def test_count_project_samples(self):
pa = ProjectAdaptor(**{'session_class': self.session_class})
pa.start_session()
sample1 = pa.count_project_samples(
project_igf_id='IGFP0001_test_22-8-2017_rna')
self.assertEqual(sample1, 3)
sample2 = pa.count_project_samples(
project_igf_id='IGFP0002_test_22-8-2017_rna')
self.assertEqual(sample2, 0)
sample3 = pa.count_project_samples(
project_igf_id='IGFP0001_test_22-8-2017_rna', only_active=False)
self.assertEqual(sample3, 4)
pa.close_session()
def run(self):
try:
seqrun_igf_id = self.param_required('seqrun_igf_id')
project_name = self.param_required('project_name')
remote_project_path = self.param_required('remote_project_path')
remote_user = self.param_required('remote_user')
remote_host = self.param_required('remote_host')
template_dir = self.param_required('template_dir')
igf_session_class = self.param_required('igf_session_class')
htaccess_template_path = self.param('htaccess_template_path')
htaccess_template = self.param('htaccess_template')
htpasswd_template = self.param('htpasswd_template')
htaccess_filename = self.param('htaccess_filename')
htpasswd_filename = self.param('htpasswd_filename')
project_template = self.param('project_template')
status_template = self.param('status_template')
analysis_template = self.param('analysis_template')
analysis_viewer_template = self.param('analysis_viewer_template')
seqruninfofile = self.param('seqruninfofile')
samplereadcountfile = self.param('samplereadcountfile')
samplereadcountcsvfile = self.param('samplereadcountcsvfile')
status_data_json = self.param('status_data_json')
analysis_data_json = self.param('analysis_data_json')
analysis_data_csv = self.param('analysis_data_csv')
analysis_chart_data_csv = self.param('analysis_chart_data_csv')
analysis_chart_data_json = self.param('analysis_chart_data_json')
analysis_view_js = self.param('analysis_view_js')
image_height = self.param('image_height')
sample_count_threshold = self.param('sample_count_threshold')
use_ephemeral_space = self.param('use_ephemeral_space')
htaccess_template_path = \
os.path.join(\
template_dir,
htaccess_template_path) # set path for template dir
project_template_path = \
os.path.join(\
template_dir,
project_template) # set path for project template
status_template_path = \
os.path.join(\
template_dir,
status_template) # set path for project status template
analysis_template_path = \
os.path.join(\
template_dir,
analysis_template) # set path for project analysis template
analysis_viewer_template = \
os.path.join(\
template_dir,
analysis_viewer_template) # set path for analysis viewer template
pa = ProjectAdaptor(**{'session_class': igf_session_class})
pa.start_session()
user_info = \
pa.get_project_user_info(project_igf_id=project_name) # fetch user info from db
sample_counts = \
pa.count_project_samples(\
project_igf_id=project_name,
only_active=True) # get sample counts for the project
pa.close_session()
image_height = \
self._calculate_image_height(\
sample_count=sample_counts,
height=image_height,
threshold=sample_count_threshold) # change image height based on sample count
user_info = user_info.to_dict(
orient='records') # convert dataframe to list of dictionaries
if len(user_info) == 0:
raise ValueError('No user found for project {0}'.\
format(project_name))
user_list = list()
user_passwd_dict = dict()
hpc_user = True # by default, load hpc user settings
for user in user_info:
username = user['username'] # get username for irods
user_list.append(username)
if 'ht_password' in user.keys():
ht_passwd = user['ht_password'] # get htaccess passwd
user_passwd_dict.update({username: ht_passwd})
if 'category' in user.keys() and \
'data_authority' in user.keys() and \
user['category'] == 'NON_HPC_USER' and \
user['data_authority']=='T':
hpc_user = False # switch to non-hpc settings if primary user is non-hpc
temp_work_dir = \
get_temp_dir(use_ephemeral_space=use_ephemeral_space) # get a temp dir
template_env = \
Environment(\
loader=FileSystemLoader(\
searchpath=htaccess_template_path),
autoescape=select_autoescape(['html', 'xml'])) # set template env
htaccess = template_env.get_template(
htaccess_template) # read htaccess template
htpasswd = template_env.get_template(
htpasswd_template) # read htpass template
htaccess_output = \
os.path.join(\
temp_work_dir,
htaccess_filename)
htpasswd_output = \
os.path.join(\
temp_work_dir,
htpasswd_filename)
htaccess.\
stream(\
remote_project_dir=remote_project_path,
project_tag=project_name,
hpcUser=hpc_user,
htpasswd_filename=htpasswd_filename,
customerUsernameList=' '.join(user_list)).\
dump(htaccess_output) # write new htacces file
htpasswd.\
stream(userDict=user_passwd_dict).\
dump(htpasswd_output) # write new htpass file
template_prj = \
Environment(\
loader=FileSystemLoader(\
searchpath=os.path.dirname(project_template_path)),
autoescape=select_autoescape(['txt', 'xml'])) # set template env for project
project_index = \
template_prj.\
get_template(os.path.basename(project_template_path)) # read htaccess template
project_output = \
os.path.join(\
temp_work_dir,
os.path.basename(project_template_path))
project_index.\
stream(\
ProjectName=project_name,
seqrunInfoFile=seqruninfofile,
sampleReadCountFile=samplereadcountfile,
sampleReadCountCsvFile=samplereadcountcsvfile,
ImageHeight=image_height).\
dump(project_output) # write new project file
template_status = \
Environment(\
loader=FileSystemLoader(\
searchpath=os.path.dirname(status_template_path)),
autoescape=select_autoescape(['txt', 'xml'])) # set template env for project
project_status = \
template_status.\
get_template(os.path.basename(status_template_path)) # read status page template
status_output = \
os.path.join(\
temp_work_dir,
os.path.basename(status_template_path))
project_status.\
stream(\
ProjectName=project_name,
status_data_json=status_data_json).\
dump(status_output) # write new project status file
template_analysis = \
Environment(\
loader=FileSystemLoader(\
searchpath=os.path.dirname(analysis_template_path)),
autoescape=select_autoescape(['txt', 'xml'])) # set template env for analysis
project_analysis = \
template_analysis.\
get_template(os.path.basename(analysis_template_path)) # read analysis page template
analysis_output = \
os.path.join(\
temp_work_dir,
os.path.basename(analysis_template_path))
project_analysis.\
stream(\
ProjectName=project_name,
analysisInfoFile=analysis_data_json,
analysisInfoCsvFile=analysis_data_csv,
analysisCsvDataFile=analysis_chart_data_csv,
analysisPlotFile=analysis_chart_data_json).\
dump(analysis_output) # write new project analysis file
template_analysis_viewer = \
Environment(\
loader=FileSystemLoader(\
searchpath=os.path.dirname(analysis_viewer_template)),
autoescape=select_autoescape(['txt', 'xml'])) # set template env for analysis viewer
project_analysis_viewer = \
template_analysis_viewer.\
get_template(os.path.basename(analysis_viewer_template)) # read analysis viewer page template
analysis_viewer_output = \
os.path.join(\
temp_work_dir,
os.path.basename(analysis_viewer_template))
project_analysis_viewer.\
stream(\
ProjectName=project_name,
analysisJsFile=analysis_view_js).\
dump(analysis_viewer_output) # write new project analysis viewer file
remote_project_dir = \
os.path.join(\
remote_project_path,
project_name) # ger remote project dir path
remote_htaccess_file = \
os.path.join(\
remote_project_dir,
htaccess_filename) # remote htaccess filepath
self._check_and_copy_remote_file(\
remote_user=remote_user,
remote_host=remote_host,
source_file=htaccess_output,
remote_file=remote_htaccess_file) # copy htaccess file to remote dir
remote_htpasswd_file = \
os.path.join(\
remote_project_dir,
htpasswd_filename) # remote htpasswd filepath
self._check_and_copy_remote_file(\
remote_user=remote_user,
remote_host=remote_host,
source_file=htpasswd_output,
remote_file=remote_htpasswd_file) # copy htpasswd file to remote dir
remote_project_output_file = \
os.path.join(\
remote_project_dir,
os.path.basename(project_output)) # remote project output filepath
self._check_and_copy_remote_file(\
remote_user=remote_user,
remote_host=remote_host,
source_file=project_output,
remote_file=remote_project_output_file) # copy project output file to remote dir
remote_status_output_file = \
os.path.join(\
remote_project_dir,
os.path.basename(status_output)) # remote project status output filepath
self._check_and_copy_remote_file(\
remote_user=remote_user,
remote_host=remote_host,
source_file=status_output,
remote_file=remote_status_output_file) # copy project status output file to remote dir
remote_analysis_output_file = \
os.path.join(\
remote_project_dir,
os.path.basename(analysis_output)) # remote project analysis output filepath
self._check_and_copy_remote_file(\
remote_user=remote_user,
remote_host=remote_host,
source_file=analysis_output,
remote_file=remote_analysis_output_file) # copy project analysis output file to remote dir
remote_analysis_viewer_output_file = \
os.path.join(\
remote_project_dir,
os.path.basename(analysis_viewer_output)) # remote project analysis viewer output filepath
self._check_and_copy_remote_file(\
remote_user=remote_user,
remote_host=remote_host,
source_file=analysis_viewer_output,
remote_file=remote_analysis_viewer_output_file) # copy project analysis viewer output file to remote dir
self.param('dataflow_params', {'remote_dir_status': 'done'})
remove_dir(temp_work_dir)
except Exception as e:
message = \
'seqrun: {2}, Error in {0}: {1}'.\
format(\
self.__class__.__name__,
e,
seqrun_igf_id)
self.warning(message)
self.post_message_to_slack(
message, reaction='fail') # post msg to slack for failed jobs
raise