def test_fetch_all_project_igf_ids(self):
pa = ProjectAdaptor(**{'session_class': self.session_class})
pa.start_session()
project_list = pa.fetch_all_project_igf_ids()
pa.close_session()
self.assertTrue('IGFP0002_test_22-8-2017_rna' in
project_list['project_igf_id'].values)
self.assertTrue('IGFP0001_test_22-8-2017_rna' in
project_list['project_igf_id'].values)
self.assertEqual(len(project_list['project_igf_id'].values), 2)
def test_check_data_authority_for_project(self):
pa = ProjectAdaptor(**{'session_class': self.session_class})
pa.start_session()
pa_results1 = pa.check_data_authority_for_project(
project_igf_id='IGFP0001_test_22-8-2017_rna')
self.assertTrue(pa_results1)
pa_results2 = pa.check_data_authority_for_project(
project_igf_id='IGFP0002_test_22-8-2017_rna')
self.assertFalse(pa_results2)
pa.close_session()
def test_fetch_data_authority_for_project(self):
pa = ProjectAdaptor(**{'session_class': self.session_class})
pa.start_session()
pa_results1 = pa.fetch_data_authority_for_project(
project_igf_id='IGFP0001_test_22-8-2017_rna')
self.assertEqual(pa_results1.email_id, '*****@*****.**')
pa_results2 = pa.fetch_data_authority_for_project(
project_igf_id='IGFP0002_test_22-8-2017_rna')
self.assertEqual(pa_results2, None)
pa.close_session()
def run(self):
'''
A ehive runnable method for uploading analysis files to irods server
:param file_list: A list of file paths to upload to irods
:param irods_exe_dir: Irods executable directory
:param project_igf_id: Name of the project
:param analysis_name: A string for analysis name, default is 'default'
:param dir_path_list: A list of directory structure for irod server, default None for using datestamp
:param file_tag: A text string for adding tag to collection, default None for only project_name
'''
try:
project_igf_id = self.param_required('project_igf_id')
igf_session_class = self.param_required('igf_session_class')
irods_exe_dir = self.param_required('irods_exe_dir')
file_list = self.param_required('file_list')
analysis_name = self.param_required('analysis_name')
dir_path_list = self.param_required('dir_path_list')
file_tag = self.param_required('file_tag')
pa = ProjectAdaptor(**{'session_class':igf_session_class})
pa.start_session()
user = \
pa.fetch_data_authority_for_project(
project_igf_id=project_igf_id) # fetch user info from db
pa.close_session()
if user is None:
raise ValueError('No user found for project {0}'.format(project_igf_id))
username = user.username # get username for irods
irods_upload = IGF_irods_uploader(irods_exe_dir) # create instance for irods upload
for file_path in file_list:
if not os.path.exists(file_path):
raise IOError('Failed to find file {0} for irods upload'.\
format(file_path))
irods_upload.\
upload_analysis_results_and_create_collection(
file_list=file_list,
irods_user=username,
project_name=project_igf_id,
analysis_name=analysis_name,
dir_path_list=dir_path_list,
file_tag=file_tag) # upload analysis results to irods and build collection
except Exception as e:
message = \
'project: {2}, Error in {0}: {1}'.format(
self.__class__.__name__,
e,
project_igf_id)
self.warning(message)
self.post_message_to_slack(message,reaction='fail') # post msg to slack for failed jobs
raise
def test_count_project_samples(self):
pa = ProjectAdaptor(**{'session_class': self.session_class})
pa.start_session()
sample1 = pa.count_project_samples(
project_igf_id='IGFP0001_test_22-8-2017_rna')
self.assertEqual(sample1, 3)
sample2 = pa.count_project_samples(
project_igf_id='IGFP0002_test_22-8-2017_rna')
self.assertEqual(sample2, 0)
sample3 = pa.count_project_samples(
project_igf_id='IGFP0001_test_22-8-2017_rna', only_active=False)
self.assertEqual(sample3, 4)
pa.close_session()
def mark_project_barcode_check_off(project_igf_id,
session_class,
barcode_check_attribute='barcode_check',
barcode_check_val='OFF'):
'''
A utility method for marking project barcode check as off using the project_igf_id
:param project_igf_id: A project_igf_id string
:param session_class: A db session class object
:param barcode_check_attribute: A text keyword for barcode check attribute, default barcode_check
:param barcode_check_val: A text for barcode check attribute value, default is 'OFF'
'''
try:
db_connected = False
pr = ProjectAdaptor(**{'session_class': session_class})
pr.start_session()
db_connected = True
pr_attributes = pr.check_project_attributes(
project_igf_id=project_igf_id,
attribute_name=barcode_check_attribute
) # check for the existing project attribute
if pr_attributes: # if attribute present, then modify it
project = pr.fetch_project_records_igf_id(
project_igf_id=project_igf_id) # fetch project info
query=pr.session.\
query(Project_attribute).\
filter(Project_attribute.attribute_name==barcode_check_attribute).\
filter(Project_attribute.project_id==project.project_id).\
update({Project_attribute.attribute_value:barcode_check_val,},
synchronize_session=False) # create query for fetching attribute records and modify attribute records
else: # if project attribute is not present, store it
data = [{
'project_igf_id': project_igf_id,
'attribute_name': barcode_check_attribute,
'attribute_value': barcode_check_val
}] # create data structure for the attribute table
pr.store_project_attributes(
data, autosave=False
) # store data to attribute table without auto commit
pr.commit_session()
except:
if db_connected:
pr.rollback_session()
raise
finally:
if db_connected:
pr.close_session()
def get_project_read_count(project_igf_id,
session_class,
run_attribute_name='R1_READ_COUNT',
active_status='ACTIVE'):
'''
A utility method for fetching sample read counts for an input project_igf_id
:param project_igf_id: A project_igf_id string
:param session_class: A db session class object
:param run_attribute_name: Attribute name from Run_attribute table for read count lookup
:param active_status: text label for active runs, default ACTIVE
:returns: A pandas dataframe containing following columns
project_igf_id
sample_igf_id
flowcell_id
attribute_value
'''
try:
read_count = pd.DataFrame()
pr = ProjectAdaptor(**{'session_class': session_class})
pr.start_session()
query=pr.session.query(Project.project_igf_id,
Sample.sample_igf_id,
Seqrun.flowcell_id,
Run_attribute.attribute_value).\
join(Sample,Project.project_id==Sample.project_id).\
join(Experiment,Sample.sample_id==Experiment.sample_id).\
join(Run,Experiment.experiment_id==Run.experiment_id).\
join(Seqrun,Seqrun.seqrun_id==Run.seqrun_id).\
join(Run_attribute,Run.run_id==Run_attribute.run_id).\
filter(Project.project_igf_id==project_igf_id).\
filter(Sample.project_id==Project.project_id).\
filter(Experiment.sample_id==Sample.sample_id).\
filter(Run.experiment_id==Experiment.experiment_id).\
filter(Seqrun.seqrun_id==Run.seqrun_id).\
filter(Run_attribute.run_id==Run.run_id).\
filter(Run_attribute.attribute_name==run_attribute_name).\
filter(Run.status==active_status).\
filter(Experiment.status==active_status).\
filter(Sample.status==active_status)
results = pr.fetch_records(query=query)
pr.close_session()
if len(results.index) > 0:
read_count = results
return read_count
except:
raise
def test_fetch_project_samples(self):
pa = ProjectAdaptor(**{'session_class': self.session_class})
pa.start_session()
sample1 = pa.fetch_project_samples(
project_igf_id='IGFP0001_test_22-8-2017_rna',
output_mode='dataframe')
self.assertEqual(len(sample1.index), 3)
sample2 = pa.fetch_project_samples(
project_igf_id='IGFP0002_test_22-8-2017_rna',
output_mode='dataframe')
self.assertEqual(len(sample2.index), 0)
sample3 = pa.fetch_project_samples(
project_igf_id='IGFP0001_test_22-8-2017_rna',
only_active=False,
output_mode='dataframe')
self.assertEqual(len(sample3.index), 4)
pa.close_session()
def test_mark_project_barcode_check_off(self):
pr = ProjectAdaptor(**{'session_class': self.session_class})
pr.start_session()
pr.store_project_and_attribute_data(self.data)
pr.close_session()
mark_project_barcode_check_off(
project_igf_id='IGFP001_test1_24-1-18',
session_class=self.session_class) # no attribute record
pr.start_session()
attribute_check = pr.check_project_attributes(
project_igf_id='IGFP001_test1_24-1-18',
attribute_name='barcode_check')
self.assertTrue(attribute_check)
pr_attributes = pr.get_project_attributes(
project_igf_id='IGFP001_test1_24-1-18',
attribute_name='barcode_check')
for pr_attribute in pr_attributes.to_dict(orient='records'):
self.assertEqual(pr_attribute['attribute_value'], 'OFF')
pr_attributes = pr.get_project_attributes(
project_igf_id='IGFP002_test1_24-1-18',
attribute_name='barcode_check')
for pr_attribute in pr_attributes.to_dict(orient='records'):
self.assertEqual(pr_attribute['attribute_value'], 'ON')
pr.close_session()
mark_project_barcode_check_off(
project_igf_id='IGFP002_test1_24-1-18',
session_class=self.session_class) # barcode check ON
pr.start_session()
pr_attributes = pr.get_project_attributes(
project_igf_id='IGFP002_test1_24-1-18',
attribute_name='barcode_check')
for pr_attribute in pr_attributes.to_dict(orient='records'):
self.assertEqual(pr_attribute['attribute_value'], 'OFF')
pr_attributes = pr.get_project_attributes(
project_igf_id='IGFP003_test1_24-1-18',
attribute_name='barcode_check')
for pr_attribute in pr_attributes.to_dict(orient='records'):
self.assertEqual(pr_attribute['attribute_value'], 'OFF')
pr.close_session()
mark_project_barcode_check_off(
project_igf_id='IGFP003_test1_24-1-18',
session_class=self.session_class) # barcode check OFF
pr.start_session()
pr_attributes = pr.get_project_attributes(
project_igf_id='IGFP003_test1_24-1-18',
attribute_name='barcode_check')
for pr_attribute in pr_attributes.to_dict(orient='records'):
self.assertEqual(pr_attribute['attribute_value'], 'OFF')
pr.close_session()
def get_seqrun_info_for_project(project_igf_id, session_class):
'''
A utility method for fetching seqrun_igf_id and flowcell_id which are linked
to a specific project_igf_id
required params:
project_igf_id: A project_igf_id string
session_class: A db session class object
returns a pandas dataframe containing following columns
seqrun_igf_id
flowcell_id
'''
try:
seqrun_info = pd.DataFrame()
pr = ProjectAdaptor(**{'session_class': session_class})
pr.start_session()
query=pr.session.query(distinct(Seqrun.seqrun_igf_id).\
label('seqrun_igf_id'),
Seqrun.flowcell_id).\
join(Run,Seqrun.seqrun_id==Run.seqrun_id).\
join(Experiment,Experiment.experiment_id==Run.experiment_id).\
join(Sample,Sample.sample_id==Experiment.sample_id).\
join(Project,Project.project_id==Sample.project_id).\
filter(Project.project_id==Sample.project_id).\
filter(Sample.sample_id==Experiment.sample_id).\
filter(Experiment.experiment_id==Run.experiment_id).\
filter(Run.seqrun_id==Seqrun.seqrun_id).\
filter(Project.project_igf_id==project_igf_id)
results = pr.fetch_records(query=query)
pr.close_session()
if len(results.index) > 0:
seqrun_info = results
return seqrun_info
except:
raise
def run(self):
try:
fastq_dir = self.param_required('fastq_dir')
seqrun_igf_id = self.param_required('seqrun_igf_id')
project_name = self.param_required('project_name')
igf_session_class = self.param_required('igf_session_class')
irods_exe_dir = self.param_required('irods_exe_dir')
flowcell_id = self.param_required('flowcell_id')
samplesheet_filename = self.param('samplesheet_filename')
manifest_name = self.param_required('manifest_name')
report_html = self.param('report_html')
use_ephemeral_space = self.param('use_ephemeral_space')
pa = ProjectAdaptor(**{'session_class':igf_session_class})
pa.start_session()
user = \
pa.fetch_data_authority_for_project(\
project_igf_id=project_name) # fetch user info from db
pa.close_session()
if user is None:
raise ValueError('No user found for project {0}'.\
format(project_name))
username = user.username # get username for irods
report_htmlname = os.path.basename(report_html)
seqrun_date = seqrun_igf_id.split('_')[0] # collect seqrun date from igf id
seqrun_date = datetime.datetime.strptime(seqrun_date,'%y%m%d').date() # identify actual date
seqrun_date = str(seqrun_date) # convert object to string
irods_upload = IGF_irods_uploader(irods_exe_dir) # create instance for irods upload
base_seq_dir = os.path.basename(fastq_dir) # get base name for the source dir
tarfile_name = \
'{0}_{1}_{2}.tar'.\
format(\
project_name,
base_seq_dir,
seqrun_date) # construct name of the tarfile
temp_work_dir = \
get_temp_dir(use_ephemeral_space=use_ephemeral_space) # get a temp dir
tarfile_name = \
os.path.join(
temp_work_dir,
tarfile_name) # create tarfile in the temp dir
with tarfile.open(tarfile_name, "w") as tar:
for root,_, files in os.walk(top=fastq_dir):
if samplesheet_filename in files:
samplesheet_file = \
os.path.join(os.path.abspath(root),
samplesheet_filename) # get samplesheet filepath
tmp_samplesheet_file = \
os.path.join(
temp_work_dir,
'{0}_{1}_{2}_{3}'.\
format(
project_name,
base_seq_dir,
seqrun_date,
samplesheet_filename))
copy2(
samplesheet_file,
tmp_samplesheet_file) # change samplesheet filename
tar.add(
tmp_samplesheet_file,
arcname=\
os.path.relpath(
tmp_samplesheet_file,
start=temp_work_dir)) # add samplesheet file to tar
if report_htmlname in files:
for file in files:
if fnmatch.fnmatch(os.path.join(root,file),report_html):
report_file = os.path.join(os.path.abspath(root),file) # get filepath for the report
tmp_report_file = \
os.path.join(\
temp_work_dir,
'{0}_{1}_{2}_{3}'.\
format(\
project_name,
base_seq_dir,
seqrun_date,
os.path.basename(report_file))) # change report name
copy2(report_file, tmp_report_file) # copy report file to temp
tar.add(tmp_report_file,
arcname=os.path.relpath(tmp_report_file,
start=temp_work_dir)) # add demultiplexing report to tar
if manifest_name in files:
manifest_file = \
os.path.join(os.path.abspath(root),
manifest_name) # get samplesheet filepath
tmp_manifest_file = \
os.path.join(\
temp_work_dir,
'{0}_{1}_{2}_{3}'.\
format(\
project_name,
base_seq_dir,
seqrun_date,
manifest_name)) # change manifest name
copy2(manifest_file,tmp_manifest_file) # copy manifest to temp
tar.add(tmp_manifest_file,
arcname=os.path.relpath(tmp_manifest_file,
start=temp_work_dir)) # add samplesheet file to tar
for file in files:
if fnmatch.fnmatch(file, '*.fastq.gz') and \
not fnmatch.fnmatch(file, 'Undetermined_*'):
fastq_file_path = os.path.join(os.path.abspath(root),file) # get filepath for the fastq files
tar.add(fastq_file_path,
arcname=os.path.relpath(fastq_file_path,
start=fastq_dir)) # add fastq file to tar
irods_upload.\
upload_fastqfile_and_create_collection(\
filepath=tarfile_name,
irods_user=username,
project_name=project_name,
run_igf_id=seqrun_igf_id,
flowcell_id=flowcell_id,
run_date=seqrun_date) # upload fastq data to irods
remove_dir(temp_work_dir) # remove temp dir once data uoload is done
except Exception as e:
message = \
'seqrun: {2}, Error in {0}: {1}'.\
format(\
self.__class__.__name__,
e,
seqrun_igf_id)
self.warning(message)
self.post_message_to_slack(message,reaction='fail') # post msg to slack for failed jobs
raise
def run(self):
try:
seqrun_igf_id = self.param_required('seqrun_igf_id')
project_name = self.param_required('project_name')
seqrun_date = self.param_required('seqrun_date')
flowcell_id = self.param_required('flowcell_id')
igf_session_class = self.param_required('igf_session_class')
template_dir = self.param_required('template_dir')
email_template_path = self.param('email_template_path')
email_template = self.param('email_template')
sendmail_exe = self.param('sendmail_exe')
use_ephemeral_space = self.param('use_ephemeral_space')
hpcUser = False # default value for hpc users
pa = ProjectAdaptor(**{'session_class':igf_session_class})
pa.start_session()
user_info = pa.get_project_user_info(project_igf_id=project_name) # fetch user info from db
pa.close_session()
user_info = user_info[user_info['data_authority']=='T'] # filter dataframe for data authority
user_info = user_info.to_dict(orient='records') # convert dataframe to list of dictionaries
if len(user_info) == 0:
raise ValueError('No user found for project {0}'.format(project_name))
user_info = user_info[0]
user_name = user_info['name'] # get username for irods
login_name = user_info['username']
user_email = user_info['email_id']
user_category = user_info['category']
if user_category=='HPC_USER':
hpcUser = True # set value for hpc user
message = 'loading hpc user specific settings for {0}:{1}'.\
format(user_name,login_name)
self.post_message_to_slack(message,reaction='pass') # send message to slack
email_template_path = \
os.path.join(\
template_dir,
email_template_path)
template_env = \
Environment(\
loader=FileSystemLoader(\
searchpath=email_template_path),
autoescape=select_autoescape(['html','xml'])) # set template env
template_file = template_env.get_template(email_template)
temp_work_dir = \
get_temp_dir(use_ephemeral_space=use_ephemeral_space) # get a temp dir
report_output_file = \
os.path.join(\
temp_work_dir,
email_template)
template_file.\
stream(\
projectName=project_name,
customerEmail=user_email,
customerName=user_name,
customerUsername=login_name,
projectRunDate=seqrun_date,
flowcellId=flowcell_id,
hpcUser=hpcUser).\
dump(report_output_file)
proc = \
subprocess.\
Popen(\
['cat',
report_output_file
],
stdout=subprocess.PIPE)
sendmail_cmd = \
[sendmail_exe,
'-t',
]
subprocess.\
check_call(\
sendmail_cmd,
stdin=proc.stdout)
proc.stdout.close()
remove_dir(temp_work_dir)
message = \
'finished data processing for seqrun: {0}, project: {1}, sent mail to igf'.\
format(seqrun_igf_id, project_name)
self.post_message_to_slack(message,reaction='pass')
except Exception as e:
message = \
'seqrun: {2}, Error in {0}: {1}'.\
format(\
self.__class__.__name__,
e,
seqrun_igf_id)
self.warning(message)
self.post_message_to_slack(message,reaction='fail') # post msg to slack for failed jobs
raise
def setUp(self):
self.dbconfig = 'data/dbconfig.json'
dbparam = read_dbconf_json(self.dbconfig)
base = BaseAdaptor(**dbparam)
self.engine = base.engine
self.dbname = dbparam['dbname']
Base.metadata.create_all(self.engine)
self.session_class = base.get_session_class()
# load platform data
platform_data=\
[{"platform_igf_id" : "M03291" ,
"model_name" : "MISEQ" ,
"vendor_name" : "ILLUMINA" ,
"software_name" : "RTA" ,
"software_version" : "RTA1.18.54"
},
{"platform_igf_id" : "NB501820",
"model_name" : "NEXTSEQ",
"vendor_name" : "ILLUMINA",
"software_name" : "RTA",
"software_version" : "RTA2"
},
{"platform_igf_id" : "K00345",
"model_name" : "HISEQ4000",
"vendor_name" : "ILLUMINA",
"software_name" : "RTA",
"software_version" : "RTA2"
}]
flowcell_rule_data=\
[{"platform_igf_id":"K00345",
"flowcell_type":"HiSeq 3000/4000 SR",
"index_1":"NO_CHANGE",
"index_2":"NO_CHANGE"},
{"platform_igf_id":"K00345",
"flowcell_type":"HiSeq 3000/4000 PE",
"index_1":"NO_CHANGE",
"index_2":"REVCOMP"},
{"platform_igf_id":"NB501820",
"flowcell_type":"NEXTSEQ",
"index_1":"NO_CHANGE",
"index_2":"REVCOMP"},
{"platform_igf_id":"M03291",
"flowcell_type":"MISEQ",
"index_1":"NO_CHANGE",
"index_2":"NO_CHANGE"}]
pl = PlatformAdaptor(**{'session_class': base.session_class})
pl.start_session()
pl.store_platform_data(data=platform_data)
pl.store_flowcell_barcode_rule(data=flowcell_rule_data)
pl.close_session()
# load project data
project_data = [{'project_igf_id': 'IGFQ000472_avik_28-3-2018_RNA'}]
pa = ProjectAdaptor(**{'session_class': base.session_class})
pa.start_session()
pa.store_project_and_attribute_data(data=project_data)
pa.close_session()
# load samples
sample_data = [
{
'project_igf_id': 'IGFQ000472_avik_28-3-2018_RNA',
'sample_igf_id': 'IGF109792',
'expected_read': 40000000
},
{
'project_igf_id': 'IGFQ000472_avik_28-3-2018_RNA',
'sample_igf_id': 'IGF109793',
'expected_read': 40000000
},
{
'project_igf_id': 'IGFQ000472_avik_28-3-2018_RNA',
'sample_igf_id': 'IGF109794',
'expected_read': 40000000
},
{
'project_igf_id': 'IGFQ000472_avik_28-3-2018_RNA',
'sample_igf_id': 'IGF109795',
'expected_read': 40000000
},
{
'project_igf_id': 'IGFQ000472_avik_28-3-2018_RNA',
'sample_igf_id': 'IGF109796',
'expected_read': 40000000
},
{
'project_igf_id': 'IGFQ000472_avik_28-3-2018_RNA',
'sample_igf_id': 'IGF109797',
'expected_read': 40000000
},
{
'project_igf_id': 'IGFQ000472_avik_28-3-2018_RNA',
'sample_igf_id': 'IGF109797_1',
'expected_read': 40000000
},
]
sa = SampleAdaptor(**{'session_class': base.session_class})
sa.start_session()
sa.store_sample_and_attribute_data(data=sample_data)
sa.close_session()
# load seqrun data
seqrun_data = [{
'flowcell_id': 'HV2GJBBXX',
'platform_igf_id': 'K00345',
'seqrun_igf_id': '180518_K00345_0047_BHV2GJBBXX'
}]
sra = SeqrunAdaptor(**{'session_class': base.session_class})
sra.start_session()
sra.store_seqrun_and_attribute_data(data=seqrun_data)
sra.close_session()
# load experiment data
experiment_data=\
[{'experiment_igf_id': 'IGF109792_HISEQ4000',
'library_name': 'IGF109792',
'platform_name': 'HISEQ4000',
'project_igf_id': 'IGFQ000472_avik_28-3-2018_RNA',
'sample_igf_id': 'IGF109792',
},
{'experiment_igf_id': 'IGF109793_HISEQ4000',
'library_name': 'IGF109793',
'platform_name': 'HISEQ4000',
'project_igf_id': 'IGFQ000472_avik_28-3-2018_RNA',
'sample_igf_id': 'IGF109793',
},
{'experiment_igf_id': 'IGF109794_HISEQ4000',
'library_name': 'IGF109794',
'platform_name': 'HISEQ4000',
'project_igf_id': 'IGFQ000472_avik_28-3-2018_RNA',
'sample_igf_id': 'IGF109794',
},
{'experiment_igf_id': 'IGF109795_HISEQ4000',
'library_name': 'IGF109795',
'platform_name': 'HISEQ4000',
'project_igf_id': 'IGFQ000472_avik_28-3-2018_RNA',
'sample_igf_id': 'IGF109795',
},
{'experiment_igf_id': 'IGF109796_HISEQ4000',
'library_name': 'IGF109796',
'platform_name': 'HISEQ4000',
'project_igf_id': 'IGFQ000472_avik_28-3-2018_RNA',
'sample_igf_id': 'IGF109796',
},
{'experiment_igf_id': 'IGF109797_HISEQ4000',
'library_name': 'IGF109797',
'platform_name': 'HISEQ4000',
'project_igf_id': 'IGFQ000472_avik_28-3-2018_RNA',
'sample_igf_id': 'IGF109797',
},
]
ea = ExperimentAdaptor(**{'session_class': base.session_class})
ea.start_session()
ea.store_project_and_attribute_data(data=experiment_data)
ea.close_session()
# load run data
run_data=\
[{'experiment_igf_id': 'IGF109792_HISEQ4000',
'lane_number': '7',
'run_igf_id': 'IGF109792_HISEQ4000_H2N3MBBXY_7',
'seqrun_igf_id': '180518_K00345_0047_BHV2GJBBXX',
'R1_READ_COUNT':288046541
},
{'experiment_igf_id': 'IGF109793_HISEQ4000',
'lane_number': '7',
'run_igf_id': 'IGF109793_HISEQ4000_H2N3MBBXY_7',
'seqrun_igf_id': '180518_K00345_0047_BHV2GJBBXX',
'R1_READ_COUNT':14666330
},
{'experiment_igf_id': 'IGF109794_HISEQ4000',
'lane_number': '7',
'run_igf_id': 'IGF109794_HISEQ4000_H2N3MBBXY_7',
'seqrun_igf_id': '180518_K00345_0047_BHV2GJBBXX',
'R1_READ_COUNT':5009143
},
{'experiment_igf_id': 'IGF109795_HISEQ4000',
'lane_number': '7',
'run_igf_id': 'IGF109795_HISEQ4000_H2N3MBBXY_7',
'seqrun_igf_id': '180518_K00345_0047_BHV2GJBBXX',
'R1_READ_COUNT':1391747
},
{'experiment_igf_id': 'IGF109796_HISEQ4000',
'lane_number': '7',
'run_igf_id': ' IGF109796_HISEQ4000_H2N3MBBXY_7',
'seqrun_igf_id': '180518_K00345_0047_BHV2GJBBXX',
'R1_READ_COUNT':1318008
},
{'experiment_igf_id': 'IGF109797_HISEQ4000',
'lane_number': '7',
'run_igf_id': 'IGF109797_HISEQ4000_H2N3MBBXY_7',
'seqrun_igf_id': '180518_K00345_0047_BHV2GJBBXX',
'R1_READ_COUNT':1216324
},
]
ra = RunAdaptor(**{'session_class': base.session_class})
ra.start_session()
ra.store_run_and_attribute_data(data=run_data)
ra.close_session()
def run(self):
try:
seqrun_igf_id = self.param_required('seqrun_igf_id')
project_name = self.param_required('project_name')
remote_project_path = self.param_required('remote_project_path')
remote_user = self.param_required('remote_user')
remote_host = self.param_required('remote_host')
template_dir = self.param_required('template_dir')
igf_session_class = self.param_required('igf_session_class')
htaccess_template_path = self.param('htaccess_template_path')
htaccess_template = self.param('htaccess_template')
htpasswd_template = self.param('htpasswd_template')
htaccess_filename = self.param('htaccess_filename')
htpasswd_filename = self.param('htpasswd_filename')
project_template = self.param('project_template')
status_template = self.param('status_template')
analysis_template = self.param('analysis_template')
analysis_viewer_template = self.param('analysis_viewer_template')
seqruninfofile = self.param('seqruninfofile')
samplereadcountfile = self.param('samplereadcountfile')
samplereadcountcsvfile = self.param('samplereadcountcsvfile')
status_data_json = self.param('status_data_json')
analysis_data_json = self.param('analysis_data_json')
analysis_data_csv = self.param('analysis_data_csv')
analysis_chart_data_csv = self.param('analysis_chart_data_csv')
analysis_chart_data_json = self.param('analysis_chart_data_json')
analysis_view_js = self.param('analysis_view_js')
image_height = self.param('image_height')
sample_count_threshold = self.param('sample_count_threshold')
use_ephemeral_space = self.param('use_ephemeral_space')
htaccess_template_path = \
os.path.join(\
template_dir,
htaccess_template_path) # set path for template dir
project_template_path = \
os.path.join(\
template_dir,
project_template) # set path for project template
status_template_path = \
os.path.join(\
template_dir,
status_template) # set path for project status template
analysis_template_path = \
os.path.join(\
template_dir,
analysis_template) # set path for project analysis template
analysis_viewer_template = \
os.path.join(\
template_dir,
analysis_viewer_template) # set path for analysis viewer template
pa = ProjectAdaptor(**{'session_class': igf_session_class})
pa.start_session()
user_info = \
pa.get_project_user_info(project_igf_id=project_name) # fetch user info from db
sample_counts = \
pa.count_project_samples(\
project_igf_id=project_name,
only_active=True) # get sample counts for the project
pa.close_session()
image_height = \
self._calculate_image_height(\
sample_count=sample_counts,
height=image_height,
threshold=sample_count_threshold) # change image height based on sample count
user_info = user_info.to_dict(
orient='records') # convert dataframe to list of dictionaries
if len(user_info) == 0:
raise ValueError('No user found for project {0}'.\
format(project_name))
user_list = list()
user_passwd_dict = dict()
hpc_user = True # by default, load hpc user settings
for user in user_info:
username = user['username'] # get username for irods
user_list.append(username)
if 'ht_password' in user.keys():
ht_passwd = user['ht_password'] # get htaccess passwd
user_passwd_dict.update({username: ht_passwd})
if 'category' in user.keys() and \
'data_authority' in user.keys() and \
user['category'] == 'NON_HPC_USER' and \
user['data_authority']=='T':
hpc_user = False # switch to non-hpc settings if primary user is non-hpc
temp_work_dir = \
get_temp_dir(use_ephemeral_space=use_ephemeral_space) # get a temp dir
template_env = \
Environment(\
loader=FileSystemLoader(\
searchpath=htaccess_template_path),
autoescape=select_autoescape(['html', 'xml'])) # set template env
htaccess = template_env.get_template(
htaccess_template) # read htaccess template
htpasswd = template_env.get_template(
htpasswd_template) # read htpass template
htaccess_output = \
os.path.join(\
temp_work_dir,
htaccess_filename)
htpasswd_output = \
os.path.join(\
temp_work_dir,
htpasswd_filename)
htaccess.\
stream(\
remote_project_dir=remote_project_path,
project_tag=project_name,
hpcUser=hpc_user,
htpasswd_filename=htpasswd_filename,
customerUsernameList=' '.join(user_list)).\
dump(htaccess_output) # write new htacces file
htpasswd.\
stream(userDict=user_passwd_dict).\
dump(htpasswd_output) # write new htpass file
template_prj = \
Environment(\
loader=FileSystemLoader(\
searchpath=os.path.dirname(project_template_path)),
autoescape=select_autoescape(['txt', 'xml'])) # set template env for project
project_index = \
template_prj.\
get_template(os.path.basename(project_template_path)) # read htaccess template
project_output = \
os.path.join(\
temp_work_dir,
os.path.basename(project_template_path))
project_index.\
stream(\
ProjectName=project_name,
seqrunInfoFile=seqruninfofile,
sampleReadCountFile=samplereadcountfile,
sampleReadCountCsvFile=samplereadcountcsvfile,
ImageHeight=image_height).\
dump(project_output) # write new project file
template_status = \
Environment(\
loader=FileSystemLoader(\
searchpath=os.path.dirname(status_template_path)),
autoescape=select_autoescape(['txt', 'xml'])) # set template env for project
project_status = \
template_status.\
get_template(os.path.basename(status_template_path)) # read status page template
status_output = \
os.path.join(\
temp_work_dir,
os.path.basename(status_template_path))
project_status.\
stream(\
ProjectName=project_name,
status_data_json=status_data_json).\
dump(status_output) # write new project status file
template_analysis = \
Environment(\
loader=FileSystemLoader(\
searchpath=os.path.dirname(analysis_template_path)),
autoescape=select_autoescape(['txt', 'xml'])) # set template env for analysis
project_analysis = \
template_analysis.\
get_template(os.path.basename(analysis_template_path)) # read analysis page template
analysis_output = \
os.path.join(\
temp_work_dir,
os.path.basename(analysis_template_path))
project_analysis.\
stream(\
ProjectName=project_name,
analysisInfoFile=analysis_data_json,
analysisInfoCsvFile=analysis_data_csv,
analysisCsvDataFile=analysis_chart_data_csv,
analysisPlotFile=analysis_chart_data_json).\
dump(analysis_output) # write new project analysis file
template_analysis_viewer = \
Environment(\
loader=FileSystemLoader(\
searchpath=os.path.dirname(analysis_viewer_template)),
autoescape=select_autoescape(['txt', 'xml'])) # set template env for analysis viewer
project_analysis_viewer = \
template_analysis_viewer.\
get_template(os.path.basename(analysis_viewer_template)) # read analysis viewer page template
analysis_viewer_output = \
os.path.join(\
temp_work_dir,
os.path.basename(analysis_viewer_template))
project_analysis_viewer.\
stream(\
ProjectName=project_name,
analysisJsFile=analysis_view_js).\
dump(analysis_viewer_output) # write new project analysis viewer file
remote_project_dir = \
os.path.join(\
remote_project_path,
project_name) # ger remote project dir path
remote_htaccess_file = \
os.path.join(\
remote_project_dir,
htaccess_filename) # remote htaccess filepath
self._check_and_copy_remote_file(\
remote_user=remote_user,
remote_host=remote_host,
source_file=htaccess_output,
remote_file=remote_htaccess_file) # copy htaccess file to remote dir
remote_htpasswd_file = \
os.path.join(\
remote_project_dir,
htpasswd_filename) # remote htpasswd filepath
self._check_and_copy_remote_file(\
remote_user=remote_user,
remote_host=remote_host,
source_file=htpasswd_output,
remote_file=remote_htpasswd_file) # copy htpasswd file to remote dir
remote_project_output_file = \
os.path.join(\
remote_project_dir,
os.path.basename(project_output)) # remote project output filepath
self._check_and_copy_remote_file(\
remote_user=remote_user,
remote_host=remote_host,
source_file=project_output,
remote_file=remote_project_output_file) # copy project output file to remote dir
remote_status_output_file = \
os.path.join(\
remote_project_dir,
os.path.basename(status_output)) # remote project status output filepath
self._check_and_copy_remote_file(\
remote_user=remote_user,
remote_host=remote_host,
source_file=status_output,
remote_file=remote_status_output_file) # copy project status output file to remote dir
remote_analysis_output_file = \
os.path.join(\
remote_project_dir,
os.path.basename(analysis_output)) # remote project analysis output filepath
self._check_and_copy_remote_file(\
remote_user=remote_user,
remote_host=remote_host,
source_file=analysis_output,
remote_file=remote_analysis_output_file) # copy project analysis output file to remote dir
remote_analysis_viewer_output_file = \
os.path.join(\
remote_project_dir,
os.path.basename(analysis_viewer_output)) # remote project analysis viewer output filepath
self._check_and_copy_remote_file(\
remote_user=remote_user,
remote_host=remote_host,
source_file=analysis_viewer_output,
remote_file=remote_analysis_viewer_output_file) # copy project analysis viewer output file to remote dir
self.param('dataflow_params', {'remote_dir_status': 'done'})
remove_dir(temp_work_dir)
except Exception as e:
message = \
'seqrun: {2}, Error in {0}: {1}'.\
format(\
self.__class__.__name__,
e,
seqrun_igf_id)
self.warning(message)
self.post_message_to_slack(
message, reaction='fail') # post msg to slack for failed jobs
raise