def test_fetch_data_authority_for_project(self):
pa = ProjectAdaptor(**{'session_class': self.session_class})
pa.start_session()
pa_results1 = pa.fetch_data_authority_for_project(
project_igf_id='IGFP0001_test_22-8-2017_rna')
self.assertEqual(pa_results1.email_id, '*****@*****.**')
pa_results2 = pa.fetch_data_authority_for_project(
project_igf_id='IGFP0002_test_22-8-2017_rna')
self.assertEqual(pa_results2, None)
pa.close_session()
def run(self):
'''
A ehive runnable method for uploading analysis files to irods server
:param file_list: A list of file paths to upload to irods
:param irods_exe_dir: Irods executable directory
:param project_igf_id: Name of the project
:param analysis_name: A string for analysis name, default is 'default'
:param dir_path_list: A list of directory structure for irod server, default None for using datestamp
:param file_tag: A text string for adding tag to collection, default None for only project_name
'''
try:
project_igf_id = self.param_required('project_igf_id')
igf_session_class = self.param_required('igf_session_class')
irods_exe_dir = self.param_required('irods_exe_dir')
file_list = self.param_required('file_list')
analysis_name = self.param_required('analysis_name')
dir_path_list = self.param_required('dir_path_list')
file_tag = self.param_required('file_tag')
pa = ProjectAdaptor(**{'session_class':igf_session_class})
pa.start_session()
user = \
pa.fetch_data_authority_for_project(
project_igf_id=project_igf_id) # fetch user info from db
pa.close_session()
if user is None:
raise ValueError('No user found for project {0}'.format(project_igf_id))
username = user.username # get username for irods
irods_upload = IGF_irods_uploader(irods_exe_dir) # create instance for irods upload
for file_path in file_list:
if not os.path.exists(file_path):
raise IOError('Failed to find file {0} for irods upload'.\
format(file_path))
irods_upload.\
upload_analysis_results_and_create_collection(
file_list=file_list,
irods_user=username,
project_name=project_igf_id,
analysis_name=analysis_name,
dir_path_list=dir_path_list,
file_tag=file_tag) # upload analysis results to irods and build collection
except Exception as e:
message = \
'project: {2}, Error in {0}: {1}'.format(
self.__class__.__name__,
e,
project_igf_id)
self.warning(message)
self.post_message_to_slack(message,reaction='fail') # post msg to slack for failed jobs
raise
def run(self):
try:
fastq_dir = self.param_required('fastq_dir')
seqrun_igf_id = self.param_required('seqrun_igf_id')
project_name = self.param_required('project_name')
igf_session_class = self.param_required('igf_session_class')
irods_exe_dir = self.param_required('irods_exe_dir')
flowcell_id = self.param_required('flowcell_id')
samplesheet_filename = self.param('samplesheet_filename')
manifest_name = self.param_required('manifest_name')
report_html = self.param('report_html')
use_ephemeral_space = self.param('use_ephemeral_space')
pa = ProjectAdaptor(**{'session_class':igf_session_class})
pa.start_session()
user = \
pa.fetch_data_authority_for_project(\
project_igf_id=project_name) # fetch user info from db
pa.close_session()
if user is None:
raise ValueError('No user found for project {0}'.\
format(project_name))
username = user.username # get username for irods
report_htmlname = os.path.basename(report_html)
seqrun_date = seqrun_igf_id.split('_')[0] # collect seqrun date from igf id
seqrun_date = datetime.datetime.strptime(seqrun_date,'%y%m%d').date() # identify actual date
seqrun_date = str(seqrun_date) # convert object to string
irods_upload = IGF_irods_uploader(irods_exe_dir) # create instance for irods upload
base_seq_dir = os.path.basename(fastq_dir) # get base name for the source dir
tarfile_name = \
'{0}_{1}_{2}.tar'.\
format(\
project_name,
base_seq_dir,
seqrun_date) # construct name of the tarfile
temp_work_dir = \
get_temp_dir(use_ephemeral_space=use_ephemeral_space) # get a temp dir
tarfile_name = \
os.path.join(
temp_work_dir,
tarfile_name) # create tarfile in the temp dir
with tarfile.open(tarfile_name, "w") as tar:
for root,_, files in os.walk(top=fastq_dir):
if samplesheet_filename in files:
samplesheet_file = \
os.path.join(os.path.abspath(root),
samplesheet_filename) # get samplesheet filepath
tmp_samplesheet_file = \
os.path.join(
temp_work_dir,
'{0}_{1}_{2}_{3}'.\
format(
project_name,
base_seq_dir,
seqrun_date,
samplesheet_filename))
copy2(
samplesheet_file,
tmp_samplesheet_file) # change samplesheet filename
tar.add(
tmp_samplesheet_file,
arcname=\
os.path.relpath(
tmp_samplesheet_file,
start=temp_work_dir)) # add samplesheet file to tar
if report_htmlname in files:
for file in files:
if fnmatch.fnmatch(os.path.join(root,file),report_html):
report_file = os.path.join(os.path.abspath(root),file) # get filepath for the report
tmp_report_file = \
os.path.join(\
temp_work_dir,
'{0}_{1}_{2}_{3}'.\
format(\
project_name,
base_seq_dir,
seqrun_date,
os.path.basename(report_file))) # change report name
copy2(report_file, tmp_report_file) # copy report file to temp
tar.add(tmp_report_file,
arcname=os.path.relpath(tmp_report_file,
start=temp_work_dir)) # add demultiplexing report to tar
if manifest_name in files:
manifest_file = \
os.path.join(os.path.abspath(root),
manifest_name) # get samplesheet filepath
tmp_manifest_file = \
os.path.join(\
temp_work_dir,
'{0}_{1}_{2}_{3}'.\
format(\
project_name,
base_seq_dir,
seqrun_date,
manifest_name)) # change manifest name
copy2(manifest_file,tmp_manifest_file) # copy manifest to temp
tar.add(tmp_manifest_file,
arcname=os.path.relpath(tmp_manifest_file,
start=temp_work_dir)) # add samplesheet file to tar
for file in files:
if fnmatch.fnmatch(file, '*.fastq.gz') and \
not fnmatch.fnmatch(file, 'Undetermined_*'):
fastq_file_path = os.path.join(os.path.abspath(root),file) # get filepath for the fastq files
tar.add(fastq_file_path,
arcname=os.path.relpath(fastq_file_path,
start=fastq_dir)) # add fastq file to tar
irods_upload.\
upload_fastqfile_and_create_collection(\
filepath=tarfile_name,
irods_user=username,
project_name=project_name,
run_igf_id=seqrun_igf_id,
flowcell_id=flowcell_id,
run_date=seqrun_date) # upload fastq data to irods
remove_dir(temp_work_dir) # remove temp dir once data uoload is done
except Exception as e:
message = \
'seqrun: {2}, Error in {0}: {1}'.\
format(\
self.__class__.__name__,
e,
seqrun_igf_id)
self.warning(message)
self.post_message_to_slack(message,reaction='fail') # post msg to slack for failed jobs
raise