def test_fetch_data_authority_for_project(self): pa = ProjectAdaptor(**{'session_class': self.session_class}) pa.start_session() pa_results1 = pa.fetch_data_authority_for_project( project_igf_id='IGFP0001_test_22-8-2017_rna') self.assertEqual(pa_results1.email_id, '*****@*****.**') pa_results2 = pa.fetch_data_authority_for_project( project_igf_id='IGFP0002_test_22-8-2017_rna') self.assertEqual(pa_results2, None) pa.close_session()
def run(self): ''' A ehive runnable method for uploading analysis files to irods server :param file_list: A list of file paths to upload to irods :param irods_exe_dir: Irods executable directory :param project_igf_id: Name of the project :param analysis_name: A string for analysis name, default is 'default' :param dir_path_list: A list of directory structure for irod server, default None for using datestamp :param file_tag: A text string for adding tag to collection, default None for only project_name ''' try: project_igf_id = self.param_required('project_igf_id') igf_session_class = self.param_required('igf_session_class') irods_exe_dir = self.param_required('irods_exe_dir') file_list = self.param_required('file_list') analysis_name = self.param_required('analysis_name') dir_path_list = self.param_required('dir_path_list') file_tag = self.param_required('file_tag') pa = ProjectAdaptor(**{'session_class':igf_session_class}) pa.start_session() user = \ pa.fetch_data_authority_for_project( project_igf_id=project_igf_id) # fetch user info from db pa.close_session() if user is None: raise ValueError('No user found for project {0}'.format(project_igf_id)) username = user.username # get username for irods irods_upload = IGF_irods_uploader(irods_exe_dir) # create instance for irods upload for file_path in file_list: if not os.path.exists(file_path): raise IOError('Failed to find file {0} for irods upload'.\ format(file_path)) irods_upload.\ upload_analysis_results_and_create_collection( file_list=file_list, irods_user=username, project_name=project_igf_id, analysis_name=analysis_name, dir_path_list=dir_path_list, file_tag=file_tag) # upload analysis results to irods and build collection except Exception as e: message = \ 'project: {2}, Error in {0}: {1}'.format( self.__class__.__name__, e, project_igf_id) self.warning(message) self.post_message_to_slack(message,reaction='fail') # post msg to slack for failed jobs raise
def run(self): try: fastq_dir = self.param_required('fastq_dir') seqrun_igf_id = self.param_required('seqrun_igf_id') project_name = self.param_required('project_name') igf_session_class = self.param_required('igf_session_class') irods_exe_dir = self.param_required('irods_exe_dir') flowcell_id = self.param_required('flowcell_id') samplesheet_filename = self.param('samplesheet_filename') manifest_name = self.param_required('manifest_name') report_html = self.param('report_html') use_ephemeral_space = self.param('use_ephemeral_space') pa = ProjectAdaptor(**{'session_class':igf_session_class}) pa.start_session() user = \ pa.fetch_data_authority_for_project(\ project_igf_id=project_name) # fetch user info from db pa.close_session() if user is None: raise ValueError('No user found for project {0}'.\ format(project_name)) username = user.username # get username for irods report_htmlname = os.path.basename(report_html) seqrun_date = seqrun_igf_id.split('_')[0] # collect seqrun date from igf id seqrun_date = datetime.datetime.strptime(seqrun_date,'%y%m%d').date() # identify actual date seqrun_date = str(seqrun_date) # convert object to string irods_upload = IGF_irods_uploader(irods_exe_dir) # create instance for irods upload base_seq_dir = os.path.basename(fastq_dir) # get base name for the source dir tarfile_name = \ '{0}_{1}_{2}.tar'.\ format(\ project_name, base_seq_dir, seqrun_date) # construct name of the tarfile temp_work_dir = \ get_temp_dir(use_ephemeral_space=use_ephemeral_space) # get a temp dir tarfile_name = \ os.path.join( temp_work_dir, tarfile_name) # create tarfile in the temp dir with tarfile.open(tarfile_name, "w") as tar: for root,_, files in os.walk(top=fastq_dir): if samplesheet_filename in files: samplesheet_file = \ os.path.join(os.path.abspath(root), samplesheet_filename) # get samplesheet filepath tmp_samplesheet_file = \ os.path.join( temp_work_dir, '{0}_{1}_{2}_{3}'.\ format( project_name, base_seq_dir, seqrun_date, samplesheet_filename)) copy2( samplesheet_file, tmp_samplesheet_file) # change samplesheet filename tar.add( tmp_samplesheet_file, arcname=\ os.path.relpath( tmp_samplesheet_file, start=temp_work_dir)) # add samplesheet file to tar if report_htmlname in files: for file in files: if fnmatch.fnmatch(os.path.join(root,file),report_html): report_file = os.path.join(os.path.abspath(root),file) # get filepath for the report tmp_report_file = \ os.path.join(\ temp_work_dir, '{0}_{1}_{2}_{3}'.\ format(\ project_name, base_seq_dir, seqrun_date, os.path.basename(report_file))) # change report name copy2(report_file, tmp_report_file) # copy report file to temp tar.add(tmp_report_file, arcname=os.path.relpath(tmp_report_file, start=temp_work_dir)) # add demultiplexing report to tar if manifest_name in files: manifest_file = \ os.path.join(os.path.abspath(root), manifest_name) # get samplesheet filepath tmp_manifest_file = \ os.path.join(\ temp_work_dir, '{0}_{1}_{2}_{3}'.\ format(\ project_name, base_seq_dir, seqrun_date, manifest_name)) # change manifest name copy2(manifest_file,tmp_manifest_file) # copy manifest to temp tar.add(tmp_manifest_file, arcname=os.path.relpath(tmp_manifest_file, start=temp_work_dir)) # add samplesheet file to tar for file in files: if fnmatch.fnmatch(file, '*.fastq.gz') and \ not fnmatch.fnmatch(file, 'Undetermined_*'): fastq_file_path = os.path.join(os.path.abspath(root),file) # get filepath for the fastq files tar.add(fastq_file_path, arcname=os.path.relpath(fastq_file_path, start=fastq_dir)) # add fastq file to tar irods_upload.\ upload_fastqfile_and_create_collection(\ filepath=tarfile_name, irods_user=username, project_name=project_name, run_igf_id=seqrun_igf_id, flowcell_id=flowcell_id, run_date=seqrun_date) # upload fastq data to irods remove_dir(temp_work_dir) # remove temp dir once data uoload is done except Exception as e: message = \ 'seqrun: {2}, Error in {0}: {1}'.\ format(\ self.__class__.__name__, e, seqrun_igf_id) self.warning(message) self.post_message_to_slack(message,reaction='fail') # post msg to slack for failed jobs raise