def test_process_project_data_and_account(self): fa=Find_and_register_new_project_data(projet_info_path=os.path.join('.','data/check_project_data'),\ dbconfig=self.dbconfig,\ user_account_template='template/email_notification/send_new_account_info.txt',\ log_slack=False,\ setup_irods=False,\ notify_user=False,\ check_hpc_user=False,\ ) fa.process_project_data_and_account() dbparam = None with open(self.dbconfig, 'r') as json_data: dbparam = json.load(json_data) base = BaseAdaptor(**dbparam) base.start_session() pa=ProjectAdaptor(**{'session':base.session}) project_exists=pa.check_project_records_igf_id(project_igf_id='IGFP0002_test_23-5-2017_rna') self.assertTrue(project_exists) ua=UserAdaptor(**{'session':base.session}) user_exists=ua.check_user_records_email_id(email_id='*****@*****.**') self.assertTrue(user_exists) user1=ua.fetch_user_records_email_id(user_email_id='*****@*****.**') self.assertEqual(user1.name,'User2') sa=SampleAdaptor(**{'session':base.session}) sample_exists=sa.check_sample_records_igf_id(sample_igf_id='IGF00006') self.assertTrue(sample_exists) project_user_exists=pa.check_existing_project_user(project_igf_id='IGFP0002_test_23-5-2017_rna',\ email_id='*****@*****.**') self.assertTrue(project_user_exists) project_user_exists=pa.check_existing_project_user(project_igf_id='IGFP0002_test_23-5-2017_rna',\ email_id='*****@*****.**') self.assertTrue(project_user_exists) base.close_session()
def test_store_sample_and_attribute_data(self): sa = SampleAdaptor(**{'session_class': self.session_class}) sample_data = [ { 'sample_igf_id': 'IGFS001', 'library_id': 'IGFS001', 'project_igf_id': 'IGFP0001_test_22-8-2017_rna', }, { 'sample_igf_id': 'IGFS002', 'library_id': 'IGFS002', 'project_igf_id': 'IGFP0001_test_22-8-2017_rna', }, { 'sample_igf_id': 'IGFS003', 'library_id': 'IGFS003', 'project_igf_id': 'IGFP0001_test_22-8-2017_rna', }, { 'sample_igf_id': 'IGFS004', 'library_id': 'IGFS004', 'project_igf_id': 'IGFP0001_test_22-8-2017_rna', }, ] sa.start_session() sa.store_sample_and_attribute_data(data=sample_data) sa1 = sa.check_sample_records_igf_id(sample_igf_id='IGFS001') sa.close_session() self.assertEqual(sa1, True)
def _check_existing_data(self, data, dbsession, table_name, check_column='EXISTS'): ''' An internal function for checking and registering project info :param data: A pandas data series :param dbsession: A sqlalchemy database session object :param table_name: A database table name :param check_column: Column name for existing data ''' try: if not isinstance(data, pd.Series): raise ValueError('Expecting a data series and got {0}'.format( type(data))) if table_name == 'project': if self.project_lookup_column in data and \ not pd.isnull(data[self.project_lookup_column]): project_igf_id = data[self.project_lookup_column] pa = ProjectAdaptor(**{'session': dbsession }) # connect to project adaptor project_exists = pa.check_project_records_igf_id( project_igf_id) if project_exists: # store data only if project is not existing data[check_column] = True else: data[check_column] = False return data else: raise ValueError('Missing or empty required column {0}'.\ format(self.project_lookup_column)) elif table_name == 'user': if self.user_lookup_column in data and \ not pd.isnull(data[self.user_lookup_column]): user_email = data[self.user_lookup_column] ua = UserAdaptor(**{'session': dbsession}) # connect to user adaptor user_exists = ua.check_user_records_email_id( email_id=user_email) if user_exists: # store data only if user is not existing data[check_column] = True else: data[check_column] = False return data else: raise ValueError('Missing or empty required column {0}'.\ format(self.user_lookup_column)) elif table_name == 'sample': if self.sample_lookup_column in data and \ not pd.isnull(data[self.sample_lookup_column]): project_igf_id = data[self.project_lookup_column] sample_igf_id = data[self.sample_lookup_column] sa = SampleAdaptor(**{'session': dbsession }) # connect to sample adaptor sample_project_exists=sa.check_project_and_sample(project_igf_id=project_igf_id,\ sample_igf_id=sample_igf_id) # check for existing sample_id and project-id combination if sample_project_exists: # store data only if sample is not existing data[check_column] = True else: sample_exists = sa.check_sample_records_igf_id( sample_igf_id) # check for existing sample if sample_exists: raise ValueError('Sample {0} exists in database but not associated with project {1}'.\ format(sample_igf_id,project_igf_id)) # inconsistency in sample project combination data[check_column] = False return data else: raise ValueError('Missing or empty required column {0}'.\ format(self.sample_lookup_column)) elif table_name == 'project_user': if self.user_lookup_column in data and \ not pd.isnull(data[self.user_lookup_column]) and \ self.project_lookup_column in data and \ not pd.isnull(data[self.project_lookup_column]): project_igf_id = data[self.project_lookup_column] user_email = data[self.user_lookup_column] pa = ProjectAdaptor(**{'session': dbsession }) # connect to project adaptor project_user_exists=pa.check_existing_project_user(project_igf_id,\ email_id=user_email) if user_email != self.default_user_email and \ (self.data_authority_column not in data or \ pd.isnull(data[self.data_authority_column])): data[ self. data_authority_column] = True # set user as data authority, filter default user if project_user_exists: # store data only if sample is not existing data[check_column] = True else: data[check_column] = False return data else: raise ValueError('Missing or empty required column {0}, {1}'.\ format(self.project_lookup_column,\ self.user_lookup_column)) else: raise ValueError('table {0} not supported'.format(table_name)) except: raise
def load_file_to_disk_and_db(self, input_file_list, withdraw_exisitng_collection=True, autosave_db=True, file_suffix=None, force=True, remove_file=False): ''' A method for loading analysis results to disk and database. File will be moved to a new path if base_path is present. Directory structure of the final path is based on the collection_table information. Following will be the final directory structure if base_path is present project - base_path/project_igf_id/analysis_name sample - base_path/project_igf_id/sample_igf_id/analysis_name experiment - base_path/project_igf_id/sample_igf_id/experiment_igf_id/analysis_name run - base_path/project_igf_id/sample_igf_id/experiment_igf_id/run_igf_id/analysis_name :param input_file_list: A list of input file to load, all using the same collection info :param withdraw_exisitng_collection: Remove existing collection group, DO NOT use this while loading a list of files :param autosave_db: Save changes to database, default True :param file_suffix: Use a specific file suffix, use None if it should be same as original file e.g. input.vcf.gz to output.vcf.gz :param force: Toggle for removing existing file, default True :param remove_file: A toggle for removing existing file from disk, default False :returns: A list of final filepath ''' try: project_igf_id = None sample_igf_id = None experiment_igf_id = None experiment_igf_id = None run_igf_id = None output_path_list = list() # define empty output list dbconnected = False if self.collection_name is None or \ self.collection_type is None or \ self.collection_table is None: raise ValueError('File collection information is incomplete' ) # check for collection information base = BaseAdaptor(**{'session_class': self.dbsession_class}) base.start_session() # connect to db dbconnected = True if self.base_path is not None: if self.collection_table == 'sample': sa = SampleAdaptor(**{'session': base.session}) sample_igf_id = self.collection_name sample_exists = sa.check_sample_records_igf_id( sample_igf_id=sample_igf_id) if not sample_exists: raise ValueError('Sample {0} not found in db'.\ format(sample_igf_id)) project_igf_id = \ sa.fetch_sample_project(sample_igf_id=sample_igf_id) # fetch project id for sample elif self.collection_table == 'experiment': ea = ExperimentAdaptor(**{'session': base.session}) experiment_igf_id = self.collection_name experiment_exists = \ ea.check_experiment_records_id( experiment_igf_id=experiment_igf_id) if not experiment_exists: raise ValueError('Experiment {0} not present in database'.\ format(experiment_igf_id)) (project_igf_id,sample_igf_id) = \ ea.fetch_project_and_sample_for_experiment( experiment_igf_id=experiment_igf_id) # fetch project and sample id for experiment elif self.collection_table == 'run': ra = RunAdaptor(**{'session': base.session}) run_igf_id = self.collection_name run_exists = ra.check_run_records_igf_id( run_igf_id=run_igf_id) if not run_exists: raise ValueError('Run {0} not found in database'.\ format(run_igf_id)) (project_igf_id,sample_igf_id,experiment_igf_id) = \ ra.fetch_project_sample_and_experiment_for_run( run_igf_id=run_igf_id) # fetch project, sample and experiment id for run elif self.collection_table == 'project': pa = ProjectAdaptor(**{'session': base.session}) project_igf_id = self.collection_name project_exists = \ pa.check_project_records_igf_id( project_igf_id=project_igf_id) if not project_exists: raise ValueError('Project {0} not found in database'.\ format(project_igf_id)) if self.rename_file and self.analysis_name is None: raise ValueError('Analysis name is required for renaming file' ) # check analysis name for input_file in input_file_list: final_path = '' if self.base_path is None: # do not move file if base_path is absent final_path = os.path.dirname(input_file) else: # move file path if self.collection_table == 'project': if project_igf_id is None: raise ValueError('Missing project id for collection {0}'.\ format(self.collection_name)) final_path = \ os.path.join( self.base_path, project_igf_id, self.analysis_name) # final path for project elif self.collection_table == 'sample': if project_igf_id is None or \ sample_igf_id is None: raise ValueError('Missing project and sample id for collection {0}'.\ format(self.collection_name)) final_path = \ os.path.join( self.base_path, project_igf_id, sample_igf_id, self.analysis_name) # final path for sample elif self.collection_table == 'experiment': if project_igf_id is None or \ sample_igf_id is None or \ experiment_igf_id is None: raise ValueError('Missing project,sample and experiment id for collection {0}'.\ format(self.collection_name)) final_path = \ os.path.join( self.base_path, project_igf_id, sample_igf_id, experiment_igf_id, self.analysis_name) # final path for experiment elif self.collection_table == 'run': if project_igf_id is None or \ sample_igf_id is None or \ experiment_igf_id is None or \ run_igf_id is None: raise ValueError('Missing project,sample,experiment and run id for collection {0}'.\ format(self.collection_name)) final_path = \ os.path.join(\ self.base_path, project_igf_id, sample_igf_id, experiment_igf_id, run_igf_id, self.analysis_name) # final path for run if self.rename_file: new_filename = \ self.get_new_file_name( input_file=input_file, file_suffix=file_suffix) final_path = \ os.path.join( final_path, new_filename) # get new filepath else: final_path = \ os.path.join( final_path, os.path.basename(input_file)) if final_path != input_file: # move file if its required final_path = preprocess_path_name( input_path=final_path ) # remove unexpected characters from file path move_file(source_path=input_file, destinationa_path=final_path, force=force ) # move or overwrite file to destination dir output_path_list.append( final_path) # add final path to the output list self.create_or_update_analysis_collection( file_path=final_path, dbsession=base.session, withdraw_exisitng_collection=withdraw_exisitng_collection, remove_file=remove_file, autosave_db=autosave_db) # load new file collection in db if autosave_db: base.commit_session() # save changes to db for each file base.commit_session() # save changes to db base.close_session() # close db connection return output_path_list except: if dbconnected: base.rollback_session() base.close_session() raise