def create_sample_template(job): """Creates a sample template Parameters ---------- job : qiita_db.processing_job.ProcessingJob The processing job performing the task """ with qdb.sql_connection.TRN: params = job.parameters.values fp = params['fp'] study = qdb.study.Study(int(params['study_id'])) is_mapping_file = params['is_mapping_file'] data_type = params['data_type'] with warnings.catch_warnings(record=True) as warns: if is_mapping_file: create_templates_from_qiime_mapping_file(fp, study, data_type) else: qdb.metadata_template.sample_template.SampleTemplate.create( qdb.metadata_template.util.load_template_to_dataframe(fp), study) remove(fp) if warns: msg = '\n'.join(set(str(w) for w in warns)) r_client.set( "sample_template_%s" % study.id, dumps({ 'job_id': job.id, 'alert_type': 'warning', 'alert_msg': msg })) job._set_status('success')
def create_sample_template(job): """Creates a sample template Parameters ---------- job : qiita_db.processing_job.ProcessingJob The processing job performing the task """ with qdb.sql_connection.TRN: params = job.parameters.values fp = params['fp'] study = qdb.study.Study(int(params['study_id'])) is_mapping_file = params['is_mapping_file'] data_type = params['data_type'] with warnings.catch_warnings(record=True) as warns: if is_mapping_file: create_templates_from_qiime_mapping_file(fp, study, data_type) else: qdb.metadata_template.sample_template.SampleTemplate.create( qdb.metadata_template.util.load_template_to_dataframe(fp), study) remove(fp) if warns: msg = '\n'.join(set(str(w.message) for w in warns)) r_client.set("sample_template_%s" % study.id, dumps({'job_id': job.id, 'alert_type': 'warning', 'alert_msg': msg})) job._set_status('success')
def create_sample_template(fp, study, is_mapping_file, data_type=None): """Creates a sample template Parameters ---------- fp : str The file path to the template file study : qiita_db.study.Study The study to add the sample template to is_mapping_file : bool Whether `fp` contains a mapping file or a sample template data_type : str, optional If `is_mapping_file` is True, the data type of the prep template to be created Returns ------- dict of {str: str} A dict of the form {'status': str, 'message': str} """ # The imports need to be in here because this code is executed in # the ipython workers import warnings from os import remove from qiita_db.metadata_template.sample_template import SampleTemplate from qiita_db.metadata_template.util import load_template_to_dataframe from qiita_ware.metadata_pipeline import ( create_templates_from_qiime_mapping_file) status = 'success' msg = '' try: with warnings.catch_warnings(record=True) as warns: if is_mapping_file: create_templates_from_qiime_mapping_file(fp, study, data_type) else: SampleTemplate.create(load_template_to_dataframe(fp), study) remove(fp) # join all the warning messages into one. Note that this # info will be ignored if an exception is raised if warns: msg = '\n'.join(set(str(w.message) for w in warns)) status = 'warning' except Exception as e: # Some error occurred while processing the sample template # Show the error to the user so they can fix the template status = 'danger' msg = str(e) return {'status': status, 'message': msg}
def test_create_templates_from_qiime_mapping_file_reverse_linker(self): curr_id = self.conn_handler.execute_fetchone( "SELECT last_value FROM " "qiita.prep_template_prep_template_id_seq")[0] obs_st, obs_pt = create_templates_from_qiime_mapping_file( StringIO(QIIME_MAP_WITH_REVERSE_LINKER_PRIMER), self.new_study, "16S") # Be green: clean the environment for template in [obs_st, obs_pt]: for _, fp in template.get_filepaths(): self._clean_up_files.append(fp) self.assertEqual(obs_st.id, self.new_study.id) self.assertEqual(obs_pt.id, curr_id + 1) # Check that each template has the correct columns exp = { "physical_specimen_location", "physical_specimen_remaining", "dna_extracted", "sample_type", "host_subject_id", "latitude", "longitude", "taxon_id", "scientific_name", "collection_timestamp", "description" } self.assertEqual(set(obs_st.categories()), exp) exp = { "barcode", "primer", "center_name", "run_prefix", "platform", "library_construction_protocol", "instrument_model", "experiment_design_description", "reverselinkerprimer" } self.assertEqual(set(obs_pt.categories()), exp)
def test_create_templates_from_qiime_mapping_file(self): new_pt_id = get_count('qiita.prep_template') + 1 obs_st, obs_pt = create_templates_from_qiime_mapping_file( StringIO(QIIME_MAP), self.new_study, "16S") # Be green: clean the environment for template in [obs_st, obs_pt]: for _, fp in template.get_filepaths(): self._clean_up_files.append(fp) self.assertEqual(obs_st.id, self.new_study.id) self.assertEqual(obs_pt.id, new_pt_id) # Check that each template has the correct columns exp = { "physical_specimen_location", "physical_specimen_remaining", "dna_extracted", "sample_type", "host_subject_id", "latitude", "longitude", "taxon_id", "scientific_name", "collection_timestamp", "description" } self.assertEqual(set(obs_st.categories()), exp) exp = { "barcode", "primer", "center_name", "run_prefix", "platform", "library_construction_protocol", "instrument_model", "experiment_design_description" } self.assertEqual(set(obs_pt.categories()), exp)
def test_create_templates_from_qiime_mapping_file_reverse_linker(self): new_pt_id = get_count('qiita.prep_template') + 1 obs_st, obs_pt = create_templates_from_qiime_mapping_file( StringIO(QIIME_MAP_WITH_REVERSE_LINKER_PRIMER), self.new_study, "16S") # Be green: clean the environment for template in [obs_st, obs_pt]: for _, fp in template.get_filepaths(): self._clean_up_files.append(fp) self.assertEqual(obs_st.id, self.new_study.id) self.assertEqual(obs_pt.id, new_pt_id) # Check that each template has the correct columns exp = {"physical_specimen_location", "physical_specimen_remaining", "dna_extracted", "sample_type", "host_subject_id", "latitude", "longitude", "taxon_id", "scientific_name", "collection_timestamp", "description"} self.assertEqual(set(obs_st.categories()), exp) exp = {"barcode", "primer", "center_name", "run_prefix", "platform", "library_construction_protocol", "experiment_design_description", "reverselinkerprimer"} self.assertEqual(set(obs_pt.categories()), exp)
def remove_add_study_template(self, raw_data, study_id, fp_rsp, data_type, is_mapping_file): """Replace prep templates, raw data, and sample template with a new one """ if is_mapping_file and data_type == "": raise ValueError("Please, choose a data type if uploading a QIIME " "mapping file") for rd in raw_data(): rd = RawData(rd) for pt in rd.prep_templates: if PrepTemplate.exists(pt): PrepTemplate.delete(pt) if SampleTemplate.exists(study_id): SampleTemplate.delete(study_id) if is_mapping_file: create_templates_from_qiime_mapping_file(fp_rsp, Study(study_id), int(data_type)) else: SampleTemplate.create(load_template_to_dataframe(fp_rsp), Study(study_id)) remove(fp_rsp)
def process_sample_template(self, study, user, callback): """Process a sample template from the POST method Parameters ---------- study : Study The current study object user : User The current user object callback : function The callback function to call with the results once the processing is done Raises ------ HTTPError If the sample template file does not exists """ # If we are on this function, the arguments "sample_template" and # "data_type" must be defined. If not, let tornado raise its error sample_template = self.get_argument('sample_template') data_type = self.get_argument('data_type') # Get the uploads folder _, base_fp = get_mountpoint("uploads")[0] # Get the path of the sample template in the uploads folder fp_rsp = join(base_fp, str(study.id), sample_template) if not exists(fp_rsp): # The file does not exist, fail nicely raise HTTPError(404, "This file doesn't exist: %s" % fp_rsp) # Define here the message and message level in case of success msg = "The sample template '%s' has been added" % sample_template msg_level = "success" is_mapping_file = looks_like_qiime_mapping_file(fp_rsp) try: if is_mapping_file and not data_type: raise ValueError("Please, choose a data type if uploading a " "QIIME mapping file") with warnings.catch_warnings(record=True) as warns: if is_mapping_file: create_templates_from_qiime_mapping_file(fp_rsp, study, int(data_type)) else: SampleTemplate.create(load_template_to_dataframe(fp_rsp), study) remove(fp_rsp) # join all the warning messages into one. Note that this # info will be ignored if an exception is raised if warns: msg = '; '.join([convert_text_html(str(w.message)) for w in warns]) msg_level = 'warning' except (TypeError, QiitaDBColumnError, QiitaDBExecutionError, QiitaDBDuplicateError, IOError, ValueError, KeyError, CParserError, QiitaDBDuplicateHeaderError, QiitaDBError, QiitaWareError) as e: # Some error occurred while processing the sample template # Show the error to the user so they can fix the template error_msg = ('parsing the QIIME mapping file' if is_mapping_file else 'parsing the sample template') msg = html_error_message % (error_msg, basename(fp_rsp), str(e)) msg = convert_text_html(msg) msg_level = "danger" callback((msg, msg_level, None, None, None))
def test_create_templates_from_qiime_mapping_file_error(self): with self.assertRaises(QiitaWareError): create_templates_from_qiime_mapping_file(StringIO(QIIME_MAP_ERROR), self.new_study, "16S")
def test_create_templates_from_qiime_mapping_file_error(self): with self.assertRaises(QiitaWareError): create_templates_from_qiime_mapping_file( StringIO(QIIME_MAP_ERROR), self.new_study, "16S")
def process_sample_template(self, study, user, callback): """Process a sample template from the POST method Parameters ---------- study : Study The current study object user : User The current user object callback : function The callback function to call with the results once the processing is done Raises ------ HTTPError If the sample template file does not exists """ # If we are on this function, the arguments "sample_template" and # "data_type" must be defined. If not, let tornado raise its error sample_template = self.get_argument('sample_template') data_type = self.get_argument('data_type') # Get the uploads folder _, base_fp = get_mountpoint("uploads")[0] # Get the path of the sample template in the uploads folder fp_rsp = join(base_fp, str(study.id), sample_template) if not exists(fp_rsp): # The file does not exist, fail nicely raise HTTPError(404, "This file doesn't exist: %s" % fp_rsp) # Define here the message and message level in case of success msg = "The sample template '%s' has been added" % sample_template msg_level = "success" is_mapping_file = looks_like_qiime_mapping_file(fp_rsp) try: if is_mapping_file and not data_type: raise ValueError("Please, choose a data type if uploading a " "QIIME mapping file") with warnings.catch_warnings(record=True) as warns: if is_mapping_file: create_templates_from_qiime_mapping_file( fp_rsp, study, int(data_type)) else: SampleTemplate.create(load_template_to_dataframe(fp_rsp), study) remove(fp_rsp) # join all the warning messages into one. Note that this # info will be ignored if an exception is raised if warns: msg = '; '.join( [convert_text_html(str(w.message)) for w in warns]) msg_level = 'warning' except (TypeError, QiitaDBColumnError, QiitaDBExecutionError, QiitaDBDuplicateError, IOError, ValueError, KeyError, CParserError, QiitaDBDuplicateHeaderError, QiitaDBError, QiitaWareError) as e: # Some error occurred while processing the sample template # Show the error to the user so they can fix the template error_msg = ('parsing the QIIME mapping file' if is_mapping_file else 'parsing the sample template') msg = html_error_message % (error_msg, basename(fp_rsp), str(e)) msg = convert_text_html(msg) msg_level = "danger" callback((msg, msg_level, None, None, None))