def test_prep_template_post_req(self): obs = prep_template_post_req(1, '*****@*****.**', 'update.txt', '16S') exp = {'status': 'warning', 'message': [ 'Sample names were already prefixed with the study id.', ('Some columns required to generate a QIIME-compliant ' 'mapping file are not present in the template. A ' 'placeholder value (XXQIITAXX) has been used to populate ' 'these columns. Missing columns: BarcodeSequence, ' 'LinkerPrimerSequence'), ('Some functionality will be disabled due to missing ' 'columns:'), ('\tDemultiplexing with multiple input files disabled.: ' 'barcode, primer, run_prefix;'), '\tDemultiplexing disabled.: barcode, primer;', ('\tEBI submission disabled: center_name, ' 'experiment_design_description, instrument_model, ' 'library_construction_protocol, platform, primer.'), ('See the Templates tutorial for a description of these ' 'fields.')], 'file': 'update.txt', 'id': 'ignored in test'} self.assertItemsEqual(obs['message'].split('\n'), exp['message']) self.assertEqual(obs['status'], exp['status']) self.assertEqual(obs['file'], exp['file']) self.assertIsInstance(obs['id'], int) # Make sure new prep template added prep = PrepTemplate(obs['id']) self.assertEqual(prep.data_type(), '16S') self.assertEqual([x for x in prep.keys()], ['1.SKD6.640190']) self.assertEqual([x._to_dict() for x in prep.values()], [{'new_col': 'new_value'}])
def test_prep_template_post_req(self): obs = prep_template_post_req(1, '*****@*****.**', 'update.txt', '16S', name=" ") exp = {'status': 'warning', 'message': [ ('Some columns required to generate a QIIME-compliant ' 'mapping file are not present in the template. A ' 'placeholder value (XXQIITAXX) has been used to populate ' 'these columns. Missing columns: BarcodeSequence, ' 'LinkerPrimerSequence'), ('Some functionality will be disabled due to missing ' 'columns:'), ('\tDemultiplexing with multiple input files disabled.: ' 'barcode, primer, run_prefix;'), '\tDemultiplexing disabled.: barcode;', ('\tEBI submission disabled: center_name, ' 'experiment_design_description, instrument_model, ' 'library_construction_protocol, platform.'), ('See the Templates tutorial for a description of these ' 'fields.')], 'file': 'update.txt', 'id': 'ignored in test'} self.assertItemsEqual(obs['message'].split('\n'), exp['message']) self.assertEqual(obs['status'], exp['status']) self.assertEqual(obs['file'], exp['file']) self.assertIsInstance(obs['id'], int) # Make sure new prep template added prep = PrepTemplate(obs['id']) self.assertEqual(prep.data_type(), '16S') self.assertEqual([x for x in prep.keys()], ['1.SKD6.640190']) self.assertEqual([x._to_dict() for x in prep.values()], [{'new_col': 'new_value'}]) self.assertEqual(prep.name, "Prep information %s" % prep.id)
def test_prep_template_post_req(self): obs = prep_template_post_req(1, '*****@*****.**', 'update.txt', '16S', name=" ") exp = { 'status': 'warning', 'message': [ 'Both a converter and dtype were specified for column ' 'sample_name - only the converter will be used', 'Some ' 'functionality will be disabled due to missing columns:', '\tEBI submission disabled: center_name, ' 'experiment_design_description, instrument_model, ' 'library_construction_protocol, platform;', '\tDemultiplexing disabled.: barcode;', '\tDemultiplexing ' 'with multiple input files disabled.: barcode, primer, ' 'run_prefix.', 'See the Templates tutorial for a ' 'description of these fields.', 'Some columns required to ' 'generate a QIIME-compliant mapping file are not present ' 'in the template. A placeholder value (XXQIITAXX) ' 'has been used to populate these columns. Missing columns: ' 'BarcodeSequence, LinkerPrimerSequence' ], 'file': 'update.txt', 'id': 'ignored in test' } self.assertCountEqual(obs['message'].split('\n'), exp['message']) self.assertEqual(obs['status'], exp['status']) self.assertEqual(obs['file'], exp['file']) self.assertIsInstance(obs['id'], int) # Make sure new prep template added prep = PrepTemplate(obs['id']) self.assertEqual(prep.data_type(), '16S') self.assertEqual([x for x in prep.keys()], ['1.SKD6.640190']) self.assertEqual([x._to_dict() for x in prep.values()], [{ 'new_col': 'new_value' }]) self.assertEqual(prep.name, "Prep information %s" % prep.id)
def test_prep_template_post_req(self): new_id = get_count('qiita.prep_template') + 1 obs = prep_template_post_req(1, '*****@*****.**', 'update.txt', '16S') exp = {'status': 'warning', 'message': 'Sample names were already prefixed with the study ' 'id.\nSome functionality will be disabled due to ' 'missing columns:\n\tDemultiplexing with multiple ' 'input files disabled. If your raw data includes ' 'multiple raw input files, you will not be able to ' 'preprocess your raw data: barcode, primer, ' 'run_prefix;\n\tDemultiplexing disabled. You will ' 'not be able to preprocess your raw data: barcode, ' 'primer;\n\tEBI submission disabled: center_name, ' 'experiment_design_description, instrument_model, ' 'library_construction_protocol, platform, primer.' '\nSee the Templates tutorial for a description of ' 'these fields.\nSome columns required to generate a ' 'QIIME-compliant mapping file are not present in the' ' template. A placeholder value (XXQIITAXX) has been' ' used to populate these columns. Missing columns: ' 'BarcodeSequence, LinkerPrimerSequence', 'file': 'update.txt', 'id': new_id} self.assertItemsEqual(obs['message'].split('\n'), exp['message'].split('\n')) self.assertEqual(obs['status'], exp['status']) self.assertEqual(obs['file'], exp['file']) self.assertEqual(obs['id'], exp['id']) # Make sure new prep template added prep = PrepTemplate(new_id) self.assertEqual(prep.data_type(), '16S') self.assertEqual([x for x in prep.keys()], ['1.SKD6.640190']) self.assertEqual([x._to_dict() for x in prep.values()], [{'new_col': 'new_value'}])
def test_delete_sample_or_column(self): st = SampleTemplate(1) # Delete a sample template column job = self._create_job( 'delete_sample_or_column', { 'obj_class': 'SampleTemplate', 'obj_id': 1, 'sample_or_col': 'columns', 'name': 'season_environment' }) private_task(job.id) self.assertEqual(job.status, 'success') self.assertNotIn('season_environment', st.categories()) # Delete a sample template sample - need to add one # sample that we will remove npt.assert_warns( QiitaDBWarning, st.extend, pd.DataFrame.from_dict({'Sample1': { 'taxon_id': '9606' }}, orient='index', dtype=str)) self.assertIn('1.Sample1', st.keys()) job = self._create_job( 'delete_sample_or_column', { 'obj_class': 'SampleTemplate', 'obj_id': 1, 'sample_or_col': 'samples', 'name': '1.Sample1' }) private_task(job.id) self.assertEqual(job.status, 'success') self.assertNotIn('1.Sample1', st.keys()) # Delete a prep template column pt = PrepTemplate(1) job = self._create_job( 'delete_sample_or_column', { 'obj_class': 'PrepTemplate', 'obj_id': 1, 'sample_or_col': 'columns', 'name': 'target_subfragment' }) private_task(job.id) self.assertEqual(job.status, 'success') self.assertNotIn('target_subfragment', pt.categories()) # Delete a prep template sample metadata = pd.DataFrame.from_dict( { '1.SKB8.640193': { 'barcode': 'GTCCGCAAGTTA', 'primer': 'GTGCCAGCMGCCGCGGTAA' }, '1.SKD8.640184': { 'barcode': 'CGTAGAGCTCTC', 'primer': 'GTGCCAGCMGCCGCGGTAA' } }, orient='index', dtype=str) pt = npt.assert_warns(QiitaDBWarning, PrepTemplate.create, metadata, Study(1), "16S") job = self._create_job( 'delete_sample_or_column', { 'obj_class': 'PrepTemplate', 'obj_id': pt.id, 'sample_or_col': 'samples', 'name': '1.SKD8.640184' }) private_task(job.id) self.assertNotIn('1.SKD8.640184', pt.keys()) # Test exceptions job = self._create_job( 'delete_sample_or_column', { 'obj_class': 'UnknownClass', 'obj_id': 1, 'sample_or_col': 'columns', 'name': 'column' }) private_task(job.id) self.assertEqual(job.status, 'error') self.assertIn( 'Unknown value "UnknownClass". Choose between ' '"SampleTemplate" and "PrepTemplate"', job.log.msg) job = self._create_job( 'delete_sample_or_column', { 'obj_class': 'SampleTemplate', 'obj_id': 1, 'sample_or_col': 'unknown', 'name': 'column' }) private_task(job.id) self.assertEqual(job.status, 'error') self.assertIn( 'Unknown value "unknown". Choose between "samples" ' 'and "columns"', job.log.msg)
def prep_template_ajax_get_req(user_id, prep_id): """Returns the prep tempalte information needed for the AJAX handler Parameters ---------- user_id : str The user id prep_id : int The prep template id Returns ------- dict of {str: object} A dictionary with the following keys: - status: str, whether the request is successful or not - message: str, if the request is unsuccessful, a human readable error - name: str, the name of the prep template - files: list of str, the files available to update the prep template - download_prep: int, the filepath_id of the prep file - download_qiime, int, the filepath_id of the qiime mapping file - num_samples: int, the number of samples present in the template - num_columns: int, the number of columns present in the template - investigation_type: str, the investigation type of the template - ontology: str, dict of {str, list of str} containing the information of the ENA ontology - artifact_attached: bool, whether the template has an artifact attached - study_id: int, the study id of the template """ # Currently there is no name attribute, but it will be soon name = "Prep information %d" % prep_id pt = PrepTemplate(prep_id) job_info = r_client.get(PREP_TEMPLATE_KEY_FORMAT % prep_id) if job_info: job_info = loads(job_info) job_id = job_info['job_id'] if job_id: redis_info = loads(r_client.get(job_id)) processing = redis_info['status_msg'] == 'Running' if processing: alert_type = 'info' alert_msg = 'This prep template is currently being updated' elif redis_info['status_msg'] == 'Success': alert_type = redis_info['return']['status'] alert_msg = redis_info['return']['message'].replace('\n', '</br>') payload = {'job_id': None, 'status': alert_type, 'message': alert_msg} r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id, dumps(payload)) else: alert_type = redis_info['return']['status'] alert_msg = redis_info['return']['message'].replace('\n', '</br>') else: processing = False alert_type = job_info['status'] alert_msg = job_info['message'].replace('\n', '</br>') else: processing = False alert_type = '' alert_msg = '' artifact_attached = pt.artifact is not None study_id = pt.study_id files = [f for _, f in get_files_from_uploads_folders(study_id) if f.endswith(('.txt', '.tsv'))] # The call to list is needed because keys is an iterator num_samples = len(list(pt.keys())) num_columns = len(pt.categories()) investigation_type = pt.investigation_type # Retrieve the information to download the prep template and QIIME # mapping file. See issue https://github.com/biocore/qiita/issues/1675 download_prep = [] download_qiime = [] for fp_id, fp in pt.get_filepaths(): if 'qiime' in basename(fp): download_qiime.append(fp_id) else: download_prep.append(fp_id) download_prep = download_prep[0] download_qiime = download_qiime[0] ontology = _get_ENA_ontology() editable = Study(study_id).can_edit(User(user_id)) and not processing return {'status': 'success', 'message': '', 'name': name, 'files': files, 'download_prep': download_prep, 'download_qiime': download_qiime, 'num_samples': num_samples, 'num_columns': num_columns, 'investigation_type': investigation_type, 'ontology': ontology, 'artifact_attached': artifact_attached, 'study_id': study_id, 'editable': editable, 'data_type': pt.data_type(), 'alert_type': alert_type, 'is_submitted_to_ebi': pt.is_submitted_to_ebi, 'alert_message': alert_msg}
def prep_template_ajax_get_req(user_id, prep_id): """Returns the prep tempalte information needed for the AJAX handler Parameters ---------- user_id : str The user id prep_id : int The prep template id Returns ------- dict of {str: object} A dictionary with the following keys: - status: str, whether the request is successful or not - message: str, if the request is unsuccessful, a human readable error - name: str, the name of the prep template - files: list of str, the files available to update the prep template - download_prep: int, the filepath_id of the prep file - download_qiime, int, the filepath_id of the qiime mapping file - num_samples: int, the number of samples present in the template - num_columns: int, the number of columns present in the template - investigation_type: str, the investigation type of the template - ontology: str, dict of {str, list of str} containing the information of the ENA ontology - artifact_attached: bool, whether the template has an artifact attached - study_id: int, the study id of the template """ pt = PrepTemplate(prep_id) name = pt.name # Initialize variables here processing = False alert_type = '' alert_msg = '' job_info = r_client.get(PREP_TEMPLATE_KEY_FORMAT % prep_id) if job_info: job_info = defaultdict(lambda: '', loads(job_info)) job_id = job_info['job_id'] job = ProcessingJob(job_id) job_status = job.status processing = job_status not in ('success', 'error') if processing: alert_type = 'info' alert_msg = 'This prep template is currently being updated' elif job_status == 'error': alert_type = 'danger' alert_msg = job.log.msg.replace('\n', '</br>') else: alert_type = job_info['alert_type'] alert_msg = job_info['alert_msg'].replace('\n', '</br>') artifact_attached = pt.artifact is not None study_id = pt.study_id files = [f for _, f, _ in get_files_from_uploads_folders(study_id) if f.endswith(('.txt', '.tsv'))] # The call to list is needed because keys is an iterator num_samples = len(list(pt.keys())) num_columns = len(pt.categories()) investigation_type = pt.investigation_type download_prep_id = None download_qiime_id = None other_filepaths = [] for fp_id, fp in pt.get_filepaths(): fp = basename(fp) if 'qiime' in fp: if download_qiime_id is None: download_qiime_id = fp_id else: if download_prep_id is None: download_prep_id = fp_id else: other_filepaths.append(fp) ontology = _get_ENA_ontology() editable = Study(study_id).can_edit(User(user_id)) and not processing return {'status': 'success', 'message': '', 'name': name, 'files': files, 'download_prep_id': download_prep_id, 'download_qiime_id': download_qiime_id, 'other_filepaths': other_filepaths, 'num_samples': num_samples, 'num_columns': num_columns, 'investigation_type': investigation_type, 'ontology': ontology, 'artifact_attached': artifact_attached, 'study_id': study_id, 'editable': editable, 'data_type': pt.data_type(), 'alert_type': alert_type, 'is_submitted_to_ebi': pt.is_submitted_to_ebi, 'alert_message': alert_msg}
def test_delete_sample_or_column(self): st = SampleTemplate(1) # Delete a sample template column job = self._create_job('delete_sample_or_column', {'obj_class': 'SampleTemplate', 'obj_id': 1, 'sample_or_col': 'columns', 'name': 'season_environment'}) private_task(job.id) self.assertEqual(job.status, 'success') self.assertNotIn('season_environment', st.categories()) # Delete a sample template sample - need to add one # sample that we will remove npt.assert_warns( QiitaDBWarning, st.extend, pd.DataFrame.from_dict({'Sample1': {'taxon_id': '9606'}}, orient='index', dtype=str)) self.assertIn('1.Sample1', st.keys()) job = self._create_job('delete_sample_or_column', {'obj_class': 'SampleTemplate', 'obj_id': 1, 'sample_or_col': 'samples', 'name': '1.Sample1'}) private_task(job.id) self.assertEqual(job.status, 'success') self.assertNotIn('1.Sample1', st.keys()) # Delete a prep template column pt = PrepTemplate(1) job = self._create_job('delete_sample_or_column', {'obj_class': 'PrepTemplate', 'obj_id': 1, 'sample_or_col': 'columns', 'name': 'target_subfragment'}) private_task(job.id) self.assertEqual(job.status, 'success') self.assertNotIn('target_subfragment', pt.categories()) # Delete a prep template sample metadata = pd.DataFrame.from_dict( {'1.SKB8.640193': {'barcode': 'GTCCGCAAGTTA', 'primer': 'GTGCCAGCMGCCGCGGTAA'}, '1.SKD8.640184': {'barcode': 'CGTAGAGCTCTC', 'primer': 'GTGCCAGCMGCCGCGGTAA'}}, orient='index', dtype=str) pt = npt.assert_warns(QiitaDBWarning, PrepTemplate.create, metadata, Study(1), "16S") job = self._create_job('delete_sample_or_column', {'obj_class': 'PrepTemplate', 'obj_id': pt.id, 'sample_or_col': 'samples', 'name': '1.SKD8.640184'}) private_task(job.id) self.assertNotIn('1.SKD8.640184', pt.keys()) # Test exceptions job = self._create_job('delete_sample_or_column', {'obj_class': 'UnknownClass', 'obj_id': 1, 'sample_or_col': 'columns', 'name': 'column'}) private_task(job.id) self.assertEqual(job.status, 'error') self.assertIn('Unknown value "UnknownClass". Choose between ' '"SampleTemplate" and "PrepTemplate"', job.log.msg) job = self._create_job('delete_sample_or_column', {'obj_class': 'SampleTemplate', 'obj_id': 1, 'sample_or_col': 'unknown', 'name': 'column'}) private_task(job.id) self.assertEqual(job.status, 'error') self.assertIn('Unknown value "unknown". Choose between "samples" ' 'and "columns"', job.log.msg)
def prep_template_ajax_get_req(user_id, prep_id): """Returns the prep tempalte information needed for the AJAX handler Parameters ---------- user_id : str The user id prep_id : int The prep template id Returns ------- dict of {str: object} A dictionary with the following keys: - status: str, whether the request is successful or not - message: str, if the request is unsuccessful, a human readable error - name: str, the name of the prep template - files: list of str, the files available to update the prep template - download_prep: int, the filepath_id of the prep file - download_qiime, int, the filepath_id of the qiime mapping file - num_samples: int, the number of samples present in the template - num_columns: int, the number of columns present in the template - investigation_type: str, the investigation type of the template - ontology: str, dict of {str, list of str} containing the information of the ENA ontology - artifact_attached: bool, whether the template has an artifact attached - study_id: int, the study id of the template """ # Currently there is no name attribute, but it will be soon name = "Prep information %d" % prep_id pt = PrepTemplate(prep_id) artifact_attached = pt.artifact is not None study_id = pt.study_id files = [f for _, f in get_files_from_uploads_folders(study_id) if f.endswith(('.txt', '.tsv'))] # The call to list is needed because keys is an iterator num_samples = len(list(pt.keys())) num_columns = len(pt.categories()) investigation_type = pt.investigation_type # Retrieve the information to download the prep template and QIIME # mapping file. See issue https://github.com/biocore/qiita/issues/1675 download_prep = [] download_qiime = [] for fp_id, fp in pt.get_filepaths(): if 'qiime' in basename(fp): download_qiime.append(fp_id) else: download_prep.append(fp_id) download_prep = download_prep[0] download_qiime = download_qiime[0] ontology = _get_ENA_ontology() job_id = r_client.get(PREP_TEMPLATE_KEY_FORMAT % prep_id) if job_id: redis_info = loads(r_client.get(job_id)) processing = redis_info['status_msg'] == 'Running' if processing: alert_type = 'info' alert_msg = 'This prep template is currently being updated' else: alert_type = redis_info['return']['status'] alert_msg = redis_info['return']['message'].replace('\n', '</br>') else: processing = False alert_type = '' alert_msg = '' editable = Study(study_id).can_edit(User(user_id)) and not processing return {'status': 'success', 'message': '', 'name': name, 'files': files, 'download_prep': download_prep, 'download_qiime': download_qiime, 'num_samples': num_samples, 'num_columns': num_columns, 'investigation_type': investigation_type, 'ontology': ontology, 'artifact_attached': artifact_attached, 'study_id': study_id, 'editable': editable, 'data_type': pt.data_type(), 'alert_type': alert_type, 'alert_message': alert_msg}