def test_prep_template_post_req(self):
obs = prep_template_post_req(1, '*****@*****.**', 'update.txt',
'16S')
exp = {'status': 'warning',
'message': [
'Sample names were already prefixed with the study id.',
('Some columns required to generate a QIIME-compliant '
'mapping file are not present in the template. A '
'placeholder value (XXQIITAXX) has been used to populate '
'these columns. Missing columns: BarcodeSequence, '
'LinkerPrimerSequence'),
('Some functionality will be disabled due to missing '
'columns:'),
('\tDemultiplexing with multiple input files disabled.: '
'barcode, primer, run_prefix;'),
'\tDemultiplexing disabled.: barcode, primer;',
('\tEBI submission disabled: center_name, '
'experiment_design_description, instrument_model, '
'library_construction_protocol, platform, primer.'),
('See the Templates tutorial for a description of these '
'fields.')],
'file': 'update.txt',
'id': 'ignored in test'}
self.assertItemsEqual(obs['message'].split('\n'), exp['message'])
self.assertEqual(obs['status'], exp['status'])
self.assertEqual(obs['file'], exp['file'])
self.assertIsInstance(obs['id'], int)
# Make sure new prep template added
prep = PrepTemplate(obs['id'])
self.assertEqual(prep.data_type(), '16S')
self.assertEqual([x for x in prep.keys()], ['1.SKD6.640190'])
self.assertEqual([x._to_dict() for x in prep.values()],
[{'new_col': 'new_value'}])
def test_prep_template_post_req(self):
obs = prep_template_post_req(1, '*****@*****.**', 'update.txt',
'16S', name=" ")
exp = {'status': 'warning',
'message': [
('Some columns required to generate a QIIME-compliant '
'mapping file are not present in the template. A '
'placeholder value (XXQIITAXX) has been used to populate '
'these columns. Missing columns: BarcodeSequence, '
'LinkerPrimerSequence'),
('Some functionality will be disabled due to missing '
'columns:'),
('\tDemultiplexing with multiple input files disabled.: '
'barcode, primer, run_prefix;'),
'\tDemultiplexing disabled.: barcode;',
('\tEBI submission disabled: center_name, '
'experiment_design_description, instrument_model, '
'library_construction_protocol, platform.'),
('See the Templates tutorial for a description of these '
'fields.')],
'file': 'update.txt',
'id': 'ignored in test'}
self.assertItemsEqual(obs['message'].split('\n'), exp['message'])
self.assertEqual(obs['status'], exp['status'])
self.assertEqual(obs['file'], exp['file'])
self.assertIsInstance(obs['id'], int)
# Make sure new prep template added
prep = PrepTemplate(obs['id'])
self.assertEqual(prep.data_type(), '16S')
self.assertEqual([x for x in prep.keys()], ['1.SKD6.640190'])
self.assertEqual([x._to_dict() for x in prep.values()],
[{'new_col': 'new_value'}])
self.assertEqual(prep.name, "Prep information %s" % prep.id)
def test_prep_template_post_req(self):
obs = prep_template_post_req(1,
'*****@*****.**',
'update.txt',
'16S',
name=" ")
exp = {
'status':
'warning',
'message': [
'Both a converter and dtype were specified for column '
'sample_name - only the converter will be used', 'Some '
'functionality will be disabled due to missing columns:',
'\tEBI submission disabled: center_name, '
'experiment_design_description, instrument_model, '
'library_construction_protocol, platform;',
'\tDemultiplexing disabled.: barcode;', '\tDemultiplexing '
'with multiple input files disabled.: barcode, primer, '
'run_prefix.', 'See the Templates tutorial for a '
'description of these fields.', 'Some columns required to '
'generate a QIIME-compliant mapping file are not present '
'in the template. A placeholder value (XXQIITAXX) '
'has been used to populate these columns. Missing columns: '
'BarcodeSequence, LinkerPrimerSequence'
],
'file':
'update.txt',
'id':
'ignored in test'
}
self.assertCountEqual(obs['message'].split('\n'), exp['message'])
self.assertEqual(obs['status'], exp['status'])
self.assertEqual(obs['file'], exp['file'])
self.assertIsInstance(obs['id'], int)
# Make sure new prep template added
prep = PrepTemplate(obs['id'])
self.assertEqual(prep.data_type(), '16S')
self.assertEqual([x for x in prep.keys()], ['1.SKD6.640190'])
self.assertEqual([x._to_dict() for x in prep.values()],
[{
'new_col': 'new_value'
}])
self.assertEqual(prep.name, "Prep information %s" % prep.id)
def test_prep_template_post_req(self):
new_id = get_count('qiita.prep_template') + 1
obs = prep_template_post_req(1, '*****@*****.**', 'update.txt',
'16S')
exp = {'status': 'warning',
'message': 'Sample names were already prefixed with the study '
'id.\nSome functionality will be disabled due to '
'missing columns:\n\tDemultiplexing with multiple '
'input files disabled. If your raw data includes '
'multiple raw input files, you will not be able to '
'preprocess your raw data: barcode, primer, '
'run_prefix;\n\tDemultiplexing disabled. You will '
'not be able to preprocess your raw data: barcode, '
'primer;\n\tEBI submission disabled: center_name, '
'experiment_design_description, instrument_model, '
'library_construction_protocol, platform, primer.'
'\nSee the Templates tutorial for a description of '
'these fields.\nSome columns required to generate a '
'QIIME-compliant mapping file are not present in the'
' template. A placeholder value (XXQIITAXX) has been'
' used to populate these columns. Missing columns: '
'BarcodeSequence, LinkerPrimerSequence',
'file': 'update.txt',
'id': new_id}
self.assertItemsEqual(obs['message'].split('\n'),
exp['message'].split('\n'))
self.assertEqual(obs['status'], exp['status'])
self.assertEqual(obs['file'], exp['file'])
self.assertEqual(obs['id'], exp['id'])
# Make sure new prep template added
prep = PrepTemplate(new_id)
self.assertEqual(prep.data_type(), '16S')
self.assertEqual([x for x in prep.keys()], ['1.SKD6.640190'])
self.assertEqual([x._to_dict() for x in prep.values()],
[{'new_col': 'new_value'}])
def test_delete_sample_or_column(self):
st = SampleTemplate(1)
# Delete a sample template column
job = self._create_job(
'delete_sample_or_column', {
'obj_class': 'SampleTemplate',
'obj_id': 1,
'sample_or_col': 'columns',
'name': 'season_environment'
})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertNotIn('season_environment', st.categories())
# Delete a sample template sample - need to add one
# sample that we will remove
npt.assert_warns(
QiitaDBWarning, st.extend,
pd.DataFrame.from_dict({'Sample1': {
'taxon_id': '9606'
}},
orient='index',
dtype=str))
self.assertIn('1.Sample1', st.keys())
job = self._create_job(
'delete_sample_or_column', {
'obj_class': 'SampleTemplate',
'obj_id': 1,
'sample_or_col': 'samples',
'name': '1.Sample1'
})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertNotIn('1.Sample1', st.keys())
# Delete a prep template column
pt = PrepTemplate(1)
job = self._create_job(
'delete_sample_or_column', {
'obj_class': 'PrepTemplate',
'obj_id': 1,
'sample_or_col': 'columns',
'name': 'target_subfragment'
})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertNotIn('target_subfragment', pt.categories())
# Delete a prep template sample
metadata = pd.DataFrame.from_dict(
{
'1.SKB8.640193': {
'barcode': 'GTCCGCAAGTTA',
'primer': 'GTGCCAGCMGCCGCGGTAA'
},
'1.SKD8.640184': {
'barcode': 'CGTAGAGCTCTC',
'primer': 'GTGCCAGCMGCCGCGGTAA'
}
},
orient='index',
dtype=str)
pt = npt.assert_warns(QiitaDBWarning, PrepTemplate.create, metadata,
Study(1), "16S")
job = self._create_job(
'delete_sample_or_column', {
'obj_class': 'PrepTemplate',
'obj_id': pt.id,
'sample_or_col': 'samples',
'name': '1.SKD8.640184'
})
private_task(job.id)
self.assertNotIn('1.SKD8.640184', pt.keys())
# Test exceptions
job = self._create_job(
'delete_sample_or_column', {
'obj_class': 'UnknownClass',
'obj_id': 1,
'sample_or_col': 'columns',
'name': 'column'
})
private_task(job.id)
self.assertEqual(job.status, 'error')
self.assertIn(
'Unknown value "UnknownClass". Choose between '
'"SampleTemplate" and "PrepTemplate"', job.log.msg)
job = self._create_job(
'delete_sample_or_column', {
'obj_class': 'SampleTemplate',
'obj_id': 1,
'sample_or_col': 'unknown',
'name': 'column'
})
private_task(job.id)
self.assertEqual(job.status, 'error')
self.assertIn(
'Unknown value "unknown". Choose between "samples" '
'and "columns"', job.log.msg)
def prep_template_ajax_get_req(user_id, prep_id):
"""Returns the prep tempalte information needed for the AJAX handler
Parameters
----------
user_id : str
The user id
prep_id : int
The prep template id
Returns
-------
dict of {str: object}
A dictionary with the following keys:
- status: str, whether the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
- name: str, the name of the prep template
- files: list of str, the files available to update the prep template
- download_prep: int, the filepath_id of the prep file
- download_qiime, int, the filepath_id of the qiime mapping file
- num_samples: int, the number of samples present in the template
- num_columns: int, the number of columns present in the template
- investigation_type: str, the investigation type of the template
- ontology: str, dict of {str, list of str} containing the information
of the ENA ontology
- artifact_attached: bool, whether the template has an artifact
attached
- study_id: int, the study id of the template
"""
# Currently there is no name attribute, but it will be soon
name = "Prep information %d" % prep_id
pt = PrepTemplate(prep_id)
job_info = r_client.get(PREP_TEMPLATE_KEY_FORMAT % prep_id)
if job_info:
job_info = loads(job_info)
job_id = job_info['job_id']
if job_id:
redis_info = loads(r_client.get(job_id))
processing = redis_info['status_msg'] == 'Running'
if processing:
alert_type = 'info'
alert_msg = 'This prep template is currently being updated'
elif redis_info['status_msg'] == 'Success':
alert_type = redis_info['return']['status']
alert_msg = redis_info['return']['message'].replace('\n',
'</br>')
payload = {'job_id': None,
'status': alert_type,
'message': alert_msg}
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id,
dumps(payload))
else:
alert_type = redis_info['return']['status']
alert_msg = redis_info['return']['message'].replace('\n',
'</br>')
else:
processing = False
alert_type = job_info['status']
alert_msg = job_info['message'].replace('\n', '</br>')
else:
processing = False
alert_type = ''
alert_msg = ''
artifact_attached = pt.artifact is not None
study_id = pt.study_id
files = [f for _, f in get_files_from_uploads_folders(study_id)
if f.endswith(('.txt', '.tsv'))]
# The call to list is needed because keys is an iterator
num_samples = len(list(pt.keys()))
num_columns = len(pt.categories())
investigation_type = pt.investigation_type
# Retrieve the information to download the prep template and QIIME
# mapping file. See issue https://github.com/biocore/qiita/issues/1675
download_prep = []
download_qiime = []
for fp_id, fp in pt.get_filepaths():
if 'qiime' in basename(fp):
download_qiime.append(fp_id)
else:
download_prep.append(fp_id)
download_prep = download_prep[0]
download_qiime = download_qiime[0]
ontology = _get_ENA_ontology()
editable = Study(study_id).can_edit(User(user_id)) and not processing
return {'status': 'success',
'message': '',
'name': name,
'files': files,
'download_prep': download_prep,
'download_qiime': download_qiime,
'num_samples': num_samples,
'num_columns': num_columns,
'investigation_type': investigation_type,
'ontology': ontology,
'artifact_attached': artifact_attached,
'study_id': study_id,
'editable': editable,
'data_type': pt.data_type(),
'alert_type': alert_type,
'is_submitted_to_ebi': pt.is_submitted_to_ebi,
'alert_message': alert_msg}
def prep_template_ajax_get_req(user_id, prep_id):
"""Returns the prep tempalte information needed for the AJAX handler
Parameters
----------
user_id : str
The user id
prep_id : int
The prep template id
Returns
-------
dict of {str: object}
A dictionary with the following keys:
- status: str, whether the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
- name: str, the name of the prep template
- files: list of str, the files available to update the prep template
- download_prep: int, the filepath_id of the prep file
- download_qiime, int, the filepath_id of the qiime mapping file
- num_samples: int, the number of samples present in the template
- num_columns: int, the number of columns present in the template
- investigation_type: str, the investigation type of the template
- ontology: str, dict of {str, list of str} containing the information
of the ENA ontology
- artifact_attached: bool, whether the template has an artifact
attached
- study_id: int, the study id of the template
"""
pt = PrepTemplate(prep_id)
name = pt.name
# Initialize variables here
processing = False
alert_type = ''
alert_msg = ''
job_info = r_client.get(PREP_TEMPLATE_KEY_FORMAT % prep_id)
if job_info:
job_info = defaultdict(lambda: '', loads(job_info))
job_id = job_info['job_id']
job = ProcessingJob(job_id)
job_status = job.status
processing = job_status not in ('success', 'error')
if processing:
alert_type = 'info'
alert_msg = 'This prep template is currently being updated'
elif job_status == 'error':
alert_type = 'danger'
alert_msg = job.log.msg.replace('\n', '</br>')
else:
alert_type = job_info['alert_type']
alert_msg = job_info['alert_msg'].replace('\n', '</br>')
artifact_attached = pt.artifact is not None
study_id = pt.study_id
files = [f for _, f, _ in get_files_from_uploads_folders(study_id)
if f.endswith(('.txt', '.tsv'))]
# The call to list is needed because keys is an iterator
num_samples = len(list(pt.keys()))
num_columns = len(pt.categories())
investigation_type = pt.investigation_type
download_prep_id = None
download_qiime_id = None
other_filepaths = []
for fp_id, fp in pt.get_filepaths():
fp = basename(fp)
if 'qiime' in fp:
if download_qiime_id is None:
download_qiime_id = fp_id
else:
if download_prep_id is None:
download_prep_id = fp_id
else:
other_filepaths.append(fp)
ontology = _get_ENA_ontology()
editable = Study(study_id).can_edit(User(user_id)) and not processing
return {'status': 'success',
'message': '',
'name': name,
'files': files,
'download_prep_id': download_prep_id,
'download_qiime_id': download_qiime_id,
'other_filepaths': other_filepaths,
'num_samples': num_samples,
'num_columns': num_columns,
'investigation_type': investigation_type,
'ontology': ontology,
'artifact_attached': artifact_attached,
'study_id': study_id,
'editable': editable,
'data_type': pt.data_type(),
'alert_type': alert_type,
'is_submitted_to_ebi': pt.is_submitted_to_ebi,
'alert_message': alert_msg}
def test_delete_sample_or_column(self):
st = SampleTemplate(1)
# Delete a sample template column
job = self._create_job('delete_sample_or_column',
{'obj_class': 'SampleTemplate', 'obj_id': 1,
'sample_or_col': 'columns',
'name': 'season_environment'})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertNotIn('season_environment', st.categories())
# Delete a sample template sample - need to add one
# sample that we will remove
npt.assert_warns(
QiitaDBWarning, st.extend,
pd.DataFrame.from_dict({'Sample1': {'taxon_id': '9606'}},
orient='index', dtype=str))
self.assertIn('1.Sample1', st.keys())
job = self._create_job('delete_sample_or_column',
{'obj_class': 'SampleTemplate', 'obj_id': 1,
'sample_or_col': 'samples',
'name': '1.Sample1'})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertNotIn('1.Sample1', st.keys())
# Delete a prep template column
pt = PrepTemplate(1)
job = self._create_job('delete_sample_or_column',
{'obj_class': 'PrepTemplate', 'obj_id': 1,
'sample_or_col': 'columns',
'name': 'target_subfragment'})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertNotIn('target_subfragment', pt.categories())
# Delete a prep template sample
metadata = pd.DataFrame.from_dict(
{'1.SKB8.640193': {'barcode': 'GTCCGCAAGTTA',
'primer': 'GTGCCAGCMGCCGCGGTAA'},
'1.SKD8.640184': {'barcode': 'CGTAGAGCTCTC',
'primer': 'GTGCCAGCMGCCGCGGTAA'}},
orient='index', dtype=str)
pt = npt.assert_warns(QiitaDBWarning, PrepTemplate.create, metadata,
Study(1), "16S")
job = self._create_job('delete_sample_or_column',
{'obj_class': 'PrepTemplate', 'obj_id': pt.id,
'sample_or_col': 'samples',
'name': '1.SKD8.640184'})
private_task(job.id)
self.assertNotIn('1.SKD8.640184', pt.keys())
# Test exceptions
job = self._create_job('delete_sample_or_column',
{'obj_class': 'UnknownClass', 'obj_id': 1,
'sample_or_col': 'columns', 'name': 'column'})
private_task(job.id)
self.assertEqual(job.status, 'error')
self.assertIn('Unknown value "UnknownClass". Choose between '
'"SampleTemplate" and "PrepTemplate"', job.log.msg)
job = self._create_job('delete_sample_or_column',
{'obj_class': 'SampleTemplate', 'obj_id': 1,
'sample_or_col': 'unknown', 'name': 'column'})
private_task(job.id)
self.assertEqual(job.status, 'error')
self.assertIn('Unknown value "unknown". Choose between "samples" '
'and "columns"', job.log.msg)
def prep_template_ajax_get_req(user_id, prep_id):
"""Returns the prep tempalte information needed for the AJAX handler
Parameters
----------
user_id : str
The user id
prep_id : int
The prep template id
Returns
-------
dict of {str: object}
A dictionary with the following keys:
- status: str, whether the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
- name: str, the name of the prep template
- files: list of str, the files available to update the prep template
- download_prep: int, the filepath_id of the prep file
- download_qiime, int, the filepath_id of the qiime mapping file
- num_samples: int, the number of samples present in the template
- num_columns: int, the number of columns present in the template
- investigation_type: str, the investigation type of the template
- ontology: str, dict of {str, list of str} containing the information
of the ENA ontology
- artifact_attached: bool, whether the template has an artifact
attached
- study_id: int, the study id of the template
"""
# Currently there is no name attribute, but it will be soon
name = "Prep information %d" % prep_id
pt = PrepTemplate(prep_id)
artifact_attached = pt.artifact is not None
study_id = pt.study_id
files = [f for _, f in get_files_from_uploads_folders(study_id)
if f.endswith(('.txt', '.tsv'))]
# The call to list is needed because keys is an iterator
num_samples = len(list(pt.keys()))
num_columns = len(pt.categories())
investigation_type = pt.investigation_type
# Retrieve the information to download the prep template and QIIME
# mapping file. See issue https://github.com/biocore/qiita/issues/1675
download_prep = []
download_qiime = []
for fp_id, fp in pt.get_filepaths():
if 'qiime' in basename(fp):
download_qiime.append(fp_id)
else:
download_prep.append(fp_id)
download_prep = download_prep[0]
download_qiime = download_qiime[0]
ontology = _get_ENA_ontology()
job_id = r_client.get(PREP_TEMPLATE_KEY_FORMAT % prep_id)
if job_id:
redis_info = loads(r_client.get(job_id))
processing = redis_info['status_msg'] == 'Running'
if processing:
alert_type = 'info'
alert_msg = 'This prep template is currently being updated'
else:
alert_type = redis_info['return']['status']
alert_msg = redis_info['return']['message'].replace('\n', '</br>')
else:
processing = False
alert_type = ''
alert_msg = ''
editable = Study(study_id).can_edit(User(user_id)) and not processing
return {'status': 'success',
'message': '',
'name': name,
'files': files,
'download_prep': download_prep,
'download_qiime': download_qiime,
'num_samples': num_samples,
'num_columns': num_columns,
'investigation_type': investigation_type,
'ontology': ontology,
'artifact_attached': artifact_attached,
'study_id': study_id,
'editable': editable,
'data_type': pt.data_type(),
'alert_type': alert_type,
'alert_message': alert_msg}
