def create_sample_template(job):
"""Creates a sample template
Parameters
----------
job : qiita_db.processing_job.ProcessingJob
The processing job performing the task
"""
with qdb.sql_connection.TRN:
params = job.parameters.values
fp = params['fp']
study = qdb.study.Study(int(params['study_id']))
is_mapping_file = params['is_mapping_file']
data_type = params['data_type']
with warnings.catch_warnings(record=True) as warns:
if is_mapping_file:
create_templates_from_qiime_mapping_file(fp, study, data_type)
else:
qdb.metadata_template.sample_template.SampleTemplate.create(
qdb.metadata_template.util.load_template_to_dataframe(fp),
study)
remove(fp)
if warns:
msg = '\n'.join(set(str(w) for w in warns))
r_client.set(
"sample_template_%s" % study.id,
dumps({
'job_id': job.id,
'alert_type': 'warning',
'alert_msg': msg
}))
job._set_status('success')
def create_sample_template(job):
"""Creates a sample template
Parameters
----------
job : qiita_db.processing_job.ProcessingJob
The processing job performing the task
"""
with qdb.sql_connection.TRN:
params = job.parameters.values
fp = params['fp']
study = qdb.study.Study(int(params['study_id']))
is_mapping_file = params['is_mapping_file']
data_type = params['data_type']
with warnings.catch_warnings(record=True) as warns:
if is_mapping_file:
create_templates_from_qiime_mapping_file(fp, study, data_type)
else:
qdb.metadata_template.sample_template.SampleTemplate.create(
qdb.metadata_template.util.load_template_to_dataframe(fp),
study)
remove(fp)
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
r_client.set("sample_template_%s" % study.id,
dumps({'job_id': job.id, 'alert_type': 'warning',
'alert_msg': msg}))
job._set_status('success')
def create_sample_template(fp, study, is_mapping_file, data_type=None):
"""Creates a sample template
Parameters
----------
fp : str
The file path to the template file
study : qiita_db.study.Study
The study to add the sample template to
is_mapping_file : bool
Whether `fp` contains a mapping file or a sample template
data_type : str, optional
If `is_mapping_file` is True, the data type of the prep template to be
created
Returns
-------
dict of {str: str}
A dict of the form {'status': str, 'message': str}
"""
# The imports need to be in here because this code is executed in
# the ipython workers
import warnings
from os import remove
from qiita_db.metadata_template.sample_template import SampleTemplate
from qiita_db.metadata_template.util import load_template_to_dataframe
from qiita_ware.metadata_pipeline import (
create_templates_from_qiime_mapping_file)
status = 'success'
msg = ''
try:
with warnings.catch_warnings(record=True) as warns:
if is_mapping_file:
create_templates_from_qiime_mapping_file(fp, study,
data_type)
else:
SampleTemplate.create(load_template_to_dataframe(fp),
study)
remove(fp)
# join all the warning messages into one. Note that this
# info will be ignored if an exception is raised
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
status = 'warning'
except Exception as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
status = 'danger'
msg = str(e)
return {'status': status, 'message': msg}
def create_sample_template(fp, study, is_mapping_file, data_type=None):
"""Creates a sample template
Parameters
----------
fp : str
The file path to the template file
study : qiita_db.study.Study
The study to add the sample template to
is_mapping_file : bool
Whether `fp` contains a mapping file or a sample template
data_type : str, optional
If `is_mapping_file` is True, the data type of the prep template to be
created
Returns
-------
dict of {str: str}
A dict of the form {'status': str, 'message': str}
"""
# The imports need to be in here because this code is executed in
# the ipython workers
import warnings
from os import remove
from qiita_db.metadata_template.sample_template import SampleTemplate
from qiita_db.metadata_template.util import load_template_to_dataframe
from qiita_ware.metadata_pipeline import (
create_templates_from_qiime_mapping_file)
status = 'success'
msg = ''
try:
with warnings.catch_warnings(record=True) as warns:
if is_mapping_file:
create_templates_from_qiime_mapping_file(fp, study,
data_type)
else:
SampleTemplate.create(load_template_to_dataframe(fp),
study)
remove(fp)
# join all the warning messages into one. Note that this
# info will be ignored if an exception is raised
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
status = 'warning'
except Exception as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
status = 'danger'
msg = str(e)
return {'status': status, 'message': msg}
def test_create_templates_from_qiime_mapping_file_reverse_linker(self):
curr_id = self.conn_handler.execute_fetchone(
"SELECT last_value FROM "
"qiita.prep_template_prep_template_id_seq")[0]
obs_st, obs_pt = create_templates_from_qiime_mapping_file(
StringIO(QIIME_MAP_WITH_REVERSE_LINKER_PRIMER), self.new_study,
"16S")
# Be green: clean the environment
for template in [obs_st, obs_pt]:
for _, fp in template.get_filepaths():
self._clean_up_files.append(fp)
self.assertEqual(obs_st.id, self.new_study.id)
self.assertEqual(obs_pt.id, curr_id + 1)
# Check that each template has the correct columns
exp = {
"physical_specimen_location", "physical_specimen_remaining",
"dna_extracted", "sample_type", "host_subject_id", "latitude",
"longitude", "taxon_id", "scientific_name", "collection_timestamp",
"description"
}
self.assertEqual(set(obs_st.categories()), exp)
exp = {
"barcode", "primer", "center_name", "run_prefix", "platform",
"library_construction_protocol", "instrument_model",
"experiment_design_description", "reverselinkerprimer"
}
self.assertEqual(set(obs_pt.categories()), exp)
def test_create_templates_from_qiime_mapping_file(self):
new_pt_id = get_count('qiita.prep_template') + 1
obs_st, obs_pt = create_templates_from_qiime_mapping_file(
StringIO(QIIME_MAP), self.new_study, "16S")
# Be green: clean the environment
for template in [obs_st, obs_pt]:
for _, fp in template.get_filepaths():
self._clean_up_files.append(fp)
self.assertEqual(obs_st.id, self.new_study.id)
self.assertEqual(obs_pt.id, new_pt_id)
# Check that each template has the correct columns
exp = {
"physical_specimen_location", "physical_specimen_remaining",
"dna_extracted", "sample_type", "host_subject_id", "latitude",
"longitude", "taxon_id", "scientific_name", "collection_timestamp",
"description"
}
self.assertEqual(set(obs_st.categories()), exp)
exp = {
"barcode", "primer", "center_name", "run_prefix", "platform",
"library_construction_protocol", "instrument_model",
"experiment_design_description"
}
self.assertEqual(set(obs_pt.categories()), exp)
def test_create_templates_from_qiime_mapping_file_reverse_linker(self):
new_pt_id = get_count('qiita.prep_template') + 1
obs_st, obs_pt = create_templates_from_qiime_mapping_file(
StringIO(QIIME_MAP_WITH_REVERSE_LINKER_PRIMER),
self.new_study, "16S")
# Be green: clean the environment
for template in [obs_st, obs_pt]:
for _, fp in template.get_filepaths():
self._clean_up_files.append(fp)
self.assertEqual(obs_st.id, self.new_study.id)
self.assertEqual(obs_pt.id, new_pt_id)
# Check that each template has the correct columns
exp = {"physical_specimen_location", "physical_specimen_remaining",
"dna_extracted", "sample_type", "host_subject_id", "latitude",
"longitude", "taxon_id", "scientific_name",
"collection_timestamp", "description"}
self.assertEqual(set(obs_st.categories()), exp)
exp = {"barcode", "primer", "center_name", "run_prefix", "platform",
"library_construction_protocol",
"experiment_design_description", "reverselinkerprimer"}
self.assertEqual(set(obs_pt.categories()), exp)
def remove_add_study_template(self, raw_data, study_id, fp_rsp, data_type, is_mapping_file):
"""Replace prep templates, raw data, and sample template with a new one
"""
if is_mapping_file and data_type == "":
raise ValueError("Please, choose a data type if uploading a QIIME " "mapping file")
for rd in raw_data():
rd = RawData(rd)
for pt in rd.prep_templates:
if PrepTemplate.exists(pt):
PrepTemplate.delete(pt)
if SampleTemplate.exists(study_id):
SampleTemplate.delete(study_id)
if is_mapping_file:
create_templates_from_qiime_mapping_file(fp_rsp, Study(study_id), int(data_type))
else:
SampleTemplate.create(load_template_to_dataframe(fp_rsp), Study(study_id))
remove(fp_rsp)
def process_sample_template(self, study, user, callback):
"""Process a sample template from the POST method
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
Raises
------
HTTPError
If the sample template file does not exists
"""
# If we are on this function, the arguments "sample_template" and
# "data_type" must be defined. If not, let tornado raise its error
sample_template = self.get_argument('sample_template')
data_type = self.get_argument('data_type')
# Get the uploads folder
_, base_fp = get_mountpoint("uploads")[0]
# Get the path of the sample template in the uploads folder
fp_rsp = join(base_fp, str(study.id), sample_template)
if not exists(fp_rsp):
# The file does not exist, fail nicely
raise HTTPError(404, "This file doesn't exist: %s" % fp_rsp)
# Define here the message and message level in case of success
msg = "The sample template '%s' has been added" % sample_template
msg_level = "success"
is_mapping_file = looks_like_qiime_mapping_file(fp_rsp)
try:
if is_mapping_file and not data_type:
raise ValueError("Please, choose a data type if uploading a "
"QIIME mapping file")
with warnings.catch_warnings(record=True) as warns:
if is_mapping_file:
create_templates_from_qiime_mapping_file(fp_rsp, study,
int(data_type))
else:
SampleTemplate.create(load_template_to_dataframe(fp_rsp),
study)
remove(fp_rsp)
# join all the warning messages into one. Note that this
# info will be ignored if an exception is raised
if warns:
msg = '; '.join([convert_text_html(str(w.message))
for w in warns])
msg_level = 'warning'
except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
QiitaDBDuplicateError, IOError, ValueError, KeyError,
CParserError, QiitaDBDuplicateHeaderError,
QiitaDBError, QiitaWareError) as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
error_msg = ('parsing the QIIME mapping file'
if is_mapping_file
else 'parsing the sample template')
msg = html_error_message % (error_msg, basename(fp_rsp),
str(e))
msg = convert_text_html(msg)
msg_level = "danger"
callback((msg, msg_level, None, None, None))
def test_create_templates_from_qiime_mapping_file_error(self):
with self.assertRaises(QiitaWareError):
create_templates_from_qiime_mapping_file(StringIO(QIIME_MAP_ERROR),
self.new_study, "16S")
def test_create_templates_from_qiime_mapping_file_error(self):
with self.assertRaises(QiitaWareError):
create_templates_from_qiime_mapping_file(
StringIO(QIIME_MAP_ERROR), self.new_study, "16S")
def process_sample_template(self, study, user, callback):
"""Process a sample template from the POST method
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
Raises
------
HTTPError
If the sample template file does not exists
"""
# If we are on this function, the arguments "sample_template" and
# "data_type" must be defined. If not, let tornado raise its error
sample_template = self.get_argument('sample_template')
data_type = self.get_argument('data_type')
# Get the uploads folder
_, base_fp = get_mountpoint("uploads")[0]
# Get the path of the sample template in the uploads folder
fp_rsp = join(base_fp, str(study.id), sample_template)
if not exists(fp_rsp):
# The file does not exist, fail nicely
raise HTTPError(404, "This file doesn't exist: %s" % fp_rsp)
# Define here the message and message level in case of success
msg = "The sample template '%s' has been added" % sample_template
msg_level = "success"
is_mapping_file = looks_like_qiime_mapping_file(fp_rsp)
try:
if is_mapping_file and not data_type:
raise ValueError("Please, choose a data type if uploading a "
"QIIME mapping file")
with warnings.catch_warnings(record=True) as warns:
if is_mapping_file:
create_templates_from_qiime_mapping_file(
fp_rsp, study, int(data_type))
else:
SampleTemplate.create(load_template_to_dataframe(fp_rsp),
study)
remove(fp_rsp)
# join all the warning messages into one. Note that this
# info will be ignored if an exception is raised
if warns:
msg = '; '.join(
[convert_text_html(str(w.message)) for w in warns])
msg_level = 'warning'
except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
QiitaDBDuplicateError, IOError, ValueError, KeyError,
CParserError, QiitaDBDuplicateHeaderError, QiitaDBError,
QiitaWareError) as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
error_msg = ('parsing the QIIME mapping file'
if is_mapping_file else 'parsing the sample template')
msg = html_error_message % (error_msg, basename(fp_rsp), str(e))
msg = convert_text_html(msg)
msg_level = "danger"
callback((msg, msg_level, None, None, None))
