def remove_add_prep_template(self, fp_rpt, study, data_type_id, investigation_type):
"""add prep templates"""
pt_id = PrepTemplate.create(
load_template_to_dataframe(fp_rpt), study, _to_int(data_type_id), investigation_type=investigation_type
).id
remove(fp_rpt)
return pt_id
def update_prep_template(self, study, user, callback):
"""Update a prep template from the POST method
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
Raises
------
HTTPError
If the prep template file does not exists
"""
# If we are on this function, the arguments "prep_template_id",
# "update_prep_template_file" must defined. If not, let tornado
# raise its error
pt_id = int(self.get_argument('prep_template_id'))
prep_template = self.get_argument('update_prep_template_file')
# Define here the message and message level in case of success
msg = "The prep template '%s' has been updated" % prep_template
msg_level = "success"
# Get the uploads folder
_, base_fp = get_mountpoint("uploads")[0]
# Get the path of the prep template in the uploads folder
fp = join(base_fp, str(study.id), prep_template)
if not exists(fp):
# The file does not exist, fail nicely
# Using 400 because we want the user to get the error in the GUI
raise HTTPError(400, "This file doesn't exist: %s" % fp)
try:
with warnings.catch_warnings(record=True) as warns:
pt = PrepTemplate(pt_id)
pt.update(load_template_to_dataframe(fp))
remove(fp)
# join all the warning messages into one. Note that this info
# will be ignored if an exception is raised
if warns:
msg = '; '.join([str(w.message) for w in warns])
msg_level = 'warning'
except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
QiitaDBDuplicateError, IOError, ValueError, KeyError,
CParserError, QiitaDBDuplicateHeaderError, QiitaDBError) as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
msg = html_error_message % ('updating the prep template:',
basename(fp), str(e))
msg = convert_text_html(msg)
msg_level = "danger"
callback((msg, msg_level, 'prep_template_tab', pt_id, None))
def update_sample_template(self, study, user, callback):
"""Update a sample template from the POST method
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
Raises
------
HTTPError
If the sample template file does not exists
"""
# If we are on this function, the argument "sample_template" must
# defined. If not, let tornado raise its error
sample_template = self.get_argument('sample_template')
# Define here the message and message level in case of success
msg = "The sample template '%s' has been updated" % sample_template
msg_level = "success"
# Get the uploads folder
_, base_fp = get_mountpoint("uploads")[0]
# Get the path of the sample template in the uploads folder
fp_rsp = join(base_fp, str(study.id), sample_template)
if not exists(fp_rsp):
# The file does not exist, fail nicely
raise HTTPError(400, "This file doesn't exist: %s" % fp_rsp)
try:
with warnings.catch_warnings(record=True) as warns:
# deleting previous uploads and inserting new one
st = SampleTemplate(study.id)
df = load_template_to_dataframe(fp_rsp)
st.extend(df)
st.update(df)
remove(fp_rsp)
# join all the warning messages into one. Note that this info
# will be ignored if an exception is raised
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
msg_level = 'warning'
except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
QiitaDBDuplicateError, IOError, ValueError, KeyError,
CParserError, QiitaDBDuplicateHeaderError, QiitaDBError) as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
msg = html_error_message % ('updating the sample template:',
basename(fp_rsp), str(e))
msg = convert_text_html(msg)
msg_level = "danger"
callback((msg, msg_level, None, None, None))
def remove_add_prep_template(self, fp_rpt, raw_data_id, study,
data_type_id, investigation_type):
"""add prep templates"""
pt_id = PrepTemplate.create(load_template_to_dataframe(fp_rpt),
RawData(raw_data_id), study,
_to_int(data_type_id),
investigation_type=investigation_type).id
remove(fp_rpt)
return pt_id
def remove_add_prep_template(self, fp_rpt, raw_data_id, study,
data_type_id, investigation_type, callback):
"""add prep templates
"""
PrepTemplate.create(load_template_to_dataframe(fp_rpt),
RawData(raw_data_id), study, int(data_type_id),
investigation_type=investigation_type)
remove(fp_rpt)
callback()
def update_sample_template(self, study, user, callback):
"""Update a sample template from the POST method
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
Raises
------
HTTPError
If the sample template file does not exists
"""
# If we are on this function, the argument "sample_template" must
# defined. If not, let tornado raise its error
sample_template = self.get_argument('sample_template')
# Define here the message and message level in case of success
msg = "The sample template '%s' has been updated" % sample_template
msg_level = "success"
# Get the uploads folder
_, base_fp = get_mountpoint("uploads")[0]
# Get the path of the sample template in the uploads folder
fp_rsp = join(base_fp, str(study.id), sample_template)
if not exists(fp_rsp):
# The file does not exist, fail nicely
raise HTTPError(400, "This file doesn't exist: %s" % fp_rsp)
try:
with warnings.catch_warnings(record=True) as warns:
# deleting previous uploads and inserting new one
st = SampleTemplate(study.id)
st.update(load_template_to_dataframe(fp_rsp))
# join all the warning messages into one. Note that this info
# will be ignored if an exception is raised
if warns:
msg = '; '.join([str(w.message) for w in warns])
msg_level = 'warning'
except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
QiitaDBDuplicateError, IOError, ValueError, KeyError,
CParserError, QiitaDBDuplicateHeaderError, QiitaDBError) as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
msg = html_error_message % ('updating the sample template:',
basename(fp_rsp), str(e))
msg_level = "danger"
callback((msg, msg_level, None, None, None))
def remove_add_study_template(self, raw_data, study_id, fp_rsp):
"""Replace prep templates, raw data, and sample template with a new one
"""
for rd in raw_data():
rd = RawData(rd)
for pt in rd.prep_templates:
if PrepTemplate.exists(pt):
PrepTemplate.delete(pt)
if SampleTemplate.exists(study_id):
SampleTemplate.delete(study_id)
SampleTemplate.create(load_template_to_dataframe(fp_rsp),
Study(study_id))
remove(fp_rsp)
def remove_add_study_template(self, raw_data, study_id, fp_rsp):
"""Replace prep templates, raw data, and sample template with a new one
"""
for rd in raw_data():
rd = RawData(rd)
for pt in rd.prep_templates:
if PrepTemplate.exists(pt):
PrepTemplate.delete(pt)
if SampleTemplate.exists(study_id):
SampleTemplate.delete(study_id)
SampleTemplate.create(load_template_to_dataframe(fp_rsp),
Study(study_id))
remove(fp_rsp)
def remove_add_study_template(self, raw_data, study_id, fp_rsp, data_type, is_mapping_file):
"""Replace prep templates, raw data, and sample template with a new one
"""
if is_mapping_file and data_type == "":
raise ValueError("Please, choose a data type if uploading a QIIME " "mapping file")
for rd in raw_data():
rd = RawData(rd)
for pt in rd.prep_templates:
if PrepTemplate.exists(pt):
PrepTemplate.delete(pt)
if SampleTemplate.exists(study_id):
SampleTemplate.delete(study_id)
if is_mapping_file:
create_templates_from_qiime_mapping_file(fp_rsp, Study(study_id), int(data_type))
else:
SampleTemplate.create(load_template_to_dataframe(fp_rsp), Study(study_id))
remove(fp_rsp)
def _get_sample_names_by_run_prefix(prep_template):
"""Generates a dictionary of run_prefix and sample names
Parameters
----------
prep_template : PrepTemplate
The prep template from which we need to generate the minimal mapping
file
Returns
-------
dict
Dict mapping run_prefix to path of the minimal qiime mapping file
Raises
------
ValueError
If there is more than 1 sample per run_prefix
"""
from qiita_db.metadata_template import load_template_to_dataframe
qiime_map = load_template_to_dataframe(prep_template.qiime_map_fp,
index='#SampleID')
samples = {}
errors = []
for prefix, df in qiime_map.groupby('run_prefix'):
len_df = len(df)
if len_df != 1:
errors.append('%s has %d samples (%s)' % (prefix, len_df,
', '.join(df.index)))
else:
samples[prefix] = df.index.values[0]
if errors:
raise ValueError("You have run_prefix values with multiple "
"samples: %s" % ' -- '.join(errors))
return samples
def process_sample_template(self, study, user, callback):
"""Process a sample template from the POST method
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
Raises
------
HTTPError
If the sample template file does not exists
"""
# If we are on this function, the arguments "sample_template" and
# "data_type" must be defined. If not, let tornado raise its error
sample_template = self.get_argument('sample_template')
data_type = self.get_argument('data_type')
# Get the uploads folder
_, base_fp = get_mountpoint("uploads")[0]
# Get the path of the sample template in the uploads folder
fp_rsp = join(base_fp, str(study.id), sample_template)
if not exists(fp_rsp):
# The file does not exist, fail nicely
raise HTTPError(404, "This file doesn't exist: %s" % fp_rsp)
# Define here the message and message level in case of success
msg = "The sample template '%s' has been added" % sample_template
msg_level = "success"
is_mapping_file = looks_like_qiime_mapping_file(fp_rsp)
try:
if is_mapping_file and not data_type:
raise ValueError("Please, choose a data type if uploading a "
"QIIME mapping file")
with warnings.catch_warnings(record=True) as warns:
if is_mapping_file:
create_templates_from_qiime_mapping_file(fp_rsp, study,
int(data_type))
else:
SampleTemplate.create(load_template_to_dataframe(fp_rsp),
study)
remove(fp_rsp)
# join all the warning messages into one. Note that this
# info will be ignored if an exception is raised
if warns:
msg = '; '.join([convert_text_html(str(w.message))
for w in warns])
msg_level = 'warning'
except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
QiitaDBDuplicateError, IOError, ValueError, KeyError,
CParserError, QiitaDBDuplicateHeaderError,
QiitaDBError, QiitaWareError) as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
error_msg = ('parsing the QIIME mapping file'
if is_mapping_file
else 'parsing the sample template')
msg = html_error_message % (error_msg, basename(fp_rsp),
str(e))
msg = convert_text_html(msg)
msg_level = "danger"
callback((msg, msg_level, None, None, None))
def create_templates_from_qiime_mapping_file(fp, study, data_type):
"""Creates a sample template and a prep template from qiime mapping file
Parameters
----------
fp : str or file-like object
Path to the QIIME mapping file
study : Study
The study to which the sample template belongs to
data_type : str or int
The data_type of the prep_template
Returns
-------
(SampleTemplate, PrepTemplate)
The templates created from the QIIME mapping file
"""
qiime_map = load_template_to_dataframe(fp, index='#SampleID')
# There are a few columns in the QIIME mapping file that are special and
# we know how to deal with them
rename_cols = {
'BarcodeSequence': 'barcode',
'LinkerPrimerSequence': 'primer',
'Description': 'description',
}
if 'ReverseLinkerPrimer' in qiime_map:
rename_cols['ReverseLinkerPrimer'] = 'reverselinkerprimer'
missing = set(rename_cols).difference(qiime_map.columns)
if missing:
raise QiitaWareError(
"Error generating the templates from the QIIME mapping file. "
"Missing QIIME mapping file columns: %s" % ', '.join(missing))
qiime_map.rename(columns=rename_cols, inplace=True)
# Fix the casing in the columns that we control
qiime_map.columns = [c.lower() if c.lower() in CONTROLLED_COLS else c
for c in qiime_map.columns]
# Figure out which columns belong to the prep template
def _col_iterator(restriction_set):
for restriction in viewvalues(restriction_set):
for cols in viewkeys(restriction.columns):
yield cols
pt_cols = set(col for col in _col_iterator(PREP_TEMPLATE_COLUMNS))
data_type_str = (convert_from_id(data_type, "data_type")
if isinstance(data_type, (int, long)) else data_type)
if data_type_str in TARGET_GENE_DATA_TYPES:
pt_cols.update(
col for col in _col_iterator(PREP_TEMPLATE_COLUMNS_TARGET_GENE))
pt_cols.add('reverselinkerprimer')
qiime_cols = set(qiime_map.columns)
pt_cols = qiime_cols.intersection(pt_cols)
st_cols = qiime_cols.difference(pt_cols)
st_md = qiime_map.ix[:, st_cols]
pt_md = qiime_map.ix[:, pt_cols]
return (SampleTemplate.create(st_md, study),
PrepTemplate.create(pt_md, study, data_type))
def _extend_sample_template(self, st_id, fp_rpt):
SampleTemplate(st_id).extend(load_template_to_dataframe(fp_rpt))
def _extend_sample_template(self, st_id, fp_rpt):
SampleTemplate(st_id).extend(load_template_to_dataframe(fp_rpt))
