def test_load_template_to_dataframe_empty_sample_names(self):
obs = load_template_to_dataframe(
StringIO(SAMPLE_TEMPLATE_NO_SAMPLE_NAMES))
exp = pd.DataFrame.from_dict(SAMPLE_TEMPLATE_DICT_FORM)
exp.index.name = 'sample_name'
assert_frame_equal(obs, exp)
obs = load_template_to_dataframe(
StringIO(SAMPLE_TEMPLATE_NO_SAMPLE_NAMES_SOME_SPACES))
exp = pd.DataFrame.from_dict(SAMPLE_TEMPLATE_DICT_FORM)
exp.index.name = 'sample_name'
assert_frame_equal(obs, exp)
def test_invalid_lat_long(self):
with self.assertRaises(QiitaDBColumnError):
obs = load_template_to_dataframe(
StringIO(SAMPLE_TEMPLATE_INVALID_LATITUDE_COLUMNS))
# prevent flake8 from complaining
str(obs)
with self.assertRaises(QiitaDBColumnError):
obs = load_template_to_dataframe(
StringIO(SAMPLE_TEMPLATE_INVALID_LONGITUDE_COLUMNS))
# prevent flake8 from complaining
str(obs)
def test_load_template_to_dataframe_typechecking(self):
obs = load_template_to_dataframe(
StringIO(EXP_SAMPLE_TEMPLATE_LAT_ALL_INT))
exp = pd.DataFrame.from_dict(SAMPLE_TEMPLATE_LAT_ALL_INT_DICT)
exp.index.name = 'sample_name'
assert_frame_equal(obs, exp)
obs = load_template_to_dataframe(
StringIO(EXP_SAMPLE_TEMPLATE_LAT_MIXED_FLOAT_INT))
exp = pd.DataFrame.from_dict(SAMPLE_TEMPLATE_MIXED_FLOAT_INT_DICT)
exp.index.name = 'sample_name'
assert_frame_equal(obs, exp)
def test_post_valid(self):
dontcare, uploads_dir = get_mountpoint('uploads')[0]
foo_fp = os.path.join(uploads_dir, '1', 'foo.txt')
bar_fp = os.path.join(uploads_dir, '1', 'bar.txt')
with open(foo_fp, 'w') as fp:
fp.write("@x\nATGC\n+\nHHHH\n")
with open(bar_fp, 'w') as fp:
fp.write("@x\nATGC\n+\nHHHH\n")
prep = StringIO(EXP_PREP_TEMPLATE.format(1))
prep_table = load_template_to_dataframe(prep)
response = self.post('/api/v1/study/1/preparation?data_type=16S',
data=prep_table.T.to_dict(),
headers=self.headers,
asjson=True)
prepid = json_decode(response.body)['id']
uri = '/api/v1/study/1/preparation/%d/artifact' % prepid
# 1 -> fwd or rev sequences in fastq
# 3 -> barcodes
body = {
'artifact_type': 'FASTQ',
'filepaths': [['foo.txt', 1], ['bar.txt', 'raw_barcodes']],
'artifact_name': 'a name is a name'
}
response = self.post(uri, data=body, headers=self.headers, asjson=True)
self.assertEqual(response.code, 201)
obs = json_decode(response.body)['id']
prep_instance = PrepTemplate(prepid)
exp = prep_instance.artifact.id
self.assertEqual(obs, exp)
def test_load_template_to_dataframe_exception(self):
with self.assertRaises(QiitaDBColumnError):
x = load_template_to_dataframe(
StringIO(SAMPLE_TEMPLATE_NO_SAMPLE_NAME))
# prevent flake8 from complaining
x.strip()
def test_post_valid(self):
dontcare, uploads_dir = get_mountpoint('uploads')[0]
foo_fp = os.path.join(uploads_dir, '1', 'foo.txt')
bar_fp = os.path.join(uploads_dir, '1', 'bar.txt')
with open(foo_fp, 'w') as fp:
fp.write("@x\nATGC\n+\nHHHH\n")
with open(bar_fp, 'w') as fp:
fp.write("@x\nATGC\n+\nHHHH\n")
prep = StringIO(EXP_PREP_TEMPLATE.format(1))
prep_table = load_template_to_dataframe(prep)
response = self.post('/api/v1/study/1/preparation?data_type=16S',
data=prep_table.T.to_dict(),
headers=self.headers, asjson=True)
prepid = json_decode(response.body)['id']
uri = '/api/v1/study/1/preparation/%d/artifact' % prepid
# 1 -> fwd or rev sequences in fastq
# 3 -> barcodes
body = {'artifact_type': 'FASTQ', 'filepaths': [['foo.txt', 1],
['bar.txt',
'raw_barcodes']],
'artifact_name': 'a name is a name'}
response = self.post(uri, data=body, headers=self.headers, asjson=True)
self.assertEqual(response.code, 201)
obs = json_decode(response.body)['id']
prep_instance = PrepTemplate(prepid)
exp = prep_instance.artifact.id
self.assertEqual(obs, exp)
def test_load_template_to_dataframe_lowercase(self):
obs = load_template_to_dataframe(
StringIO(EXP_SAMPLE_TEMPLATE_MULTICASE))
exp = pd.DataFrame.from_dict(SAMPLE_TEMPLATE_DICT_FORM)
exp.index.name = 'sample_name'
exp.rename(columns={"str_column": "str_CoLumn"}, inplace=True)
assert_frame_equal(obs, exp)
def patch(self, study_id):
study = self.safe_get_study(study_id)
if study is None:
return
if study.sample_template is None:
self.fail('No sample information found', 404)
return
else:
sample_info = study.sample_template.to_dataframe()
# convert from json into a format that qiita can validate
rawdata = pd.DataFrame.from_dict(json_decode(self.request.body),
orient='index')
rawdata.index.name = 'sample_name'
if len(rawdata.index) == 0:
self.fail('No samples provided', 400)
return
buffer = io.StringIO()
rawdata.to_csv(buffer, sep='\t', index=True, header=True)
buffer.seek(0)
# validate on load
data = load_template_to_dataframe(buffer)
categories = set(study.sample_template.categories)
if set(data.columns) != categories:
if set(data.columns).issubset(categories):
self.fail('Not all sample information categories provided',
400)
else:
unknown = set(data.columns) - categories
self.fail(
"Some categories do not exist in the sample "
"information",
400,
categories_not_found=sorted(unknown))
return
existing_samples = set(sample_info.index)
overlapping_ids = set(data.index).intersection(existing_samples)
new_ids = set(data.index) - existing_samples
status = 500
# warnings generated are not currently caught
# see https://github.com/biocore/qiita/issues/2096
if overlapping_ids:
to_update = data.loc[overlapping_ids]
study.sample_template.update(to_update)
status = 200
if new_ids:
to_extend = data.loc[new_ids]
study.sample_template.extend(to_extend)
status = 201
self.set_status(status)
self.finish()
def remove_add_prep_template(self, fp_rpt, study, data_type_id,
investigation_type):
"""add prep templates"""
pt_id = PrepTemplate.create(load_template_to_dataframe(fp_rpt),
study, _to_int(data_type_id),
investigation_type=investigation_type).id
remove(fp_rpt)
return pt_id
def update_sample_template(self, study, user, callback):
"""Update a sample template from the POST method
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
Raises
------
HTTPError
If the sample template file does not exists
"""
# If we are on this function, the argument "sample_template" must
# defined. If not, let tornado raise its error
sample_template = self.get_argument('sample_template')
# Define here the message and message level in case of success
msg = "The sample template '%s' has been updated" % sample_template
msg_level = "success"
# Get the uploads folder
_, base_fp = get_mountpoint("uploads")[0]
# Get the path of the sample template in the uploads folder
fp_rsp = join(base_fp, str(study.id), sample_template)
if not exists(fp_rsp):
# The file does not exist, fail nicely
raise HTTPError(400, "This file doesn't exist: %s" % fp_rsp)
try:
with warnings.catch_warnings(record=True) as warns:
# deleting previous uploads and inserting new one
st = SampleTemplate(study.id)
df = load_template_to_dataframe(fp_rsp)
st.extend(df)
st.update(df)
remove(fp_rsp)
# join all the warning messages into one. Note that this info
# will be ignored if an exception is raised
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
msg_level = 'warning'
except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
QiitaDBDuplicateError, IOError, ValueError, KeyError,
CParserError, QiitaDBDuplicateHeaderError, QiitaDBError) as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
msg = html_error_message % ('updating the sample template:',
basename(fp_rsp), str(e))
msg = convert_text_html(msg)
msg_level = "danger"
callback((msg, msg_level, None, None, None))
def update_sample_template(self, study, user, callback):
"""Update a sample template from the POST method
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
Raises
------
HTTPError
If the sample template file does not exists
"""
# If we are on this function, the argument "sample_template" must
# defined. If not, let tornado raise its error
sample_template = self.get_argument('sample_template')
# Define here the message and message level in case of success
msg = "The sample template '%s' has been updated" % sample_template
msg_level = "success"
# Get the uploads folder
_, base_fp = get_mountpoint("uploads")[0]
# Get the path of the sample template in the uploads folder
fp_rsp = join(base_fp, str(study.id), sample_template)
if not exists(fp_rsp):
# The file does not exist, fail nicely
raise HTTPError(400, "This file doesn't exist: %s" % fp_rsp)
try:
with warnings.catch_warnings(record=True) as warns:
# deleting previous uploads and inserting new one
st = SampleTemplate(study.id)
df = load_template_to_dataframe(fp_rsp)
st.extend(df)
st.update(df)
remove(fp_rsp)
# join all the warning messages into one. Note that this info
# will be ignored if an exception is raised
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
msg_level = 'warning'
except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
QiitaDBDuplicateError, IOError, ValueError, KeyError,
CParserError, QiitaDBDuplicateHeaderError, QiitaDBError) as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
msg = html_error_message % ('updating the sample template:',
basename(fp_rsp), str(e))
msg = convert_text_html(msg)
msg_level = "danger"
callback((msg, msg_level, None, None, None))
def test_load_template_to_dataframe_duplicate_cols(self):
obs = load_template_to_dataframe(
StringIO(EXP_SAMPLE_TEMPLATE_DUPE_COLS))
obs = list(obs.columns)
exp = ['collection_timestamp', 'description', 'has_extracted_data',
'has_physical_specimen', 'host_subject_id', 'latitude',
'longitude', 'physical_location', 'required_sample_info_status',
'sample_type', 'str_column', 'str_column']
self.assertEqual(obs, exp)
def test_load_template_to_dataframe_no_sample_name_cast(self):
obs = load_template_to_dataframe(
StringIO(EXP_SAMPLE_TEMPLATE_NUMBER_SAMPLE_NAMES))
exp = pd.DataFrame.from_dict(
SAMPLE_TEMPLATE_NUMBER_SAMPLE_NAMES_DICT_FORM)
exp.index.name = 'sample_name'
obs.sort_index(inplace=True)
exp.sort_index(inplace=True)
assert_frame_equal(obs, exp)
def remove_add_prep_template(self, fp_rpt, study, data_type_id,
investigation_type):
"""add prep templates"""
pt_id = PrepTemplate.create(load_template_to_dataframe(fp_rpt),
study,
_to_int(data_type_id),
investigation_type=investigation_type).id
remove(fp_rpt)
return pt_id
def test_post_non_existant_study(self):
# study id that does not exist
prep = StringIO(EXP_PREP_TEMPLATE.format(0))
prep_table = load_template_to_dataframe(prep)
response = self.post('/api/v1/study/0/preparation?'
'&data_type=16S',
data=prep_table.T.to_dict(),
headers=self.headers, asjson=True)
self.assertEqual(response.code, 404)
def test_load_template_to_dataframe_qiime_map(self):
obs = load_template_to_dataframe(StringIO(QIIME_TUTORIAL_MAP_SUBSET),
index='#SampleID')
exp = pd.DataFrame.from_dict(QIIME_TUTORIAL_MAP_DICT_FORM)
exp.index.name = 'SampleID'
obs.sort_index(axis=0, inplace=True)
obs.sort_index(axis=1, inplace=True)
exp.sort_index(axis=0, inplace=True)
exp.sort_index(axis=1, inplace=True)
assert_frame_equal(obs, exp)
def test_post_non_existant_study(self):
# study id that does not exist
prep = StringIO(EXP_PREP_TEMPLATE.format(0))
prep_table = load_template_to_dataframe(prep)
response = self.post('/api/v1/study/0/preparation?'
'&data_type=16S',
data=prep_table.T.to_dict(),
headers=self.headers,
asjson=True)
self.assertEqual(response.code, 404)
def create_sample_template(fp, study, is_mapping_file, data_type=None):
"""Creates a sample template
Parameters
----------
fp : str
The file path to the template file
study : qiita_db.study.Study
The study to add the sample template to
is_mapping_file : bool
Whether `fp` contains a mapping file or a sample template
data_type : str, optional
If `is_mapping_file` is True, the data type of the prep template to be
created
Returns
-------
dict of {str: str}
A dict of the form {'status': str, 'message': str}
"""
# The imports need to be in here because this code is executed in
# the ipython workers
import warnings
from os import remove
from qiita_db.metadata_template.sample_template import SampleTemplate
from qiita_db.metadata_template.util import load_template_to_dataframe
from qiita_ware.metadata_pipeline import (
create_templates_from_qiime_mapping_file)
status = 'success'
msg = ''
try:
with warnings.catch_warnings(record=True) as warns:
if is_mapping_file:
create_templates_from_qiime_mapping_file(fp, study,
data_type)
else:
SampleTemplate.create(load_template_to_dataframe(fp),
study)
remove(fp)
# join all the warning messages into one. Note that this
# info will be ignored if an exception is raised
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
status = 'warning'
except Exception as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
status = 'danger'
msg = str(e)
return {'status': status, 'message': msg}
def create_sample_template(fp, study, is_mapping_file, data_type=None):
"""Creates a sample template
Parameters
----------
fp : str
The file path to the template file
study : qiita_db.study.Study
The study to add the sample template to
is_mapping_file : bool
Whether `fp` contains a mapping file or a sample template
data_type : str, optional
If `is_mapping_file` is True, the data type of the prep template to be
created
Returns
-------
dict of {str: str}
A dict of the form {'status': str, 'message': str}
"""
# The imports need to be in here because this code is executed in
# the ipython workers
import warnings
from os import remove
from qiita_db.metadata_template.sample_template import SampleTemplate
from qiita_db.metadata_template.util import load_template_to_dataframe
from qiita_ware.metadata_pipeline import (
create_templates_from_qiime_mapping_file)
status = 'success'
msg = ''
try:
with warnings.catch_warnings(record=True) as warns:
if is_mapping_file:
create_templates_from_qiime_mapping_file(fp, study,
data_type)
else:
SampleTemplate.create(load_template_to_dataframe(fp),
study)
remove(fp)
# join all the warning messages into one. Note that this
# info will be ignored if an exception is raised
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
status = 'warning'
except Exception as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
status = 'danger'
msg = str(e)
return {'status': status, 'message': msg}
def test_post_non_matching_identifiers(self):
prep = StringIO(EXP_PREP_TEMPLATE.format(100))
prep_table = load_template_to_dataframe(prep)
response = self.post('/api/v1/study/1/preparation?'
'data_type=16S',
data=prep_table.T.to_dict(),
headers=self.headers, asjson=True)
self.assertEqual(response.code, 406)
obs = json_decode(response.body)
self.assertCountEqual(obs.keys(), ['message'])
self.assertGreater(len(obs['message']), 0)
def test_post_non_matching_identifiers(self):
prep = StringIO(EXP_PREP_TEMPLATE.format(100))
prep_table = load_template_to_dataframe(prep)
response = self.post('/api/v1/study/1/preparation?'
'data_type=16S',
data=prep_table.T.to_dict(),
headers=self.headers,
asjson=True)
self.assertEqual(response.code, 406)
obs = json_decode(response.body)
self.assertEqual(list(obs.keys()), ['message'])
self.assertGreater(len(obs['message']), 0)
def test_post_valid_study(self):
prep = StringIO(EXP_PREP_TEMPLATE.format(1))
prep_table = load_template_to_dataframe(prep)
response = self.post('/api/v1/study/1/preparation?data_type=16S',
data=prep_table.T.to_dict(),
headers=self.headers, asjson=True)
self.assertEqual(response.code, 201)
exp = json_decode(response.body)
exp_prep = PrepTemplate(exp['id']).to_dataframe()
prep_table.index.name = 'sample_id'
# sort columns to be comparable
prep_table = prep_table[sorted(prep_table.columns.tolist())]
exp_prep = exp_prep[sorted(exp_prep.columns.tolist())]
exp_prep.drop('qiita_prep_id', axis=1, inplace=True)
pd.util.testing.assert_frame_equal(prep_table, exp_prep)
def update_sample_template(study_id, fp):
"""Updates a sample template
Parameters
----------
study_id : int
Study id whose template is going to be updated
fp : str
The file path to the template file
Returns
-------
dict of {str: str}
A dict of the form {'status': str, 'message': str}
"""
import warnings
from os import remove
from qiita_db.metadata_template.util import load_template_to_dataframe
from qiita_db.metadata_template.sample_template import SampleTemplate
msg = ''
status = 'success'
try:
with warnings.catch_warnings(record=True) as warns:
# deleting previous uploads and inserting new one
st = SampleTemplate(study_id)
df = load_template_to_dataframe(fp)
st.extend(df)
st.update(df)
remove(fp)
# join all the warning messages into one. Note that this info
# will be ignored if an exception is raised
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
status = 'warning'
except Exception as e:
status = 'danger'
msg = str(e)
return {'status': status, 'message': msg}
def update_sample_template(study_id, fp):
"""Updates a sample template
Parameters
----------
study_id : int
Study id whose template is going to be updated
fp : str
The file path to the template file
Returns
-------
dict of {str: str}
A dict of the form {'status': str, 'message': str}
"""
import warnings
from os import remove
from qiita_db.metadata_template.util import load_template_to_dataframe
from qiita_db.metadata_template.sample_template import SampleTemplate
msg = ''
status = 'success'
try:
with warnings.catch_warnings(record=True) as warns:
# deleting previous uploads and inserting new one
st = SampleTemplate(study_id)
df = load_template_to_dataframe(fp)
st.extend(df)
st.update(df)
remove(fp)
# join all the warning messages into one. Note that this info
# will be ignored if an exception is raised
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
status = 'warning'
except Exception as e:
status = 'danger'
msg = str(e)
return {'status': status, 'message': msg}
def test_post_valid_study(self):
prep = StringIO(EXP_PREP_TEMPLATE.format(1))
prep_table = load_template_to_dataframe(prep)
response = self.post('/api/v1/study/1/preparation?data_type=16S',
data=prep_table.T.to_dict(),
headers=self.headers,
asjson=True)
self.assertEqual(response.code, 201)
exp = json_decode(response.body)
exp_prep = PrepTemplate(exp['id']).to_dataframe()
prep_table.index.name = 'sample_id'
# sort columns to be comparable
prep_table = prep_table[sorted(prep_table.columns.tolist())]
exp_prep = exp_prep[sorted(exp_prep.columns.tolist())]
exp_prep.drop('qiita_prep_id', axis=1, inplace=True)
pd.util.testing.assert_frame_equal(prep_table, exp_prep)
def update_prep_template(prep_id, fp):
"""Updates a prep template
Parameters
----------
prep_id : int
Prep template id to be updated
fp : str
The file path to the template file
Returns
-------
dict of {str: str}
A dict of the form {'status': str, 'message': str}
"""
import warnings
from os import remove
from qiita_db.metadata_template.util import load_template_to_dataframe
from qiita_db.metadata_template.prep_template import PrepTemplate
msg = ''
status = 'success'
prep = PrepTemplate(prep_id)
try:
with warnings.catch_warnings(record=True) as warns:
df = load_template_to_dataframe(fp)
prep.extend(df)
prep.update(df)
remove(fp)
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
status = 'warning'
except Exception as e:
status = 'danger'
msg = str(e)
return {'status': status, 'message': msg}
def update_prep_template(prep_id, fp):
"""Updates a prep template
Parameters
----------
prep_id : int
Prep template id to be updated
fp : str
The file path to the template file
Returns
-------
dict of {str: str}
A dict of the form {'status': str, 'message': str}
"""
import warnings
from os import remove
from qiita_db.metadata_template.util import load_template_to_dataframe
from qiita_db.metadata_template.prep_template import PrepTemplate
msg = ''
status = 'success'
prep = PrepTemplate(prep_id)
try:
with warnings.catch_warnings(record=True) as warns:
df = load_template_to_dataframe(fp)
prep.extend(df)
prep.update(df)
remove(fp)
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
status = 'warning'
except Exception as e:
status = 'danger'
msg = str(e)
return {'status': status, 'message': msg}
def prep_template_post_req(study_id, user_id, prep_template, data_type,
investigation_type=None,
user_defined_investigation_type=None,
new_investigation_type=None):
"""Adds a prep template to the system
Parameters
----------
study_id : int
Study to attach the prep template to
user_id : str
User adding the prep template
prep_template : str
Filepath to the prep template being added
data_type : str
Data type of the processed samples
investigation_type: str, optional
Existing investigation type to attach to the prep template
user_defined_investigation_type: str, optional
Existing user added investigation type to attach to the prep template
new_investigation_type: str, optional
Investigation type to add to the system
Returns
-------
dict of str
{'status': status,
'message': message,
'file': prep_template,
'id': id}
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
fp_rpt = check_fp(study_id, prep_template)
if fp_rpt['status'] != 'success':
# Unknown filepath, so return the error message
return fp_rpt
fp_rpt = fp_rpt['file']
# Add new investigation type if needed
investigation_type = _process_investigation_type(
investigation_type, user_defined_investigation_type,
new_investigation_type)
msg = ''
status = 'success'
prep = None
try:
with warnings.catch_warnings(record=True) as warns:
# deleting previous uploads and inserting new one
prep = PrepTemplate.create(
load_template_to_dataframe(fp_rpt), Study(study_id), data_type,
investigation_type=investigation_type)
remove(fp_rpt)
# join all the warning messages into one. Note that this info
# will be ignored if an exception is raised
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
status = 'warning'
except Exception as e:
# Some error occurred while processing the prep template
# Show the error to the user so he can fix the template
status = 'error'
msg = str(e)
info = {'status': status,
'message': msg,
'file': prep_template,
'id': prep.id if prep is not None else None}
return info
def test_load_template_to_dataframe_empty_rows(self):
obs = load_template_to_dataframe(
StringIO(EXP_SAMPLE_TEMPLATE_SPACES_EMPTY_ROW))
exp = pd.DataFrame.from_dict(SAMPLE_TEMPLATE_DICT_FORM)
exp.index.name = 'sample_name'
assert_frame_equal(obs, exp)
def create_templates_from_qiime_mapping_file(fp, study, data_type):
"""Creates a sample template and a prep template from qiime mapping file
Parameters
----------
fp : str or file-like object
Path to the QIIME mapping file
study : Study
The study to which the sample template belongs to
data_type : str or int
The data_type of the prep_template
Returns
-------
(SampleTemplate, PrepTemplate)
The templates created from the QIIME mapping file
"""
qiime_map = load_template_to_dataframe(fp, index='#SampleID')
# There are a few columns in the QIIME mapping file that are special and
# we know how to deal with them
rename_cols = {
'BarcodeSequence': 'barcode',
'LinkerPrimerSequence': 'primer',
'Description': 'description',
}
if 'ReverseLinkerPrimer' in qiime_map:
rename_cols['ReverseLinkerPrimer'] = 'reverselinkerprimer'
missing = set(rename_cols).difference(qiime_map.columns)
if missing:
raise QiitaWareError(
"Error generating the templates from the QIIME mapping file. "
"Missing QIIME mapping file columns: %s" % ', '.join(missing))
qiime_map.rename(columns=rename_cols, inplace=True)
# Fix the casing in the columns that we control
qiime_map.columns = [c.lower() if c.lower() in CONTROLLED_COLS else c
for c in qiime_map.columns]
# Figure out which columns belong to the prep template
def _col_iterator(restriction_set):
for restriction in viewvalues(restriction_set):
for cols in viewkeys(restriction.columns):
yield cols
pt_cols = set(col for col in _col_iterator(PREP_TEMPLATE_COLUMNS))
data_type_str = (convert_from_id(data_type, "data_type")
if isinstance(data_type, int) else data_type)
if data_type_str in TARGET_GENE_DATA_TYPES:
pt_cols.update(
col for col in _col_iterator(PREP_TEMPLATE_COLUMNS_TARGET_GENE))
pt_cols.add('reverselinkerprimer')
qiime_cols = set(qiime_map.columns)
pt_cols = qiime_cols.intersection(pt_cols)
st_cols = qiime_cols.difference(pt_cols)
st_md = qiime_map.ix[:, st_cols]
pt_md = qiime_map.ix[:, pt_cols]
return (SampleTemplate.create(st_md, study),
PrepTemplate.create(pt_md, study, data_type))
def create_templates_from_qiime_mapping_file(fp, study, data_type):
"""Creates a sample template and a prep template from qiime mapping file
Parameters
----------
fp : str or file-like object
Path to the QIIME mapping file
study : Study
The study to which the sample template belongs to
data_type : str or int
The data_type of the prep_template
Returns
-------
(SampleTemplate, PrepTemplate)
The templates created from the QIIME mapping file
"""
qiime_map = load_template_to_dataframe(fp, index='#SampleID')
# There are a few columns in the QIIME mapping file that are special and
# we know how to deal with them
rename_cols = {
'BarcodeSequence': 'barcode',
'LinkerPrimerSequence': 'primer',
'Description': 'description',
}
if 'ReverseLinkerPrimer' in qiime_map:
rename_cols['ReverseLinkerPrimer'] = 'reverselinkerprimer'
missing = set(rename_cols).difference(qiime_map.columns)
if missing:
raise QiitaWareError(
"Error generating the templates from the QIIME mapping file. "
"Missing QIIME mapping file columns: %s" % ', '.join(missing))
qiime_map.rename(columns=rename_cols, inplace=True)
# Fix the casing in the columns that we control
qiime_map.columns = [
c.lower() if c.lower() in CONTROLLED_COLS else c
for c in qiime_map.columns
]
# Figure out which columns belong to the prep template
def _col_iterator(restriction_set):
for restriction in restriction_set.values():
for cols in restriction.columns.keys():
yield cols
pt_cols = set(col for col in _col_iterator(PREP_TEMPLATE_COLUMNS))
data_type_str = (convert_from_id(data_type, "data_type") if isinstance(
data_type, int) else data_type)
if data_type_str in TARGET_GENE_DATA_TYPES:
pt_cols.update(
col for col in _col_iterator(PREP_TEMPLATE_COLUMNS_TARGET_GENE))
pt_cols.add('reverselinkerprimer')
qiime_cols = set(qiime_map.columns)
pt_cols = qiime_cols.intersection(pt_cols)
st_cols = qiime_cols.difference(pt_cols)
st_md = qiime_map.loc[:, st_cols]
pt_md = qiime_map.loc[:, pt_cols]
return (SampleTemplate.create(st_md, study),
PrepTemplate.create(pt_md, study, data_type))
def process_sample_template(self, study, user, callback):
"""Process a sample template from the POST method
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
Raises
------
HTTPError
If the sample template file does not exists
"""
# If we are on this function, the arguments "sample_template" and
# "data_type" must be defined. If not, let tornado raise its error
sample_template = self.get_argument('sample_template')
data_type = self.get_argument('data_type')
# Get the uploads folder
_, base_fp = get_mountpoint("uploads")[0]
# Get the path of the sample template in the uploads folder
fp_rsp = join(base_fp, str(study.id), sample_template)
if not exists(fp_rsp):
# The file does not exist, fail nicely
raise HTTPError(404, "This file doesn't exist: %s" % fp_rsp)
# Define here the message and message level in case of success
msg = "The sample template '%s' has been added" % sample_template
msg_level = "success"
is_mapping_file = looks_like_qiime_mapping_file(fp_rsp)
try:
if is_mapping_file and not data_type:
raise ValueError("Please, choose a data type if uploading a "
"QIIME mapping file")
with warnings.catch_warnings(record=True) as warns:
if is_mapping_file:
create_templates_from_qiime_mapping_file(fp_rsp, study,
int(data_type))
else:
SampleTemplate.create(load_template_to_dataframe(fp_rsp),
study)
remove(fp_rsp)
# join all the warning messages into one. Note that this
# info will be ignored if an exception is raised
if warns:
msg = '; '.join([convert_text_html(str(w.message))
for w in warns])
msg_level = 'warning'
except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
QiitaDBDuplicateError, IOError, ValueError, KeyError,
CParserError, QiitaDBDuplicateHeaderError,
QiitaDBError, QiitaWareError) as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
error_msg = ('parsing the QIIME mapping file'
if is_mapping_file
else 'parsing the sample template')
msg = html_error_message % (error_msg, basename(fp_rsp),
str(e))
msg = convert_text_html(msg)
msg_level = "danger"
callback((msg, msg_level, None, None, None))
def test_load_template_to_dataframe_non_utf8(self):
bad = EXP_SAMPLE_TEMPLATE.replace('Test Sample 2', 'Test Sample\x962')
with self.assertRaises(QiitaDBError):
load_template_to_dataframe(StringIO(bad))
def test_load_template_to_dataframe_whitespace(self):
obs = load_template_to_dataframe(
StringIO(EXP_SAMPLE_TEMPLATE_WHITESPACE))
exp = pd.DataFrame.from_dict(SAMPLE_TEMPLATE_DICT_FORM)
exp.index.name = 'sample_name'
assert_frame_equal(obs, exp)
def prep_template_post_req(study_id, user_id, prep_template, data_type,
investigation_type=None,
user_defined_investigation_type=None,
new_investigation_type=None, name=None):
"""Adds a prep template to the system
Parameters
----------
study_id : int
Study to attach the prep template to
user_id : str
User adding the prep template
prep_template : str
Filepath to the prep template being added
data_type : str
Data type of the processed samples
investigation_type: str, optional
Existing investigation type to attach to the prep template
user_defined_investigation_type: str, optional
Existing user added investigation type to attach to the prep template
new_investigation_type: str, optional
Investigation type to add to the system
name : str, optional
The name of the new prep template
Returns
-------
dict of str
{'status': status,
'message': message,
'file': prep_template,
'id': id}
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
fp_rpt = check_fp(study_id, prep_template)
if fp_rpt['status'] != 'success':
# Unknown filepath, so return the error message
return fp_rpt
fp_rpt = fp_rpt['file']
# Add new investigation type if needed
investigation_type = _process_investigation_type(
investigation_type, user_defined_investigation_type,
new_investigation_type)
msg = ''
status = 'success'
prep = None
if name:
name = name if name.strip() else None
try:
with warnings.catch_warnings(record=True) as warns:
# deleting previous uploads and inserting new one
prep = PrepTemplate.create(
load_template_to_dataframe(fp_rpt), Study(study_id), data_type,
investigation_type=investigation_type, name=name)
remove(fp_rpt)
# join all the warning messages into one. Note that this info
# will be ignored if an exception is raised
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
status = 'warning'
except Exception as e:
# Some error occurred while processing the prep template
# Show the error to the user so he can fix the template
status = 'error'
msg = str(e)
info = {'status': status,
'message': msg,
'file': prep_template,
'id': prep.id if prep is not None else None}
return info
def test_load_template_to_dataframe_column_with_nas(self):
obs = load_template_to_dataframe(
StringIO(SAMPLE_TEMPLATE_COLUMN_WITH_NAS))
exp = pd.DataFrame.from_dict(ST_COLUMN_WITH_NAS_DICT_FORM)
exp.index.name = 'sample_name'
assert_frame_equal(obs, exp)
def process_sample_template(self, study, user, callback):
"""Process a sample template from the POST method
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
Raises
------
HTTPError
If the sample template file does not exists
"""
# If we are on this function, the arguments "sample_template" and
# "data_type" must be defined. If not, let tornado raise its error
sample_template = self.get_argument('sample_template')
data_type = self.get_argument('data_type')
# Get the uploads folder
_, base_fp = get_mountpoint("uploads")[0]
# Get the path of the sample template in the uploads folder
fp_rsp = join(base_fp, str(study.id), sample_template)
if not exists(fp_rsp):
# The file does not exist, fail nicely
raise HTTPError(404, "This file doesn't exist: %s" % fp_rsp)
# Define here the message and message level in case of success
msg = "The sample template '%s' has been added" % sample_template
msg_level = "success"
is_mapping_file = looks_like_qiime_mapping_file(fp_rsp)
try:
if is_mapping_file and not data_type:
raise ValueError("Please, choose a data type if uploading a "
"QIIME mapping file")
with warnings.catch_warnings(record=True) as warns:
if is_mapping_file:
create_templates_from_qiime_mapping_file(
fp_rsp, study, int(data_type))
else:
SampleTemplate.create(load_template_to_dataframe(fp_rsp),
study)
remove(fp_rsp)
# join all the warning messages into one. Note that this
# info will be ignored if an exception is raised
if warns:
msg = '; '.join(
[convert_text_html(str(w.message)) for w in warns])
msg_level = 'warning'
except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
QiitaDBDuplicateError, IOError, ValueError, KeyError,
CParserError, QiitaDBDuplicateHeaderError, QiitaDBError,
QiitaWareError) as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
error_msg = ('parsing the QIIME mapping file'
if is_mapping_file else 'parsing the sample template')
msg = html_error_message % (error_msg, basename(fp_rsp), str(e))
msg = convert_text_html(msg)
msg_level = "danger"
callback((msg, msg_level, None, None, None))
