def test_copy_artifact(self):
# Failure test
job = self._create_job('copy_artifact',
{'artifact': 1, 'prep_template': 1})
private_task(job.id)
self.assertEqual(job.status, 'error')
self.assertIn("Prep template 1 already has an artifact associated",
job.log.msg)
# Success test
metadata_dict = {
'SKB8.640193': {'center_name': 'ANL',
'primer': 'GTGCCAGCMGCCGCGGTAA',
'barcode': 'GTCCGCAAGTTA',
'run_prefix': "s_G1_L001_sequences",
'platform': 'ILLUMINA',
'instrument_model': 'Illumina MiSeq',
'library_construction_protocol': 'AAAA',
'experiment_design_description': 'BBBB'}}
metadata = pd.DataFrame.from_dict(metadata_dict, orient='index',
dtype=str)
prep = PrepTemplate.create(metadata, Study(1), "16S")
job = self._create_job('copy_artifact', {'artifact': 1,
'prep_template': prep.id})
private_task(job.id)
self.assertEqual(job.status, 'success')
def test_copy_artifact(self):
# Failure test
job = self._create_job('copy_artifact', {
'artifact': 1,
'prep_template': 1
})
private_task(job.id)
self.assertEqual(job.status, 'error')
self.assertIn("Prep template 1 already has an artifact associated",
job.log.msg)
# Success test
metadata_dict = {
'SKB8.640193': {
'center_name': 'ANL',
'primer': 'GTGCCAGCMGCCGCGGTAA',
'barcode': 'GTCCGCAAGTTA',
'run_prefix': "s_G1_L001_sequences",
'platform': 'Illumina',
'instrument_model': 'Illumina MiSeq',
'library_construction_protocol': 'AAAA',
'experiment_design_description': 'BBBB'
}
}
metadata = pd.DataFrame.from_dict(metadata_dict,
orient='index',
dtype=str)
prep = PrepTemplate.create(metadata, Study(1), "16S")
job = self._create_job('copy_artifact', {
'artifact': 1,
'prep_template': prep.id
})
private_task(job.id)
self.assertEqual(job.status, 'success')
def remove_add_prep_template(self, fp_rpt, study, data_type_id,
investigation_type):
"""add prep templates"""
pt_id = PrepTemplate.create(load_template_to_dataframe(fp_rpt),
study, _to_int(data_type_id),
investigation_type=investigation_type).id
remove(fp_rpt)
return pt_id
def remove_add_prep_template(self, fp_rpt, study, data_type_id,
investigation_type):
"""add prep templates"""
pt_id = PrepTemplate.create(load_template_to_dataframe(fp_rpt),
study,
_to_int(data_type_id),
investigation_type=investigation_type).id
remove(fp_rpt)
return pt_id
def test_prep_template_jobs_get_req(self):
# Create a new template:
metadata = pd.DataFrame.from_dict(
{
'SKD6.640190': {
'center_name': 'ANL',
'target_subfragment': 'V4',
'center_project_name': 'Test Project',
'ebi_submission_accession': None,
'EMP_status': 'EMP',
'str_column': 'Value for sample 1',
'primer': 'GTGCCAGCMGCCGCGGTAA',
'barcode': 'GTCCGCAAGTTA',
'run_prefix': "s_G1_L001_sequences",
'platform': 'Illumina',
'instrument_model': 'Illumina MiSeq',
'library_construction_protocol': 'AAAA',
'experiment_design_description': 'BBBB'
}
},
orient='index',
dtype=str)
pt = PrepTemplate.create(metadata, Study(1), '16S')
# Check that it returns an empty dictionary when there are no jobs
# attached to the prep template
self.assertEqual(prep_template_jobs_get_req(pt.id, '*****@*****.**'), {})
# Create a job on the template
prep_template_patch_req('*****@*****.**', 'remove',
'/%s/10/columns/target_subfragment/' % pt.id)
# To ensure a deterministic result, wait until the job is completed
self._wait_for_parallel_job('prep_template_%s' % pt.id)
obs = prep_template_jobs_get_req(pt.id, '*****@*****.**')
self.assertEqual(len(obs), 1)
self.assertCountEqual(obs.values(), [{
'error': '',
'status': 'success',
'step': None
}])
obs = prep_template_jobs_get_req(pt.id, '*****@*****.**')
exp = {
'status': 'error',
'message': 'User does not have access to study'
}
self.assertEqual(obs, exp)
def post(self, study_id, *args, **kwargs):
data_type = self.get_argument('data_type')
investigation_type = self.get_argument('investigation_type', None)
study_id = self.safe_get_study(study_id)
if study_id is None:
return
data = pd.DataFrame.from_dict(json_decode(self.request.body),
orient='index')
try:
p = PrepTemplate.create(data, study_id, data_type,
investigation_type)
except QiitaError as e:
self.fail(str(e), 406)
return
self.write({'id': p.id})
self.set_status(201)
self.finish()
def test_prep_template_jobs_get_req(self):
# Create a new template:
metadata = pd.DataFrame.from_dict(
{'SKD6.640190': {'center_name': 'ANL',
'target_subfragment': 'V4',
'center_project_name': 'Test Project',
'ebi_submission_accession': None,
'EMP_status': 'EMP',
'str_column': 'Value for sample 1',
'primer': 'GTGCCAGCMGCCGCGGTAA',
'barcode': 'GTCCGCAAGTTA',
'run_prefix': "s_G1_L001_sequences",
'platform': 'ILLUMINA',
'instrument_model': 'Illumina MiSeq',
'library_construction_protocol': 'AAAA',
'experiment_design_description': 'BBBB'}},
orient='index', dtype=str)
pt = PrepTemplate.create(metadata, Study(1), '16S')
# Check that it returns an empty dictionary when there are no jobs
# attached to the prep template
self.assertEqual(prep_template_jobs_get_req(pt.id, '*****@*****.**'), {})
# Create a job on the template
prep_template_patch_req(
'*****@*****.**', 'remove',
'/%s/10/columns/target_subfragment/' % pt.id)
# To ensure a deterministic result, wait until the job is completed
self._wait_for_parallel_job('prep_template_%s' % pt.id)
obs = prep_template_jobs_get_req(pt.id, '*****@*****.**')
self.assertEqual(len(obs), 1)
self.assertEqual(obs.values(),
[{'error': '', 'status': 'success', 'step': None}])
obs = prep_template_jobs_get_req(pt.id, '*****@*****.**')
exp = {'status': 'error',
'message': 'User does not have access to study'}
self.assertEqual(obs, exp)
def generate_new_study_with_preprocessed_data(self):
"""Creates a new study up to the processed data for testing"""
info = {
"timeseries_type_id": 1,
"metadata_complete": True,
"mixs_compliant": True,
"number_samples_collected": 3,
"number_samples_promised": 3,
"study_alias": "Test EBI",
"study_description": "Study for testing EBI",
"study_abstract": "Study for testing EBI",
"emp_person_id": StudyPerson(2),
"principal_investigator_id": StudyPerson(3),
"lab_person_id": StudyPerson(1)
}
study = Study.create(User('*****@*****.**'), "Test EBI study", info)
metadata_dict = {
'Sample1': {
'collection_timestamp': datetime(2015, 6, 1, 7, 0, 0),
'physical_specimen_location': 'location1',
'taxon_id': 9606,
'scientific_name': 'h**o sapiens',
'Description': 'Test Sample 1'
},
'Sample2': {
'collection_timestamp': datetime(2015, 6, 2, 7, 0, 0),
'physical_specimen_location': 'location1',
'taxon_id': 9606,
'scientific_name': 'h**o sapiens',
'Description': 'Test Sample 2'
},
'Sample3': {
'collection_timestamp': datetime(2015, 6, 3, 7, 0, 0),
'physical_specimen_location': 'location1',
'taxon_id': 9606,
'scientific_name': 'h**o sapiens',
'Description': 'Test Sample 3'
}
}
metadata = pd.DataFrame.from_dict(metadata_dict,
orient='index',
dtype=str)
SampleTemplate.create(metadata, study)
metadata_dict = {
'Sample1': {
'primer': 'GTGCCAGCMGCCGCGGTAA',
'barcode': 'CGTAGAGCTCTC',
'center_name': 'KnightLab',
'platform': 'ILLUMINA',
'instrument_model': 'Illumina MiSeq',
'library_construction_protocol': 'Protocol ABC',
'experiment_design_description': "Random value 1"
},
'Sample2': {
'primer': 'GTGCCAGCMGCCGCGGTAA',
'barcode': 'CGTAGAGCTCTA',
'center_name': 'KnightLab',
'platform': 'ILLUMINA',
'instrument_model': 'Illumina MiSeq',
'library_construction_protocol': 'Protocol ABC',
'experiment_design_description': "Random value 2"
},
'Sample3': {
'primer': 'GTGCCAGCMGCCGCGGTAA',
'barcode': 'CGTAGAGCTCTT',
'center_name': 'KnightLab',
'platform': 'ILLUMINA',
'instrument_model': 'Illumina MiSeq',
'library_construction_protocol': 'Protocol ABC',
'experiment_design_description': "Random value 3"
},
}
metadata = pd.DataFrame.from_dict(metadata_dict,
orient='index',
dtype=str)
pt = PrepTemplate.create(metadata, study, "16S", 'Metagenomics')
fna_fp = join(self.temp_dir, 'seqs.fna')
demux_fp = join(self.temp_dir, 'demux.seqs')
with open(fna_fp, 'w') as f:
f.write(FASTA_EXAMPLE_2.format(study.id))
with File(demux_fp, 'w') as f:
to_hdf5(fna_fp, f)
ppd = Artifact.create([(demux_fp, 6)],
"Demultiplexed",
prep_template=pt)
return ppd
def prep_template_post_req(study_id, user_id, prep_template, data_type,
investigation_type=None,
user_defined_investigation_type=None,
new_investigation_type=None):
"""Adds a prep template to the system
Parameters
----------
study_id : int
Study to attach the prep template to
user_id : str
User adding the prep template
prep_template : str
Filepath to the prep template being added
data_type : str
Data type of the processed samples
investigation_type: str, optional
Existing investigation type to attach to the prep template
user_defined_investigation_type: str, optional
Existing user added investigation type to attach to the prep template
new_investigation_type: str, optional
Investigation type to add to the system
Returns
-------
dict of str
{'status': status,
'message': message,
'file': prep_template,
'id': id}
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
fp_rpt = check_fp(study_id, prep_template)
if fp_rpt['status'] != 'success':
# Unknown filepath, so return the error message
return fp_rpt
fp_rpt = fp_rpt['file']
# Add new investigation type if needed
investigation_type = _process_investigation_type(
investigation_type, user_defined_investigation_type,
new_investigation_type)
msg = ''
status = 'success'
prep = None
try:
with warnings.catch_warnings(record=True) as warns:
# deleting previous uploads and inserting new one
prep = PrepTemplate.create(
load_template_to_dataframe(fp_rpt), Study(study_id), data_type,
investigation_type=investigation_type)
remove(fp_rpt)
# join all the warning messages into one. Note that this info
# will be ignored if an exception is raised
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
status = 'warning'
except Exception as e:
# Some error occurred while processing the prep template
# Show the error to the user so he can fix the template
status = 'error'
msg = str(e)
info = {'status': status,
'message': msg,
'file': prep_template,
'id': prep.id if prep is not None else None}
return info
def generate_new_study_with_preprocessed_data(self):
"""Creates a new study up to the processed data for testing"""
# ignoring warnings generated when adding templates
simplefilter("ignore")
info = {
"timeseries_type_id": 1,
"metadata_complete": True,
"mixs_compliant": True,
"number_samples_collected": 3,
"number_samples_promised": 3,
"study_alias": "Test EBI",
"study_description": "Study for testing EBI",
"study_abstract": "Study for testing EBI",
"emp_person_id": StudyPerson(2),
"principal_investigator_id": StudyPerson(3),
"lab_person_id": StudyPerson(1)
}
study = Study.create(User('*****@*****.**'), "Test EBI study", [1], info)
metadata_dict = {
'Sample1': {'collection_timestamp': datetime(2015, 6, 1, 7, 0, 0),
'physical_specimen_location': 'location1',
'taxon_id': 9606,
'scientific_name': 'h**o sapiens',
'Description': 'Test Sample 1'},
'Sample2': {'collection_timestamp': datetime(2015, 6, 2, 7, 0, 0),
'physical_specimen_location': 'location1',
'taxon_id': 9606,
'scientific_name': 'h**o sapiens',
'Description': 'Test Sample 2'},
'Sample3': {'collection_timestamp': datetime(2015, 6, 3, 7, 0, 0),
'physical_specimen_location': 'location1',
'taxon_id': 9606,
'scientific_name': 'h**o sapiens',
'Description': 'Test Sample 3'}
}
metadata = pd.DataFrame.from_dict(metadata_dict, orient='index',
dtype=str)
SampleTemplate.create(metadata, study)
metadata_dict = {
'Sample1': {'primer': 'GTGCCAGCMGCCGCGGTAA',
'barcode': 'CGTAGAGCTCTC',
'center_name': 'KnightLab',
'platform': 'ILLUMINA',
'instrument_model': 'Illumina MiSeq',
'library_construction_protocol': 'Protocol ABC',
'experiment_design_description': "Random value 1"},
'Sample2': {'primer': 'GTGCCAGCMGCCGCGGTAA',
'barcode': 'CGTAGAGCTCTA',
'center_name': 'KnightLab',
'platform': 'ILLUMINA',
'instrument_model': 'Illumina MiSeq',
'library_construction_protocol': 'Protocol ABC',
'experiment_design_description': "Random value 2"},
'Sample3': {'primer': 'GTGCCAGCMGCCGCGGTAA',
'barcode': 'CGTAGAGCTCTT',
'center_name': 'KnightLab',
'platform': 'ILLUMINA',
'instrument_model': 'Illumina MiSeq',
'library_construction_protocol': 'Protocol ABC',
'experiment_design_description': "Random value 3"},
}
metadata = pd.DataFrame.from_dict(metadata_dict, orient='index',
dtype=str)
pt = PrepTemplate.create(metadata, study, "16S", 'Metagenomics')
fna_fp = join(self.temp_dir, 'seqs.fna')
demux_fp = join(self.temp_dir, 'demux.seqs')
with open(fna_fp, 'w') as f:
f.write(FASTA_EXAMPLE_2.format(study.id))
with File(demux_fp, 'w') as f:
to_hdf5(fna_fp, f)
ppd = Artifact.create(
[(demux_fp, 6)], "Demultiplexed", prep_template=pt)
return ppd
def test_prep_template_patch_req(self):
metadata = pd.DataFrame.from_dict(
{
'SKD6.640190': {
'center_name': 'ANL',
'target_subfragment': 'V4',
'center_project_name': 'Test Project',
'ebi_submission_accession': None,
'EMP_status': 'EMP',
'str_column': 'Value for sample 1',
'primer': 'GTGCCAGCMGCCGCGGTAA',
'barcode': 'GTCCGCAAGTTA',
'run_prefix': "s_G1_L001_sequences",
'platform': 'ILLUMINA',
'instrument_model': 'Illumina MiSeq',
'library_construction_protocol': 'AAAA',
'experiment_design_description': 'BBBB'
}
},
orient='index',
dtype=str)
pt = PrepTemplate.create(metadata, Study(1), '16S')
# Update investigation type
obs = prep_template_patch_req('*****@*****.**', 'replace',
'/%s/investigation_type' % pt.id,
'Cancer Genomics')
exp = {'status': 'success', 'message': ''}
self.assertEqual(obs, exp)
self.assertEqual(pt.investigation_type, 'Cancer Genomics')
# Update prep template data
obs = prep_template_patch_req('*****@*****.**', 'replace',
'/%s/data' % pt.id, 'update.txt')
self.assertEqual(obs, exp)
obs = r_client.get('prep_template_%s' % pt.id)
self.assertIsNotNone(obs)
self._wait_for_parallel_job('prep_template_%s' % pt.id)
# Delete a prep template column
obs = prep_template_patch_req(
'*****@*****.**', 'remove',
'/%s/10/columns/target_subfragment/' % pt.id)
exp = {'status': 'success', 'message': '', 'row_id': '10'}
self.assertEqual(obs, exp)
self._wait_for_parallel_job('prep_template_%s' % pt.id)
self.assertNotIn('target_subfragment', pt.categories())
# Test all the errors
# Operation not supported
obs = prep_template_patch_req('*****@*****.**', 'add',
'/1/investigation_type',
'Cancer Genomics')
exp = {
'status': 'error',
'message': 'Operation "add" not supported. '
'Current supported operations: replace, remove',
'row_id': '0'
}
self.assertEqual(obs, exp)
# Incorrect path parameter
obs = prep_template_patch_req('*****@*****.**', 'replace',
'/investigation_type', 'Cancer Genomics')
exp = {'status': 'error', 'message': 'Incorrect path parameter'}
self.assertEqual(obs, exp)
# Incorrect attribute
obs = prep_template_patch_req('*****@*****.**', 'replace',
'/1/other_attribute', 'Cancer Genomics')
exp = {
'status':
'error',
'message':
'Attribute "other_attribute" not found. '
'Please, check the path parameter'
}
self.assertEqual(obs, exp)
# User doesn't have access
obs = prep_template_patch_req('*****@*****.**', 'replace',
'/%s/investigation_type' % pt.id,
'Cancer Genomics')
exp = {
'status': 'error',
'message': 'User does not have access to study'
}
self.assertEqual(obs, exp)
# File does not exists
obs = prep_template_patch_req('*****@*****.**', 'replace', '/1/data',
'unknown_file.txt')
exp = {
'status': 'error',
'message': 'file does not exist',
'file': 'unknown_file.txt'
}
self.assertEqual(obs, exp)
def test_prep_template_patch_req(self):
metadata = pd.DataFrame.from_dict(
{'SKD6.640190': {'center_name': 'ANL',
'target_subfragment': 'V4',
'center_project_name': 'Test Project',
'ebi_submission_accession': None,
'EMP_status': 'EMP',
'str_column': 'Value for sample 1',
'primer': 'GTGCCAGCMGCCGCGGTAA',
'barcode': 'GTCCGCAAGTTA',
'run_prefix': "s_G1_L001_sequences",
'platform': 'ILLUMINA',
'instrument_model': 'Illumina MiSeq',
'library_construction_protocol': 'AAAA',
'experiment_design_description': 'BBBB'}},
orient='index', dtype=str)
pt = PrepTemplate.create(metadata, Study(1), '16S')
# Update investigation type
obs = prep_template_patch_req(
'*****@*****.**', 'replace', '/%s/investigation_type' % pt.id,
'Cancer Genomics')
exp = {'status': 'success', 'message': ''}
self.assertEqual(obs, exp)
self.assertEqual(pt.investigation_type, 'Cancer Genomics')
# Update prep template data
obs = prep_template_patch_req(
'*****@*****.**', 'replace', '/%s/data' % pt.id, 'update.txt')
self.assertEqual(obs, exp)
obs = r_client.get('prep_template_%s' % pt.id)
self.assertIsNotNone(obs)
self._wait_for_parallel_job('prep_template_%s' % pt.id)
# Delete a prep template column
obs = prep_template_patch_req(
'*****@*****.**', 'remove',
'/%s/10/columns/target_subfragment/' % pt.id)
exp = {'status': 'success', 'message': '', 'row_id': '10'}
self.assertEqual(obs, exp)
self._wait_for_parallel_job('prep_template_%s' % pt.id)
self.assertNotIn('target_subfragment', pt.categories())
# Change the name of the prep template
obs = prep_template_patch_req(
'*****@*****.**', 'replace', '/%s/name' % pt.id, ' My New Name ')
exp = {'status': 'success', 'message': ''}
self.assertEqual(obs, exp)
self.assertEqual(pt.name, 'My New Name')
# Test all the errors
# Operation not supported
obs = prep_template_patch_req(
'*****@*****.**', 'add', '/1/investigation_type',
'Cancer Genomics')
exp = {'status': 'error',
'message': 'Operation "add" not supported. '
'Current supported operations: replace, remove',
'row_id': '0'}
self.assertEqual(obs, exp)
# Incorrect path parameter
obs = prep_template_patch_req(
'*****@*****.**', 'replace', '/investigation_type',
'Cancer Genomics')
exp = {'status': 'error',
'message': 'Incorrect path parameter'}
self.assertEqual(obs, exp)
# Incorrect attribute
obs = prep_template_patch_req(
'*****@*****.**', 'replace', '/1/other_attribute',
'Cancer Genomics')
exp = {'status': 'error',
'message': 'Attribute "other_attribute" not found. '
'Please, check the path parameter'}
self.assertEqual(obs, exp)
# User doesn't have access
obs = prep_template_patch_req(
'*****@*****.**', 'replace', '/%s/investigation_type' % pt.id,
'Cancer Genomics')
exp = {'status': 'error',
'message': 'User does not have access to study'}
self.assertEqual(obs, exp)
# File does not exists
obs = prep_template_patch_req(
'*****@*****.**', 'replace', '/1/data', 'unknown_file.txt')
exp = {'status': 'error',
'message': 'file does not exist',
'file': 'unknown_file.txt'}
self.assertEqual(obs, exp)
def generate_new_study_with_preprocessed_data(self):
"""Creates a new study up to the processed data for testing"""
info = {
"timeseries_type_id": 1,
"metadata_complete": True,
"mixs_compliant": True,
"number_samples_collected": 3,
"number_samples_promised": 3,
"study_alias": "Test EBI",
"study_description": "Study for testing EBI",
"study_abstract": "Study for testing EBI",
"emp_person_id": StudyPerson(2),
"principal_investigator_id": StudyPerson(3),
"lab_person_id": StudyPerson(1),
}
study = Study.create(User("*****@*****.**"), "Test EBI study", [1], info)
metadata_dict = {
"Sample1": {
"collection_timestamp": datetime(2015, 6, 1, 7, 0, 0),
"physical_specimen_location": "location1",
"taxon_id": 9606,
"scientific_name": "h**o sapiens",
"Description": "Test Sample 1",
},
"Sample2": {
"collection_timestamp": datetime(2015, 6, 2, 7, 0, 0),
"physical_specimen_location": "location1",
"taxon_id": 9606,
"scientific_name": "h**o sapiens",
"Description": "Test Sample 2",
},
"Sample3": {
"collection_timestamp": datetime(2015, 6, 3, 7, 0, 0),
"physical_specimen_location": "location1",
"taxon_id": 9606,
"scientific_name": "h**o sapiens",
"Description": "Test Sample 3",
},
}
metadata = pd.DataFrame.from_dict(metadata_dict, orient="index", dtype=str)
SampleTemplate.create(metadata, study)
metadata_dict = {
"Sample1": {
"primer": "GTGCCAGCMGCCGCGGTAA",
"barcode": "CGTAGAGCTCTC",
"center_name": "KnightLab",
"platform": "ILLUMINA",
"instrument_model": "Illumina MiSeq",
"library_construction_protocol": "Protocol ABC",
"experiment_design_description": "Random value 1",
},
"Sample2": {
"primer": "GTGCCAGCMGCCGCGGTAA",
"barcode": "CGTAGAGCTCTA",
"center_name": "KnightLab",
"platform": "ILLUMINA",
"instrument_model": "Illumina MiSeq",
"library_construction_protocol": "Protocol ABC",
"experiment_design_description": "Random value 2",
},
"Sample3": {
"primer": "GTGCCAGCMGCCGCGGTAA",
"barcode": "CGTAGAGCTCTT",
"center_name": "KnightLab",
"platform": "ILLUMINA",
"instrument_model": "Illumina MiSeq",
"library_construction_protocol": "Protocol ABC",
"experiment_design_description": "Random value 3",
},
}
metadata = pd.DataFrame.from_dict(metadata_dict, orient="index", dtype=str)
pt = PrepTemplate.create(metadata, study, "16S", "Metagenomics")
fna_fp = join(self.temp_dir, "seqs.fna")
demux_fp = join(self.temp_dir, "demux.seqs")
with open(fna_fp, "w") as f:
f.write(FASTA_EXAMPLE_2.format(study.id))
with File(demux_fp, "w") as f:
to_hdf5(fna_fp, f)
ppd = Artifact.create([(demux_fp, 6)], "Demultiplexed", prep_template=pt)
return ppd
def test_delete_study(self):
# as samples have been submitted to EBI, this will fail
job = self._create_job('delete_study', {'study': 1})
private_task(job.id)
self.assertEqual(job.status, 'error')
self.assertIn("Artifact 2 has been submitted to EBI", job.log.msg)
# making sure the analysis, first thing to delete, still exists
self.assertTrue(Analysis.exists(1))
info = {
"timeseries_type_id": '1',
"metadata_complete": 'true',
"mixs_compliant": 'true',
"study_alias": "FCM",
"study_description": "Microbiome of people who eat nothing but "
"fried chicken",
"study_abstract": "Exploring how a high fat diet changes the "
"gut microbiome",
"principal_investigator_id": StudyPerson(3),
"lab_person_id": StudyPerson(1)
}
new_study = Study.create(User('*****@*****.**'),
"Fried Chicken Microbiome %s" % time(), info)
# adding tags
new_study.update_tags(User('*****@*****.**'), ['my new tag!'])
# creating a sample information file
metadata = pd.DataFrame.from_dict(
{
'Sample1': {
'physical_specimen_location': 'location1',
'taxon_id': '9606',
'scientific_name': 'h**o sapiens'
},
'Sample2': {
'physical_specimen_location': 'location1',
'taxon_id': '9606',
'scientific_name': 'h**o sapiens'
},
'Sample3': {
'physical_specimen_location': 'location1',
'taxon_id': '9606',
'scientific_name': 'h**o sapiens'
}
},
orient='index')
SampleTemplate.create(metadata, new_study)
# creating a preparation information file
metadata = pd.DataFrame.from_dict(
{
'Sample1': {
'center_name': 'ANL',
'target_subfragment': 'V4',
'center_project_name': 'Test Project'
}
},
orient='index',
dtype=str)
PrepTemplate.create(metadata, new_study, '16S')
job = self._create_job('delete_study', {'study': new_study.id})
private_task(job.id)
self.assertEqual(job.status, 'success')
# making sure the study doesn't exist
with self.assertRaises(QiitaDBUnknownIDError):
Study(new_study.id)
def prep_template_post_req(study_id, user_id, prep_template, data_type,
investigation_type=None,
user_defined_investigation_type=None,
new_investigation_type=None, name=None):
"""Adds a prep template to the system
Parameters
----------
study_id : int
Study to attach the prep template to
user_id : str
User adding the prep template
prep_template : str
Filepath to the prep template being added
data_type : str
Data type of the processed samples
investigation_type: str, optional
Existing investigation type to attach to the prep template
user_defined_investigation_type: str, optional
Existing user added investigation type to attach to the prep template
new_investigation_type: str, optional
Investigation type to add to the system
name : str, optional
The name of the new prep template
Returns
-------
dict of str
{'status': status,
'message': message,
'file': prep_template,
'id': id}
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
fp_rpt = check_fp(study_id, prep_template)
if fp_rpt['status'] != 'success':
# Unknown filepath, so return the error message
return fp_rpt
fp_rpt = fp_rpt['file']
# Add new investigation type if needed
investigation_type = _process_investigation_type(
investigation_type, user_defined_investigation_type,
new_investigation_type)
msg = ''
status = 'success'
prep = None
if name:
name = name if name.strip() else None
try:
with warnings.catch_warnings(record=True) as warns:
# deleting previous uploads and inserting new one
prep = PrepTemplate.create(
load_template_to_dataframe(fp_rpt), Study(study_id), data_type,
investigation_type=investigation_type, name=name)
remove(fp_rpt)
# join all the warning messages into one. Note that this info
# will be ignored if an exception is raised
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
status = 'warning'
except Exception as e:
# Some error occurred while processing the prep template
# Show the error to the user so he can fix the template
status = 'error'
msg = str(e)
info = {'status': status,
'message': msg,
'file': prep_template,
'id': prep.id if prep is not None else None}
return info
def create_templates_from_qiime_mapping_file(fp, study, data_type):
"""Creates a sample template and a prep template from qiime mapping file
Parameters
----------
fp : str or file-like object
Path to the QIIME mapping file
study : Study
The study to which the sample template belongs to
data_type : str or int
The data_type of the prep_template
Returns
-------
(SampleTemplate, PrepTemplate)
The templates created from the QIIME mapping file
"""
qiime_map = load_template_to_dataframe(fp, index='#SampleID')
# There are a few columns in the QIIME mapping file that are special and
# we know how to deal with them
rename_cols = {
'BarcodeSequence': 'barcode',
'LinkerPrimerSequence': 'primer',
'Description': 'description',
}
if 'ReverseLinkerPrimer' in qiime_map:
rename_cols['ReverseLinkerPrimer'] = 'reverselinkerprimer'
missing = set(rename_cols).difference(qiime_map.columns)
if missing:
raise QiitaWareError(
"Error generating the templates from the QIIME mapping file. "
"Missing QIIME mapping file columns: %s" % ', '.join(missing))
qiime_map.rename(columns=rename_cols, inplace=True)
# Fix the casing in the columns that we control
qiime_map.columns = [c.lower() if c.lower() in CONTROLLED_COLS else c
for c in qiime_map.columns]
# Figure out which columns belong to the prep template
def _col_iterator(restriction_set):
for restriction in viewvalues(restriction_set):
for cols in viewkeys(restriction.columns):
yield cols
pt_cols = set(col for col in _col_iterator(PREP_TEMPLATE_COLUMNS))
data_type_str = (convert_from_id(data_type, "data_type")
if isinstance(data_type, int) else data_type)
if data_type_str in TARGET_GENE_DATA_TYPES:
pt_cols.update(
col for col in _col_iterator(PREP_TEMPLATE_COLUMNS_TARGET_GENE))
pt_cols.add('reverselinkerprimer')
qiime_cols = set(qiime_map.columns)
pt_cols = qiime_cols.intersection(pt_cols)
st_cols = qiime_cols.difference(pt_cols)
st_md = qiime_map.ix[:, st_cols]
pt_md = qiime_map.ix[:, pt_cols]
return (SampleTemplate.create(st_md, study),
PrepTemplate.create(pt_md, study, data_type))
def create_templates_from_qiime_mapping_file(fp, study, data_type):
"""Creates a sample template and a prep template from qiime mapping file
Parameters
----------
fp : str or file-like object
Path to the QIIME mapping file
study : Study
The study to which the sample template belongs to
data_type : str or int
The data_type of the prep_template
Returns
-------
(SampleTemplate, PrepTemplate)
The templates created from the QIIME mapping file
"""
qiime_map = load_template_to_dataframe(fp, index='#SampleID')
# There are a few columns in the QIIME mapping file that are special and
# we know how to deal with them
rename_cols = {
'BarcodeSequence': 'barcode',
'LinkerPrimerSequence': 'primer',
'Description': 'description',
}
if 'ReverseLinkerPrimer' in qiime_map:
rename_cols['ReverseLinkerPrimer'] = 'reverselinkerprimer'
missing = set(rename_cols).difference(qiime_map.columns)
if missing:
raise QiitaWareError(
"Error generating the templates from the QIIME mapping file. "
"Missing QIIME mapping file columns: %s" % ', '.join(missing))
qiime_map.rename(columns=rename_cols, inplace=True)
# Fix the casing in the columns that we control
qiime_map.columns = [
c.lower() if c.lower() in CONTROLLED_COLS else c
for c in qiime_map.columns
]
# Figure out which columns belong to the prep template
def _col_iterator(restriction_set):
for restriction in restriction_set.values():
for cols in restriction.columns.keys():
yield cols
pt_cols = set(col for col in _col_iterator(PREP_TEMPLATE_COLUMNS))
data_type_str = (convert_from_id(data_type, "data_type") if isinstance(
data_type, int) else data_type)
if data_type_str in TARGET_GENE_DATA_TYPES:
pt_cols.update(
col for col in _col_iterator(PREP_TEMPLATE_COLUMNS_TARGET_GENE))
pt_cols.add('reverselinkerprimer')
qiime_cols = set(qiime_map.columns)
pt_cols = qiime_cols.intersection(pt_cols)
st_cols = qiime_cols.difference(pt_cols)
st_md = qiime_map.loc[:, st_cols]
pt_md = qiime_map.loc[:, pt_cols]
return (SampleTemplate.create(st_md, study),
PrepTemplate.create(pt_md, study, data_type))
