def sample_template_checks(study_id, user, check_exists=False):
"""Performs different checks and raises errors if any of the checks fail
Parameters
----------
study_id : int
The study id
user : qiita_db.user.User
The user trying to access the study
check_exists : bool, optional
If true, check if the sample template exists
Raises
------
HTTPError
404 if the study does not exist
403 if the user does not have access to the study
404 if check_exists == True and the sample template doesn't exist
"""
try:
study = Study(int(study_id))
except QiitaDBUnknownIDError:
raise HTTPError(404, reason='Study does not exist')
if not study.has_access(user):
raise HTTPError(403, reason='User does not have access to study')
# Check if the sample template exists
if check_exists and not SampleTemplate.exists(study_id):
raise HTTPError(404,
reason="Study %s doesn't have sample information" %
study_id)
def test_build_study_info(self):
for a in Study(1).artifacts():
a.visibility = 'private'
obs = _build_study_info(User('*****@*****.**'), 'user')
self.assertEqual(obs, self.exp)
obs = _build_study_info(User('*****@*****.**'), 'public')
self.assertEqual(obs, [])
obs = _build_study_info(User('*****@*****.**'), 'public')
self.assertEqual(obs, [])
# make all the artifacts public - (1) the only study in the tests,
for a in Study(1).artifacts():
a.visibility = 'public'
self.exp[0]['status'] = 'public'
obs = _build_study_info(User('*****@*****.**'), 'user')
self.assertEqual(obs, self.exp)
obs = _build_study_info(User('*****@*****.**'), 'public')
self.assertEqual(obs, [])
obs = _build_study_info(User('*****@*****.**'), 'public')
self.assertEqual(obs, self.exp)
# return to it's private status
for a in Study(1).artifacts():
a.visibility = 'private'
def test_create_nonqiita_portal(self):
qiita_config.portal = "EMP"
Study.create(User("*****@*****.**"), "NEW!", [1], self.info, Investigation(1))
# make sure portal is associated
obs = self.conn_handler.execute_fetchall("SELECT * from qiita.study_portal WHERE study_id = 2")
self.assertEqual(obs, [[2, 2], [2, 1]])
def setUp(self):
fd, self.seqs_fp = mkstemp(suffix='_seqs.fastq')
close(fd)
fd, self.barcodes_fp = mkstemp(suffix='_barcodes.fastq')
close(fd)
self.filetype = 2
self.filepaths = [(self.seqs_fp, 1), (self.barcodes_fp, 2)]
self.studies = [Study(1)]
_, self.db_test_raw_dir = get_mountpoint('raw_data')[0]
with open(self.seqs_fp, "w") as f:
f.write("\n")
with open(self.barcodes_fp, "w") as f:
f.write("\n")
self._clean_up_files = []
# Create a new study
info = {
"timeseries_type_id": 1,
"metadata_complete": True,
"mixs_compliant": True,
"number_samples_collected": 25,
"number_samples_promised": 28,
"portal_type_id": 3,
"study_alias": "FCM",
"study_description": "Microbiome of people who eat nothing but "
"fried chicken",
"study_abstract": "Exploring how a high fat diet changes the "
"gut microbiome",
"emp_person_id": StudyPerson(2),
"principal_investigator_id": StudyPerson(3),
"lab_person_id": StudyPerson(1)
}
Study.create(User("*****@*****.**"), "Test study 2", [1], info)
def delete_study(self, study, user, callback):
"""Delete study
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done and it fails
"""
study_id = study.id
study_title = study.title
try:
Study.delete(study_id)
# redirecting to list but also passing messages
# we need to change the request.method to GET
self.request.method = 'GET'
ListStudiesHandler(self.application, self.request)._execute(
[t(self.request) for t in self.application.transforms],
message=('Study "%s" has been deleted' % study_title),
msg_level='success')
except Exception as e:
msg = "Couldn't remove study %d: %s" % (study_id, str(e))
msg_level = "danger"
callback((msg, msg_level, 'study_information_tab', None, None))
def test_delete_study(self):
# as samples have been submitted to EBI, this will fail
job = self._create_job('delete_study', {'study': 1})
private_task(job.id)
self.assertEqual(job.status, 'error')
self.assertIn(
"Cannot delete artifact 2: Artifact 2 has been "
"submitted to EBI", job.log.msg)
# making sure the analysis, first thing to delete, still exists
self.assertTrue(Analysis.exists(1))
# delete everything from the EBI submissions and the processing job so
# we can try again: test success (with tags)
with TRN:
sql = """DELETE FROM qiita.ebi_run_accession"""
TRN.add(sql)
sql = """DELETE FROM qiita.artifact_processing_job"""
TRN.add(sql)
TRN.execute()
# adding tags
Study(1).update_tags(self.user, ['my new tag!'])
job = self._create_job('delete_study', {'study': 1})
private_task(job.id)
self.assertEqual(job.status, 'success')
with self.assertRaises(QiitaDBUnknownIDError):
Study(1)
def check_access(study_id, user_id):
"""Checks if user given has access to the study given
Parameters
----------
study_id : int
ID of the study to check access to
user_id : str
ID of the user to check access for
Returns
-------
dict
Empty dict if access allowed, else a dict in the form
{'status': 'error',
'message': reason for error}
"""
try:
study = Study(int(study_id))
except QiitaDBUnknownIDError:
return {'status': 'error',
'message': 'Study does not exist'}
if not study.has_access(User(user_id)):
return {'status': 'error',
'message': 'User does not have access to study'}
return {}
def test_build_study_info_new_study(self):
info = {
'timeseries_type_id': 1,
'lab_person_id': None,
'principal_investigator_id': 3,
'metadata_complete': False,
'mixs_compliant': True,
'study_description': 'desc',
'study_alias': 'alias',
'study_abstract': 'abstract'
}
user = User('*****@*****.**')
Study.create(user, 'test_study_1', efo=[1], info=info)
obs = _build_study_info(user)
self.exp.append({
'study_id': 2,
'status': 'sandbox',
'study_abstract': 'abstract',
'metadata_complete': False,
'study_title': 'test_study_1',
'num_raw_data': 0,
'number_samples_collected': 0,
'shared': '',
'pmid': '',
'publication_doi': '',
'pi':
'<a target="_blank" href="mailto:[email protected]">PIDude</a>',
'proc_data_info': []
})
self.assertEqual(obs, self.exp)
def test_build_study_info_new_study(self):
info = {
'timeseries_type_id': 1,
'lab_person_id': None,
'principal_investigator_id': 3,
'metadata_complete': False,
'mixs_compliant': True,
'study_description': 'desc',
'study_alias': 'alias',
'study_abstract': 'abstract'}
user = User('*****@*****.**')
Study.create(user, 'test_study_1', efo=[1], info=info)
obs = _build_study_info(user)
self.exp.append({
'study_id': 2,
'status': 'sandbox',
'study_abstract': 'abstract',
'metadata_complete': False,
'study_title': 'test_study_1',
'num_raw_data': 0,
'number_samples_collected': 0,
'shared': '',
'pmid': '',
'publication_doi': '',
'pi':
'<a target="_blank" href="mailto:[email protected]">PIDude</a>',
'proc_data_info': []})
self.assertEqual(obs, self.exp)
def study_get_req(study_id, user_id):
"""Returns information available for the given study
Parameters
----------
study_id : int
Study id to get prep template info for
user_id : str
User requesting the info
Returns
-------
dict
Data types information in the form
{'status': status,
'message': message,
'info': dict of objects
status can be success, warning, or error depending on result
message has the warnings or errors
info contains study information seperated by data type, in the form
{col_name: value, ...} with value being a string, int, or list of
strings or ints
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
# Can only pass ids over API, so need to instantiate object
study = Study(study_id)
study_info = study.info
# Add needed info that is not part of the initial info pull
study_info['publications'] = study.publications
study_info['study_id'] = study.id
study_info['study_title'] = study.title
study_info['shared_with'] = [s.id for s in study.shared_with]
study_info['status'] = study.status
study_info['ebi_study_accession'] = study.ebi_study_accession
study_info['ebi_submission_status'] = study.ebi_submission_status
# Clean up StudyPerson objects to string for display
pi = study_info['principal_investigator']
study_info['principal_investigator'] = {
'name': pi.name,
'email': pi.email,
'affiliation': pi.affiliation}
lab_person = study_info['lab_person']
if lab_person:
study_info['lab_person'] = {
'name': lab_person.name,
'email': lab_person.email,
'affiliation': lab_person.affiliation}
samples = study.sample_template
study_info['num_samples'] = 0 if samples is None else len(list(samples))
study_info['owner'] = study.owner.id
return {'status': 'success',
'message': '',
'study_info': study_info,
'editable': study.can_edit(User(user_id))}
def test_build_study_info_new_study(self):
ProcessedData(1).status = 'public'
info = {
'timeseries_type_id': 1,
'portal_type_id': 1,
'lab_person_id': None,
'principal_investigator_id': 3,
'metadata_complete': False,
'mixs_compliant': True,
'study_description': 'desc',
'study_alias': 'alias',
'study_abstract': 'abstract'}
user = User('*****@*****.**')
Study.create(user, 'test_study_1', efo=[1], info=info)
obs = _build_study_info(user)
self.exp.append({
'status': 'sandbox',
'checkbox': "<input type='checkbox' value='2' />",
'abstract': 'abstract',
'meta_complete': "<span class='glyphicon glyphicon-remove'>"
"</span>",
'title': '<a href=\'#\' data-toggle=\'modal\' data-target=\'#study'
'-abstract-modal\' onclick=\'fillAbstract("studies-table"'
', 1)\'><span class=\'glyphicon glyphicon-file\' aria-hidden=\''
'true\'></span></a> | <a href=\'/study/description/2\' id=\''
'study1-title\'>test_study_1</a>',
'num_raw_data': 0, 'id': 2, 'num_samples': '0',
'shared': "<span id='shared_html_2'></span><br/><a class='btn "
"btn-primary btn-xs' data-toggle='modal' data-target='#share-study"
"-modal-view' onclick='modify_sharing(2);'>Modify</a>",
'pmid': '', 'pi':
'<a target="_blank" href="mailto:[email protected]">PIDude</a>'})
self.assertEqual(obs, self.exp)
def check_access(study_id, user_id):
"""Checks if user given has access to the study given
Parameters
----------
study_id : int
ID of the study to check access to
user_id : str
ID of the user to check access for
Returns
-------
dict
Empty dict if access allowed, else a dict in the form
{'status': 'error',
'message': reason for error}
"""
try:
study = Study(int(study_id))
except QiitaDBUnknownIDError:
return {'status': 'error', 'message': 'Study does not exist'}
if not study.has_access(User(user_id)):
return {
'status': 'error',
'message': 'User does not have access to study'
}
return {}
def get(self, arguments):
study_id = int(self.get_argument('study_id'))
# this block is tricky because you can either pass the sample or the
# prep template and if none is passed then we will let an exception
# be raised because template will not be declared for the logic below
if self.get_argument('prep_template', None):
template = PrepTemplate(int(self.get_argument('prep_template')))
if self.get_argument('sample_template', None):
template = None
tid = int(self.get_argument('sample_template'))
try:
template = SampleTemplate(tid)
except QiitaDBUnknownIDError:
raise HTTPError(404, "SampleTemplate %d does not exist" % tid)
study = Study(template.study_id)
# check whether or not the user has access to the requested information
if not study.has_access(User(self.current_user)):
raise HTTPError(403, "You do not have access to access this "
"information.")
df = dataframe_from_template(template)
stats = stats_from_df(df)
self.render('metadata_summary.html', user=self.current_user,
study_title=study.title, stats=stats,
study_id=study_id)
def test_delete_study_empty_study(self):
info = {
"timeseries_type_id": '1',
"metadata_complete": 'true',
"mixs_compliant": 'true',
"number_samples_collected": 25,
"number_samples_promised": 28,
"study_alias": "FCM",
"study_description": "Microbiome of people who eat nothing but "
"fried chicken",
"study_abstract": "Exploring how a high fat diet changes the "
"gut microbiome",
"emp_person_id": StudyPerson(2),
"principal_investigator_id": StudyPerson(3),
"lab_person_id": StudyPerson(1)
}
new_study = Study.create(User('*****@*****.**'),
"Fried Chicken Microbiome %s" % time(), info)
job = self._create_job('delete_study', {'study': new_study.id})
private_task(job.id)
self.assertEqual(job.status, 'success')
# making sure the study doesn't exist
with self.assertRaises(QiitaDBUnknownIDError):
Study(new_study.id)
def _build_study_info(studytype, user=None):
"""builds list of namedtuples for study listings"""
if studytype == "private":
studylist = user.user_studies
elif studytype == "shared":
studylist = user.shared_studies
elif studytype == "public":
studylist = Study.get_by_status('public')
else:
raise IncompetentQiitaDeveloperError("Must use private, shared, "
"or public!")
StudyTuple = namedtuple(
'StudyInfo', 'id title meta_complete '
'num_samples_collected shared num_raw_data pi '
'pmids owner status')
infolist = []
for s_id in studylist:
study = Study(s_id)
status = study.status
# Just passing the email address as the name here, since
# name is not a required field in qiita.qiita_user
owner = study_person_linkifier((study.owner, study.owner))
info = study.info
PI = StudyPerson(info['principal_investigator_id'])
PI = study_person_linkifier((PI.email, PI.name))
pmids = ", ".join([pubmed_linkifier([pmid]) for pmid in study.pmids])
shared = _get_shared_links_for_study(study)
infolist.append(
StudyTuple(study.id, study.title, info["metadata_complete"],
info["number_samples_collected"], shared,
len(study.raw_data()), PI, pmids, owner, status))
return infolist
def delete_study(self, study, user, callback):
"""Delete study
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done and it fails
"""
study_id = study.id
study_title = study.title
try:
Study.delete(study_id)
# redirecting to list but also passing messages
# we need to change the request.method to GET
self.request.method = 'GET'
ListStudiesHandler(self.application, self.request)._execute(
[t(self.request) for t in self.application.transforms],
message=('Study "%s" has been deleted' % study_title),
msg_level='success')
except Exception as e:
msg = "Couldn't remove study %d: %s" % (study_id, str(e))
msg_level = "danger"
callback((msg, msg_level, 'study_information_tab', None, None))
def test_artifact_post_req(self):
# Create new prep template to attach artifact to
pt = npt.assert_warns(
QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {'1.SKD6.640190': 1}}), Study(1), '16S')
self._files_to_remove.extend([fp for _, fp in pt.get_filepaths()])
filepaths = {'raw_forward_seqs': 'uploaded_file.txt',
'raw_barcodes': 'update.txt'}
obs = artifact_post_req(
'*****@*****.**', filepaths, 'FASTQ', 'New Test Artifact', pt.id)
exp = {'status': 'success',
'message': ''}
self.assertEqual(obs, exp)
wait_for_prep_information_job(pt.id)
# Test importing an artifact
# Create new prep template to attach artifact to
pt = npt.assert_warns(
QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {'1.SKD6.640190': 1}}), Study(1), '16S')
self._files_to_remove.extend([fp for _, fp in pt.get_filepaths()])
obs = artifact_post_req(
'*****@*****.**', {}, 'Demultiplexed', 'New Test Artifact 2',
pt.id, 3)
exp = {'status': 'success',
'message': ''}
self.assertEqual(obs, exp)
wait_for_prep_information_job(pt.id)
# Instantiate the artifact to make sure it was made and
# to clean the environment
a = Artifact(pt.artifact.id)
self._files_to_remove.extend([fp for _, fp, _ in a.filepaths])
def test_build_study_info_new_study(self):
info = {
'timeseries_type_id': 1,
'portal_type_id': 1,
'lab_person_id': None,
'principal_investigator_id': 3,
'metadata_complete': False,
'mixs_compliant': True,
'study_description': 'desc',
'study_alias': 'alias',
'study_abstract': 'abstract'}
user = User('*****@*****.**')
Study.create(user, 'test_study_1', efo=[1], info=info)
obs = _build_study_info('private', user)
StudyTuple = namedtuple('StudyInfo', 'id title meta_complete '
'num_samples_collected shared num_raw_data pi '
'pmids owner status')
exp = [
StudyTuple(
id=2,
title='test_study_1',
meta_complete=False, num_samples_collected=None,
shared='',
num_raw_data=0,
pi='<a target="_blank" href="mailto:[email protected]">'
'PIDude</a>',
pmids='',
owner='<a target="_blank" href="mailto:[email protected]">'
'[email protected]</a>',
status='sandbox')]
self.assertEqual(obs, exp)
def setUp(self):
fd, self.seqs_fp = mkstemp(suffix='_seqs.fastq')
close(fd)
fd, self.barcodes_fp = mkstemp(suffix='_barcodes.fastq')
close(fd)
self.filetype = 2
self.filepaths = [(self.seqs_fp, 1), (self.barcodes_fp, 2)]
self.studies = [Study(1)]
_, self.db_test_raw_dir = get_mountpoint('raw_data')[0]
with open(self.seqs_fp, "w") as f:
f.write("\n")
with open(self.barcodes_fp, "w") as f:
f.write("\n")
self._clean_up_files = []
# Create a new study
info = {
"timeseries_type_id": 1,
"metadata_complete": True,
"mixs_compliant": True,
"number_samples_collected": 25,
"number_samples_promised": 28,
"portal_type_id": 3,
"study_alias": "FCM",
"study_description": "Microbiome of people who eat nothing but "
"fried chicken",
"study_abstract": "Exploring how a high fat diet changes the "
"gut microbiome",
"emp_person_id": StudyPerson(2),
"principal_investigator_id": StudyPerson(3),
"lab_person_id": StudyPerson(1)
}
Study.create(User("*****@*****.**"), "Test study 2", [1], info)
def test_build_study_info_empty_study(self):
info = {
'timeseries_type_id': 1,
'lab_person_id': None,
'principal_investigator_id': 3,
'metadata_complete': False,
'mixs_compliant': True,
'study_description': 'desc',
'study_alias': 'alias',
'study_abstract': 'abstract'}
Study.create(User('*****@*****.**'), "My study", efo=[1], info=info)
obs = _build_study_info(User('*****@*****.**'), 'user')
self.exp.append({
'metadata_complete': False,
'ebi_submission_status':
'not submitted',
'shared': [],
'publication_pid': [],
'pi': ('*****@*****.**', 'PIDude'),
'status': 'sandbox',
'proc_data_info': [],
'publication_doi': [],
'study_abstract': 'abstract',
'study_id': 2,
'ebi_study_accession': None,
'study_title': 'My study',
'number_samples_collected': 0})
self.assertItemsEqual(obs, self.exp)
# Now testing that admin also sees this study
obs = _build_study_info(User('*****@*****.**'), 'user')
self.assertEqual(obs, self.exp)
def sample_template_checks(study_id, user, check_exists=False):
"""Performs different checks and raises errors if any of the checks fail
Parameters
----------
study_id : int
The study id
user : qiita_db.user.User
The user trying to access the study
check_exists : bool, optional
If true, check if the sample template exists
Raises
------
HTTPError
404 if the study does not exist
403 if the user does not have access to the study
404 if check_exists == True and the sample template doesn't exist
"""
try:
study = Study(int(study_id))
except QiitaDBUnknownIDError:
raise HTTPError(404, reason='Study does not exist')
if not study.has_access(user):
raise HTTPError(403, reason='User does not have access to study')
# Check if the sample template exists
if check_exists and not SampleTemplate.exists(study_id):
raise HTTPError(404, reason="Study %s doesn't have sample information"
% study_id)
def test_get_accessible_filepath_ids(self):
self._set_processed_data_private()
# shared has access to all study files and analysis files
obs = get_accessible_filepath_ids(User('*****@*****.**'))
self.assertEqual(obs, set([1, 2, 5, 6, 7, 11, 12, 13, 14, 15, 16, 17,
18, 19, 20]))
# Now shared should not have access to the study files
self._unshare_studies()
obs = get_accessible_filepath_ids(User('*****@*****.**'))
self.assertEqual(obs, set([12, 13, 14, 15]))
# Now shared should not have access to any files
self._unshare_analyses()
obs = get_accessible_filepath_ids(User('*****@*****.**'))
self.assertEqual(obs, set())
# Now shared has access to public study files
self._set_processed_data_public()
obs = get_accessible_filepath_ids(User('*****@*****.**'))
self.assertEqual(obs, set([1, 2, 5, 6, 7, 11, 16, 19, 20]))
# Test that it doesn't break: if the SampleTemplate hasn't been added
exp = set([1, 2, 5, 6, 7, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20])
obs = get_accessible_filepath_ids(User('*****@*****.**'))
self.assertEqual(obs, exp)
info = {
"timeseries_type_id": 1,
"metadata_complete": True,
"mixs_compliant": True,
"number_samples_collected": 4,
"number_samples_promised": 4,
"portal_type_id": 3,
"study_alias": "TestStudy",
"study_description": "Description of a test study",
"study_abstract": "No abstract right now...",
"emp_person_id": 1,
"principal_investigator_id": 1,
"lab_person_id": 1
}
Study.create(User('*****@*****.**'), "Test study", [1], info)
obs = get_accessible_filepath_ids(User('*****@*****.**'))
self.assertEqual(obs, exp)
# test in case there is a prep template that failed
self.conn_handler.execute(
"INSERT INTO qiita.prep_template (data_type_id, raw_data_id) "
"VALUES (2,1)")
obs = get_accessible_filepath_ids(User('*****@*****.**'))
self.assertEqual(obs, exp)
# admin should have access to everything
count = self.conn_handler.execute_fetchone("SELECT count(*) FROM "
"qiita.filepath")[0]
exp = set(range(1, count + 1))
obs = get_accessible_filepath_ids(User('*****@*****.**'))
self.assertEqual(obs, exp)
def _build_study_info(studytype, user=None):
"""builds list of namedtuples for study listings"""
if studytype == "private":
studylist = user.user_studies
elif studytype == "shared":
studylist = user.shared_studies
elif studytype == "public":
studylist = Study.get_by_status('public')
else:
raise IncompetentQiitaDeveloperError("Must use private, shared, "
"or public!")
StudyTuple = namedtuple('StudyInfo', 'id title meta_complete '
'num_samples_collected shared num_raw_data pi '
'pmids owner status')
infolist = []
for s_id in studylist:
study = Study(s_id)
status = study.status
# Just passing the email address as the name here, since
# name is not a required field in qiita.qiita_user
owner = study_person_linkifier((study.owner, study.owner))
info = study.info
PI = StudyPerson(info['principal_investigator_id'])
PI = study_person_linkifier((PI.email, PI.name))
pmids = ", ".join([pubmed_linkifier([pmid])
for pmid in study.pmids])
shared = _get_shared_links_for_study(study)
infolist.append(StudyTuple(study.id, study.title,
info["metadata_complete"],
info["number_samples_collected"],
shared, len(study.raw_data()),
PI, pmids, owner, status))
return infolist
def test_build_study_info_empty_study(self):
info = {
'timeseries_type_id': 1,
'lab_person_id': None,
'principal_investigator_id': 3,
'metadata_complete': False,
'mixs_compliant': True,
'study_description': 'desc',
'study_alias': 'alias',
'study_abstract': 'abstract'
}
Study.create(User('*****@*****.**'), "My study", efo=[1], info=info)
obs = _build_study_info(User('*****@*****.**'), 'user')
self.exp.append({
'metadata_complete': False,
'ebi_submission_status': 'not submitted',
'shared': [],
'pmid': [],
'pi': ('*****@*****.**', 'PIDude'),
'status': 'private',
'proc_data_info': [],
'publication_doi': [],
'study_abstract': 'abstract',
'study_id': 2,
'ebi_study_accession': None,
'study_title': 'My study',
'number_samples_collected': 0
})
self.assertItemsEqual(obs, self.exp)
def study_delete_req(study_id, user_id):
"""Delete a given study
Parameters
----------
study_id : int
Study id to delete
user_id : str
User requesting the deletion
Returns
-------
dict
Status of deletion, in the format
{status: status,
message: message}
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
status = 'success'
try:
Study.delete(int(study_id))
msg = ''
except Exception as e:
status = 'error'
msg = 'Unable to delete study: %s' % str(e)
return {'status': status, 'message': msg}
def study_delete_req(study_id, user_id):
"""Delete a given study
Parameters
----------
study_id : int
Study id to delete
user_id : str
User requesting the deletion
Returns
-------
dict
Status of deletion, in the format
{status: status,
message: message}
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
status = 'success'
try:
Study.delete(int(study_id))
msg = ''
except Exception as e:
status = 'error'
msg = 'Unable to delete study: %s' % str(e)
return {
'status': status,
'message': msg
}
def test_remove_portal(self):
Portal.create("NEWPORTAL", "SOMEDESC")
# Select some samples on a default analysis
qiita_config.portal = "NEWPORTAL"
a = Analysis(User("*****@*****.**").default_analysis)
a.add_samples({1: ['1.SKB8.640193', '1.SKD5.640186']})
Portal.delete("NEWPORTAL")
obs = self.conn_handler.execute_fetchall(
"SELECT * FROM qiita.portal_type")
exp = [[1, 'QIITA', 'QIITA portal. Access to all data stored '
'in database.'],
[2, 'EMP', 'EMP portal']]
self.assertItemsEqual(obs, exp)
obs = self.conn_handler.execute_fetchall(
"SELECT * FROM qiita.analysis_portal")
exp = [[1, 1], [2, 1], [3, 1], [4, 1], [5, 1], [6, 1], [7, 2], [8, 2],
[9, 2], [10, 2]]
self.assertItemsEqual(obs, exp)
with self.assertRaises(QiitaDBLookupError):
Portal.delete("NOEXISTPORTAL")
with self.assertRaises(QiitaDBError):
Portal.delete("QIITA")
Portal.create("NEWPORTAL2", "SOMEDESC")
# Add analysis to this new portal and make sure error raised
qiita_config.portal = "NEWPORTAL2"
Analysis.create(User("*****@*****.**"), "newportal analysis", "desc")
qiita_config.portal = "QIITA"
with self.assertRaises(QiitaDBError):
Portal.delete("NEWPORTAL2")
# Add study to this new portal and make sure error raised
info = {
"timeseries_type_id": 1,
"metadata_complete": True,
"mixs_compliant": True,
"number_samples_collected": 25,
"number_samples_promised": 28,
"study_alias": "FCM",
"study_description": "Microbiome of people who eat nothing but "
"fried chicken",
"study_abstract": "Exploring how a high fat diet changes the "
"gut microbiome",
"emp_person_id": StudyPerson(2),
"principal_investigator_id": StudyPerson(3),
"lab_person_id": StudyPerson(1)
}
Portal.create("NEWPORTAL3", "SOMEDESC")
qiita_config.portal = "NEWPORTAL3"
Study.create(User('*****@*****.**'), "Fried chicken microbiome",
[1], info)
qiita_config.portal = "QIITA"
with self.assertRaises(QiitaDBError):
Portal.delete("NEWPORTAL3")
def _build_study_info(user, search_type, study_proc=None, proc_samples=None):
"""Builds list of dicts for studies table, with all HTML formatted
Parameters
----------
user : User object
logged in user
search_type : choice, ['user', 'public']
what kind of search to perform
study_proc : dict of lists, optional
Dictionary keyed on study_id that lists all processed data associated
with that study. Required if proc_samples given.
proc_samples : dict of lists, optional
Dictionary keyed on proc_data_id that lists all samples associated with
that processed data. Required if study_proc given.
Returns
-------
infolist: list of dict of lists and dicts
study and processed data info for JSON serialiation for datatables
Each dict in the list is a single study, and contains the text
Notes
-----
Both study_proc and proc_samples must be passed, or neither passed.
"""
build_samples = False
# Logic check to make sure both needed parts passed
if study_proc is not None and proc_samples is None:
raise IncompetentQiitaDeveloperError(
'Must pass proc_samples when study_proc given')
elif proc_samples is not None and study_proc is None:
raise IncompetentQiitaDeveloperError(
'Must pass study_proc when proc_samples given')
elif study_proc is None:
build_samples = True
# get list of studies for table
if search_type == 'user':
user_study_set = user.user_studies.union(user.shared_studies)
if user.level == 'admin':
user_study_set = (user_study_set |
Study.get_by_status('sandbox') |
Study.get_by_status('private'))
study_set = user_study_set - Study.get_by_status('public')
elif search_type == 'public':
study_set = Study.get_by_status('public')
else:
raise ValueError('Not a valid search type')
if study_proc is not None:
study_set = study_set.intersection(study_proc)
if not study_set:
# No studies left so no need to continue
return []
return generate_study_list([s.id for s in study_set], build_samples,
public_only=(search_type == 'public'))
def _build_study_info(user, search_type, study_proc=None, proc_samples=None):
"""Builds list of dicts for studies table, with all HTML formatted
Parameters
----------
user : User object
logged in user
search_type : choice, ['user', 'public']
what kind of search to perform
study_proc : dict of lists, optional
Dictionary keyed on study_id that lists all processed data associated
with that study. Required if proc_samples given.
proc_samples : dict of lists, optional
Dictionary keyed on proc_data_id that lists all samples associated with
that processed data. Required if study_proc given.
Returns
-------
infolist: list of dict of lists and dicts
study and processed data info for JSON serialiation for datatables
Each dict in the list is a single study, and contains the text
Notes
-----
Both study_proc and proc_samples must be passed, or neither passed.
"""
build_samples = False
# Logic check to make sure both needed parts passed
if study_proc is not None and proc_samples is None:
raise IncompetentQiitaDeveloperError(
'Must pass proc_samples when study_proc given')
elif proc_samples is not None and study_proc is None:
raise IncompetentQiitaDeveloperError(
'Must pass study_proc when proc_samples given')
elif study_proc is None:
build_samples = True
# get list of studies for table
if search_type == 'user':
user_study_set = user.user_studies.union(user.shared_studies)
if user.level == 'admin':
user_study_set = (user_study_set |
Study.get_by_status('sandbox') |
Study.get_by_status('private'))
study_set = user_study_set - Study.get_by_status('public')
elif search_type == 'public':
study_set = Study.get_by_status('public')
else:
raise ValueError('Not a valid search type')
if study_proc is not None:
study_set = study_set.intersection(study_proc)
if not study_set:
# No studies left so no need to continue
return []
return generate_study_list([s.id for s in study_set], build_samples,
public_only=(search_type == 'public'))
def get(self):
data = self.get_argument("data", None)
study_id = self.get_argument("study_id", None)
data_type = self.get_argument("data_type", None)
dtypes = get_data_types().keys()
if data is None or study_id is None or data not in ('raw', 'biom'):
raise HTTPError(422,
reason='You need to specify both data (the '
'data type you want to download - raw/biom) and '
'study_id')
elif data_type is not None and data_type not in dtypes:
raise HTTPError(422,
reason='Not a valid data_type. Valid types '
'are: %s' % ', '.join(dtypes))
else:
study_id = int(study_id)
try:
study = Study(study_id)
except QiitaDBUnknownIDError:
raise HTTPError(422, reason='Study does not exist')
else:
public_raw_download = study.public_raw_download
if study.status != 'public':
raise HTTPError(422,
reason='Study is not public. If this '
'is a mistake contact: '
'*****@*****.**')
elif data == 'raw' and not public_raw_download:
raise HTTPError(422,
reason='No raw data access. If this '
'is a mistake contact: '
'*****@*****.**')
else:
to_download = []
for a in study.artifacts(
dtype=data_type,
artifact_type='BIOM' if data == 'biom' else None):
if a.visibility != 'public':
continue
to_download.extend(self._list_artifact_files_nginx(a))
if not to_download:
raise HTTPError(422,
reason='Nothing to download. If '
'this is a mistake contact: '
'*****@*****.**')
else:
self._write_nginx_file_list(to_download)
zip_fn = 'study_%d_%s_%s.zip' % (
study_id, data,
datetime.now().strftime('%m%d%y-%H%M%S'))
self._set_nginx_headers(zip_fn)
self.finish()
def get(self):
self.write(self.render_string('waiting.html'))
self.flush()
u = User(self.current_user)
user_studies = [Study(s_id) for s_id in u.private_studies]
share_dict = {s.id: s.shared_with for s in user_studies}
shared_studies = [Study(s_id) for s_id in u.shared_studies]
self.render('private_studies.html', user=self.current_user,
user_studies=user_studies, shared_studies=shared_studies,
share_dict=share_dict)
def test_get_edit_utf8(self):
"""Make sure the page loads when utf8 characters are present"""
study = Study(1)
study.title = "TEST_ø"
study.alias = "TEST_ø"
study.description = "TEST_ø"
study.abstract = "TEST_ø"
response = self.get('/study/edit/1')
self.assertEqual(response.code, 200)
self.assertNotEqual(str(response.body), "")
def test_get_accessible_filepath_ids(self):
self._set_studies_private()
# shared has access to all study files and analysis files
obs = get_accessible_filepath_ids(User('*****@*****.**'))
self.assertEqual(obs,
set([1, 2, 5, 6, 7, 11, 12, 13, 14, 15, 16, 17, 18]))
# Now shared should not have access to the study files
self._unshare_studies()
obs = get_accessible_filepath_ids(User('*****@*****.**'))
self.assertEqual(obs, set([12, 13, 14, 15]))
# Now shared should not have access to any files
self._unshare_analyses()
obs = get_accessible_filepath_ids(User('*****@*****.**'))
self.assertEqual(obs, set())
# Test that it doesn't break: if the SampleTemplate hasn't been added
exp = set([1, 2, 5, 6, 7, 11, 12, 13, 14, 15, 16, 17, 18])
obs = get_accessible_filepath_ids(User('*****@*****.**'))
self.assertEqual(obs, exp)
info = {
"timeseries_type_id": 1,
"metadata_complete": True,
"mixs_compliant": True,
"number_samples_collected": 4,
"number_samples_promised": 4,
"portal_type_id": 3,
"study_alias": "TestStudy",
"study_description": "Description of a test study",
"study_abstract": "No abstract right now...",
"emp_person_id": 1,
"principal_investigator_id": 1,
"lab_person_id": 1
}
Study.create(User('*****@*****.**'), "Test study", [1], info)
obs = get_accessible_filepath_ids(User('*****@*****.**'))
self.assertEqual(obs, exp)
# test in case there is a prep template that failed
self.conn_handler.execute(
"INSERT INTO qiita.prep_template (data_type_id, raw_data_id) "
"VALUES (2,1)")
obs = get_accessible_filepath_ids(User('*****@*****.**'))
self.assertEqual(obs, exp)
# admin should have access to everything
count = self.conn_handler.execute_fetchone("SELECT count(*) FROM "
"qiita.filepath")[0]
exp = set(range(1, count + 1))
obs = get_accessible_filepath_ids(User('*****@*****.**'))
self.assertEqual(obs, exp)
def test_download_raw_data(self):
# it's possible that one of the tests is deleting the raw data
# so we will make sure that the files exists so this test passes
study = Study(1)
all_files = [x['fp'] for a in study.artifacts()
for x in a.filepaths]
for fp in all_files:
if not exists(fp):
with open(fp, 'w') as f:
f.write('')
response = self.get('/download_raw_data/1')
self.assertEqual(response.code, 200)
exp = (
'2125826711 58 /protected/raw_data/1_s_G1_L001_sequences.fastq.gz '
'raw_data/1_s_G1_L001_sequences.fastq.gz\n'
'2125826711 58 /protected/raw_data/'
'1_s_G1_L001_sequences_barcodes.fastq.gz '
'raw_data/1_s_G1_L001_sequences_barcodes.fastq.gz\n'
'- [0-9]* /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]*.txt '
'mapping_files/1_mapping_file.txt\n'
'1756512010 1093210 /protected/BIOM/7/biom_table.biom '
'BIOM/7/biom_table.biom\n'
'- [0-9]* /protected/templates/1_prep_2_qiime_[0-9]*-[0-9]*.txt '
'mapping_files/7_mapping_file.txt\n')
self.assertRegex(response.body.decode('ascii'), exp)
response = self.get('/download_study_bioms/200')
self.assertEqual(response.code, 405)
# changing user so we can test the failures
BaseHandler.get_current_user = Mock(
return_value=User("*****@*****.**"))
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 405)
# now, let's make sure that when artifacts are public AND the
# public_raw_download any user can download the files
study.public_raw_download = True
BaseHandler.get_current_user = Mock(
return_value=User("*****@*****.**"))
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 405)
# 7 is an uploaded biom, which should now be available but as it's a
# biom, only the prep info file will be retrieved
Artifact(7).visibility = 'public'
BaseHandler.get_current_user = Mock(
return_value=User("*****@*****.**"))
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 200)
exp = (
'- [0-9]* /protected/templates/1_prep_2_qiime_[0-9]*-[0-9]*.txt '
'mapping_files/7_mapping_file.txt\n')
self.assertRegex(response.body.decode('ascii'), exp)
def test_delete(self):
title = "Fried chicken microbiome"
study = Study.create(User('*****@*****.**'), title, [1], self.info)
study.delete(study.id)
self.assertFalse(study.exists(title))
with self.assertRaises(QiitaDBError):
Study.delete(1)
with self.assertRaises(QiitaDBUnknownIDError):
Study.delete(41)
def test_download_raw_data(self):
# it's possible that one of the tests is deleting the raw data
# so we will make sure that the files exists so this test passes
study = Study(1)
all_files = [x['fp'] for a in study.artifacts()
for x in a.filepaths]
for fp in all_files:
if not exists(fp):
with open(fp, 'w') as f:
f.write('')
response = self.get('/download_raw_data/1')
self.assertEqual(response.code, 200)
exp = (
'2125826711 58 /protected/raw_data/1_s_G1_L001_sequences.fastq.gz '
'raw_data/1_s_G1_L001_sequences.fastq.gz\n'
'2125826711 58 /protected/raw_data/'
'1_s_G1_L001_sequences_barcodes.fastq.gz '
'raw_data/1_s_G1_L001_sequences_barcodes.fastq.gz\n'
'- [0-9]* /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]*.txt '
'mapping_files/1_mapping_file.txt\n'
'1756512010 1093210 /protected/BIOM/7/biom_table.biom '
'BIOM/7/biom_table.biom\n'
'- [0-9]* /protected/templates/1_prep_2_qiime_[0-9]*-[0-9]*.txt '
'mapping_files/7_mapping_file.txt\n')
self.assertRegex(response.body.decode('ascii'), exp)
response = self.get('/download_study_bioms/200')
self.assertEqual(response.code, 405)
# changing user so we can test the failures
BaseHandler.get_current_user = Mock(
return_value=User("*****@*****.**"))
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 405)
# now, let's make sure that when artifacts are public AND the
# public_raw_download any user can download the files
study.public_raw_download = True
BaseHandler.get_current_user = Mock(
return_value=User("*****@*****.**"))
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 405)
# 7 is an uploaded biom, which should now be available but as it's a
# biom, only the prep info file will be retrieved
Artifact(7).visibility = 'public'
BaseHandler.get_current_user = Mock(
return_value=User("*****@*****.**"))
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 200)
exp = (
'- [0-9]* /protected/templates/1_prep_2_qiime_[0-9]*-[0-9]*.txt '
'mapping_files/7_mapping_file.txt\n')
self.assertRegex(response.body.decode('ascii'), exp)
def test_artifact_post_req(self):
# Create new prep template to attach artifact to
pt = npt.assert_warns(QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {
'1.SKD6.640190': 1
}}), Study(1), '16S')
self._files_to_remove.extend([fp for _, fp in pt.get_filepaths()])
new_artifact_id = get_count('qiita.artifact') + 1
filepaths = {
'raw_forward_seqs': 'uploaded_file.txt',
'raw_barcodes': 'update.txt'
}
obs = artifact_post_req('*****@*****.**', filepaths, 'FASTQ',
'New Test Artifact', pt.id)
exp = {'status': 'success', 'message': ''}
self.assertEqual(obs, exp)
obs = r_client.get('prep_template_%d' % pt.id)
self.assertIsNotNone(obs)
redis_info = loads(r_client.get(loads(obs)['job_id']))
while redis_info['status_msg'] == 'Running':
sleep(0.05)
redis_info = loads(r_client.get(loads(obs)['job_id']))
# Instantiate the artifact to make sure it was made and
# to clean the environment
a = Artifact(new_artifact_id)
self._files_to_remove.extend([fp for _, fp, _ in a.filepaths])
# Test importing an artifact
# Create new prep template to attach artifact to
pt = npt.assert_warns(QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {
'1.SKD6.640190': 1
}}), Study(1), '16S')
self._files_to_remove.extend([fp for _, fp in pt.get_filepaths()])
new_artifact_id_2 = get_count('qiita.artifact') + 1
obs = artifact_post_req('*****@*****.**', {}, 'FASTQ',
'New Test Artifact 2', pt.id, new_artifact_id)
exp = {'status': 'success', 'message': ''}
self.assertEqual(obs, exp)
obs = r_client.get('prep_template_%d' % pt.id)
self.assertIsNotNone(obs)
redis_info = loads(r_client.get(loads(obs)['job_id']))
while redis_info['status_msg'] == 'Running':
sleep(0.05)
redis_info = loads(r_client.get(loads(obs)['job_id']))
# Instantiate the artifact to make sure it was made and
# to clean the environment
a = Artifact(new_artifact_id_2)
self._files_to_remove.extend([fp for _, fp, _ in a.filepaths])
def test_get_valid_one_arg(self):
df = Study(1).sample_template.to_dataframe()
df = df[['ph', 'country']]
df = {idx: [row['country']] for idx, row in df.iterrows()}
exp = {'header': ['country'], 'samples': df}
response = self.get('/api/v1/study/1/samples/categories=country',
headers=self.headers)
self.assertEqual(response.code, 200)
obs = json_decode(response.body)
self.assertEqual(obs, exp)
def tearDown(self):
for fp in self._clean_up_files:
if exists(fp):
remove(fp)
study_id = self.new_study.id
for pt in self.new_study.prep_templates():
PrepTemplate.delete(pt.id)
if SampleTemplate.exists(study_id):
SampleTemplate.delete(study_id)
Study.delete(study_id)
def tearDown(self):
for fp in self._clean_up_files:
if exists(fp):
remove(fp)
study_id = self.new_study.id
for pt in self.new_study.prep_templates():
PrepTemplate.delete(pt.id)
if SampleTemplate.exists(study_id):
SampleTemplate.delete(study_id)
Study.delete(study_id)
def get_raw_data_from_other_studies(user, study):
"""Retrieves a tuple of raw_data_id and the last study title for that
raw_data
"""
d = {}
for sid in user.user_studies:
if sid == study.id:
continue
for rdid in Study(sid).raw_data():
d[int(rdid)] = Study(RawData(rdid).studies[-1]).title
return d
def test_get_valid_one_arg(self):
df = Study(1).sample_template.to_dataframe()
df = df[['ph', 'country']]
df = {idx: [row['country']] for idx, row in df.iterrows()}
exp = {'header': ['country'], 'samples': df}
response = self.get('/api/v1/study/1/samples/categories=country',
headers=self.headers)
self.assertEqual(response.code, 200)
obs = json_decode(response.body)
self.assertEqual(obs, exp)
def get(self):
user = self.current_user
if user.level != 'admin':
raise HTTPError(403, 'User %s is not admin' % self.current_user)
parsed_studies = []
for sid in Study.get_by_status('awaiting_approval'):
study = Study(sid)
parsed_studies.append((study.id, study.title, study.owner))
self.render('admin_approval.html', study_info=parsed_studies)
def study_prep_get_req(study_id, user_id):
"""Gives a summary of each prep template attached to the study
Parameters
----------
study_id : int
Study id to get prep template info for
user_id : str
User id requesting the prep templates
Returns
-------
dict of list of dict
prep template information seperated by data type, in the form
{data_type: [{prep 1 info dict}, ....], ...}
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
# Can only pass ids over API, so need to instantiate object
study = Study(int(study_id))
prep_info = defaultdict(list)
editable = study.can_edit(User(user_id))
for dtype in study.data_types:
for prep in study.prep_templates(dtype):
if prep.status != 'public' and not editable:
continue
start_artifact = prep.artifact
info = {
'name': 'PREP %d NAME' % prep.id,
'id': prep.id,
'status': prep.status,
}
if start_artifact is not None:
youngest_artifact = prep.artifact.youngest_artifact
info['start_artifact'] = start_artifact.artifact_type
info['start_artifact_id'] = start_artifact.id
info['youngest_artifact'] = '%s - %s' % (
youngest_artifact.name, youngest_artifact.artifact_type)
info['ebi_experiment'] = bool(
[v for _, v in viewitems(prep.ebi_experiment_accessions)
if v is not None])
else:
info['start_artifact'] = None
info['start_artifact_id'] = None
info['youngest_artifact'] = None
info['ebi_experiment'] = False
prep_info[dtype].append(info)
return {'status': 'success',
'message': '',
'info': prep_info}
def _get_accessible_raw_data(user):
"""Retrieves a tuple of raw_data_id and one study title for that
raw_data
"""
d = {}
accessible_studies = user.user_studies.union(user.shared_studies)
for sid in accessible_studies:
study = Study(sid)
study_title = study.title
for rdid in study.raw_data():
d[int(rdid)] = study_title
return d
def study_tags_patch_request(user_id, study_id,
req_op, req_path, req_value=None, req_from=None):
"""Modifies an attribute of the artifact
Parameters
----------
user_id : int
The id of the user performing the patch operation
study_id : int
The id of the study on which we will be performing the patch operation
req_op : str
The operation to perform on the study
req_path : str
The attribute to patch
req_value : str, optional
The value that needs to be modified
req_from : str, optional
The original path of the element
Returns
-------
dict of {str, str}
A dictionary with the following keys:
- status: str, whether if the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
"""
if req_op == 'replace':
req_path = [v for v in req_path.split('/') if v]
if len(req_path) != 1:
return {'status': 'error',
'message': 'Incorrect path parameter'}
attribute = req_path[0]
# Check if the user actually has access to the study
access_error = check_access(study_id, user_id)
if access_error:
return access_error
study = Study(study_id)
if attribute == 'tags':
message = study.update_tags(User(user_id), req_value)
return {'status': 'success',
'message': message}
else:
# We don't understand the attribute so return an error
return {'status': 'error',
'message': 'Attribute "%s" not found. '
'Please, check the path parameter' % attribute}
else:
return {'status': 'error',
'message': 'Operation "%s" not supported. '
'Current supported operations: replace' % req_op}
def study_tags_patch_request(user_id, study_id,
req_op, req_path, req_value=None, req_from=None):
"""Modifies an attribute of the artifact
Parameters
----------
user_id : int
The id of the user performing the patch operation
study_id : int
The id of the study on which we will be performing the patch operation
req_op : str
The operation to perform on the study
req_path : str
The attribute to patch
req_value : str, optional
The value that needs to be modified
req_from : str, optional
The original path of the element
Returns
-------
dict of {str, str}
A dictionary with the following keys:
- status: str, whether if the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
"""
if req_op == 'replace':
req_path = [v for v in req_path.split('/') if v]
if len(req_path) != 1:
return {'status': 'error',
'message': 'Incorrect path parameter'}
attribute = req_path[0]
# Check if the user actually has access to the study
access_error = check_access(study_id, user_id)
if access_error:
return access_error
study = Study(study_id)
if attribute == 'tags':
message = study.update_tags(User(user_id), req_value)
return {'status': 'success',
'message': message}
else:
# We don't understand the attribute so return an error
return {'status': 'error',
'message': 'Attribute "%s" not found. '
'Please, check the path parameter' % attribute}
else:
return {'status': 'error',
'message': 'Operation "%s" not supported. '
'Current supported operations: replace' % req_op}
def filter_by_processed_data(self, datatypes=None):
"""Filters results to what is available in each processed data
Parameters
----------
datatypes : list of str, optional
Datatypes to selectively return. Default all datatypes available
Returns
-------
study_proc_ids : dict of dicts of lists
Processed data ids with samples for each study, in the format
{study_id: {datatype: [proc_id, proc_id, ...], ...}, ...}
proc_data_samples : dict of lists
Samples available in each processed data id, in the format
{proc_data_id: [samp_id1, samp_id2, ...], ...}
samples_meta : dict of pandas DataFrames
metadata for the found samples, keyed by study. Pandas indexed on
sample_id, column headers are the metadata categories searched
over
"""
with TRN:
if datatypes is not None:
# convert to set for easy lookups
datatypes = set(datatypes)
study_proc_ids = {}
proc_data_samples = {}
samples_meta = {}
headers = {c: val for c, val in enumerate(self.meta_headers)}
for study_id, study_meta in viewitems(self.results):
# add metadata to dataframe and dict
# use from_dict because pandas doesn't like cursor objects
samples_meta[study_id] = pd.DataFrame.from_dict(
{s[0]: s[1:] for s in study_meta}, orient='index')
samples_meta[study_id].rename(columns=headers, inplace=True)
# set up study-based data needed
study = Study(study_id)
study_sample_ids = {s[0] for s in study_meta}
study_proc_ids[study_id] = defaultdict(list)
for proc_data_id in study.processed_data():
proc_data = ProcessedData(proc_data_id)
datatype = proc_data.data_type()
# skip processed data if it doesn't fit the given datatypes
if datatypes is not None and datatype not in datatypes:
continue
filter_samps = proc_data.samples.intersection(
study_sample_ids)
if filter_samps:
proc_data_samples[proc_data_id] = sorted(filter_samps)
study_proc_ids[study_id][datatype].append(proc_data_id)
return study_proc_ids, proc_data_samples, samples_meta
def test_delete(self):
title = "Fried chicken microbiome"
# the study is assigned to investigation 1
study = Study.create(User("*****@*****.**"), title, [1], self.info, Investigation(1))
# sharing with other user
study.share(User("*****@*****.**"))
study.delete(study.id)
self.assertFalse(study.exists(title))
with self.assertRaises(QiitaDBError):
Study.delete(1)
with self.assertRaises(QiitaDBUnknownIDError):
Study.delete(41)
def get_info(self, portal="QIITA"):
# Add the portals and, optionally, checkbox to the information
studies = [s.id for s in Portal(portal).get_studies()]
if not studies:
return []
study_info = Study.get_info(studies, info_cols=self.study_cols)
info = []
for s in study_info:
# Make sure in correct order
hold = dict(s)
hold['portals'] = ', '.join(sorted(Study(s['study_id'])._portals))
info.append(hold)
return info
def get(self):
"""Show the sample summary page"""
study_id = int(self.get_argument('study_id'))
email = self.current_user.id
res = sample_template_meta_cats_get_req(study_id, email)
if res['status'] == 'error':
if 'does not exist' in res['message']:
raise HTTPError(404, reason=res['message'])
elif 'User does not have access to study' in res['message']:
raise HTTPError(403, reason=res['message'])
else:
raise HTTPError(500, reason=res['message'])
categories = res['categories']
columns, rows = _build_sample_summary(study_id, email)
_, alert_type, alert_msg = get_sample_template_processing_status(
study_id)
self.render('study_ajax/sample_prep_summary.html',
rows=rows,
columns=columns,
categories=categories,
study_id=study_id,
alert_type=alert_type,
alert_message=alert_msg,
user_can_edit=Study(study_id).can_edit(self.current_user))
def get(self):
stats = yield Task(self._get_stats)
# Pull a random public study from the database
public_studies = Study.get_by_status('public')
study = choice(list(public_studies)) if public_studies else None
if study is None:
random_study_info = None
random_study_title = None
random_study_id = None
else:
random_study_info = study.info
random_study_title = study.title
random_study_id = study.id
self.render('stats.html',
number_studies=stats['number_studies'],
number_of_samples=stats['number_of_samples'],
num_users=stats['num_users'],
lat_longs=eval(
stats['lat_longs']) if stats['lat_longs'] else [],
num_studies_ebi=stats['num_studies_ebi'],
num_samples_ebi=stats['num_samples_ebi'],
number_samples_ebi_prep=stats['number_samples_ebi_prep'],
img=stats['img'], time=stats['time'],
random_study_info=random_study_info,
random_study_title=random_study_title,
random_study_id=random_study_id)
def _create_study(self, study_title):
"""Creates a new study
Parameters
----------
study_title: str
The title of the new study
Returns
-------
qiita_db.study.Study
The newly created study
"""
info = {
"timeseries_type_id": 1,
"metadata_complete": True,
"mixs_compliant": True,
"number_samples_collected": 25,
"number_samples_promised": 28,
"study_alias": "ALIAS",
"study_description": "DESC",
"study_abstract": "ABS",
"principal_investigator_id": StudyPerson(3),
"lab_person_id": StudyPerson(1)
}
return Study.create(User('*****@*****.**'), study_title, info)
def setUp(self):
super(NewArtifactHandlerTests, self).setUp()
tmp_dir = join(get_mountpoint('uploads')[0][1], '1')
# Create prep test file to point at
fd, prep_fp = mkstemp(dir=tmp_dir, suffix='.txt')
close(fd)
with open(prep_fp, 'w') as f:
f.write("""sample_name\tnew_col\n1.SKD6.640190\tnew_value\n""")
self.prep = npt.assert_warns(
QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {
'1.SKD6.640190': 1
}}), Study(1), "16S")
fd, self.fwd_fp = mkstemp(dir=tmp_dir, suffix=".fastq")
close(fd)
with open(self.fwd_fp, 'w') as f:
f.write("@seq\nTACGA\n+ABBBB\n")
fd, self.barcodes_fp = mkstemp(dir=tmp_dir, suffix=".fastq")
close(fd)
with open(self.barcodes_fp, 'w') as f:
f.write("@seq\nTACGA\n+ABBBB\n")
self._files_to_remove = [prep_fp, self.fwd_fp, self.barcodes_fp]
def test_patch_no_sample_template(self):
info = {
"timeseries_type_id": 1,
"metadata_complete": True,
"mixs_compliant": True,
"study_alias": "FCM",
"study_description": "DESC",
"study_abstract": "ABS",
"principal_investigator_id": StudyPerson(3),
'first_contact': datetime(2015, 5, 19, 16, 10),
'most_recent_contact': datetime(2015, 5, 19, 16, 11),
}
new_study = Study.create(User('*****@*****.**'),
"Some New Study for test jr", info)
body = {'sampleid1': {'category_a': 'value_a'},
'sampleid2': {'category_b': 'value_b'}}
exp = {'message': 'No sample information found'}
response = self.patch('/api/v1/study/%d/samples' % new_study.id,
headers=self.headers, data=body, asjson=True)
self.assertEqual(response.code, 404)
obs = json_decode(response.body)
self.assertEqual(obs, exp)
def get(self):
"""Show the sample summary page"""
study_id = self.get_argument('study_id')
res = sample_template_meta_cats_get_req(int(study_id),
self.current_user.id)
if res['status'] == 'error':
if 'does not exist' in res['message']:
raise HTTPError(404, res['message'])
elif 'User does not have access to study' in res['message']:
raise HTTPError(403, res['message'])
else:
raise HTTPError(500, res['message'])
meta_cats = res['categories']
cols, samps_table = _build_sample_summary(study_id,
self.current_user.id)
_, alert_type, alert_msg = get_sample_template_processing_status(
study_id)
self.render('study_ajax/sample_prep_summary.html',
table=samps_table,
cols=cols,
meta_available=meta_cats,
study_id=study_id,
alert_type=alert_type,
alert_message=alert_msg,
user_can_edit=Study(study_id).can_edit(self.current_user))
def test_copy_artifact(self):
# Failure test
job = self._create_job('copy_artifact', {
'artifact': 1,
'prep_template': 1
})
private_task(job.id)
self.assertEqual(job.status, 'error')
self.assertIn("Prep template 1 already has an artifact associated",
job.log.msg)
# Success test
metadata_dict = {
'SKB8.640193': {
'center_name': 'ANL',
'primer': 'GTGCCAGCMGCCGCGGTAA',
'barcode': 'GTCCGCAAGTTA',
'run_prefix': "s_G1_L001_sequences",
'platform': 'Illumina',
'instrument_model': 'Illumina MiSeq',
'library_construction_protocol': 'AAAA',
'experiment_design_description': 'BBBB'
}
}
metadata = pd.DataFrame.from_dict(metadata_dict,
orient='index',
dtype=str)
prep = PrepTemplate.create(metadata, Study(1), "16S")
job = self._create_job('copy_artifact', {
'artifact': 1,
'prep_template': prep.id
})
private_task(job.id)
self.assertEqual(job.status, 'success')
def test_status_error(self):
# Let's create a new study, so we can check that the error is raised
# because the new study does not have access to the raw data
info = {
"timeseries_type_id": 1,
"metadata_complete": True,
"mixs_compliant": True,
"number_samples_collected": 25,
"number_samples_promised": 28,
"portal_type_id": 3,
"study_alias": "FCM",
"study_description": "Microbiome of people who eat nothing but "
"fried chicken",
"study_abstract": "Exploring how a high fat diet changes the "
"gut microbiome",
"emp_person_id": StudyPerson(2),
"principal_investigator_id": StudyPerson(3),
"lab_person_id": StudyPerson(1)
}
s = Study.create(User('*****@*****.**'), "Fried chicken microbiome",
[1], info)
rd = RawData(1)
with self.assertRaises(QiitaDBStatusError):
rd.status(s)