def get(self):
data = self.get_argument("data", None)
study_id = self.get_argument("study_id", None)
data_type = self.get_argument("data_type", None)
dtypes = get_data_types().keys()
if data is None or study_id is None or data not in ('raw', 'biom'):
raise HTTPError(422,
reason='You need to specify both data (the '
'data type you want to download - raw/biom) and '
'study_id')
elif data_type is not None and data_type not in dtypes:
raise HTTPError(422,
reason='Not a valid data_type. Valid types '
'are: %s' % ', '.join(dtypes))
else:
study_id = int(study_id)
try:
study = Study(study_id)
except QiitaDBUnknownIDError:
raise HTTPError(422, reason='Study does not exist')
else:
public_raw_download = study.public_raw_download
if study.status != 'public':
raise HTTPError(422,
reason='Study is not public. If this '
'is a mistake contact: '
'*****@*****.**')
elif data == 'raw' and not public_raw_download:
raise HTTPError(422,
reason='No raw data access. If this '
'is a mistake contact: '
'*****@*****.**')
else:
to_download = []
for a in study.artifacts(
dtype=data_type,
artifact_type='BIOM' if data == 'biom' else None):
if a.visibility != 'public':
continue
to_download.extend(self._list_artifact_files_nginx(a))
if not to_download:
raise HTTPError(422,
reason='Nothing to download. If '
'this is a mistake contact: '
'*****@*****.**')
else:
self._write_nginx_file_list(to_download)
zip_fn = 'study_%d_%s_%s.zip' % (
study_id, data,
datetime.now().strftime('%m%d%y-%H%M%S'))
self._set_nginx_headers(zip_fn)
self.finish()
def test_download_raw_data(self):
# it's possible that one of the tests is deleting the raw data
# so we will make sure that the files exists so this test passes
study = Study(1)
all_files = [x['fp'] for a in study.artifacts()
for x in a.filepaths]
for fp in all_files:
if not exists(fp):
with open(fp, 'w') as f:
f.write('')
response = self.get('/download_raw_data/1')
self.assertEqual(response.code, 200)
exp = (
'2125826711 58 /protected/raw_data/1_s_G1_L001_sequences.fastq.gz '
'raw_data/1_s_G1_L001_sequences.fastq.gz\n'
'2125826711 58 /protected/raw_data/'
'1_s_G1_L001_sequences_barcodes.fastq.gz '
'raw_data/1_s_G1_L001_sequences_barcodes.fastq.gz\n'
'- [0-9]* /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]*.txt '
'mapping_files/1_mapping_file.txt\n'
'1756512010 1093210 /protected/BIOM/7/biom_table.biom '
'BIOM/7/biom_table.biom\n'
'- [0-9]* /protected/templates/1_prep_2_qiime_[0-9]*-[0-9]*.txt '
'mapping_files/7_mapping_file.txt\n')
self.assertRegex(response.body.decode('ascii'), exp)
response = self.get('/download_study_bioms/200')
self.assertEqual(response.code, 405)
# changing user so we can test the failures
BaseHandler.get_current_user = Mock(
return_value=User("*****@*****.**"))
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 405)
# now, let's make sure that when artifacts are public AND the
# public_raw_download any user can download the files
study.public_raw_download = True
BaseHandler.get_current_user = Mock(
return_value=User("*****@*****.**"))
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 405)
# 7 is an uploaded biom, which should now be available but as it's a
# biom, only the prep info file will be retrieved
Artifact(7).visibility = 'public'
BaseHandler.get_current_user = Mock(
return_value=User("*****@*****.**"))
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 200)
exp = (
'- [0-9]* /protected/templates/1_prep_2_qiime_[0-9]*-[0-9]*.txt '
'mapping_files/7_mapping_file.txt\n')
self.assertRegex(response.body.decode('ascii'), exp)
def test_download_raw_data(self):
# it's possible that one of the tests is deleting the raw data
# so we will make sure that the files exists so this test passes
study = Study(1)
all_files = [x['fp'] for a in study.artifacts()
for x in a.filepaths]
for fp in all_files:
if not exists(fp):
with open(fp, 'w') as f:
f.write('')
response = self.get('/download_raw_data/1')
self.assertEqual(response.code, 200)
exp = (
'2125826711 58 /protected/raw_data/1_s_G1_L001_sequences.fastq.gz '
'raw_data/1_s_G1_L001_sequences.fastq.gz\n'
'2125826711 58 /protected/raw_data/'
'1_s_G1_L001_sequences_barcodes.fastq.gz '
'raw_data/1_s_G1_L001_sequences_barcodes.fastq.gz\n'
'- [0-9]* /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]*.txt '
'mapping_files/1_mapping_file.txt\n'
'1756512010 1093210 /protected/BIOM/7/biom_table.biom '
'BIOM/7/biom_table.biom\n'
'- [0-9]* /protected/templates/1_prep_2_qiime_[0-9]*-[0-9]*.txt '
'mapping_files/7_mapping_file.txt\n')
self.assertRegex(response.body.decode('ascii'), exp)
response = self.get('/download_study_bioms/200')
self.assertEqual(response.code, 405)
# changing user so we can test the failures
BaseHandler.get_current_user = Mock(
return_value=User("*****@*****.**"))
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 405)
# now, let's make sure that when artifacts are public AND the
# public_raw_download any user can download the files
study.public_raw_download = True
BaseHandler.get_current_user = Mock(
return_value=User("*****@*****.**"))
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 405)
# 7 is an uploaded biom, which should now be available but as it's a
# biom, only the prep info file will be retrieved
Artifact(7).visibility = 'public'
BaseHandler.get_current_user = Mock(
return_value=User("*****@*****.**"))
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 200)
exp = (
'- [0-9]* /protected/templates/1_prep_2_qiime_[0-9]*-[0-9]*.txt '
'mapping_files/7_mapping_file.txt\n')
self.assertRegex(response.body.decode('ascii'), exp)
def study_files_get_req(user_id, study_id, prep_template_id, artifact_type):
"""Returns the uploaded files for the study id categorized by artifact_type
It retrieves the files uploaded for the given study and tries to do a
guess on how those files should be added to the artifact of the given
type. Uses information on the prep template to try to do a better guess.
Parameters
----------
user_id : str
The id of the user making the request
study_id : int
The study id
prep_template_id : int
The prep template id
artifact_type : str
The artifact type
Returns
-------
dict of {str: object}
A dict of the form {'status': str,
'message': str,
'remaining': list of str,
'file_types': list of (str, bool, list of str),
'num_prefixes': int}
where 'status' is a string specifying whether the query is successfull,
'message' is a human-readable description of the error (optional),
'remaining' is the list of files that could not be categorized,
'file_types' is a list of the available filetypes, if it is required
or not and the list of categorized files for the given artifact type
and 'num_prefixes' is the number of different run prefix values in
the given prep template.
"""
supp_file_types = supported_filepath_types(artifact_type)
selected = []
remaining = []
uploaded = get_files_from_uploads_folders(study_id)
pt = PrepTemplate(prep_template_id).to_dataframe()
if (any(ft.startswith('raw_') for ft, _ in supp_file_types)
and 'run_prefix' in pt.columns):
prep_prefixes = tuple(set(pt['run_prefix']))
num_prefixes = len(prep_prefixes)
for _, filename in uploaded:
if filename.startswith(prep_prefixes):
selected.append(filename)
else:
remaining.append(filename)
else:
num_prefixes = 0
remaining = [f for _, f in uploaded]
# At this point we can't do anything smart about selecting by default
# the files for each type. The only thing that we can do is assume that
# the first in the supp_file_types list is the default one where files
# should be added in case of 'run_prefix' being present
file_types = [(fp_type, req, []) for fp_type, req in supp_file_types[1:]]
first = supp_file_types[0]
# Note that this works even if `run_prefix` is not in the prep template
# because selected is initialized to the empty list
file_types.insert(0, (first[0], first[1], selected))
# Create a list of artifacts that the user has access to, in case that
# he wants to import the files from another artifact
user = User(user_id)
artifact_options = []
user_artifacts = user.user_artifacts(artifact_type=artifact_type)
study = Study(study_id)
if study not in user_artifacts:
user_artifacts[study] = study.artifacts(artifact_type=artifact_type)
for study, artifacts in viewitems(user_artifacts):
study_label = "%s (%d)" % (study.title, study.id)
for a in artifacts:
artifact_options.append(
(a.id, "%s - %s (%d)" % (study_label, a.name, a.id)))
return {
'status': 'success',
'message': '',
'remaining': remaining,
'file_types': file_types,
'num_prefixes': num_prefixes,
'artifacts': artifact_options
}
def get(self):
data = self.get_argument("data", None)
study_id = self.get_argument("study_id", None)
prep_id = self.get_argument("prep_id", None)
data_type = self.get_argument("data_type", None)
dtypes = get_data_types().keys()
templates = ['sample_information', 'prep_information']
valid_data = ['raw', 'biom'] + templates
to_download = []
if data is None or (study_id is None and prep_id is None) or \
data not in valid_data:
raise HTTPError(422, reason='You need to specify both data (the '
'data type you want to download - %s) and '
'study_id or prep_id' % '/'.join(valid_data))
elif data_type is not None and data_type not in dtypes:
raise HTTPError(422, reason='Not a valid data_type. Valid types '
'are: %s' % ', '.join(dtypes))
elif data in templates and prep_id is None and study_id is None:
raise HTTPError(422, reason='If downloading a sample or '
'preparation file you need to define study_id or'
' prep_id')
elif data in templates:
if data_type is not None:
raise HTTPError(422, reason='If requesting an information '
'file you cannot specify the data_type')
elif prep_id is not None and data == 'prep_information':
fname = 'preparation_information_%s' % prep_id
prep_id = int(prep_id)
try:
infofile = PrepTemplate(prep_id)
except QiitaDBUnknownIDError:
raise HTTPError(
422, reason='Preparation information does not exist')
elif study_id is not None and data == 'sample_information':
fname = 'sample_information_%s' % study_id
study_id = int(study_id)
try:
infofile = SampleTemplate(study_id)
except QiitaDBUnknownIDError:
raise HTTPError(
422, reason='Sample information does not exist')
else:
raise HTTPError(422, reason='Review your parameters, not a '
'valid combination')
x = retrieve_filepaths(
infofile._filepath_table, infofile._id_column, infofile.id,
sort='descending')[0]
basedir_len = len(get_db_files_base_dir()) + 1
fp = x['fp'][basedir_len:]
to_download.append((fp, fp, str(x['checksum']), str(x['fp_size'])))
self._write_nginx_file_list(to_download)
zip_fn = '%s_%s.zip' % (
fname, datetime.now().strftime('%m%d%y-%H%M%S'))
self._set_nginx_headers(zip_fn)
else:
study_id = int(study_id)
try:
study = Study(study_id)
except QiitaDBUnknownIDError:
raise HTTPError(422, reason='Study does not exist')
else:
public_raw_download = study.public_raw_download
if study.status != 'public':
raise HTTPError(404, reason='Study is not public. If this '
'is a mistake contact: '
'*****@*****.**')
elif data == 'raw' and not public_raw_download:
raise HTTPError(422, reason='No raw data access. If this '
'is a mistake contact: '
'*****@*****.**')
else:
# raw data
artifacts = [a for a in study.artifacts(dtype=data_type)
if not a.parents]
# bioms
if data == 'biom':
artifacts = study.artifacts(
dtype=data_type, artifact_type='BIOM')
for a in artifacts:
if a.visibility != 'public':
continue
to_download.extend(self._list_artifact_files_nginx(a))
if not to_download:
raise HTTPError(422, reason='Nothing to download. If '
'this is a mistake contact: '
'*****@*****.**')
else:
self._write_nginx_file_list(to_download)
zip_fn = 'study_%d_%s_%s.zip' % (
study_id, data, datetime.now().strftime(
'%m%d%y-%H%M%S'))
self._set_nginx_headers(zip_fn)
self.finish()
def _build_study_info(user, study_proc=None, proc_samples=None):
"""Builds list of dicts for studies table, with all HTML formatted
Parameters
----------
user : User object
logged in user
study_proc : dict of lists, optional
Dictionary keyed on study_id that lists all processed data associated
with that study. Required if proc_samples given.
proc_samples : dict of lists, optional
Dictionary keyed on proc_data_id that lists all samples associated with
that processed data. Required if study_proc given.
Returns
-------
infolist: list of dict of lists and dicts
study and processed data info for JSON serialiation for datatables
Each dict in the list is a single study, and contains the text
Notes
-----
Both study_proc and proc_samples must be passed, or neither passed.
"""
build_samples = False
# Logic check to make sure both needed parts passed
if study_proc is not None and proc_samples is None:
raise IncompetentQiitaDeveloperError(
'Must pass proc_samples when study_proc given')
elif proc_samples is not None and study_proc is None:
raise IncompetentQiitaDeveloperError(
'Must pass study_proc when proc_samples given')
elif study_proc is None:
build_samples = True
# get list of studies for table
study_set = user.user_studies.union(
Study.get_by_status('public')).union(user.shared_studies)
if study_proc is not None:
study_set = study_set.intersection(study_proc)
if not study_set:
# No studies left so no need to continue
return []
cols = ['study_id', 'email', 'principal_investigator_id',
'publication_doi', 'study_title', 'metadata_complete',
'number_samples_collected', 'study_abstract']
study_info = Study.get_info([s.id for s in study_set], cols)
# get info for the studies
infolist = []
for info in study_info:
# Convert DictCursor to proper dict
info = dict(info)
study = Study(info['study_id'])
# Build the processed data info for the study if none passed
if build_samples:
proc_data_list = [ar for ar in study.artifacts()
if ar.artifact_type == 'BIOM']
proc_samples = {}
study_proc = {study.id: defaultdict(list)}
for proc_data in proc_data_list:
study_proc[study.id][proc_data.data_type].append(proc_data.id)
# there is only one prep template for each processed data
proc_samples[proc_data.id] = proc_data.prep_templates[0].keys()
study_info = _build_single_study_info(study, info, study_proc,
proc_samples)
infolist.append(study_info)
return infolist
def study_files_get_req(user_id, study_id, prep_template_id, artifact_type):
"""Returns the uploaded files for the study id categorized by artifact_type
It retrieves the files uploaded for the given study and tries to do a
guess on how those files should be added to the artifact of the given
type. Uses information on the prep template to try to do a better guess.
Parameters
----------
user_id : str
The id of the user making the request
study_id : int
The study id
prep_template_id : int
The prep template id
artifact_type : str
The artifact type
Returns
-------
dict of {str: object}
A dict of the form {'status': str,
'message': str,
'remaining': list of str,
'file_types': list of (str, bool, list of str),
'num_prefixes': int}
where 'status' is a string specifying whether the query is successfull,
'message' is a human-readable description of the error (optional),
'remaining' is the list of files that could not be categorized,
'file_types' is a list of the available filetypes, if it is required
or not and the list of categorized files for the given artifact type
and 'num_prefixes' is the number of different run prefix values in
the given prep template.
"""
supp_file_types = supported_filepath_types(artifact_type)
selected = []
remaining = []
uploaded = get_files_from_uploads_folders(study_id)
pt = PrepTemplate(prep_template_id).to_dataframe()
ftypes_if = (ft.startswith('raw_') for ft, _ in supp_file_types
if ft != 'raw_sff')
if any(ftypes_if) and 'run_prefix' in pt.columns:
prep_prefixes = tuple(set(pt['run_prefix']))
num_prefixes = len(prep_prefixes)
for _, filename in uploaded:
if filename.startswith(prep_prefixes):
selected.append(filename)
else:
remaining.append(filename)
else:
num_prefixes = 0
remaining = [f for _, f in uploaded]
# At this point we can't do anything smart about selecting by default
# the files for each type. The only thing that we can do is assume that
# the first in the supp_file_types list is the default one where files
# should be added in case of 'run_prefix' being present
file_types = [(fp_type, req, []) for fp_type, req in supp_file_types[1:]]
first = supp_file_types[0]
# Note that this works even if `run_prefix` is not in the prep template
# because selected is initialized to the empty list
file_types.insert(0, (first[0], first[1], selected))
# Create a list of artifacts that the user has access to, in case that
# he wants to import the files from another artifact
user = User(user_id)
artifact_options = []
user_artifacts = user.user_artifacts(artifact_type=artifact_type)
study = Study(study_id)
if study not in user_artifacts:
user_artifacts[study] = study.artifacts(artifact_type=artifact_type)
for study, artifacts in viewitems(user_artifacts):
study_label = "%s (%d)" % (study.title, study.id)
for a in artifacts:
artifact_options.append(
(a.id, "%s - %s (%d)" % (study_label, a.name, a.id)))
return {'status': 'success',
'message': '',
'remaining': sorted(remaining),
'file_types': file_types,
'num_prefixes': num_prefixes,
'artifacts': artifact_options}
def study_get_req(study_id, user_id):
"""Returns information available for the given study
Parameters
----------
study_id : int
Study id to get prep template info for
user_id : str
User requesting the info
Returns
-------
dict
Data types information in the form
{'status': status,
'message': message,
'info': dict of objects
status can be success, warning, or error depending on result
message has the warnings or errors
info contains study information seperated by data type, in the form
{col_name: value, ...} with value being a string, int, or list of
strings or ints
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
# Can only pass ids over API, so need to instantiate object
study = Study(study_id)
study_info = study.info
# Add needed info that is not part of the initial info pull
study_info['publication_doi'] = []
study_info['publication_pid'] = []
for pub, is_doi in study.publications:
if is_doi:
study_info['publication_doi'].append(pub)
else:
study_info['publication_pid'].append(pub)
study_info['study_id'] = study.id
study_info['study_title'] = study.title
study_info['shared_with'] = [s.id for s in study.shared_with]
study_info['status'] = study.status
study_info['ebi_study_accession'] = study.ebi_study_accession
study_info['ebi_submission_status'] = study.ebi_submission_status
study_info['public_raw_download'] = study.public_raw_download
study_info['notes'] = study.notes
# Clean up StudyPerson objects to string for display
pi = study_info['principal_investigator']
study_info['principal_investigator'] = {
'name': pi.name,
'email': pi.email,
'affiliation': pi.affiliation
}
lab_person = study_info['lab_person']
if lab_person:
study_info['lab_person'] = {
'name': lab_person.name,
'email': lab_person.email,
'affiliation': lab_person.affiliation
}
samples = study.sample_template
study_info['num_samples'] = 0 if samples is None else len(list(samples))
study_info['owner'] = study.owner.id
# Study.has_access no_public=True, will return True only if the user_id is
# the owner of the study or if the study is shared with the user_id; this
# with study.public_raw_download will define has_access_to_raw_data
study_info['has_access_to_raw_data'] = study.has_access(
User(user_id), True) or study.public_raw_download
study_info['show_biom_download_button'] = 'BIOM' in [
a.artifact_type for a in study.artifacts()
]
study_info['show_raw_download_button'] = any(
[True for pt in study.prep_templates() if pt.artifact is not None])
# getting study processing status from redis
processing = False
study_info['level'] = ''
study_info['message'] = ''
job_info = r_client.get(STUDY_KEY_FORMAT % study_id)
if job_info:
job_info = defaultdict(lambda: '', loads(job_info))
job_id = job_info['job_id']
job = ProcessingJob(job_id)
job_status = job.status
processing = job_status not in ('success', 'error')
if processing:
study_info['level'] = 'info'
study_info['message'] = 'This study is currently being processed'
elif job_status == 'error':
study_info['level'] = 'danger'
study_info['message'] = job.log.msg.replace('\n', '</br>')
else:
study_info['level'] = job_info['alert_type']
study_info['message'] = job_info['alert_msg'].replace(
'\n', '</br>')
return {
'status': 'success',
'message': '',
'study_info': study_info,
'editable': study.can_edit(User(user_id))
}
def study_files_get_req(user_id, study_id, prep_template_id, artifact_type):
"""Returns the uploaded files for the study id categorized by artifact_type
It retrieves the files uploaded for the given study and tries to
guess on how those files should be added to the artifact of the given
type. Uses information on the prep template to try to do a better guess.
Parameters
----------
user_id : str
The id of the user making the request
study_id : int
The study id
prep_template_id : int
The prep template id
artifact_type : str
The artifact type
Returns
-------
dict of {str: object}
A dict of the form {'status': str,
'message': str,
'remaining': list of str,
'file_types': list of (str, bool, list of str),
'num_prefixes': int}
where 'status' is a string specifying whether the query is successfull,
'message' is a human-readable description of the error (optional),
'remaining' is the list of files that could not be categorized,
'file_types' is a list of the available filetypes, if it is required
or not and the list of categorized files for the given artifact type
and 'num_prefixes' is the number of different run prefix values in
the given prep template.
"""
supp_file_types = supported_filepath_types(artifact_type)
selected = []
remaining = []
message = []
pt = PrepTemplate(prep_template_id)
if pt.study_id != study_id:
raise IncompetentQiitaDeveloperError(
"The requested prep id (%d) doesn't belong to the study "
"(%d)" % (pt.study_id, study_id))
uploaded = get_files_from_uploads_folders(study_id)
pt = pt.to_dataframe()
ftypes_if = (ft.startswith('raw_') for ft, _ in supp_file_types
if ft != 'raw_sff')
if any(ftypes_if) and 'run_prefix' in pt.columns:
prep_prefixes = tuple(set(pt['run_prefix']))
num_prefixes = len(prep_prefixes)
# sorting prefixes by length to avoid collisions like: 100 1002
# 10003
prep_prefixes = sorted(prep_prefixes, key=len, reverse=True)
# group files by prefix
sfiles = defaultdict(list)
for p in prep_prefixes:
to_remove = []
for fid, f, _ in uploaded:
if f.startswith(p):
sfiles[p].append(f)
to_remove.append((fid, f))
uploaded = [x for x in uploaded if x not in to_remove]
inuse = [y for x in sfiles.values() for y in x]
remaining.extend([f for _, f, _ in uploaded if f not in inuse])
supp_file_types_len = len(supp_file_types)
for k, v in sfiles.items():
len_files = len(v)
# if the number of files in the k group is larger than the
# available columns add to the remaining group, if not put them in
# the selected group
if len_files > supp_file_types_len:
remaining.extend(v)
message.append("'%s' has %d matches." % (k, len_files))
else:
v.sort()
selected.append(v)
else:
num_prefixes = 0
remaining = [f for _, f, _ in uploaded]
# get file_types, format: filetype, required, list of files
file_types = [(t, req, [x[i] for x in selected if i + 1 <= len(x)])
for i, (t, req) in enumerate(supp_file_types)]
# Create a list of artifacts that the user has access to, in case that
# he wants to import the files from another artifact
user = User(user_id)
artifact_options = []
user_artifacts = user.user_artifacts(artifact_type=artifact_type)
study = Study(study_id)
if study not in user_artifacts:
user_artifacts[study] = study.artifacts(artifact_type=artifact_type)
for study, artifacts in user_artifacts.items():
study_label = "%s (%d)" % (study.title, study.id)
for a in artifacts:
artifact_options.append(
(a.id, "%s - %s (%d)" % (study_label, a.name, a.id)))
message = ('' if not message else '\n'.join(['Check these run_prefix:'] +
message))
return {
'status': 'success',
'message': message,
'remaining': sorted(remaining),
'file_types': file_types,
'num_prefixes': num_prefixes,
'artifacts': artifact_options
}
def _build_study_info(user, study_proc=None, proc_samples=None):
"""Builds list of dicts for studies table, with all HTML formatted
Parameters
----------
user : User object
logged in user
study_proc : dict of lists, optional
Dictionary keyed on study_id that lists all processed data associated
with that study. Required if proc_samples given.
proc_samples : dict of lists, optional
Dictionary keyed on proc_data_id that lists all samples associated with
that processed data. Required if study_proc given.
Returns
-------
infolist: list of dict of lists and dicts
study and processed data info for JSON serialiation for datatables
Each dict in the list is a single study, and contains the text
Notes
-----
Both study_proc and proc_samples must be passed, or neither passed.
"""
build_samples = False
# Logic check to make sure both needed parts passed
if study_proc is not None and proc_samples is None:
raise IncompetentQiitaDeveloperError(
'Must pass proc_samples when study_proc given')
elif proc_samples is not None and study_proc is None:
raise IncompetentQiitaDeveloperError(
'Must pass study_proc when proc_samples given')
elif study_proc is None:
build_samples = True
# get list of studies for table
study_set = user.user_studies.union(
Study.get_by_status('public')).union(user.shared_studies)
if study_proc is not None:
study_set = study_set.intersection(study_proc)
if not study_set:
# No studies left so no need to continue
return []
cols = ['study_id', 'email', 'principal_investigator_id',
'publication_doi', 'study_title', 'metadata_complete',
'number_samples_collected', 'study_abstract']
study_info = Study.get_info([s.id for s in study_set], cols)
# get info for the studies
infolist = []
for info in study_info:
# Convert DictCursor to proper dict
info = dict(info)
study = Study(info['study_id'])
# Build the processed data info for the study if none passed
if build_samples:
proc_data_list = [ar for ar in study.artifacts()
if ar.artifact_type == 'BIOM']
proc_samples = {}
study_proc = {study.id: defaultdict(list)}
for proc_data in proc_data_list:
study_proc[study.id][proc_data.data_type].append(proc_data.id)
# there is only one prep template for each processed data
proc_samples[proc_data.id] = proc_data.prep_templates[0].keys()
study_info = _build_single_study_info(study, info, study_proc,
proc_samples)
infolist.append(study_info)
return infolist
def study_get_req(study_id, user_id):
"""Returns information available for the given study
Parameters
----------
study_id : int
Study id to get prep template info for
user_id : str
User requesting the info
Returns
-------
dict
Data types information in the form
{'status': status,
'message': message,
'info': dict of objects
status can be success, warning, or error depending on result
message has the warnings or errors
info contains study information seperated by data type, in the form
{col_name: value, ...} with value being a string, int, or list of
strings or ints
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
# Can only pass ids over API, so need to instantiate object
study = Study(study_id)
study_info = study.info
# Add needed info that is not part of the initial info pull
study_info['publication_doi'] = []
study_info['publication_pid'] = []
for pub, is_doi in study.publications:
if is_doi:
study_info['publication_doi'].append(pub)
else:
study_info['publication_pid'].append(pub)
study_info['study_id'] = study.id
study_info['study_title'] = study.title
study_info['shared_with'] = [s.id for s in study.shared_with]
study_info['status'] = study.status
study_info['ebi_study_accession'] = study.ebi_study_accession
study_info['ebi_submission_status'] = study.ebi_submission_status
# Clean up StudyPerson objects to string for display
pi = study_info['principal_investigator']
study_info['principal_investigator'] = {
'name': pi.name,
'email': pi.email,
'affiliation': pi.affiliation}
lab_person = study_info['lab_person']
if lab_person:
study_info['lab_person'] = {
'name': lab_person.name,
'email': lab_person.email,
'affiliation': lab_person.affiliation}
samples = study.sample_template
study_info['num_samples'] = 0 if samples is None else len(list(samples))
study_info['owner'] = study.owner.id
# Study.has_access no_public=True, will return True only if the user_id is
# the owner of the study or if the study is shared with the user_id
study_info['has_access_to_raw_data'] = study.has_access(
User(user_id), True)
study_info['show_biom_download_button'] = 'BIOM' in [
a.artifact_type for a in study.artifacts()]
study_info['show_raw_download_button'] = any([
True for pt in study.prep_templates() if pt.artifact is not None])
# getting study processing status from redis
processing = False
study_info['level'] = ''
study_info['message'] = ''
job_info = r_client.get(STUDY_KEY_FORMAT % study_id)
if job_info:
job_info = defaultdict(lambda: '', loads(job_info))
job_id = job_info['job_id']
job = ProcessingJob(job_id)
job_status = job.status
processing = job_status not in ('success', 'error')
if processing:
study_info['level'] = 'info'
study_info['message'] = 'This study is currently being processed'
elif job_status == 'error':
study_info['level'] = 'danger'
study_info['message'] = job.log.msg.replace('\n', '</br>')
else:
study_info['level'] = job_info['alert_type']
study_info['message'] = job_info['alert_msg'].replace(
'\n', '</br>')
return {'status': 'success',
'message': '',
'study_info': study_info,
'editable': study.can_edit(User(user_id))}
def study_files_get_req(user_id, study_id, prep_template_id, artifact_type):
"""Returns the uploaded files for the study id categorized by artifact_type
It retrieves the files uploaded for the given study and tries to
guess on how those files should be added to the artifact of the given
type. Uses information on the prep template to try to do a better guess.
Parameters
----------
user_id : str
The id of the user making the request
study_id : int
The study id
prep_template_id : int
The prep template id
artifact_type : str
The artifact type
Returns
-------
dict of {str: object}
A dict of the form {'status': str,
'message': str,
'remaining': list of str,
'file_types': list of (str, bool, list of str),
'num_prefixes': int}
where 'status' is a string specifying whether the query is successfull,
'message' is a human-readable description of the error (optional),
'remaining' is the list of files that could not be categorized,
'file_types' is a list of the available filetypes, if it is required
or not and the list of categorized files for the given artifact type
and 'num_prefixes' is the number of different run prefix values in
the given prep template.
"""
supp_file_types = supported_filepath_types(artifact_type)
selected = []
remaining = []
message = []
pt = PrepTemplate(prep_template_id)
if pt.study_id != study_id:
raise IncompetentQiitaDeveloperError(
"The requested prep id (%d) doesn't belong to the study "
"(%d)" % (pt.study_id, study_id))
uploaded = get_files_from_uploads_folders(study_id)
pt = pt.to_dataframe()
ftypes_if = (ft.startswith('raw_') for ft, _ in supp_file_types
if ft != 'raw_sff')
if any(ftypes_if) and 'run_prefix' in pt.columns:
prep_prefixes = tuple(set(pt['run_prefix']))
num_prefixes = len(prep_prefixes)
# sorting prefixes by length to avoid collisions like: 100 1002
# 10003
prep_prefixes = sorted(prep_prefixes, key=len, reverse=True)
# group files by prefix
sfiles = defaultdict(list)
for p in prep_prefixes:
to_remove = []
for fid, f in uploaded:
if f.startswith(p):
sfiles[p].append(f)
to_remove.append((fid, f))
uploaded = [x for x in uploaded if x not in to_remove]
inuse = [y for x in sfiles.values() for y in x]
remaining.extend([f for _, f in uploaded if f not in inuse])
supp_file_types_len = len(supp_file_types)
for k, v in viewitems(sfiles):
len_files = len(v)
# if the number of files in the k group is larger than the
# available columns add to the remaining group, if not put them in
# the selected group
if len_files > supp_file_types_len:
remaining.extend(v)
message.append("'%s' has %d matches." % (k, len_files))
else:
v.sort()
selected.append(v)
else:
num_prefixes = 0
remaining = [f for _, f in uploaded]
# get file_types, format: filetype, required, list of files
file_types = [(t, req, [x[i] for x in selected if i+1 <= len(x)])
for i, (t, req) in enumerate(supp_file_types)]
# Create a list of artifacts that the user has access to, in case that
# he wants to import the files from another artifact
user = User(user_id)
artifact_options = []
user_artifacts = user.user_artifacts(artifact_type=artifact_type)
study = Study(study_id)
if study not in user_artifacts:
user_artifacts[study] = study.artifacts(artifact_type=artifact_type)
for study, artifacts in viewitems(user_artifacts):
study_label = "%s (%d)" % (study.title, study.id)
for a in artifacts:
artifact_options.append(
(a.id, "%s - %s (%d)" % (study_label, a.name, a.id)))
message = ('' if not message
else '\n'.join(['Check these run_prefix:'] + message))
return {'status': 'success',
'message': message,
'remaining': sorted(remaining),
'file_types': file_types,
'num_prefixes': num_prefixes,
'artifacts': artifact_options}
<<<<<<< HEAD
cols = ['study_id', 'email', 'principal_investigator_id',
'publication_doi', 'study_title', 'metadata_complete',
'number_samples_collected', 'study_abstract']
study_info = Study.get_info([s.id for s in study_set], cols)
# get info for the studies
infolist = []
for info in study_info:
# Convert DictCursor to proper dict
info = dict(info)
study = Study(info['study_id'])
# Build the processed data info for the study if none passed
if build_samples:
proc_data_list = [ar for ar in study.artifacts()
if ar.artifact_type == 'BIOM']
proc_samples = {}
study_proc = {study.id: defaultdict(list)}
for proc_data in proc_data_list:
study_proc[study.id][proc_data.data_type].append(proc_data.id)
# there is only one prep template for each processed data
proc_samples[proc_data.id] = proc_data.prep_templates[0].keys()
study_info = _build_single_study_info(study, info, study_proc,
proc_samples)
infolist.append(study_info)
return infolist
=======
return generate_study_list([s.id for s in study_set],
def get(self):
study_id = self.get_argument("study_id", None)
artifact_id = self.get_argument("artifact_id", None)
if study_id is None and artifact_id is None:
raise HTTPError(
422, reason='You need to specify study_id or artifact_id')
self.finish()
elif study_id is not None:
try:
study = Study(int(study_id))
except QiitaDBUnknownIDError:
raise HTTPError(422,
reason="Study %s doesn't exist" % study_id)
self.finish()
artifact_ids = [
a.id for a in study.artifacts() if a.visibility == 'public'
]
else:
try:
artifact = Artifact(int(artifact_id))
except QiitaDBUnknownIDError:
raise HTTPError(422,
reason="Artifact %s doesn't exist" %
artifact_id)
self.finish()
if artifact.visibility != 'public':
raise HTTPError(422,
reason="Artifact %s is not public" %
artifact_id)
self.finish()
study = artifact.study
if study is None:
raise HTTPError(422,
reason="Artifact %s doesn't belong to "
"a study" % artifact_id)
self.finish()
artifact_ids = [artifact.id]
if study.status != 'public':
raise HTTPError(422, reason='Not a public study')
self.finish()
study_info = study.info
study_info['study_id'] = study.id
study_info['study_title'] = study.title
study_info['shared_with'] = [s.id for s in study.shared_with]
study_info['status'] = study.status
study_info['ebi_study_accession'] = study.ebi_study_accession
study_info['ebi_submission_status'] = study.ebi_submission_status
# Clean up StudyPerson objects to string for display
email = '<a href="mailto:{email}">{name} ({affiliation})</a>'
pi = study.info['principal_investigator']
study_info['principal_investigator'] = email.format(
**{
'name': pi.name,
'email': pi.email,
'affiliation': pi.affiliation
})
study_info['owner'] = study.owner.id
# Add needed info that is not part of the initial info pull
study_info['publications'] = []
for pub, is_doi in study.publications:
if is_doi:
study_info['publications'].append(pubmed_linkifier([pub]))
else:
study_info['publications'].append(doi_linkifier([pub]))
study_info['publications'] = ', '.join(study_info['publications'])
if study_info['ebi_study_accession']:
links = ''.join([
EBI_LINKIFIER.format(a)
for a in study_info['ebi_study_accession'].split(',')
])
study_info['ebi_study_accession'] = '%s (%s)' % (
links, study_info['ebi_submission_status'])
self.render("public.html",
study_info=study_info,
artifacts_info=get_artifacts_information(
artifact_ids, False))
def study_get_req(study_id, user_id):
"""Returns information available for the given study
Parameters
----------
study_id : int
Study id to get prep template info for
user_id : str
User requesting the info
Returns
-------
dict
Data types information in the form
{'status': status,
'message': message,
'info': dict of objects
status can be success, warning, or error depending on result
message has the warnings or errors
info contains study information seperated by data type, in the form
{col_name: value, ...} with value being a string, int, or list of
strings or ints
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
# Can only pass ids over API, so need to instantiate object
study = Study(study_id)
study_info = study.info
# Add needed info that is not part of the initial info pull
study_info['publication_doi'] = []
study_info['publication_pid'] = []
for pub, is_doi in study.publications:
if is_doi:
study_info['publication_doi'].append(pub)
else:
study_info['publication_pid'].append(pub)
study_info['study_id'] = study.id
study_info['study_title'] = study.title
study_info['shared_with'] = [s.id for s in study.shared_with]
study_info['status'] = study.status
study_info['ebi_study_accession'] = study.ebi_study_accession
study_info['ebi_submission_status'] = study.ebi_submission_status
# Clean up StudyPerson objects to string for display
pi = study_info['principal_investigator']
study_info['principal_investigator'] = {
'name': pi.name,
'email': pi.email,
'affiliation': pi.affiliation}
lab_person = study_info['lab_person']
if lab_person:
study_info['lab_person'] = {
'name': lab_person.name,
'email': lab_person.email,
'affiliation': lab_person.affiliation}
samples = study.sample_template
study_info['num_samples'] = 0 if samples is None else len(list(samples))
study_info['owner'] = study.owner.id
# Study.has_access no_public=True, will return True only if the user_id is
# the owner of the study or if the study is shared with the user_id
study_info['has_access_to_raw_data'] = study.has_access(
User(user_id), True)
study_info['show_biom_download_button'] = 'BIOM' in [
a.artifact_type for a in study.artifacts()]
study_info['show_raw_download_button'] = any([
True for pt in study.prep_templates() if pt.artifact is not None])
return {'status': 'success',
'message': '',
'study_info': study_info,
'editable': study.can_edit(User(user_id))}
def get(self, study_id):
study_id = int(study_id)
# Check access to study
study_info = study_get_req(study_id, self.current_user.id)
if study_info['status'] != 'success':
raise HTTPError(
405,
"%s: %s, %s" % (study_info['message'], self.current_user.email,
str(study_id)))
study = Study(study_id)
basedir = get_db_files_base_dir()
basedir_len = len(basedir) + 1
# loop over artifacts and retrieve those that we have access to
to_download = []
for a in study.artifacts():
if a.artifact_type == 'BIOM':
for i, (fid, path, data_type) in enumerate(a.filepaths):
# ignore if tgz as they could create problems and the
# raw data is in the folder
if data_type == 'tgz':
continue
if data_type == 'directory':
# If we have a directory, we actually need to list
# all the files from the directory so NGINX can
# actually download all of them
for dp, _, fps in walk(path):
for fname in fps:
fullpath = join(dp, fname)
spath = fullpath
if fullpath.startswith(basedir):
spath = fullpath[basedir_len:]
to_download.append((fullpath, spath, spath))
elif path.startswith(basedir):
spath = path[basedir_len:]
to_download.append((path, spath, spath))
else:
# We are not aware of any case that can trigger this
# situation, but we wanted to be overly cautious
# There is no test for this line cause we don't know
# how to trigger it
to_download.append((path, path, path))
for pt in a.prep_templates:
qmf = pt.qiime_map_fp
if qmf is not None:
sqmf = qmf
if qmf.startswith(basedir):
sqmf = qmf[basedir_len:]
to_download.append(
(qmf, sqmf,
'mapping_files/%s_mapping_file.txt' % a.id))
# If we don't have nginx, write a file that indicates this
all_files = '\n'.join([
"- %s /protected/%s %s" % (getsize(fp), sfp, n)
for fp, sfp, n in to_download
])
self.write("%s\n" % all_files)
zip_fn = 'study_%d_%s.zip' % (study_id,
datetime.now().strftime('%m%d%y-%H%M%S'))
self.set_header('Content-Description', 'File Transfer')
self.set_header('Expires', '0')
self.set_header('Cache-Control', 'no-cache')
self.set_header('X-Archive-Files', 'zip')
self.set_header('Content-Disposition',
'attachment; filename=%s' % zip_fn)
self.finish()
