def get_release_info(study_status='public'):
"""Returns the study status release MD5
Parameters
----------
study_status : str, optional
The study status to search for. Note that this should always be set
to 'public' but having this exposed helps with testing. The other
options are 'private' and 'sandbox'
Returns
------
str, str, str
The release MD5, filepath and timestamp
"""
portal = qiita_config.portal
md5sum = r_client.get('%s:release:%s:md5sum' % (portal, study_status))
filepath = r_client.get('%s:release:%s:filepath' % (portal, study_status))
timestamp = r_client.get('%s:release:%s:time' % (portal, study_status))
if md5sum is None:
md5sum = ''
if filepath is None:
filepath = ''
if timestamp is None:
timestamp = ''
return md5sum, filepath, timestamp
def test_sample_template_handler_post_request(self):
# Test user doesn't have access
with self.assertRaisesRegex(HTTPError,
'User does not have access to study'):
sample_template_handler_post_request(
1, User('*****@*****.**'), 'ignored')
# Test study doesn't exist
user = User('*****@*****.**')
with self.assertRaisesRegex(HTTPError, 'Study does not exist'):
sample_template_handler_post_request(1000000, user, 'ignored')
# Test file doesn't exist
with self.assertRaisesRegex(HTTPError, 'Filepath not found'):
sample_template_handler_post_request(1, user, 'DoesNotExist.txt')
# Test looks like mapping file and no data_type provided
uploads_dir = join(get_mountpoint('uploads')[0][1], '1')
fd, fp = mkstemp(suffix='.txt', dir=uploads_dir)
self._clean_up_files.append(fp)
close(fd)
with open(fp, 'w') as f:
f.write('#SampleID\tCol1\nSample1\tVal1')
with self.assertRaisesRegex(
HTTPError, 'Please, choose a data type if uploading a QIIME '
'mapping file'):
sample_template_handler_post_request(1, user, fp)
# Test success
obs = sample_template_handler_post_request(
1, user, 'uploaded_file.txt')
self.assertCountEqual(obs.keys(), ['job'])
job_info = r_client.get('sample_template_1')
self.assertIsNotNone(job_info)
# Wait until the job is done
wait_for_processing_job(loads(job_info)['job_id'])
# Test direct upload
obs = sample_template_handler_post_request(
1, user, fp, data_type='16S', direct_upload=True)
self.assertCountEqual(obs.keys(), ['job'])
job_info = r_client.get('sample_template_1')
self.assertIsNotNone(job_info)
# Wait until the job is done
wait_for_processing_job(loads(job_info)['job_id'])
def test_create_sample_template(self):
# Test error
job = self._create_job('create_sample_template', {
'fp': self.fp, 'study_id': 1, 'is_mapping_file': False,
'data_type': None})
private_task(job.id)
self.assertEqual(job.status, 'error')
self.assertIn("The 'SampleTemplate' object with attributes (id: 1) "
"already exists.", job.log.msg)
# Test success with a warning
info = {"timeseries_type_id": '1',
"metadata_complete": 'true',
"mixs_compliant": 'true',
"study_alias": "TDST",
"study_description": "Test create sample template",
"study_abstract": "Test create sample template",
"principal_investigator_id": StudyPerson(1)}
study = Study.create(User('*****@*****.**'),
"Create Sample Template test", info)
job = self._create_job('create_sample_template',
{'fp': self.fp, 'study_id': study.id,
'is_mapping_file': False, 'data_type': None})
private_task(job.id)
self.assertEqual(job.status, 'success')
obs = r_client.get("sample_template_%d" % study.id)
self.assertIsNotNone(obs)
obs = loads(obs)
self.assertCountEqual(obs, ['job_id', 'alert_type', 'alert_msg'])
self.assertEqual(obs['job_id'], job.id)
self.assertEqual(obs['alert_type'], 'warning')
self.assertIn(
'Some functionality will be disabled due to missing columns:',
obs['alert_msg'])
def prep_template_jobs_get_req(prep_id, user_id):
"""Returns graph of all artifacts created from the prep base artifact
Parameters
----------
prep_id : int
Prep template ID to get graph for
user_id : str
User making the request
Returns
-------
dict with the jobs information
Notes
-----
Nodes are identified by the corresponding Artifact ID.
"""
prep = PrepTemplate(int(prep_id))
access_error = check_access(prep.study_id, user_id)
if access_error:
return access_error
job_info = r_client.get(PREP_TEMPLATE_KEY_FORMAT % prep_id)
result = {}
if job_info:
job_info = defaultdict(lambda: '', loads(job_info))
job_id = job_info['job_id']
job = ProcessingJob(job_id)
result[job.id] = {'status': job.status, 'step': job.step,
'error': job.log.msg if job.log else ""}
return result
def wait_for_prep_information_job(prep_id, raise_if_none=True):
"""Waits until a prep information job is completed
Parameters
----------
prep_id : int
Prep template id
raise_if_none : bool, optional
If True, raise an AssertionError if the correspondent redis key
is empty. Default: True
Raises
------
AssertionError
If `raise_if_none` is True and the correspondent redis key is not set
"""
res = r_client.get('prep_template_%d' % prep_id)
if raise_if_none and res is None:
raise AssertionError("unexpectedly None")
if res is not None:
payload = loads(res)
job_id = payload['job_id']
wait_for_processing_job(job_id)
def test_sample_template_handler_delete_request(self):
# Test user doesn't have access
with self.assertRaisesRegexp(HTTPError,
'User does not have access to study'):
sample_template_handler_delete_request(
1, User('*****@*****.**'))
# Test study doesn't exist
user = User('*****@*****.**')
with self.assertRaisesRegexp(HTTPError, 'Study does not exist'):
sample_template_handler_delete_request(1000000, user)
# Test sample information doesn't exist
new_study = self._create_study('Study for deleting test')
with self.assertRaisesRegexp(HTTPError, "Study %s doesn't have sample "
"information" % new_study.id):
sample_template_handler_delete_request(new_study.id, user)
# Test success
user = User('*****@*****.**')
obs = sample_template_handler_delete_request(1, user)
self.assertEqual(obs.keys(), ['job'])
job_info = r_client.get('sample_template_1')
self.assertIsNotNone(job_info)
# Wait until the job is done
wait_for_processing_job(loads(job_info)['job_id'])
def analysis_description_handler_get_request(analysis_id, user):
"""Returns the analysis information
Parameters
----------
analysis_id : int
The analysis id
user : qiita_db.user.User
The user performing the request
"""
analysis = Analysis(analysis_id)
check_analysis_access(user, analysis)
job_info = r_client.get("analysis_%s" % analysis.id)
alert_type = 'info'
alert_msg = ''
if job_info:
job_info = loads(job_info)
job_id = job_info['job_id']
if job_id:
r_payload = r_client.get(job_id)
if r_payload:
redis_info = loads(r_client.get(job_id))
if redis_info['status_msg'] == 'running':
alert_msg = ('An artifact is being deleted from this '
'analysis')
elif redis_info['return'] is not None:
alert_type = redis_info['return']['status']
alert_msg = redis_info['return']['message'].replace(
'\n', '</br>')
artifacts = {}
for aid, samples in analysis.samples.items():
artifact = Artifact(aid)
study = artifact.study
artifacts[aid] = (
study.id, study.title, artifact.merging_scheme, samples)
return {'analysis_name': analysis.name,
'analysis_id': analysis.id,
'analysis_is_public': analysis.is_public,
'analysis_description': analysis.description,
'analysis_mapping_id': analysis.mapping_file,
'alert_type': alert_type,
'artifacts': artifacts,
'alert_msg': alert_msg}
def sample_template_overview_handler_get_request(study_id, user):
# Check if the current user has access to the sample template
sample_template_checks(study_id, user)
# Check if the sample template exists
exists = SampleTemplate.exists(study_id)
# The following information should always be provided:
# The files that have been uploaded to the system and can be a
# sample template file
files = [f for _, f, _ in get_files_from_uploads_folders(study_id)
if f.endswith(('txt', 'tsv', 'xlsx'))]
# If there is a job associated with the sample information, the job id
job = None
job_info = r_client.get(SAMPLE_TEMPLATE_KEY_FORMAT % study_id)
if job_info:
job = loads(job_info)['job_id']
# Specific information if it exists or not:
data_types = []
st_fp_id = None
old_files = []
num_samples = 0
num_cols = 0
if exists:
# If it exists we need to provide:
# The id of the sample template file so the user can download it and
# the list of old filepaths
st = SampleTemplate(study_id)
all_st_files = st.get_filepaths()
# The current sample template file is the first one in the list
# (pop(0)) and we are interested only in the id ([0])
st_fp_id = all_st_files.pop(0)[0]
# For the old filepaths we are only interested in their basename
old_files = [basename(fp) for _, fp in all_st_files]
# The number of samples - this is a space efficient way of counting
# the number of samples. Doing len(list(st.keys())) creates a list
# that we are not using
num_samples = sum(1 for _ in st.keys())
# The number of columns
num_cols = len(st.categories())
else:
# It doesn't exist, we also need to provide the data_types in case
# the user uploads a QIIME mapping file
data_types = sorted(data_types_get_req()['data_types'])
return {'exists': exists,
'uploaded_files': files,
'data_types': data_types,
'user_can_edit': Study(study_id).can_edit(user),
'job': job,
'download_id': st_fp_id,
'old_files': old_files,
'num_samples': num_samples,
'num_columns': num_cols}
def test_artifact_post_request(self):
# No access
with self.assertRaises(QiitaHTTPError):
artifact_post_req(User('*****@*****.**'), 1)
obs = artifact_post_req(User('*****@*****.**'), 2)
self.assertEqual(obs.keys(), ['job'])
# Wait until the job is completed
wait_for_prep_information_job(1)
# Check that the delete function has been actually called
job = ProcessingJob(loads(r_client.get('prep_template_1'))['job_id'])
self.assertEqual(job.status, 'error')
self.assertIn('Cannot delete artifact 2', job.log.msg)
def wrapper(handler, *args, **kwargs):
header = handler.request.headers.get('Authorization', None)
if header is None:
_oauth_error(handler, 'Oauth2 error: invalid access token',
'invalid_request')
return
token_info = header.split()
# Based on RFC6750 if reply is not 2 elements in the format of:
# ['Bearer', token] we assume a wrong reply
if len(token_info) != 2 or token_info[0] != 'Bearer':
_oauth_error(handler, 'Oauth2 error: invalid access token',
'invalid_grant')
return
token = token_info[1]
db_token = r_client.hgetall(token)
if not db_token:
# token has timed out or never existed
_oauth_error(handler, 'Oauth2 error: token has timed out',
'invalid_grant')
return
# Check daily rate limit for key if password style key
if db_token[b'grant_type'] == b'password':
limit_key = '%s_%s_daily_limit' % (
db_token[b'client_id'].decode('ascii'),
db_token[b'user'].decode('ascii'))
limiter = r_client.get(limit_key)
if limiter is None:
# Set limit to 5,000 requests per day
r_client.setex(limit_key, 86400, 5000)
else:
r_client.decr(limit_key)
if int(r_client.get(limit_key)) <= 0:
_oauth_error(
handler, 'Oauth2 error: daily request limit reached',
'invalid_grant')
return
return f(handler, *args, **kwargs)
def test_authenticate_header_username(self):
obs = self.get('/qiita_db/artifacts/1/', headers={
'Authorization': 'Bearer ' + self.user_token})
self.assertEqual(obs.code, 200)
# Check rate limiting works
self.assertEqual(int(r_client.get(self.user_rate_key)), 1)
r_client.setex('[email protected]_daily_limit', 1, 0)
obs = self.get('/qiita_db/artifacts/100/', headers={
'Authorization': 'Bearer ' + self.user_token})
exp = {'error': 'invalid_grant',
'error_description': 'Oauth2 error: daily request limit reached'
}
self.assertEqual(loads(obs.body), exp)
def set_token(self, client_id, grant_type, user=None, timeout=3600):
"""Create access token for the client on redis and send json response
Parameters
----------
client_id : str
Client that requested the token
grant_type : str
Type of key being requested
user : str, optional
If password grant type requested, the user requesting the key.
timeout : int, optional
The timeout, in seconds, for the token. Default 3600
Returns
-------
Writes token information JSON in the form expected by RFC6750:
{'access_token': token,
'token_type': 'Bearer',
'expires_in': timeout}
access_token: the actual token to use
token_type: 'Bearer', which is the expected token type for Oauth2
expires_in: time to token expiration, in seconds.
"""
token = self.generate_access_token()
token_info = {
'timestamp': datetime.datetime.now().strftime('%m-%d-%y %H:%M:%S'),
'client_id': client_id,
'grant_type': grant_type
}
if user:
token_info['user'] = user
r_client.hmset(token, token_info)
r_client.expire(token, timeout)
if grant_type == 'password':
# Check if client has access limit key, and if not, create it
limit_key = '%s_%s_daily_limit' % (client_id, user)
limiter = r_client.get(limit_key)
if limiter is None:
# Set limit to 5,000 requests per day
r_client.setex(limit_key, 86400, 5000)
self.write({'access_token': token,
'token_type': 'Bearer',
'expires_in': timeout})
self.finish()
def display_template(self, study_id, msg):
"""Simple function to avoid duplication of code"""
study_id = int(study_id)
study = Study(study_id)
user = self.current_user
level = 'info'
message = ''
remote_url = ''
remote_files = []
check_access(user, study, no_public=True, raise_error=True)
job_info = r_client.get(UPLOAD_STUDY_FORMAT % study_id)
if job_info:
job_info = defaultdict(lambda: '', loads(job_info))
job_id = job_info['job_id']
job = ProcessingJob(job_id)
job_status = job.status
processing = job_status not in ('success', 'error')
url = job.parameters.values['url']
if processing:
if job.command.name == 'list_remote_files':
message = 'Retrieving remote files: listing %s' % url
else:
message = 'Retrieving remote files: download %s' % url
elif job_status == 'error':
level = 'danger'
message = job.log.msg.replace('\n', '</br>')
# making errors nicer for users
if 'No such file' in message:
message = 'URL not valid: <i>%s</i>, please review.' % url
else:
remote_url = job_info['url']
remote_files = job_info['files']
level = job_info['alert_type']
message = job_info['alert_msg'].replace('\n', '</br>')
# getting the ontologies
self.render('upload.html',
study_title=study.title, study_info=study.info,
study_id=study_id, is_admin=user.level == 'admin',
extensions=','.join(qiita_config.valid_upload_extension),
max_upload_size=qiita_config.max_upload_size, level=level,
message=message, remote_url=remote_url,
remote_files=remote_files,
files=get_files_from_uploads_folders(str(study_id)))
def test_update_prep_template(self):
fd, fp = mkstemp(suffix=".txt")
close(fd)
with open(fp, 'w') as f:
f.write("sample_name\tnew_col\n1.SKD6.640190\tnew_value")
job = self._create_job('update_prep_template', {'prep_template': 1,
'template_fp': fp})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertEqual(PrepTemplate(1)['1.SKD6.640190']['new_col'],
'new_value')
obs = r_client.get("prep_template_1")
self.assertIsNotNone(obs)
obs = loads(obs)
self.assertItemsEqual(obs, ['job_id', 'alert_type', 'alert_msg'])
self.assertEqual(obs['job_id'], job.id)
self.assertEqual(obs['alert_type'], 'warning')
self.assertIn('The following columns have been added to the existing '
'template: new_col', obs['alert_msg'])
def get(self):
user = self.current_user
is_local_request = is_localhost(self.request.headers['host'])
uanalyses = user.shared_analyses | user.private_analyses
user_analysis_ids = set([a.id for a in uanalyses])
panalyses = Analysis.get_by_status('public')
public_analysis_ids = set([a.id for a in panalyses])
public_analysis_ids = public_analysis_ids - user_analysis_ids
user_analyses = generate_analysis_list(user_analysis_ids)
public_analyses = generate_analysis_list(public_analysis_ids, True)
dlop = partial(download_link_or_path, is_local_request)
messages = {'info': '', 'danger': ''}
for analysis_id in user_analysis_ids:
job_info = r_client.get('analysis_delete_%d' % analysis_id)
if job_info:
job_info = defaultdict(lambda: '', loads(job_info))
job_id = job_info['job_id']
job = ProcessingJob(job_id)
job_status = job.status
processing = job_status not in ('success', 'error')
if processing:
messages['info'] += (
'Analysis %s is being deleted<br/>' % analysis_id)
elif job_status == 'error':
messages['danger'] += (
job.log.msg.replace('\n', '<br/>') + '<br/>')
else:
if job_info['alert_type'] not in messages:
messages[job_info['alert_type']] = []
messages[job_info['alert_type']] += (
job.log.msg.replace('\n', '<br/>') + '<br/>')
self.render("list_analyses.html", user_analyses=user_analyses,
public_analyses=public_analyses, messages=messages,
dlop=dlop)
def get_release_info(study_status='public'):
"""Returns the studies and the archive release details
Parameters
----------
study_status : str, optional
The study status to search for. Note that this should always be set
to 'public' but having this exposed helps with testing. The other
options are 'private' and 'sandbox'
Returns
------
((str, str, str), (str, str, str))
The release MD5, filepath and timestamp
"""
portal = qiita_config.portal
md5sum = r_client.get('%s:release:%s:md5sum' % (portal, study_status))
filepath = r_client.get('%s:release:%s:filepath' % (portal, study_status))
timestamp = r_client.get('%s:release:%s:time' % (portal, study_status))
# replacing None values for empty strings as the text is displayed nicely
# in the GUI
if md5sum is None:
md5sum = ''
if filepath is None:
filepath = ''
if timestamp is None:
timestamp = ''
biom_metadata_release = ((md5sum, filepath, timestamp))
md5sum = r_client.get('release-archive:md5sum')
filepath = r_client.get('release-archive:filepath')
timestamp = r_client.get('release-archive:time')
# replacing None values for empty strings as the text is displayed nicely
# in the GUI
if md5sum is None:
md5sum = ''
if filepath is None:
filepath = ''
if timestamp is None:
timestamp = ''
archive_release = ((md5sum, filepath, timestamp))
return (biom_metadata_release, archive_release)
def sample_template_overview_handler_get_request(study_id, user):
# Check if the current user has access to the sample template
sample_template_checks(study_id, user)
# Check if the sample template exists
exists = SampleTemplate.exists(study_id)
# The following information should always be provided:
# The files that have been uploaded to the system and can be a
# sample template file
files = [
f for _, f, _ in get_files_from_uploads_folders(study_id)
if f.endswith(('txt', 'tsv', 'xlsx'))
]
# If there is a job associated with the sample information, the job id
job = None
job_info = r_client.get(SAMPLE_TEMPLATE_KEY_FORMAT % study_id)
if job_info:
job = loads(job_info)['job_id']
# Specific information if it exists or not:
data_types = []
st_fp_id = None
old_files = []
num_samples = 0
num_cols = 0
columns = []
specimen_id_column = None
sample_restrictions = ''
if exists:
# If it exists we need to provide:
# The id of the sample template file so the user can download it and
# the list of old filepaths
st = SampleTemplate(study_id)
all_st_files = st.get_filepaths()
# The current sample template file is the first one in the list
# (pop(0)) and we are interested only in the id ([0])
st_fp_id = all_st_files.pop(0)[0]
# For the old filepaths we are only interested in their basename
old_files = [basename(fp) for _, fp in all_st_files]
# The number of samples - this is a space efficient way of counting
# the number of samples. Doing len(list(st.keys())) creates a list
# that we are not using
num_samples = sum(1 for _ in st.keys())
columns = st.categories
# The number of columns
num_cols = len(columns)
specimen_id_column = Study(study_id).specimen_id_column
_, sample_restrictions = st.validate_restrictions()
else:
# It doesn't exist, we also need to provide the data_types in case
# the user uploads a QIIME mapping file
data_types = sorted(data_types_get_req()['data_types'])
return {
'exists': exists,
'uploaded_files': files,
'data_types': data_types,
'user_can_edit': Study(study_id).can_edit(user),
'job': job,
'download_id': st_fp_id,
'old_files': old_files,
'num_samples': num_samples,
'num_columns': num_cols,
'columns': columns,
'sample_restrictions': sample_restrictions,
'specimen_id_column': specimen_id_column
}
def correct_redis_data(key, cmd, values_dict, user):
"""Corrects the data stored in the redis DB
Parameters
----------
key: str
The redis key to fix
cmd : qiita_db.software.Command
Command to use to create the processing job
values_dict : dict
Dictionary used to instantiate the parameters of the command
user : qiita_db.user. User
The user that will own the job
"""
info = r_client.get(key)
if info:
info = loads(info)
if info['job_id'] is not None:
if 'is_qiita_job' in info:
if info['is_qiita_job']:
try:
job = ProcessingJob(info['job_id'])
payload = {
'job_id': info['job_id'],
'alert_type': info['status'],
'alert_msg': info['alert_msg']
}
r_client.set(key, dumps(payload))
except (QiitaDBUnknownIDError, KeyError):
# We shomehow lost the information of this job
# Simply delete the key
r_client.delete(key)
else:
# These jobs don't contain any information on the live
# dump. We can safely delete the key
r_client.delete(key)
else:
# These jobs don't contain any information on the live
# dump. We can safely delete the key
r_client.delete(key)
else:
# Job is null, we have the information here
if info['status'] == 'success':
# In the success case no information is stored. We can
# safely delete the key
r_client.delete(key)
elif info['status'] == 'warning':
# In case of warning the key message stores the warning
# message. We need to create a new job, mark it as
# successful and store the error message as expected by
# the new structure
params = Parameters.load(cmd, values_dict=values_dict)
job = ProcessingJob.create(user, params)
job._set_status('success')
payload = {
'job_id': job.id,
'alert_type': 'warning',
'alert_msg': info['message']
}
r_client.set(key, dumps(payload))
else:
# The status is error. The key message stores the error
# message. We need to create a new job and mark it as
# failed with the given error message
params = Parameters.load(cmd, values_dict=values_dict)
job = ProcessingJob.create(user, params)
job._set_error(info['message'])
payload = {'job_id': job.id}
r_client.set(key, dumps(payload))
else:
# The key doesn't contain any information. Delete the key
r_client.delete(key)
def study_get_req(study_id, user_id):
"""Returns information available for the given study
Parameters
----------
study_id : int
Study id to get prep template info for
user_id : str
User requesting the info
Returns
-------
dict
Data types information in the form
{'status': status,
'message': message,
'info': dict of objects
status can be success, warning, or error depending on result
message has the warnings or errors
info contains study information seperated by data type, in the form
{col_name: value, ...} with value being a string, int, or list of
strings or ints
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
# Can only pass ids over API, so need to instantiate object
study = Study(study_id)
study_info = study.info
# Add needed info that is not part of the initial info pull
study_info['publication_doi'] = []
study_info['publication_pid'] = []
for pub, is_doi in study.publications:
if is_doi:
study_info['publication_doi'].append(pub)
else:
study_info['publication_pid'].append(pub)
study_info['study_id'] = study.id
study_info['study_title'] = study.title
study_info['shared_with'] = [s.id for s in study.shared_with]
study_info['status'] = study.status
study_info['ebi_study_accession'] = study.ebi_study_accession
study_info['ebi_submission_status'] = study.ebi_submission_status
# Clean up StudyPerson objects to string for display
pi = study_info['principal_investigator']
study_info['principal_investigator'] = {
'name': pi.name,
'email': pi.email,
'affiliation': pi.affiliation
}
lab_person = study_info['lab_person']
if lab_person:
study_info['lab_person'] = {
'name': lab_person.name,
'email': lab_person.email,
'affiliation': lab_person.affiliation
}
samples = study.sample_template
study_info['num_samples'] = 0 if samples is None else len(list(samples))
study_info['owner'] = study.owner.id
# Study.has_access no_public=True, will return True only if the user_id is
# the owner of the study or if the study is shared with the user_id
study_info['has_access_to_raw_data'] = study.has_access(
User(user_id), True)
study_info['show_biom_download_button'] = 'BIOM' in [
a.artifact_type for a in study.artifacts()
]
study_info['show_raw_download_button'] = any(
[True for pt in study.prep_templates() if pt.artifact is not None])
# getting study processing status from redis
processing = False
study_info['level'] = ''
study_info['message'] = ''
job_info = r_client.get(STUDY_KEY_FORMAT % study_id)
if job_info:
job_info = defaultdict(lambda: '', loads(job_info))
job_id = job_info['job_id']
job = ProcessingJob(job_id)
job_status = job.status
processing = job_status not in ('success', 'error')
if processing:
study_info['level'] = 'info'
study_info['message'] = 'This study is currently being processed'
elif job_status == 'error':
study_info['level'] = 'danger'
study_info['message'] = job.log.msg.replace('\n', '</br>')
else:
study_info['level'] = job_info['alert_type']
study_info['message'] = job_info['alert_msg'].replace(
'\n', '</br>')
return {
'status': 'success',
'message': '',
'study_info': study_info,
'editable': study.can_edit(User(user_id))
}
def correct_redis_data(key, cmd, values_dict, user):
"""Corrects the data stored in the redis DB
Parameters
----------
key: str
The redis key to fix
cmd : qiita_db.software.Command
Command to use to create the processing job
values_dict : dict
Dictionary used to instantiate the parameters of the command
user : qiita_db.user. User
The user that will own the job
"""
info = r_client.get(key)
if info:
info = loads(info)
if info['job_id'] is not None:
if 'is_qiita_job' in info:
if info['is_qiita_job']:
try:
job = ProcessingJob(info['job_id'])
payload = {'job_id': info['job_id'],
'alert_type': info['status'],
'alert_msg': info['alert_msg']}
r_client.set(key, dumps(payload))
except (QiitaDBUnknownIDError, KeyError):
# We shomehow lost the information of this job
# Simply delete the key
r_client.delete(key)
else:
# These jobs don't contain any information on the live
# dump. We can safely delete the key
r_client.delete(key)
else:
# These jobs don't contain any information on the live
# dump. We can safely delete the key
r_client.delete(key)
else:
# Job is null, we have the information here
if info['status'] == 'success':
# In the success case no information is stored. We can
# safely delete the key
r_client.delete(key)
elif info['status'] == 'warning':
# In case of warning the key message stores the warning
# message. We need to create a new job, mark it as
# successful and store the error message as expected by
# the new structure
params = Parameters.load(cmd, values_dict=values_dict)
job = ProcessingJob.create(user, params)
job._set_status('success')
payload = {'job_id': job.id,
'alert_type': 'warning',
'alert_msg': info['message']}
r_client.set(key, dumps(payload))
else:
# The status is error. The key message stores the error
# message. We need to create a new job and mark it as
# failed with the given error message
params = Parameters.load(cmd, values_dict=values_dict)
job = ProcessingJob.create(user, params)
job._set_error(info['message'])
payload = {'job_id': job.id}
r_client.set(key, dumps(payload))
else:
# The key doesn't contain any information. Delete the key
r_client.delete(key)
def test_prep_template_patch_req(self):
metadata = pd.DataFrame.from_dict(
{'SKD6.640190': {'center_name': 'ANL',
'target_subfragment': 'V4',
'center_project_name': 'Test Project',
'ebi_submission_accession': None,
'EMP_status': 'EMP',
'str_column': 'Value for sample 1',
'primer': 'GTGCCAGCMGCCGCGGTAA',
'barcode': 'GTCCGCAAGTTA',
'run_prefix': "s_G1_L001_sequences",
'platform': 'ILLUMINA',
'instrument_model': 'Illumina MiSeq',
'library_construction_protocol': 'AAAA',
'experiment_design_description': 'BBBB'}},
orient='index', dtype=str)
pt = PrepTemplate.create(metadata, Study(1), '16S')
# Update investigation type
obs = prep_template_patch_req(
'*****@*****.**', 'replace', '/%s/investigation_type' % pt.id,
'Cancer Genomics')
exp = {'status': 'success', 'message': ''}
self.assertEqual(obs, exp)
self.assertEqual(pt.investigation_type, 'Cancer Genomics')
# Update prep template data
obs = prep_template_patch_req(
'*****@*****.**', 'replace', '/%s/data' % pt.id, 'update.txt')
self.assertEqual(obs, exp)
obs = r_client.get('prep_template_%s' % pt.id)
self.assertIsNotNone(obs)
self._wait_for_parallel_job('prep_template_%s' % pt.id)
# Delete a prep template column
obs = prep_template_patch_req(
'*****@*****.**', 'remove',
'/%s/10/columns/target_subfragment/' % pt.id)
exp = {'status': 'success', 'message': '', 'row_id': '10'}
self.assertEqual(obs, exp)
self._wait_for_parallel_job('prep_template_%s' % pt.id)
self.assertNotIn('target_subfragment', pt.categories())
# Change the name of the prep template
obs = prep_template_patch_req(
'*****@*****.**', 'replace', '/%s/name' % pt.id, ' My New Name ')
exp = {'status': 'success', 'message': ''}
self.assertEqual(obs, exp)
self.assertEqual(pt.name, 'My New Name')
# Test all the errors
# Operation not supported
obs = prep_template_patch_req(
'*****@*****.**', 'add', '/1/investigation_type',
'Cancer Genomics')
exp = {'status': 'error',
'message': 'Operation "add" not supported. '
'Current supported operations: replace, remove',
'row_id': '0'}
self.assertEqual(obs, exp)
# Incorrect path parameter
obs = prep_template_patch_req(
'*****@*****.**', 'replace', '/investigation_type',
'Cancer Genomics')
exp = {'status': 'error',
'message': 'Incorrect path parameter'}
self.assertEqual(obs, exp)
# Incorrect attribute
obs = prep_template_patch_req(
'*****@*****.**', 'replace', '/1/other_attribute',
'Cancer Genomics')
exp = {'status': 'error',
'message': 'Attribute "other_attribute" not found. '
'Please, check the path parameter'}
self.assertEqual(obs, exp)
# User doesn't have access
obs = prep_template_patch_req(
'*****@*****.**', 'replace', '/%s/investigation_type' % pt.id,
'Cancer Genomics')
exp = {'status': 'error',
'message': 'User does not have access to study'}
self.assertEqual(obs, exp)
# File does not exists
obs = prep_template_patch_req(
'*****@*****.**', 'replace', '/1/data', 'unknown_file.txt')
exp = {'status': 'error',
'message': 'file does not exist',
'file': 'unknown_file.txt'}
self.assertEqual(obs, exp)
def _wait_for_parallel_job(self, key):
# This is needed so the clean up works - this is a distributed system
# so we need to make sure that all processes are done before we reset
# the test database
obs = r_client.get(key)
wait_for_processing_job(loads(obs)['job_id'])
def prep_template_ajax_get_req(user_id, prep_id):
"""Returns the prep tempalte information needed for the AJAX handler
Parameters
----------
user_id : str
The user id
prep_id : int
The prep template id
Returns
-------
dict of {str: object}
A dictionary with the following keys:
- status: str, whether the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
- name: str, the name of the prep template
- files: list of str, the files available to update the prep template
- download_prep: int, the filepath_id of the prep file
- download_qiime, int, the filepath_id of the qiime mapping file
- num_samples: int, the number of samples present in the template
- num_columns: int, the number of columns present in the template
- investigation_type: str, the investigation type of the template
- ontology: str, dict of {str, list of str} containing the information
of the ENA ontology
- artifact_attached: bool, whether the template has an artifact
attached
- study_id: int, the study id of the template
"""
pt = PrepTemplate(prep_id)
name = pt.name
# Initialize variables here
processing = False
alert_type = ''
alert_msg = ''
job_info = r_client.get(PREP_TEMPLATE_KEY_FORMAT % prep_id)
if job_info:
job_info = defaultdict(lambda: '', loads(job_info))
job_id = job_info['job_id']
job = ProcessingJob(job_id)
job_status = job.status
processing = job_status not in ('success', 'error')
if processing:
alert_type = 'info'
alert_msg = 'This prep template is currently being updated'
elif job_status == 'error':
alert_type = 'danger'
alert_msg = job.log.msg.replace('\n', '</br>')
else:
alert_type = job_info['alert_type']
alert_msg = job_info['alert_msg'].replace('\n', '</br>')
artifact_attached = pt.artifact is not None
study_id = pt.study_id
files = [f for _, f, _ in get_files_from_uploads_folders(study_id)
if f.endswith(('.txt', '.tsv'))]
# The call to list is needed because keys is an iterator
num_samples = len(list(pt.keys()))
num_columns = len(pt.categories())
investigation_type = pt.investigation_type
download_prep_id = None
download_qiime_id = None
other_filepaths = []
for fp_id, fp in pt.get_filepaths():
fp = basename(fp)
if 'qiime' in fp:
if download_qiime_id is None:
download_qiime_id = fp_id
else:
if download_prep_id is None:
download_prep_id = fp_id
else:
other_filepaths.append(fp)
ontology = _get_ENA_ontology()
editable = Study(study_id).can_edit(User(user_id)) and not processing
return {'status': 'success',
'message': '',
'name': name,
'files': files,
'download_prep_id': download_prep_id,
'download_qiime_id': download_qiime_id,
'other_filepaths': other_filepaths,
'num_samples': num_samples,
'num_columns': num_columns,
'investigation_type': investigation_type,
'ontology': ontology,
'artifact_attached': artifact_attached,
'study_id': study_id,
'editable': editable,
'data_type': pt.data_type(),
'alert_type': alert_type,
'is_submitted_to_ebi': pt.is_submitted_to_ebi,
'alert_message': alert_msg}
def prep_template_ajax_get_req(user_id, prep_id):
"""Returns the prep tempalte information needed for the AJAX handler
Parameters
----------
user_id : str
The user id
prep_id : int
The prep template id
Returns
-------
dict of {str: object}
A dictionary with the following keys:
- status: str, whether the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
- name: str, the name of the prep template
- files: list of str, the files available to update the prep template
- download_prep: int, the filepath_id of the prep file
- download_qiime, int, the filepath_id of the qiime mapping file
- num_samples: int, the number of samples present in the template
- num_columns: int, the number of columns present in the template
- investigation_type: str, the investigation type of the template
- ontology: str, dict of {str, list of str} containing the information
of the ENA ontology
- artifact_attached: bool, whether the template has an artifact
attached
- study_id: int, the study id of the template
"""
pt = PrepTemplate(prep_id)
name = pt.name
deprecated = pt.deprecated
# Initialize variables here
processing = False
alert_type = ''
alert_msg = ''
job_info = r_client.get(PREP_TEMPLATE_KEY_FORMAT % prep_id)
if job_info:
job_info = defaultdict(lambda: '', loads(job_info))
job_id = job_info['job_id']
job = ProcessingJob(job_id)
job_status = job.status
processing = job_status not in ('success', 'error')
if processing:
alert_type = 'info'
alert_msg = 'This prep template is currently being updated'
elif job_status == 'error':
alert_type = 'danger'
alert_msg = job.log.msg.replace('\n', '</br>')
else:
alert_type = job_info['alert_type']
alert_msg = job_info['alert_msg'].replace('\n', '</br>')
artifact_attached = pt.artifact is not None
study_id = pt.study_id
files = [
f for _, f, _ in get_files_from_uploads_folders(study_id)
if f.endswith(('.txt', '.tsv', '.xlsx'))
]
# The call to list is needed because keys is an iterator
num_samples = len(list(pt.keys()))
num_columns = len(pt.categories())
investigation_type = pt.investigation_type
download_prep_id = None
download_qiime_id = None
other_filepaths = []
for fp_id, fp in pt.get_filepaths():
fp = basename(fp)
if 'qiime' in fp:
if download_qiime_id is None:
download_qiime_id = fp_id
else:
if download_prep_id is None:
download_prep_id = fp_id
else:
other_filepaths.append(fp)
ontology = _get_ENA_ontology()
editable = Study(study_id).can_edit(User(user_id)) and not processing
success, restrictions = pt.validate_restrictions()
return {
'status': 'success',
'message': '',
'name': name,
'files': files,
'download_prep_id': download_prep_id,
'download_qiime_id': download_qiime_id,
'other_filepaths': other_filepaths,
'num_samples': num_samples,
'num_columns': num_columns,
'investigation_type': investigation_type,
'ontology': ontology,
'artifact_attached': artifact_attached,
'study_id': study_id,
'editable': editable,
'data_type': pt.data_type(),
'alert_type': alert_type,
'is_submitted_to_ebi': pt.is_submitted_to_ebi,
'prep_restrictions': restrictions,
'samples': sorted(list(pt.keys())),
'deprecated': deprecated,
'alert_message': alert_msg
}
def test_sample_template_handler_patch_request(self):
user = User('*****@*****.**')
# Test user doesn't have access
with self.assertRaisesRegexp(HTTPError,
'User does not have access to study'):
sample_template_handler_patch_request(
User('*****@*****.**'), "remove",
"/1/columns/season_environment/")
# Test study doesn't exist
with self.assertRaisesRegexp(HTTPError, 'Study does not exist'):
sample_template_handler_patch_request(
user, "remove", "/10000/columns/season_environment/")
# Test sample template doesn't exist
new_study = self._create_study('Patching test')
with self.assertRaisesRegexp(
HTTPError,
"Study %s doesn't have sample information" % new_study.id):
sample_template_handler_patch_request(
user, "remove",
"/%s/columns/season_environment/" % new_study.id)
# Test wrong operation value
with self.assertRaisesRegexp(
HTTPError, 'Operation add not supported. Current supported '
'operations: remove.'):
sample_template_handler_patch_request(
user, 'add', '/1/columns/season_environment')
# Test wrong path parameter < 2
with self.assertRaisesRegexp(HTTPError, 'Incorrect path parameter'):
sample_template_handler_patch_request(user, 'ignored', '1')
# TESTS FOR OPERATION: remove
# Test wrong path parameter
with self.assertRaisesRegexp(HTTPError, 'Incorrect path parameter'):
sample_template_handler_patch_request(user, 'remove',
'/1/season_environment/')
# Add sample information to the new study so we can delete one column
# without affecting the other tests
md = pd.DataFrame.from_dict(
{'Sample1': {
'col1': 'val1',
'col2': 'val2'
}},
orient='index',
dtype=str)
st = SampleTemplate.create(md, new_study)
# Test success
obs = sample_template_handler_patch_request(
user, "remove", "/%s/columns/col2/" % new_study.id)
self.assertEqual(obs.keys(), ['job'])
job_info = r_client.get('sample_template_%s' % new_study.id)
self.assertIsNotNone(job_info)
# Wait until the job is done
wait_for_processing_job(loads(job_info)['job_id'])
self.assertNotIn('col2', st.categories())
# TESTS FOR OPERATION: replace
# Test incorrect path parameter with replace
with self.assertRaisesRegexp(HTTPError, 'Incorrect path parameter'):
sample_template_handler_patch_request(user, "replace", "/1/")
# Test attribute not found
with self.assertRaisesRegexp(HTTPError, 'Attribute name not found'):
sample_template_handler_patch_request(user, "replace", "/1/name")
# Test missing value
with self.assertRaisesRegexp(
HTTPError, 'Value is required when updating sample '
'information'):
sample_template_handler_patch_request(user, "replace", "/1/data")
# Test file doesn't exist
with self.assertRaisesRegexp(HTTPError, 'Filepath not found'):
sample_template_handler_patch_request(user,
"replace",
"/1/data",
req_value='DoesNotExist')
# Test success
obs = sample_template_handler_patch_request(
user, "replace", "/1/data", req_value='uploaded_file.txt')
self.assertEqual(obs.keys(), ['job'])
job_info = r_client.get('sample_template_1')
self.assertIsNotNone(job_info)
# Wait until the job is done
wait_for_processing_job(loads(job_info)['job_id'])
def test_prep_template_patch_req(self):
metadata = pd.DataFrame.from_dict(
{
'SKD6.640190': {
'center_name': 'ANL',
'target_subfragment': 'V4',
'center_project_name': 'Test Project',
'ebi_submission_accession': None,
'EMP_status': 'EMP',
'str_column': 'Value for sample 1',
'primer': 'GTGCCAGCMGCCGCGGTAA',
'barcode': 'GTCCGCAAGTTA',
'run_prefix': "s_G1_L001_sequences",
'platform': 'Illumina',
'instrument_model': 'Illumina MiSeq',
'library_construction_protocol': 'AAAA',
'experiment_design_description': 'BBBB'
}
},
orient='index',
dtype=str)
pt = PrepTemplate.create(metadata, Study(1), '16S')
# Update investigation type
obs = prep_template_patch_req('*****@*****.**', 'replace',
'/%s/investigation_type' % pt.id,
'Cancer Genomics')
exp = {'status': 'success', 'message': ''}
self.assertEqual(obs, exp)
self.assertEqual(pt.investigation_type, 'Cancer Genomics')
# Update prep template data
obs = prep_template_patch_req('*****@*****.**', 'replace',
'/%s/data' % pt.id, 'update.txt')
self.assertEqual(obs, exp)
obs = r_client.get('prep_template_%s' % pt.id)
self.assertIsNotNone(obs)
self._wait_for_parallel_job('prep_template_%s' % pt.id)
# Delete a prep template column
obs = prep_template_patch_req(
'*****@*****.**', 'remove',
'/%s/10/columns/target_subfragment/' % pt.id)
exp = {'status': 'success', 'message': '', 'row_id': '10'}
self.assertEqual(obs, exp)
self._wait_for_parallel_job('prep_template_%s' % pt.id)
self.assertNotIn('target_subfragment', pt.categories())
# Change the name of the prep template
obs = prep_template_patch_req('*****@*****.**', 'replace',
'/%s/name' % pt.id, ' My New Name ')
exp = {'status': 'success', 'message': ''}
self.assertEqual(obs, exp)
self.assertEqual(pt.name, 'My New Name')
# Test all the errors
# Operation not supported
obs = prep_template_patch_req('*****@*****.**', 'add',
'/1/investigation_type',
'Cancer Genomics')
exp = {
'status': 'error',
'message': 'Operation "add" not supported. '
'Current supported operations: replace, remove',
'row_id': '0'
}
self.assertEqual(obs, exp)
# Incorrect path parameter
obs = prep_template_patch_req('*****@*****.**', 'replace',
'/investigation_type', 'Cancer Genomics')
exp = {'status': 'error', 'message': 'Incorrect path parameter'}
self.assertEqual(obs, exp)
# Incorrect attribute
obs = prep_template_patch_req('*****@*****.**', 'replace',
'/1/other_attribute', 'Cancer Genomics')
exp = {
'status':
'error',
'message':
'Attribute "other_attribute" not found. '
'Please, check the path parameter'
}
self.assertEqual(obs, exp)
# User doesn't have access
obs = prep_template_patch_req('*****@*****.**', 'replace',
'/%s/investigation_type' % pt.id,
'Cancer Genomics')
exp = {
'status': 'error',
'message': 'User does not have access to study'
}
self.assertEqual(obs, exp)
# File does not exists
obs = prep_template_patch_req('*****@*****.**', 'replace', '/1/data',
'unknown_file.txt')
exp = {
'status': 'error',
'message': 'file does not exist',
'file': 'unknown_file.txt'
}
self.assertEqual(obs, exp)
def test_sample_template_handler_patch_request(self):
user = User('*****@*****.**')
# Test user doesn't have access
with self.assertRaisesRegexp(HTTPError,
'User does not have access to study'):
sample_template_handler_patch_request(
User('*****@*****.**'), "remove",
"/1/columns/season_environment/")
# Test study doesn't exist
with self.assertRaisesRegexp(HTTPError, 'Study does not exist'):
sample_template_handler_patch_request(
user, "remove", "/10000/columns/season_environment/")
# Test sample template doesn't exist
new_study = self._create_study('Patching test')
with self.assertRaisesRegexp(HTTPError,
"Study %s doesn't have sample information"
% new_study.id):
sample_template_handler_patch_request(
user, "remove", "/%s/columns/season_environment/"
% new_study.id)
# Test wrong operation value
with self.assertRaisesRegexp(
HTTPError, 'Operation add not supported. Current supported '
'operations: remove.'):
sample_template_handler_patch_request(
user, 'add', '/1/columns/season_environment')
# Test wrong path parameter < 2
with self.assertRaisesRegexp(HTTPError, 'Incorrect path parameter'):
sample_template_handler_patch_request(user, 'ignored', '1')
# TESTS FOR OPERATION: remove
# Test wrong path parameter
with self.assertRaisesRegexp(HTTPError, 'Incorrect path parameter'):
sample_template_handler_patch_request(
user, 'remove', '/1/season_environment/')
# Add sample information to the new study so we can delete one column
# without affecting the other tests
md = pd.DataFrame.from_dict(
{'Sample1': {'col1': 'val1', 'col2': 'val2'}},
orient='index', dtype=str)
st = SampleTemplate.create(md, new_study)
# Test success
obs = sample_template_handler_patch_request(
user, "remove", "/%s/columns/col2/"
% new_study.id)
self.assertEqual(obs.keys(), ['job'])
job_info = r_client.get('sample_template_%s' % new_study.id)
self.assertIsNotNone(job_info)
# Wait until the job is done
wait_for_processing_job(loads(job_info)['job_id'])
self.assertNotIn('col2', st.categories())
# TESTS FOR OPERATION: replace
# Test incorrect path parameter with replace
with self.assertRaisesRegexp(HTTPError, 'Incorrect path parameter'):
sample_template_handler_patch_request(user, "replace", "/1/")
# Test attribute not found
with self.assertRaisesRegexp(HTTPError, 'Attribute name not found'):
sample_template_handler_patch_request(user, "replace", "/1/name")
# Test missing value
with self.assertRaisesRegexp(HTTPError,
'Value is required when updating sample '
'information'):
sample_template_handler_patch_request(user, "replace", "/1/data")
# Test file doesn't exist
with self.assertRaisesRegexp(HTTPError, 'Filepath not found'):
sample_template_handler_patch_request(user, "replace", "/1/data",
req_value='DoesNotExist')
# Test success
obs = sample_template_handler_patch_request(
user, "replace", "/1/data", req_value='uploaded_file.txt')
self.assertEqual(obs.keys(), ['job'])
job_info = r_client.get('sample_template_1')
self.assertIsNotNone(job_info)
# Wait until the job is done
wait_for_processing_job(loads(job_info)['job_id'])
def _wait_for_parallel_job(self, key):
# This is needed so the clean up works - this is a distributed system
# so we need to make sure that all processes are done before we reset
# the test database
obs = r_client.get(key)
wait_for_processing_job(loads(obs)['job_id'])
def study_get_req(study_id, user_id):
"""Returns information available for the given study
Parameters
----------
study_id : int
Study id to get prep template info for
user_id : str
User requesting the info
Returns
-------
dict
Data types information in the form
{'status': status,
'message': message,
'info': dict of objects
status can be success, warning, or error depending on result
message has the warnings or errors
info contains study information seperated by data type, in the form
{col_name: value, ...} with value being a string, int, or list of
strings or ints
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
# Can only pass ids over API, so need to instantiate object
study = Study(study_id)
study_info = study.info
# Add needed info that is not part of the initial info pull
study_info['publication_doi'] = []
study_info['publication_pid'] = []
for pub, is_doi in study.publications:
if is_doi:
study_info['publication_doi'].append(pub)
else:
study_info['publication_pid'].append(pub)
study_info['study_id'] = study.id
study_info['study_title'] = study.title
study_info['shared_with'] = [s.id for s in study.shared_with]
study_info['status'] = study.status
study_info['ebi_study_accession'] = study.ebi_study_accession
study_info['ebi_submission_status'] = study.ebi_submission_status
# Clean up StudyPerson objects to string for display
pi = study_info['principal_investigator']
study_info['principal_investigator'] = {
'name': pi.name,
'email': pi.email,
'affiliation': pi.affiliation}
lab_person = study_info['lab_person']
if lab_person:
study_info['lab_person'] = {
'name': lab_person.name,
'email': lab_person.email,
'affiliation': lab_person.affiliation}
samples = study.sample_template
study_info['num_samples'] = 0 if samples is None else len(list(samples))
study_info['owner'] = study.owner.id
# Study.has_access no_public=True, will return True only if the user_id is
# the owner of the study or if the study is shared with the user_id
study_info['has_access_to_raw_data'] = study.has_access(
User(user_id), True)
study_info['show_biom_download_button'] = 'BIOM' in [
a.artifact_type for a in study.artifacts()]
study_info['show_raw_download_button'] = any([
True for pt in study.prep_templates() if pt.artifact is not None])
# getting study processing status from redis
processing = False
study_info['level'] = ''
study_info['message'] = ''
job_info = r_client.get(STUDY_KEY_FORMAT % study_id)
if job_info:
job_info = defaultdict(lambda: '', loads(job_info))
job_id = job_info['job_id']
job = ProcessingJob(job_id)
job_status = job.status
processing = job_status not in ('success', 'error')
if processing:
study_info['level'] = 'info'
study_info['message'] = 'This study is currently being processed'
elif job_status == 'error':
study_info['level'] = 'danger'
study_info['message'] = job.log.msg.replace('\n', '</br>')
else:
study_info['level'] = job_info['alert_type']
study_info['message'] = job_info['alert_msg'].replace(
'\n', '</br>')
return {'status': 'success',
'message': '',
'study_info': study_info,
'editable': study.can_edit(User(user_id))}