def test_status_change(self):
new_status = 'new status'
r_client.set(self.test_id, json.dumps(self.test_job_info))
obs = _status_change(self.test_id, new_status)
self.assertEqual(obs, self.test_job_info['status'])
def artifact_delete_req(artifact_id, user_id):
"""Deletes the artifact
Parameters
----------
artifact_id : int
Artifact being acted on
user_id : str
The user requesting the action
Returns
-------
dict
Status of action, in the form {'status': status, 'message': msg}
status: status of the action, either success or error
message: Human readable message for status
"""
pd = Artifact(int(artifact_id))
pt_id = pd.prep_templates[0].id
access_error = check_access(pd.study.id, user_id)
if access_error:
return access_error
job_id = safe_submit(user_id, delete_artifact, artifact_id)
r_client.set(PREP_TEMPLATE_KEY_FORMAT % pt_id,
dumps({'job_id': job_id}))
return {'status': 'success',
'message': ''}
def artifact_post_req(user, artifact_id):
"""Deletes the artifact
Parameters
----------
user : qiita_db.user.User
The user requesting the action
artifact_id : int
Id of the artifact being deleted
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
check_artifact_access(user, artifact)
analysis = artifact.analysis
if analysis:
# Do something when deleting in the analysis part to keep track of it
redis_key = "analysis_%s" % analysis.id
else:
pt_id = artifact.prep_templates[0].id
redis_key = PREP_TEMPLATE_KEY_FORMAT % pt_id
job_id = safe_submit(user.id, delete_artifact, artifact_id)
r_client.set(redis_key, dumps({'job_id': job_id, 'is_qiita_job': False}))
def test_status_change(self):
new_status = 'new status'
r_client.set(self.test_id, json.dumps(self.test_job_info))
obs = _status_change(self.test_id, new_status)
self.assertEqual(obs, self.test_job_info['status'])
def sample_template_patch_request(user_id,
req_op,
req_path,
req_value=None,
req_from=None):
"""Modifies an attribute of the artifact
Parameters
----------
user_id : str
The id of the user performing the patch operation
req_op : str
The operation to perform on the artifact
req_path : str
The prep information and attribute to patch
req_value : str, optional
The value that needs to be modified
req_from : str, optional
The original path of the element
Returns
-------
dict of {str, str}
A dictionary with the following keys:
- status: str, whether if the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
"""
if req_op == 'remove':
req_path = [v for v in req_path.split('/') if v]
if len(req_path) != 3:
return {'status': 'error', 'message': 'Incorrect path parameter'}
st_id = req_path[0]
attribute = req_path[1]
attr_id = req_path[2]
# Check if the user actually has access to the template
st = SampleTemplate(st_id)
access_error = check_access(st.study_id, user_id)
if access_error:
return access_error
# Offload the deletion of the sample or column to the cluster
job_id = safe_submit(user_id, delete_sample_or_column, SampleTemplate,
int(st_id), attribute, attr_id)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % st_id,
dumps({'job_id': job_id}))
return {'status': 'success', 'message': ''}
else:
return {
'status':
'error',
'message':
'Operation "%s" not supported. '
'Current supported operations: remove' % req_op
}
def artifact_delete_req(artifact_id, user_id):
"""Deletes the artifact
Parameters
----------
artifact_id : int
Artifact being acted on
user_id : str
The user requesting the action
Returns
-------
dict
Status of action, in the form {'status': status, 'message': msg}
status: status of the action, either success or error
message: Human readable message for status
"""
pd = Artifact(int(artifact_id))
pt_id = pd.prep_templates[0].id
access_error = check_access(pd.study.id, user_id)
if access_error:
return access_error
job_id = safe_submit(user_id, delete_artifact, artifact_id)
r_client.set(PREP_TEMPLATE_KEY_FORMAT % pt_id, dumps({'job_id': job_id}))
return {'status': 'success', 'message': ''}
def sample_template_delete_req(study_id, user_id):
"""Deletes the sample template attached to the study
Parameters
----------
study_id : int
The current study object id
user_id : str
The current user object id
Returns
-------
dict
results dictonary in the format
{'status': status,
'message': msg}
status can be success, warning, or error depending on result
message has the warnings or errors
"""
exists = _check_sample_template_exists(int(study_id))
if exists['status'] != 'success':
return exists
access_error = check_access(int(study_id), user_id)
if access_error:
return access_error
# Offload the deletion of the sample template to the cluster
job_id = safe_submit(user_id, delete_sample_template, int(study_id))
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job_id}))
return {'status': 'success', 'message': ''}
def create_info(name, info_type, url=None, parent=None, id=None,
context=ctx_default, store=False):
"""Return a group object"""
id = str(uuid4()) if id is None else id
pubsub = _pubsub_key(id)
info = {'id': id,
'type': info_type,
'pubsub': pubsub,
'url': url,
'parent': parent,
'context': context,
'name': name,
'status': 'Queued' if info_type == 'job' else None,
'date_start': None,
'date_end': None,
'date_created': str(datetime.now()),
'result': None}
if store:
r_client.set(id, json_encode(info))
if parent is not None:
r_client.sadd(_children_key(parent), id)
return info
def _deposit_payload(redis_deets, payload):
"""Drop messages into redis
This is being defined inline as we need to use it multiple times, and
for an undiagnosed reason, having this function call it as a first
class function does not work.
"""
from json import dumps
from moi import r_client
job_id = redis_deets['job_id']
pubsub = redis_deets['pubsub']
messages = redis_deets['messages']
serialized = dumps(payload)
# First, we need to push the message on to the messages queue which is
# in place in the event of a race-condition where a websocket client
# may not be already listening to the pubsub.
r_client.rpush(messages, serialized)
# Next, in support of our "normal" and desired means of communicating,
# we "publish" our payload. Anyone listening on the pubsub will see
# this and fire an event (e.g., WebSocketHandler.callback)
r_client.publish(pubsub, serialized)
# Finally, we dump the payload keyed by job ID so that subsequent
# handlers who are not listening on the channel can examine the results
r_client.set(job_id, serialized, ex=86400 * 7) # expire at 1 week
def get_sample_template_processing_status(st_id):
job_info = r_client.get(SAMPLE_TEMPLATE_KEY_FORMAT % st_id)
if job_info:
job_info = loads(job_info)
job_id = job_info['job_id']
if job_id:
redis_info = loads(r_client.get(job_id))
processing = redis_info['status_msg'] == 'Running'
if processing:
alert_type = 'info'
alert_msg = 'This sample template is currently being processed'
elif redis_info['status_msg'] == 'Success':
alert_type = redis_info['return']['status']
alert_msg = redis_info['return']['message'].replace('\n',
'</br>')
payload = {'job_id': None,
'status': alert_type,
'message': alert_msg}
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % st_id,
dumps(payload))
else:
alert_type = redis_info['return']['status']
alert_msg = redis_info['return']['message'].replace('\n',
'</br>')
else:
processing = False
alert_type = job_info['status']
alert_msg = job_info['message'].replace('\n', '</br>')
else:
processing = False
alert_type = ''
alert_msg = ''
return processing, alert_type, alert_msg
def _deposit_payload(redis_deets, payload):
"""Drop messages into redis
This is being defined inline as we need to use it multiple times, and
for an undiagnosed reason, having this function call it as a first
class function does not work.
"""
from json import dumps
from moi import r_client
job_id = redis_deets['job_id']
pubsub = redis_deets['pubsub']
messages = redis_deets['messages']
serialized = dumps(payload)
# First, we need to push the message on to the messages queue which is
# in place in the event of a race-condition where a websocket client
# may not be already listening to the pubsub.
r_client.rpush(messages, serialized)
# Next, in support of our "normal" and desired means of communicating,
# we "publish" our payload. Anyone listening on the pubsub will see
# this and fire an event (e.g., WebSocketHandler.callback)
r_client.publish(pubsub, serialized)
# Finally, we dump the payload keyed by job ID so that subsequent
# handlers who are not listening on the channel can examine the results
r_client.set(job_id, serialized, ex=86400 * 7) # expire at 1 week
def create_info(name,
info_type,
url=None,
parent=None,
id=None,
context=ctx_default,
store=False):
"""Return a group object"""
id = str(uuid4()) if id is None else id
pubsub = _pubsub_key(id)
info = {
'id': id,
'type': info_type,
'pubsub': pubsub,
'url': url,
'parent': parent,
'context': context,
'name': name,
'status': 'Queued' if info_type == 'job' else None,
'date_start': None,
'date_end': None,
'date_created': str(datetime.now()),
'result': None
}
if store:
r_client.set(id, json_encode(info))
if parent is not None:
r_client.sadd(_children_key(parent), id)
return info
def sample_template_delete_req(study_id, user_id):
"""Deletes the sample template attached to the study
Parameters
----------
study_id : int
The current study object id
user_id : str
The current user object id
Returns
-------
dict
results dictonary in the format
{'status': status,
'message': msg}
status can be success, warning, or error depending on result
message has the warnings or errors
"""
exists = _check_sample_template_exists(int(study_id))
if exists['status'] != 'success':
return exists
access_error = check_access(int(study_id), user_id)
if access_error:
return access_error
# Offload the deletion of the sample template to the cluster
job_id = safe_submit(user_id, delete_sample_template, int(study_id))
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job_id}))
return {'status': 'success', 'message': ''}
def sample_template_post_req(study_id, user_id, data_type,
sample_template):
"""Creates the sample template from the given file
Parameters
----------
study_id : int
The current study object id
user_id : str
The current user object id
data_type : str
Data type for the sample template
sample_template : str
filename to use for creation
Returns
-------
dict
results dictonary in the format
{'status': status,
'message': msg,
'file': sample_template}
status can be success, warning, or error depending on result
message has the warnings or errors
file has the file name
"""
access_error = check_access(int(study_id), user_id)
if access_error:
return access_error
fp_rsp = check_fp(study_id, sample_template)
if fp_rsp['status'] != 'success':
# Unknown filepath, so return the error message
return fp_rsp
fp_rsp = fp_rsp['file']
# Define here the message and message level in case of success
msg = ''
status = 'success'
is_mapping_file = looks_like_qiime_mapping_file(fp_rsp)
if is_mapping_file and not data_type:
return {'status': 'error',
'message': 'Please, choose a data type if uploading a '
'QIIME mapping file',
'file': sample_template}
study = Study(int(study_id))
# Offload the creation of the sample template to the cluster
job_id = safe_submit(user_id, create_sample_template, fp_rsp, study,
is_mapping_file, data_type)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study.id,
dumps({'job_id': job_id}))
return {'status': status,
'message': msg,
'file': sample_template}
def sample_template_post_req(study_id, user_id, data_type, sample_template):
"""Creates the sample template from the given file
Parameters
----------
study_id : int
The current study object id
user_id : str
The current user object id
data_type : str
Data type for the sample template
sample_template : str
filename to use for creation
Returns
-------
dict
results dictonary in the format
{'status': status,
'message': msg,
'file': sample_template}
status can be success, warning, or error depending on result
message has the warnings or errors
file has the file name
"""
access_error = check_access(int(study_id), user_id)
if access_error:
return access_error
fp_rsp = check_fp(study_id, sample_template)
if fp_rsp['status'] != 'success':
# Unknown filepath, so return the error message
return fp_rsp
fp_rsp = fp_rsp['file']
# Define here the message and message level in case of success
msg = ''
status = 'success'
is_mapping_file = looks_like_qiime_mapping_file(fp_rsp)
if is_mapping_file and not data_type:
return {
'status': 'error',
'message': 'Please, choose a data type if uploading a '
'QIIME mapping file',
'file': sample_template
}
study = Study(int(study_id))
# Offload the creation of the sample template to the cluster
job_id = safe_submit(user_id, create_sample_template, fp_rsp, study,
is_mapping_file, data_type)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study.id,
dumps({'job_id': job_id}))
return {'status': status, 'message': msg, 'file': sample_template}
def test_status_change(self):
new_status = 'new status'
r_client.set(self.test_id, json.dumps(self.test_job_info))
# _status_change will return the old status
obs = _status_change(self.test_id, new_status)
self.assertEqual(obs, self.test_job_info['status'])
obs = json.loads(r_client.get(self.test_id))
self.assertEqual(obs['status'], new_status)
def sample_template_patch_request(user_id, req_op, req_path, req_value=None,
req_from=None):
"""Modifies an attribute of the artifact
Parameters
----------
user_id : str
The id of the user performing the patch operation
req_op : str
The operation to perform on the artifact
req_path : str
The prep information and attribute to patch
req_value : str, optional
The value that needs to be modified
req_from : str, optional
The original path of the element
Returns
-------
dict of {str, str}
A dictionary with the following keys:
- status: str, whether if the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
"""
if req_op == 'remove':
req_path = [v for v in req_path.split('/') if v]
if len(req_path) != 3:
return {'status': 'error',
'message': 'Incorrect path parameter'}
st_id = req_path[0]
attribute = req_path[1]
attr_id = req_path[2]
# Check if the user actually has access to the template
st = SampleTemplate(st_id)
access_error = check_access(st.study_id, user_id)
if access_error:
return access_error
# Offload the deletion of the sample or column to the cluster
job_id = safe_submit(user_id, delete_sample_or_column, SampleTemplate,
int(st_id), attribute, attr_id)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % st_id,
dumps({'job_id': job_id}))
return {'status': 'success', 'message': ''}
else:
return {'status': 'error',
'message': 'Operation "%s" not supported. '
'Current supported operations: remove' % req_op}
def test_status_change(self):
new_status = 'new status'
r_client.set(self.test_id, json.dumps(self.test_job_info))
# _status_change will return the old status
obs = _status_change(self.test_id, new_status)
self.assertEqual(obs, self.test_job_info['status'])
obs = json.loads(r_client.get(self.test_id))
self.assertEqual(obs['status'], new_status)
def setUp(self):
r_client.hset('user-id-map', 'testing', 'testing')
r_client.sadd('testing:children', 'a')
r_client.sadd('testing:children', 'b')
r_client.sadd('testing:children', 'c')
r_client.set('a', '{"type": "job", "id": "a", "name": "a"}')
r_client.set('b', '{"type": "job", "id": "b", "name": "b"}')
r_client.set('c', '{"type": "job", "id": "c", "name": "c"}')
r_client.set('d', '{"type": "job", "id": "d", "name": "other job"}')
r_client.set('e', '{"type": "job", "id": "e", "name": "other job e"}')
self.obj = Group('testing')
def setUp(self):
r_client.hset('user-id-map', 'testing', 'testing')
r_client.sadd('testing:children', 'a')
r_client.sadd('testing:children', 'b')
r_client.sadd('testing:children', 'c')
r_client.set('a', '{"type": "job", "id": "a", "name": "a"}')
r_client.set('b', '{"type": "job", "id": "b", "name": "b"}')
r_client.set('c', '{"type": "job", "id": "c", "name": "c"}')
r_client.set('d', '{"type": "job", "id": "d", "name": "other job"}')
r_client.set('e', '{"type": "job", "id": "e", "name": "other job e"}')
self.obj = Group('testing')
self.to_delete = ['testing', 'testing:jobs', 'testing:children',
'user-id-map', 'a', 'b', 'c', 'd', 'e']
def sample_template_put_req(study_id, user_id, sample_template):
"""Updates a sample template using the given file
Parameters
----------
study_id : int
The current study object id
user_id : str
The current user object id
sample_template : str
filename to use for updating
Returns
-------
dict
results dictonary in the format
{'status': status,
'message': msg,
'file': sample_template}
status can be success, warning, or error depending on result
message has the warnings or errors
file has the file name
"""
exists = _check_sample_template_exists(int(study_id))
if exists['status'] != 'success':
return exists
access_error = check_access(int(study_id), user_id)
if access_error:
return access_error
fp_rsp = check_fp(study_id, sample_template)
if fp_rsp['status'] != 'success':
# Unknown filepath, so return the error message
return fp_rsp
fp_rsp = fp_rsp['file']
msg = ''
status = 'success'
# Offload the update of the sample template to the cluster
job_id = safe_submit(user_id, update_sample_template, int(study_id),
fp_rsp)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job_id}))
return {'status': status,
'message': msg,
'file': sample_template}
def post(self):
if r_client.get('maintenance') is not None:
raise HTTPError(503, "Site is down for maintenance")
username = self.get_argument("username", "").strip().lower()
passwd = self.get_argument("password", "")
nextpage = self.get_argument("next", None)
if nextpage is None:
if "auth/" not in self.request.headers['Referer']:
nextpage = self.request.headers['Referer']
else:
nextpage = "%s/" % qiita_config.portal_dir
msg = ""
# check the user level
try:
if User(username).level == "unverified":
# email not verified so dont log in
msg = ("Email not verified. Please check your email and click "
"the verify link. You may need to check your spam "
"folder to find the email.<br/>If a verification email"
" has not arrived in 15 minutes, please email <a href='"
"mailto:[email protected]'>[email protected]</a>")
except QiitaDBUnknownIDError:
msg = "Unknown user"
except RuntimeError:
# means DB not available, so set maintenance mode and failover
r_client.set(
"maintenance", "Database connection unavailable, "
"please try again later.")
self.redirect("%s/" % qiita_config.portal_dir)
return
# Check the login information
login = None
try:
login = User.login(username, passwd)
except IncorrectEmailError:
msg = "Unknown user"
except IncorrectPasswordError:
msg = "Incorrect password"
except UnverifiedEmailError:
msg = "You have not verified your email address"
if login:
# everything good so log in
self.set_current_user(username)
self.redirect(nextpage)
else:
self.render("index.html", message=msg, level='danger')
def _submit(ctx, parent_id, name, url, func, *args, **kwargs):
"""Submit a function to a cluster
Parameters
----------
parent_id : str
The ID of the group that the job is a part of.
name : str
The name of the job
url : str
The handler that can take the results (e.g., /beta_diversity/)
func : function
The function to execute. Any returns from this function will be
serialized and deposited into Redis using the uuid for a key. This
function should raise if the method fails.
args : tuple or None
Any args for ``func``
kwargs : dict or None
Any kwargs for ``func``
Returns
-------
tuple, (str, str, AsyncResult)
The job ID, parent ID and the IPython's AsyncResult object of the job
"""
parent_info = r_client.get(parent_id)
if parent_info is None:
parent_info = create_info('unnamed', 'group', id=parent_id)
parent_id = parent_info['id']
r_client.set(parent_id, json.dumps(parent_info))
parent_pubsub_key = parent_id + ':pubsub'
job_info = create_info(name,
'job',
url=url,
parent=parent_id,
context=ctx.name,
store=True)
job_info['status'] = 'Queued'
job_id = job_info['id']
with r_client.pipeline() as pipe:
pipe.set(job_id, json.dumps(job_info))
pipe.publish(parent_pubsub_key, json.dumps({'add': [job_id]}))
pipe.execute()
ar = ctx.bv.apply_async(_redis_wrap, job_info, func, *args, **kwargs)
return job_id, parent_id, ar
def post(self):
if r_client.get('maintenance') is not None:
raise HTTPError(503, "Site is down for maintenance")
username = self.get_argument("username", "").strip().lower()
passwd = self.get_argument("password", "")
nextpage = self.get_argument("next", None)
if nextpage is None:
if "auth/" not in self.request.headers['Referer']:
nextpage = self.request.headers['Referer']
else:
nextpage = "%s/" % qiita_config.portal_dir
msg = ""
# check the user level
try:
if User(username).level == "unverified":
# email not verified so dont log in
msg = ("Email not verified. Please check your email and click "
"the verify link. You may need to check your spam "
"folder to find the email.<br/>If a verification email"
" has not arrived in 15 minutes, please email <a href='"
"mailto:[email protected]'>[email protected]</a>")
except QiitaDBUnknownIDError:
msg = "Unknown user"
except RuntimeError:
# means DB not available, so set maintenance mode and failover
r_client.set("maintenance", "Database connection unavailable, "
"please try again later.")
self.redirect("%s/" % qiita_config.portal_dir)
return
# Check the login information
login = None
try:
login = User.login(username, passwd)
except IncorrectEmailError:
msg = "Unknown user"
except IncorrectPasswordError:
msg = "Incorrect password"
except UnverifiedEmailError:
msg = "You have not verified your email address"
if login:
# everything good so log in
self.set_current_user(username)
self.redirect(nextpage)
else:
self.render("index.html", message=msg, level='danger')
def sample_template_put_req(study_id, user_id, sample_template):
"""Updates a sample template using the given file
Parameters
----------
study_id : int
The current study object id
user_id : str
The current user object id
sample_template : str
filename to use for updating
Returns
-------
dict
results dictonary in the format
{'status': status,
'message': msg,
'file': sample_template}
status can be success, warning, or error depending on result
message has the warnings or errors
file has the file name
"""
exists = _check_sample_template_exists(int(study_id))
if exists['status'] != 'success':
return exists
access_error = check_access(int(study_id), user_id)
if access_error:
return access_error
fp_rsp = check_fp(study_id, sample_template)
if fp_rsp['status'] != 'success':
# Unknown filepath, so return the error message
return fp_rsp
fp_rsp = fp_rsp['file']
msg = ''
status = 'success'
# Offload the update of the sample template to the cluster
job_id = safe_submit(user_id, update_sample_template, int(study_id),
fp_rsp)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job_id}))
return {'status': status, 'message': msg, 'file': sample_template}
def test_redis_wrap(self):
def foo(a, b, **kwargs):
return a + b
r_client.set(self.test_job_info['id'], json.dumps(self.test_job_info))
obs_ret = _redis_wrap(self.test_job_info, foo, 1, 2)
sleep(1)
obs = json.loads(r_client.get(self.test_job_info['id']))
self.assertEqual(obs['result'], 3)
self.assertEqual(obs_ret, obs['result'])
self.assertEqual(obs['status'], 'Success')
self.assertNotEqual(obs['date_start'], None)
self.assertNotEqual(obs['date_end'], None)
r_client.set(self.test_job_info['id'], json.dumps(self.test_job_info))
def test_redis_wrap(self):
def foo(a, b, **kwargs):
return a+b
r_client.set(self.test_job_info['id'], json.dumps(self.test_job_info))
obs_ret = _redis_wrap(self.test_job_info, foo, 1, 2)
sleep(1)
obs = json.loads(r_client.get(self.test_job_info['id']))
self.assertEqual(obs['result'], 3)
self.assertEqual(obs_ret, obs['result'])
self.assertEqual(obs['status'], 'Success')
self.assertNotEqual(obs['date_start'], None)
self.assertNotEqual(obs['date_end'], None)
r_client.set(self.test_job_info['id'], json.dumps(self.test_job_info))
def test_post_select_samples(self):
# just making sure that the key is not set in redis
r_client.delete('maintenance')
response = self.get('/auth/reset/')
self.assertEqual(response.code, 200)
self.assertIn(('<label for="newpass2" class="col-sm-2 '
'control-label">Repeat New Password'
'</label>'), response.body)
# not displaying due to maintenance
r_client.set('maintenance', 'This is my error message')
response = self.get('/auth/reset/')
self.assertEqual(response.code, 200)
self.assertNotIn(('<label for="newpass2" class="col-sm-2 '
'control-label">Repeat New Password'
'</label>'), response.body)
r_client.delete('maintenance')
def post(self):
if r_client.get('maintenance') is not None:
raise HTTPError(503, "Site is down for maintenance")
username = self.get_argument("username", "").strip().lower()
passwd = self.get_argument("password", "")
nextpage = self.get_argument("next", None)
if nextpage is None:
if "auth/" not in self.request.headers['Referer']:
nextpage = self.request.headers['Referer']
else:
nextpage = "/"
msg = ""
# check the user level
try:
if User(username).level == "unverified":
# email not verified so dont log in
msg = "Email not verified"
except QiitaDBUnknownIDError:
msg = "Unknown user"
except RuntimeError:
# means DB not available, so set maintenance mode and failover
r_client.set(
"maintenance", "Database connection unavailable, "
"please try again later.")
self.redirect("/")
return
# Check the login information
login = None
try:
login = User.login(username, passwd)
except IncorrectEmailError:
msg = "Unknown user"
except IncorrectPasswordError:
msg = "Incorrect password"
except UnverifiedEmailError:
msg = "You have not verified your email address"
if login:
# everything good so log in
self.set_current_user(username)
self.redirect(nextpage)
else:
self.render("index.html", message=msg, level='danger')
def test_post_select_samples(self):
# just making sure that the key is not set in redis
r_client.delete('maintenance')
response = self.get('/auth/reset/')
self.assertEqual(response.code, 200)
self.assertIn(('<label for="newpass2" class="col-sm-2 '
'control-label">Repeat New Password'
'</label>'), response.body)
# not displaying due to maintenance
r_client.set('maintenance', 'This is my error message')
response = self.get('/auth/reset/')
self.assertEqual(response.code, 200)
self.assertNotIn(('<label for="newpass2" class="col-sm-2 '
'control-label">Repeat New Password'
'</label>'), response.body)
r_client.delete('maintenance')
def test_post_select_samples(self):
# just making sure that the key is not set in redis
r_client.delete("maintenance")
response = self.get("/auth/reset/")
self.assertEqual(response.code, 200)
self.assertTrue(
('<label for="newpass2" class="col-sm-10 ' 'control-label">Repeat New Password' "</label>") in response.body
)
# not displaying due to maintenance
r_client.set("maintenance", "This is my error message")
response = self.get("/auth/reset/")
self.assertEqual(response.code, 200)
self.assertFalse(
('<label for="newpass2" class="col-sm-10 ' 'control-label">Repeat New Password' "</label>") in response.body
)
r_client.delete("maintenance")
def _submit(ctx, parent_id, name, url, func, *args, **kwargs):
"""Submit a function to a cluster
Parameters
----------
parent_id : str
The ID of the group that the job is a part of.
name : str
The name of the job
url : str
The handler that can take the results (e.g., /beta_diversity/)
func : function
The function to execute. Any returns from this function will be
serialized and deposited into Redis using the uuid for a key. This
function should raise if the method fails.
args : tuple or None
Any args for ``func``
kwargs : dict or None
Any kwargs for ``func``
Returns
-------
tuple, (str, str, AsyncResult)
The job ID, parent ID and the IPython's AsyncResult object of the job
"""
parent_info = r_client.get(parent_id)
if parent_info is None:
parent_info = create_info('unnamed', 'group', id=parent_id)
parent_id = parent_info['id']
r_client.set(parent_id, json.dumps(parent_info))
parent_pubsub_key = parent_id + ':pubsub'
job_info = create_info(name, 'job', url=url, parent=parent_id,
context=ctx.name, store=True)
job_info['status'] = 'Queued'
job_id = job_info['id']
with r_client.pipeline() as pipe:
pipe.set(job_id, json.dumps(job_info))
pipe.publish(parent_pubsub_key, json.dumps({'add': [job_id]}))
pipe.execute()
ar = ctx.bv.apply_async(_redis_wrap, job_info, func, *args, **kwargs)
return job_id, parent_id, ar
def post(self):
if r_client.get('maintenance') is not None:
raise HTTPError(503, "Site is down for maintenance")
username = self.get_argument("username", "").strip().lower()
passwd = self.get_argument("password", "")
nextpage = self.get_argument("next", None)
if nextpage is None:
if "auth/" not in self.request.headers['Referer']:
nextpage = self.request.headers['Referer']
else:
nextpage = "/"
msg = ""
# check the user level
try:
if User(username).level == "unverified":
# email not verified so dont log in
msg = "Email not verified"
except QiitaDBUnknownIDError:
msg = "Unknown user"
except RuntimeError:
# means DB not available, so set maintenance mode and failover
r_client.set("maintenance", "Database connection unavailable, "
"please try again later.")
self.redirect("/")
return
# Check the login information
login = None
try:
login = User.login(username, passwd)
except IncorrectEmailError:
msg = "Unknown user"
except IncorrectPasswordError:
msg = "Incorrect password"
except UnverifiedEmailError:
msg = "You have not verified your email address"
if login:
# everything good so log in
self.set_current_user(username)
self.redirect(nextpage)
else:
self.render("index.html", message=msg, level='danger')
def get_sample_template_processing_status(st_id):
job_info = r_client.get(SAMPLE_TEMPLATE_KEY_FORMAT % st_id)
if job_info:
job_info = loads(job_info)
job_id = job_info['job_id']
if job_id:
redis_info = r_client.get(job_id)
if redis_info:
redis_info = loads(redis_info)
processing = redis_info['status_msg'] == 'Running'
if processing:
alert_type = 'info'
alert_msg = ('This sample template is currently being '
'processed')
elif redis_info['status_msg'] == 'Success':
alert_type = redis_info['return']['status']
alert_msg = redis_info['return']['message'].replace(
'\n', '</br>')
payload = {
'job_id': None,
'status': alert_type,
'message': alert_msg
}
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % st_id,
dumps(payload))
else:
alert_type = redis_info['return']['status']
alert_msg = redis_info['return']['message'].replace(
'\n', '</br>')
else:
processing = False
alert_type = ''
alert_msg = ''
else:
processing = False
alert_type = job_info['status']
alert_msg = job_info['message'].replace('\n', '</br>')
else:
processing = False
alert_type = ''
alert_msg = ''
return processing, alert_type, alert_msg
def test_redis_wrap(self):
def foo(a, b, **kwargs):
return a+b
r_client.set(self.test_job_info['id'], json.dumps(self.test_job_info))
_redis_wrap(self.test_job_info, foo, 1, 2)
sleep(2)
obs = json.loads(r_client.get(self.test_job_info['id']))
self.assertEqual(obs['result'], 3)
self.assertEqual(obs['status'], 'Success')
self.assertNotEqual(obs['date_start'], None)
self.assertNotEqual(obs['date_end'], None)
r_client.set(self.test_job_info['id'], json.dumps(self.test_job_info))
_redis_wrap(self.test_job_info, foo, 1, 2, 3)
sleep(2)
obs = json.loads(r_client.get(self.test_job_info['id']))
self.assertEqual(obs['result'][0],
u'Traceback (most recent call last):\n')
self.assertEqual(obs['status'], 'Failed')
self.assertNotEqual(obs['date_start'], None)
self.assertNotEqual(obs['date_end'], None)
def test_redis_wrap(self):
def foo(a, b, **kwargs):
return a + b
r_client.set(self.test_job_info['id'], json.dumps(self.test_job_info))
_redis_wrap(self.test_job_info, foo, 1, 2)
sleep(2)
obs = json.loads(r_client.get(self.test_job_info['id']))
self.assertEqual(obs['result'], 3)
self.assertEqual(obs['status'], 'Success')
self.assertNotEqual(obs['date_start'], None)
self.assertNotEqual(obs['date_end'], None)
r_client.set(self.test_job_info['id'], json.dumps(self.test_job_info))
_redis_wrap(self.test_job_info, foo, 1, 2, 3)
sleep(2)
obs = json.loads(r_client.get(self.test_job_info['id']))
self.assertEqual(obs['result'][0],
u'Traceback (most recent call last):\n')
self.assertEqual(obs['status'], 'Failed')
self.assertNotEqual(obs['date_start'], None)
self.assertNotEqual(obs['date_end'], None)
def test_traverse_complex(self):
r_client.sadd('testing:children', 'd')
r_client.sadd('d:children', 'd_a', 'd_b')
r_client.set('d', '{"type": "group", "id": "d", "name": "d"}')
r_client.set('d_a', '{"type": "job", "id": "d_a", "name": "d_a"}')
r_client.set('d_b', '{"type": "job", "id": "d_b", "name": "d_b"}')
self.to_delete.append('d:children')
self.to_delete.append('d_a')
self.to_delete.append('d_b')
exp = {'a', 'b', 'c', 'd', 'd_a', 'd_b'}
obs = {obj['id'] for obj in self.obj.traverse('testing')}
self.assertEqual(obs, exp)
def test_traverse_complex(self):
r_client.sadd('testing:children', 'd')
r_client.sadd('d:children', 'd_a', 'd_b')
r_client.set('d', '{"type": "group", "id": "d", "name": "d"}')
r_client.set('d_a', '{"type": "job", "id": "d_a", "name": "d_a"}')
r_client.set('d_b', '{"type": "job", "id": "d_b", "name": "d_b"}')
self.to_delete.append('d:children')
self.to_delete.append('d_a')
self.to_delete.append('d_b')
exp = {'a', 'b', 'c', 'd', 'd_a', 'd_b'}
obs = {obj['id'] for obj in self.obj.traverse('testing')}
self.assertEqual(obs, exp)
def test_analysis_description_handler_get_request(self):
obs = analysis_description_handler_get_request(1, User('*****@*****.**'))
exp = {
'analysis_name': 'SomeAnalysis',
'analysis_id': 1,
'analysis_description': 'A test analysis',
'alert_type': 'info',
'alert_msg': ''
}
self.assertEqual(obs, exp)
r_client.set('analysis_1', dumps({'job_id': 'job_id'}))
r_client.set('job_id', dumps({'status_msg': 'running'}))
obs = analysis_description_handler_get_request(1, User('*****@*****.**'))
exp = {
'analysis_name': 'SomeAnalysis',
'analysis_id': 1,
'analysis_description': 'A test analysis',
'alert_type': 'info',
'alert_msg': 'An artifact is being deleted from this analysis'
}
self.assertEqual(obs, exp)
r_client.set(
'job_id',
dumps({
'status_msg': 'Success',
'return': {
'status': 'danger',
'message': 'Error deleting artifact'
}
}))
obs = analysis_description_handler_get_request(1, User('*****@*****.**'))
exp = {
'analysis_name': 'SomeAnalysis',
'analysis_id': 1,
'analysis_description': 'A test analysis',
'alert_type': 'danger',
'alert_msg': 'Error deleting artifact'
}
self.assertEqual(obs, exp)
def test_get_sample_template_processing_status(self):
key = SAMPLE_TEMPLATE_KEY_FORMAT % 1
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "")
self.assertEqual(obs_am, "")
# Without job id
r_client.set(key, dumps({'job_id': None, 'status': "success",
'message': ""}))
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "success")
self.assertEqual(obs_am, "")
# With job id and processing
r_client.set(key, dumps({'job_id': "test_job_id"}))
r_client.set("test_job_id", dumps({'status_msg': 'Running'}))
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertTrue(obs_proc)
self.assertEqual(obs_at, "info")
self.assertEqual(
obs_am, "This sample template is currently being processed")
# With job id and success
r_client.set(key, dumps({'job_id': "test_job_id"}))
r_client.set("test_job_id",
dumps({'status_msg': 'Success',
'return': {'status': 'success',
'message': 'Some\nwarning'}}))
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "success")
self.assertEqual(obs_am, "Some</br>warning")
obs = loads(r_client.get(key))
self.assertEqual(obs, {'job_id': None, 'status': 'success',
'message': 'Some</br>warning'})
# With job and not success
r_client.set(key, dumps({'job_id': "test_job_id"}))
r_client.set("test_job_id",
dumps({'status_msg': 'Failed',
'return': {'status': 'error',
'message': 'Some\nerror'}}))
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "error")
self.assertEqual(obs_am, "Some</br>error")
def prep_template_ajax_get_req(user_id, prep_id):
"""Returns the prep tempalte information needed for the AJAX handler
Parameters
----------
user_id : str
The user id
prep_id : int
The prep template id
Returns
-------
dict of {str: object}
A dictionary with the following keys:
- status: str, whether the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
- name: str, the name of the prep template
- files: list of str, the files available to update the prep template
- download_prep: int, the filepath_id of the prep file
- download_qiime, int, the filepath_id of the qiime mapping file
- num_samples: int, the number of samples present in the template
- num_columns: int, the number of columns present in the template
- investigation_type: str, the investigation type of the template
- ontology: str, dict of {str, list of str} containing the information
of the ENA ontology
- artifact_attached: bool, whether the template has an artifact
attached
- study_id: int, the study id of the template
"""
# Currently there is no name attribute, but it will be soon
name = "Prep information %d" % prep_id
pt = PrepTemplate(prep_id)
job_info = r_client.get(PREP_TEMPLATE_KEY_FORMAT % prep_id)
if job_info:
job_info = loads(job_info)
job_id = job_info['job_id']
if job_id:
redis_info = loads(r_client.get(job_id))
processing = redis_info['status_msg'] == 'Running'
if processing:
alert_type = 'info'
alert_msg = 'This prep template is currently being updated'
elif redis_info['status_msg'] == 'Success':
alert_type = redis_info['return']['status']
alert_msg = redis_info['return']['message'].replace('\n',
'</br>')
payload = {'job_id': None,
'status': alert_type,
'message': alert_msg}
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id,
dumps(payload))
else:
alert_type = redis_info['return']['status']
alert_msg = redis_info['return']['message'].replace('\n',
'</br>')
else:
processing = False
alert_type = job_info['status']
alert_msg = job_info['message'].replace('\n', '</br>')
else:
processing = False
alert_type = ''
alert_msg = ''
artifact_attached = pt.artifact is not None
study_id = pt.study_id
files = [f for _, f in get_files_from_uploads_folders(study_id)
if f.endswith(('.txt', '.tsv'))]
# The call to list is needed because keys is an iterator
num_samples = len(list(pt.keys()))
num_columns = len(pt.categories())
investigation_type = pt.investigation_type
# Retrieve the information to download the prep template and QIIME
# mapping file. See issue https://github.com/biocore/qiita/issues/1675
download_prep = []
download_qiime = []
for fp_id, fp in pt.get_filepaths():
if 'qiime' in basename(fp):
download_qiime.append(fp_id)
else:
download_prep.append(fp_id)
download_prep = download_prep[0]
download_qiime = download_qiime[0]
ontology = _get_ENA_ontology()
editable = Study(study_id).can_edit(User(user_id)) and not processing
return {'status': 'success',
'message': '',
'name': name,
'files': files,
'download_prep': download_prep,
'download_qiime': download_qiime,
'num_samples': num_samples,
'num_columns': num_columns,
'investigation_type': investigation_type,
'ontology': ontology,
'artifact_attached': artifact_attached,
'study_id': study_id,
'editable': editable,
'data_type': pt.data_type(),
'alert_type': alert_type,
'is_submitted_to_ebi': pt.is_submitted_to_ebi,
'alert_message': alert_msg}
def prep_template_patch_req(user_id, req_op, req_path, req_value=None,
req_from=None):
"""Modifies an attribute of the prep template
Parameters
----------
user_id : str
The id of the user performing the patch operation
req_op : str
The operation to perform on the prep information
req_path : str
The prep information and attribute to patch
req_value : str, optional
The value that needs to be modified
req_from : str, optional
The original path of the element
Returns
-------
dict of {str, str}
A dictionary with the following keys:
- status: str, whether if the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
"""
req_path = [v for v in req_path.split('/') if v]
if req_op == 'replace':
# The structure of the path should be /prep_id/attribute_to_modify/
# so if we don't have those 2 elements, we should return an error
if len(req_path) != 2:
return {'status': 'error',
'message': 'Incorrect path parameter'}
prep_id = int(req_path[0])
attribute = req_path[1]
# Check if the user actually has access to the prep template
prep = PrepTemplate(prep_id)
access_error = check_access(prep.study_id, user_id)
if access_error:
return access_error
status = 'success'
msg = ''
if attribute == 'investigation_type':
prep.investigation_type = req_value
elif attribute == 'data':
fp = check_fp(prep.study_id, req_value)
if fp['status'] != 'success':
return fp
fp = fp['file']
job_id = safe_submit(user_id, update_prep_template, prep_id, fp)
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id,
dumps({'job_id': job_id}))
else:
# We don't understand the attribute so return an error
return {'status': 'error',
'message': 'Attribute "%s" not found. '
'Please, check the path parameter' % attribute}
return {'status': status, 'message': msg}
elif req_op == 'remove':
# The structure of the path should be /prep_id/{columns|samples}/name
if len(req_path) != 3:
return {'status': 'error',
'message': 'Incorrect path parameter'}
prep_id = int(req_path[0])
attribute = req_path[1]
attr_id = req_path[2]
# Check if the user actually has access to the study
pt = PrepTemplate(prep_id)
access_error = check_access(pt.study_id, user_id)
if access_error:
return access_error
# Offload the deletion of the column to the cluster
job_id = safe_submit(user_id, delete_sample_or_column, PrepTemplate,
prep_id, attribute, attr_id)
# Store the job id attaching it to the sample template id
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id,
dumps({'job_id': job_id}))
return {'status': 'success', 'message': ''}
else:
return {'status': 'error',
'message': 'Operation "%s" not supported. '
'Current supported operations: replace, remove'
% req_op}
def test_get_sample_template_processing_status(self):
key = SAMPLE_TEMPLATE_KEY_FORMAT % 1
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "")
self.assertEqual(obs_am, "")
# Without job id
r_client.set(key, dumps({"job_id": None, "status": "success", "message": ""}))
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "success")
self.assertEqual(obs_am, "")
# With job id and processing
r_client.set(key, dumps({"job_id": "test_job_id"}))
r_client.set("test_job_id", dumps({"status_msg": "Running"}))
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertTrue(obs_proc)
self.assertEqual(obs_at, "info")
self.assertEqual(obs_am, "This sample template is currently being processed")
# With job id and success
r_client.set(key, dumps({"job_id": "test_job_id"}))
r_client.set(
"test_job_id", dumps({"status_msg": "Success", "return": {"status": "success", "message": "Some\nwarning"}})
)
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "success")
self.assertEqual(obs_am, "Some</br>warning")
obs = loads(r_client.get(key))
self.assertEqual(obs, {"job_id": None, "status": "success", "message": "Some</br>warning"})
# With job and not success
r_client.set(key, dumps({"job_id": "test_job_id"}))
r_client.set(
"test_job_id", dumps({"status_msg": "Failed", "return": {"status": "error", "message": "Some\nerror"}})
)
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "error")
self.assertEqual(obs_am, "Some</br>error")
# With job expired
r_client.set(key, dumps({"job_id": "non_existent_job"}))
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "")
self.assertEqual(obs_am, "")
def sample_template_summary_get_req(samp_id, user_id):
"""Returns a summary of the sample template metadata columns
Parameters
----------
samp_id : int
SampleTemplate id to get info for
user_id : str
User requesting the sample template info
Returns
-------
dict
Returns summary information in the form
{'status': str,
'message': str,
'info': dict of {str: object}
status can be success, warning, or error depending on result
message has the warnings or errors
info dictionary contains the keys as the metadata categories
and the values are list of tuples. Each tuple is an observed value in
the category and the number of times its seen.
Format {num_samples: value,
category: [(val1, count1), (val2, count2), ...], ...}
"""
access_error = check_access(samp_id, user_id)
if access_error:
return access_error
job_info = r_client.get(SAMPLE_TEMPLATE_KEY_FORMAT % samp_id)
if job_info:
job_info = loads(job_info)
job_id = job_info['job_id']
if job_id:
redis_info = loads(r_client.get(job_id))
processing = redis_info['status_msg'] == 'Running'
if processing:
alert_type = 'info'
alert_msg = 'This sample template is currently being processed'
elif redis_info['status_msg'] == 'Success':
alert_type = redis_info['return']['status']
alert_msg = redis_info['return']['message'].replace('\n',
'</br>')
payload = {'job_id': None,
'status': alert_type,
'message': alert_msg}
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % samp_id,
dumps(payload))
else:
alert_type = redis_info['return']['status']
alert_msg = redis_info['return']['message'].replace('\n',
'</br>')
else:
processing = False
alert_type = job_info['status']
alert_msg = job_info['message'].replace('\n', '</br>')
else:
processing = False
alert_type = ''
alert_msg = ''
exists = _check_sample_template_exists(int(samp_id))
if exists['status'] != 'success':
return {'status': 'success',
'message': '',
'num_samples': 0,
'num_columns': 0,
'editable': not processing,
'alert_type': alert_type,
'alert_message': alert_msg,
'stats': {}}
template = SampleTemplate(int(samp_id))
df = template.to_dataframe()
editable = (Study(template.study_id).can_edit(User(user_id)) and not
processing)
out = {'status': 'success',
'message': '',
'num_samples': df.shape[0],
'num_columns': df.shape[1],
'editable': editable,
'alert_type': alert_type,
'alert_message': alert_msg,
'stats': {}}
# drop the samp_id column if it exists
if 'study_id' in df.columns:
df.drop('study_id', axis=1, inplace=True)
for column in df.columns:
counts = df[column].value_counts()
out['stats'][str(column)] = [(str(key), counts[key])
for key in natsorted(counts.index)]
return out
def artifact_post_req(user_id,
filepaths,
artifact_type,
name,
prep_template_id,
artifact_id=None):
"""Creates the initial artifact for the prep template
Parameters
----------
user_id : str
User adding the atrifact
filepaths : dict of str
Comma-separated list of files to attach to the artifact,
keyed by file type
artifact_type : str
The type of the artifact
name : str
Name to give the artifact
prep_template_id : int or str castable to int
Prep template to attach the artifact to
artifact_id : int or str castable to int, optional
The id of the imported artifact
Returns
-------
dict of objects
A dictionary containing the new artifact ID
{'status': status,
'message': message,
'artifact': id}
"""
prep = PrepTemplate(int(prep_template_id))
study_id = prep.study_id
# First check if the user has access to the study
access_error = check_access(study_id, user_id)
if access_error:
return access_error
if artifact_id:
# if the artifact id has been provided, import the artifact
job_id = safe_submit(user_id, copy_raw_data, prep, artifact_id)
else:
uploads_path = get_mountpoint('uploads')[0][1]
path_builder = partial(join, uploads_path, str(study_id))
cleaned_filepaths = []
for ftype, file_list in viewitems(filepaths):
# JavaScript sends us this list as a comma-separated list
for fp in file_list.split(','):
# JavaScript will send this value as an empty string if the
# list of files was empty. In such case, the split will
# generate a single element containing the empty string. Check
# for that case here and, if fp is not the empty string,
# proceed to check if the file exists
if fp:
# Check if filepath being passed exists for study
full_fp = path_builder(fp)
exists = check_fp(study_id, full_fp)
if exists['status'] != 'success':
return {
'status': 'error',
'message': 'File does not exist: %s' % fp
}
cleaned_filepaths.append((full_fp, ftype))
# This should never happen, but it doesn't hurt to actually have
# a explicit check, in case there is something odd with the JS
if not cleaned_filepaths:
return {
'status': 'error',
'message': "Can't create artifact, no files provided."
}
job_id = safe_submit(user_id,
create_raw_data,
artifact_type,
prep,
cleaned_filepaths,
name=name)
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep.id, dumps({'job_id': job_id}))
return {'status': 'success', 'message': ''}
def sample_template_summary_get_req(samp_id, user_id):
"""Returns a summary of the sample template metadata columns
Parameters
----------
samp_id : int
SampleTemplate id to get info for
user_id : str
User requesting the sample template info
Returns
-------
dict
Returns summary information in the form
{'status': str,
'message': str,
'info': dict of {str: object}
status can be success, warning, or error depending on result
message has the warnings or errors
info dictionary contains the keys as the metadata categories
and the values are list of tuples. Each tuple is an observed value in
the category and the number of times its seen.
Format {num_samples: value,
category: [(val1, count1), (val2, count2), ...], ...}
"""
access_error = check_access(samp_id, user_id)
if access_error:
return access_error
job_info = r_client.get(SAMPLE_TEMPLATE_KEY_FORMAT % samp_id)
if job_info:
job_info = loads(job_info)
job_id = job_info['job_id']
if job_id:
redis_info = loads(r_client.get(job_id))
processing = redis_info['status_msg'] == 'Running'
if processing:
alert_type = 'info'
alert_msg = 'This sample template is currently being processed'
elif redis_info['status_msg'] == 'Success':
alert_type = redis_info['return']['status']
alert_msg = redis_info['return']['message'].replace(
'\n', '</br>')
payload = {
'job_id': None,
'status': alert_type,
'message': alert_msg
}
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % samp_id,
dumps(payload))
else:
alert_type = redis_info['return']['status']
alert_msg = redis_info['return']['message'].replace(
'\n', '</br>')
else:
processing = False
alert_type = job_info['status']
alert_msg = job_info['message'].replace('\n', '</br>')
else:
processing = False
alert_type = ''
alert_msg = ''
exists = _check_sample_template_exists(int(samp_id))
if exists['status'] != 'success':
return {
'status': 'success',
'message': '',
'num_samples': 0,
'num_columns': 0,
'editable': not processing,
'alert_type': alert_type,
'alert_message': alert_msg,
'stats': {}
}
template = SampleTemplate(int(samp_id))
df = template.to_dataframe()
editable = (Study(template.study_id).can_edit(User(user_id))
and not processing)
out = {
'status': 'success',
'message': '',
'num_samples': df.shape[0],
'num_columns': df.shape[1],
'editable': editable,
'alert_type': alert_type,
'alert_message': alert_msg,
'stats': {}
}
# drop the samp_id column if it exists
if 'study_id' in df.columns:
df.drop('study_id', axis=1, inplace=True)
for column in df.columns:
counts = df[column].value_counts()
out['stats'][str(column)] = [(str(key), counts[key])
for key in natsorted(counts.index)]
return out
def test_get_sample_template_processing_status(self):
key = SAMPLE_TEMPLATE_KEY_FORMAT % 1
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "")
self.assertEqual(obs_am, "")
# Without job id
r_client.set(
key, dumps({
'job_id': None,
'status': "success",
'message': ""
}))
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "success")
self.assertEqual(obs_am, "")
# With job id and processing
r_client.set(key, dumps({'job_id': "test_job_id"}))
r_client.set("test_job_id", dumps({'status_msg': 'Running'}))
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertTrue(obs_proc)
self.assertEqual(obs_at, "info")
self.assertEqual(obs_am,
"This sample template is currently being processed")
# With job id and success
r_client.set(key, dumps({'job_id': "test_job_id"}))
r_client.set(
"test_job_id",
dumps({
'status_msg': 'Success',
'return': {
'status': 'success',
'message': 'Some\nwarning'
}
}))
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "success")
self.assertEqual(obs_am, "Some</br>warning")
obs = loads(r_client.get(key))
self.assertEqual(obs, {
'job_id': None,
'status': 'success',
'message': 'Some</br>warning'
})
# With job and not success
r_client.set(key, dumps({'job_id': "test_job_id"}))
r_client.set(
"test_job_id",
dumps({
'status_msg': 'Failed',
'return': {
'status': 'error',
'message': 'Some\nerror'
}
}))
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "error")
self.assertEqual(obs_am, "Some</br>error")
def prep_template_ajax_get_req(user_id, prep_id):
"""Returns the prep tempalte information needed for the AJAX handler
Parameters
----------
user_id : str
The user id
prep_id : int
The prep template id
Returns
-------
dict of {str: object}
A dictionary with the following keys:
- status: str, whether the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
- name: str, the name of the prep template
- files: list of str, the files available to update the prep template
- download_prep: int, the filepath_id of the prep file
- download_qiime, int, the filepath_id of the qiime mapping file
- num_samples: int, the number of samples present in the template
- num_columns: int, the number of columns present in the template
- investigation_type: str, the investigation type of the template
- ontology: str, dict of {str, list of str} containing the information
of the ENA ontology
- artifact_attached: bool, whether the template has an artifact
attached
- study_id: int, the study id of the template
"""
# Currently there is no name attribute, but it will be soon
name = "Prep information %d" % prep_id
pt = PrepTemplate(prep_id)
job_info = r_client.get(PREP_TEMPLATE_KEY_FORMAT % prep_id)
if job_info:
job_info = loads(job_info)
job_id = job_info['job_id']
if job_id:
if job_info['is_qiita_job']:
job = ProcessingJob(job_id)
processing = job.status in ('queued', 'running')
success = job.status == 'success'
alert_type = 'info' if processing or success else 'danger'
alert_msg = (job.log.msg.replace('\n', '</br>')
if job.log is not None else "")
else:
redis_info = loads(r_client.get(job_id))
processing = redis_info['status_msg'] == 'Running'
success = redis_info['status_msg'] == 'Success'
if redis_info['return'] is not None:
alert_type = redis_info['return']['status']
alert_msg = redis_info['return']['message'].replace(
'\n', '</br>')
else:
alert_type = 'info'
alert_msg = ''
if processing:
alert_type = 'info'
alert_msg = 'This prep template is currently being updated'
elif success:
payload = {'job_id': None,
'status': alert_type,
'message': alert_msg,
'is_qiita_job': job_info['is_qiita_job']}
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id,
dumps(payload))
else:
processing = False
alert_type = job_info['status']
alert_msg = job_info['message'].replace('\n', '</br>')
else:
processing = False
alert_type = ''
alert_msg = ''
artifact_attached = pt.artifact is not None
study_id = pt.study_id
files = [f for _, f in get_files_from_uploads_folders(study_id)
if f.endswith(('.txt', '.tsv'))]
# The call to list is needed because keys is an iterator
num_samples = len(list(pt.keys()))
num_columns = len(pt.categories())
investigation_type = pt.investigation_type
# Retrieve the information to download the prep template and QIIME
# mapping file. See issue https://github.com/biocore/qiita/issues/1675
download_prep = []
download_qiime = []
for fp_id, fp in pt.get_filepaths():
if 'qiime' in basename(fp):
download_qiime.append(fp_id)
else:
download_prep.append(fp_id)
download_prep = download_prep[0]
download_qiime = download_qiime[0]
ontology = _get_ENA_ontology()
editable = Study(study_id).can_edit(User(user_id)) and not processing
return {'status': 'success',
'message': '',
'name': name,
'files': files,
'download_prep': download_prep,
'download_qiime': download_qiime,
'num_samples': num_samples,
'num_columns': num_columns,
'investigation_type': investigation_type,
'ontology': ontology,
'artifact_attached': artifact_attached,
'study_id': study_id,
'editable': editable,
'data_type': pt.data_type(),
'alert_type': alert_type,
'is_submitted_to_ebi': pt.is_submitted_to_ebi,
'alert_message': alert_msg}
def prep_template_patch_req(user_id, req_op, req_path, req_value=None,
req_from=None):
"""Modifies an attribute of the prep template
Parameters
----------
user_id : str
The id of the user performing the patch operation
req_op : str
The operation to perform on the prep information
req_path : str
The prep information and attribute to patch
req_value : str, optional
The value that needs to be modified
req_from : str, optional
The original path of the element
Returns
-------
dict of {str, str}
A dictionary with the following keys:
- status: str, whether if the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
"""
if req_op == 'replace':
req_path = [v for v in req_path.split('/') if v]
# The structure of the path should be /prep_id/attribute_to_modify/
# so if we don't have those 2 elements, we should return an error
if len(req_path) != 2:
return {'status': 'error',
'message': 'Incorrect path parameter'}
prep_id = int(req_path[0])
attribute = req_path[1]
# Check if the user actually has access to the prep template
prep = PrepTemplate(prep_id)
access_error = check_access(prep.study_id, user_id)
if access_error:
return access_error
status = 'success'
msg = ''
if attribute == 'investigation_type':
prep.investigation_type = req_value
elif attribute == 'data':
fp = check_fp(prep.study_id, req_value)
if fp['status'] != 'success':
return fp
fp = fp['file']
job_id = safe_submit(user_id, update_prep_template, prep_id, fp)
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id, job_id)
else:
# We don't understand the attribute so return an error
return {'status': 'error',
'message': 'Attribute "%s" not found. '
'Please, check the path parameter' % attribute}
return {'status': status, 'message': msg}
else:
return {'status': 'error',
'message': 'Operation "%s" not supported. '
'Current supported operations: replace' % req_op}
def artifact_post_req(user_id, filepaths, artifact_type, name,
prep_template_id, artifact_id=None):
"""Creates the initial artifact for the prep template
Parameters
----------
user_id : str
User adding the atrifact
filepaths : dict of str
Comma-separated list of files to attach to the artifact,
keyed by file type
artifact_type : str
The type of the artifact
name : str
Name to give the artifact
prep_template_id : int or str castable to int
Prep template to attach the artifact to
artifact_id : int or str castable to int, optional
The id of the imported artifact
Returns
-------
dict of objects
A dictionary containing the new artifact ID
{'status': status,
'message': message,
'artifact': id}
"""
prep = PrepTemplate(int(prep_template_id))
study_id = prep.study_id
# First check if the user has access to the study
access_error = check_access(study_id, user_id)
if access_error:
return access_error
if artifact_id:
# if the artifact id has been provided, import the artifact
job_id = safe_submit(user_id, copy_raw_data, prep, artifact_id)
else:
uploads_path = get_mountpoint('uploads')[0][1]
path_builder = partial(join, uploads_path, str(study_id))
cleaned_filepaths = []
for ftype, file_list in viewitems(filepaths):
# JavaScript sends us this list as a comma-separated list
for fp in file_list.split(','):
# JavaScript will send this value as an empty string if the
# list of files was empty. In such case, the split will
# generate a single element containing the empty string. Check
# for that case here and, if fp is not the empty string,
# proceed to check if the file exists
if fp:
# Check if filepath being passed exists for study
full_fp = path_builder(fp)
exists = check_fp(study_id, full_fp)
if exists['status'] != 'success':
return {'status': 'error',
'message': 'File does not exist: %s' % fp}
cleaned_filepaths.append((full_fp, ftype))
# This should never happen, but it doesn't hurt to actually have
# a explicit check, in case there is something odd with the JS
if not cleaned_filepaths:
return {'status': 'error',
'message': "Can't create artifact, no files provided."}
job_id = safe_submit(user_id, create_raw_data, artifact_type, prep,
cleaned_filepaths, name=name)
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep.id, dumps({'job_id': job_id}))
return {'status': 'success',
'message': ''}
