def post(self, preprocessed_data_id):
user = self.current_user
# make sure user is admin and can therefore actually submit to EBI
if user.level != 'admin':
raise HTTPError(403, reason="User %s cannot submit to EBI!" %
user.id)
submission_type = self.get_argument('submission_type')
if submission_type not in ['ADD', 'MODIFY']:
raise HTTPError(403, reason="User: %s, %s is not a recognized "
"submission type" % (user.id, submission_type))
study = Artifact(preprocessed_data_id).study
state = study.ebi_submission_status
if state == 'submitting':
message = "Cannot resubmit! Current state is: %s" % state
self.display_template(preprocessed_data_id, message, 'danger')
else:
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('submit_to_EBI')
params = Parameters.load(
cmd, values_dict={'artifact': preprocessed_data_id,
'submission_type': submission_type})
job = ProcessingJob.create(user, params, True)
r_client.set('ebi_submission_%s' % preprocessed_data_id,
dumps({'job_id': job.id, 'is_qiita_job': True}))
job.submit()
level = 'success'
message = 'EBI submission started. Job id: %s' % job.id
self.redirect("%s/study/description/%d?level=%s&message=%s" % (
qiita_config.portal_dir, study.id, level, url_escape(message)))
def study_delete_req(study_id, user_id):
"""Delete a given study
Parameters
----------
study_id : int
Study id to delete
user_id : str
User requesting the deletion
Returns
-------
dict
Status of deletion, in the format
{status: status,
message: message}
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_study')
params = Parameters.load(cmd, values_dict={'study': study_id})
job = ProcessingJob.create(User(user_id), params, True)
# Store the job id attaching it to the sample template id
r_client.set(STUDY_KEY_FORMAT % study_id, dumps({'job_id': job.id}))
job.submit()
return {'status': 'success', 'message': ''}
def post(self, preprocessed_data_id):
user = self.current_user
# make sure user is admin and can therefore actually submit to EBI
if user.level != 'admin':
raise HTTPError(403, reason="User %s cannot submit to EBI!" %
user.id)
submission_type = self.get_argument('submission_type')
if submission_type not in ['ADD', 'MODIFY']:
raise HTTPError(403, reason="User: %s, %s is not a recognized "
"submission type" % (user.id, submission_type))
study = Artifact(preprocessed_data_id).study
state = study.ebi_submission_status
if state == 'submitting':
message = "Cannot resubmit! Current state is: %s" % state
self.display_template(preprocessed_data_id, message, 'danger')
else:
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('submit_to_EBI')
params = Parameters.load(
cmd, values_dict={'artifact': preprocessed_data_id,
'submission_type': submission_type})
job = ProcessingJob.create(user, params, True)
r_client.set('ebi_submission_%s' % preprocessed_data_id,
dumps({'job_id': job.id, 'is_qiita_job': True}))
job.submit()
level = 'success'
message = 'EBI submission started. Job id: %s' % job.id
self.redirect("%s/study/description/%d?level=%s&message=%s" % (
qiita_config.portal_dir, study.id, level, url_escape(message)))
def create_sample_template(job):
"""Creates a sample template
Parameters
----------
job : qiita_db.processing_job.ProcessingJob
The processing job performing the task
"""
with qdb.sql_connection.TRN:
params = job.parameters.values
fp = params['fp']
study = qdb.study.Study(int(params['study_id']))
is_mapping_file = params['is_mapping_file']
data_type = params['data_type']
with warnings.catch_warnings(record=True) as warns:
if is_mapping_file:
create_templates_from_qiime_mapping_file(fp, study, data_type)
else:
qdb.metadata_template.sample_template.SampleTemplate.create(
qdb.metadata_template.util.load_template_to_dataframe(fp),
study)
remove(fp)
if warns:
msg = '\n'.join(set(str(w) for w in warns))
r_client.set(
"sample_template_%s" % study.id,
dumps({
'job_id': job.id,
'alert_type': 'warning',
'alert_msg': msg
}))
job._set_status('success')
def create_sample_template(job):
"""Creates a sample template
Parameters
----------
job : qiita_db.processing_job.ProcessingJob
The processing job performing the task
"""
with qdb.sql_connection.TRN:
params = job.parameters.values
fp = params['fp']
study = qdb.study.Study(int(params['study_id']))
is_mapping_file = params['is_mapping_file']
data_type = params['data_type']
with warnings.catch_warnings(record=True) as warns:
if is_mapping_file:
create_templates_from_qiime_mapping_file(fp, study, data_type)
else:
qdb.metadata_template.sample_template.SampleTemplate.create(
qdb.metadata_template.util.load_template_to_dataframe(fp),
study)
remove(fp)
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
r_client.set("sample_template_%s" % study.id,
dumps({'job_id': job.id, 'alert_type': 'warning',
'alert_msg': msg}))
job._set_status('success')
def artifact_post_req(user, artifact_id):
"""Deletes the artifact
Parameters
----------
user : qiita_db.user.User
The user requesting the action
artifact_id : int
Id of the artifact being deleted
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
check_artifact_access(user, artifact)
analysis = artifact.analysis
if analysis:
# Do something when deleting in the analysis part to keep track of it
redis_key = "analysis_%s" % analysis.id
else:
pt_id = artifact.prep_templates[0].id
redis_key = PREP_TEMPLATE_KEY_FORMAT % pt_id
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_artifact')
params = Parameters.load(cmd, values_dict={'artifact': artifact_id})
job = ProcessingJob.create(user, params)
r_client.set(redis_key, dumps({'job_id': job.id, 'is_qiita_job': True}))
job.submit()
def update_prep_template(job):
"""Updates a prep template
Parameters
----------
job : qiita_db.processing_job.ProcessingJob
The processing job performing the task
"""
with qdb.sql_connection.TRN:
param_vals = job.parameters.values
prep_id = param_vals['prep_template']
fp = param_vals['template_fp']
prep = qdb.metadata_template.prep_template.PrepTemplate(prep_id)
with warnings.catch_warnings(record=True) as warns:
df = qdb.metadata_template.util.load_template_to_dataframe(fp)
prep.extend_and_update(df)
remove(fp)
# Join all the warning messages into one. Note that this info
# will be ignored if an exception is raised
if warns:
msg = '\n'.join(set(str(w) for w in warns))
r_client.set(
"prep_template_%s" % prep_id,
dumps({
'job_id': job.id,
'alert_type': 'warning',
'alert_msg': msg
}))
job._set_status('success')
def list_remote_files(job):
"""Lists valid study files on a remote server
Parameters
----------
job : qiita_db.processing_job.ProcessingJob
The processing job performing the task
"""
with qdb.sql_connection.TRN:
url = job.parameters.values['url']
private_key = job.parameters.values['private_key']
study_id = job.parameters.values['study_id']
try:
files = list_remote(url, private_key)
r_client.set("upload_study_%s" % study_id,
dumps({
'job_id': job.id,
'url': url,
'files': files
}))
except Exception:
job._set_error(traceback.format_exception(*exc_info()))
else:
job._set_status('success')
finally:
# making sure to always delete the key so Qiita never keeps it
remove(private_key)
def study_delete_req(study_id, user_id):
"""Delete a given study
Parameters
----------
study_id : int
Study id to delete
user_id : str
User requesting the deletion
Returns
-------
dict
Status of deletion, in the format
{status: status,
message: message}
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_study')
params = Parameters.load(cmd, values_dict={'study': study_id})
job = ProcessingJob.create(User(user_id), params, True)
# Store the job id attaching it to the sample template id
r_client.set(STUDY_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'status': 'success', 'message': ''}
def update_prep_template(job):
"""Updates a prep template
Parameters
----------
job : qiita_db.processing_job.ProcessingJob
The processing job performing the task
"""
with qdb.sql_connection.TRN:
param_vals = job.parameters.values
prep_id = param_vals['prep_template']
fp = param_vals['template_fp']
prep = qdb.metadata_template.prep_template.PrepTemplate(prep_id)
with warnings.catch_warnings(record=True) as warns:
df = qdb.metadata_template.util.load_template_to_dataframe(fp)
prep.extend_and_update(df)
remove(fp)
# Join all the warning messages into one. Note that this info
# will be ignored if an exception is raised
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
r_client.set("prep_template_%s" % prep_id,
dumps({'job_id': job.id, 'alert_type': 'warning',
'alert_msg': msg}))
job._set_status('success')
def artifact_post_req(user, artifact_id):
"""Deletes the artifact
Parameters
----------
user : qiita_db.user.User
The user requesting the action
artifact_id : int
Id of the artifact being deleted
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
check_artifact_access(user, artifact)
analysis = artifact.analysis
if analysis:
# Do something when deleting in the analysis part to keep track of it
redis_key = "analysis_%s" % analysis.id
else:
pt_id = artifact.prep_templates[0].id
redis_key = PREP_TEMPLATE_KEY_FORMAT % pt_id
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_artifact')
params = Parameters.load(cmd, values_dict={'artifact': artifact_id})
job = ProcessingJob.create(user, params, True)
r_client.set(redis_key, dumps({'job_id': job.id, 'is_qiita_job': True}))
job.submit()
return {'job': job.id}
def sample_template_put_req(study_id, user_id, sample_template):
"""Updates a sample template using the given file
Parameters
----------
study_id : int
The current study object id
user_id : str
The current user object id
sample_template : str
filename to use for updating
Returns
-------
dict
results dictonary in the format
{'status': status,
'message': msg,
'file': sample_template}
status can be success, warning, or error depending on result
message has the warnings or errors
file has the file name
"""
exists = _check_sample_template_exists(int(study_id))
if exists['status'] != 'success':
return exists
access_error = check_access(int(study_id), user_id)
if access_error:
return access_error
fp_rsp = check_fp(study_id, sample_template)
if fp_rsp['status'] != 'success':
# Unknown filepath, so return the error message
return fp_rsp
fp_rsp = fp_rsp['file']
msg = ''
status = 'success'
# Offload the update of the sample template to the cluster
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('update_sample_template')
params = Parameters.load(cmd,
values_dict={
'study': int(study_id),
'template_fp': fp_rsp
})
job = ProcessingJob.create(User(user_id), params)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'status': status, 'message': msg, 'file': sample_template}
def sample_template_handler_post_request(study_id,
user,
filepath,
data_type=None):
"""Creates a new sample template
Parameters
----------
study_id: int
The study to add the sample information
user: qiita_db.user import User
The user performing the request
filepath: str
The path to the sample template file
data_type: str, optional
If filepath is a QIIME mapping file, the data type of the prep
information file
Returns
-------
dict of {'job': str}
job: the id of the job adding the sample information to the study
Raises
------
HTTPError
404 if the filepath doesn't exist
"""
# Check if the current user has access to the study
sample_template_checks(study_id, user)
# Check if the file exists
fp_rsp = check_fp(study_id, filepath)
if fp_rsp['status'] != 'success':
raise HTTPError(404, 'Filepath not found')
filepath = fp_rsp['file']
is_mapping_file = looks_like_qiime_mapping_file(filepath)
if is_mapping_file and not data_type:
raise HTTPError(
400, 'Please, choose a data type if uploading a '
'QIIME mapping file')
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('create_sample_template')
params = Parameters.load(cmd,
values_dict={
'fp': filepath,
'study_id': study_id,
'is_mapping_file': is_mapping_file,
'data_type': data_type
})
job = ProcessingJob.create(user, params, True)
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
def sample_template_handler_post_request(study_id, user, filepath,
data_type=None, direct_upload=False):
"""Creates a new sample template
Parameters
----------
study_id: int
The study to add the sample information
user: qiita_db.user import User
The user performing the request
filepath: str
The path to the sample template file
data_type: str, optional
If filepath is a QIIME mapping file, the data type of the prep
information file
direct_upload: boolean, optional
If filepath is a direct upload; if False we need to process the
filepath as part of the study upload folder
Returns
-------
dict of {'job': str}
job: the id of the job adding the sample information to the study
Raises
------
HTTPError
404 if the filepath doesn't exist
"""
# Check if the current user has access to the study
sample_template_checks(study_id, user)
# Check if the file exists
if not direct_upload:
fp_rsp = check_fp(study_id, filepath)
if fp_rsp['status'] != 'success':
raise HTTPError(404, reason='Filepath not found')
filepath = fp_rsp['file']
is_mapping_file = looks_like_qiime_mapping_file(filepath)
if is_mapping_file and not data_type:
raise HTTPError(400, reason='Please, choose a data type if uploading '
'a QIIME mapping file')
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('create_sample_template')
params = Parameters.load(
cmd, values_dict={'fp': filepath, 'study_id': study_id,
'is_mapping_file': is_mapping_file,
'data_type': data_type})
job = ProcessingJob.create(user, params, True)
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
def post(self, study_id):
method = self.get_argument('remote-request-type')
url = self.get_argument('inputURL')
ssh_key = self.request.files['ssh-key'][0]['body']
status = 'success'
message = ''
try:
study = Study(int(study_id))
except QiitaDBUnknownIDError:
raise HTTPError(404, reason="Study %s does not exist" % study_id)
check_access(self.current_user,
study,
no_public=True,
raise_error=True)
_, upload_folder = get_mountpoint("uploads")[0]
upload_folder = join(upload_folder, study_id)
ssh_key_fp = join(upload_folder, '.key.txt')
create_nested_path(upload_folder)
with open(ssh_key_fp, 'wb') as f:
f.write(ssh_key)
chmod(ssh_key_fp, 0o600)
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
if method == 'list':
cmd = qiita_plugin.get_command('list_remote_files')
params = Parameters.load(cmd,
values_dict={
'url': url,
'private_key': ssh_key_fp,
'study_id': study_id
})
elif method == 'transfer':
cmd = qiita_plugin.get_command('download_remote_files')
params = Parameters.load(cmd,
values_dict={
'url': url,
'private_key': ssh_key_fp,
'destination': upload_folder
})
else:
status = 'error'
message = 'Not a valid method'
if status == 'success':
job = ProcessingJob.create(self.current_user, params, True)
job.submit()
r_client.set(UPLOAD_STUDY_FORMAT % study_id,
dumps({'job_id': job.id}))
self.write({'status': status, 'message': message})
def post(self):
if r_client.get('maintenance') is not None:
raise HTTPError(503, "Site is down for maintenance")
username = self.get_argument("username", "").strip().lower()
passwd = self.get_argument("password", "")
nextpage = self.get_argument("next", None)
if nextpage is None:
if "auth/" not in self.request.headers['Referer']:
nextpage = self.request.headers['Referer']
else:
nextpage = "%s/" % qiita_config.portal_dir
msg = ""
# check the user level
try:
if User(username).level == "unverified":
# email not verified so dont log in
msg = ("Email not verified. Please check your email and click "
"the verify link. You may need to check your spam "
"folder to find the email.<br/>If a verification email"
" has not arrived in 15 minutes, please email <a href='"
"mailto:[email protected]'>[email protected]</a>")
except QiitaDBUnknownIDError:
msg = "Unknown user"
except RuntimeError:
# means DB not available, so set maintenance mode and failover
r_client.set(
"maintenance", "Database connection unavailable, "
"please try again later.")
self.redirect("%s/" % qiita_config.portal_dir)
return
# Check the login information
login = None
try:
login = User.login(username, passwd)
except IncorrectEmailError:
msg = "Unknown user"
except IncorrectPasswordError:
msg = "Incorrect password"
except UnverifiedEmailError:
msg = "You have not verified your email address"
if login:
# everything good so log in
self.set_current_user(username)
self.redirect(nextpage)
else:
self.render("index.html", message=msg, level='danger')
def post(self):
analysis_id = int(self.get_argument('analysis_id'))
user = self.current_user
check_analysis_access(user, Analysis(analysis_id))
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_analysis')
params = Parameters.load(cmd, values_dict={'analysis_id': analysis_id})
job = ProcessingJob.create(user, params, True)
# Store the job id attaching it to the sample template id
r_client.set('analysis_delete_%d' % analysis_id,
dumps({'job_id': job.id}))
job.submit()
self.redirect("%s/analysis/list/" % (qiita_config.portal_dir))
def test_post_select_samples(self):
# just making sure that the key is not set in redis
r_client.delete('maintenance')
response = self.get('/auth/reset/')
self.assertEqual(response.code, 200)
self.assertIn(('<label for="newpass2" class="col-sm-2 '
'control-label">Repeat New Password'
'</label>'), response.body)
# not displaying due to maintenance
r_client.set('maintenance', 'This is my error message')
response = self.get('/auth/reset/')
self.assertEqual(response.code, 200)
self.assertNotIn(('<label for="newpass2" class="col-sm-2 '
'control-label">Repeat New Password'
'</label>'), response.body)
r_client.delete('maintenance')
def post(self):
username = self.get_argument("username", "").strip().lower()
passwd = self.get_argument("password", "")
nextpage = self.get_argument("next", None)
if nextpage is None:
if "auth/" not in self.request.headers['Referer']:
nextpage = self.request.headers['Referer']
else:
nextpage = "%s/" % qiita_config.portal_dir
msg = ""
# check the user level
try:
if User(username).level == "unverified":
# email not verified so dont log in
msg = ("Email not verified. Please check your email and click "
"the verify link. You may need to check your spam "
"folder to find the email.<br/>If a verification email"
" has not arrived in 15 minutes, please email <a href='"
"mailto:[email protected]'>[email protected]</a>")
except QiitaDBUnknownIDError:
msg = "Unknown user"
except RuntimeError:
# means DB not available, so set maintenance mode and failover
r_client.set("maintenance", "Database connection unavailable, "
"please try again later.")
self.redirect("%s/" % qiita_config.portal_dir)
return
# Check the login information
login = None
try:
login = User.login(username, passwd)
except IncorrectEmailError:
msg = "Unknown user"
except IncorrectPasswordError:
msg = "Incorrect password"
except UnverifiedEmailError:
msg = "You have not verified your email address"
if login:
# everything good so log in
self.set_current_user(username)
self.redirect(nextpage)
else:
self.render("index.html", message=msg, level='danger')
def test_post_select_samples(self):
# just making sure that the key is not set in redis
r_client.delete('maintenance')
response = self.get('/auth/reset/')
self.assertEqual(response.code, 200)
self.assertIn(('<label for="newpass2" class="col-sm-2 '
'control-label">Repeat New Password'
'</label>'), response.body)
# not displaying due to maintenance
r_client.set('maintenance', 'This is my error message')
response = self.get('/auth/reset/')
self.assertEqual(response.code, 200)
self.assertNotIn(('<label for="newpass2" class="col-sm-2 '
'control-label">Repeat New Password'
'</label>'), response.body)
r_client.delete('maintenance')
def post(self, study_id):
method = self.get_argument('remote-request-type')
url = self.get_argument('inputURL')
ssh_key = self.request.files['ssh-key'][0]['body']
status = 'success'
message = ''
try:
study = Study(int(study_id))
except QiitaDBUnknownIDError:
raise HTTPError(404, reason="Study %s does not exist" % study_id)
check_access(
self.current_user, study, no_public=True, raise_error=True)
_, upload_folder = get_mountpoint("uploads")[0]
upload_folder = join(upload_folder, study_id)
ssh_key_fp = join(upload_folder, '.key.txt')
create_nested_path(upload_folder)
with open(ssh_key_fp, 'w') as f:
f.write(ssh_key)
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
if method == 'list':
cmd = qiita_plugin.get_command('list_remote_files')
params = Parameters.load(cmd, values_dict={
'url': url, 'private_key': ssh_key_fp, 'study_id': study_id})
elif method == 'transfer':
cmd = qiita_plugin.get_command('download_remote_files')
params = Parameters.load(cmd, values_dict={
'url': url, 'private_key': ssh_key_fp,
'destination': upload_folder})
else:
status = 'error'
message = 'Not a valid method'
if status == 'success':
job = ProcessingJob.create(self.current_user, params, True)
job.submit()
r_client.set(
UPLOAD_STUDY_FORMAT % study_id, dumps({'job_id': job.id}))
self.write({'status': status, 'message': message})
def test_analysis_description_handler_get_request(self):
obs = analysis_description_handler_get_request(1, User('*****@*****.**'))
exp = {'analysis_name': 'SomeAnalysis',
'analysis_id': 1,
'analysis_description': 'A test analysis',
'alert_type': 'info',
'alert_msg': ''}
self.assertEqual(obs, exp)
r_client.set('analysis_1', dumps({'job_id': 'job_id'}))
r_client.set('job_id', dumps({'status_msg': 'running'}))
obs = analysis_description_handler_get_request(1, User('*****@*****.**'))
exp = {'analysis_name': 'SomeAnalysis',
'analysis_id': 1,
'analysis_description': 'A test analysis',
'alert_type': 'info',
'alert_msg': 'An artifact is being deleted from this analysis'}
self.assertEqual(obs, exp)
r_client.set('job_id', dumps(
{'status_msg': 'Success',
'return': {'status': 'danger',
'message': 'Error deleting artifact'}}))
obs = analysis_description_handler_get_request(1, User('*****@*****.**'))
exp = {'analysis_name': 'SomeAnalysis',
'analysis_id': 1,
'analysis_description': 'A test analysis',
'alert_type': 'danger',
'alert_msg': 'Error deleting artifact'}
self.assertEqual(obs, exp)
def sample_template_delete_req(study_id, user_id):
"""Deletes the sample template attached to the study
Parameters
----------
study_id : int
The current study object id
user_id : str
The current user object id
Returns
-------
dict
results dictonary in the format
{'status': status,
'message': msg}
status can be success, warning, or error depending on result
message has the warnings or errors
"""
exists = _check_sample_template_exists(int(study_id))
if exists['status'] != 'success':
return exists
access_error = check_access(int(study_id), user_id)
if access_error:
return access_error
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_template')
params = Parameters.load(cmd, values_dict={'study': int(study_id)})
job = ProcessingJob.create(User(user_id), params)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'status': 'success', 'message': ''}
def list_remote_files(job):
"""Lists valid study files on a remote server
Parameters
----------
job : qiita_db.processing_job.ProcessingJob
The processing job performing the task
"""
with qdb.sql_connection.TRN:
url = job.parameters.values['url']
private_key = job.parameters.values['private_key']
study_id = job.parameters.values['study_id']
try:
files = list_remote(url, private_key)
r_client.set("upload_study_%s" % study_id,
dumps({'job_id': job.id, 'url': url, 'files': files}))
except Exception:
job._set_error(traceback.format_exception(*exc_info()))
else:
job._set_status('success')
finally:
# making sure to always delete the key so Qiita never keeps it
remove(private_key)
def sample_template_handler_delete_request(study_id, user):
"""Deletes the sample template
Parameters
----------
study_id: int
The study to delete the sample information
user: qiita_db.user
The user performing the request
Returns
-------
dict of {'job': str}
job: the id of the job deleting the sample information to the study
Raises
------
HTTPError
404 If the sample template doesn't exist
"""
# Check if the current user has access to the study and if the sample
# template exists
sample_template_checks(study_id, user, check_exists=True)
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_template')
params = Parameters.load(cmd, values_dict={'study': int(study_id)})
job = ProcessingJob.create(user, params, True)
# Store the job if deleteing the sample template
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
def sample_template_handler_delete_request(study_id, user):
"""Deletes the sample template
Parameters
----------
study_id: int
The study to delete the sample information
user: qiita_db.user
The user performing the request
Returns
-------
dict of {'job': str}
job: the id of the job deleting the sample information to the study
Raises
------
HTTPError
404 If the sample template doesn't exist
"""
# Check if the current user has access to the study and if the sample
# template exists
sample_template_checks(study_id, user, check_exists=True)
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_template')
params = Parameters.load(cmd, values_dict={'study': int(study_id)})
job = ProcessingJob.create(user, params, True)
# Store the job if deleteing the sample template
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
def test_get_sample_template_processing_status(self):
key = SAMPLE_TEMPLATE_KEY_FORMAT % 1
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "")
self.assertEqual(obs_am, "")
# With job id and processing
qiita_plugin = qdb.software.Software.from_name_and_version(
'Qiita', 'alpha')
cmd = qiita_plugin.get_command('update_sample_template')
params = qdb.software.Parameters.load(cmd,
values_dict={
'study': 1,
'template_fp': 'ignored'
})
job = qdb.processing_job.ProcessingJob.create(
qdb.user.User('*****@*****.**'), params)
job._set_status('running')
r_client.set(key, dumps({'job_id': job.id}))
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertTrue(obs_proc)
self.assertEqual(obs_at, "info")
self.assertEqual(obs_am,
"This sample template is currently being processed")
# With job id and success
job._set_status('success')
r_client.set(
key,
dumps({
'job_id': job.id,
'alert_type': 'warning',
'alert_msg': 'Some\nwarning'
}))
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "warning")
self.assertEqual(obs_am, "Some</br>warning")
# With job and not success
job = qdb.processing_job.ProcessingJob.create(
qdb.user.User('*****@*****.**'), params)
job._set_status('running')
job._set_error('Some\nerror')
r_client.set(key, dumps({'job_id': job.id}))
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "danger")
self.assertEqual(obs_am, "Some</br>error")
def test_get_sample_template_processing_status(self):
key = SAMPLE_TEMPLATE_KEY_FORMAT % 1
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "")
self.assertEqual(obs_am, "")
# With job id and processing
qiita_plugin = qdb.software.Software.from_name_and_version('Qiita',
'alpha')
cmd = qiita_plugin.get_command('update_sample_template')
params = qdb.software.Parameters.load(
cmd, values_dict={'study': 1, 'template_fp': 'ignored'})
job = qdb.processing_job.ProcessingJob.create(
qdb.user.User('*****@*****.**'), params, True)
job._set_status('running')
r_client.set(key, dumps({'job_id': job.id}))
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertTrue(obs_proc)
self.assertEqual(obs_at, "info")
self.assertEqual(
obs_am, "This sample template is currently being processed")
# With job id and success
job._set_status('success')
r_client.set(key, dumps({'job_id': job.id, 'alert_type': 'warning',
'alert_msg': 'Some\nwarning'}))
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "warning")
self.assertEqual(obs_am, "Some</br>warning")
# With job and not success
job = qdb.processing_job.ProcessingJob.create(
qdb.user.User('*****@*****.**'), params, True)
job._set_status('running')
job._set_error('Some\nerror')
r_client.set(key, dumps({'job_id': job.id}))
obs_proc, obs_at, obs_am = get_sample_template_processing_status(1)
self.assertFalse(obs_proc)
self.assertEqual(obs_at, "danger")
self.assertEqual(obs_am, "Some</br>error")
def test_analysis_description_handler_get_request(self):
obs = analysis_description_handler_get_request(1, User('*****@*****.**'))
exp = {'analysis_name': 'SomeAnalysis',
'analysis_id': 1,
'analysis_description': 'A test analysis',
'analysis_mapping_id': 16,
'analysis_is_public': False,
'alert_type': 'info',
'alert_msg': ''}
self.assertEqual(obs, exp)
r_client.set('analysis_1', dumps({'job_id': 'job_id'}))
r_client.set('job_id', dumps({'status_msg': 'running'}))
obs = analysis_description_handler_get_request(1, User('*****@*****.**'))
exp = {'analysis_name': 'SomeAnalysis',
'analysis_id': 1,
'analysis_description': 'A test analysis',
'analysis_mapping_id': 16,
'analysis_is_public': False,
'alert_type': 'info',
'alert_msg': 'An artifact is being deleted from this analysis'}
self.assertEqual(obs, exp)
r_client.set('job_id', dumps(
{'status_msg': 'Success',
'return': {'status': 'danger',
'message': 'Error deleting artifact'}}))
obs = analysis_description_handler_get_request(1, User('*****@*****.**'))
exp = {'analysis_name': 'SomeAnalysis',
'analysis_id': 1,
'analysis_description': 'A test analysis',
'analysis_mapping_id': 16,
'analysis_is_public': False,
'alert_type': 'danger',
'alert_msg': 'Error deleting artifact'}
self.assertEqual(obs, exp)
def artifact_post_req(user_id, filepaths, artifact_type, name,
prep_template_id, artifact_id=None):
"""Creates the initial artifact for the prep template
Parameters
----------
user_id : str
User adding the atrifact
filepaths : dict of str
Comma-separated list of files to attach to the artifact,
keyed by file type
artifact_type : str
The type of the artifact
name : str
Name to give the artifact
prep_template_id : int or str castable to int
Prep template to attach the artifact to
artifact_id : int or str castable to int, optional
The id of the imported artifact
Returns
-------
dict of objects
A dictionary containing the new artifact ID
{'status': status,
'message': message,
'artifact': id}
"""
prep_template_id = int(prep_template_id)
prep = PrepTemplate(prep_template_id)
study_id = prep.study_id
# First check if the user has access to the study
access_error = check_access(study_id, user_id)
if access_error:
return access_error
user = User(user_id)
if artifact_id:
# if the artifact id has been provided, import the artifact
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('copy_artifact')
params = Parameters.load(cmd, values_dict={'artifact': artifact_id,
'prep_template': prep.id})
job = ProcessingJob.create(user, params, True)
else:
uploads_path = get_mountpoint('uploads')[0][1]
path_builder = partial(join, uploads_path, str(study_id))
cleaned_filepaths = {}
for ftype, file_list in viewitems(filepaths):
# JavaScript sends us this list as a comma-separated list
for fp in file_list.split(','):
# JavaScript will send this value as an empty string if the
# list of files was empty. In such case, the split will
# generate a single element containing the empty string. Check
# for that case here and, if fp is not the empty string,
# proceed to check if the file exists
if fp:
# Check if filepath being passed exists for study
full_fp = path_builder(fp)
exists = check_fp(study_id, full_fp)
if exists['status'] != 'success':
return {'status': 'error',
'message': 'File does not exist: %s' % fp}
if ftype not in cleaned_filepaths:
cleaned_filepaths[ftype] = []
cleaned_filepaths[ftype].append(full_fp)
# This should never happen, but it doesn't hurt to actually have
# a explicit check, in case there is something odd with the JS
if not cleaned_filepaths:
return {'status': 'error',
'message': "Can't create artifact, no files provided."}
# This try/except will catch the case when the plugins are not
# activated so there is no Validate for the given artifact_type
try:
command = Command.get_validator(artifact_type)
except QiitaDBError as e:
return {'status': 'error', 'message': str(e)}
job = ProcessingJob.create(
user,
Parameters.load(command, values_dict={
'template': prep_template_id,
'files': dumps(cleaned_filepaths),
'artifact_type': artifact_type,
'name': name,
'analysis': None,
}), True)
# Submit the job
job.submit()
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep.id,
dumps({'job_id': job.id, 'is_qiita_job': True}))
return {'status': 'success', 'message': ''}
def sample_template_handler_patch_request(user,
req_op,
req_path,
req_value=None,
req_from=None,
direct_upload=False):
"""Patches the sample template
Parameters
----------
user: qiita_db.user.User
The user performing the request
req_op : str
The operation to perform on the sample template
req_path : str
The path to the attribute to patch
req_value : str, optional
The new value
req_from : str, optional
The original path of the element
direct_upload : boolean, optional
If the file being uploaded comes from a direct upload (True)
Returns
-------
Raises
------
HTTPError
400 If the path parameter doens't follow the expected format
400 If the given operation is not supported
"""
req_path = [v for v in req_path.split('/') if v]
# At this point we know the path should be at least length 2
if len(req_path) < 2:
raise HTTPError(400, reason='Incorrect path parameter')
study_id = int(req_path[0])
# Check if the current user has access to the study and if the sample
# template exists
sample_template_checks(study_id, user, check_exists=True)
if req_op == 'remove':
# Path format
# column: study_id/columns/column_name
# sample: study_id/samples/sample_id
if len(req_path) != 3:
raise HTTPError(400, reason='Incorrect path parameter')
attribute = req_path[1]
attr_id = req_path[2]
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_or_column')
params = Parameters.load(cmd,
values_dict={
'obj_class': 'SampleTemplate',
'obj_id': study_id,
'sample_or_col': attribute,
'name': attr_id
})
job = ProcessingJob.create(user, params, True)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
elif req_op == 'replace':
# WARNING: Although the patch operation is a replace, is not a full
# true replace. A replace is in theory equivalent to a remove + add.
# In this case, the replace operation doesn't necessarily removes
# anything (e.g. when only new columns/samples are being added to the)
# sample information.
# Path format: study_id/data
# Forcing to specify data for extensibility. In the future we may want
# to use this function to replace other elements of the sample
# information
if len(req_path) != 2:
raise HTTPError(400, reason='Incorrect path parameter')
attribute = req_path[1]
if attribute == 'data':
# Update the sample information
if req_value is None:
raise HTTPError(400,
reason="Value is required when updating "
"sample information")
if direct_upload:
# We can assume that the file exist as it was generated by
# the system
filepath = req_value
if not exists(filepath):
reason = ('Upload file not found (%s), please report to '
'*****@*****.**' % filepath)
raise HTTPError(404, reason=reason)
else:
# Check if the file exists
fp_rsp = check_fp(study_id, req_value)
if fp_rsp['status'] != 'success':
raise HTTPError(404, reason='Filepath not found')
filepath = fp_rsp['file']
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('update_sample_template')
params = Parameters.load(cmd,
values_dict={
'study': study_id,
'template_fp': filepath
})
job = ProcessingJob.create(user, params, True)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
else:
raise HTTPError(404, reason='Attribute %s not found' % attribute)
else:
raise HTTPError(400,
reason='Operation %s not supported. Current '
'supported operations: remove, replace' % req_op)
<<<<<<< HEAD
msg = ''
msg_level = 'success'
study_id = Artifact(preprocessed_data_id).study
study = Study(study_id)
=======
study = Artifact(preprocessed_data_id).study
>>>>>>> 405cbef0c9f71c620da95a0c1ba6c7d3d588b3ed
state = study.ebi_submission_status
if state == 'submitting':
message = "Cannot resubmit! Current state is: %s" % state
self.display_template(preprocessed_data_id, message, 'danger')
else:
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('submit_to_EBI')
params = Parameters.load(
cmd, values_dict={'artifact': preprocessed_data_id,
'submission_type': submission_type})
job = ProcessingJob.create(user, params, True)
r_client.set('ebi_submission_%s' % preprocessed_data_id,
dumps({'job_id': job.id, 'is_qiita_job': True}))
job.submit()
level = 'success'
message = 'EBI submission started. Job id: %s' % job.id
self.redirect("%s/study/description/%d?level=%s&message=%s" % (
qiita_config.portal_dir, study.id, level, url_escape(message)))
def correct_redis_data(key, cmd, values_dict, user):
"""Corrects the data stored in the redis DB
Parameters
----------
key: str
The redis key to fix
cmd : qiita_db.software.Command
Command to use to create the processing job
values_dict : dict
Dictionary used to instantiate the parameters of the command
user : qiita_db.user. User
The user that will own the job
"""
info = r_client.get(key)
if info:
info = loads(info)
if info['job_id'] is not None:
if 'is_qiita_job' in info:
if info['is_qiita_job']:
try:
job = ProcessingJob(info['job_id'])
payload = {
'job_id': info['job_id'],
'alert_type': info['status'],
'alert_msg': info['alert_msg']
}
r_client.set(key, dumps(payload))
except (QiitaDBUnknownIDError, KeyError):
# We shomehow lost the information of this job
# Simply delete the key
r_client.delete(key)
else:
# These jobs don't contain any information on the live
# dump. We can safely delete the key
r_client.delete(key)
else:
# These jobs don't contain any information on the live
# dump. We can safely delete the key
r_client.delete(key)
else:
# Job is null, we have the information here
if info['status'] == 'success':
# In the success case no information is stored. We can
# safely delete the key
r_client.delete(key)
elif info['status'] == 'warning':
# In case of warning the key message stores the warning
# message. We need to create a new job, mark it as
# successful and store the error message as expected by
# the new structure
params = Parameters.load(cmd, values_dict=values_dict)
job = ProcessingJob.create(user, params)
job._set_status('success')
payload = {
'job_id': job.id,
'alert_type': 'warning',
'alert_msg': info['message']
}
r_client.set(key, dumps(payload))
else:
# The status is error. The key message stores the error
# message. We need to create a new job and mark it as
# failed with the given error message
params = Parameters.load(cmd, values_dict=values_dict)
job = ProcessingJob.create(user, params)
job._set_error(info['message'])
payload = {'job_id': job.id}
r_client.set(key, dumps(payload))
else:
# The key doesn't contain any information. Delete the key
r_client.delete(key)
def sample_template_handler_patch_request(user, req_op, req_path,
req_value=None, req_from=None):
"""Patches the sample template
Parameters
----------
user: qiita_db.user.User
The user performing the request
req_op : str
The operation to perform on the sample template
req_path : str
The path to the attribute to patch
req_value : str, optional
The new value
req_from : str, optional
The original path of the element
Returns
-------
Raises
------
HTTPError
400 If the path parameter doens't follow the expected format
400 If the given operation is not supported
"""
req_path = [v for v in req_path.split('/') if v]
# At this point we know the path should be at least length 2
if len(req_path) < 2:
raise HTTPError(400, reason='Incorrect path parameter')
study_id = int(req_path[0])
# Check if the current user has access to the study and if the sample
# template exists
sample_template_checks(study_id, user, check_exists=True)
if req_op == 'remove':
# Path format
# column: study_id/columns/column_name
# sample: study_id/samples/sample_id
if len(req_path) != 3:
raise HTTPError(400, reason='Incorrect path parameter')
attribute = req_path[1]
attr_id = req_path[2]
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_or_column')
params = Parameters.load(
cmd, values_dict={'obj_class': 'SampleTemplate',
'obj_id': study_id,
'sample_or_col': attribute,
'name': attr_id})
job = ProcessingJob.create(user, params, True)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
elif req_op == 'replace':
# WARNING: Although the patch operation is a replace, is not a full
# true replace. A replace is in theory equivalent to a remove + add.
# In this case, the replace operation doesn't necessarily removes
# anything (e.g. when only new columns/samples are being added to the)
# sample information.
# Path format: study_id/data
# Forcing to specify data for extensibility. In the future we may want
# to use this function to replace other elements of the sample
# information
if len(req_path) != 2:
raise HTTPError(400, reason='Incorrect path parameter')
attribute = req_path[1]
if attribute == 'data':
# Update the sample information
if req_value is None:
raise HTTPError(400, reason="Value is required when updating "
"sample information")
# Check if the file exists
fp_rsp = check_fp(study_id, req_value)
if fp_rsp['status'] != 'success':
raise HTTPError(404, reason='Filepath not found')
filepath = fp_rsp['file']
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('update_sample_template')
params = Parameters.load(
cmd, values_dict={'study': study_id,
'template_fp': filepath})
job = ProcessingJob.create(user, params, True)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
else:
raise HTTPError(404, reason='Attribute %s not found' % attribute)
else:
raise HTTPError(400, reason='Operation %s not supported. Current '
'supported operations: remove, replace' % req_op)
def test_analysis_description_handler_get_request(self):
obs = analysis_description_handler_get_request(1, User('*****@*****.**'))
exp = {
'analysis_name': 'SomeAnalysis',
'analysis_id': 1,
'analysis_description': 'A test analysis',
'analysis_mapping_id': 16,
'analysis_is_public': False,
'alert_type': 'info',
'artifacts': {
4: (1, 'Identification of the Microbiomes for Cannabis '
'Soils', ('Pick closed-reference OTUs | Split '
'libraries FASTQ', 'QIIME v1.9.1'), [
'1.SKB7.640196', '1.SKB8.640193',
'1.SKD8.640184', '1.SKM4.640180',
'1.SKM9.640192'
]),
5: (1, 'Identification of the Microbiomes for Cannabis '
'Soils', ('Pick closed-reference OTUs | Split '
'libraries FASTQ', 'QIIME v1.9.1'), [
'1.SKB7.640196', '1.SKB8.640193',
'1.SKD8.640184', '1.SKM4.640180',
'1.SKM9.640192'
]),
6: (1, 'Identification of the Microbiomes for Cannabis '
'Soils', ('Pick closed-reference OTUs | Split '
'libraries FASTQ', 'QIIME v1.9.1'), [
'1.SKB7.640196', '1.SKB8.640193',
'1.SKD8.640184', '1.SKM4.640180',
'1.SKM9.640192'
])
},
'alert_msg': ''
}
self.assertEqual(obs, exp)
r_client.set('analysis_1', dumps({'job_id': 'job_id'}))
r_client.set('job_id', dumps({'status_msg': 'running'}))
obs = analysis_description_handler_get_request(1, User('*****@*****.**'))
exp = {
'analysis_name': 'SomeAnalysis',
'analysis_id': 1,
'analysis_description': 'A test analysis',
'analysis_mapping_id': 16,
'analysis_is_public': False,
'alert_type': 'info',
'artifacts': {
4: (1, 'Identification of the Microbiomes for Cannabis '
'Soils', ('Pick closed-reference OTUs | Split '
'libraries FASTQ', 'QIIME v1.9.1'), [
'1.SKB7.640196', '1.SKB8.640193',
'1.SKD8.640184', '1.SKM4.640180',
'1.SKM9.640192'
]),
5: (1, 'Identification of the Microbiomes for Cannabis '
'Soils', ('Pick closed-reference OTUs | Split '
'libraries FASTQ', 'QIIME v1.9.1'), [
'1.SKB7.640196', '1.SKB8.640193',
'1.SKD8.640184', '1.SKM4.640180',
'1.SKM9.640192'
]),
6: (1, 'Identification of the Microbiomes for Cannabis '
'Soils', ('Pick closed-reference OTUs | Split '
'libraries FASTQ', 'QIIME v1.9.1'), [
'1.SKB7.640196', '1.SKB8.640193',
'1.SKD8.640184', '1.SKM4.640180',
'1.SKM9.640192'
])
},
'alert_msg': 'An artifact is being deleted from this analysis'
}
self.assertEqual(obs, exp)
r_client.set(
'job_id',
dumps({
'status_msg': 'Success',
'return': {
'status': 'danger',
'message': 'Error deleting artifact'
}
}))
obs = analysis_description_handler_get_request(1, User('*****@*****.**'))
exp = {
'analysis_name': 'SomeAnalysis',
'analysis_id': 1,
'analysis_description': 'A test analysis',
'analysis_mapping_id': 16,
'analysis_is_public': False,
'alert_type': 'danger',
'artifacts': {
4: (1, 'Identification of the Microbiomes for Cannabis '
'Soils', ('Pick closed-reference OTUs | Split '
'libraries FASTQ', 'QIIME v1.9.1'), [
'1.SKB7.640196', '1.SKB8.640193',
'1.SKD8.640184', '1.SKM4.640180',
'1.SKM9.640192'
]),
5: (1, 'Identification of the Microbiomes for Cannabis '
'Soils', ('Pick closed-reference OTUs | Split '
'libraries FASTQ', 'QIIME v1.9.1'), [
'1.SKB7.640196', '1.SKB8.640193',
'1.SKD8.640184', '1.SKM4.640180',
'1.SKM9.640192'
]),
6: (1, 'Identification of the Microbiomes for Cannabis '
'Soils', ('Pick closed-reference OTUs | Split '
'libraries FASTQ', 'QIIME v1.9.1'), [
'1.SKB7.640196', '1.SKB8.640193',
'1.SKD8.640184', '1.SKM4.640180',
'1.SKM9.640192'
])
},
'alert_msg': 'Error deleting artifact'
}
self.assertEqual(obs, exp)
def post(self):
user_email = self.get_argument('user_email')
job_id = self.get_argument('job_id', None)
prep_id = self.get_argument('prep_id', None)
atype = self.get_argument('artifact_type')
aname = self.get_argument('command_artifact_name', 'Name')
files = self.get_argument('files')
if job_id is None and prep_id is None:
raise HTTPError(400,
reason='You need to specify a job_id or a prep_id')
if job_id is not None and prep_id is not None:
raise HTTPError(
400, reason='You need to specify only a job_id or a prep_id')
user = qdb.user.User(user_email)
values = {
'files': files,
'artifact_type': atype,
'name': aname,
# leaving here in case we need to add a way to add an artifact
# directly to an analysis, for more information see
# ProcessingJob._complete_artifact_transformation
'analysis': None
}
PJ = qdb.processing_job.ProcessingJob
if job_id is not None:
TN = qdb.sql_connection.TRN
job = PJ(job_id)
with TN:
sql = """SELECT command_output_id
FROM qiita.command_output
WHERE name = %s AND command_id = %s"""
TN.add(sql, [aname, job.command.id])
results = TN.execute_fetchflatten()
if len(results) < 1:
raise HTTPError(
400, 'The command_artifact_name does not '
'exist in the command')
cmd_out_id = results[0]
provenance = {
'job': job_id,
'cmd_out_id': cmd_out_id,
# direct_creation is a flag to avoid having to wait
# for the complete job to create the new artifact,
# which is normally ran during regular processing.
# Skipping is fine because we are adding an artifact
# to an existing job outside of regular processing
'direct_creation': True,
'name': aname
}
values['provenance'] = dumps(provenance)
# inherint the first prep info file from the first input artifact
prep_id = job.input_artifacts[0].prep_templates[0].id
else:
prep_id = int(prep_id)
values['template'] = prep_id
cmd = qdb.software.Command.get_validator(atype)
params = qdb.software.Parameters.load(cmd, values_dict=values)
new_job = PJ.create(user, params, True)
new_job.submit()
r_client.set('prep_template_%d' % prep_id,
dumps({
'job_id': new_job.id,
'is_qiita_job': True
}))
self.write(new_job.id)
self.finish()
def artifact_post_req(user_id,
filepaths,
artifact_type,
name,
prep_template_id,
artifact_id=None):
"""Creates the initial artifact for the prep template
Parameters
----------
user_id : str
User adding the atrifact
filepaths : dict of str
Comma-separated list of files to attach to the artifact,
keyed by file type
artifact_type : str
The type of the artifact
name : str
Name to give the artifact
prep_template_id : int or str castable to int
Prep template to attach the artifact to
artifact_id : int or str castable to int, optional
The id of the imported artifact
Returns
-------
dict of objects
A dictionary containing the new artifact ID
{'status': status,
'message': message,
'artifact': id}
"""
prep_template_id = int(prep_template_id)
prep = PrepTemplate(prep_template_id)
study_id = prep.study_id
# First check if the user has access to the study
access_error = check_access(study_id, user_id)
if access_error:
return access_error
user = User(user_id)
if artifact_id:
# if the artifact id has been provided, import the artifact
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('copy_artifact')
params = Parameters.load(cmd,
values_dict={
'artifact': artifact_id,
'prep_template': prep.id
})
job = ProcessingJob.create(user, params, True)
else:
uploads_path = get_mountpoint('uploads')[0][1]
path_builder = partial(join, uploads_path, str(study_id))
cleaned_filepaths = {}
for ftype, file_list in filepaths.items():
# JavaScript sends us this list as a comma-separated list
for fp in file_list.split(','):
# JavaScript will send this value as an empty string if the
# list of files was empty. In such case, the split will
# generate a single element containing the empty string. Check
# for that case here and, if fp is not the empty string,
# proceed to check if the file exists
if fp:
# Check if filepath being passed exists for study
full_fp = path_builder(fp)
exists = check_fp(study_id, full_fp)
if exists['status'] != 'success':
return {
'status': 'error',
'message': 'File does not exist: %s' % fp
}
if ftype not in cleaned_filepaths:
cleaned_filepaths[ftype] = []
cleaned_filepaths[ftype].append(full_fp)
# This should never happen, but it doesn't hurt to actually have
# a explicit check, in case there is something odd with the JS
if not cleaned_filepaths:
return {
'status': 'error',
'message': "Can't create artifact, no files provided."
}
# This try/except will catch the case when the plugins are not
# activated so there is no Validate for the given artifact_type
try:
command = Command.get_validator(artifact_type)
except QiitaDBError as e:
return {'status': 'error', 'message': str(e)}
job = ProcessingJob.create(
user,
Parameters.load(command,
values_dict={
'template': prep_template_id,
'files': dumps(cleaned_filepaths),
'artifact_type': artifact_type,
'name': name,
'analysis': None,
}), True)
# Submit the job
job.submit()
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep.id,
dumps({
'job_id': job.id,
'is_qiita_job': True
}))
return {'status': 'success', 'message': ''}
def sample_template_patch_request(user_id,
req_op,
req_path,
req_value=None,
req_from=None):
"""Modifies an attribute of the artifact
Parameters
----------
user_id : str
The id of the user performing the patch operation
req_op : str
The operation to perform on the artifact
req_path : str
The prep information and attribute to patch
req_value : str, optional
The value that needs to be modified
req_from : str, optional
The original path of the element
Returns
-------
dict of {str, str}
A dictionary with the following keys:
- status: str, whether if the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
"""
if req_op == 'remove':
req_path = [v for v in req_path.split('/') if v]
# format
# column: study_id/row_id/columns/column_name
# sample: study_id/row_id/samples/sample_id
if len(req_path) != 4:
return {'status': 'error', 'message': 'Incorrect path parameter'}
st_id = req_path[0]
row_id = req_path[1]
attribute = req_path[2]
attr_id = req_path[3]
# Check if the user actually has access to the template
st = SampleTemplate(st_id)
access_error = check_access(st.study_id, user_id)
if access_error:
return access_error
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_or_column')
params = Parameters.load(cmd,
values_dict={
'obj_class': 'SampleTemplate',
'obj_id': int(st_id),
'sample_or_col': attribute,
'name': attr_id
})
job = ProcessingJob.create(User(user_id), params)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % st_id,
dumps({'job_id': job.id}))
job.submit()
return {'status': 'success', 'message': '', 'row_id': row_id}
else:
return {
'status':
'error',
'message':
'Operation "%s" not supported. '
'Current supported operations: remove' % req_op,
'row_id':
0
}
def prep_template_patch_req(user_id, req_op, req_path, req_value=None,
req_from=None):
"""Modifies an attribute of the prep template
Parameters
----------
user_id : str
The id of the user performing the patch operation
req_op : str
The operation to perform on the prep information
req_path : str
The prep information and attribute to patch
req_value : str, optional
The value that needs to be modified
req_from : str, optional
The original path of the element
Returns
-------
dict of {str, str, str}
A dictionary with the following keys:
- status: str, whether if the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
- row_id: str, the row_id that we tried to delete
"""
req_path = [v for v in req_path.split('/') if v]
if req_op == 'replace':
# The structure of the path should be /prep_id/attribute_to_modify/
# so if we don't have those 2 elements, we should return an error
if len(req_path) != 2:
return {'status': 'error',
'message': 'Incorrect path parameter'}
prep_id = int(req_path[0])
attribute = req_path[1]
# Check if the user actually has access to the prep template
prep = PrepTemplate(prep_id)
access_error = check_access(prep.study_id, user_id)
if access_error:
return access_error
status = 'success'
msg = ''
if attribute == 'investigation_type':
prep.investigation_type = req_value
elif attribute == 'data':
fp = check_fp(prep.study_id, req_value)
if fp['status'] != 'success':
return fp
fp = fp['file']
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('update_prep_template')
params = Parameters.load(
cmd, values_dict={'prep_template': prep_id, 'template_fp': fp})
job = ProcessingJob.create(User(user_id), params, True)
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id,
dumps({'job_id': job.id}))
job.submit()
elif attribute == 'name':
prep.name = req_value.strip()
else:
# We don't understand the attribute so return an error
return {'status': 'error',
'message': 'Attribute "%s" not found. '
'Please, check the path parameter' % attribute}
return {'status': status, 'message': msg}
elif req_op == 'remove':
# The structure of the path should be:
# /prep_id/row_id/{columns|samples}/name
if len(req_path) != 4:
return {'status': 'error',
'message': 'Incorrect path parameter'}
prep_id = int(req_path[0])
row_id = req_path[1]
attribute = req_path[2]
attr_id = req_path[3]
# Check if the user actually has access to the study
pt = PrepTemplate(prep_id)
access_error = check_access(pt.study_id, user_id)
if access_error:
return access_error
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_or_column')
params = Parameters.load(
cmd, values_dict={'obj_class': 'PrepTemplate',
'obj_id': prep_id,
'sample_or_col': attribute,
'name': attr_id})
job = ProcessingJob.create(User(user_id), params, True)
# Store the job id attaching it to the sample template id
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id,
dumps({'job_id': job.id}))
job.submit()
return {'status': 'success', 'message': '', 'row_id': row_id}
else:
return {'status': 'error',
'message': 'Operation "%s" not supported. '
'Current supported operations: replace, remove'
% req_op,
'row_id': '0'}
def sample_template_post_req(study_id, user_id, data_type, sample_template):
"""Creates the sample template from the given file
Parameters
----------
study_id : int
The current study object id
user_id : str
The current user object id
data_type : str
Data type for the sample template
sample_template : str
filename to use for creation
Returns
-------
dict
results dictonary in the format
{'status': status,
'message': msg,
'file': sample_template}
status can be success, warning, or error depending on result
message has the warnings or errors
file has the file name
"""
study_id = int(study_id)
access_error = check_access(study_id, user_id)
if access_error:
return access_error
fp_rsp = check_fp(study_id, sample_template)
if fp_rsp['status'] != 'success':
# Unknown filepath, so return the error message
return fp_rsp
fp_rsp = fp_rsp['file']
# Define here the message and message level in case of success
is_mapping_file = looks_like_qiime_mapping_file(fp_rsp)
if is_mapping_file and not data_type:
return {
'status': 'error',
'message': 'Please, choose a data type if uploading a '
'QIIME mapping file',
'file': sample_template
}
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('create_sample_template')
params = Parameters.load(cmd,
values_dict={
'fp': fp_rsp,
'study_id': study_id,
'is_mapping_file': is_mapping_file,
'data_type': data_type
})
job = ProcessingJob.create(User(user_id), params)
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
# Store the job id attaching it to the sample template id
job.submit()
return {'status': 'success', 'message': '', 'file': sample_template}
def correct_redis_data(key, cmd, values_dict, user):
"""Corrects the data stored in the redis DB
Parameters
----------
key: str
The redis key to fix
cmd : qiita_db.software.Command
Command to use to create the processing job
values_dict : dict
Dictionary used to instantiate the parameters of the command
user : qiita_db.user. User
The user that will own the job
"""
info = r_client.get(key)
if info:
info = loads(info)
if info['job_id'] is not None:
if 'is_qiita_job' in info:
if info['is_qiita_job']:
try:
job = ProcessingJob(info['job_id'])
payload = {'job_id': info['job_id'],
'alert_type': info['status'],
'alert_msg': info['alert_msg']}
r_client.set(key, dumps(payload))
except (QiitaDBUnknownIDError, KeyError):
# We shomehow lost the information of this job
# Simply delete the key
r_client.delete(key)
else:
# These jobs don't contain any information on the live
# dump. We can safely delete the key
r_client.delete(key)
else:
# These jobs don't contain any information on the live
# dump. We can safely delete the key
r_client.delete(key)
else:
# Job is null, we have the information here
if info['status'] == 'success':
# In the success case no information is stored. We can
# safely delete the key
r_client.delete(key)
elif info['status'] == 'warning':
# In case of warning the key message stores the warning
# message. We need to create a new job, mark it as
# successful and store the error message as expected by
# the new structure
params = Parameters.load(cmd, values_dict=values_dict)
job = ProcessingJob.create(user, params)
job._set_status('success')
payload = {'job_id': job.id,
'alert_type': 'warning',
'alert_msg': info['message']}
r_client.set(key, dumps(payload))
else:
# The status is error. The key message stores the error
# message. We need to create a new job and mark it as
# failed with the given error message
params = Parameters.load(cmd, values_dict=values_dict)
job = ProcessingJob.create(user, params)
job._set_error(info['message'])
payload = {'job_id': job.id}
r_client.set(key, dumps(payload))
else:
# The key doesn't contain any information. Delete the key
r_client.delete(key)