def post(self, study_id):
method = self.get_argument('remote-request-type')
url = self.get_argument('inputURL')
ssh_key = self.request.files['ssh-key'][0]['body']
status = 'success'
message = ''
try:
study = Study(int(study_id))
except QiitaDBUnknownIDError:
raise HTTPError(404, reason="Study %s does not exist" % study_id)
check_access(self.current_user,
study,
no_public=True,
raise_error=True)
_, upload_folder = get_mountpoint("uploads")[0]
upload_folder = join(upload_folder, study_id)
ssh_key_fp = join(upload_folder, '.key.txt')
create_nested_path(upload_folder)
with open(ssh_key_fp, 'wb') as f:
f.write(ssh_key)
chmod(ssh_key_fp, 0o600)
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
if method == 'list':
cmd = qiita_plugin.get_command('list_remote_files')
params = Parameters.load(cmd,
values_dict={
'url': url,
'private_key': ssh_key_fp,
'study_id': study_id
})
elif method == 'transfer':
cmd = qiita_plugin.get_command('download_remote_files')
params = Parameters.load(cmd,
values_dict={
'url': url,
'private_key': ssh_key_fp,
'destination': upload_folder
})
else:
status = 'error'
message = 'Not a valid method'
if status == 'success':
job = ProcessingJob.create(self.current_user, params, True)
job.submit()
r_client.set(UPLOAD_STUDY_FORMAT % study_id,
dumps({'job_id': job.id}))
self.write({'status': status, 'message': message})
def artifact_post_req(user, artifact_id):
"""Deletes the artifact
Parameters
----------
user : qiita_db.user.User
The user requesting the action
artifact_id : int
Id of the artifact being deleted
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
check_artifact_access(user, artifact)
analysis = artifact.analysis
if analysis:
# Do something when deleting in the analysis part to keep track of it
redis_key = "analysis_%s" % analysis.id
else:
pt_id = artifact.prep_templates[0].id
redis_key = PREP_TEMPLATE_KEY_FORMAT % pt_id
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_artifact')
params = Parameters.load(cmd, values_dict={'artifact': artifact_id})
job = ProcessingJob.create(user, params, True)
r_client.set(redis_key, dumps({'job_id': job.id, 'is_qiita_job': True}))
job.submit()
return {'job': job.id}
def study_delete_req(study_id, user_id):
"""Delete a given study
Parameters
----------
study_id : int
Study id to delete
user_id : str
User requesting the deletion
Returns
-------
dict
Status of deletion, in the format
{status: status,
message: message}
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_study')
params = Parameters.load(cmd, values_dict={'study': study_id})
job = ProcessingJob.create(User(user_id), params, True)
# Store the job id attaching it to the sample template id
r_client.set(STUDY_KEY_FORMAT % study_id, dumps({'job_id': job.id}))
job.submit()
return {'status': 'success', 'message': ''}
def post(self, preprocessed_data_id):
user = self.current_user
# make sure user is admin and can therefore actually submit to EBI
if user.level != 'admin':
raise HTTPError(403, reason="User %s cannot submit to EBI!" %
user.id)
submission_type = self.get_argument('submission_type')
if submission_type not in ['ADD', 'MODIFY']:
raise HTTPError(403, reason="User: %s, %s is not a recognized "
"submission type" % (user.id, submission_type))
study = Artifact(preprocessed_data_id).study
state = study.ebi_submission_status
if state == 'submitting':
message = "Cannot resubmit! Current state is: %s" % state
self.display_template(preprocessed_data_id, message, 'danger')
else:
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('submit_to_EBI')
params = Parameters.load(
cmd, values_dict={'artifact': preprocessed_data_id,
'submission_type': submission_type})
job = ProcessingJob.create(user, params, True)
r_client.set('ebi_submission_%s' % preprocessed_data_id,
dumps({'job_id': job.id, 'is_qiita_job': True}))
job.submit()
level = 'success'
message = 'EBI submission started. Job id: %s' % job.id
self.redirect("%s/study/description/%d?level=%s&message=%s" % (
qiita_config.portal_dir, study.id, level, url_escape(message)))
def post(self, preprocessed_data_id):
user = self.current_user
# make sure user is admin and can therefore actually submit to EBI
if user.level != 'admin':
raise HTTPError(403, reason="User %s cannot submit to EBI!" %
user.id)
submission_type = self.get_argument('submission_type')
if submission_type not in ['ADD', 'MODIFY']:
raise HTTPError(403, reason="User: %s, %s is not a recognized "
"submission type" % (user.id, submission_type))
study = Artifact(preprocessed_data_id).study
state = study.ebi_submission_status
if state == 'submitting':
message = "Cannot resubmit! Current state is: %s" % state
self.display_template(preprocessed_data_id, message, 'danger')
else:
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('submit_to_EBI')
params = Parameters.load(
cmd, values_dict={'artifact': preprocessed_data_id,
'submission_type': submission_type})
job = ProcessingJob.create(user, params, True)
r_client.set('ebi_submission_%s' % preprocessed_data_id,
dumps({'job_id': job.id, 'is_qiita_job': True}))
job.submit()
level = 'success'
message = 'EBI submission started. Job id: %s' % job.id
self.redirect("%s/study/description/%d?level=%s&message=%s" % (
qiita_config.portal_dir, study.id, level, url_escape(message)))
def post(self, preprocessed_data_id):
user = self.current_user
# make sure user is admin and can therefore actually submit to VAMPS
if user.level != 'admin':
raise HTTPError(403, "User %s cannot submit to VAMPS!" %
user.id)
msg = ''
msg_level = 'success'
plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = plugin.get_command('submit_to_VAMPS')
artifact = Artifact(preprocessed_data_id)
# Check if the artifact is already being submitted to VAMPS
is_being_submitted = any(
[j.status in ('queued', 'running')
for j in artifact.jobs(cmd=cmd)])
if is_being_submitted == 'submitting':
msg = "Cannot resubmit! Data is already being submitted"
msg_level = 'danger'
self.display_template(preprocessed_data_id, msg, msg_level)
else:
params = Parameters.load(
cmd, values_dict={'artifact': preprocessed_data_id})
job = ProcessingJob.create(user, params, True)
job.submit()
self.redirect('/study/description/%s' % artifact.study.study_id)
def artifact_post_req(user, artifact_id):
"""Deletes the artifact
Parameters
----------
user : qiita_db.user.User
The user requesting the action
artifact_id : int
Id of the artifact being deleted
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
check_artifact_access(user, artifact)
analysis = artifact.analysis
if analysis:
# Do something when deleting in the analysis part to keep track of it
redis_key = "analysis_%s" % analysis.id
else:
pt_id = artifact.prep_templates[0].id
redis_key = PREP_TEMPLATE_KEY_FORMAT % pt_id
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_artifact')
params = Parameters.load(cmd, values_dict={'artifact': artifact_id})
job = ProcessingJob.create(user, params)
r_client.set(redis_key, dumps({'job_id': job.id, 'is_qiita_job': True}))
job.submit()
def study_delete_req(study_id, user_id):
"""Delete a given study
Parameters
----------
study_id : int
Study id to delete
user_id : str
User requesting the deletion
Returns
-------
dict
Status of deletion, in the format
{status: status,
message: message}
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_study')
params = Parameters.load(cmd, values_dict={'study': study_id})
job = ProcessingJob.create(User(user_id), params, True)
# Store the job id attaching it to the sample template id
r_client.set(STUDY_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'status': 'success', 'message': ''}
def post(self, preprocessed_data_id):
user = self.current_user
# make sure user is admin and can therefore actually submit to VAMPS
if user.level != 'admin':
raise HTTPError(403, "User %s cannot submit to VAMPS!" % user.id)
msg = ''
msg_level = 'success'
plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = plugin.get_command('submit_to_VAMPS')
artifact = Artifact(preprocessed_data_id)
# Check if the artifact is already being submitted to VAMPS
is_being_submitted = any([
j.status in ('queued', 'running') for j in artifact.jobs(cmd=cmd)
])
if is_being_submitted == 'submitting':
msg = "Cannot resubmit! Data is already being submitted"
msg_level = 'danger'
self.display_template(preprocessed_data_id, msg, msg_level)
else:
params = Parameters.load(
cmd, values_dict={'artifact': preprocessed_data_id})
job = ProcessingJob.create(user, params)
job.submit()
self.redirect('/study/description/%s' % artifact.study.study_id)
def _create_job(self, cmd_name, values_dict):
self.user = User('*****@*****.**')
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command(cmd_name)
params = Parameters.load(cmd, values_dict=values_dict)
job = ProcessingJob.create(self.user, params, True)
job._set_status('queued')
return job
def _create_job(self, cmd_name, values_dict):
self.user = User('*****@*****.**')
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command(cmd_name)
params = Parameters.load(cmd, values_dict=values_dict)
job = ProcessingJob.create(self.user, params, True)
job._set_status('queued')
return job
def sample_template_put_req(study_id, user_id, sample_template):
"""Updates a sample template using the given file
Parameters
----------
study_id : int
The current study object id
user_id : str
The current user object id
sample_template : str
filename to use for updating
Returns
-------
dict
results dictonary in the format
{'status': status,
'message': msg,
'file': sample_template}
status can be success, warning, or error depending on result
message has the warnings or errors
file has the file name
"""
exists = _check_sample_template_exists(int(study_id))
if exists['status'] != 'success':
return exists
access_error = check_access(int(study_id), user_id)
if access_error:
return access_error
fp_rsp = check_fp(study_id, sample_template)
if fp_rsp['status'] != 'success':
# Unknown filepath, so return the error message
return fp_rsp
fp_rsp = fp_rsp['file']
msg = ''
status = 'success'
# Offload the update of the sample template to the cluster
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('update_sample_template')
params = Parameters.load(cmd,
values_dict={
'study': int(study_id),
'template_fp': fp_rsp
})
job = ProcessingJob.create(User(user_id), params)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'status': status, 'message': msg, 'file': sample_template}
def sample_template_handler_post_request(study_id,
user,
filepath,
data_type=None):
"""Creates a new sample template
Parameters
----------
study_id: int
The study to add the sample information
user: qiita_db.user import User
The user performing the request
filepath: str
The path to the sample template file
data_type: str, optional
If filepath is a QIIME mapping file, the data type of the prep
information file
Returns
-------
dict of {'job': str}
job: the id of the job adding the sample information to the study
Raises
------
HTTPError
404 if the filepath doesn't exist
"""
# Check if the current user has access to the study
sample_template_checks(study_id, user)
# Check if the file exists
fp_rsp = check_fp(study_id, filepath)
if fp_rsp['status'] != 'success':
raise HTTPError(404, 'Filepath not found')
filepath = fp_rsp['file']
is_mapping_file = looks_like_qiime_mapping_file(filepath)
if is_mapping_file and not data_type:
raise HTTPError(
400, 'Please, choose a data type if uploading a '
'QIIME mapping file')
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('create_sample_template')
params = Parameters.load(cmd,
values_dict={
'fp': filepath,
'study_id': study_id,
'is_mapping_file': is_mapping_file,
'data_type': data_type
})
job = ProcessingJob.create(user, params, True)
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
def post(self, study_id):
method = self.get_argument('remote-request-type')
url = self.get_argument('inputURL')
ssh_key = self.request.files['ssh-key'][0]['body']
status = 'success'
message = ''
try:
study = Study(int(study_id))
except QiitaDBUnknownIDError:
raise HTTPError(404, reason="Study %s does not exist" % study_id)
check_access(
self.current_user, study, no_public=True, raise_error=True)
_, upload_folder = get_mountpoint("uploads")[0]
upload_folder = join(upload_folder, study_id)
ssh_key_fp = join(upload_folder, '.key.txt')
create_nested_path(upload_folder)
with open(ssh_key_fp, 'w') as f:
f.write(ssh_key)
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
if method == 'list':
cmd = qiita_plugin.get_command('list_remote_files')
params = Parameters.load(cmd, values_dict={
'url': url, 'private_key': ssh_key_fp, 'study_id': study_id})
elif method == 'transfer':
cmd = qiita_plugin.get_command('download_remote_files')
params = Parameters.load(cmd, values_dict={
'url': url, 'private_key': ssh_key_fp,
'destination': upload_folder})
else:
status = 'error'
message = 'Not a valid method'
if status == 'success':
job = ProcessingJob.create(self.current_user, params, True)
job.submit()
r_client.set(
UPLOAD_STUDY_FORMAT % study_id, dumps({'job_id': job.id}))
self.write({'status': status, 'message': message})
def sample_template_handler_post_request(study_id, user, filepath,
data_type=None, direct_upload=False):
"""Creates a new sample template
Parameters
----------
study_id: int
The study to add the sample information
user: qiita_db.user import User
The user performing the request
filepath: str
The path to the sample template file
data_type: str, optional
If filepath is a QIIME mapping file, the data type of the prep
information file
direct_upload: boolean, optional
If filepath is a direct upload; if False we need to process the
filepath as part of the study upload folder
Returns
-------
dict of {'job': str}
job: the id of the job adding the sample information to the study
Raises
------
HTTPError
404 if the filepath doesn't exist
"""
# Check if the current user has access to the study
sample_template_checks(study_id, user)
# Check if the file exists
if not direct_upload:
fp_rsp = check_fp(study_id, filepath)
if fp_rsp['status'] != 'success':
raise HTTPError(404, reason='Filepath not found')
filepath = fp_rsp['file']
is_mapping_file = looks_like_qiime_mapping_file(filepath)
if is_mapping_file and not data_type:
raise HTTPError(400, reason='Please, choose a data type if uploading '
'a QIIME mapping file')
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('create_sample_template')
params = Parameters.load(
cmd, values_dict={'fp': filepath, 'study_id': study_id,
'is_mapping_file': is_mapping_file,
'data_type': data_type})
job = ProcessingJob.create(user, params, True)
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
def workflow_handler_post_req(user_id, command_id, params):
"""Creates a new workflow in the system
Parameters
----------
user_id : str
The user creating the workflow
command_id : int
The first command to execute in the workflow
params : str
JSON representations of the parameters for the first command of
the workflow
Returns
-------
dict of objects
A dictionary containing the commands information
{'status': str,
'message': str,
'workflow_id': int}
"""
parameters = Parameters.load(Command(command_id), json_str=params)
status = 'success'
message = ''
try:
wf = ProcessingWorkflow.from_scratch(User(user_id), parameters)
except Exception as exc:
wf = None
wf_id = None
job_info = None
status = 'error'
message = str(exc)
if wf is not None:
# this is safe as we are creating the workflow for the first time
# and there is only one node. Remember networkx doesn't assure order
# of nodes
job = list(wf.graph.nodes())[0]
inputs = [a.id for a in job.input_artifacts]
job_cmd = job.command
wf_id = wf.id
job_info = {
'id': job.id,
'inputs': inputs,
'label': job_cmd.name,
'outputs': job_cmd.outputs
}
return {
'status': status,
'message': message,
'workflow_id': wf_id,
'job': job_info
}
def test_workflow_handler_patch_req(self):
# Create a new workflow so it is in construction
exp_command = Command(1)
json_str = (
'{"input_data": 1, "max_barcode_errors": 1.5, '
'"barcode_type": "golay_12", "max_bad_run_length": 3, '
'"rev_comp": false, "phred_quality_threshold": 3, '
'"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
'"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
exp_params = Parameters.load(exp_command, json_str=json_str)
exp_user = User('*****@*****.**')
name = "Test processing workflow"
# tests success
wf = ProcessingWorkflow.from_scratch(exp_user,
exp_params,
name=name,
force=True)
graph = wf.graph
nodes = list(graph.nodes())
job_id = nodes[0].id
value = {
'dflt_params': 10,
'connections': {
job_id: {
'demultiplexed': 'input_data'
}
}
}
obs = workflow_handler_patch_req('add',
'/%s/' % wf.id,
req_value=dumps(value))
new_jobs = set(wf.graph.nodes()) - set(nodes)
self.assertEqual(len(new_jobs), 1)
new_job = new_jobs.pop()
exp = {
'status': 'success',
'message': '',
'job': {
'id': new_job.id,
'inputs': [job_id],
'label': 'Pick closed-reference OTUs',
'outputs': [['OTU table', 'BIOM']]
}
}
self.assertEqual(obs, exp)
obs = workflow_handler_patch_req('remove',
'/%s/%s/' % (wf.id, new_job.id))
exp = {'status': 'success', 'message': ''}
jobs = set(wf.graph.nodes()) - set(nodes)
self.assertEqual(jobs, set())
def test_artifact_summary_post_request(self):
# No access
with self.assertRaises(QiitaHTTPError):
artifact_summary_post_request(User('*****@*****.**'), 1)
# Returns already existing job
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 2}))
job._set_status('queued')
obs = artifact_summary_post_request(User('*****@*****.**'), 2)
exp = {'job': [job.id, 'queued', None]}
self.assertEqual(obs, exp)
def test_artifact_summary_post_request(self):
# No access
with self.assertRaises(QiitaHTTPError):
artifact_summary_post_request(User('*****@*****.**'), 1)
# Returns already existing job
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 2})
)
job._set_status('queued')
obs = artifact_summary_post_request(User('*****@*****.**'), 2)
exp = {'job': [job.id, 'queued', None]}
self.assertEqual(obs, exp)
def post(self):
analysis_id = int(self.get_argument('analysis_id'))
user = self.current_user
check_analysis_access(user, Analysis(analysis_id))
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_analysis')
params = Parameters.load(cmd, values_dict={'analysis_id': analysis_id})
job = ProcessingJob.create(user, params, True)
# Store the job id attaching it to the sample template id
r_client.set('analysis_delete_%d' % analysis_id,
dumps({'job_id': job.id}))
job.submit()
self.redirect("%s/analysis/list/" % (qiita_config.portal_dir))
def artifact_summary_post_request(user_id, artifact_id):
"""Launches the HTML summary generation and returns the job information
Parameters
----------
user_id : str
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the artifact summary information
{'status': str,
'message': str,
'job': list of [str, str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
access_error = check_access(artifact.study.id, user_id)
if access_error:
return access_error
# Check if the summary is being generated or has been already generated
command = Command.get_html_generator(artifact.artifact_type)
jobs = artifact.jobs(cmd=command)
jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
if jobs:
# The HTML summary is either being generated or already generated.
# Return the information of that job so we only generate the HTML
# once
job = jobs[0]
else:
# Create a new job to generate the HTML summary and return the newly
# created job information
job = ProcessingJob.create(
User(user_id),
Parameters.load(command, values_dict={'input_data': artifact_id}))
job.submit()
return {
'status': 'success',
'message': '',
'job': [job.id, job.status, job.step]
}
def test_patch(self):
# Create a new job - through a workflow since that is the only way
# of creating jobs in the interface
exp_command = Command(1)
json_str = (
'{"input_data": 1, "max_barcode_errors": 1.5, '
'"barcode_type": "golay_12", "max_bad_run_length": 3, '
'"rev_comp": false, "phred_quality_threshold": 3, '
'"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
'"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
exp_params = Parameters.load(exp_command, json_str=json_str)
exp_user = User('*****@*****.**')
name = "Test processing workflow"
# tests success
wf = ProcessingWorkflow.from_scratch(exp_user,
exp_params,
name=name,
force=True)
graph = wf.graph
nodes = graph.nodes()
job_id = nodes[0].id
response = self.patch('/study/process/job/', {
'op': 'remove',
'path': job_id
})
self.assertEqual(response.code, 200)
exp = {
'status':
'error',
'message':
"Can't delete job %s. It is 'in_construction' "
"status. Please use /study/process/workflow/" % job_id
}
self.assertEqual(loads(response.body), exp)
# Test success
ProcessingJob(job_id)._set_error('Killed for testing')
response = self.patch('/study/process/job/', {
'op': 'remove',
'path': job_id
})
self.assertEqual(response.code, 200)
exp = {'status': 'success', 'message': ''}
self.assertEqual(loads(response.body), exp)
def artifact_summary_post_request(user_id, artifact_id):
"""Launches the HTML summary generation and returns the job information
Parameters
----------
user_id : str
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the artifact summary information
{'status': str,
'message': str,
'job': list of [str, str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
access_error = check_access(artifact.study.id, user_id)
if access_error:
return access_error
# Check if the summary is being generated or has been already generated
command = Command.get_html_generator(artifact.artifact_type)
jobs = artifact.jobs(cmd=command)
jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
if jobs:
# The HTML summary is either being generated or already generated.
# Return the information of that job so we only generate the HTML
# once
job = jobs[0]
else:
# Create a new job to generate the HTML summary and return the newly
# created job information
job = ProcessingJob.create(
User(user_id),
Parameters.load(command, values_dict={'input_data': artifact_id}))
job.submit()
return {'status': 'success',
'message': '',
'job': [job.id, job.status, job.step]}
def test_artifact_summary_post_request(self):
# No access
obs = artifact_summary_post_request('*****@*****.**', 1)
exp = {'status': 'error',
'message': 'User does not have access to study'}
self.assertEqual(obs, exp)
# Returns already existing job
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 2})
)
job._set_status('queued')
obs = artifact_summary_post_request('*****@*****.**', 2)
exp = {'status': 'success',
'message': '',
'job': [job.id, 'queued', None]}
self.assertEqual(obs, exp)
def artifact_summary_post_request(user, artifact_id):
"""Launches the HTML summary generation and returns the job information
Parameters
----------
user : qiita_db.user.User
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the job summary information
{'job': [str, str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
check_artifact_access(user, artifact)
# Check if the summary is being generated or has been already generated
command = Command.get_html_generator(artifact.artifact_type)
jobs = artifact.jobs(cmd=command)
jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
if jobs:
# The HTML summary is either being generated or already generated.
# Return the information of that job so we only generate the HTML
# once - Magic number 0 -> we are ensuring that there is only one
# job generating the summary, so we can use the index 0 to access to
# that job
job = jobs[0]
else:
# Create a new job to generate the HTML summary and return the newly
# created job information
job = ProcessingJob.create(
user,
Parameters.load(command, values_dict={'input_data': artifact_id}),
True)
job.submit()
return {'job': [job.id, job.status, job.step]}
def test_artifact_summary_post_request(self):
# No access
obs = artifact_summary_post_request('*****@*****.**', 1)
exp = {
'status': 'error',
'message': 'User does not have access to study'
}
self.assertEqual(obs, exp)
# Returns already existing job
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 2}))
job._set_status('queued')
obs = artifact_summary_post_request('*****@*****.**', 2)
exp = {
'status': 'success',
'message': '',
'job': [job.id, 'queued', None]
}
self.assertEqual(obs, exp)
def artifact_summary_post_request(user, artifact_id):
"""Launches the HTML summary generation and returns the job information
Parameters
----------
user : qiita_db.user.User
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the job summary information
{'job': [str, str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
check_artifact_access(user, artifact)
# Check if the summary is being generated or has been already generated
command = Command.get_html_generator(artifact.artifact_type)
jobs = artifact.jobs(cmd=command)
jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
if jobs:
# The HTML summary is either being generated or already generated.
# Return the information of that job so we only generate the HTML
# once - Magic number 0 -> we are ensuring that there is only one
# job generating the summary, so we can use the index 0 to access to
# that job
job = jobs[0]
else:
# Create a new job to generate the HTML summary and return the newly
# created job information
job = ProcessingJob.create(user, Parameters.load(
command, values_dict={'input_data': artifact_id}), True)
job.submit()
return {'job': [job.id, job.status, job.step]}
def test_workflow_handler_patch_req(self):
# Create a new workflow so it is in construction
exp_command = Command(1)
json_str = (
'{"input_data": 1, "max_barcode_errors": 1.5, '
'"barcode_type": "golay_12", "max_bad_run_length": 3, '
'"rev_comp": false, "phred_quality_threshold": 3, '
'"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
'"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
exp_params = Parameters.load(exp_command, json_str=json_str)
exp_user = User('*****@*****.**')
name = "Test processing workflow"
# tests success
wf = ProcessingWorkflow.from_scratch(
exp_user, exp_params, name=name, force=True)
graph = wf.graph
nodes = list(graph.nodes())
job_id = nodes[0].id
value = {'dflt_params': 10,
'connections': {job_id: {'demultiplexed': 'input_data'}}}
obs = workflow_handler_patch_req(
'add', '/%s/' % wf.id, req_value=dumps(value))
new_jobs = set(wf.graph.nodes()) - set(nodes)
self.assertEqual(len(new_jobs), 1)
new_job = new_jobs.pop()
exp = {'status': 'success',
'message': '',
'job': {'id': new_job.id,
'inputs': [job_id],
'label': 'Pick closed-reference OTUs',
'outputs': [['OTU table', 'BIOM']]}}
self.assertEqual(obs, exp)
obs = workflow_handler_patch_req(
'remove', '/%s/%s/' % (wf.id, new_job.id))
exp = {'status': 'success', 'message': ''}
jobs = set(wf.graph.nodes()) - set(nodes)
self.assertEqual(jobs, set())
def sample_template_delete_req(study_id, user_id):
"""Deletes the sample template attached to the study
Parameters
----------
study_id : int
The current study object id
user_id : str
The current user object id
Returns
-------
dict
results dictonary in the format
{'status': status,
'message': msg}
status can be success, warning, or error depending on result
message has the warnings or errors
"""
exists = _check_sample_template_exists(int(study_id))
if exists['status'] != 'success':
return exists
access_error = check_access(int(study_id), user_id)
if access_error:
return access_error
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_template')
params = Parameters.load(cmd, values_dict={'study': int(study_id)})
job = ProcessingJob.create(User(user_id), params)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'status': 'success', 'message': ''}
def workflow_handler_post_req(user_id, command_id, params):
"""Creates a new workflow in the system
Parameters
----------
user_id : str
The user creating the workflow
command_id : int
The first command to execute in the workflow
params : str
JSON representations of the parameters for the first command of
the workflow
Returns
-------
dict of objects
A dictionary containing the commands information
{'status': str,
'message': str,
'workflow_id': int}
"""
parameters = Parameters.load(Command(command_id), json_str=params)
wf = ProcessingWorkflow.from_scratch(User(user_id), parameters)
# this is safe as we are creating the workflow for the first time and there
# is only one node. Remember networkx doesn't assure order of nodes
job = wf.graph.nodes()[0]
inputs = [a.id for a in job.input_artifacts]
job_cmd = job.command
return {
'status': 'success',
'message': '',
'workflow_id': wf.id,
'job': {
'id': job.id,
'inputs': inputs,
'label': job_cmd.name,
'outputs': job_cmd.outputs
}
}
def test_patch(self):
# Create a new job - through a workflow since that is the only way
# of creating jobs in the interface
exp_command = Command(1)
json_str = (
'{"input_data": 1, "max_barcode_errors": 1.5, '
'"barcode_type": "golay_12", "max_bad_run_length": 3, '
'"rev_comp": false, "phred_quality_threshold": 3, '
'"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
'"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
exp_params = Parameters.load(exp_command, json_str=json_str)
exp_user = User('*****@*****.**')
name = "Test processing workflow"
# tests success
wf = ProcessingWorkflow.from_scratch(
exp_user, exp_params, name=name, force=True)
graph = wf.graph
nodes = graph.nodes()
job_id = nodes[0].id
response = self.patch('/study/process/job/',
{'op': 'remove', 'path': job_id})
self.assertEqual(response.code, 200)
exp = {'status': 'error',
'message': "Can't delete job %s. It is 'in_construction' "
"status. Please use /study/process/workflow/"
% job_id}
self.assertEqual(loads(response.body), exp)
# Test success
ProcessingJob(job_id)._set_error('Killed for testing')
response = self.patch('/study/process/job/',
{'op': 'remove', 'path': job_id})
self.assertEqual(response.code, 200)
exp = {'status': 'success',
'message': ''}
self.assertEqual(loads(response.body), exp)
def sample_template_handler_delete_request(study_id, user):
"""Deletes the sample template
Parameters
----------
study_id: int
The study to delete the sample information
user: qiita_db.user
The user performing the request
Returns
-------
dict of {'job': str}
job: the id of the job deleting the sample information to the study
Raises
------
HTTPError
404 If the sample template doesn't exist
"""
# Check if the current user has access to the study and if the sample
# template exists
sample_template_checks(study_id, user, check_exists=True)
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_template')
params = Parameters.load(cmd, values_dict={'study': int(study_id)})
job = ProcessingJob.create(user, params, True)
# Store the job if deleteing the sample template
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
def sample_template_handler_delete_request(study_id, user):
"""Deletes the sample template
Parameters
----------
study_id: int
The study to delete the sample information
user: qiita_db.user
The user performing the request
Returns
-------
dict of {'job': str}
job: the id of the job deleting the sample information to the study
Raises
------
HTTPError
404 If the sample template doesn't exist
"""
# Check if the current user has access to the study and if the sample
# template exists
sample_template_checks(study_id, user, check_exists=True)
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_template')
params = Parameters.load(cmd, values_dict={'study': int(study_id)})
job = ProcessingJob.create(user, params, True)
# Store the job if deleteing the sample template
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
def sample_template_handler_patch_request(user, req_op, req_path,
req_value=None, req_from=None):
"""Patches the sample template
Parameters
----------
user: qiita_db.user.User
The user performing the request
req_op : str
The operation to perform on the sample template
req_path : str
The path to the attribute to patch
req_value : str, optional
The new value
req_from : str, optional
The original path of the element
Returns
-------
Raises
------
HTTPError
400 If the path parameter doens't follow the expected format
400 If the given operation is not supported
"""
req_path = [v for v in req_path.split('/') if v]
# At this point we know the path should be at least length 2
if len(req_path) < 2:
raise HTTPError(400, reason='Incorrect path parameter')
study_id = int(req_path[0])
# Check if the current user has access to the study and if the sample
# template exists
sample_template_checks(study_id, user, check_exists=True)
if req_op == 'remove':
# Path format
# column: study_id/columns/column_name
# sample: study_id/samples/sample_id
if len(req_path) != 3:
raise HTTPError(400, reason='Incorrect path parameter')
attribute = req_path[1]
attr_id = req_path[2]
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_or_column')
params = Parameters.load(
cmd, values_dict={'obj_class': 'SampleTemplate',
'obj_id': study_id,
'sample_or_col': attribute,
'name': attr_id})
job = ProcessingJob.create(user, params, True)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
elif req_op == 'replace':
# WARNING: Although the patch operation is a replace, is not a full
# true replace. A replace is in theory equivalent to a remove + add.
# In this case, the replace operation doesn't necessarily removes
# anything (e.g. when only new columns/samples are being added to the)
# sample information.
# Path format: study_id/data
# Forcing to specify data for extensibility. In the future we may want
# to use this function to replace other elements of the sample
# information
if len(req_path) != 2:
raise HTTPError(400, reason='Incorrect path parameter')
attribute = req_path[1]
if attribute == 'data':
# Update the sample information
if req_value is None:
raise HTTPError(400, reason="Value is required when updating "
"sample information")
# Check if the file exists
fp_rsp = check_fp(study_id, req_value)
if fp_rsp['status'] != 'success':
raise HTTPError(404, reason='Filepath not found')
filepath = fp_rsp['file']
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('update_sample_template')
params = Parameters.load(
cmd, values_dict={'study': study_id,
'template_fp': filepath})
job = ProcessingJob.create(user, params, True)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
else:
raise HTTPError(404, reason='Attribute %s not found' % attribute)
else:
raise HTTPError(400, reason='Operation %s not supported. Current '
'supported operations: remove, replace' % req_op)
def test_complete_job(self):
# Complete success
pt = npt.assert_warns(
QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {'1.SKD6.640190': 1}}),
Study(1), '16S')
c_job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(
Command.get_validator('BIOM'),
values_dict={'template': pt.id,
'files': dumps({'BIOM': ['file']}),
'artifact_type': 'BIOM'}), True)
c_job._set_status('running')
fd, fp = mkstemp(suffix='_table.biom')
close(fd)
with open(fp, 'w') as f:
f.write('\n')
self._clean_up_files.append(fp)
exp_artifact_count = get_count('qiita.artifact') + 1
payload = dumps(
{'success': True, 'error': '',
'artifacts': {'OTU table': {'filepaths': [(fp, 'biom')],
'artifact_type': 'BIOM'}}})
job = self._create_job('complete_job', {'job_id': c_job.id,
'payload': payload})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertEqual(c_job.status, 'success')
self.assertEqual(get_count('qiita.artifact'), exp_artifact_count)
# Complete job error
payload = dumps({'success': False, 'error': 'Job failure'})
job = self._create_job(
'complete_job', {'job_id': 'bcc7ebcd-39c1-43e4-af2d-822e3589f14d',
'payload': payload})
private_task(job.id)
self.assertEqual(job.status, 'success')
c_job = ProcessingJob('bcc7ebcd-39c1-43e4-af2d-822e3589f14d')
self.assertEqual(c_job.status, 'error')
self.assertEqual(c_job.log, LogEntry.newest_records(numrecords=1)[0])
self.assertEqual(c_job.log.msg, 'Job failure')
# Complete internal error
pt = npt.assert_warns(
QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {'1.SKD6.640190': 1}}),
Study(1), '16S')
c_job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(
Command.get_validator('BIOM'),
values_dict={'template': pt.id,
'files': dumps({'BIOM': ['file']}),
'artifact_type': 'BIOM'}), True)
c_job._set_status('running')
fp = '/surprised/if/this/path/exists.biom'
payload = dumps(
{'success': True, 'error': '',
'artifacts': {'OTU table': {'filepaths': [(fp, 'biom')],
'artifact_type': 'BIOM'}}})
job = self._create_job('complete_job', {'job_id': c_job.id,
'payload': payload})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertEqual(c_job.status, 'error')
self.assertIn('No such file or directory', c_job.log.msg)
def test_artifact_summary_get_request(self):
# Artifact w/o summary
obs = artifact_summary_get_request('*****@*****.**', 1)
exp_p_jobs = [
['063e553b-327c-4818-ab4a-adfe58e49860', 'Split libraries FASTQ',
'queued', None, None],
['bcc7ebcd-39c1-43e4-af2d-822e3589f14d', 'Split libraries',
'running', 'demultiplexing', None]]
exp_files = [
(1L, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)'),
(2L, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)')]
exp = {'status': 'success',
'message': '',
'name': 'Raw data 1',
'summary': None,
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'private',
'buttons': '<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public</button>'
' <button onclick="if (confirm(\'Are you sure you '
'want to revert to sandbox artifact id: 1?\')) '
'{ set_artifact_visibility(\'sandbox\', 1) }" '
'class="btn btn-primary btn-sm">Revert to '
'sandbox</button>',
'files': exp_files,
'editable': True}
self.assertEqual(obs, exp)
# Artifact with summary being generated
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 1})
)
job._set_status('queued')
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {'status': 'success',
'message': '',
'name': 'Raw data 1',
'summary': None,
'job': [job.id, 'queued', None],
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'private',
'buttons': '<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public</button>'
' <button onclick="if (confirm(\'Are you sure you '
'want to revert to sandbox artifact id: 1?\')) { '
'set_artifact_visibility(\'sandbox\', 1) }" '
'class="btn btn-primary btn-sm">Revert to '
'sandbox</button>',
'files': exp_files,
'editable': True}
self.assertEqual(obs, exp)
# Artifact with summary
fd, fp = mkstemp(suffix=".html")
close(fd)
with open(fp, 'w') as f:
f.write('<b>HTML TEST - not important</b>\n')
a = Artifact(1)
a.html_summary_fp = fp
self._files_to_remove.extend([fp, a.html_summary_fp[1]])
exp_files.append(
(a.html_summary_fp[0],
'%s (html summary)' % basename(a.html_summary_fp[1])))
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {'status': 'success',
'message': '',
'name': 'Raw data 1',
'summary': '<b>HTML TEST - not important</b>\n',
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'private',
'buttons': '<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public</button>'
' <button onclick="if (confirm(\'Are you sure you '
'want to revert to sandbox artifact id: 1?\')) { '
'set_artifact_visibility(\'sandbox\', 1) }" '
'class="btn btn-primary btn-sm">Revert to '
'sandbox</button>',
'files': exp_files,
'editable': True}
self.assertEqual(obs, exp)
# No access
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {'status': 'error',
'message': 'User does not have access to study'}
self.assertEqual(obs, exp)
# A non-owner/share user can't see the files
a.visibility = 'public'
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {'status': 'success',
'message': '',
'name': 'Raw data 1',
'summary': '<b>HTML TEST - not important</b>\n',
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'public',
'buttons': '',
'files': [],
'editable': False}
self.assertEqual(obs, exp)
def test_get_analysis_graph_handler(self):
response = self.get('/analysis/description/1/graph/')
self.assertEqual(response.code, 200)
# The job id is randomly generated in the test environment. Gather
# it here. There is only 1 job in the first artifact of the analysis
job_id = Analysis(1).artifacts[0].jobs()[0].id
obs = loads(response.body)
exp = {
'edges': [[8, job_id], [job_id, 9]],
'nodes': [['job', 'job', job_id, 'Single Rarefaction', 'success'],
['artifact', 'BIOM', 9, 'noname\n(BIOM)', 'artifact'],
['artifact', 'BIOM', 8, 'noname\n(BIOM)', 'artifact']],
'workflow':
None
}
self.assertItemsEqual(obs, exp)
self.assertItemsEqual(obs['edges'], exp['edges'])
self.assertItemsEqual(obs['nodes'], exp['nodes'])
self.assertIsNone(obs['workflow'])
# Create a new analysis with 2 starting BIOMs to be able to test
# the different if statements of the request
BaseHandler.get_current_user = Mock(
return_value=User('*****@*****.**'))
user = User('*****@*****.**')
dflt_analysis = user.default_analysis
dflt_analysis.add_samples({
4: ['1.SKB8.640193', '1.SKD8.640184', '1.SKB7.640196'],
6: ['1.SKB8.640193', '1.SKD8.640184', '1.SKB7.640196']
})
args = {'name': 'New Test Graph Analysis', 'description': 'Desc'}
response = self.post('/analysis/create/', args)
new_id = response.effective_url.split('/')[-2]
a = Analysis(new_id)
# Wait until all the jobs are done so the BIOM tables exist
for j in a.jobs:
wait_for_processing_job(j.id)
artifacts = a.artifacts
self.assertEqual(len(artifacts), 2)
# Create a new workflow starting on the first artifact
# Magic number 9 -> Summarize Taxa command
params = Parameters.load(Command(9),
values_dict={
'metadata_category': 'None',
'sort': 'False',
'biom_table': artifacts[0].id
})
wf = ProcessingWorkflow.from_scratch(user, params)
# There is only one job in the workflow
job_id = wf.graph.nodes()[0].id
response = self.get('/analysis/description/%s/graph/' % new_id)
self.assertEqual(response.code, 200)
obs = loads(response.body)
exp = {
'edges': [[artifacts[0].id, job_id],
[job_id, '%s:taxa_summary' % job_id]],
'nodes':
[['job', 'job', job_id, 'Summarize Taxa', 'in_construction'],
[
'artifact', 'BIOM', artifacts[0].id, 'noname\n(BIOM)',
'artifact'
],
[
'artifact', 'BIOM', artifacts[1].id, 'noname\n(BIOM)',
'artifact'
],
[
'type', 'taxa_summary',
'%s:taxa_summary' % job_id, 'taxa_summary\n(taxa_summary)',
'type'
]],
'workflow':
wf.id
}
# Check that the keys are the same
self.assertItemsEqual(obs, exp)
# Check the edges
self.assertItemsEqual(obs['edges'], exp['edges'])
# Check the edges
self.assertItemsEqual(obs['nodes'], exp['nodes'])
# Check the edges
self.assertEqual(obs['workflow'], exp['workflow'])
# Add a job to the second BIOM to make sure that the edges and nodes
# are respected. Magic number 12 -> Single Rarefaction
job2 = wf.add(DefaultParameters(16),
req_params={
'depth': '100',
'biom_table': artifacts[1].id
})
job_id_2 = job2.id
response = self.get('/analysis/description/%s/graph/' % new_id)
self.assertEqual(response.code, 200)
obs = loads(response.body)
exp = {
'edges': [[artifacts[0].id, job_id],
[job_id, '%s:taxa_summary' % job_id],
[artifacts[1].id, job_id_2],
[job_id_2, '%s:rarefied_table' % job_id_2]],
'nodes':
[['job', 'job', job_id, 'Summarize Taxa', 'in_construction'],
['job', 'job', job_id_2, 'Single Rarefaction', 'in_construction'],
[
'artifact', 'BIOM', artifacts[0].id, 'noname\n(BIOM)',
'artifact'
],
[
'artifact', 'BIOM', artifacts[1].id, 'noname\n(BIOM)',
'artifact'
],
[
'type', 'taxa_summary',
'%s:taxa_summary' % job_id, 'taxa_summary\n(taxa_summary)',
'type'
],
[
'type', 'BIOM',
'%s:rarefied_table' % job_id_2, 'rarefied_table\n(BIOM)',
'type'
]],
'workflow':
wf.id
}
# Check that the keys are the same
self.assertItemsEqual(obs, exp)
# Check the edges
self.assertItemsEqual(obs['edges'], exp['edges'])
# Check the edges
self.assertItemsEqual(obs['nodes'], exp['nodes'])
# Check the edges
self.assertEqual(obs['workflow'], exp['workflow'])
# Add a second Workflow to the second artifact to force the raise of
# the error. This situation should never happen when using
# the interface
wf.remove(job2)
params = Parameters.load(Command(9),
values_dict={
'metadata_category': 'None',
'sort': 'False',
'biom_table': artifacts[1].id
})
wf = ProcessingWorkflow.from_scratch(user, params)
response = self.get('/analysis/description/%s/graph/' % new_id)
self.assertEqual(response.code, 500)
def test_job_ajax_patch_req(self):
# Create a new job - through a workflow since that is the only way
# of creating jobs in the interface
exp_command = Command(1)
json_str = (
'{"input_data": 1, "max_barcode_errors": 1.5, '
'"barcode_type": "golay_12", "max_bad_run_length": 3, '
'"rev_comp": false, "phred_quality_threshold": 3, '
'"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
'"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
exp_params = Parameters.load(exp_command, json_str=json_str)
exp_user = User('*****@*****.**')
name = "Test processing workflow"
# tests success
wf = ProcessingWorkflow.from_scratch(
exp_user, exp_params, name=name, force=True)
graph = wf.graph
nodes = list(graph.nodes())
job_id = nodes[0].id
# Incorrect path parameter
obs = job_ajax_patch_req('remove', '/%s/somethingelse' % job_id)
exp = {'status': 'error',
'message': 'Incorrect path parameter: missing job id'}
self.assertEqual(obs, exp)
obs = job_ajax_patch_req('remove', '/')
exp = {'status': 'error',
'message': 'Incorrect path parameter: missing job id'}
self.assertEqual(obs, exp)
# Job id is not like a job id
obs = job_ajax_patch_req('remove', '/notAJobId')
exp = {'status': 'error',
'message': 'Incorrect path parameter: '
'notAJobId is not a recognized job id'}
self.assertEqual(obs, exp)
# Job doesn't exist
obs = job_ajax_patch_req('remove',
'/6d368e16-2242-4cf8-87b4-a5dc40bc890b')
exp = {'status': 'error',
'message': 'Incorrect path parameter: '
'6d368e16-2242-4cf8-87b4-a5dc40bc890b is not a '
'recognized job id'}
self.assertEqual(obs, exp)
# in_construction job
obs = job_ajax_patch_req('remove', '/%s' % job_id)
exp = {'status': 'error',
'message': "Can't delete job %s. It is 'in_construction' "
"status. Please use /study/process/workflow/"
% job_id}
self.assertEqual(obs, exp)
# job status != 'error'
job = ProcessingJob(job_id)
job._set_status('queued')
obs = job_ajax_patch_req('remove', '/%s' % job_id)
exp = {'status': 'error',
'message': 'Only jobs in "error" status can be deleted.'}
self.assertEqual(obs, exp)
# Operation not supported
job._set_status('queued')
obs = job_ajax_patch_req('add', '/%s' % job_id)
exp = {'status': 'error',
'message': 'Operation "add" not supported. Current supported '
'operations: remove'}
self.assertEqual(obs, exp)
# Test success
job._set_error('Killed for testing')
obs = job_ajax_patch_req('remove', '/%s' % job_id)
exp = {'status': 'success',
'message': ''}
self.assertEqual(obs, exp)
def test_complete_job(self):
# Complete success
pt = npt.assert_warns(QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {
'1.SKD6.640190': 1
}}), Study(1), '16S')
c_job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command.get_validator('BIOM'),
values_dict={
'template': pt.id,
'files': dumps({'BIOM': ['file']}),
'artifact_type': 'BIOM'
}), True)
c_job._set_status('running')
fd, fp = mkstemp(suffix='_table.biom')
close(fd)
with open(fp, 'w') as f:
f.write('\n')
self._clean_up_files.append(fp)
exp_artifact_count = get_count('qiita.artifact') + 1
payload = dumps({
'success': True,
'error': '',
'artifacts': {
'OTU table': {
'filepaths': [(fp, 'biom')],
'artifact_type': 'BIOM'
}
}
})
job = self._create_job('complete_job', {
'job_id': c_job.id,
'payload': payload
})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertEqual(c_job.status, 'success')
self.assertEqual(get_count('qiita.artifact'), exp_artifact_count)
# Complete job error
payload = dumps({'success': False, 'error': 'Job failure'})
job = self._create_job('complete_job', {
'job_id': 'bcc7ebcd-39c1-43e4-af2d-822e3589f14d',
'payload': payload
})
private_task(job.id)
self.assertEqual(job.status, 'success')
c_job = ProcessingJob('bcc7ebcd-39c1-43e4-af2d-822e3589f14d')
self.assertEqual(c_job.status, 'error')
self.assertEqual(c_job.log, LogEntry.newest_records(numrecords=1)[0])
self.assertEqual(c_job.log.msg, 'Job failure')
# Complete internal error
pt = npt.assert_warns(QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {
'1.SKD6.640190': 1
}}), Study(1), '16S')
c_job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command.get_validator('BIOM'),
values_dict={
'template': pt.id,
'files': dumps({'BIOM': ['file']}),
'artifact_type': 'BIOM'
}), True)
c_job._set_status('running')
fp = '/surprised/if/this/path/exists.biom'
payload = dumps({
'success': True,
'error': '',
'artifacts': {
'OTU table': {
'filepaths': [(fp, 'biom')],
'artifact_type': 'BIOM'
}
}
})
job = self._create_job('complete_job', {
'job_id': c_job.id,
'payload': payload
})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertEqual(c_job.status, 'error')
self.assertIn('No such file or directory', c_job.log.msg)
job_id=job_id, title='VAMPS Submission',
completion_redirect='/compute_complete/%s' % job_id)
return
=======
if user.level != 'admin':
raise HTTPError(403, reason="User %s cannot submit to VAMPS!" %
user.id)
msg = ''
msg_level = 'success'
plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = plugin.get_command('submit_to_VAMPS')
artifact = Artifact(preprocessed_data_id)
# Check if the artifact is already being submitted to VAMPS
is_being_submitted = any(
[j.status in ('queued', 'running')
for j in artifact.jobs(cmd=cmd)])
>>>>>>> 405cbef0c9f71c620da95a0c1ba6c7d3d588b3ed
if is_being_submitted == 'submitting':
msg = "Cannot resubmit! Data is already being submitted"
msg_level = 'danger'
self.display_template(preprocessed_data_id, msg, msg_level)
else:
params = Parameters.load(
cmd, values_dict={'artifact': preprocessed_data_id})
job = ProcessingJob.create(user, params, True)
job.submit()
self.redirect('/study/description/%s' % artifact.study.study_id)
def artifact_post_req(user_id, filepaths, artifact_type, name,
prep_template_id, artifact_id=None):
"""Creates the initial artifact for the prep template
Parameters
----------
user_id : str
User adding the atrifact
filepaths : dict of str
Comma-separated list of files to attach to the artifact,
keyed by file type
artifact_type : str
The type of the artifact
name : str
Name to give the artifact
prep_template_id : int or str castable to int
Prep template to attach the artifact to
artifact_id : int or str castable to int, optional
The id of the imported artifact
Returns
-------
dict of objects
A dictionary containing the new artifact ID
{'status': status,
'message': message,
'artifact': id}
"""
prep_template_id = int(prep_template_id)
prep = PrepTemplate(prep_template_id)
study_id = prep.study_id
# First check if the user has access to the study
access_error = check_access(study_id, user_id)
if access_error:
return access_error
if artifact_id:
# if the artifact id has been provided, import the artifact
job_id = safe_submit(user_id, copy_raw_data, prep, artifact_id)
is_qiita_job = False
else:
uploads_path = get_mountpoint('uploads')[0][1]
path_builder = partial(join, uploads_path, str(study_id))
cleaned_filepaths = {}
for ftype, file_list in viewitems(filepaths):
# JavaScript sends us this list as a comma-separated list
for fp in file_list.split(','):
# JavaScript will send this value as an empty string if the
# list of files was empty. In such case, the split will
# generate a single element containing the empty string. Check
# for that case here and, if fp is not the empty string,
# proceed to check if the file exists
if fp:
# Check if filepath being passed exists for study
full_fp = path_builder(fp)
exists = check_fp(study_id, full_fp)
if exists['status'] != 'success':
return {'status': 'error',
'message': 'File does not exist: %s' % fp}
if ftype not in cleaned_filepaths:
cleaned_filepaths[ftype] = []
cleaned_filepaths[ftype].append(full_fp)
# This should never happen, but it doesn't hurt to actually have
# a explicit check, in case there is something odd with the JS
if not cleaned_filepaths:
return {'status': 'error',
'message': "Can't create artifact, no files provided."}
command = Command.get_validator(artifact_type)
job = ProcessingJob.create(
User(user_id),
Parameters.load(command, values_dict={
'template': prep_template_id,
'files': dumps(cleaned_filepaths),
'artifact_type': artifact_type
}))
job.submit()
job_id = job.id
is_qiita_job = True
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep.id,
dumps({'job_id': job_id, 'is_qiita_job': is_qiita_job}))
return {'status': 'success', 'message': ''}
def sample_template_handler_patch_request(user,
req_op,
req_path,
req_value=None,
req_from=None,
direct_upload=False):
"""Patches the sample template
Parameters
----------
user: qiita_db.user.User
The user performing the request
req_op : str
The operation to perform on the sample template
req_path : str
The path to the attribute to patch
req_value : str, optional
The new value
req_from : str, optional
The original path of the element
direct_upload : boolean, optional
If the file being uploaded comes from a direct upload (True)
Returns
-------
Raises
------
HTTPError
400 If the path parameter doens't follow the expected format
400 If the given operation is not supported
"""
req_path = [v for v in req_path.split('/') if v]
# At this point we know the path should be at least length 2
if len(req_path) < 2:
raise HTTPError(400, reason='Incorrect path parameter')
study_id = int(req_path[0])
# Check if the current user has access to the study and if the sample
# template exists
sample_template_checks(study_id, user, check_exists=True)
if req_op == 'remove':
# Path format
# column: study_id/columns/column_name
# sample: study_id/samples/sample_id
if len(req_path) != 3:
raise HTTPError(400, reason='Incorrect path parameter')
attribute = req_path[1]
attr_id = req_path[2]
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_or_column')
params = Parameters.load(cmd,
values_dict={
'obj_class': 'SampleTemplate',
'obj_id': study_id,
'sample_or_col': attribute,
'name': attr_id
})
job = ProcessingJob.create(user, params, True)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
elif req_op == 'replace':
# WARNING: Although the patch operation is a replace, is not a full
# true replace. A replace is in theory equivalent to a remove + add.
# In this case, the replace operation doesn't necessarily removes
# anything (e.g. when only new columns/samples are being added to the)
# sample information.
# Path format: study_id/data
# Forcing to specify data for extensibility. In the future we may want
# to use this function to replace other elements of the sample
# information
if len(req_path) != 2:
raise HTTPError(400, reason='Incorrect path parameter')
attribute = req_path[1]
if attribute == 'data':
# Update the sample information
if req_value is None:
raise HTTPError(400,
reason="Value is required when updating "
"sample information")
if direct_upload:
# We can assume that the file exist as it was generated by
# the system
filepath = req_value
if not exists(filepath):
reason = ('Upload file not found (%s), please report to '
'*****@*****.**' % filepath)
raise HTTPError(404, reason=reason)
else:
# Check if the file exists
fp_rsp = check_fp(study_id, req_value)
if fp_rsp['status'] != 'success':
raise HTTPError(404, reason='Filepath not found')
filepath = fp_rsp['file']
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('update_sample_template')
params = Parameters.load(cmd,
values_dict={
'study': study_id,
'template_fp': filepath
})
job = ProcessingJob.create(user, params, True)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
else:
raise HTTPError(404, reason='Attribute %s not found' % attribute)
else:
raise HTTPError(400,
reason='Operation %s not supported. Current '
'supported operations: remove, replace' % req_op)
def test_artifact_summary_get_request(self):
user = User('*****@*****.**')
main_buttons = (
'<button onclick="if (confirm('
"\'Are you sure you want to make "
"public artifact id: 1?')) { set_artifact_visibility('public', 1) "
'}" class="btn btn-primary btn-sm">Make public</button> <button '
'onclick="if (confirm('
"'Are you sure you want to revert to "
"sandbox artifact id: 1?')) { set_artifact_visibility('sandbox', 1"
') }" class="btn btn-primary btn-sm">Revert to sandbox</button> ')
private_download_button = (
'<button class="btn btn-primary btn-sm" type="button" '
'aria-expanded="false" aria-controls="privateDownloadLink" '
'onclick="generate_private_download_link(%s)">Generate Download '
'Link</button><div class="collapse" id="privateDownloadLink"><div '
'class="card card-body" id="privateDownloadText">Generating '
'Download Link...</div></div>')
# Artifact w/o summary
obs = artifact_summary_get_request(user, 1)
exp_files = [
(1, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)',
'2125826711', '58 Bytes'),
(2, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)',
'2125826711', '58 Bytes')
]
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'private',
'editable': True,
'buttons': main_buttons + private_download_button % 1,
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': None,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# Artifact with summary being generated
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 1}))
job._set_status('queued')
obs = artifact_summary_get_request(user, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'private',
'editable': True,
'buttons': main_buttons + private_download_button % 1,
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': None,
'job': [job.id, 'queued', None],
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# Artifact with summary
fd, fp = mkstemp(suffix=".html")
close(fd)
with open(fp, 'w') as f:
f.write('<b>HTML TEST - not important</b>\n')
a = Artifact(1)
a.set_html_summary(fp)
self._files_to_remove.extend([fp, a.html_summary_fp[1]])
exp_files.append((a.html_summary_fp[0],
'%s (html summary)' % basename(a.html_summary_fp[1]),
'1642196267', '33 Bytes'))
exp_summary_path = relpath(a.html_summary_fp[1],
qiita_config.base_data_dir)
obs = artifact_summary_get_request(user, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'private',
'editable': True,
'buttons': main_buttons + private_download_button % 1,
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': exp_summary_path,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# No access
demo_u = User('*****@*****.**')
with self.assertRaises(QiitaHTTPError):
obs = artifact_summary_get_request(demo_u, 1)
# A non-owner/share user can't see the files
a.visibility = 'public'
obs = artifact_summary_get_request(demo_u, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'public',
'editable': False,
'buttons': '',
'processing_info': {},
'files': [],
'is_from_analysis': False,
'summary': exp_summary_path,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# testing sandbox
a.visibility = 'sandbox'
obs = artifact_summary_get_request(user, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'sandbox',
'editable': True,
'buttons': private_download_button % 1,
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': exp_summary_path,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# returnig to private
a.visibility = 'private'
# admin gets buttons
obs = artifact_summary_get_request(User('*****@*****.**'), 2)
exp_files = [(3, '1_seqs.fna (preprocessed fasta)', '', '0 Bytes'),
(4, '1_seqs.qual (preprocessed fastq)', '', '0 Bytes'),
(5, '1_seqs.demux (preprocessed demux)', '', '0 Bytes')]
exp = {
'name':
'Demultiplexed 1',
'artifact_id':
2,
'artifact_type':
'Demultiplexed',
'artifact_timestamp':
'2012-10-01 10:10',
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 2?\')) { '
'set_artifact_visibility(\'public\', 2) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'2?\')) { set_artifact_visibility(\'sandbox\', 2) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button> <a class="btn btn-primary '
'btn-sm" href="/ebi_submission/2"><span '
'class="glyphicon glyphicon-export"></span> '
'Submit to EBI</a> <a class="btn btn-primary '
'btn-sm" href="/vamps/2"><span class="glyphicon '
'glyphicon-export"></span> Submit to VAMPS</a> ' +
private_download_button % 2),
'processing_info': {
'command_active': True,
'software_deprecated': False,
'command': 'Split libraries FASTQ',
'processing_parameters': {
'max_barcode_errors': '1.5',
'sequence_max_n': '0',
'max_bad_run_length': '3',
'phred_offset': 'auto',
'rev_comp': 'False',
'phred_quality_threshold': '3',
'input_data': '1',
'rev_comp_barcode': 'False',
'rev_comp_mapping_barcodes': 'False',
'min_per_read_length_fraction': '0.75',
'barcode_type': 'golay_12'
},
'software_version': '1.9.1',
'software': 'QIIME'
},
'files':
exp_files,
'is_from_analysis':
False,
'summary':
None,
'job':
None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# analysis artifact
obs = artifact_summary_get_request(user, 8)
exp = {
'name': 'noname',
'artifact_id': 8,
'artifact_type': 'BIOM',
# this value changes on build so copy from obs
'artifact_timestamp': obs['artifact_timestamp'],
'visibility': 'sandbox',
'editable': True,
'buttons': private_download_button % 8,
'processing_info': {},
'files': [(22, 'biom_table.biom (biom)', '1756512010', '1.1 MB')],
'is_from_analysis': True,
'summary': None,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
def sample_template_patch_request(user_id,
req_op,
req_path,
req_value=None,
req_from=None):
"""Modifies an attribute of the artifact
Parameters
----------
user_id : str
The id of the user performing the patch operation
req_op : str
The operation to perform on the artifact
req_path : str
The prep information and attribute to patch
req_value : str, optional
The value that needs to be modified
req_from : str, optional
The original path of the element
Returns
-------
dict of {str, str}
A dictionary with the following keys:
- status: str, whether if the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
"""
if req_op == 'remove':
req_path = [v for v in req_path.split('/') if v]
# format
# column: study_id/row_id/columns/column_name
# sample: study_id/row_id/samples/sample_id
if len(req_path) != 4:
return {'status': 'error', 'message': 'Incorrect path parameter'}
st_id = req_path[0]
row_id = req_path[1]
attribute = req_path[2]
attr_id = req_path[3]
# Check if the user actually has access to the template
st = SampleTemplate(st_id)
access_error = check_access(st.study_id, user_id)
if access_error:
return access_error
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_or_column')
params = Parameters.load(cmd,
values_dict={
'obj_class': 'SampleTemplate',
'obj_id': int(st_id),
'sample_or_col': attribute,
'name': attr_id
})
job = ProcessingJob.create(User(user_id), params)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % st_id,
dumps({'job_id': job.id}))
job.submit()
return {'status': 'success', 'message': '', 'row_id': row_id}
else:
return {
'status':
'error',
'message':
'Operation "%s" not supported. '
'Current supported operations: remove' % req_op,
'row_id':
0
}
def test_artifact_summary_get_request(self):
# Artifact w/o summary
obs = artifact_summary_get_request('*****@*****.**', 1)
exp_p_jobs = [
['063e553b-327c-4818-ab4a-adfe58e49860', 'Split libraries FASTQ',
'queued', None, None],
['bcc7ebcd-39c1-43e4-af2d-822e3589f14d', 'Split libraries',
'running', 'demultiplexing', None]]
exp_files = [
(1L, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)'),
(2L, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)')]
exp = {'status': 'success',
'message': '',
'name': 'Raw data 1',
'processing_parameters': {},
'summary': None,
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'private',
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'files': exp_files,
'editable': True,
'prep_id': 1,
'study_id': 1}
self.assertEqual(obs, exp)
# Artifact with summary being generated
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 1})
)
job._set_status('queued')
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {'status': 'success',
'message': '',
'name': 'Raw data 1',
'processing_parameters': {},
'summary': None,
'job': [job.id, 'queued', None],
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'private',
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'files': exp_files,
'editable': True,
'prep_id': 1,
'study_id': 1}
self.assertEqual(obs, exp)
# Artifact with summary
fd, fp = mkstemp(suffix=".html")
close(fd)
with open(fp, 'w') as f:
f.write('<b>HTML TEST - not important</b>\n')
a = Artifact(1)
a.html_summary_fp = fp
self._files_to_remove.extend([fp, a.html_summary_fp[1]])
exp_files.append(
(a.html_summary_fp[0],
'%s (html summary)' % basename(a.html_summary_fp[1])))
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {'status': 'success',
'message': '',
'name': 'Raw data 1',
'processing_parameters': {},
'summary': '<b>HTML TEST - not important</b>\n',
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'private',
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'files': exp_files,
'editable': True,
'prep_id': 1,
'study_id': 1}
self.assertEqual(obs, exp)
# No access
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {'status': 'error',
'message': 'User does not have access to study'}
self.assertEqual(obs, exp)
# A non-owner/share user can't see the files
a.visibility = 'public'
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {'status': 'success',
'message': '',
'name': 'Raw data 1',
'processing_parameters': {},
'summary': '<b>HTML TEST - not important</b>\n',
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'public',
'buttons': '',
'files': [],
'editable': False,
'prep_id': 1,
'study_id': 1}
self.assertEqual(obs, exp)
# returnig to private
a.visibility = 'sandbox'
# admin gets buttons
obs = artifact_summary_get_request('*****@*****.**', 2)
exp_p_jobs = [
['d19f76ee-274e-4c1b-b3a2-a12d73507c55',
'Pick closed-reference OTUs', 'error', 'generating demux file',
'Error message']]
exp_files = [
(3L, '1_seqs.fna (preprocessed fasta)'),
(4L, '1_seqs.qual (preprocessed fastq)'),
(5L, '1_seqs.demux (preprocessed demux)')]
exp = {'status': 'success',
'files': exp_files,
'errored_jobs': [],
'editable': True,
'visibility': 'private',
'job': None,
'message': '',
'name': 'Demultiplexed 1',
'processing_jobs': exp_p_jobs,
'processing_parameters': {
'max_barcode_errors': 1.5, 'sequence_max_n': 0,
'max_bad_run_length': 3, 'phred_offset': u'auto',
'rev_comp': False, 'phred_quality_threshold': 3,
'input_data': 1, 'rev_comp_barcode': False,
'rev_comp_mapping_barcodes': False,
'min_per_read_length_fraction': 0.75,
'barcode_type': u'golay_12'},
'summary': None,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 2?\')) { '
'set_artifact_visibility(\'public\', 2) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'2?\')) { set_artifact_visibility(\'sandbox\', 2) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button> <a class="btn btn-primary '
'btn-sm" href="/vamps/2"><span class="glyphicon '
'glyphicon-export"></span> Submit to VAMPS</a>'),
'study_id': 1,
'prep_id': 1}
self.assertEqual(obs, exp)
def artifact_post_req(user_id,
filepaths,
artifact_type,
name,
prep_template_id,
artifact_id=None):
"""Creates the initial artifact for the prep template
Parameters
----------
user_id : str
User adding the atrifact
filepaths : dict of str
Comma-separated list of files to attach to the artifact,
keyed by file type
artifact_type : str
The type of the artifact
name : str
Name to give the artifact
prep_template_id : int or str castable to int
Prep template to attach the artifact to
artifact_id : int or str castable to int, optional
The id of the imported artifact
Returns
-------
dict of objects
A dictionary containing the new artifact ID
{'status': status,
'message': message,
'artifact': id}
"""
prep_template_id = int(prep_template_id)
prep = PrepTemplate(prep_template_id)
study_id = prep.study_id
# First check if the user has access to the study
access_error = check_access(study_id, user_id)
if access_error:
return access_error
user = User(user_id)
if artifact_id:
# if the artifact id has been provided, import the artifact
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('copy_artifact')
params = Parameters.load(cmd,
values_dict={
'artifact': artifact_id,
'prep_template': prep.id
})
job = ProcessingJob.create(user, params, True)
else:
uploads_path = get_mountpoint('uploads')[0][1]
path_builder = partial(join, uploads_path, str(study_id))
cleaned_filepaths = {}
for ftype, file_list in filepaths.items():
# JavaScript sends us this list as a comma-separated list
for fp in file_list.split(','):
# JavaScript will send this value as an empty string if the
# list of files was empty. In such case, the split will
# generate a single element containing the empty string. Check
# for that case here and, if fp is not the empty string,
# proceed to check if the file exists
if fp:
# Check if filepath being passed exists for study
full_fp = path_builder(fp)
exists = check_fp(study_id, full_fp)
if exists['status'] != 'success':
return {
'status': 'error',
'message': 'File does not exist: %s' % fp
}
if ftype not in cleaned_filepaths:
cleaned_filepaths[ftype] = []
cleaned_filepaths[ftype].append(full_fp)
# This should never happen, but it doesn't hurt to actually have
# a explicit check, in case there is something odd with the JS
if not cleaned_filepaths:
return {
'status': 'error',
'message': "Can't create artifact, no files provided."
}
# This try/except will catch the case when the plugins are not
# activated so there is no Validate for the given artifact_type
try:
command = Command.get_validator(artifact_type)
except QiitaDBError as e:
return {'status': 'error', 'message': str(e)}
job = ProcessingJob.create(
user,
Parameters.load(command,
values_dict={
'template': prep_template_id,
'files': dumps(cleaned_filepaths),
'artifact_type': artifact_type,
'name': name,
'analysis': None,
}), True)
# Submit the job
job.submit()
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep.id,
dumps({
'job_id': job.id,
'is_qiita_job': True
}))
return {'status': 'success', 'message': ''}
def prep_template_patch_req(user_id, req_op, req_path, req_value=None,
req_from=None):
"""Modifies an attribute of the prep template
Parameters
----------
user_id : str
The id of the user performing the patch operation
req_op : str
The operation to perform on the prep information
req_path : str
The prep information and attribute to patch
req_value : str, optional
The value that needs to be modified
req_from : str, optional
The original path of the element
Returns
-------
dict of {str, str, str}
A dictionary with the following keys:
- status: str, whether if the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
- row_id: str, the row_id that we tried to delete
"""
req_path = [v for v in req_path.split('/') if v]
if req_op == 'replace':
# The structure of the path should be /prep_id/attribute_to_modify/
# so if we don't have those 2 elements, we should return an error
if len(req_path) != 2:
return {'status': 'error',
'message': 'Incorrect path parameter'}
prep_id = int(req_path[0])
attribute = req_path[1]
# Check if the user actually has access to the prep template
prep = PrepTemplate(prep_id)
access_error = check_access(prep.study_id, user_id)
if access_error:
return access_error
status = 'success'
msg = ''
if attribute == 'investigation_type':
prep.investigation_type = req_value
elif attribute == 'data':
fp = check_fp(prep.study_id, req_value)
if fp['status'] != 'success':
return fp
fp = fp['file']
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('update_prep_template')
params = Parameters.load(
cmd, values_dict={'prep_template': prep_id, 'template_fp': fp})
job = ProcessingJob.create(User(user_id), params, True)
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id,
dumps({'job_id': job.id}))
job.submit()
elif attribute == 'name':
prep.name = req_value.strip()
else:
# We don't understand the attribute so return an error
return {'status': 'error',
'message': 'Attribute "%s" not found. '
'Please, check the path parameter' % attribute}
return {'status': status, 'message': msg}
elif req_op == 'remove':
# The structure of the path should be:
# /prep_id/row_id/{columns|samples}/name
if len(req_path) != 4:
return {'status': 'error',
'message': 'Incorrect path parameter'}
prep_id = int(req_path[0])
row_id = req_path[1]
attribute = req_path[2]
attr_id = req_path[3]
# Check if the user actually has access to the study
pt = PrepTemplate(prep_id)
access_error = check_access(pt.study_id, user_id)
if access_error:
return access_error
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_or_column')
params = Parameters.load(
cmd, values_dict={'obj_class': 'PrepTemplate',
'obj_id': prep_id,
'sample_or_col': attribute,
'name': attr_id})
job = ProcessingJob.create(User(user_id), params, True)
# Store the job id attaching it to the sample template id
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id,
dumps({'job_id': job.id}))
job.submit()
return {'status': 'success', 'message': '', 'row_id': row_id}
else:
return {'status': 'error',
'message': 'Operation "%s" not supported. '
'Current supported operations: replace, remove'
% req_op,
'row_id': '0'}
def test_artifact_summary_get_request(self):
user = User('*****@*****.**')
# Artifact w/o summary
obs = artifact_summary_get_request(user, 1)
exp_files = [
(1L, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)'),
(2L, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)')]
exp = {'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'private',
'editable': True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': None,
'job': None,
'errored_summary_jobs': []}
self.assertEqual(obs, exp)
# Artifact with summary being generated
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 1})
)
job._set_status('queued')
obs = artifact_summary_get_request(user, 1)
exp = {'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'private',
'editable': True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': None,
'job': [job.id, 'queued', None],
'errored_summary_jobs': []}
self.assertEqual(obs, exp)
# Artifact with summary
fd, fp = mkstemp(suffix=".html")
close(fd)
with open(fp, 'w') as f:
f.write('<b>HTML TEST - not important</b>\n')
a = Artifact(1)
a.set_html_summary(fp)
self._files_to_remove.extend([fp, a.html_summary_fp[1]])
exp_files.append(
(a.html_summary_fp[0],
'%s (html summary)' % basename(a.html_summary_fp[1])))
exp_summary_path = relpath(
a.html_summary_fp[1], qiita_config.base_data_dir)
obs = artifact_summary_get_request(user, 1)
exp = {'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'private',
'editable': True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': exp_summary_path,
'job': None,
'errored_summary_jobs': []}
self.assertEqual(obs, exp)
# No access
demo_u = User('*****@*****.**')
with self.assertRaises(QiitaHTTPError):
obs = artifact_summary_get_request(demo_u, 1)
# A non-owner/share user can't see the files
a.visibility = 'public'
obs = artifact_summary_get_request(demo_u, 1)
exp = {'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'public',
'editable': False,
'buttons': '',
'processing_info': {},
'files': [],
'is_from_analysis': False,
'summary': exp_summary_path,
'job': None,
'errored_summary_jobs': []}
self.assertEqual(obs, exp)
# returnig to private
a.visibility = 'private'
# admin gets buttons
obs = artifact_summary_get_request(User('*****@*****.**'), 2)
exp_files = [
(3L, '1_seqs.fna (preprocessed fasta)'),
(4L, '1_seqs.qual (preprocessed fastq)'),
(5L, '1_seqs.demux (preprocessed demux)')]
exp = {'name': 'Demultiplexed 1',
'artifact_id': 2,
'artifact_type': 'Demultiplexed',
'artifact_timestamp': '2012-10-01 10:10',
'visibility': 'private',
'editable': True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 2?\')) { '
'set_artifact_visibility(\'public\', 2) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'2?\')) { set_artifact_visibility(\'sandbox\', 2) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button> <a class="btn btn-primary '
'btn-sm" href="/ebi_submission/2"><span '
'class="glyphicon glyphicon-export"></span> '
'Submit to EBI</a> <a class="btn btn-primary '
'btn-sm" href="/vamps/2"><span class="glyphicon '
'glyphicon-export"></span> Submit to VAMPS</a>'),
'processing_info': {
'command': 'Split libraries FASTQ',
'software': 'QIIME',
'software_version': '1.9.1',
'processing_parameters': {
'max_barcode_errors': '1.5', 'sequence_max_n': '0',
'max_bad_run_length': '3', 'phred_offset': u'auto',
'rev_comp': 'False', 'phred_quality_threshold': '3',
'input_data': '1', 'rev_comp_barcode': 'False',
'rev_comp_mapping_barcodes': 'False',
'min_per_read_length_fraction': '0.75',
'barcode_type': u'golay_12'}},
'files': exp_files,
'is_from_analysis': False,
'summary': None,
'job': None,
'errored_summary_jobs': []}
self.assertEqual(obs, exp)
# analysis artifact
obs = artifact_summary_get_request(user, 8)
exp = {'name': 'noname',
'artifact_id': 8,
'artifact_type': 'BIOM',
# this value changes on build so copy from obs
'artifact_timestamp': obs['artifact_timestamp'],
'visibility': 'sandbox',
'editable': True,
'buttons': '',
'processing_info': {},
'files': [(27, 'biom_table.biom (biom)')],
'is_from_analysis': True,
'summary': None,
'job': None,
'errored_summary_jobs': []}
self.assertEqual(obs, exp)
def test_job_ajax_patch_req(self):
# Create a new job - through a workflow since that is the only way
# of creating jobs in the interface
exp_command = Command(1)
json_str = (
'{"input_data": 1, "max_barcode_errors": 1.5, '
'"barcode_type": "golay_12", "max_bad_run_length": 3, '
'"rev_comp": false, "phred_quality_threshold": 3, '
'"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
'"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
exp_params = Parameters.load(exp_command, json_str=json_str)
exp_user = User('*****@*****.**')
name = "Test processing workflow"
# tests success
wf = ProcessingWorkflow.from_scratch(exp_user,
exp_params,
name=name,
force=True)
graph = wf.graph
nodes = list(graph.nodes())
job_id = nodes[0].id
# Incorrect path parameter
obs = job_ajax_patch_req('remove', '/%s/somethingelse' % job_id)
exp = {
'status': 'error',
'message': 'Incorrect path parameter: missing job id'
}
self.assertEqual(obs, exp)
obs = job_ajax_patch_req('remove', '/')
exp = {
'status': 'error',
'message': 'Incorrect path parameter: missing job id'
}
self.assertEqual(obs, exp)
# Job id is not like a job id
obs = job_ajax_patch_req('remove', '/notAJobId')
exp = {
'status':
'error',
'message':
'Incorrect path parameter: '
'notAJobId is not a recognized job id'
}
self.assertEqual(obs, exp)
# Job doesn't exist
obs = job_ajax_patch_req('remove',
'/6d368e16-2242-4cf8-87b4-a5dc40bc890b')
exp = {
'status':
'error',
'message':
'Incorrect path parameter: '
'6d368e16-2242-4cf8-87b4-a5dc40bc890b is not a '
'recognized job id'
}
self.assertEqual(obs, exp)
# in_construction job
obs = job_ajax_patch_req('remove', '/%s' % job_id)
exp = {
'status':
'error',
'message':
"Can't delete job %s. It is 'in_construction' "
"status. Please use /study/process/workflow/" % job_id
}
self.assertEqual(obs, exp)
# job status != 'error'
job = ProcessingJob(job_id)
job._set_status('queued')
obs = job_ajax_patch_req('remove', '/%s' % job_id)
exp = {
'status': 'error',
'message': 'Only jobs in "error" status can be deleted.'
}
self.assertEqual(obs, exp)
# Operation not supported
job._set_status('queued')
obs = job_ajax_patch_req('add', '/%s' % job_id)
exp = {
'status':
'error',
'message':
'Operation "add" not supported. Current supported '
'operations: remove'
}
self.assertEqual(obs, exp)
# Test success
job._set_error('Killed for testing')
obs = job_ajax_patch_req('remove', '/%s' % job_id)
exp = {'status': 'success', 'message': ''}
self.assertEqual(obs, exp)
def test_artifact_summary_get_request(self):
user = User('*****@*****.**')
# Artifact w/o summary
obs = artifact_summary_get_request(user, 1)
exp_p_jobs = [[
'063e553b-327c-4818-ab4a-adfe58e49860', 'Split libraries FASTQ',
'queued', None, None
],
[
'bcc7ebcd-39c1-43e4-af2d-822e3589f14d',
'Split libraries', 'running', 'demultiplexing', None
]]
exp_files = [
(1L, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)'),
(2L, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)')
]
exp = {
'name':
'Raw data 1',
'artifact_id':
1,
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'processing_parameters': {},
'files':
exp_files,
'summary':
None,
'job':
None,
'processing_jobs':
exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# Artifact with summary being generated
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 1}))
job._set_status('queued')
obs = artifact_summary_get_request(user, 1)
exp = {
'name':
'Raw data 1',
'artifact_id':
1,
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'processing_parameters': {},
'files':
exp_files,
'summary':
None,
'job': [job.id, 'queued', None],
'processing_jobs':
exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# Artifact with summary
fd, fp = mkstemp(suffix=".html")
close(fd)
with open(fp, 'w') as f:
f.write('<b>HTML TEST - not important</b>\n')
a = Artifact(1)
a.set_html_summary(fp)
self._files_to_remove.extend([fp, a.html_summary_fp[1]])
exp_files.append(
(a.html_summary_fp[0],
'%s (html summary)' % basename(a.html_summary_fp[1])))
exp_summary_path = relpath(a.html_summary_fp[1],
qiita_config.base_data_dir)
obs = artifact_summary_get_request(user, 1)
exp = {
'name':
'Raw data 1',
'artifact_id':
1,
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'processing_parameters': {},
'files':
exp_files,
'summary':
exp_summary_path,
'job':
None,
'processing_jobs':
exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# No access
demo_u = User('*****@*****.**')
with self.assertRaises(QiitaHTTPError):
obs = artifact_summary_get_request(demo_u, 1)
# A non-owner/share user can't see the files
a.visibility = 'public'
obs = artifact_summary_get_request(demo_u, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'visibility': 'public',
'editable': False,
'buttons': '',
'processing_parameters': {},
'files': [],
'summary': exp_summary_path,
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# returnig to private
a.visibility = 'private'
# admin gets buttons
obs = artifact_summary_get_request(User('*****@*****.**'), 2)
exp_p_jobs = [[
'd19f76ee-274e-4c1b-b3a2-a12d73507c55',
'Pick closed-reference OTUs', 'error', 'generating demux file',
'Error message'
]]
exp_files = [(3L, '1_seqs.fna (preprocessed fasta)'),
(4L, '1_seqs.qual (preprocessed fastq)'),
(5L, '1_seqs.demux (preprocessed demux)')]
exp = {
'name':
'Demultiplexed 1',
'artifact_id':
2,
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 2?\')) { '
'set_artifact_visibility(\'public\', 2) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'2?\')) { set_artifact_visibility(\'sandbox\', 2) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button> <a class="btn btn-primary '
'btn-sm" href="/ebi_submission/2"><span '
'class="glyphicon glyphicon-export"></span> '
'Submit to EBI</a> <a class="btn btn-primary '
'btn-sm" href="/vamps/2"><span class="glyphicon '
'glyphicon-export"></span> Submit to VAMPS</a>'),
'processing_parameters': {
'max_barcode_errors': 1.5,
'sequence_max_n': 0,
'max_bad_run_length': 3,
'phred_offset': u'auto',
'rev_comp': False,
'phred_quality_threshold': 3,
'input_data': 1,
'rev_comp_barcode': False,
'rev_comp_mapping_barcodes': False,
'min_per_read_length_fraction': 0.75,
'barcode_type': u'golay_12'
},
'files':
exp_files,
'summary':
None,
'job':
None,
'processing_jobs':
exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# analysis artifact
obs = artifact_summary_get_request(user, 8)
exp = {
'name': 'noname',
'artifact_id': 8,
'visibility': 'sandbox',
'editable': True,
'buttons': '',
'processing_parameters': {},
'files': [(27, 'biom_table.biom (biom)')],
'summary': None,
'job': None,
'processing_jobs': [],
'errored_jobs': []
}
self.assertEqual(obs, exp)
def sample_template_post_req(study_id, user_id, data_type, sample_template):
"""Creates the sample template from the given file
Parameters
----------
study_id : int
The current study object id
user_id : str
The current user object id
data_type : str
Data type for the sample template
sample_template : str
filename to use for creation
Returns
-------
dict
results dictonary in the format
{'status': status,
'message': msg,
'file': sample_template}
status can be success, warning, or error depending on result
message has the warnings or errors
file has the file name
"""
study_id = int(study_id)
access_error = check_access(study_id, user_id)
if access_error:
return access_error
fp_rsp = check_fp(study_id, sample_template)
if fp_rsp['status'] != 'success':
# Unknown filepath, so return the error message
return fp_rsp
fp_rsp = fp_rsp['file']
# Define here the message and message level in case of success
is_mapping_file = looks_like_qiime_mapping_file(fp_rsp)
if is_mapping_file and not data_type:
return {
'status': 'error',
'message': 'Please, choose a data type if uploading a '
'QIIME mapping file',
'file': sample_template
}
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('create_sample_template')
params = Parameters.load(cmd,
values_dict={
'fp': fp_rsp,
'study_id': study_id,
'is_mapping_file': is_mapping_file,
'data_type': data_type
})
job = ProcessingJob.create(User(user_id), params)
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
# Store the job id attaching it to the sample template id
job.submit()
return {'status': 'success', 'message': '', 'file': sample_template}
def correct_redis_data(key, cmd, values_dict, user):
"""Corrects the data stored in the redis DB
Parameters
----------
key: str
The redis key to fix
cmd : qiita_db.software.Command
Command to use to create the processing job
values_dict : dict
Dictionary used to instantiate the parameters of the command
user : qiita_db.user. User
The user that will own the job
"""
info = r_client.get(key)
if info:
info = loads(info)
if info['job_id'] is not None:
if 'is_qiita_job' in info:
if info['is_qiita_job']:
try:
job = ProcessingJob(info['job_id'])
payload = {'job_id': info['job_id'],
'alert_type': info['status'],
'alert_msg': info['alert_msg']}
r_client.set(key, dumps(payload))
except (QiitaDBUnknownIDError, KeyError):
# We shomehow lost the information of this job
# Simply delete the key
r_client.delete(key)
else:
# These jobs don't contain any information on the live
# dump. We can safely delete the key
r_client.delete(key)
else:
# These jobs don't contain any information on the live
# dump. We can safely delete the key
r_client.delete(key)
else:
# Job is null, we have the information here
if info['status'] == 'success':
# In the success case no information is stored. We can
# safely delete the key
r_client.delete(key)
elif info['status'] == 'warning':
# In case of warning the key message stores the warning
# message. We need to create a new job, mark it as
# successful and store the error message as expected by
# the new structure
params = Parameters.load(cmd, values_dict=values_dict)
job = ProcessingJob.create(user, params)
job._set_status('success')
payload = {'job_id': job.id,
'alert_type': 'warning',
'alert_msg': info['message']}
r_client.set(key, dumps(payload))
else:
# The status is error. The key message stores the error
# message. We need to create a new job and mark it as
# failed with the given error message
params = Parameters.load(cmd, values_dict=values_dict)
job = ProcessingJob.create(user, params)
job._set_error(info['message'])
payload = {'job_id': job.id}
r_client.set(key, dumps(payload))
else:
# The key doesn't contain any information. Delete the key
r_client.delete(key)
def test_get_analysis_graph_handler(self):
response = self.get('/analysis/description/1/graph/')
self.assertEqual(response.code, 200)
# The job id is randomly generated in the test environment. Gather
# it here. There is only 1 job in the first artifact of the analysis
job_id = Analysis(1).artifacts[0].jobs()[0].id
obs = loads(response.body)
exp = {'edges': [[8, job_id], [job_id, 9]],
'nodes': [
['job', 'job', job_id, 'Single Rarefaction', 'success'],
['artifact', 'BIOM', 9, 'noname\n(BIOM)', 'artifact'],
['artifact', 'BIOM', 8, 'noname\n(BIOM)', 'artifact']],
'workflow': None}
self.assertItemsEqual(obs, exp)
self.assertItemsEqual(obs['edges'], exp['edges'])
self.assertItemsEqual(obs['nodes'], exp['nodes'])
self.assertIsNone(obs['workflow'])
# Create a new analysis with 2 starting BIOMs to be able to test
# the different if statements of the request
BaseHandler.get_current_user = Mock(
return_value=User('*****@*****.**'))
user = User('*****@*****.**')
dflt_analysis = user.default_analysis
dflt_analysis.add_samples(
{4: ['1.SKB8.640193', '1.SKD8.640184', '1.SKB7.640196'],
6: ['1.SKB8.640193', '1.SKD8.640184', '1.SKB7.640196']})
args = {'name': 'New Test Graph Analysis', 'description': 'Desc'}
response = self.post('/analysis/create/', args)
new_id = response.effective_url.split('/')[-2]
a = Analysis(new_id)
# Wait until all the jobs are done so the BIOM tables exist
for j in a.jobs:
wait_for_processing_job(j.id)
artifacts = a.artifacts
self.assertEqual(len(artifacts), 2)
# Create a new workflow starting on the first artifact
# Magic number 9 -> Summarize Taxa command
params = Parameters.load(
Command(9), values_dict={'metadata_category': 'None',
'sort': 'False',
'biom_table': artifacts[0].id})
wf = ProcessingWorkflow.from_scratch(user, params)
# There is only one job in the workflow
job_id = wf.graph.nodes()[0].id
response = self.get('/analysis/description/%s/graph/' % new_id)
self.assertEqual(response.code, 200)
obs = loads(response.body)
exp = {'edges': [[artifacts[0].id, job_id],
[job_id, '%s:taxa_summary' % job_id]],
'nodes': [
['job', 'job', job_id, 'Summarize Taxa',
'in_construction'],
['artifact', 'BIOM', artifacts[0].id, 'noname\n(BIOM)',
'artifact'],
['artifact', 'BIOM', artifacts[1].id, 'noname\n(BIOM)',
'artifact'],
['type', 'taxa_summary', '%s:taxa_summary' % job_id,
'taxa_summary\n(taxa_summary)', 'type']],
'workflow': wf.id}
# Check that the keys are the same
self.assertItemsEqual(obs, exp)
# Check the edges
self.assertItemsEqual(obs['edges'], exp['edges'])
# Check the edges
self.assertItemsEqual(obs['nodes'], exp['nodes'])
# Check the edges
self.assertEqual(obs['workflow'], exp['workflow'])
# Add a job to the second BIOM to make sure that the edges and nodes
# are respected. Magic number 12 -> Single Rarefaction
job2 = wf.add(
DefaultParameters(16), req_params={'depth': '100',
'biom_table': artifacts[1].id})
job_id_2 = job2.id
response = self.get('/analysis/description/%s/graph/' % new_id)
self.assertEqual(response.code, 200)
obs = loads(response.body)
exp = {'edges': [[artifacts[0].id, job_id],
[job_id, '%s:taxa_summary' % job_id],
[artifacts[1].id, job_id_2],
[job_id_2, '%s:rarefied_table' % job_id_2]],
'nodes': [
['job', 'job', job_id, 'Summarize Taxa',
'in_construction'],
['job', 'job', job_id_2, 'Single Rarefaction',
'in_construction'],
['artifact', 'BIOM', artifacts[0].id, 'noname\n(BIOM)',
'artifact'],
['artifact', 'BIOM', artifacts[1].id, 'noname\n(BIOM)',
'artifact'],
['type', 'taxa_summary', '%s:taxa_summary' % job_id,
'taxa_summary\n(taxa_summary)', 'type'],
['type', 'BIOM', '%s:rarefied_table' % job_id_2,
'rarefied_table\n(BIOM)', 'type']],
'workflow': wf.id}
# Check that the keys are the same
self.assertItemsEqual(obs, exp)
# Check the edges
self.assertItemsEqual(obs['edges'], exp['edges'])
# Check the edges
self.assertItemsEqual(obs['nodes'], exp['nodes'])
# Check the edges
self.assertEqual(obs['workflow'], exp['workflow'])
# Add a second Workflow to the second artifact to force the raise of
# the error. This situation should never happen when using
# the interface
wf.remove(job2)
params = Parameters.load(
Command(9), values_dict={'metadata_category': 'None',
'sort': 'False',
'biom_table': artifacts[1].id})
wf = ProcessingWorkflow.from_scratch(user, params)
response = self.get('/analysis/description/%s/graph/' % new_id)
self.assertEqual(response.code, 500)
