def post(self, study_id):
method = self.get_argument('remote-request-type')
url = self.get_argument('inputURL')
ssh_key = self.request.files['ssh-key'][0]['body']
status = 'success'
message = ''
try:
study = Study(int(study_id))
except QiitaDBUnknownIDError:
raise HTTPError(404, reason="Study %s does not exist" % study_id)
check_access(self.current_user,
study,
no_public=True,
raise_error=True)
_, upload_folder = get_mountpoint("uploads")[0]
upload_folder = join(upload_folder, study_id)
ssh_key_fp = join(upload_folder, '.key.txt')
create_nested_path(upload_folder)
with open(ssh_key_fp, 'wb') as f:
f.write(ssh_key)
chmod(ssh_key_fp, 0o600)
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
if method == 'list':
cmd = qiita_plugin.get_command('list_remote_files')
params = Parameters.load(cmd,
values_dict={
'url': url,
'private_key': ssh_key_fp,
'study_id': study_id
})
elif method == 'transfer':
cmd = qiita_plugin.get_command('download_remote_files')
params = Parameters.load(cmd,
values_dict={
'url': url,
'private_key': ssh_key_fp,
'destination': upload_folder
})
else:
status = 'error'
message = 'Not a valid method'
if status == 'success':
job = ProcessingJob.create(self.current_user, params, True)
job.submit()
r_client.set(UPLOAD_STUDY_FORMAT % study_id,
dumps({'job_id': job.id}))
self.write({'status': status, 'message': message})
def post(self, preprocessed_data_id):
user = self.current_user
# make sure user is admin and can therefore actually submit to VAMPS
if user.level != 'admin':
raise HTTPError(403, "User %s cannot submit to VAMPS!" %
user.id)
msg = ''
msg_level = 'success'
plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = plugin.get_command('submit_to_VAMPS')
artifact = Artifact(preprocessed_data_id)
# Check if the artifact is already being submitted to VAMPS
is_being_submitted = any(
[j.status in ('queued', 'running')
for j in artifact.jobs(cmd=cmd)])
if is_being_submitted == 'submitting':
msg = "Cannot resubmit! Data is already being submitted"
msg_level = 'danger'
self.display_template(preprocessed_data_id, msg, msg_level)
else:
params = Parameters.load(
cmd, values_dict={'artifact': preprocessed_data_id})
job = ProcessingJob.create(user, params, True)
job.submit()
self.redirect('/study/description/%s' % artifact.study.study_id)
def artifact_post_req(user, artifact_id):
"""Deletes the artifact
Parameters
----------
user : qiita_db.user.User
The user requesting the action
artifact_id : int
Id of the artifact being deleted
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
check_artifact_access(user, artifact)
analysis = artifact.analysis
if analysis:
# Do something when deleting in the analysis part to keep track of it
redis_key = "analysis_%s" % analysis.id
else:
pt_id = artifact.prep_templates[0].id
redis_key = PREP_TEMPLATE_KEY_FORMAT % pt_id
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_artifact')
params = Parameters.load(cmd, values_dict={'artifact': artifact_id})
job = ProcessingJob.create(user, params)
r_client.set(redis_key, dumps({'job_id': job.id, 'is_qiita_job': True}))
job.submit()
def post(self, preprocessed_data_id):
user = self.current_user
# make sure user is admin and can therefore actually submit to EBI
if user.level != 'admin':
raise HTTPError(403, reason="User %s cannot submit to EBI!" %
user.id)
submission_type = self.get_argument('submission_type')
if submission_type not in ['ADD', 'MODIFY']:
raise HTTPError(403, reason="User: %s, %s is not a recognized "
"submission type" % (user.id, submission_type))
study = Artifact(preprocessed_data_id).study
state = study.ebi_submission_status
if state == 'submitting':
message = "Cannot resubmit! Current state is: %s" % state
self.display_template(preprocessed_data_id, message, 'danger')
else:
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('submit_to_EBI')
params = Parameters.load(
cmd, values_dict={'artifact': preprocessed_data_id,
'submission_type': submission_type})
job = ProcessingJob.create(user, params, True)
r_client.set('ebi_submission_%s' % preprocessed_data_id,
dumps({'job_id': job.id, 'is_qiita_job': True}))
job.submit()
level = 'success'
message = 'EBI submission started. Job id: %s' % job.id
self.redirect("%s/study/description/%d?level=%s&message=%s" % (
qiita_config.portal_dir, study.id, level, url_escape(message)))
def write_demux_files(self, prep_template, generate_hdf5=True):
"""Writes a demux test file to avoid duplication of code"""
fna_fp = join(self.temp_dir, 'seqs.fna')
demux_fp = join(self.temp_dir, 'demux.seqs')
if generate_hdf5:
with open(fna_fp, 'w') as f:
f.write(FASTA_EXAMPLE)
with File(demux_fp, "w") as f:
to_hdf5(fna_fp, f)
else:
with open(demux_fp, 'w') as f:
f.write('')
if prep_template.artifact is None:
ppd = Artifact.create([(demux_fp, 6)],
"Demultiplexed",
prep_template=prep_template)
else:
params = Parameters.from_default_params(
DefaultParameters(1),
{'input_data': prep_template.artifact.id})
ppd = Artifact.create([(demux_fp, 6)],
"Demultiplexed",
parents=[prep_template.artifact],
processing_parameters=params)
return ppd
def study_delete_req(study_id, user_id):
"""Delete a given study
Parameters
----------
study_id : int
Study id to delete
user_id : str
User requesting the deletion
Returns
-------
dict
Status of deletion, in the format
{status: status,
message: message}
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_study')
params = Parameters.load(cmd, values_dict={'study': study_id})
job = ProcessingJob.create(User(user_id), params, True)
# Store the job id attaching it to the sample template id
r_client.set(STUDY_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'status': 'success', 'message': ''}
def setUp(self):
uploads_path = get_mountpoint('uploads')[0][1]
# Create prep test file to point at
self.update_fp = join(uploads_path, '1', 'update.txt')
with open(self.update_fp, 'w') as f:
f.write("""sample_name\tnew_col\n1.SKD6.640190\tnew_value\n""")
self._files_to_remove = [self.update_fp]
self._files_to_remove = []
# creating temporal files and artifact
# NOTE: we don't need to remove the artifact created cause it's
# used to test the delete functionality
fd, fp = mkstemp(suffix='_seqs.fna')
close(fd)
with open(fp, 'w') as f:
f.write(">1.sid_r4_0 M02034:17:000000000-A5U18:1:1101:15370:1394 "
"1:N:0:1 orig_bc=CATGAGCT new_bc=CATGAGCT bc_diffs=0\n"
"GTGTGCCAGCAGCCGCGGTAATACGTAGGG\n")
# 4 Demultiplexed
filepaths_processed = [(fp, 4)]
# 1 for default parameters and input data
exp_params = Parameters.from_default_params(DefaultParameters(1),
{'input_data': 1})
self.artifact = Artifact.create(filepaths_processed, "Demultiplexed",
parents=[Artifact(1)],
processing_parameters=exp_params)
def setUp(self):
uploads_path = get_mountpoint('uploads')[0][1]
# Create prep test file to point at
self.update_fp = join(uploads_path, '1', 'update.txt')
with open(self.update_fp, 'w') as f:
f.write("""sample_name\tnew_col\n1.SKD6.640190\tnew_value\n""")
self._files_to_remove = [self.update_fp]
self._files_to_remove = []
# creating temporal files and artifact
# NOTE: we don't need to remove the artifact created cause it's
# used to test the delete functionality
fd, fp = mkstemp(suffix='_seqs.fna')
close(fd)
with open(fp, 'w') as f:
f.write(">1.sid_r4_0 M02034:17:000000000-A5U18:1:1101:15370:1394 "
"1:N:0:1 orig_bc=CATGAGCT new_bc=CATGAGCT bc_diffs=0\n"
"GTGTGCCAGCAGCCGCGGTAATACGTAGGG\n")
# 4 Demultiplexed
filepaths_processed = [(fp, 4)]
# 1 for default parameters and input data
exp_params = Parameters.from_default_params(DefaultParameters(1),
{'input_data': 1})
self.artifact = Artifact.create(filepaths_processed, "Demultiplexed",
parents=[Artifact(1)],
processing_parameters=exp_params)
def artifact_post_req(user, artifact_id):
"""Deletes the artifact
Parameters
----------
user : qiita_db.user.User
The user requesting the action
artifact_id : int
Id of the artifact being deleted
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
check_artifact_access(user, artifact)
analysis = artifact.analysis
if analysis:
# Do something when deleting in the analysis part to keep track of it
redis_key = "analysis_%s" % analysis.id
else:
pt_id = artifact.prep_templates[0].id
redis_key = PREP_TEMPLATE_KEY_FORMAT % pt_id
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_artifact')
params = Parameters.load(cmd, values_dict={'artifact': artifact_id})
job = ProcessingJob.create(user, params, True)
r_client.set(redis_key, dumps({'job_id': job.id, 'is_qiita_job': True}))
job.submit()
return {'job': job.id}
def write_demux_files(self, prep_template, generate_hdf5=True):
"""Writes a demux test file to avoid duplication of code"""
fna_fp = join(self.temp_dir, 'seqs.fna')
demux_fp = join(self.temp_dir, 'demux.seqs')
if generate_hdf5:
with open(fna_fp, 'w') as f:
f.write(FASTA_EXAMPLE)
with File(demux_fp, "w") as f:
to_hdf5(fna_fp, f)
else:
with open(demux_fp, 'w') as f:
f.write('')
if prep_template.artifact is None:
ppd = Artifact.create(
[(demux_fp, 6)], "Demultiplexed", prep_template=prep_template,
can_be_submitted_to_ebi=True, can_be_submitted_to_vamps=True)
else:
params = Parameters.from_default_params(
DefaultParameters(1),
{'input_data': prep_template.artifact.id})
ppd = Artifact.create(
[(demux_fp, 6)], "Demultiplexed",
parents=[prep_template.artifact], processing_parameters=params,
can_be_submitted_to_ebi=True, can_be_submitted_to_vamps=True)
return ppd
def study_delete_req(study_id, user_id):
"""Delete a given study
Parameters
----------
study_id : int
Study id to delete
user_id : str
User requesting the deletion
Returns
-------
dict
Status of deletion, in the format
{status: status,
message: message}
"""
access_error = check_access(study_id, user_id)
if access_error:
return access_error
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_study')
params = Parameters.load(cmd, values_dict={'study': study_id})
job = ProcessingJob.create(User(user_id), params, True)
# Store the job id attaching it to the sample template id
r_client.set(STUDY_KEY_FORMAT % study_id, dumps({'job_id': job.id}))
job.submit()
return {'status': 'success', 'message': ''}
def post(self, preprocessed_data_id):
user = self.current_user
# make sure user is admin and can therefore actually submit to EBI
if user.level != 'admin':
raise HTTPError(403, reason="User %s cannot submit to EBI!" %
user.id)
submission_type = self.get_argument('submission_type')
if submission_type not in ['ADD', 'MODIFY']:
raise HTTPError(403, reason="User: %s, %s is not a recognized "
"submission type" % (user.id, submission_type))
study = Artifact(preprocessed_data_id).study
state = study.ebi_submission_status
if state == 'submitting':
message = "Cannot resubmit! Current state is: %s" % state
self.display_template(preprocessed_data_id, message, 'danger')
else:
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('submit_to_EBI')
params = Parameters.load(
cmd, values_dict={'artifact': preprocessed_data_id,
'submission_type': submission_type})
job = ProcessingJob.create(user, params, True)
r_client.set('ebi_submission_%s' % preprocessed_data_id,
dumps({'job_id': job.id, 'is_qiita_job': True}))
job.submit()
level = 'success'
message = 'EBI submission started. Job id: %s' % job.id
self.redirect("%s/study/description/%d?level=%s&message=%s" % (
qiita_config.portal_dir, study.id, level, url_escape(message)))
def post(self, preprocessed_data_id):
user = self.current_user
# make sure user is admin and can therefore actually submit to VAMPS
if user.level != 'admin':
raise HTTPError(403, "User %s cannot submit to VAMPS!" % user.id)
msg = ''
msg_level = 'success'
plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = plugin.get_command('submit_to_VAMPS')
artifact = Artifact(preprocessed_data_id)
# Check if the artifact is already being submitted to VAMPS
is_being_submitted = any([
j.status in ('queued', 'running') for j in artifact.jobs(cmd=cmd)
])
if is_being_submitted == 'submitting':
msg = "Cannot resubmit! Data is already being submitted"
msg_level = 'danger'
self.display_template(preprocessed_data_id, msg, msg_level)
else:
params = Parameters.load(
cmd, values_dict={'artifact': preprocessed_data_id})
job = ProcessingJob.create(user, params)
job.submit()
self.redirect('/study/description/%s' % artifact.study.study_id)
def _create_job(self, cmd_name, values_dict):
self.user = User('*****@*****.**')
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command(cmd_name)
params = Parameters.load(cmd, values_dict=values_dict)
job = ProcessingJob.create(self.user, params, True)
job._set_status('queued')
return job
def _create_job(self, cmd_name, values_dict):
self.user = User('*****@*****.**')
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command(cmd_name)
params = Parameters.load(cmd, values_dict=values_dict)
job = ProcessingJob.create(self.user, params, True)
job._set_status('queued')
return job
def sample_template_handler_post_request(study_id,
user,
filepath,
data_type=None):
"""Creates a new sample template
Parameters
----------
study_id: int
The study to add the sample information
user: qiita_db.user import User
The user performing the request
filepath: str
The path to the sample template file
data_type: str, optional
If filepath is a QIIME mapping file, the data type of the prep
information file
Returns
-------
dict of {'job': str}
job: the id of the job adding the sample information to the study
Raises
------
HTTPError
404 if the filepath doesn't exist
"""
# Check if the current user has access to the study
sample_template_checks(study_id, user)
# Check if the file exists
fp_rsp = check_fp(study_id, filepath)
if fp_rsp['status'] != 'success':
raise HTTPError(404, 'Filepath not found')
filepath = fp_rsp['file']
is_mapping_file = looks_like_qiime_mapping_file(filepath)
if is_mapping_file and not data_type:
raise HTTPError(
400, 'Please, choose a data type if uploading a '
'QIIME mapping file')
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('create_sample_template')
params = Parameters.load(cmd,
values_dict={
'fp': filepath,
'study_id': study_id,
'is_mapping_file': is_mapping_file,
'data_type': data_type
})
job = ProcessingJob.create(user, params, True)
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
def sample_template_put_req(study_id, user_id, sample_template):
"""Updates a sample template using the given file
Parameters
----------
study_id : int
The current study object id
user_id : str
The current user object id
sample_template : str
filename to use for updating
Returns
-------
dict
results dictonary in the format
{'status': status,
'message': msg,
'file': sample_template}
status can be success, warning, or error depending on result
message has the warnings or errors
file has the file name
"""
exists = _check_sample_template_exists(int(study_id))
if exists['status'] != 'success':
return exists
access_error = check_access(int(study_id), user_id)
if access_error:
return access_error
fp_rsp = check_fp(study_id, sample_template)
if fp_rsp['status'] != 'success':
# Unknown filepath, so return the error message
return fp_rsp
fp_rsp = fp_rsp['file']
msg = ''
status = 'success'
# Offload the update of the sample template to the cluster
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('update_sample_template')
params = Parameters.load(cmd,
values_dict={
'study': int(study_id),
'template_fp': fp_rsp
})
job = ProcessingJob.create(User(user_id), params)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'status': status, 'message': msg, 'file': sample_template}
def post(self, study_id):
method = self.get_argument('remote-request-type')
url = self.get_argument('inputURL')
ssh_key = self.request.files['ssh-key'][0]['body']
status = 'success'
message = ''
try:
study = Study(int(study_id))
except QiitaDBUnknownIDError:
raise HTTPError(404, reason="Study %s does not exist" % study_id)
check_access(
self.current_user, study, no_public=True, raise_error=True)
_, upload_folder = get_mountpoint("uploads")[0]
upload_folder = join(upload_folder, study_id)
ssh_key_fp = join(upload_folder, '.key.txt')
create_nested_path(upload_folder)
with open(ssh_key_fp, 'w') as f:
f.write(ssh_key)
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
if method == 'list':
cmd = qiita_plugin.get_command('list_remote_files')
params = Parameters.load(cmd, values_dict={
'url': url, 'private_key': ssh_key_fp, 'study_id': study_id})
elif method == 'transfer':
cmd = qiita_plugin.get_command('download_remote_files')
params = Parameters.load(cmd, values_dict={
'url': url, 'private_key': ssh_key_fp,
'destination': upload_folder})
else:
status = 'error'
message = 'Not a valid method'
if status == 'success':
job = ProcessingJob.create(self.current_user, params, True)
job.submit()
r_client.set(
UPLOAD_STUDY_FORMAT % study_id, dumps({'job_id': job.id}))
self.write({'status': status, 'message': message})
def workflow_handler_post_req(user_id, command_id, params):
"""Creates a new workflow in the system
Parameters
----------
user_id : str
The user creating the workflow
command_id : int
The first command to execute in the workflow
params : str
JSON representations of the parameters for the first command of
the workflow
Returns
-------
dict of objects
A dictionary containing the commands information
{'status': str,
'message': str,
'workflow_id': int}
"""
parameters = Parameters.load(Command(command_id), json_str=params)
status = 'success'
message = ''
try:
wf = ProcessingWorkflow.from_scratch(User(user_id), parameters)
except Exception as exc:
wf = None
wf_id = None
job_info = None
status = 'error'
message = str(exc)
if wf is not None:
# this is safe as we are creating the workflow for the first time
# and there is only one node. Remember networkx doesn't assure order
# of nodes
job = list(wf.graph.nodes())[0]
inputs = [a.id for a in job.input_artifacts]
job_cmd = job.command
wf_id = wf.id
job_info = {
'id': job.id,
'inputs': inputs,
'label': job_cmd.name,
'outputs': job_cmd.outputs
}
return {
'status': status,
'message': message,
'workflow_id': wf_id,
'job': job_info
}
def sample_template_handler_post_request(study_id, user, filepath,
data_type=None, direct_upload=False):
"""Creates a new sample template
Parameters
----------
study_id: int
The study to add the sample information
user: qiita_db.user import User
The user performing the request
filepath: str
The path to the sample template file
data_type: str, optional
If filepath is a QIIME mapping file, the data type of the prep
information file
direct_upload: boolean, optional
If filepath is a direct upload; if False we need to process the
filepath as part of the study upload folder
Returns
-------
dict of {'job': str}
job: the id of the job adding the sample information to the study
Raises
------
HTTPError
404 if the filepath doesn't exist
"""
# Check if the current user has access to the study
sample_template_checks(study_id, user)
# Check if the file exists
if not direct_upload:
fp_rsp = check_fp(study_id, filepath)
if fp_rsp['status'] != 'success':
raise HTTPError(404, reason='Filepath not found')
filepath = fp_rsp['file']
is_mapping_file = looks_like_qiime_mapping_file(filepath)
if is_mapping_file and not data_type:
raise HTTPError(400, reason='Please, choose a data type if uploading '
'a QIIME mapping file')
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('create_sample_template')
params = Parameters.load(
cmd, values_dict={'fp': filepath, 'study_id': study_id,
'is_mapping_file': is_mapping_file,
'data_type': data_type})
job = ProcessingJob.create(user, params, True)
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
def test_workflow_handler_patch_req(self):
# Create a new workflow so it is in construction
exp_command = Command(1)
json_str = (
'{"input_data": 1, "max_barcode_errors": 1.5, '
'"barcode_type": "golay_12", "max_bad_run_length": 3, '
'"rev_comp": false, "phred_quality_threshold": 3, '
'"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
'"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
exp_params = Parameters.load(exp_command, json_str=json_str)
exp_user = User('*****@*****.**')
name = "Test processing workflow"
# tests success
wf = ProcessingWorkflow.from_scratch(exp_user,
exp_params,
name=name,
force=True)
graph = wf.graph
nodes = list(graph.nodes())
job_id = nodes[0].id
value = {
'dflt_params': 10,
'connections': {
job_id: {
'demultiplexed': 'input_data'
}
}
}
obs = workflow_handler_patch_req('add',
'/%s/' % wf.id,
req_value=dumps(value))
new_jobs = set(wf.graph.nodes()) - set(nodes)
self.assertEqual(len(new_jobs), 1)
new_job = new_jobs.pop()
exp = {
'status': 'success',
'message': '',
'job': {
'id': new_job.id,
'inputs': [job_id],
'label': 'Pick closed-reference OTUs',
'outputs': [['OTU table', 'BIOM']]
}
}
self.assertEqual(obs, exp)
obs = workflow_handler_patch_req('remove',
'/%s/%s/' % (wf.id, new_job.id))
exp = {'status': 'success', 'message': ''}
jobs = set(wf.graph.nodes()) - set(nodes)
self.assertEqual(jobs, set())
def test_artifact_summary_post_request(self):
# No access
with self.assertRaises(QiitaHTTPError):
artifact_summary_post_request(User('*****@*****.**'), 1)
# Returns already existing job
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 2}))
job._set_status('queued')
obs = artifact_summary_post_request(User('*****@*****.**'), 2)
exp = {'job': [job.id, 'queued', None]}
self.assertEqual(obs, exp)
def test_artifact_summary_post_request(self):
# No access
with self.assertRaises(QiitaHTTPError):
artifact_summary_post_request(User('*****@*****.**'), 1)
# Returns already existing job
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 2})
)
job._set_status('queued')
obs = artifact_summary_post_request(User('*****@*****.**'), 2)
exp = {'job': [job.id, 'queued', None]}
self.assertEqual(obs, exp)
def post(self):
study_id = int(self.get_argument('study_id'))
preprocessed_data_id = int(self.get_argument('preprocessed_data_id'))
param_id = self.get_argument('parameter-set-%s' % preprocessed_data_id)
parameters = Parameters.from_default_params(
DefaultParameters(param_id), {'input_data': preprocessed_data_id})
job_id = plugin_submit(self.current_user, parameters)
self.render('compute_wait.html',
job_id=job_id, title='Processing',
completion_redirect='/study/description/%d?top_tab='
'preprocessed_data_tab&sub_tab=%s'
% (study_id, preprocessed_data_id))
def post(self):
study_id = int(self.get_argument('study_id'))
preprocessed_data_id = int(self.get_argument('preprocessed_data_id'))
param_id = self.get_argument('parameter-set-%s' % preprocessed_data_id)
parameters = Parameters.from_default_params(
DefaultParameters(param_id), {'input_data': preprocessed_data_id})
job_id = plugin_submit(self.current_user, parameters)
self.render('compute_wait.html',
job_id=job_id,
title='Processing',
completion_redirect='/study/description/%d?top_tab='
'preprocessed_data_tab&sub_tab=%s' %
(study_id, preprocessed_data_id))
def post(self):
analysis_id = int(self.get_argument('analysis_id'))
user = self.current_user
check_analysis_access(user, Analysis(analysis_id))
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_analysis')
params = Parameters.load(cmd, values_dict={'analysis_id': analysis_id})
job = ProcessingJob.create(user, params, True)
# Store the job id attaching it to the sample template id
r_client.set('analysis_delete_%d' % analysis_id,
dumps({'job_id': job.id}))
job.submit()
self.redirect("%s/analysis/list/" % (qiita_config.portal_dir))
def artifact_summary_post_request(user_id, artifact_id):
"""Launches the HTML summary generation and returns the job information
Parameters
----------
user_id : str
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the artifact summary information
{'status': str,
'message': str,
'job': list of [str, str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
access_error = check_access(artifact.study.id, user_id)
if access_error:
return access_error
# Check if the summary is being generated or has been already generated
command = Command.get_html_generator(artifact.artifact_type)
jobs = artifact.jobs(cmd=command)
jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
if jobs:
# The HTML summary is either being generated or already generated.
# Return the information of that job so we only generate the HTML
# once
job = jobs[0]
else:
# Create a new job to generate the HTML summary and return the newly
# created job information
job = ProcessingJob.create(
User(user_id),
Parameters.load(command, values_dict={'input_data': artifact_id}))
job.submit()
return {
'status': 'success',
'message': '',
'job': [job.id, job.status, job.step]
}
def test_patch(self):
# Create a new job - through a workflow since that is the only way
# of creating jobs in the interface
exp_command = Command(1)
json_str = (
'{"input_data": 1, "max_barcode_errors": 1.5, '
'"barcode_type": "golay_12", "max_bad_run_length": 3, '
'"rev_comp": false, "phred_quality_threshold": 3, '
'"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
'"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
exp_params = Parameters.load(exp_command, json_str=json_str)
exp_user = User('*****@*****.**')
name = "Test processing workflow"
# tests success
wf = ProcessingWorkflow.from_scratch(exp_user,
exp_params,
name=name,
force=True)
graph = wf.graph
nodes = graph.nodes()
job_id = nodes[0].id
response = self.patch('/study/process/job/', {
'op': 'remove',
'path': job_id
})
self.assertEqual(response.code, 200)
exp = {
'status':
'error',
'message':
"Can't delete job %s. It is 'in_construction' "
"status. Please use /study/process/workflow/" % job_id
}
self.assertEqual(loads(response.body), exp)
# Test success
ProcessingJob(job_id)._set_error('Killed for testing')
response = self.patch('/study/process/job/', {
'op': 'remove',
'path': job_id
})
self.assertEqual(response.code, 200)
exp = {'status': 'success', 'message': ''}
self.assertEqual(loads(response.body), exp)
def post(self):
study_id = int(self.get_argument('study_id'))
prep_template_id = int(self.get_argument('prep_template_id'))
raw_data = PrepTemplate(prep_template_id).artifact
param_id = int(self.get_argument('preprocessing_parameters_id'))
parameters = Parameters.from_default_params(
DefaultParameters(param_id), {'input_data': raw_data.id})
job_id = plugin_submit(self.current_user, parameters)
self.render('compute_wait.html',
job_id=job_id,
title='Preprocessing',
completion_redirect='/study/description/%d?top_tab='
'prep_template_tab&sub_tab=%s' %
(study_id, prep_template_id))
def post(self):
study_id = int(self.get_argument("study_id"))
prep_template_id = int(self.get_argument("prep_template_id"))
raw_data = PrepTemplate(prep_template_id).artifact
param_id = int(self.get_argument("preprocessing_parameters_id"))
parameters = Parameters.from_default_params(DefaultParameters(param_id), {"input_data": raw_data.id})
job_id = plugin_submit(self.current_user, parameters)
self.render(
"compute_wait.html",
job_id=job_id,
title="Preprocessing",
completion_redirect="/study/description/%d?top_tab="
"prep_template_tab&sub_tab=%s" % (study_id, prep_template_id),
)
def artifact_summary_post_request(user_id, artifact_id):
"""Launches the HTML summary generation and returns the job information
Parameters
----------
user_id : str
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the artifact summary information
{'status': str,
'message': str,
'job': list of [str, str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
access_error = check_access(artifact.study.id, user_id)
if access_error:
return access_error
# Check if the summary is being generated or has been already generated
command = Command.get_html_generator(artifact.artifact_type)
jobs = artifact.jobs(cmd=command)
jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
if jobs:
# The HTML summary is either being generated or already generated.
# Return the information of that job so we only generate the HTML
# once
job = jobs[0]
else:
# Create a new job to generate the HTML summary and return the newly
# created job information
job = ProcessingJob.create(
User(user_id),
Parameters.load(command, values_dict={'input_data': artifact_id}))
job.submit()
return {'status': 'success',
'message': '',
'job': [job.id, job.status, job.step]}
def test_artifact_summary_post_request(self):
# No access
obs = artifact_summary_post_request('*****@*****.**', 1)
exp = {'status': 'error',
'message': 'User does not have access to study'}
self.assertEqual(obs, exp)
# Returns already existing job
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 2})
)
job._set_status('queued')
obs = artifact_summary_post_request('*****@*****.**', 2)
exp = {'status': 'success',
'message': '',
'job': [job.id, 'queued', None]}
self.assertEqual(obs, exp)
def test_submit_to_EBI(self):
# setting up test
fna_fp = join(self.temp_dir, 'seqs.fna')
demux_fp = join(self.temp_dir, 'demux.seqs')
with open(fna_fp, 'w') as f:
f.write(FASTA_EXAMPLE)
with File(demux_fp, "w") as f:
to_hdf5(fna_fp, f)
pt = PrepTemplate(1)
params = Parameters.from_default_params(DefaultParameters(1),
{'input_data': pt.artifact.id})
artifact = Artifact.create([(demux_fp, 6)],
"Demultiplexed",
parents=[pt.artifact],
processing_parameters=params)
# submit job
job = self._create_job('submit_to_EBI', {
'artifact': artifact.id,
'submission_type': 'VALIDATE'
})
job._set_status('in_construction')
job.submit()
# wait for the job to fail, and check that the status is submitting
checked_submitting = True
while job.status != 'error':
if checked_submitting:
self.assertEqual('submitting',
artifact.study.ebi_submission_status)
checked_submitting = False
# once it fails wait for a few to check status again
sleep(5)
exp = 'Some artifact submissions failed: %d' % artifact.id
obs = artifact.study.ebi_submission_status
self.assertEqual(obs, exp)
# make sure that the error is correct, we have 2 options
if environ.get('ASPERA_SCP_PASS', '') != '':
self.assertIn('1.SKM2.640199', job.log.msg)
else:
self.assertIn('ASCP Error:', job.log.msg)
# wait for everything to finish to avoid DB deadlocks
sleep(5)
def artifact_summary_post_request(user, artifact_id):
"""Launches the HTML summary generation and returns the job information
Parameters
----------
user : qiita_db.user.User
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the job summary information
{'job': [str, str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
check_artifact_access(user, artifact)
# Check if the summary is being generated or has been already generated
command = Command.get_html_generator(artifact.artifact_type)
jobs = artifact.jobs(cmd=command)
jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
if jobs:
# The HTML summary is either being generated or already generated.
# Return the information of that job so we only generate the HTML
# once - Magic number 0 -> we are ensuring that there is only one
# job generating the summary, so we can use the index 0 to access to
# that job
job = jobs[0]
else:
# Create a new job to generate the HTML summary and return the newly
# created job information
job = ProcessingJob.create(
user,
Parameters.load(command, values_dict={'input_data': artifact_id}),
True)
job.submit()
return {'job': [job.id, job.status, job.step]}
def workflow_handler_post_req(user_id, dflt_params_id, req_params):
"""Creates a new workflow in the system
Parameters
----------
user_id : str
The user creating the workflow
dflt_params_id : int
The default parameters to use for the first command of the workflow
req_params : str
JSON representations of the required parameters for the first
command of the workflow
Returns
-------
dict of objects
A dictionary containing the commands information
{'status': str,
'message': str,
'workflow_id': int}
"""
dflt_params = DefaultParameters(dflt_params_id)
req_params = loads(req_params)
parameters = Parameters.from_default_params(dflt_params, req_params)
wf = ProcessingWorkflow.from_scratch(User(user_id), parameters)
# this is safe as we are creating the workflow for the first time and there
# is only one node. Remember networkx doesn't assure order of nodes
job = wf.graph.nodes()[0]
inputs = [a.id for a in job.input_artifacts]
job_cmd = job.command
return {
'status': 'success',
'message': '',
'workflow_id': wf.id,
'job': {
'id': job.id,
'inputs': inputs,
'label': job_cmd.name,
'outputs': job_cmd.outputs
}
}
def test_submit_to_EBI(self):
# setting up test
fna_fp = join(self.temp_dir, 'seqs.fna')
demux_fp = join(self.temp_dir, 'demux.seqs')
with open(fna_fp, 'w') as f:
f.write(FASTA_EXAMPLE)
with File(demux_fp, "w") as f:
to_hdf5(fna_fp, f)
pt = PrepTemplate(1)
params = Parameters.from_default_params(
DefaultParameters(1), {'input_data': pt.artifact.id})
artifact = Artifact.create(
[(demux_fp, 6)], "Demultiplexed", parents=[pt.artifact],
processing_parameters=params)
# submit job
job = self._create_job('submit_to_EBI', {
'artifact': artifact.id, 'submission_type': 'VALIDATE'})
job._set_status('in_construction')
job.submit()
# wait for the job to fail, and check that the status is submitting
checked_submitting = True
while job.status != 'error':
if checked_submitting:
self.assertEqual('submitting',
artifact.study.ebi_submission_status)
checked_submitting = False
# once it fails wait for a few to check status again
sleep(5)
exp = 'Some artifact submissions failed: %d' % artifact.id
obs = artifact.study.ebi_submission_status
self.assertEqual(obs, exp)
# make sure that the error is correct, we have 2 options
if environ.get('ASPERA_SCP_PASS', '') != '':
self.assertIn('1.SKM2.640199', job.log.msg)
else:
self.assertIn('ASCP Error:', job.log.msg)
# wait for everything to finish to avoid DB deadlocks
sleep(5)
def test_artifact_summary_post_request(self):
# No access
obs = artifact_summary_post_request('*****@*****.**', 1)
exp = {
'status': 'error',
'message': 'User does not have access to study'
}
self.assertEqual(obs, exp)
# Returns already existing job
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 2}))
job._set_status('queued')
obs = artifact_summary_post_request('*****@*****.**', 2)
exp = {
'status': 'success',
'message': '',
'job': [job.id, 'queued', None]
}
self.assertEqual(obs, exp)
def artifact_summary_post_request(user, artifact_id):
"""Launches the HTML summary generation and returns the job information
Parameters
----------
user : qiita_db.user.User
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the job summary information
{'job': [str, str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
check_artifact_access(user, artifact)
# Check if the summary is being generated or has been already generated
command = Command.get_html_generator(artifact.artifact_type)
jobs = artifact.jobs(cmd=command)
jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
if jobs:
# The HTML summary is either being generated or already generated.
# Return the information of that job so we only generate the HTML
# once - Magic number 0 -> we are ensuring that there is only one
# job generating the summary, so we can use the index 0 to access to
# that job
job = jobs[0]
else:
# Create a new job to generate the HTML summary and return the newly
# created job information
job = ProcessingJob.create(user, Parameters.load(
command, values_dict={'input_data': artifact_id}), True)
job.submit()
return {'job': [job.id, job.status, job.step]}
def test_workflow_handler_patch_req(self):
# Create a new workflow so it is in construction
exp_command = Command(1)
json_str = (
'{"input_data": 1, "max_barcode_errors": 1.5, '
'"barcode_type": "golay_12", "max_bad_run_length": 3, '
'"rev_comp": false, "phred_quality_threshold": 3, '
'"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
'"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
exp_params = Parameters.load(exp_command, json_str=json_str)
exp_user = User('*****@*****.**')
name = "Test processing workflow"
# tests success
wf = ProcessingWorkflow.from_scratch(
exp_user, exp_params, name=name, force=True)
graph = wf.graph
nodes = list(graph.nodes())
job_id = nodes[0].id
value = {'dflt_params': 10,
'connections': {job_id: {'demultiplexed': 'input_data'}}}
obs = workflow_handler_patch_req(
'add', '/%s/' % wf.id, req_value=dumps(value))
new_jobs = set(wf.graph.nodes()) - set(nodes)
self.assertEqual(len(new_jobs), 1)
new_job = new_jobs.pop()
exp = {'status': 'success',
'message': '',
'job': {'id': new_job.id,
'inputs': [job_id],
'label': 'Pick closed-reference OTUs',
'outputs': [['OTU table', 'BIOM']]}}
self.assertEqual(obs, exp)
obs = workflow_handler_patch_req(
'remove', '/%s/%s/' % (wf.id, new_job.id))
exp = {'status': 'success', 'message': ''}
jobs = set(wf.graph.nodes()) - set(nodes)
self.assertEqual(jobs, set())
def sample_template_delete_req(study_id, user_id):
"""Deletes the sample template attached to the study
Parameters
----------
study_id : int
The current study object id
user_id : str
The current user object id
Returns
-------
dict
results dictonary in the format
{'status': status,
'message': msg}
status can be success, warning, or error depending on result
message has the warnings or errors
"""
exists = _check_sample_template_exists(int(study_id))
if exists['status'] != 'success':
return exists
access_error = check_access(int(study_id), user_id)
if access_error:
return access_error
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_template')
params = Parameters.load(cmd, values_dict={'study': int(study_id)})
job = ProcessingJob.create(User(user_id), params)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'status': 'success', 'message': ''}
def test_patch(self):
# Create a new job - through a workflow since that is the only way
# of creating jobs in the interface
exp_command = Command(1)
json_str = (
'{"input_data": 1, "max_barcode_errors": 1.5, '
'"barcode_type": "golay_12", "max_bad_run_length": 3, '
'"rev_comp": false, "phred_quality_threshold": 3, '
'"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
'"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
exp_params = Parameters.load(exp_command, json_str=json_str)
exp_user = User('*****@*****.**')
name = "Test processing workflow"
# tests success
wf = ProcessingWorkflow.from_scratch(
exp_user, exp_params, name=name, force=True)
graph = wf.graph
nodes = graph.nodes()
job_id = nodes[0].id
response = self.patch('/study/process/job/',
{'op': 'remove', 'path': job_id})
self.assertEqual(response.code, 200)
exp = {'status': 'error',
'message': "Can't delete job %s. It is 'in_construction' "
"status. Please use /study/process/workflow/"
% job_id}
self.assertEqual(loads(response.body), exp)
# Test success
ProcessingJob(job_id)._set_error('Killed for testing')
response = self.patch('/study/process/job/',
{'op': 'remove', 'path': job_id})
self.assertEqual(response.code, 200)
exp = {'status': 'success',
'message': ''}
self.assertEqual(loads(response.body), exp)
def workflow_handler_post_req(user_id, dflt_params_id, req_params):
"""Creates a new workflow in the system
Parameters
----------
user_id : str
The user creating the workflow
dflt_params_id : int
The default parameters to use for the first command of the workflow
req_params : str
JSON representations of the required parameters for the first
command of the workflow
Returns
-------
dict of objects
A dictionary containing the commands information
{'status': str,
'message': str,
'workflow_id': int}
"""
dflt_params = DefaultParameters(dflt_params_id)
req_params = loads(req_params)
parameters = Parameters.from_default_params(dflt_params, req_params)
wf = ProcessingWorkflow.from_scratch(User(user_id), parameters)
# this is safe as we are creating the workflow for the first time and there
# is only one node. Remember networkx doesn't assure order of nodes
job = wf.graph.nodes()[0]
inputs = [a.id for a in job.input_artifacts]
job_cmd = job.command
return {'status': 'success',
'message': '',
'workflow_id': wf.id,
'job': {'id': job.id,
'inputs': inputs,
'label': job_cmd.name,
'outputs': job_cmd.outputs}}
def sample_template_handler_delete_request(study_id, user):
"""Deletes the sample template
Parameters
----------
study_id: int
The study to delete the sample information
user: qiita_db.user
The user performing the request
Returns
-------
dict of {'job': str}
job: the id of the job deleting the sample information to the study
Raises
------
HTTPError
404 If the sample template doesn't exist
"""
# Check if the current user has access to the study and if the sample
# template exists
sample_template_checks(study_id, user, check_exists=True)
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_template')
params = Parameters.load(cmd, values_dict={'study': int(study_id)})
job = ProcessingJob.create(user, params, True)
# Store the job if deleteing the sample template
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
def sample_template_handler_delete_request(study_id, user):
"""Deletes the sample template
Parameters
----------
study_id: int
The study to delete the sample information
user: qiita_db.user
The user performing the request
Returns
-------
dict of {'job': str}
job: the id of the job deleting the sample information to the study
Raises
------
HTTPError
404 If the sample template doesn't exist
"""
# Check if the current user has access to the study and if the sample
# template exists
sample_template_checks(study_id, user, check_exists=True)
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_template')
params = Parameters.load(cmd, values_dict={'study': int(study_id)})
job = ProcessingJob.create(user, params, True)
# Store the job if deleteing the sample template
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
def correct_redis_data(key, cmd, values_dict, user):
"""Corrects the data stored in the redis DB
Parameters
----------
key: str
The redis key to fix
cmd : qiita_db.software.Command
Command to use to create the processing job
values_dict : dict
Dictionary used to instantiate the parameters of the command
user : qiita_db.user. User
The user that will own the job
"""
info = r_client.get(key)
if info:
info = loads(info)
if info['job_id'] is not None:
if 'is_qiita_job' in info:
if info['is_qiita_job']:
try:
job = ProcessingJob(info['job_id'])
payload = {'job_id': info['job_id'],
'alert_type': info['status'],
'alert_msg': info['alert_msg']}
r_client.set(key, dumps(payload))
except (QiitaDBUnknownIDError, KeyError):
# We shomehow lost the information of this job
# Simply delete the key
r_client.delete(key)
else:
# These jobs don't contain any information on the live
# dump. We can safely delete the key
r_client.delete(key)
else:
# These jobs don't contain any information on the live
# dump. We can safely delete the key
r_client.delete(key)
else:
# Job is null, we have the information here
if info['status'] == 'success':
# In the success case no information is stored. We can
# safely delete the key
r_client.delete(key)
elif info['status'] == 'warning':
# In case of warning the key message stores the warning
# message. We need to create a new job, mark it as
# successful and store the error message as expected by
# the new structure
params = Parameters.load(cmd, values_dict=values_dict)
job = ProcessingJob.create(user, params)
job._set_status('success')
payload = {'job_id': job.id,
'alert_type': 'warning',
'alert_msg': info['message']}
r_client.set(key, dumps(payload))
else:
# The status is error. The key message stores the error
# message. We need to create a new job and mark it as
# failed with the given error message
params = Parameters.load(cmd, values_dict=values_dict)
job = ProcessingJob.create(user, params)
job._set_error(info['message'])
payload = {'job_id': job.id}
r_client.set(key, dumps(payload))
else:
# The key doesn't contain any information. Delete the key
r_client.delete(key)
def sample_template_handler_patch_request(user,
req_op,
req_path,
req_value=None,
req_from=None,
direct_upload=False):
"""Patches the sample template
Parameters
----------
user: qiita_db.user.User
The user performing the request
req_op : str
The operation to perform on the sample template
req_path : str
The path to the attribute to patch
req_value : str, optional
The new value
req_from : str, optional
The original path of the element
direct_upload : boolean, optional
If the file being uploaded comes from a direct upload (True)
Returns
-------
Raises
------
HTTPError
400 If the path parameter doens't follow the expected format
400 If the given operation is not supported
"""
req_path = [v for v in req_path.split('/') if v]
# At this point we know the path should be at least length 2
if len(req_path) < 2:
raise HTTPError(400, reason='Incorrect path parameter')
study_id = int(req_path[0])
# Check if the current user has access to the study and if the sample
# template exists
sample_template_checks(study_id, user, check_exists=True)
if req_op == 'remove':
# Path format
# column: study_id/columns/column_name
# sample: study_id/samples/sample_id
if len(req_path) != 3:
raise HTTPError(400, reason='Incorrect path parameter')
attribute = req_path[1]
attr_id = req_path[2]
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_or_column')
params = Parameters.load(cmd,
values_dict={
'obj_class': 'SampleTemplate',
'obj_id': study_id,
'sample_or_col': attribute,
'name': attr_id
})
job = ProcessingJob.create(user, params, True)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
elif req_op == 'replace':
# WARNING: Although the patch operation is a replace, is not a full
# true replace. A replace is in theory equivalent to a remove + add.
# In this case, the replace operation doesn't necessarily removes
# anything (e.g. when only new columns/samples are being added to the)
# sample information.
# Path format: study_id/data
# Forcing to specify data for extensibility. In the future we may want
# to use this function to replace other elements of the sample
# information
if len(req_path) != 2:
raise HTTPError(400, reason='Incorrect path parameter')
attribute = req_path[1]
if attribute == 'data':
# Update the sample information
if req_value is None:
raise HTTPError(400,
reason="Value is required when updating "
"sample information")
if direct_upload:
# We can assume that the file exist as it was generated by
# the system
filepath = req_value
if not exists(filepath):
reason = ('Upload file not found (%s), please report to '
'*****@*****.**' % filepath)
raise HTTPError(404, reason=reason)
else:
# Check if the file exists
fp_rsp = check_fp(study_id, req_value)
if fp_rsp['status'] != 'success':
raise HTTPError(404, reason='Filepath not found')
filepath = fp_rsp['file']
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('update_sample_template')
params = Parameters.load(cmd,
values_dict={
'study': study_id,
'template_fp': filepath
})
job = ProcessingJob.create(user, params, True)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
else:
raise HTTPError(404, reason='Attribute %s not found' % attribute)
else:
raise HTTPError(400,
reason='Operation %s not supported. Current '
'supported operations: remove, replace' % req_op)
def test_artifact_summary_get_request(self):
user = User('*****@*****.**')
# Artifact w/o summary
obs = artifact_summary_get_request(user, 1)
exp_p_jobs = [[
'063e553b-327c-4818-ab4a-adfe58e49860', 'Split libraries FASTQ',
'queued', None, None
],
[
'bcc7ebcd-39c1-43e4-af2d-822e3589f14d',
'Split libraries', 'running', 'demultiplexing', None
]]
exp_files = [
(1L, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)'),
(2L, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)')
]
exp = {
'name':
'Raw data 1',
'artifact_id':
1,
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'processing_parameters': {},
'files':
exp_files,
'summary':
None,
'job':
None,
'processing_jobs':
exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# Artifact with summary being generated
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 1}))
job._set_status('queued')
obs = artifact_summary_get_request(user, 1)
exp = {
'name':
'Raw data 1',
'artifact_id':
1,
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'processing_parameters': {},
'files':
exp_files,
'summary':
None,
'job': [job.id, 'queued', None],
'processing_jobs':
exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# Artifact with summary
fd, fp = mkstemp(suffix=".html")
close(fd)
with open(fp, 'w') as f:
f.write('<b>HTML TEST - not important</b>\n')
a = Artifact(1)
a.set_html_summary(fp)
self._files_to_remove.extend([fp, a.html_summary_fp[1]])
exp_files.append(
(a.html_summary_fp[0],
'%s (html summary)' % basename(a.html_summary_fp[1])))
exp_summary_path = relpath(a.html_summary_fp[1],
qiita_config.base_data_dir)
obs = artifact_summary_get_request(user, 1)
exp = {
'name':
'Raw data 1',
'artifact_id':
1,
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'processing_parameters': {},
'files':
exp_files,
'summary':
exp_summary_path,
'job':
None,
'processing_jobs':
exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# No access
demo_u = User('*****@*****.**')
with self.assertRaises(QiitaHTTPError):
obs = artifact_summary_get_request(demo_u, 1)
# A non-owner/share user can't see the files
a.visibility = 'public'
obs = artifact_summary_get_request(demo_u, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'visibility': 'public',
'editable': False,
'buttons': '',
'processing_parameters': {},
'files': [],
'summary': exp_summary_path,
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# returnig to private
a.visibility = 'private'
# admin gets buttons
obs = artifact_summary_get_request(User('*****@*****.**'), 2)
exp_p_jobs = [[
'd19f76ee-274e-4c1b-b3a2-a12d73507c55',
'Pick closed-reference OTUs', 'error', 'generating demux file',
'Error message'
]]
exp_files = [(3L, '1_seqs.fna (preprocessed fasta)'),
(4L, '1_seqs.qual (preprocessed fastq)'),
(5L, '1_seqs.demux (preprocessed demux)')]
exp = {
'name':
'Demultiplexed 1',
'artifact_id':
2,
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 2?\')) { '
'set_artifact_visibility(\'public\', 2) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'2?\')) { set_artifact_visibility(\'sandbox\', 2) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button> <a class="btn btn-primary '
'btn-sm" href="/ebi_submission/2"><span '
'class="glyphicon glyphicon-export"></span> '
'Submit to EBI</a> <a class="btn btn-primary '
'btn-sm" href="/vamps/2"><span class="glyphicon '
'glyphicon-export"></span> Submit to VAMPS</a>'),
'processing_parameters': {
'max_barcode_errors': 1.5,
'sequence_max_n': 0,
'max_bad_run_length': 3,
'phred_offset': u'auto',
'rev_comp': False,
'phred_quality_threshold': 3,
'input_data': 1,
'rev_comp_barcode': False,
'rev_comp_mapping_barcodes': False,
'min_per_read_length_fraction': 0.75,
'barcode_type': u'golay_12'
},
'files':
exp_files,
'summary':
None,
'job':
None,
'processing_jobs':
exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# analysis artifact
obs = artifact_summary_get_request(user, 8)
exp = {
'name': 'noname',
'artifact_id': 8,
'visibility': 'sandbox',
'editable': True,
'buttons': '',
'processing_parameters': {},
'files': [(27, 'biom_table.biom (biom)')],
'summary': None,
'job': None,
'processing_jobs': [],
'errored_jobs': []
}
self.assertEqual(obs, exp)
def test_artifact_summary_get_request(self):
user = User('*****@*****.**')
# Artifact w/o summary
obs = artifact_summary_get_request(user, 1)
exp_files = [
(1L, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)'),
(2L, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)')]
exp = {'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'private',
'editable': True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': None,
'job': None,
'errored_summary_jobs': []}
self.assertEqual(obs, exp)
# Artifact with summary being generated
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 1})
)
job._set_status('queued')
obs = artifact_summary_get_request(user, 1)
exp = {'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'private',
'editable': True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': None,
'job': [job.id, 'queued', None],
'errored_summary_jobs': []}
self.assertEqual(obs, exp)
# Artifact with summary
fd, fp = mkstemp(suffix=".html")
close(fd)
with open(fp, 'w') as f:
f.write('<b>HTML TEST - not important</b>\n')
a = Artifact(1)
a.set_html_summary(fp)
self._files_to_remove.extend([fp, a.html_summary_fp[1]])
exp_files.append(
(a.html_summary_fp[0],
'%s (html summary)' % basename(a.html_summary_fp[1])))
exp_summary_path = relpath(
a.html_summary_fp[1], qiita_config.base_data_dir)
obs = artifact_summary_get_request(user, 1)
exp = {'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'private',
'editable': True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': exp_summary_path,
'job': None,
'errored_summary_jobs': []}
self.assertEqual(obs, exp)
# No access
demo_u = User('*****@*****.**')
with self.assertRaises(QiitaHTTPError):
obs = artifact_summary_get_request(demo_u, 1)
# A non-owner/share user can't see the files
a.visibility = 'public'
obs = artifact_summary_get_request(demo_u, 1)
exp = {'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'public',
'editable': False,
'buttons': '',
'processing_info': {},
'files': [],
'is_from_analysis': False,
'summary': exp_summary_path,
'job': None,
'errored_summary_jobs': []}
self.assertEqual(obs, exp)
# returnig to private
a.visibility = 'private'
# admin gets buttons
obs = artifact_summary_get_request(User('*****@*****.**'), 2)
exp_files = [
(3L, '1_seqs.fna (preprocessed fasta)'),
(4L, '1_seqs.qual (preprocessed fastq)'),
(5L, '1_seqs.demux (preprocessed demux)')]
exp = {'name': 'Demultiplexed 1',
'artifact_id': 2,
'artifact_type': 'Demultiplexed',
'artifact_timestamp': '2012-10-01 10:10',
'visibility': 'private',
'editable': True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 2?\')) { '
'set_artifact_visibility(\'public\', 2) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'2?\')) { set_artifact_visibility(\'sandbox\', 2) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button> <a class="btn btn-primary '
'btn-sm" href="/ebi_submission/2"><span '
'class="glyphicon glyphicon-export"></span> '
'Submit to EBI</a> <a class="btn btn-primary '
'btn-sm" href="/vamps/2"><span class="glyphicon '
'glyphicon-export"></span> Submit to VAMPS</a>'),
'processing_info': {
'command': 'Split libraries FASTQ',
'software': 'QIIME',
'software_version': '1.9.1',
'processing_parameters': {
'max_barcode_errors': '1.5', 'sequence_max_n': '0',
'max_bad_run_length': '3', 'phred_offset': u'auto',
'rev_comp': 'False', 'phred_quality_threshold': '3',
'input_data': '1', 'rev_comp_barcode': 'False',
'rev_comp_mapping_barcodes': 'False',
'min_per_read_length_fraction': '0.75',
'barcode_type': u'golay_12'}},
'files': exp_files,
'is_from_analysis': False,
'summary': None,
'job': None,
'errored_summary_jobs': []}
self.assertEqual(obs, exp)
# analysis artifact
obs = artifact_summary_get_request(user, 8)
exp = {'name': 'noname',
'artifact_id': 8,
'artifact_type': 'BIOM',
# this value changes on build so copy from obs
'artifact_timestamp': obs['artifact_timestamp'],
'visibility': 'sandbox',
'editable': True,
'buttons': '',
'processing_info': {},
'files': [(27, 'biom_table.biom (biom)')],
'is_from_analysis': True,
'summary': None,
'job': None,
'errored_summary_jobs': []}
self.assertEqual(obs, exp)
def test_download_study(self):
tmp_dir = mkdtemp()
self._clean_up_files.append(tmp_dir)
biom_fp = join(tmp_dir, 'otu_table.biom')
smr_dir = join(tmp_dir, 'sortmerna_picked_otus')
log_dir = join(smr_dir, 'seqs_otus.log')
tgz = join(tmp_dir, 'sortmerna_picked_otus.tgz')
with biom_open(biom_fp, 'w') as f:
et.to_hdf5(f, "test")
makedirs(smr_dir)
with open(log_dir, 'w') as f:
f.write('\n')
with open(tgz, 'w') as f:
f.write('\n')
files_biom = [(biom_fp, 'biom'), (smr_dir, 'directory'), (tgz, 'tgz')]
params = Parameters.from_default_params(
Command(3).default_parameter_sets.next(), {'input_data': 1})
a = Artifact.create(files_biom, "BIOM", parents=[Artifact(2)],
processing_parameters=params)
for _, fp, _ in a.filepaths:
self._clean_up_files.append(fp)
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 200)
exp = (
'- 1256812 /protected/processed_data/1_study_1001_closed_'
'reference_otu_table.biom processed_data/1_study_1001_closed_'
'reference_otu_table.biom\n'
'- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-'
'[0-9]*.txt mapping_files/4_mapping_file.txt\n'
'- 1256812 /protected/processed_data/'
'1_study_1001_closed_reference_otu_table.biom processed_data/'
'1_study_1001_closed_reference_otu_table.biom\n'
'- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-'
'[0-9]*.txt mapping_files/5_mapping_file.txt\n'
'- 1256812 /protected/processed_data/'
'1_study_1001_closed_reference_otu_table_Silva.biom processed_data'
'/1_study_1001_closed_reference_otu_table_Silva.biom\n'
'- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-'
'[0-9]*.txt mapping_files/6_mapping_file.txt\n'
'- 36615 /protected/templates/1_prep_2_qiime_[0-9]*-'
'[0-9]*.txt mapping_files/7_mapping_file.txt\n'
'- [0-9]* /protected/BIOM/{0}/otu_table.biom '
'BIOM/{0}/otu_table.biom\n'
'- 1 /protected/BIOM/{0}/sortmerna_picked_otus/seqs_otus.log '
'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n'
'- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]*.'
'txt mapping_files/{0}_mapping_file.txt\n'.format(a.id))
self.assertRegexpMatches(response.body, exp)
response = self.get('/download_study_bioms/200')
self.assertEqual(response.code, 405)
# changing user so we can test the failures
BaseHandler.get_current_user = Mock(
return_value=User("*****@*****.**"))
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 405)
a.visibility = 'public'
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 200)
exp = (
'- [0-9]* /protected/BIOM/{0}/otu_table.biom '
'BIOM/{0}/otu_table.biom\n'
'- 1 /protected/BIOM/{0}/sortmerna_picked_otus/seqs_otus.log '
'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n'
'- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]*.'
'txt mapping_files/{0}_mapping_file.txt\n'.format(a.id))
self.assertRegexpMatches(response.body, exp)
def test_artifact_summary_get_request(self):
# Artifact w/o summary
obs = artifact_summary_get_request('*****@*****.**', 1)
exp_p_jobs = [
['063e553b-327c-4818-ab4a-adfe58e49860', 'Split libraries FASTQ',
'queued', None, None],
['bcc7ebcd-39c1-43e4-af2d-822e3589f14d', 'Split libraries',
'running', 'demultiplexing', None]]
exp_files = [
(1L, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)'),
(2L, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)')]
exp = {'status': 'success',
'message': '',
'name': 'Raw data 1',
'summary': None,
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'private',
'buttons': '<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public</button>'
' <button onclick="if (confirm(\'Are you sure you '
'want to revert to sandbox artifact id: 1?\')) '
'{ set_artifact_visibility(\'sandbox\', 1) }" '
'class="btn btn-primary btn-sm">Revert to '
'sandbox</button>',
'files': exp_files,
'editable': True}
self.assertEqual(obs, exp)
# Artifact with summary being generated
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 1})
)
job._set_status('queued')
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {'status': 'success',
'message': '',
'name': 'Raw data 1',
'summary': None,
'job': [job.id, 'queued', None],
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'private',
'buttons': '<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public</button>'
' <button onclick="if (confirm(\'Are you sure you '
'want to revert to sandbox artifact id: 1?\')) { '
'set_artifact_visibility(\'sandbox\', 1) }" '
'class="btn btn-primary btn-sm">Revert to '
'sandbox</button>',
'files': exp_files,
'editable': True}
self.assertEqual(obs, exp)
# Artifact with summary
fd, fp = mkstemp(suffix=".html")
close(fd)
with open(fp, 'w') as f:
f.write('<b>HTML TEST - not important</b>\n')
a = Artifact(1)
a.html_summary_fp = fp
self._files_to_remove.extend([fp, a.html_summary_fp[1]])
exp_files.append(
(a.html_summary_fp[0],
'%s (html summary)' % basename(a.html_summary_fp[1])))
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {'status': 'success',
'message': '',
'name': 'Raw data 1',
'summary': '<b>HTML TEST - not important</b>\n',
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'private',
'buttons': '<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public</button>'
' <button onclick="if (confirm(\'Are you sure you '
'want to revert to sandbox artifact id: 1?\')) { '
'set_artifact_visibility(\'sandbox\', 1) }" '
'class="btn btn-primary btn-sm">Revert to '
'sandbox</button>',
'files': exp_files,
'editable': True}
self.assertEqual(obs, exp)
# No access
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {'status': 'error',
'message': 'User does not have access to study'}
self.assertEqual(obs, exp)
# A non-owner/share user can't see the files
a.visibility = 'public'
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {'status': 'success',
'message': '',
'name': 'Raw data 1',
'summary': '<b>HTML TEST - not important</b>\n',
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'public',
'buttons': '',
'files': [],
'editable': False}
self.assertEqual(obs, exp)
def test_complete_job(self):
# Complete success
pt = npt.assert_warns(
QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {'1.SKD6.640190': 1}}),
Study(1), '16S')
c_job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(
Command.get_validator('BIOM'),
values_dict={'template': pt.id,
'files': dumps({'BIOM': ['file']}),
'artifact_type': 'BIOM'}), True)
c_job._set_status('running')
fd, fp = mkstemp(suffix='_table.biom')
close(fd)
with open(fp, 'w') as f:
f.write('\n')
self._clean_up_files.append(fp)
exp_artifact_count = get_count('qiita.artifact') + 1
payload = dumps(
{'success': True, 'error': '',
'artifacts': {'OTU table': {'filepaths': [(fp, 'biom')],
'artifact_type': 'BIOM'}}})
job = self._create_job('complete_job', {'job_id': c_job.id,
'payload': payload})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertEqual(c_job.status, 'success')
self.assertEqual(get_count('qiita.artifact'), exp_artifact_count)
# Complete job error
payload = dumps({'success': False, 'error': 'Job failure'})
job = self._create_job(
'complete_job', {'job_id': 'bcc7ebcd-39c1-43e4-af2d-822e3589f14d',
'payload': payload})
private_task(job.id)
self.assertEqual(job.status, 'success')
c_job = ProcessingJob('bcc7ebcd-39c1-43e4-af2d-822e3589f14d')
self.assertEqual(c_job.status, 'error')
self.assertEqual(c_job.log, LogEntry.newest_records(numrecords=1)[0])
self.assertEqual(c_job.log.msg, 'Job failure')
# Complete internal error
pt = npt.assert_warns(
QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {'1.SKD6.640190': 1}}),
Study(1), '16S')
c_job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(
Command.get_validator('BIOM'),
values_dict={'template': pt.id,
'files': dumps({'BIOM': ['file']}),
'artifact_type': 'BIOM'}), True)
c_job._set_status('running')
fp = '/surprised/if/this/path/exists.biom'
payload = dumps(
{'success': True, 'error': '',
'artifacts': {'OTU table': {'filepaths': [(fp, 'biom')],
'artifact_type': 'BIOM'}}})
job = self._create_job('complete_job', {'job_id': c_job.id,
'payload': payload})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertEqual(c_job.status, 'error')
self.assertIn('No such file or directory', c_job.log.msg)
def sample_template_handler_patch_request(user, req_op, req_path,
req_value=None, req_from=None):
"""Patches the sample template
Parameters
----------
user: qiita_db.user.User
The user performing the request
req_op : str
The operation to perform on the sample template
req_path : str
The path to the attribute to patch
req_value : str, optional
The new value
req_from : str, optional
The original path of the element
Returns
-------
Raises
------
HTTPError
400 If the path parameter doens't follow the expected format
400 If the given operation is not supported
"""
req_path = [v for v in req_path.split('/') if v]
# At this point we know the path should be at least length 2
if len(req_path) < 2:
raise HTTPError(400, reason='Incorrect path parameter')
study_id = int(req_path[0])
# Check if the current user has access to the study and if the sample
# template exists
sample_template_checks(study_id, user, check_exists=True)
if req_op == 'remove':
# Path format
# column: study_id/columns/column_name
# sample: study_id/samples/sample_id
if len(req_path) != 3:
raise HTTPError(400, reason='Incorrect path parameter')
attribute = req_path[1]
attr_id = req_path[2]
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_or_column')
params = Parameters.load(
cmd, values_dict={'obj_class': 'SampleTemplate',
'obj_id': study_id,
'sample_or_col': attribute,
'name': attr_id})
job = ProcessingJob.create(user, params, True)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
elif req_op == 'replace':
# WARNING: Although the patch operation is a replace, is not a full
# true replace. A replace is in theory equivalent to a remove + add.
# In this case, the replace operation doesn't necessarily removes
# anything (e.g. when only new columns/samples are being added to the)
# sample information.
# Path format: study_id/data
# Forcing to specify data for extensibility. In the future we may want
# to use this function to replace other elements of the sample
# information
if len(req_path) != 2:
raise HTTPError(400, reason='Incorrect path parameter')
attribute = req_path[1]
if attribute == 'data':
# Update the sample information
if req_value is None:
raise HTTPError(400, reason="Value is required when updating "
"sample information")
# Check if the file exists
fp_rsp = check_fp(study_id, req_value)
if fp_rsp['status'] != 'success':
raise HTTPError(404, reason='Filepath not found')
filepath = fp_rsp['file']
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('update_sample_template')
params = Parameters.load(
cmd, values_dict={'study': study_id,
'template_fp': filepath})
job = ProcessingJob.create(user, params, True)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
dumps({'job_id': job.id}))
job.submit()
return {'job': job.id}
else:
raise HTTPError(404, reason='Attribute %s not found' % attribute)
else:
raise HTTPError(400, reason='Operation %s not supported. Current '
'supported operations: remove, replace' % req_op)
def test_job_ajax_patch_req(self):
# Create a new job - through a workflow since that is the only way
# of creating jobs in the interface
exp_command = Command(1)
json_str = (
'{"input_data": 1, "max_barcode_errors": 1.5, '
'"barcode_type": "golay_12", "max_bad_run_length": 3, '
'"rev_comp": false, "phred_quality_threshold": 3, '
'"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
'"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
exp_params = Parameters.load(exp_command, json_str=json_str)
exp_user = User('*****@*****.**')
name = "Test processing workflow"
# tests success
wf = ProcessingWorkflow.from_scratch(
exp_user, exp_params, name=name, force=True)
graph = wf.graph
nodes = list(graph.nodes())
job_id = nodes[0].id
# Incorrect path parameter
obs = job_ajax_patch_req('remove', '/%s/somethingelse' % job_id)
exp = {'status': 'error',
'message': 'Incorrect path parameter: missing job id'}
self.assertEqual(obs, exp)
obs = job_ajax_patch_req('remove', '/')
exp = {'status': 'error',
'message': 'Incorrect path parameter: missing job id'}
self.assertEqual(obs, exp)
# Job id is not like a job id
obs = job_ajax_patch_req('remove', '/notAJobId')
exp = {'status': 'error',
'message': 'Incorrect path parameter: '
'notAJobId is not a recognized job id'}
self.assertEqual(obs, exp)
# Job doesn't exist
obs = job_ajax_patch_req('remove',
'/6d368e16-2242-4cf8-87b4-a5dc40bc890b')
exp = {'status': 'error',
'message': 'Incorrect path parameter: '
'6d368e16-2242-4cf8-87b4-a5dc40bc890b is not a '
'recognized job id'}
self.assertEqual(obs, exp)
# in_construction job
obs = job_ajax_patch_req('remove', '/%s' % job_id)
exp = {'status': 'error',
'message': "Can't delete job %s. It is 'in_construction' "
"status. Please use /study/process/workflow/"
% job_id}
self.assertEqual(obs, exp)
# job status != 'error'
job = ProcessingJob(job_id)
job._set_status('queued')
obs = job_ajax_patch_req('remove', '/%s' % job_id)
exp = {'status': 'error',
'message': 'Only jobs in "error" status can be deleted.'}
self.assertEqual(obs, exp)
# Operation not supported
job._set_status('queued')
obs = job_ajax_patch_req('add', '/%s' % job_id)
exp = {'status': 'error',
'message': 'Operation "add" not supported. Current supported '
'operations: remove'}
self.assertEqual(obs, exp)
# Test success
job._set_error('Killed for testing')
obs = job_ajax_patch_req('remove', '/%s' % job_id)
exp = {'status': 'success',
'message': ''}
self.assertEqual(obs, exp)
def prep_template_patch_req(user_id, req_op, req_path, req_value=None,
req_from=None):
"""Modifies an attribute of the prep template
Parameters
----------
user_id : str
The id of the user performing the patch operation
req_op : str
The operation to perform on the prep information
req_path : str
The prep information and attribute to patch
req_value : str, optional
The value that needs to be modified
req_from : str, optional
The original path of the element
Returns
-------
dict of {str, str, str}
A dictionary with the following keys:
- status: str, whether if the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
- row_id: str, the row_id that we tried to delete
"""
req_path = [v for v in req_path.split('/') if v]
if req_op == 'replace':
# The structure of the path should be /prep_id/attribute_to_modify/
# so if we don't have those 2 elements, we should return an error
if len(req_path) != 2:
return {'status': 'error',
'message': 'Incorrect path parameter'}
prep_id = int(req_path[0])
attribute = req_path[1]
# Check if the user actually has access to the prep template
prep = PrepTemplate(prep_id)
access_error = check_access(prep.study_id, user_id)
if access_error:
return access_error
status = 'success'
msg = ''
if attribute == 'investigation_type':
prep.investigation_type = req_value
elif attribute == 'data':
fp = check_fp(prep.study_id, req_value)
if fp['status'] != 'success':
return fp
fp = fp['file']
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('update_prep_template')
params = Parameters.load(
cmd, values_dict={'prep_template': prep_id, 'template_fp': fp})
job = ProcessingJob.create(User(user_id), params, True)
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id,
dumps({'job_id': job.id}))
job.submit()
elif attribute == 'name':
prep.name = req_value.strip()
else:
# We don't understand the attribute so return an error
return {'status': 'error',
'message': 'Attribute "%s" not found. '
'Please, check the path parameter' % attribute}
return {'status': status, 'message': msg}
elif req_op == 'remove':
# The structure of the path should be:
# /prep_id/row_id/{columns|samples}/name
if len(req_path) != 4:
return {'status': 'error',
'message': 'Incorrect path parameter'}
prep_id = int(req_path[0])
row_id = req_path[1]
attribute = req_path[2]
attr_id = req_path[3]
# Check if the user actually has access to the study
pt = PrepTemplate(prep_id)
access_error = check_access(pt.study_id, user_id)
if access_error:
return access_error
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_or_column')
params = Parameters.load(
cmd, values_dict={'obj_class': 'PrepTemplate',
'obj_id': prep_id,
'sample_or_col': attribute,
'name': attr_id})
job = ProcessingJob.create(User(user_id), params, True)
# Store the job id attaching it to the sample template id
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id,
dumps({'job_id': job.id}))
job.submit()
return {'status': 'success', 'message': '', 'row_id': row_id}
else:
return {'status': 'error',
'message': 'Operation "%s" not supported. '
'Current supported operations: replace, remove'
% req_op,
'row_id': '0'}
def test_artifact_summary_get_request(self):
# Artifact w/o summary
obs = artifact_summary_get_request('*****@*****.**', 1)
exp_p_jobs = [
['063e553b-327c-4818-ab4a-adfe58e49860', 'Split libraries FASTQ',
'queued', None, None],
['bcc7ebcd-39c1-43e4-af2d-822e3589f14d', 'Split libraries',
'running', 'demultiplexing', None]]
exp_files = [
(1L, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)'),
(2L, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)')]
exp = {'status': 'success',
'message': '',
'name': 'Raw data 1',
'processing_parameters': {},
'summary': None,
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'private',
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'files': exp_files,
'editable': True,
'prep_id': 1,
'study_id': 1}
self.assertEqual(obs, exp)
# Artifact with summary being generated
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 1})
)
job._set_status('queued')
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {'status': 'success',
'message': '',
'name': 'Raw data 1',
'processing_parameters': {},
'summary': None,
'job': [job.id, 'queued', None],
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'private',
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'files': exp_files,
'editable': True,
'prep_id': 1,
'study_id': 1}
self.assertEqual(obs, exp)
# Artifact with summary
fd, fp = mkstemp(suffix=".html")
close(fd)
with open(fp, 'w') as f:
f.write('<b>HTML TEST - not important</b>\n')
a = Artifact(1)
a.html_summary_fp = fp
self._files_to_remove.extend([fp, a.html_summary_fp[1]])
exp_files.append(
(a.html_summary_fp[0],
'%s (html summary)' % basename(a.html_summary_fp[1])))
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {'status': 'success',
'message': '',
'name': 'Raw data 1',
'processing_parameters': {},
'summary': '<b>HTML TEST - not important</b>\n',
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'private',
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'files': exp_files,
'editable': True,
'prep_id': 1,
'study_id': 1}
self.assertEqual(obs, exp)
# No access
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {'status': 'error',
'message': 'User does not have access to study'}
self.assertEqual(obs, exp)
# A non-owner/share user can't see the files
a.visibility = 'public'
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {'status': 'success',
'message': '',
'name': 'Raw data 1',
'processing_parameters': {},
'summary': '<b>HTML TEST - not important</b>\n',
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'public',
'buttons': '',
'files': [],
'editable': False,
'prep_id': 1,
'study_id': 1}
self.assertEqual(obs, exp)
# returnig to private
a.visibility = 'sandbox'
# admin gets buttons
obs = artifact_summary_get_request('*****@*****.**', 2)
exp_p_jobs = [
['d19f76ee-274e-4c1b-b3a2-a12d73507c55',
'Pick closed-reference OTUs', 'error', 'generating demux file',
'Error message']]
exp_files = [
(3L, '1_seqs.fna (preprocessed fasta)'),
(4L, '1_seqs.qual (preprocessed fastq)'),
(5L, '1_seqs.demux (preprocessed demux)')]
exp = {'status': 'success',
'files': exp_files,
'errored_jobs': [],
'editable': True,
'visibility': 'private',
'job': None,
'message': '',
'name': 'Demultiplexed 1',
'processing_jobs': exp_p_jobs,
'processing_parameters': {
'max_barcode_errors': 1.5, 'sequence_max_n': 0,
'max_bad_run_length': 3, 'phred_offset': u'auto',
'rev_comp': False, 'phred_quality_threshold': 3,
'input_data': 1, 'rev_comp_barcode': False,
'rev_comp_mapping_barcodes': False,
'min_per_read_length_fraction': 0.75,
'barcode_type': u'golay_12'},
'summary': None,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 2?\')) { '
'set_artifact_visibility(\'public\', 2) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'2?\')) { set_artifact_visibility(\'sandbox\', 2) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button> <a class="btn btn-primary '
'btn-sm" href="/vamps/2"><span class="glyphicon '
'glyphicon-export"></span> Submit to VAMPS</a>'),
'study_id': 1,
'prep_id': 1}
self.assertEqual(obs, exp)
def sample_template_patch_request(user_id,
req_op,
req_path,
req_value=None,
req_from=None):
"""Modifies an attribute of the artifact
Parameters
----------
user_id : str
The id of the user performing the patch operation
req_op : str
The operation to perform on the artifact
req_path : str
The prep information and attribute to patch
req_value : str, optional
The value that needs to be modified
req_from : str, optional
The original path of the element
Returns
-------
dict of {str, str}
A dictionary with the following keys:
- status: str, whether if the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
"""
if req_op == 'remove':
req_path = [v for v in req_path.split('/') if v]
# format
# column: study_id/row_id/columns/column_name
# sample: study_id/row_id/samples/sample_id
if len(req_path) != 4:
return {'status': 'error', 'message': 'Incorrect path parameter'}
st_id = req_path[0]
row_id = req_path[1]
attribute = req_path[2]
attr_id = req_path[3]
# Check if the user actually has access to the template
st = SampleTemplate(st_id)
access_error = check_access(st.study_id, user_id)
if access_error:
return access_error
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_sample_or_column')
params = Parameters.load(cmd,
values_dict={
'obj_class': 'SampleTemplate',
'obj_id': int(st_id),
'sample_or_col': attribute,
'name': attr_id
})
job = ProcessingJob.create(User(user_id), params)
# Store the job id attaching it to the sample template id
r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % st_id,
dumps({'job_id': job.id}))
job.submit()
return {'status': 'success', 'message': '', 'row_id': row_id}
else:
return {
'status':
'error',
'message':
'Operation "%s" not supported. '
'Current supported operations: remove' % req_op,
'row_id':
0
}
def test_artifact_summary_get_request(self):
user = User('*****@*****.**')
main_buttons = (
'<button onclick="if (confirm('
"\'Are you sure you want to make "
"public artifact id: 1?')) { set_artifact_visibility('public', 1) "
'}" class="btn btn-primary btn-sm">Make public</button> <button '
'onclick="if (confirm('
"'Are you sure you want to revert to "
"sandbox artifact id: 1?')) { set_artifact_visibility('sandbox', 1"
') }" class="btn btn-primary btn-sm">Revert to sandbox</button> ')
private_download_button = (
'<button class="btn btn-primary btn-sm" type="button" '
'aria-expanded="false" aria-controls="privateDownloadLink" '
'onclick="generate_private_download_link(%s)">Generate Download '
'Link</button><div class="collapse" id="privateDownloadLink"><div '
'class="card card-body" id="privateDownloadText">Generating '
'Download Link...</div></div>')
# Artifact w/o summary
obs = artifact_summary_get_request(user, 1)
exp_files = [
(1, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)',
'2125826711', '58 Bytes'),
(2, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)',
'2125826711', '58 Bytes')
]
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'private',
'editable': True,
'buttons': main_buttons + private_download_button % 1,
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': None,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# Artifact with summary being generated
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 1}))
job._set_status('queued')
obs = artifact_summary_get_request(user, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'private',
'editable': True,
'buttons': main_buttons + private_download_button % 1,
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': None,
'job': [job.id, 'queued', None],
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# Artifact with summary
fd, fp = mkstemp(suffix=".html")
close(fd)
with open(fp, 'w') as f:
f.write('<b>HTML TEST - not important</b>\n')
a = Artifact(1)
a.set_html_summary(fp)
self._files_to_remove.extend([fp, a.html_summary_fp[1]])
exp_files.append((a.html_summary_fp[0],
'%s (html summary)' % basename(a.html_summary_fp[1]),
'1642196267', '33 Bytes'))
exp_summary_path = relpath(a.html_summary_fp[1],
qiita_config.base_data_dir)
obs = artifact_summary_get_request(user, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'private',
'editable': True,
'buttons': main_buttons + private_download_button % 1,
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': exp_summary_path,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# No access
demo_u = User('*****@*****.**')
with self.assertRaises(QiitaHTTPError):
obs = artifact_summary_get_request(demo_u, 1)
# A non-owner/share user can't see the files
a.visibility = 'public'
obs = artifact_summary_get_request(demo_u, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'public',
'editable': False,
'buttons': '',
'processing_info': {},
'files': [],
'is_from_analysis': False,
'summary': exp_summary_path,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# testing sandbox
a.visibility = 'sandbox'
obs = artifact_summary_get_request(user, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'sandbox',
'editable': True,
'buttons': private_download_button % 1,
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': exp_summary_path,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# returnig to private
a.visibility = 'private'
# admin gets buttons
obs = artifact_summary_get_request(User('*****@*****.**'), 2)
exp_files = [(3, '1_seqs.fna (preprocessed fasta)', '', '0 Bytes'),
(4, '1_seqs.qual (preprocessed fastq)', '', '0 Bytes'),
(5, '1_seqs.demux (preprocessed demux)', '', '0 Bytes')]
exp = {
'name':
'Demultiplexed 1',
'artifact_id':
2,
'artifact_type':
'Demultiplexed',
'artifact_timestamp':
'2012-10-01 10:10',
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 2?\')) { '
'set_artifact_visibility(\'public\', 2) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'2?\')) { set_artifact_visibility(\'sandbox\', 2) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button> <a class="btn btn-primary '
'btn-sm" href="/ebi_submission/2"><span '
'class="glyphicon glyphicon-export"></span> '
'Submit to EBI</a> <a class="btn btn-primary '
'btn-sm" href="/vamps/2"><span class="glyphicon '
'glyphicon-export"></span> Submit to VAMPS</a> ' +
private_download_button % 2),
'processing_info': {
'command_active': True,
'software_deprecated': False,
'command': 'Split libraries FASTQ',
'processing_parameters': {
'max_barcode_errors': '1.5',
'sequence_max_n': '0',
'max_bad_run_length': '3',
'phred_offset': 'auto',
'rev_comp': 'False',
'phred_quality_threshold': '3',
'input_data': '1',
'rev_comp_barcode': 'False',
'rev_comp_mapping_barcodes': 'False',
'min_per_read_length_fraction': '0.75',
'barcode_type': 'golay_12'
},
'software_version': '1.9.1',
'software': 'QIIME'
},
'files':
exp_files,
'is_from_analysis':
False,
'summary':
None,
'job':
None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# analysis artifact
obs = artifact_summary_get_request(user, 8)
exp = {
'name': 'noname',
'artifact_id': 8,
'artifact_type': 'BIOM',
# this value changes on build so copy from obs
'artifact_timestamp': obs['artifact_timestamp'],
'visibility': 'sandbox',
'editable': True,
'buttons': private_download_button % 8,
'processing_info': {},
'files': [(22, 'biom_table.biom (biom)', '1756512010', '1.1 MB')],
'is_from_analysis': True,
'summary': None,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
def test_complete_job(self):
# Complete success
pt = npt.assert_warns(QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {
'1.SKD6.640190': 1
}}), Study(1), '16S')
c_job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command.get_validator('BIOM'),
values_dict={
'template': pt.id,
'files': dumps({'BIOM': ['file']}),
'artifact_type': 'BIOM'
}), True)
c_job._set_status('running')
fd, fp = mkstemp(suffix='_table.biom')
close(fd)
with open(fp, 'w') as f:
f.write('\n')
self._clean_up_files.append(fp)
exp_artifact_count = get_count('qiita.artifact') + 1
payload = dumps({
'success': True,
'error': '',
'artifacts': {
'OTU table': {
'filepaths': [(fp, 'biom')],
'artifact_type': 'BIOM'
}
}
})
job = self._create_job('complete_job', {
'job_id': c_job.id,
'payload': payload
})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertEqual(c_job.status, 'success')
self.assertEqual(get_count('qiita.artifact'), exp_artifact_count)
# Complete job error
payload = dumps({'success': False, 'error': 'Job failure'})
job = self._create_job('complete_job', {
'job_id': 'bcc7ebcd-39c1-43e4-af2d-822e3589f14d',
'payload': payload
})
private_task(job.id)
self.assertEqual(job.status, 'success')
c_job = ProcessingJob('bcc7ebcd-39c1-43e4-af2d-822e3589f14d')
self.assertEqual(c_job.status, 'error')
self.assertEqual(c_job.log, LogEntry.newest_records(numrecords=1)[0])
self.assertEqual(c_job.log.msg, 'Job failure')
# Complete internal error
pt = npt.assert_warns(QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {
'1.SKD6.640190': 1
}}), Study(1), '16S')
c_job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command.get_validator('BIOM'),
values_dict={
'template': pt.id,
'files': dumps({'BIOM': ['file']}),
'artifact_type': 'BIOM'
}), True)
c_job._set_status('running')
fp = '/surprised/if/this/path/exists.biom'
payload = dumps({
'success': True,
'error': '',
'artifacts': {
'OTU table': {
'filepaths': [(fp, 'biom')],
'artifact_type': 'BIOM'
}
}
})
job = self._create_job('complete_job', {
'job_id': c_job.id,
'payload': payload
})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertEqual(c_job.status, 'error')
self.assertIn('No such file or directory', c_job.log.msg)
def test_download_study(self):
tmp_dir = mkdtemp()
self._clean_up_files.append(tmp_dir)
biom_fp = join(tmp_dir, 'otu_table.biom')
smr_dir = join(tmp_dir, 'sortmerna_picked_otus')
log_dir = join(smr_dir, 'seqs_otus.log')
tgz = join(tmp_dir, 'sortmerna_picked_otus.tgz')
with biom_open(biom_fp, 'w') as f:
et.to_hdf5(f, "test")
makedirs(smr_dir)
with open(log_dir, 'w') as f:
f.write('\n')
with open(tgz, 'w') as f:
f.write('\n')
files_biom = [(biom_fp, 'biom'), (smr_dir, 'directory'), (tgz, 'tgz')]
params = Parameters.from_default_params(
next(Command(3).default_parameter_sets), {'input_data': 1})
a = Artifact.create(files_biom,
"BIOM",
parents=[Artifact(2)],
processing_parameters=params)
for x in a.filepaths:
self._clean_up_files.append(x['fp'])
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 200)
exp = ('- 1256812 /protected/processed_data/'
'1_study_1001_closed_reference_otu_table.biom processed_data/'
'1_study_1001_closed_reference_otu_table.biom\n'
'- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
'mapping_files/4_mapping_file.txt\n'
'- 1256812 /protected/processed_data/'
'1_study_1001_closed_reference_otu_table.biom processed_data/'
'1_study_1001_closed_reference_otu_table.biom\n'
'- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
'mapping_files/5_mapping_file.txt\n'
'- 1256812 /protected/processed_data/1_study_1001_'
'closed_reference_otu_table_Silva.biom processed_data/'
'1_study_1001_closed_reference_otu_table_Silva.biom\n'
'- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
'mapping_files/6_mapping_file.txt\n'
'- 1093210 /protected/BIOM/7/biom_table.biom '
'BIOM/7/biom_table.biom\n'
'- [0-9]* /protected/templates/1_prep_2_[0-9]*-[0-9]*.txt '
'mapping_files/7_mapping_file.txt\n'
'- [0-9]* /protected/BIOM/{0}/otu_table.biom '
'BIOM/{0}/otu_table.biom\n'
'- 1 /protected/BIOM/10/sortmerna_picked_otus/seqs_otus.log '
'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n'
'- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
'mapping_files/{0}_mapping_file.txt\n'.format(a.id))
self.assertRegex(response.body.decode('ascii'), exp)
response = self.get('/download_study_bioms/200')
self.assertEqual(response.code, 405)
# changing user so we can test the failures
BaseHandler.get_current_user = Mock(
return_value=User("*****@*****.**"))
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 405)
a.visibility = 'public'
response = self.get('/download_study_bioms/1')
# returning visibility
a.visibility = 'private'
self.assertEqual(response.code, 200)
# we should have the same files than the previous test, except artifact
# and mapping file 7: position 6 and 7; thus removing 6 twice
exp = exp.split('\n')
exp.pop(6)
exp.pop(6)
exp = '\n'.join(exp)
self.assertRegex(response.body.decode('ascii'), exp)
