def list_commands_handler_get_req(artifact_types):
"""Retrieves the commands that can process the given artifact types
Parameters
----------
artifact_types : str
Comma-separated list of artifact types
Returns
-------
dict of objects
A dictionary containing the commands information
{'status': str,
'message': str,
'commands': list of dicts of {'id': int,
'command': str,
'output': list of [str, str]}}
"""
artifact_types = artifact_types.split(',')
cmd_info = [
{'id': cmd.id, 'command': cmd.name, 'output': cmd.outputs}
for cmd in Command.get_commands_by_input_type(artifact_types)]
return {'status': 'success',
'message': '',
'commands': cmd_info}
def list_commands_handler_get_req(artifact_types):
"""Retrieves the commands that can process the given artifact types
Parameters
----------
artifact_types : str
Comma-separated list of artifact types
Returns
-------
dict of objects
A dictionary containing the commands information
{'status': str,
'message': str,
'commands': list of dicts of {'id': int,
'command': str,
'output': list of [str, str]}}
"""
artifact_types = artifact_types.split(',')
cmd_info = [{
'id': cmd.id,
'command': cmd.name,
'output': cmd.outputs
} for cmd in Command.get_commands_by_input_type(artifact_types)]
return {'status': 'success', 'message': '', 'commands': cmd_info}
def list_options_handler_get_req(command_id):
"""Returns the available default parameters set for the given command
Parameters
----------
command_id : int
The command id
Returns
-------
dict of objects
A dictionary containing the commands information
{'status': str,
'message': str,
'options': list of dicts of {'id: str', 'name': str,
'values': dict of {str: str}}}
"""
command = Command(command_id)
options = [{
'id': p.id,
'name': p.name,
'values': p.values
} for p in command.default_parameter_sets]
return {
'status': 'success',
'message': '',
'options': options,
'req_options': command.required_parameters
}
def workflow_handler_post_req(user_id, command_id, params):
"""Creates a new workflow in the system
Parameters
----------
user_id : str
The user creating the workflow
command_id : int
The first command to execute in the workflow
params : str
JSON representations of the parameters for the first command of
the workflow
Returns
-------
dict of objects
A dictionary containing the commands information
{'status': str,
'message': str,
'workflow_id': int}
"""
parameters = Parameters.load(Command(command_id), json_str=params)
status = 'success'
message = ''
try:
wf = ProcessingWorkflow.from_scratch(User(user_id), parameters)
except Exception as exc:
wf = None
wf_id = None
job_info = None
status = 'error'
message = str(exc)
if wf is not None:
# this is safe as we are creating the workflow for the first time
# and there is only one node. Remember networkx doesn't assure order
# of nodes
job = list(wf.graph.nodes())[0]
inputs = [a.id for a in job.input_artifacts]
job_cmd = job.command
wf_id = wf.id
job_info = {
'id': job.id,
'inputs': inputs,
'label': job_cmd.name,
'outputs': job_cmd.outputs
}
return {
'status': status,
'message': message,
'workflow_id': wf_id,
'job': job_info
}
def list_options_handler_get_req(command_id, artifact_id=None):
"""Returns the available default parameters set for the given command
Parameters
----------
command_id : int
The command id
artifact_id : int, optional
The artifact id so to limit options based on how it has already been
processed
Returns
-------
dict of objects
A dictionary containing the commands information
{'status': str,
'message': str,
'options': list of dicts of {'id: str', 'name': str,
'values': dict of {str: str}}}
"""
def _helper_process_params(params):
return dumps({k: str(v).lower()
for k, v in params.items()},
sort_keys=True)
command = Command(command_id)
rparamers = command.required_parameters.keys()
eparams = []
if artifact_id is not None:
artifact = Artifact(artifact_id)
for job in artifact.jobs(cmd=command):
jstatus = job.status
outputs = job.outputs if job.status == 'success' else None
# this ignore any jobs that weren't successful or are in
# construction, or the results have been deleted [outputs == {}]
if jstatus not in {'success', 'in_construction'} or outputs == {}:
continue
params = job.parameters.values.copy()
for k in rparamers:
del params[k]
eparams.append(_helper_process_params(params))
options = [{
'id': p.id,
'name': p.name,
'values': p.values
} for p in command.default_parameter_sets
if _helper_process_params(p.values) not in eparams]
return {
'status': 'success',
'message': '',
'options': options,
'req_options': command.required_parameters,
'opt_options': command.optional_parameters
}
def test_workflow_handler_patch_req(self):
# Create a new workflow so it is in construction
exp_command = Command(1)
json_str = (
'{"input_data": 1, "max_barcode_errors": 1.5, '
'"barcode_type": "golay_12", "max_bad_run_length": 3, '
'"rev_comp": false, "phred_quality_threshold": 3, '
'"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
'"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
exp_params = Parameters.load(exp_command, json_str=json_str)
exp_user = User('*****@*****.**')
name = "Test processing workflow"
# tests success
wf = ProcessingWorkflow.from_scratch(exp_user,
exp_params,
name=name,
force=True)
graph = wf.graph
nodes = list(graph.nodes())
job_id = nodes[0].id
value = {
'dflt_params': 10,
'connections': {
job_id: {
'demultiplexed': 'input_data'
}
}
}
obs = workflow_handler_patch_req('add',
'/%s/' % wf.id,
req_value=dumps(value))
new_jobs = set(wf.graph.nodes()) - set(nodes)
self.assertEqual(len(new_jobs), 1)
new_job = new_jobs.pop()
exp = {
'status': 'success',
'message': '',
'job': {
'id': new_job.id,
'inputs': [job_id],
'label': 'Pick closed-reference OTUs',
'outputs': [['OTU table', 'BIOM']]
}
}
self.assertEqual(obs, exp)
obs = workflow_handler_patch_req('remove',
'/%s/%s/' % (wf.id, new_job.id))
exp = {'status': 'success', 'message': ''}
jobs = set(wf.graph.nodes()) - set(nodes)
self.assertEqual(jobs, set())
def test_artifact_summary_post_request(self):
# No access
with self.assertRaises(QiitaHTTPError):
artifact_summary_post_request(User('*****@*****.**'), 1)
# Returns already existing job
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 2}))
job._set_status('queued')
obs = artifact_summary_post_request(User('*****@*****.**'), 2)
exp = {'job': [job.id, 'queued', None]}
self.assertEqual(obs, exp)
def test_patch(self):
# Create a new job - through a workflow since that is the only way
# of creating jobs in the interface
exp_command = Command(1)
json_str = (
'{"input_data": 1, "max_barcode_errors": 1.5, '
'"barcode_type": "golay_12", "max_bad_run_length": 3, '
'"rev_comp": false, "phred_quality_threshold": 3, '
'"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
'"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
exp_params = Parameters.load(exp_command, json_str=json_str)
exp_user = User('*****@*****.**')
name = "Test processing workflow"
# tests success
wf = ProcessingWorkflow.from_scratch(exp_user,
exp_params,
name=name,
force=True)
graph = wf.graph
nodes = graph.nodes()
job_id = nodes[0].id
response = self.patch('/study/process/job/', {
'op': 'remove',
'path': job_id
})
self.assertEqual(response.code, 200)
exp = {
'status':
'error',
'message':
"Can't delete job %s. It is 'in_construction' "
"status. Please use /study/process/workflow/" % job_id
}
self.assertEqual(loads(response.body), exp)
# Test success
ProcessingJob(job_id)._set_error('Killed for testing')
response = self.patch('/study/process/job/', {
'op': 'remove',
'path': job_id
})
self.assertEqual(response.code, 200)
exp = {'status': 'success', 'message': ''}
self.assertEqual(loads(response.body), exp)
def artifact_summary_post_request(user_id, artifact_id):
"""Launches the HTML summary generation and returns the job information
Parameters
----------
user_id : str
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the artifact summary information
{'status': str,
'message': str,
'job': list of [str, str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
access_error = check_access(artifact.study.id, user_id)
if access_error:
return access_error
# Check if the summary is being generated or has been already generated
command = Command.get_html_generator(artifact.artifact_type)
jobs = artifact.jobs(cmd=command)
jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
if jobs:
# The HTML summary is either being generated or already generated.
# Return the information of that job so we only generate the HTML
# once
job = jobs[0]
else:
# Create a new job to generate the HTML summary and return the newly
# created job information
job = ProcessingJob.create(
User(user_id),
Parameters.load(command, values_dict={'input_data': artifact_id}))
job.submit()
return {
'status': 'success',
'message': '',
'job': [job.id, job.status, job.step]
}
def list_commands_handler_get_req(id, exclude_analysis):
"""Retrieves the commands that can process the given artifact types
Parameters
----------
id : string
id, it can be the integer or the name of the artifact:network-root
exclude_analysis : bool
If True, return commands that are not part of the analysis pipeline
Returns
-------
dict of objects
A dictionary containing the commands information
{'status': str,
'message': str,
'commands': list of dicts of {'id': int,
'command': str,
'output': list of [str, str]}}
"""
if id.isdigit():
commands = Artifact(id).get_commands
else:
pieces = id.split(':')
if len(pieces) == 1:
aid = pieces[0]
root = ''
else:
aid = pieces[0]
root = pieces[1]
prep_type = None
if root.isdigit():
artifact = Artifact(root)
if artifact.analysis is None:
prep_type = artifact.prep_templates[0].data_type
commands = Command.get_commands_by_input_type(
[aid], exclude_analysis=exclude_analysis, prep_type=prep_type)
cmd_info = [{
'id': cmd.id,
'command': cmd.name,
'output': cmd.outputs
} for cmd in commands]
return {'status': 'success', 'message': '', 'commands': cmd_info}
def artifact_summary_post_request(user_id, artifact_id):
"""Launches the HTML summary generation and returns the job information
Parameters
----------
user_id : str
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the artifact summary information
{'status': str,
'message': str,
'job': list of [str, str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
access_error = check_access(artifact.study.id, user_id)
if access_error:
return access_error
# Check if the summary is being generated or has been already generated
command = Command.get_html_generator(artifact.artifact_type)
jobs = artifact.jobs(cmd=command)
jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
if jobs:
# The HTML summary is either being generated or already generated.
# Return the information of that job so we only generate the HTML
# once
job = jobs[0]
else:
# Create a new job to generate the HTML summary and return the newly
# created job information
job = ProcessingJob.create(
User(user_id),
Parameters.load(command, values_dict={'input_data': artifact_id}))
job.submit()
return {'status': 'success',
'message': '',
'job': [job.id, job.status, job.step]}
def artifact_summary_post_request(user, artifact_id):
"""Launches the HTML summary generation and returns the job information
Parameters
----------
user : qiita_db.user.User
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the job summary information
{'job': [str, str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
check_artifact_access(user, artifact)
# Check if the summary is being generated or has been already generated
command = Command.get_html_generator(artifact.artifact_type)
jobs = artifact.jobs(cmd=command)
jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
if jobs:
# The HTML summary is either being generated or already generated.
# Return the information of that job so we only generate the HTML
# once - Magic number 0 -> we are ensuring that there is only one
# job generating the summary, so we can use the index 0 to access to
# that job
job = jobs[0]
else:
# Create a new job to generate the HTML summary and return the newly
# created job information
job = ProcessingJob.create(
user,
Parameters.load(command, values_dict={'input_data': artifact_id}),
True)
job.submit()
return {'job': [job.id, job.status, job.step]}
def test_artifact_summary_post_request(self):
# No access
obs = artifact_summary_post_request('*****@*****.**', 1)
exp = {
'status': 'error',
'message': 'User does not have access to study'
}
self.assertEqual(obs, exp)
# Returns already existing job
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 2}))
job._set_status('queued')
obs = artifact_summary_post_request('*****@*****.**', 2)
exp = {
'status': 'success',
'message': '',
'job': [job.id, 'queued', None]
}
self.assertEqual(obs, exp)
def artifact_summary_post_request(user, artifact_id):
"""Launches the HTML summary generation and returns the job information
Parameters
----------
user : qiita_db.user.User
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the job summary information
{'job': [str, str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
check_artifact_access(user, artifact)
# Check if the summary is being generated or has been already generated
command = Command.get_html_generator(artifact.artifact_type)
jobs = artifact.jobs(cmd=command)
jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
if jobs:
# The HTML summary is either being generated or already generated.
# Return the information of that job so we only generate the HTML
# once - Magic number 0 -> we are ensuring that there is only one
# job generating the summary, so we can use the index 0 to access to
# that job
job = jobs[0]
else:
# Create a new job to generate the HTML summary and return the newly
# created job information
job = ProcessingJob.create(user, Parameters.load(
command, values_dict={'input_data': artifact_id}), True)
job.submit()
return {'job': [job.id, job.status, job.step]}
def workflow_handler_post_req(user_id, command_id, params):
"""Creates a new workflow in the system
Parameters
----------
user_id : str
The user creating the workflow
command_id : int
The first command to execute in the workflow
params : str
JSON representations of the parameters for the first command of
the workflow
Returns
-------
dict of objects
A dictionary containing the commands information
{'status': str,
'message': str,
'workflow_id': int}
"""
parameters = Parameters.load(Command(command_id), json_str=params)
wf = ProcessingWorkflow.from_scratch(User(user_id), parameters)
# this is safe as we are creating the workflow for the first time and there
# is only one node. Remember networkx doesn't assure order of nodes
job = wf.graph.nodes()[0]
inputs = [a.id for a in job.input_artifacts]
job_cmd = job.command
return {
'status': 'success',
'message': '',
'workflow_id': wf.id,
'job': {
'id': job.id,
'inputs': inputs,
'label': job_cmd.name,
'outputs': job_cmd.outputs
}
}
def artifact_post_req(user_id,
filepaths,
artifact_type,
name,
prep_template_id,
artifact_id=None):
"""Creates the initial artifact for the prep template
Parameters
----------
user_id : str
User adding the atrifact
filepaths : dict of str
Comma-separated list of files to attach to the artifact,
keyed by file type
artifact_type : str
The type of the artifact
name : str
Name to give the artifact
prep_template_id : int or str castable to int
Prep template to attach the artifact to
artifact_id : int or str castable to int, optional
The id of the imported artifact
Returns
-------
dict of objects
A dictionary containing the new artifact ID
{'status': status,
'message': message,
'artifact': id}
"""
prep_template_id = int(prep_template_id)
prep = PrepTemplate(prep_template_id)
study_id = prep.study_id
# First check if the user has access to the study
access_error = check_access(study_id, user_id)
if access_error:
return access_error
user = User(user_id)
if artifact_id:
# if the artifact id has been provided, import the artifact
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('copy_artifact')
params = Parameters.load(cmd,
values_dict={
'artifact': artifact_id,
'prep_template': prep.id
})
job = ProcessingJob.create(user, params, True)
else:
uploads_path = get_mountpoint('uploads')[0][1]
path_builder = partial(join, uploads_path, str(study_id))
cleaned_filepaths = {}
for ftype, file_list in filepaths.items():
# JavaScript sends us this list as a comma-separated list
for fp in file_list.split(','):
# JavaScript will send this value as an empty string if the
# list of files was empty. In such case, the split will
# generate a single element containing the empty string. Check
# for that case here and, if fp is not the empty string,
# proceed to check if the file exists
if fp:
# Check if filepath being passed exists for study
full_fp = path_builder(fp)
exists = check_fp(study_id, full_fp)
if exists['status'] != 'success':
return {
'status': 'error',
'message': 'File does not exist: %s' % fp
}
if ftype not in cleaned_filepaths:
cleaned_filepaths[ftype] = []
cleaned_filepaths[ftype].append(full_fp)
# This should never happen, but it doesn't hurt to actually have
# a explicit check, in case there is something odd with the JS
if not cleaned_filepaths:
return {
'status': 'error',
'message': "Can't create artifact, no files provided."
}
# This try/except will catch the case when the plugins are not
# activated so there is no Validate for the given artifact_type
try:
command = Command.get_validator(artifact_type)
except QiitaDBError as e:
return {'status': 'error', 'message': str(e)}
job = ProcessingJob.create(
user,
Parameters.load(command,
values_dict={
'template': prep_template_id,
'files': dumps(cleaned_filepaths),
'artifact_type': artifact_type,
'name': name,
'analysis': None,
}), True)
# Submit the job
job.submit()
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep.id,
dumps({
'job_id': job.id,
'is_qiita_job': True
}))
return {'status': 'success', 'message': ''}
def artifact_summary_get_request(user, artifact_id):
"""Returns the information for the artifact summary page
Parameters
----------
user : qiita_db.user.User
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the artifact summary information
{'name': str,
'artifact_id': int,
'visibility': str,
'editable': bool,
'buttons': str,
'processing_parameters': dict of {str: object},
'files': list of (int, str),
'is_from_analysis': bool,
'summary': str or None,
'job': [str, str, str],
'errored_jobs': list of [str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
artifact_type = artifact.artifact_type
check_artifact_access(user, artifact)
visibility = artifact.visibility
summary = artifact.html_summary_fp
job_info = None
errored_summary_jobs = []
# Check if the HTML summary exists
if summary:
# Magic number 1: If the artifact has a summary, the call
# artifact.html_summary_fp returns a tuple with 2 elements. The first
# element is the filepath id, while the second one is the actual
# actual filepath. We are only interested on the actual filepath,
# hence the 1 value.
summary = relpath(summary[1], qiita_config.base_data_dir)
else:
# Check if the summary is being generated
command = Command.get_html_generator(artifact_type)
all_jobs = set(artifact.jobs(cmd=command))
jobs = []
errored_summary_jobs = []
for j in all_jobs:
if j.status in ['queued', 'running']:
jobs.append(j)
elif j.status in ['error']:
errored_summary_jobs.append(j)
if jobs:
# There is already a job generating the HTML. Also, there should be
# at most one job, because we are not allowing here to start more
# than one
job = jobs[0]
job_info = [job.id, job.status, job.step]
# Check if the artifact is editable by the given user
study = artifact.study
analysis = artifact.analysis
editable = study.can_edit(user) if study else analysis.can_edit(user)
buttons = []
btn_base = (
'<button onclick="if (confirm(\'Are you sure you want to %s '
'artifact id: {0}?\')) {{ set_artifact_visibility(\'%s\', {0}) }}" '
'class="btn btn-primary btn-sm">%s</button>').format(artifact_id)
if not analysis:
# If the artifact is part of a study, the buttons shown depend in
# multiple factors (see each if statement for an explanation of those)
if qiita_config.require_approval:
if visibility == 'sandbox' and artifact.parents:
# The request approval button only appears if the artifact is
# sandboxed and the qiita_config specifies that the approval
# should be requested
buttons.append(
btn_base % ('request approval for', 'awaiting_approval',
'Request approval'))
elif user.level == 'admin' and visibility == 'awaiting_approval':
# The approve artifact button only appears if the user is an
# admin the artifact is waiting to be approvaed and the qiita
# config requires artifact approval
buttons.append(btn_base % ('approve', 'private',
'Approve artifact'))
if visibility == 'private':
# The make public button only appears if the artifact is private
buttons.append(btn_base % ('make public', 'public', 'Make public'))
# The revert to sandbox button only appears if the artifact is not
# sandboxed nor public
if visibility not in {'sandbox', 'public'}:
buttons.append(btn_base % ('revert to sandbox', 'sandbox',
'Revert to sandbox'))
if user.level == 'admin':
if artifact.can_be_submitted_to_ebi:
buttons.append(
'<a class="btn btn-primary btn-sm" '
'href="/ebi_submission/%d">'
'<span class="glyphicon glyphicon-export"></span>'
' Submit to EBI</a>' % artifact_id)
if artifact.can_be_submitted_to_vamps:
if not artifact.is_submitted_to_vamps:
buttons.append(
'<a class="btn btn-primary btn-sm" href="/vamps/%d">'
'<span class="glyphicon glyphicon-export"></span>'
' Submit to VAMPS</a>' % artifact_id)
files = [(x['fp_id'], "%s (%s)" % (basename(x['fp']),
x['fp_type'].replace('_', ' ')),
x['checksum'], naturalsize(x['fp_size']))
for x in artifact.filepaths if x['fp_type'] != 'directory']
# TODO: https://github.com/biocore/qiita/issues/1724 Remove this hardcoded
# values to actually get the information from the database once it stores
# the information
if artifact_type in ['SFF', 'FASTQ', 'FASTA', 'FASTA_Sanger',
'per_sample_FASTQ']:
# If the artifact is one of the "raw" types, only the owner of the
# study and users that has been shared with can see the files
study = artifact.study
has_access = study.has_access(user, no_public=True)
if (not study.public_raw_download and not has_access):
files = []
proc_params = artifact.processing_parameters
if proc_params:
cmd = proc_params.command
sw = cmd.software
processing_info = {
'command': cmd.name,
'software': sw.name,
'software_version': sw.version,
'processing_parameters': proc_params.values,
'command_active': cmd.active,
'software_deprecated': sw.deprecated,
}
else:
processing_info = {}
return {'name': artifact.name,
'artifact_id': artifact_id,
'artifact_type': artifact_type,
'visibility': visibility,
'editable': editable,
'buttons': ' '.join(buttons),
'processing_info': processing_info,
'files': files,
'is_from_analysis': artifact.analysis is not None,
'summary': summary,
'job': job_info,
'artifact_timestamp': artifact.timestamp.strftime(
"%Y-%m-%d %H:%m"),
'errored_summary_jobs': errored_summary_jobs}
def artifact_summary_get_request(user_id, artifact_id):
"""Returns the information for the artifact summary page
Parameters
----------
user_id : str
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the artifact summary information
{'status': str,
'message': str,
'name': str,
'summary': str,
'job': list of [str, str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
access_error = check_access(artifact.study.id, user_id)
if access_error:
return access_error
user = User(user_id)
visibility = artifact.visibility
summary = artifact.html_summary_fp
job_info = None
errored_jobs = []
processing_jobs = []
for j in artifact.jobs():
if j.command.software.type == "artifact transformation":
status = j.status
if status == 'success':
continue
j_msg = j.log.msg if status == 'error' else None
processing_jobs.append(
[j.id, j.command.name, j.status, j.step, j_msg])
# Check if the HTML summary exists
if summary:
with open(summary[1]) as f:
summary = f.read()
else:
# Check if the summary is being generated
command = Command.get_html_generator(artifact.artifact_type)
all_jobs = set(artifact.jobs(cmd=command))
jobs = [j for j in all_jobs if j.status in ['queued', 'running']]
errored_jobs = [(j.id, j.log.msg)
for j in all_jobs if j.status in ['error']]
if jobs:
# There is already a job generating the HTML. Also, there should be
# at most one job, because we are not allowing here to start more
# than one
job = jobs[0]
job_info = [job.id, job.status, job.step]
buttons = []
btn_base = (
'<button onclick="if (confirm(\'Are you sure you want to %s '
'artifact id: {0}?\')) {{ set_artifact_visibility(\'%s\', {0}) }}" '
'class="btn btn-primary btn-sm">%s</button>').format(artifact_id)
if qiita_config.require_approval:
if visibility == 'sandbox':
# The request approval button only appears if the artifact is
# sandboxed and the qiita_config specifies that the approval should
# be requested
buttons.append(
btn_base % ('request approval for', 'awaiting_approval',
'Request approval'))
elif user.level == 'admin' and visibility == 'awaiting_approval':
# The approve artifact button only appears if the user is an admin
# the artifact is waiting to be approvaed and the qiita config
# requires artifact approval
buttons.append(btn_base % ('approve', 'private',
'Approve artifact'))
if visibility == 'private':
# The make public button only appears if the artifact is private
buttons.append(btn_base % ('make public', 'public', 'Make public'))
# The revert to sandbox button only appears if the artifact is not
# sandboxed nor public
if visibility not in {'sandbox', 'public'}:
buttons.append(btn_base % ('revert to sandbox', 'sandbox',
'Revert to sandbox'))
if artifact.can_be_submitted_to_ebi:
if not artifact.is_submitted_to_ebi:
buttons.append(
'<a class="btn btn-primary btn-sm" '
'href="/ebi_submission/%d">'
'<span class="glyphicon glyphicon-export"></span>'
' Submit to EBI</a>' % artifact_id)
if artifact.can_be_submitted_to_vamps:
if not artifact.is_submitted_to_vamps:
buttons.append(
'<a class="btn btn-primary btn-sm" href="/vamps/%d">'
'<span class="glyphicon glyphicon-export"></span>'
' Submit to VAMPS</a>' % artifact_id)
files = [(f_id, "%s (%s)" % (basename(fp), f_type.replace('_', ' ')))
for f_id, fp, f_type in artifact.filepaths
if f_type != 'directory']
# TODO: https://github.com/biocore/qiita/issues/1724 Remove this hardcoded
# values to actually get the information from the database once it stores
# the information
if artifact.artifact_type in ['SFF', 'FASTQ', 'FASTA', 'FASTA_Sanger',
'per_sample_FASTQ']:
# If the artifact is one of the "raw" types, only the owner of the
# study and users that has been shared with can see the files
if not artifact.study.has_access(user, no_public=True):
files = []
return {'status': 'success',
'message': '',
'name': artifact.name,
'summary': summary,
'job': job_info,
'errored_jobs': errored_jobs,
'processing_jobs': processing_jobs,
'visibility': visibility,
'buttons': ' '.join(buttons),
'files': files,
'editable': artifact.study.can_edit(user)}
def render(self, study_id, preprocessed_data):
user = self.current_user
ppd_id = preprocessed_data.id
vamps_status = preprocessed_data.is_submitted_to_vamps
filepaths = preprocessed_data.filepaths
is_local_request = is_localhost(self.request.headers['host'])
show_ebi_btn = user.level == "admin"
processing_status, processing_status_msg = \
get_artifact_processing_status(preprocessed_data)
processed_data = sorted([pd.id for pd in preprocessed_data.children])
# Get all the ENA terms for the investigation type
ontology = Ontology(convert_to_id('ENA', 'ontology'))
# make "Other" show at the bottom of the drop down menu
ena_terms = []
for v in sorted(ontology.terms):
if v != 'Other':
ena_terms.append('<option value="%s">%s</option>' % (v, v))
ena_terms.append('<option value="Other">Other</option>')
# New Type is for users to add a new user-defined investigation type
user_defined_terms = ontology.user_defined_terms + ['New Type']
# ppd can only have 1 prep template
prep_template = preprocessed_data.prep_templates[0]
# this block might seem wrong but is here due to a possible
# pathological case that we used to have in the system: preprocessed
# data without valid prep_templates
prep_templates = preprocessed_data.prep_templates
if len(prep_templates) == 1:
prep_template_id = prep_template.id
raw_data_id = prep_template.artifact.id
inv_type = prep_template.investigation_type or "None selected"
else:
prep_template_id = None
raw_data_id = None
inv_type = "None Selected"
process_params = {
param.id: (generate_param_str(param), param.name)
for param in Command(3).default_parameter_sets
}
# We just need to provide an ID for the default parameters,
# so we can initialize the interface
default_params = min(process_params.keys())
ebi_link = None
if preprocessed_data.is_submitted_to_ebi:
ebi_link = EBI_LINKIFIER.format(
Study(study_id).ebi_study_accession)
return self.render_string(
"study_description_templates/preprocessed_data_info_tab.html",
ppd_id=ppd_id,
show_ebi_btn=show_ebi_btn,
filepaths=filepaths,
is_local_request=is_local_request,
prep_template_id=prep_template_id,
raw_data_id=raw_data_id,
inv_type=inv_type,
ena_terms=ena_terms,
vamps_status=vamps_status,
user_defined_terms=user_defined_terms,
process_params=process_params,
default_params=default_params,
study_id=preprocessed_data.study.id,
processing_status=processing_status,
processing_status_msg=processing_status_msg,
processed_data=processed_data,
ebi_link=ebi_link)
# Nov 28, 2017 (only in py file)
# Adding a new command into Qiita/Alpha: delete_analysis
from qiita_db.software import Software, Command
from qiita_db.sql_connection import TRN
# Create the delete study command
Command.create(Software.from_name_and_version('Qiita',
'alpha'), 'delete_analysis',
'Deletes a full analysis', {'analysis_id': ['integer', None]})
# Make sure that all validate commands have the "analysis" parameter
with TRN:
# Get all validate commands that are missing the analysis parameter
sql = """SELECT command_id
FROM qiita.software_command sc
WHERE name = 'Validate' AND NOT (
SELECT EXISTS(SELECT *
FROM qiita.command_parameter
WHERE parameter_name = 'analysis'
AND command_id = sc.command_id));"""
TRN.add(sql)
sql = """INSERT INTO qiita.command_parameter
(command_id, parameter_name, parameter_type,
required, default_value, name_order, check_biom_merge)
VALUES (6, 'analysis', 'analysis', false, NULL, NULL, false)"""
sql_params = [[cmd_id, 'analysis', 'analysis', False, None, None, False]
for cmd_id in TRN.execute_fetchflatten()]
TRN.add(sql, sql_params, many=True)
TRN.execute()
def artifact_summary_get_request(user_id, artifact_id):
"""Returns the information for the artifact summary page
Parameters
----------
user_id : str
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the artifact summary information
{'status': str,
'message': str,
'name': str,
'summary': str,
'job': list of [str, str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
access_error = check_access(artifact.study.id, user_id)
if access_error:
return access_error
user = User(user_id)
visibility = artifact.visibility
summary = artifact.html_summary_fp
job_info = None
errored_jobs = []
processing_jobs = []
for j in artifact.jobs():
if j.command.software.type == "artifact transformation":
status = j.status
if status == 'success':
continue
j_msg = j.log.msg if status == 'error' else None
processing_jobs.append(
[j.id, j.command.name, j.status, j.step, j_msg])
# Check if the HTML summary exists
if summary:
with open(summary[1]) as f:
summary = f.read()
else:
# Check if the summary is being generated
command = Command.get_html_generator(artifact.artifact_type)
all_jobs = set(artifact.jobs(cmd=command))
jobs = [j for j in all_jobs if j.status in ['queued', 'running']]
errored_jobs = [(j.id, j.log.msg) for j in all_jobs
if j.status in ['error']]
if jobs:
# There is already a job generating the HTML. Also, there should be
# at most one job, because we are not allowing here to start more
# than one
job = jobs[0]
job_info = [job.id, job.status, job.step]
buttons = []
btn_base = (
'<button onclick="if (confirm(\'Are you sure you want to %s '
'artifact id: {0}?\')) {{ set_artifact_visibility(\'%s\', {0}) }}" '
'class="btn btn-primary btn-sm">%s</button>').format(artifact_id)
if qiita_config.require_approval:
if visibility == 'sandbox':
# The request approval button only appears if the artifact is
# sandboxed and the qiita_config specifies that the approval should
# be requested
buttons.append(btn_base %
('request approval for', 'awaiting_approval',
'Request approval'))
elif user.level == 'admin' and visibility == 'awaiting_approval':
# The approve artifact button only appears if the user is an admin
# the artifact is waiting to be approvaed and the qiita config
# requires artifact approval
buttons.append(btn_base %
('approve', 'private', 'Approve artifact'))
if visibility == 'private':
# The make public button only appears if the artifact is private
buttons.append(btn_base % ('make public', 'public', 'Make public'))
# The revert to sandbox button only appears if the artifact is not
# sandboxed nor public
if visibility not in {'sandbox', 'public'}:
buttons.append(btn_base %
('revert to sandbox', 'sandbox', 'Revert to sandbox'))
if user.level == 'admin':
if artifact.can_be_submitted_to_ebi:
if not artifact.is_submitted_to_ebi:
buttons.append(
'<a class="btn btn-primary btn-sm" '
'href="/ebi_submission/%d">'
'<span class="glyphicon glyphicon-export"></span>'
' Submit to EBI</a>' % artifact_id)
if artifact.can_be_submitted_to_vamps:
if not artifact.is_submitted_to_vamps:
buttons.append(
'<a class="btn btn-primary btn-sm" href="/vamps/%d">'
'<span class="glyphicon glyphicon-export"></span>'
' Submit to VAMPS</a>' % artifact_id)
files = [(f_id, "%s (%s)" % (basename(fp), f_type.replace('_', ' ')))
for f_id, fp, f_type in artifact.filepaths
if f_type != 'directory']
# TODO: https://github.com/biocore/qiita/issues/1724 Remove this hardcoded
# values to actually get the information from the database once it stores
# the information
if artifact.artifact_type in [
'SFF', 'FASTQ', 'FASTA', 'FASTA_Sanger', 'per_sample_FASTQ'
]:
# If the artifact is one of the "raw" types, only the owner of the
# study and users that has been shared with can see the files
if not artifact.study.has_access(user, no_public=True):
files = []
return {
'status': 'success',
'message': '',
'name': artifact.name,
'summary': summary,
'job': job_info,
'errored_jobs': errored_jobs,
'processing_jobs': processing_jobs,
'visibility': visibility,
'buttons': ' '.join(buttons),
'files': files,
'editable': artifact.study.can_edit(user),
'study_id': artifact.study.id,
'prep_id': artifact.prep_templates[0].id
}
def artifact_summary_get_request(user, artifact_id):
"""Returns the information for the artifact summary page
Parameters
----------
user : qiita_db.user.User
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the artifact summary information
{'name': str,
'artifact_id': int,
'visibility': str,
'editable': bool,
'buttons': str,
'processing_parameters': dict of {str: object},
'files': list of (int, str),
'is_from_analysis': bool,
'processing_jobs': list of [str, str, str, str, str],
'summary': str or None,
'job': [str, str, str],
'errored_jobs': list of [str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
artifact_type = artifact.artifact_type
check_artifact_access(user, artifact)
visibility = artifact.visibility
summary = artifact.html_summary_fp
job_info = None
errored_jobs = []
processing_jobs = []
for j in artifact.jobs():
if j.command.software.type == "artifact transformation":
status = j.status
if status == 'success':
continue
j_msg = j.log.msg if status == 'error' else None
processing_jobs.append(
[j.id, j.command.name, j.status, j.step, j_msg])
# Check if the HTML summary exists
if summary:
# Magic number 1: If the artifact has a summary, the call
# artifact.html_summary_fp returns a tuple with 2 elements. The first
# element is the filepath id, while the second one is the actual
# actual filepath. We are only interested on the actual filepath,
# hence the 1 value.
summary = relpath(summary[1], qiita_config.base_data_dir)
else:
# Check if the summary is being generated
command = Command.get_html_generator(artifact_type)
all_jobs = set(artifact.jobs(cmd=command))
jobs = [j for j in all_jobs if j.status in ['queued', 'running']]
errored_jobs = [(j.id, j.log.msg) for j in all_jobs
if j.status in ['error']]
if jobs:
# There is already a job generating the HTML. Also, there should be
# at most one job, because we are not allowing here to start more
# than one
job = jobs[0]
job_info = [job.id, job.status, job.step]
# Check if the artifact is editable by the given user
study = artifact.study
analysis = artifact.analysis
editable = study.can_edit(user) if study else analysis.can_edit(user)
buttons = []
btn_base = (
'<button onclick="if (confirm(\'Are you sure you want to %s '
'artifact id: {0}?\')) {{ set_artifact_visibility(\'%s\', {0}) }}" '
'class="btn btn-primary btn-sm">%s</button>').format(artifact_id)
if analysis:
# If the artifact is part of an analysis, we don't require admin
# approval, and the artifact can be made public only if all the
# artifacts used to create the initial artifact set are public
if analysis.can_be_publicized and visibility != 'public':
buttons.append(btn_base % ('make public', 'public', 'Make public'))
else:
# If the artifact is part of a study, the buttons shown depend in
# multiple factors (see each if statement for an explanation of those)
if qiita_config.require_approval:
if visibility == 'sandbox':
# The request approval button only appears if the artifact is
# sandboxed and the qiita_config specifies that the approval
# should be requested
buttons.append(btn_base %
('request approval for', 'awaiting_approval',
'Request approval'))
elif user.level == 'admin' and visibility == 'awaiting_approval':
# The approve artifact button only appears if the user is an
# admin the artifact is waiting to be approvaed and the qiita
# config requires artifact approval
buttons.append(btn_base %
('approve', 'private', 'Approve artifact'))
if visibility == 'private':
# The make public button only appears if the artifact is private
buttons.append(btn_base % ('make public', 'public', 'Make public'))
# The revert to sandbox button only appears if the artifact is not
# sandboxed nor public
if visibility not in {'sandbox', 'public'}:
buttons.append(
btn_base %
('revert to sandbox', 'sandbox', 'Revert to sandbox'))
if user.level == 'admin':
if artifact.can_be_submitted_to_ebi:
buttons.append(
'<a class="btn btn-primary btn-sm" '
'href="/ebi_submission/%d">'
'<span class="glyphicon glyphicon-export"></span>'
' Submit to EBI</a>' % artifact_id)
if artifact.can_be_submitted_to_vamps:
if not artifact.is_submitted_to_vamps:
buttons.append(
'<a class="btn btn-primary btn-sm" href="/vamps/%d">'
'<span class="glyphicon glyphicon-export"></span>'
' Submit to VAMPS</a>' % artifact_id)
files = [(f_id, "%s (%s)" % (basename(fp), f_type.replace('_', ' ')))
for f_id, fp, f_type in artifact.filepaths
if f_type != 'directory']
# TODO: https://github.com/biocore/qiita/issues/1724 Remove this hardcoded
# values to actually get the information from the database once it stores
# the information
if artifact_type in [
'SFF', 'FASTQ', 'FASTA', 'FASTA_Sanger', 'per_sample_FASTQ'
]:
# If the artifact is one of the "raw" types, only the owner of the
# study and users that has been shared with can see the files
if not artifact.study.has_access(user, no_public=True):
files = []
processing_parameters = (artifact.processing_parameters.values
if artifact.processing_parameters is not None else
{})
return {
'name': artifact.name,
'artifact_id': artifact_id,
'artifact_type': artifact_type,
'visibility': visibility,
'editable': editable,
'buttons': ' '.join(buttons),
'processing_parameters': processing_parameters,
'files': files,
'is_from_analysis': artifact.analysis is not None,
'processing_jobs': processing_jobs,
'summary': summary,
'job': job_info,
'artifact_timestamp': artifact.timestamp.strftime("%Y-%m-%d %H:%m"),
'errored_jobs': errored_jobs
}
def artifact_summary_get_request(user, artifact_id):
"""Returns the information for the artifact summary page
Parameters
----------
user : qiita_db.user.User
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the artifact summary information
{'name': str,
'artifact_id': int,
'visibility': str,
'editable': bool,
'buttons': str,
'processing_parameters': dict of {str: object},
'files': list of (int, str),
'is_from_analysis': bool,
'summary': str or None,
'job': [str, str, str],
'errored_jobs': list of [str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
artifact_type = artifact.artifact_type
check_artifact_access(user, artifact)
visibility = artifact.visibility
summary = artifact.html_summary_fp
job_info = None
errored_summary_jobs = []
# Check if the HTML summary exists
if summary:
# Magic number 1: If the artifact has a summary, the call
# artifact.html_summary_fp returns a tuple with 2 elements. The first
# element is the filepath id, while the second one is the actual
# actual filepath. We are only interested on the actual filepath,
# hence the 1 value.
summary = relpath(summary[1], qiita_config.base_data_dir)
else:
# Check if the summary is being generated
command = Command.get_html_generator(artifact_type)
all_jobs = set(artifact.jobs(cmd=command))
jobs = []
errored_summary_jobs = []
for j in all_jobs:
if j.status in ['queued', 'running']:
jobs.append(j)
elif j.status in ['error']:
errored_summary_jobs.append(j)
if jobs:
# There is already a job generating the HTML. Also, there should be
# at most one job, because we are not allowing here to start more
# than one
job = jobs[0]
job_info = [job.id, job.status, job.step]
# Check if the artifact is editable by the given user
study = artifact.study
analysis = artifact.analysis
if artifact_type == 'job-output-folder':
editable = False
else:
editable = study.can_edit(user) if study else analysis.can_edit(user)
buttons = []
btn_base = (
'<button onclick="if (confirm(\'Are you sure you want to %s '
'artifact id: {0}?\')) {{ set_artifact_visibility(\'%s\', {0}) }}" '
'class="btn btn-primary btn-sm">%s</button>').format(artifact_id)
if not analysis and artifact_type != 'job-output-folder':
# If the artifact is part of a study, the buttons shown depend in
# multiple factors (see each if statement for an explanation of those)
if qiita_config.require_approval:
if visibility == 'sandbox' and artifact.parents:
# The request approval button only appears if the artifact is
# sandboxed and the qiita_config specifies that the approval
# should be requested
buttons.append(btn_base %
('request approval for', 'awaiting_approval',
'Request approval'))
elif user.level == 'admin' and visibility == 'awaiting_approval':
# The approve artifact button only appears if the user is an
# admin the artifact is waiting to be approvaed and the qiita
# config requires artifact approval
buttons.append(btn_base %
('approve', 'private', 'Approve artifact'))
if visibility == 'private':
# The make public button only appears if the artifact is private
buttons.append(btn_base % ('make public', 'public', 'Make public'))
# The revert to sandbox button only appears if the artifact is not
# sandboxed nor public
if visibility not in {'sandbox', 'public'}:
buttons.append(
btn_base %
('revert to sandbox', 'sandbox', 'Revert to sandbox'))
if user.level == 'admin' and not study.autoloaded:
if artifact.can_be_submitted_to_ebi:
buttons.append(
'<a class="btn btn-primary btn-sm" '
'href="/ebi_submission/%d">'
'<span class="glyphicon glyphicon-export"></span>'
' Submit to EBI</a>' % artifact_id)
if artifact.can_be_submitted_to_vamps:
if not artifact.is_submitted_to_vamps:
buttons.append(
'<a class="btn btn-primary btn-sm" href="/vamps/%d">'
'<span class="glyphicon glyphicon-export"></span>'
' Submit to VAMPS</a>' % artifact_id)
if visibility != 'public':
# Have no fear, this is just python to generate html with an onclick in
# javascript that makes an ajax call to a separate url, takes the
# response and writes it to the newly uncollapsed div. Do note that
# you have to be REALLY CAREFUL with properly escaping quotation marks.
private_download = (
'<button class="btn btn-primary btn-sm" type="button" '
'aria-expanded="false" aria-controls="privateDownloadLink" '
'onclick="generate_private_download_link(%d)">Generate '
'Download Link</button><div class="collapse" '
'id="privateDownloadLink"><div class="card card-body" '
'id="privateDownloadText">Generating Download Link...'
'</div></div>') % artifact_id
buttons.append(private_download)
files = [(x['fp_id'],
"%s (%s)" % (basename(x['fp']), x['fp_type'].replace('_', ' ')),
x['checksum'], naturalsize(x['fp_size'], gnu=True))
for x in artifact.filepaths if x['fp_type'] != 'directory']
# TODO: https://github.com/biocore/qiita/issues/1724 Remove this hardcoded
# values to actually get the information from the database once it stores
# the information
if artifact_type in [
'SFF', 'FASTQ', 'FASTA', 'FASTA_Sanger', 'per_sample_FASTQ'
]:
# If the artifact is one of the "raw" types, only the owner of the
# study and users that has been shared with can see the files
study = artifact.study
has_access = study.has_access(user, no_public=True)
if (not study.public_raw_download and not has_access):
files = []
proc_params = artifact.processing_parameters
if proc_params:
cmd = proc_params.command
sw = cmd.software
processing_info = {
'command': cmd.name,
'software': sw.name,
'software_version': sw.version,
'processing_parameters': proc_params.values,
'command_active': cmd.active,
'software_deprecated': sw.deprecated,
}
else:
processing_info = {}
return {
'name': artifact.name,
'artifact_id': artifact_id,
'artifact_type': artifact_type,
'visibility': visibility,
'editable': editable,
'buttons': ' '.join(buttons),
'processing_info': processing_info,
'files': files,
'is_from_analysis': artifact.analysis is not None,
'summary': summary,
'job': job_info,
'artifact_timestamp': artifact.timestamp.strftime("%Y-%m-%d %H:%m"),
'being_deleted': artifact.being_deleted_by is not None,
'errored_summary_jobs': errored_summary_jobs
}
def test_download_study(self):
tmp_dir = mkdtemp()
self._clean_up_files.append(tmp_dir)
biom_fp = join(tmp_dir, 'otu_table.biom')
smr_dir = join(tmp_dir, 'sortmerna_picked_otus')
log_dir = join(smr_dir, 'seqs_otus.log')
tgz = join(tmp_dir, 'sortmerna_picked_otus.tgz')
with biom_open(biom_fp, 'w') as f:
et.to_hdf5(f, "test")
makedirs(smr_dir)
with open(log_dir, 'w') as f:
f.write('\n')
with open(tgz, 'w') as f:
f.write('\n')
files_biom = [(biom_fp, 'biom'), (smr_dir, 'directory'), (tgz, 'tgz')]
params = Parameters.from_default_params(
Command(3).default_parameter_sets.next(), {'input_data': 1})
a = Artifact.create(files_biom,
"BIOM",
parents=[Artifact(2)],
processing_parameters=params)
for _, fp, _ in a.filepaths:
self._clean_up_files.append(fp)
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 200)
exp = (
'- 1256812 /protected/processed_data/1_study_1001_closed_'
'reference_otu_table.biom processed_data/1_study_1001_closed_'
'reference_otu_table.biom\n'
'- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-'
'[0-9]*.txt mapping_files/4_mapping_file.txt\n'
'- 1256812 /protected/processed_data/'
'1_study_1001_closed_reference_otu_table.biom processed_data/'
'1_study_1001_closed_reference_otu_table.biom\n'
'- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-'
'[0-9]*.txt mapping_files/5_mapping_file.txt\n'
'- 1256812 /protected/processed_data/'
'1_study_1001_closed_reference_otu_table_Silva.biom processed_data'
'/1_study_1001_closed_reference_otu_table_Silva.biom\n'
'- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-'
'[0-9]*.txt mapping_files/6_mapping_file.txt\n'
'- 36615 /protected/templates/1_prep_2_qiime_[0-9]*-'
'[0-9]*.txt mapping_files/7_mapping_file.txt\n'
'- 39752 /protected/BIOM/{0}/otu_table.biom '
'BIOM/{0}/otu_table.biom\n'
'- 1 /protected/BIOM/{0}/sortmerna_picked_otus/seqs_otus.log '
'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n'
'- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]*.'
'txt mapping_files/{0}_mapping_file.txt\n'.format(a.id))
self.assertRegexpMatches(response.body, exp)
response = self.get('/download_study_bioms/200')
self.assertEqual(response.code, 405)
# changing user so we can test the failures
BaseHandler.get_current_user = Mock(
return_value=User("*****@*****.**"))
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 405)
a.visibility = 'public'
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 200)
exp = ('- 39752 /protected/BIOM/{0}/otu_table.biom '
'BIOM/{0}/otu_table.biom\n'
'- 1 /protected/BIOM/{0}/sortmerna_picked_otus/seqs_otus.log '
'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n'
'- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]*.'
'txt mapping_files/{0}_mapping_file.txt\n'.format(a.id))
self.assertRegexpMatches(response.body, exp)
def create_command(software, name, description, parameters, outputs=None,
analysis_only=False):
r"""Replicates the Command.create code at the time the patch was written"""
# Perform some sanity checks in the parameters dictionary
if not parameters:
raise QiitaDBError(
"Error creating command %s. At least one parameter should "
"be provided." % name)
sql_param_values = []
sql_artifact_params = []
for pname, vals in parameters.items():
if len(vals) != 2:
raise QiitaDBError(
"Malformed parameters dictionary, the format should be "
"{param_name: [parameter_type, default]}. Found: "
"%s for parameter name %s" % (vals, pname))
ptype, dflt = vals
# Check that the type is one of the supported types
supported_types = ['string', 'integer', 'float', 'reference',
'boolean', 'prep_template', 'analysis']
if ptype not in supported_types and not ptype.startswith(
('choice', 'mchoice', 'artifact')):
supported_types.extend(['choice', 'mchoice', 'artifact'])
raise QiitaDBError(
"Unsupported parameters type '%s' for parameter %s. "
"Supported types are: %s"
% (ptype, pname, ', '.join(supported_types)))
if ptype.startswith(('choice', 'mchoice')) and dflt is not None:
choices = set(loads(ptype.split(':')[1]))
dflt_val = dflt
if ptype.startswith('choice'):
# In the choice case, the dflt value is a single string,
# create a list with it the string on it to use the
# issuperset call below
dflt_val = [dflt_val]
else:
# jsonize the list to store it in the DB
dflt = dumps(dflt)
if not choices.issuperset(dflt_val):
raise QiitaDBError(
"The default value '%s' for the parameter %s is not "
"listed in the available choices: %s"
% (dflt, pname, ', '.join(choices)))
if ptype.startswith('artifact'):
atypes = loads(ptype.split(':')[1])
sql_artifact_params.append(
[pname, 'artifact', atypes])
else:
if dflt is not None:
sql_param_values.append([pname, ptype, False, dflt])
else:
sql_param_values.append([pname, ptype, True, None])
with TRN:
sql = """SELECT EXISTS(SELECT *
FROM qiita.software_command
WHERE software_id = %s AND name = %s)"""
TRN.add(sql, [software.id, name])
if TRN.execute_fetchlast():
raise QiitaDBDuplicateError(
"command", "software: %d, name: %s"
% (software.id, name))
# Add the command to the DB
sql = """INSERT INTO qiita.software_command
(name, software_id, description, is_analysis)
VALUES (%s, %s, %s, %s)
RETURNING command_id"""
sql_params = [name, software.id, description, analysis_only]
TRN.add(sql, sql_params)
c_id = TRN.execute_fetchlast()
# Add the parameters to the DB
sql = """INSERT INTO qiita.command_parameter
(command_id, parameter_name, parameter_type, required,
default_value)
VALUES (%s, %s, %s, %s, %s)
RETURNING command_parameter_id"""
sql_params = [[c_id, pname, p_type, reqd, default]
for pname, p_type, reqd, default in sql_param_values]
TRN.add(sql, sql_params, many=True)
TRN.execute()
# Add the artifact parameters
sql_type = """INSERT INTO qiita.parameter_artifact_type
(command_parameter_id, artifact_type_id)
VALUES (%s, %s)"""
supported_types = []
for pname, p_type, atypes in sql_artifact_params:
sql_params = [c_id, pname, p_type, True, None]
TRN.add(sql, sql_params)
pid = TRN.execute_fetchlast()
sql_params = [[pid, convert_to_id(at, 'artifact_type')]
for at in atypes]
TRN.add(sql_type, sql_params, many=True)
supported_types.extend([atid for _, atid in sql_params])
# If the software type is 'artifact definition', there are a couple
# of extra steps
if software.type == 'artifact definition':
# If supported types is not empty, link the software with these
# types
if supported_types:
sql = """INSERT INTO qiita.software_artifact_type
(software_id, artifact_type_id)
VALUES (%s, %s)"""
sql_params = [[software.id, atid]
for atid in supported_types]
TRN.add(sql, sql_params, many=True)
# If this is the validate command, we need to add the
# provenance and name parameters. These are used internally,
# that's why we are adding them here
if name == 'Validate':
sql = """INSERT INTO qiita.command_parameter
(command_id, parameter_name, parameter_type,
required, default_value)
VALUES (%s, 'name', 'string', 'False',
'dflt_name'),
(%s, 'provenance', 'string', 'False', NULL)
"""
TRN.add(sql, [c_id, c_id])
# Add the outputs to the command
if outputs:
sql = """INSERT INTO qiita.command_output
(name, command_id, artifact_type_id)
VALUES (%s, %s, %s)"""
sql_args = [[pname, c_id, convert_to_id(at, 'artifact_type')]
for pname, at in outputs.items()]
TRN.add(sql, sql_args, many=True)
TRN.execute()
return Command(c_id)
def artifact_post_req(user_id, filepaths, artifact_type, name,
prep_template_id, artifact_id=None):
"""Creates the initial artifact for the prep template
Parameters
----------
user_id : str
User adding the atrifact
filepaths : dict of str
Comma-separated list of files to attach to the artifact,
keyed by file type
artifact_type : str
The type of the artifact
name : str
Name to give the artifact
prep_template_id : int or str castable to int
Prep template to attach the artifact to
artifact_id : int or str castable to int, optional
The id of the imported artifact
Returns
-------
dict of objects
A dictionary containing the new artifact ID
{'status': status,
'message': message,
'artifact': id}
"""
prep_template_id = int(prep_template_id)
prep = PrepTemplate(prep_template_id)
study_id = prep.study_id
# First check if the user has access to the study
access_error = check_access(study_id, user_id)
if access_error:
return access_error
if artifact_id:
# if the artifact id has been provided, import the artifact
job_id = safe_submit(user_id, copy_raw_data, prep, artifact_id)
is_qiita_job = False
else:
uploads_path = get_mountpoint('uploads')[0][1]
path_builder = partial(join, uploads_path, str(study_id))
cleaned_filepaths = {}
for ftype, file_list in viewitems(filepaths):
# JavaScript sends us this list as a comma-separated list
for fp in file_list.split(','):
# JavaScript will send this value as an empty string if the
# list of files was empty. In such case, the split will
# generate a single element containing the empty string. Check
# for that case here and, if fp is not the empty string,
# proceed to check if the file exists
if fp:
# Check if filepath being passed exists for study
full_fp = path_builder(fp)
exists = check_fp(study_id, full_fp)
if exists['status'] != 'success':
return {'status': 'error',
'message': 'File does not exist: %s' % fp}
if ftype not in cleaned_filepaths:
cleaned_filepaths[ftype] = []
cleaned_filepaths[ftype].append(full_fp)
# This should never happen, but it doesn't hurt to actually have
# a explicit check, in case there is something odd with the JS
if not cleaned_filepaths:
return {'status': 'error',
'message': "Can't create artifact, no files provided."}
command = Command.get_validator(artifact_type)
job = ProcessingJob.create(
User(user_id),
Parameters.load(command, values_dict={
'template': prep_template_id,
'files': dumps(cleaned_filepaths),
'artifact_type': artifact_type
}))
job.submit()
job_id = job.id
is_qiita_job = True
r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep.id,
dumps({'job_id': job_id, 'is_qiita_job': is_qiita_job}))
return {'status': 'success', 'message': ''}
def test_job_ajax_patch_req(self):
# Create a new job - through a workflow since that is the only way
# of creating jobs in the interface
exp_command = Command(1)
json_str = (
'{"input_data": 1, "max_barcode_errors": 1.5, '
'"barcode_type": "golay_12", "max_bad_run_length": 3, '
'"rev_comp": false, "phred_quality_threshold": 3, '
'"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
'"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
exp_params = Parameters.load(exp_command, json_str=json_str)
exp_user = User('*****@*****.**')
name = "Test processing workflow"
# tests success
wf = ProcessingWorkflow.from_scratch(exp_user,
exp_params,
name=name,
force=True)
graph = wf.graph
nodes = list(graph.nodes())
job_id = nodes[0].id
# Incorrect path parameter
obs = job_ajax_patch_req('remove', '/%s/somethingelse' % job_id)
exp = {
'status': 'error',
'message': 'Incorrect path parameter: missing job id'
}
self.assertEqual(obs, exp)
obs = job_ajax_patch_req('remove', '/')
exp = {
'status': 'error',
'message': 'Incorrect path parameter: missing job id'
}
self.assertEqual(obs, exp)
# Job id is not like a job id
obs = job_ajax_patch_req('remove', '/notAJobId')
exp = {
'status':
'error',
'message':
'Incorrect path parameter: '
'notAJobId is not a recognized job id'
}
self.assertEqual(obs, exp)
# Job doesn't exist
obs = job_ajax_patch_req('remove',
'/6d368e16-2242-4cf8-87b4-a5dc40bc890b')
exp = {
'status':
'error',
'message':
'Incorrect path parameter: '
'6d368e16-2242-4cf8-87b4-a5dc40bc890b is not a '
'recognized job id'
}
self.assertEqual(obs, exp)
# in_construction job
obs = job_ajax_patch_req('remove', '/%s' % job_id)
exp = {
'status':
'error',
'message':
"Can't delete job %s. It is 'in_construction' "
"status. Please use /study/process/workflow/" % job_id
}
self.assertEqual(obs, exp)
# job status != 'error'
job = ProcessingJob(job_id)
job._set_status('queued')
obs = job_ajax_patch_req('remove', '/%s' % job_id)
exp = {
'status': 'error',
'message': 'Only jobs in "error" status can be deleted.'
}
self.assertEqual(obs, exp)
# Operation not supported
job._set_status('queued')
obs = job_ajax_patch_req('add', '/%s' % job_id)
exp = {
'status':
'error',
'message':
'Operation "add" not supported. Current supported '
'operations: remove'
}
self.assertEqual(obs, exp)
# Test success
job._set_error('Killed for testing')
obs = job_ajax_patch_req('remove', '/%s' % job_id)
exp = {'status': 'success', 'message': ''}
self.assertEqual(obs, exp)
def test_artifact_summary_get_request(self):
# Artifact w/o summary
obs = artifact_summary_get_request('*****@*****.**', 1)
exp_p_jobs = [[
'063e553b-327c-4818-ab4a-adfe58e49860', 'Split libraries FASTQ',
'queued', None, None
],
[
'bcc7ebcd-39c1-43e4-af2d-822e3589f14d',
'Split libraries', 'running', 'demultiplexing', None
]]
exp_files = [
(1L, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)'),
(2L, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)')
]
exp = {
'status':
'success',
'message':
'',
'name':
'Raw data 1',
'summary':
None,
'job':
None,
'processing_jobs':
exp_p_jobs,
'errored_jobs': [],
'visibility':
'private',
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'files':
exp_files,
'editable':
True,
'prep_id':
1,
'study_id':
1
}
self.assertEqual(obs, exp)
# Artifact with summary being generated
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 1}))
job._set_status('queued')
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {
'status':
'success',
'message':
'',
'name':
'Raw data 1',
'summary':
None,
'job': [job.id, 'queued', None],
'processing_jobs':
exp_p_jobs,
'errored_jobs': [],
'visibility':
'private',
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'files':
exp_files,
'editable':
True,
'prep_id':
1,
'study_id':
1
}
self.assertEqual(obs, exp)
# Artifact with summary
fd, fp = mkstemp(suffix=".html")
close(fd)
with open(fp, 'w') as f:
f.write('<b>HTML TEST - not important</b>\n')
a = Artifact(1)
a.html_summary_fp = fp
self._files_to_remove.extend([fp, a.html_summary_fp[1]])
exp_files.append(
(a.html_summary_fp[0],
'%s (html summary)' % basename(a.html_summary_fp[1])))
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {
'status':
'success',
'message':
'',
'name':
'Raw data 1',
'summary':
'<b>HTML TEST - not important</b>\n',
'job':
None,
'processing_jobs':
exp_p_jobs,
'errored_jobs': [],
'visibility':
'private',
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'files':
exp_files,
'editable':
True,
'prep_id':
1,
'study_id':
1
}
self.assertEqual(obs, exp)
# No access
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {
'status': 'error',
'message': 'User does not have access to study'
}
self.assertEqual(obs, exp)
# A non-owner/share user can't see the files
a.visibility = 'public'
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {
'status': 'success',
'message': '',
'name': 'Raw data 1',
'summary': '<b>HTML TEST - not important</b>\n',
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'public',
'buttons': '',
'files': [],
'editable': False,
'prep_id': 1,
'study_id': 1
}
self.assertEqual(obs, exp)
# returnig to private
a.visibility = 'sandbox'
# admin gets buttons
obs = artifact_summary_get_request('*****@*****.**', 2)
exp_p_jobs = [[
'd19f76ee-274e-4c1b-b3a2-a12d73507c55',
'Pick closed-reference OTUs', 'error', 'generating demux file',
'Error message'
]]
exp_files = [(3L, '1_seqs.fna (preprocessed fasta)'),
(4L, '1_seqs.qual (preprocessed fastq)'),
(5L, '1_seqs.demux (preprocessed demux)')]
exp = {
'status':
'success',
'files':
exp_files,
'errored_jobs': [],
'editable':
True,
'visibility':
'private',
'job':
None,
'message':
'',
'name':
'Demultiplexed 1',
'processing_jobs':
exp_p_jobs,
'summary':
None,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 2?\')) { '
'set_artifact_visibility(\'public\', 2) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'2?\')) { set_artifact_visibility(\'sandbox\', 2) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button> <a class="btn btn-primary '
'btn-sm" href="/vamps/2"><span class="glyphicon '
'glyphicon-export"></span> Submit to VAMPS</a>'),
'study_id':
1,
'prep_id':
1
}
self.assertEqual(obs, exp)
def test_complete_job(self):
# Complete success
pt = npt.assert_warns(
QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {'1.SKD6.640190': 1}}),
Study(1), '16S')
c_job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(
Command.get_validator('BIOM'),
values_dict={'template': pt.id,
'files': dumps({'BIOM': ['file']}),
'artifact_type': 'BIOM'}), True)
c_job._set_status('running')
fd, fp = mkstemp(suffix='_table.biom')
close(fd)
with open(fp, 'w') as f:
f.write('\n')
self._clean_up_files.append(fp)
exp_artifact_count = get_count('qiita.artifact') + 1
payload = dumps(
{'success': True, 'error': '',
'artifacts': {'OTU table': {'filepaths': [(fp, 'biom')],
'artifact_type': 'BIOM'}}})
job = self._create_job('complete_job', {'job_id': c_job.id,
'payload': payload})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertEqual(c_job.status, 'success')
self.assertEqual(get_count('qiita.artifact'), exp_artifact_count)
# Complete job error
payload = dumps({'success': False, 'error': 'Job failure'})
job = self._create_job(
'complete_job', {'job_id': 'bcc7ebcd-39c1-43e4-af2d-822e3589f14d',
'payload': payload})
private_task(job.id)
self.assertEqual(job.status, 'success')
c_job = ProcessingJob('bcc7ebcd-39c1-43e4-af2d-822e3589f14d')
self.assertEqual(c_job.status, 'error')
self.assertEqual(c_job.log, LogEntry.newest_records(numrecords=1)[0])
self.assertEqual(c_job.log.msg, 'Job failure')
# Complete internal error
pt = npt.assert_warns(
QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {'1.SKD6.640190': 1}}),
Study(1), '16S')
c_job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(
Command.get_validator('BIOM'),
values_dict={'template': pt.id,
'files': dumps({'BIOM': ['file']}),
'artifact_type': 'BIOM'}), True)
c_job._set_status('running')
fp = '/surprised/if/this/path/exists.biom'
payload = dumps(
{'success': True, 'error': '',
'artifacts': {'OTU table': {'filepaths': [(fp, 'biom')],
'artifact_type': 'BIOM'}}})
job = self._create_job('complete_job', {'job_id': c_job.id,
'payload': payload})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertEqual(c_job.status, 'error')
self.assertIn('No such file or directory', c_job.log.msg)
def test_complete_job(self):
# Complete success
pt = npt.assert_warns(QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {
'1.SKD6.640190': 1
}}), Study(1), '16S')
c_job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command.get_validator('BIOM'),
values_dict={
'template': pt.id,
'files': dumps({'BIOM': ['file']}),
'artifact_type': 'BIOM'
}), True)
c_job._set_status('running')
fd, fp = mkstemp(suffix='_table.biom')
close(fd)
with open(fp, 'w') as f:
f.write('\n')
self._clean_up_files.append(fp)
exp_artifact_count = get_count('qiita.artifact') + 1
payload = dumps({
'success': True,
'error': '',
'artifacts': {
'OTU table': {
'filepaths': [(fp, 'biom')],
'artifact_type': 'BIOM'
}
}
})
job = self._create_job('complete_job', {
'job_id': c_job.id,
'payload': payload
})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertEqual(c_job.status, 'success')
self.assertEqual(get_count('qiita.artifact'), exp_artifact_count)
# Complete job error
payload = dumps({'success': False, 'error': 'Job failure'})
job = self._create_job('complete_job', {
'job_id': 'bcc7ebcd-39c1-43e4-af2d-822e3589f14d',
'payload': payload
})
private_task(job.id)
self.assertEqual(job.status, 'success')
c_job = ProcessingJob('bcc7ebcd-39c1-43e4-af2d-822e3589f14d')
self.assertEqual(c_job.status, 'error')
self.assertEqual(c_job.log, LogEntry.newest_records(numrecords=1)[0])
self.assertEqual(c_job.log.msg, 'Job failure')
# Complete internal error
pt = npt.assert_warns(QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {
'1.SKD6.640190': 1
}}), Study(1), '16S')
c_job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command.get_validator('BIOM'),
values_dict={
'template': pt.id,
'files': dumps({'BIOM': ['file']}),
'artifact_type': 'BIOM'
}), True)
c_job._set_status('running')
fp = '/surprised/if/this/path/exists.biom'
payload = dumps({
'success': True,
'error': '',
'artifacts': {
'OTU table': {
'filepaths': [(fp, 'biom')],
'artifact_type': 'BIOM'
}
}
})
job = self._create_job('complete_job', {
'job_id': c_job.id,
'payload': payload
})
private_task(job.id)
self.assertEqual(job.status, 'success')
self.assertEqual(c_job.status, 'error')
self.assertIn('No such file or directory', c_job.log.msg)
def test_get_analysis_graph_handler(self):
response = self.get('/analysis/description/1/graph/')
self.assertEqual(response.code, 200)
# The job id is randomly generated in the test environment. Gather
# it here. There is only 1 job in the first artifact of the analysis
job_id = Analysis(1).artifacts[0].jobs()[0].id
obs = loads(response.body)
exp = {
'edges': [[8, job_id], [job_id, 9]],
'nodes': [['job', 'job', job_id, 'Single Rarefaction', 'success'],
['artifact', 'BIOM', 9, 'noname\n(BIOM)', 'artifact'],
['artifact', 'BIOM', 8, 'noname\n(BIOM)', 'artifact']],
'workflow':
None
}
self.assertItemsEqual(obs, exp)
self.assertItemsEqual(obs['edges'], exp['edges'])
self.assertItemsEqual(obs['nodes'], exp['nodes'])
self.assertIsNone(obs['workflow'])
# Create a new analysis with 2 starting BIOMs to be able to test
# the different if statements of the request
BaseHandler.get_current_user = Mock(
return_value=User('*****@*****.**'))
user = User('*****@*****.**')
dflt_analysis = user.default_analysis
dflt_analysis.add_samples({
4: ['1.SKB8.640193', '1.SKD8.640184', '1.SKB7.640196'],
6: ['1.SKB8.640193', '1.SKD8.640184', '1.SKB7.640196']
})
args = {'name': 'New Test Graph Analysis', 'description': 'Desc'}
response = self.post('/analysis/create/', args)
new_id = response.effective_url.split('/')[-2]
a = Analysis(new_id)
# Wait until all the jobs are done so the BIOM tables exist
for j in a.jobs:
wait_for_processing_job(j.id)
artifacts = a.artifacts
self.assertEqual(len(artifacts), 2)
# Create a new workflow starting on the first artifact
# Magic number 9 -> Summarize Taxa command
params = Parameters.load(Command(9),
values_dict={
'metadata_category': 'None',
'sort': 'False',
'biom_table': artifacts[0].id
})
wf = ProcessingWorkflow.from_scratch(user, params)
# There is only one job in the workflow
job_id = wf.graph.nodes()[0].id
response = self.get('/analysis/description/%s/graph/' % new_id)
self.assertEqual(response.code, 200)
obs = loads(response.body)
exp = {
'edges': [[artifacts[0].id, job_id],
[job_id, '%s:taxa_summary' % job_id]],
'nodes':
[['job', 'job', job_id, 'Summarize Taxa', 'in_construction'],
[
'artifact', 'BIOM', artifacts[0].id, 'noname\n(BIOM)',
'artifact'
],
[
'artifact', 'BIOM', artifacts[1].id, 'noname\n(BIOM)',
'artifact'
],
[
'type', 'taxa_summary',
'%s:taxa_summary' % job_id, 'taxa_summary\n(taxa_summary)',
'type'
]],
'workflow':
wf.id
}
# Check that the keys are the same
self.assertItemsEqual(obs, exp)
# Check the edges
self.assertItemsEqual(obs['edges'], exp['edges'])
# Check the edges
self.assertItemsEqual(obs['nodes'], exp['nodes'])
# Check the edges
self.assertEqual(obs['workflow'], exp['workflow'])
# Add a job to the second BIOM to make sure that the edges and nodes
# are respected. Magic number 12 -> Single Rarefaction
job2 = wf.add(DefaultParameters(16),
req_params={
'depth': '100',
'biom_table': artifacts[1].id
})
job_id_2 = job2.id
response = self.get('/analysis/description/%s/graph/' % new_id)
self.assertEqual(response.code, 200)
obs = loads(response.body)
exp = {
'edges': [[artifacts[0].id, job_id],
[job_id, '%s:taxa_summary' % job_id],
[artifacts[1].id, job_id_2],
[job_id_2, '%s:rarefied_table' % job_id_2]],
'nodes':
[['job', 'job', job_id, 'Summarize Taxa', 'in_construction'],
['job', 'job', job_id_2, 'Single Rarefaction', 'in_construction'],
[
'artifact', 'BIOM', artifacts[0].id, 'noname\n(BIOM)',
'artifact'
],
[
'artifact', 'BIOM', artifacts[1].id, 'noname\n(BIOM)',
'artifact'
],
[
'type', 'taxa_summary',
'%s:taxa_summary' % job_id, 'taxa_summary\n(taxa_summary)',
'type'
],
[
'type', 'BIOM',
'%s:rarefied_table' % job_id_2, 'rarefied_table\n(BIOM)',
'type'
]],
'workflow':
wf.id
}
# Check that the keys are the same
self.assertItemsEqual(obs, exp)
# Check the edges
self.assertItemsEqual(obs['edges'], exp['edges'])
# Check the edges
self.assertItemsEqual(obs['nodes'], exp['nodes'])
# Check the edges
self.assertEqual(obs['workflow'], exp['workflow'])
# Add a second Workflow to the second artifact to force the raise of
# the error. This situation should never happen when using
# the interface
wf.remove(job2)
params = Parameters.load(Command(9),
values_dict={
'metadata_category': 'None',
'sort': 'False',
'biom_table': artifacts[1].id
})
wf = ProcessingWorkflow.from_scratch(user, params)
response = self.get('/analysis/description/%s/graph/' % new_id)
self.assertEqual(response.code, 500)
def _construct_job_graph(self, analysis, commands, comm_opts=None,
rarefaction_depth=None,
merge_duplicated_sample_ids=False):
"""Builds the job graph for running an analysis
Parameters
----------
analysis: Analysis object
Analysis to finalize.
commands : list of tuples
Commands to add as jobs in the analysis.
Format [(data_type, command name), ...]
comm_opts : dict of dicts, optional
Options for commands. Format {command name: {opt1: value,...},...}
Default None (use default options).
rarefaction_depth : int, optional
Rarefaction depth for analysis' biom tables. Default None.
merge_duplicated_sample_ids : bool, optional
If the duplicated sample ids in the selected studies should be
merged or prepended with the artifact ids. False (default) prepends
the artifact id
"""
self._logger = stderr
self.analysis = analysis
analysis_id = analysis.id
# Add jobs to analysis
if comm_opts is None:
comm_opts = {}
analysis.status = "running"
# creating bioms at this point cause all this section runs on a worker
# node, currently an ipython job
analysis.build_files(rarefaction_depth, merge_duplicated_sample_ids)
mapping_file = analysis.mapping_file
tree_commands = ["Beta Diversity", "Alpha Rarefaction"]
for data_type, biom_fp in viewitems(analysis.biom_tables):
biom_table = load_table(biom_fp)
# getting reference_id and software_command_id from the first
# sample of the biom. This decision was discussed on the qiita
# meeting on 02/24/16
metadata = biom_table.metadata(biom_table.ids()[0])
rid = metadata['reference_id']
sci = metadata['command_id']
if rid != 'na':
reference = Reference(rid)
tree = reference.tree_fp
else:
reference = None
tree = ''
cmd = Command(sci) if sci != 'na' else None
for cmd_data_type, command in commands:
if data_type != cmd_data_type:
continue
# get opts set by user, else make it empty dict
opts = comm_opts.get(command, {})
opts["--otu_table_fp"] = biom_fp
opts["--mapping_fp"] = mapping_file
if command in tree_commands:
if tree != '':
opts["--tree_fp"] = tree
else:
opts["--parameter_fp"] = join(
get_db_files_base_dir(), "reference",
"params_qiime.txt")
if command == "Alpha Rarefaction":
opts["-n"] = 4
Job.create(data_type, command, opts, analysis, reference, cmd,
return_existing=True)
# Add the jobs
job_nodes = []
for job in analysis.jobs:
node_name = "%d_JOB_%d" % (analysis_id, job.id)
job_nodes.append(node_name)
job_name = "%s: %s" % (job.datatype, job.command[0])
self._job_graph.add_node(node_name,
func=system_call_from_job,
args=(job.id,),
job_name=job_name,
requires_deps=False)
# tgz-ing the analysis results
tgz_node_name = "TGZ_ANALYSIS_%d" % (analysis_id)
job_name = "tgz_analysis_%d" % (analysis_id)
self._job_graph.add_node(tgz_node_name,
func=_generate_analysis_tgz,
args=(analysis,),
job_name=job_name,
requires_deps=False)
# Adding the dependency edges to the graph
for job_node_name in job_nodes:
self._job_graph.add_edge(job_node_name, tgz_node_name)
# Finalize the analysis.
node_name = "FINISH_ANALYSIS_%d" % analysis.id
self._job_graph.add_node(node_name,
func=_finish_analysis,
args=(analysis,),
job_name='Finalize analysis',
requires_deps=False)
self._job_graph.add_edge(tgz_node_name, node_name)
def test_artifact_summary_get_request(self):
user = User('*****@*****.**')
main_buttons = (
'<button onclick="if (confirm('
"\'Are you sure you want to make "
"public artifact id: 1?')) { set_artifact_visibility('public', 1) "
'}" class="btn btn-primary btn-sm">Make public</button> <button '
'onclick="if (confirm('
"'Are you sure you want to revert to "
"sandbox artifact id: 1?')) { set_artifact_visibility('sandbox', 1"
') }" class="btn btn-primary btn-sm">Revert to sandbox</button> ')
private_download_button = (
'<button class="btn btn-primary btn-sm" type="button" '
'aria-expanded="false" aria-controls="privateDownloadLink" '
'onclick="generate_private_download_link(%s)">Generate Download '
'Link</button><div class="collapse" id="privateDownloadLink"><div '
'class="card card-body" id="privateDownloadText">Generating '
'Download Link...</div></div>')
# Artifact w/o summary
obs = artifact_summary_get_request(user, 1)
exp_files = [
(1, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)',
'2125826711', '58 Bytes'),
(2, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)',
'2125826711', '58 Bytes')
]
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'private',
'editable': True,
'buttons': main_buttons + private_download_button % 1,
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': None,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# Artifact with summary being generated
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 1}))
job._set_status('queued')
obs = artifact_summary_get_request(user, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'private',
'editable': True,
'buttons': main_buttons + private_download_button % 1,
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': None,
'job': [job.id, 'queued', None],
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# Artifact with summary
fd, fp = mkstemp(suffix=".html")
close(fd)
with open(fp, 'w') as f:
f.write('<b>HTML TEST - not important</b>\n')
a = Artifact(1)
a.set_html_summary(fp)
self._files_to_remove.extend([fp, a.html_summary_fp[1]])
exp_files.append((a.html_summary_fp[0],
'%s (html summary)' % basename(a.html_summary_fp[1]),
'1642196267', '33 Bytes'))
exp_summary_path = relpath(a.html_summary_fp[1],
qiita_config.base_data_dir)
obs = artifact_summary_get_request(user, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'private',
'editable': True,
'buttons': main_buttons + private_download_button % 1,
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': exp_summary_path,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# No access
demo_u = User('*****@*****.**')
with self.assertRaises(QiitaHTTPError):
obs = artifact_summary_get_request(demo_u, 1)
# A non-owner/share user can't see the files
a.visibility = 'public'
obs = artifact_summary_get_request(demo_u, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'public',
'editable': False,
'buttons': '',
'processing_info': {},
'files': [],
'is_from_analysis': False,
'summary': exp_summary_path,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# testing sandbox
a.visibility = 'sandbox'
obs = artifact_summary_get_request(user, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'sandbox',
'editable': True,
'buttons': private_download_button % 1,
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': exp_summary_path,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# returnig to private
a.visibility = 'private'
# admin gets buttons
obs = artifact_summary_get_request(User('*****@*****.**'), 2)
exp_files = [(3, '1_seqs.fna (preprocessed fasta)', '', '0 Bytes'),
(4, '1_seqs.qual (preprocessed fastq)', '', '0 Bytes'),
(5, '1_seqs.demux (preprocessed demux)', '', '0 Bytes')]
exp = {
'name':
'Demultiplexed 1',
'artifact_id':
2,
'artifact_type':
'Demultiplexed',
'artifact_timestamp':
'2012-10-01 10:10',
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 2?\')) { '
'set_artifact_visibility(\'public\', 2) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'2?\')) { set_artifact_visibility(\'sandbox\', 2) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button> <a class="btn btn-primary '
'btn-sm" href="/ebi_submission/2"><span '
'class="glyphicon glyphicon-export"></span> '
'Submit to EBI</a> <a class="btn btn-primary '
'btn-sm" href="/vamps/2"><span class="glyphicon '
'glyphicon-export"></span> Submit to VAMPS</a> ' +
private_download_button % 2),
'processing_info': {
'command_active': True,
'software_deprecated': False,
'command': 'Split libraries FASTQ',
'processing_parameters': {
'max_barcode_errors': '1.5',
'sequence_max_n': '0',
'max_bad_run_length': '3',
'phred_offset': 'auto',
'rev_comp': 'False',
'phred_quality_threshold': '3',
'input_data': '1',
'rev_comp_barcode': 'False',
'rev_comp_mapping_barcodes': 'False',
'min_per_read_length_fraction': '0.75',
'barcode_type': 'golay_12'
},
'software_version': '1.9.1',
'software': 'QIIME'
},
'files':
exp_files,
'is_from_analysis':
False,
'summary':
None,
'job':
None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# analysis artifact
obs = artifact_summary_get_request(user, 8)
exp = {
'name': 'noname',
'artifact_id': 8,
'artifact_type': 'BIOM',
# this value changes on build so copy from obs
'artifact_timestamp': obs['artifact_timestamp'],
'visibility': 'sandbox',
'editable': True,
'buttons': private_download_button % 8,
'processing_info': {},
'files': [(22, 'biom_table.biom (biom)', '1756512010', '1.1 MB')],
'is_from_analysis': True,
'summary': None,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
def test_artifact_summary_get_request(self):
user = User('*****@*****.**')
# Artifact w/o summary
obs = artifact_summary_get_request(user, 1)
exp_p_jobs = [[
'063e553b-327c-4818-ab4a-adfe58e49860', 'Split libraries FASTQ',
'queued', None, None
],
[
'bcc7ebcd-39c1-43e4-af2d-822e3589f14d',
'Split libraries', 'running', 'demultiplexing', None
]]
exp_files = [
(1L, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)'),
(2L, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)')
]
exp = {
'name':
'Raw data 1',
'artifact_id':
1,
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'processing_parameters': {},
'files':
exp_files,
'summary':
None,
'job':
None,
'processing_jobs':
exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# Artifact with summary being generated
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 1}))
job._set_status('queued')
obs = artifact_summary_get_request(user, 1)
exp = {
'name':
'Raw data 1',
'artifact_id':
1,
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'processing_parameters': {},
'files':
exp_files,
'summary':
None,
'job': [job.id, 'queued', None],
'processing_jobs':
exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# Artifact with summary
fd, fp = mkstemp(suffix=".html")
close(fd)
with open(fp, 'w') as f:
f.write('<b>HTML TEST - not important</b>\n')
a = Artifact(1)
a.set_html_summary(fp)
self._files_to_remove.extend([fp, a.html_summary_fp[1]])
exp_files.append(
(a.html_summary_fp[0],
'%s (html summary)' % basename(a.html_summary_fp[1])))
exp_summary_path = relpath(a.html_summary_fp[1],
qiita_config.base_data_dir)
obs = artifact_summary_get_request(user, 1)
exp = {
'name':
'Raw data 1',
'artifact_id':
1,
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'processing_parameters': {},
'files':
exp_files,
'summary':
exp_summary_path,
'job':
None,
'processing_jobs':
exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# No access
demo_u = User('*****@*****.**')
with self.assertRaises(QiitaHTTPError):
obs = artifact_summary_get_request(demo_u, 1)
# A non-owner/share user can't see the files
a.visibility = 'public'
obs = artifact_summary_get_request(demo_u, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'visibility': 'public',
'editable': False,
'buttons': '',
'processing_parameters': {},
'files': [],
'summary': exp_summary_path,
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# returnig to private
a.visibility = 'private'
# admin gets buttons
obs = artifact_summary_get_request(User('*****@*****.**'), 2)
exp_p_jobs = [[
'd19f76ee-274e-4c1b-b3a2-a12d73507c55',
'Pick closed-reference OTUs', 'error', 'generating demux file',
'Error message'
]]
exp_files = [(3L, '1_seqs.fna (preprocessed fasta)'),
(4L, '1_seqs.qual (preprocessed fastq)'),
(5L, '1_seqs.demux (preprocessed demux)')]
exp = {
'name':
'Demultiplexed 1',
'artifact_id':
2,
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 2?\')) { '
'set_artifact_visibility(\'public\', 2) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'2?\')) { set_artifact_visibility(\'sandbox\', 2) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button> <a class="btn btn-primary '
'btn-sm" href="/ebi_submission/2"><span '
'class="glyphicon glyphicon-export"></span> '
'Submit to EBI</a> <a class="btn btn-primary '
'btn-sm" href="/vamps/2"><span class="glyphicon '
'glyphicon-export"></span> Submit to VAMPS</a>'),
'processing_parameters': {
'max_barcode_errors': 1.5,
'sequence_max_n': 0,
'max_bad_run_length': 3,
'phred_offset': u'auto',
'rev_comp': False,
'phred_quality_threshold': 3,
'input_data': 1,
'rev_comp_barcode': False,
'rev_comp_mapping_barcodes': False,
'min_per_read_length_fraction': 0.75,
'barcode_type': u'golay_12'
},
'files':
exp_files,
'summary':
None,
'job':
None,
'processing_jobs':
exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# analysis artifact
obs = artifact_summary_get_request(user, 8)
exp = {
'name': 'noname',
'artifact_id': 8,
'visibility': 'sandbox',
'editable': True,
'buttons': '',
'processing_parameters': {},
'files': [(27, 'biom_table.biom (biom)')],
'summary': None,
'job': None,
'processing_jobs': [],
'errored_jobs': []
}
self.assertEqual(obs, exp)
def test_download_study(self):
tmp_dir = mkdtemp()
self._clean_up_files.append(tmp_dir)
biom_fp = join(tmp_dir, 'otu_table.biom')
smr_dir = join(tmp_dir, 'sortmerna_picked_otus')
log_dir = join(smr_dir, 'seqs_otus.log')
tgz = join(tmp_dir, 'sortmerna_picked_otus.tgz')
with biom_open(biom_fp, 'w') as f:
et.to_hdf5(f, "test")
makedirs(smr_dir)
with open(log_dir, 'w') as f:
f.write('\n')
with open(tgz, 'w') as f:
f.write('\n')
files_biom = [(biom_fp, 'biom'), (smr_dir, 'directory'), (tgz, 'tgz')]
params = Parameters.from_default_params(
next(Command(3).default_parameter_sets), {'input_data': 1})
a = Artifact.create(files_biom,
"BIOM",
parents=[Artifact(2)],
processing_parameters=params)
for x in a.filepaths:
self._clean_up_files.append(x['fp'])
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 200)
exp = ('- 1256812 /protected/processed_data/'
'1_study_1001_closed_reference_otu_table.biom processed_data/'
'1_study_1001_closed_reference_otu_table.biom\n'
'- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
'mapping_files/4_mapping_file.txt\n'
'- 1256812 /protected/processed_data/'
'1_study_1001_closed_reference_otu_table.biom processed_data/'
'1_study_1001_closed_reference_otu_table.biom\n'
'- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
'mapping_files/5_mapping_file.txt\n'
'- 1256812 /protected/processed_data/1_study_1001_'
'closed_reference_otu_table_Silva.biom processed_data/'
'1_study_1001_closed_reference_otu_table_Silva.biom\n'
'- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
'mapping_files/6_mapping_file.txt\n'
'- 1093210 /protected/BIOM/7/biom_table.biom '
'BIOM/7/biom_table.biom\n'
'- [0-9]* /protected/templates/1_prep_2_[0-9]*-[0-9]*.txt '
'mapping_files/7_mapping_file.txt\n'
'- [0-9]* /protected/BIOM/{0}/otu_table.biom '
'BIOM/{0}/otu_table.biom\n'
'- 1 /protected/BIOM/10/sortmerna_picked_otus/seqs_otus.log '
'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n'
'- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
'mapping_files/{0}_mapping_file.txt\n'.format(a.id))
self.assertRegex(response.body.decode('ascii'), exp)
response = self.get('/download_study_bioms/200')
self.assertEqual(response.code, 405)
# changing user so we can test the failures
BaseHandler.get_current_user = Mock(
return_value=User("*****@*****.**"))
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 405)
a.visibility = 'public'
response = self.get('/download_study_bioms/1')
# returning visibility
a.visibility = 'private'
self.assertEqual(response.code, 200)
# we should have the same files than the previous test, except artifact
# and mapping file 7: position 6 and 7; thus removing 6 twice
exp = exp.split('\n')
exp.pop(6)
exp.pop(6)
exp = '\n'.join(exp)
self.assertRegex(response.body.decode('ascii'), exp)
job = ProcessingJob.create(user, params)
job._set_error(info['message'])
payload = {'job_id': job.id}
r_client.set(key, dumps(payload))
else:
# The key doesn't contain any information. Delete the key
r_client.delete(key)
with TRN:
# Retrieve the Qiita plugin
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
# Create the submit command for VAMPS command
parameters = {'artifact': ['integer', None]}
Command.create(qiita_plugin, "submit_to_VAMPS",
"submits an artifact to VAMPS", parameters)
# Create the copy artifact command
parameters = {
'artifact': ['integer', None],
'prep_template': ['prep_template', None]
}
Command.create(qiita_plugin, "copy_artifact",
"Creates a copy of an artifact", parameters)
# Create the submit command for EBI command
parameters = {
'artifact': ['integer', None],
'submission_type': ['choice:["ADD", "MODIFY"]', 'ADD']
}
Command.create(qiita_plugin, "submit_to_EBI", "submits an artifact to EBI",
def render(self, study, prep_template, full_access, ena_terms,
user_defined_terms):
user = self.current_user
is_local_request = is_localhost(self.request.headers['host'])
template_fps = []
qiime_fps = []
# Unfortunately, both the prep template and the qiime mapping files
# have the sample type. The way to differentiate them is if we have
# the substring 'qiime' in the basename
for id_, fp in prep_template.get_filepaths():
if 'qiime' in basename(fp):
qiime_fps.append(
download_link_or_path(is_local_request, fp, id_,
'Qiime mapping'))
else:
template_fps.append(
download_link_or_path(is_local_request, fp, id_,
'Prep template'))
# Since get_filepaths returns the paths sorted from newest to oldest,
# the first in both list is the latest one
current_template_fp = template_fps[0]
current_qiime_fp = qiime_fps[0]
if len(template_fps) > 1:
show_old_templates = True
old_templates = template_fps[1:]
else:
show_old_templates = False
old_templates = None
if len(qiime_fps) > 1:
show_old_qiime_fps = True
old_qiime_fps = qiime_fps[1:]
else:
show_old_qiime_fps = False
old_qiime_fps = None
filetypes = sorted(((ft, ft_id, fp_type_by_ft[ft])
for ft, ft_id in viewitems(get_artifact_types())),
key=itemgetter(1))
files = [f for _, f in get_files_from_uploads_folders(str(study.id))]
other_studies_rd = sorted(viewitems(_get_accessible_raw_data(user)))
# A prep template can be modified if its status is sandbox
is_editable = prep_template.status == 'sandbox'
raw_data = prep_template.artifact
preprocess_options = []
preprocessed_data = None
show_preprocess_btn = True
no_preprocess_msg = None
preprocessing_status = 'Not processed'
preprocessing_status_msg = ""
if raw_data:
raw_data_ft = raw_data.artifact_type
# If the prep template has a raw data associated, it can be
# preprocessed. Retrieve the pre-processing parameters
# Hardcoding the command ids until the interface is refactored
if raw_data_ft in ('SFF', 'FASTA'):
param_iter = Command(2).default_parameter_sets
elif raw_data_ft == 'FASTQ':
param_iter = [
p for p in Command(1).default_parameter_sets
if p.values['barcode_type'] != 'not-barcoded'
]
elif raw_data_ft == 'per_sample_FASTQ':
param_iter = [
p for p in Command(1).default_parameter_sets
if p.values['barcode_type'] == 'not-barcoded'
]
else:
raise NotImplementedError(
"Pre-processing of %s files currently not supported." %
raw_data_ft)
preprocess_options = []
for param in param_iter:
text = ("<b>%s:</b> %s" % (k, v)
for k, v in viewitems(param.values))
preprocess_options.append(
(param.id, param.name, '<br>'.join(text)))
preprocessed_data = raw_data.children
# Check if the template have all the required columns for
# preprocessing
raw_data_files = raw_data.filepaths
if len(raw_data_files) == 0:
show_preprocess_btn = False
no_preprocess_msg = (
"Preprocessing disabled because there are no files "
"linked with the Raw Data")
else:
if prep_template.data_type() in TARGET_GENE_DATA_TYPES:
raw_forward_fps = [
fp for _, fp, ftype in raw_data_files
if ftype == 'raw_forward_seqs'
]
key = ('demultiplex_multiple'
if len(raw_forward_fps) > 1 else 'demultiplex')
missing_cols = prep_template.check_restrictions(
[PREP_TEMPLATE_COLUMNS_TARGET_GENE[key]])
if raw_data_ft == 'per_sample_FASTQ':
show_preprocess_btn = 'run_prefix' not in missing_cols
else:
show_preprocess_btn = len(missing_cols) == 0
no_preprocess_msg = None
if not show_preprocess_btn:
no_preprocess_msg = (
"Preprocessing disabled due to missing columns in "
"the prep template: %s" % ', '.join(missing_cols))
# Check the processing status
preprocessing_status, preprocessing_status_msg = \
get_artifact_processing_status(raw_data)
ebi_link = None
if prep_template.is_submitted_to_ebi:
ebi_link = EBI_LINKIFIER.format(study.ebi_study_accession)
return self.render_string(
"study_description_templates/prep_template_info_tab.html",
raw_data=raw_data,
current_template_fp=current_template_fp,
current_qiime_fp=current_qiime_fp,
show_old_templates=show_old_templates,
old_templates=old_templates,
show_old_qiime_fps=show_old_qiime_fps,
old_qiime_fps=old_qiime_fps,
filetypes=filetypes,
files=files,
other_studies_rd=other_studies_rd,
prep_template=prep_template,
study=study,
ena_terms=ena_terms,
user_defined_terms=user_defined_terms,
investigation_type=prep_template.investigation_type,
is_editable=is_editable,
preprocess_options=preprocess_options,
preprocessed_data=preprocessed_data,
preprocessing_status=preprocessing_status,
preprocessing_status_message=preprocessing_status_msg,
show_preprocess_btn=show_preprocess_btn,
no_preprocess_msg=no_preprocess_msg,
ebi_link=ebi_link)
