def list_options_handler_get_req(command_id):
"""Returns the available default parameters set for the given command
Parameters
----------
command_id : int
The command id
Returns
-------
dict of objects
A dictionary containing the commands information
{'status': str,
'message': str,
'options': list of dicts of {'id: str', 'name': str,
'values': dict of {str: str}}}
"""
command = Command(command_id)
options = [{
'id': p.id,
'name': p.name,
'values': p.values
} for p in command.default_parameter_sets]
return {
'status': 'success',
'message': '',
'options': options,
'req_options': command.required_parameters
}
def workflow_handler_post_req(user_id, command_id, params):
"""Creates a new workflow in the system
Parameters
----------
user_id : str
The user creating the workflow
command_id : int
The first command to execute in the workflow
params : str
JSON representations of the parameters for the first command of
the workflow
Returns
-------
dict of objects
A dictionary containing the commands information
{'status': str,
'message': str,
'workflow_id': int}
"""
parameters = Parameters.load(Command(command_id), json_str=params)
status = 'success'
message = ''
try:
wf = ProcessingWorkflow.from_scratch(User(user_id), parameters)
except Exception as exc:
wf = None
wf_id = None
job_info = None
status = 'error'
message = str(exc)
if wf is not None:
# this is safe as we are creating the workflow for the first time
# and there is only one node. Remember networkx doesn't assure order
# of nodes
job = list(wf.graph.nodes())[0]
inputs = [a.id for a in job.input_artifacts]
job_cmd = job.command
wf_id = wf.id
job_info = {
'id': job.id,
'inputs': inputs,
'label': job_cmd.name,
'outputs': job_cmd.outputs
}
return {
'status': status,
'message': message,
'workflow_id': wf_id,
'job': job_info
}
def list_options_handler_get_req(command_id, artifact_id=None):
"""Returns the available default parameters set for the given command
Parameters
----------
command_id : int
The command id
artifact_id : int, optional
The artifact id so to limit options based on how it has already been
processed
Returns
-------
dict of objects
A dictionary containing the commands information
{'status': str,
'message': str,
'options': list of dicts of {'id: str', 'name': str,
'values': dict of {str: str}}}
"""
def _helper_process_params(params):
return dumps({k: str(v).lower()
for k, v in params.items()},
sort_keys=True)
command = Command(command_id)
rparamers = command.required_parameters.keys()
eparams = []
if artifact_id is not None:
artifact = Artifact(artifact_id)
for job in artifact.jobs(cmd=command):
jstatus = job.status
outputs = job.outputs if job.status == 'success' else None
# this ignore any jobs that weren't successful or are in
# construction, or the results have been deleted [outputs == {}]
if jstatus not in {'success', 'in_construction'} or outputs == {}:
continue
params = job.parameters.values.copy()
for k in rparamers:
del params[k]
eparams.append(_helper_process_params(params))
options = [{
'id': p.id,
'name': p.name,
'values': p.values
} for p in command.default_parameter_sets
if _helper_process_params(p.values) not in eparams]
return {
'status': 'success',
'message': '',
'options': options,
'req_options': command.required_parameters,
'opt_options': command.optional_parameters
}
def test_workflow_handler_patch_req(self):
# Create a new workflow so it is in construction
exp_command = Command(1)
json_str = (
'{"input_data": 1, "max_barcode_errors": 1.5, '
'"barcode_type": "golay_12", "max_bad_run_length": 3, '
'"rev_comp": false, "phred_quality_threshold": 3, '
'"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
'"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
exp_params = Parameters.load(exp_command, json_str=json_str)
exp_user = User('*****@*****.**')
name = "Test processing workflow"
# tests success
wf = ProcessingWorkflow.from_scratch(exp_user,
exp_params,
name=name,
force=True)
graph = wf.graph
nodes = list(graph.nodes())
job_id = nodes[0].id
value = {
'dflt_params': 10,
'connections': {
job_id: {
'demultiplexed': 'input_data'
}
}
}
obs = workflow_handler_patch_req('add',
'/%s/' % wf.id,
req_value=dumps(value))
new_jobs = set(wf.graph.nodes()) - set(nodes)
self.assertEqual(len(new_jobs), 1)
new_job = new_jobs.pop()
exp = {
'status': 'success',
'message': '',
'job': {
'id': new_job.id,
'inputs': [job_id],
'label': 'Pick closed-reference OTUs',
'outputs': [['OTU table', 'BIOM']]
}
}
self.assertEqual(obs, exp)
obs = workflow_handler_patch_req('remove',
'/%s/%s/' % (wf.id, new_job.id))
exp = {'status': 'success', 'message': ''}
jobs = set(wf.graph.nodes()) - set(nodes)
self.assertEqual(jobs, set())
def test_artifact_summary_post_request(self):
# No access
with self.assertRaises(QiitaHTTPError):
artifact_summary_post_request(User('*****@*****.**'), 1)
# Returns already existing job
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 2}))
job._set_status('queued')
obs = artifact_summary_post_request(User('*****@*****.**'), 2)
exp = {'job': [job.id, 'queued', None]}
self.assertEqual(obs, exp)
def test_patch(self):
# Create a new job - through a workflow since that is the only way
# of creating jobs in the interface
exp_command = Command(1)
json_str = (
'{"input_data": 1, "max_barcode_errors": 1.5, '
'"barcode_type": "golay_12", "max_bad_run_length": 3, '
'"rev_comp": false, "phred_quality_threshold": 3, '
'"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
'"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
exp_params = Parameters.load(exp_command, json_str=json_str)
exp_user = User('*****@*****.**')
name = "Test processing workflow"
# tests success
wf = ProcessingWorkflow.from_scratch(exp_user,
exp_params,
name=name,
force=True)
graph = wf.graph
nodes = graph.nodes()
job_id = nodes[0].id
response = self.patch('/study/process/job/', {
'op': 'remove',
'path': job_id
})
self.assertEqual(response.code, 200)
exp = {
'status':
'error',
'message':
"Can't delete job %s. It is 'in_construction' "
"status. Please use /study/process/workflow/" % job_id
}
self.assertEqual(loads(response.body), exp)
# Test success
ProcessingJob(job_id)._set_error('Killed for testing')
response = self.patch('/study/process/job/', {
'op': 'remove',
'path': job_id
})
self.assertEqual(response.code, 200)
exp = {'status': 'success', 'message': ''}
self.assertEqual(loads(response.body), exp)
def test_artifact_summary_post_request(self):
# No access
obs = artifact_summary_post_request('*****@*****.**', 1)
exp = {
'status': 'error',
'message': 'User does not have access to study'
}
self.assertEqual(obs, exp)
# Returns already existing job
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 2}))
job._set_status('queued')
obs = artifact_summary_post_request('*****@*****.**', 2)
exp = {
'status': 'success',
'message': '',
'job': [job.id, 'queued', None]
}
self.assertEqual(obs, exp)
def workflow_handler_post_req(user_id, command_id, params):
"""Creates a new workflow in the system
Parameters
----------
user_id : str
The user creating the workflow
command_id : int
The first command to execute in the workflow
params : str
JSON representations of the parameters for the first command of
the workflow
Returns
-------
dict of objects
A dictionary containing the commands information
{'status': str,
'message': str,
'workflow_id': int}
"""
parameters = Parameters.load(Command(command_id), json_str=params)
wf = ProcessingWorkflow.from_scratch(User(user_id), parameters)
# this is safe as we are creating the workflow for the first time and there
# is only one node. Remember networkx doesn't assure order of nodes
job = wf.graph.nodes()[0]
inputs = [a.id for a in job.input_artifacts]
job_cmd = job.command
return {
'status': 'success',
'message': '',
'workflow_id': wf.id,
'job': {
'id': job.id,
'inputs': inputs,
'label': job_cmd.name,
'outputs': job_cmd.outputs
}
}
def test_job_ajax_patch_req(self):
# Create a new job - through a workflow since that is the only way
# of creating jobs in the interface
exp_command = Command(1)
json_str = (
'{"input_data": 1, "max_barcode_errors": 1.5, '
'"barcode_type": "golay_12", "max_bad_run_length": 3, '
'"rev_comp": false, "phred_quality_threshold": 3, '
'"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
'"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
exp_params = Parameters.load(exp_command, json_str=json_str)
exp_user = User('*****@*****.**')
name = "Test processing workflow"
# tests success
wf = ProcessingWorkflow.from_scratch(exp_user,
exp_params,
name=name,
force=True)
graph = wf.graph
nodes = list(graph.nodes())
job_id = nodes[0].id
# Incorrect path parameter
obs = job_ajax_patch_req('remove', '/%s/somethingelse' % job_id)
exp = {
'status': 'error',
'message': 'Incorrect path parameter: missing job id'
}
self.assertEqual(obs, exp)
obs = job_ajax_patch_req('remove', '/')
exp = {
'status': 'error',
'message': 'Incorrect path parameter: missing job id'
}
self.assertEqual(obs, exp)
# Job id is not like a job id
obs = job_ajax_patch_req('remove', '/notAJobId')
exp = {
'status':
'error',
'message':
'Incorrect path parameter: '
'notAJobId is not a recognized job id'
}
self.assertEqual(obs, exp)
# Job doesn't exist
obs = job_ajax_patch_req('remove',
'/6d368e16-2242-4cf8-87b4-a5dc40bc890b')
exp = {
'status':
'error',
'message':
'Incorrect path parameter: '
'6d368e16-2242-4cf8-87b4-a5dc40bc890b is not a '
'recognized job id'
}
self.assertEqual(obs, exp)
# in_construction job
obs = job_ajax_patch_req('remove', '/%s' % job_id)
exp = {
'status':
'error',
'message':
"Can't delete job %s. It is 'in_construction' "
"status. Please use /study/process/workflow/" % job_id
}
self.assertEqual(obs, exp)
# job status != 'error'
job = ProcessingJob(job_id)
job._set_status('queued')
obs = job_ajax_patch_req('remove', '/%s' % job_id)
exp = {
'status': 'error',
'message': 'Only jobs in "error" status can be deleted.'
}
self.assertEqual(obs, exp)
# Operation not supported
job._set_status('queued')
obs = job_ajax_patch_req('add', '/%s' % job_id)
exp = {
'status':
'error',
'message':
'Operation "add" not supported. Current supported '
'operations: remove'
}
self.assertEqual(obs, exp)
# Test success
job._set_error('Killed for testing')
obs = job_ajax_patch_req('remove', '/%s' % job_id)
exp = {'status': 'success', 'message': ''}
self.assertEqual(obs, exp)
def _construct_job_graph(self, analysis, commands, comm_opts=None,
rarefaction_depth=None,
merge_duplicated_sample_ids=False):
"""Builds the job graph for running an analysis
Parameters
----------
analysis: Analysis object
Analysis to finalize.
commands : list of tuples
Commands to add as jobs in the analysis.
Format [(data_type, command name), ...]
comm_opts : dict of dicts, optional
Options for commands. Format {command name: {opt1: value,...},...}
Default None (use default options).
rarefaction_depth : int, optional
Rarefaction depth for analysis' biom tables. Default None.
merge_duplicated_sample_ids : bool, optional
If the duplicated sample ids in the selected studies should be
merged or prepended with the artifact ids. False (default) prepends
the artifact id
"""
self._logger = stderr
self.analysis = analysis
analysis_id = analysis.id
# Add jobs to analysis
if comm_opts is None:
comm_opts = {}
analysis.status = "running"
# creating bioms at this point cause all this section runs on a worker
# node, currently an ipython job
analysis.build_files(rarefaction_depth, merge_duplicated_sample_ids)
mapping_file = analysis.mapping_file
tree_commands = ["Beta Diversity", "Alpha Rarefaction"]
for data_type, biom_fp in viewitems(analysis.biom_tables):
biom_table = load_table(biom_fp)
# getting reference_id and software_command_id from the first
# sample of the biom. This decision was discussed on the qiita
# meeting on 02/24/16
metadata = biom_table.metadata(biom_table.ids()[0])
rid = metadata['reference_id']
sci = metadata['command_id']
if rid != 'na':
reference = Reference(rid)
tree = reference.tree_fp
else:
reference = None
tree = ''
cmd = Command(sci) if sci != 'na' else None
for cmd_data_type, command in commands:
if data_type != cmd_data_type:
continue
# get opts set by user, else make it empty dict
opts = comm_opts.get(command, {})
opts["--otu_table_fp"] = biom_fp
opts["--mapping_fp"] = mapping_file
if command in tree_commands:
if tree != '':
opts["--tree_fp"] = tree
else:
opts["--parameter_fp"] = join(
get_db_files_base_dir(), "reference",
"params_qiime.txt")
if command == "Alpha Rarefaction":
opts["-n"] = 4
Job.create(data_type, command, opts, analysis, reference, cmd,
return_existing=True)
# Add the jobs
job_nodes = []
for job in analysis.jobs:
node_name = "%d_JOB_%d" % (analysis_id, job.id)
job_nodes.append(node_name)
job_name = "%s: %s" % (job.datatype, job.command[0])
self._job_graph.add_node(node_name,
func=system_call_from_job,
args=(job.id,),
job_name=job_name,
requires_deps=False)
# tgz-ing the analysis results
tgz_node_name = "TGZ_ANALYSIS_%d" % (analysis_id)
job_name = "tgz_analysis_%d" % (analysis_id)
self._job_graph.add_node(tgz_node_name,
func=_generate_analysis_tgz,
args=(analysis,),
job_name=job_name,
requires_deps=False)
# Adding the dependency edges to the graph
for job_node_name in job_nodes:
self._job_graph.add_edge(job_node_name, tgz_node_name)
# Finalize the analysis.
node_name = "FINISH_ANALYSIS_%d" % analysis.id
self._job_graph.add_node(node_name,
func=_finish_analysis,
args=(analysis,),
job_name='Finalize analysis',
requires_deps=False)
self._job_graph.add_edge(tgz_node_name, node_name)
def test_download_study(self):
tmp_dir = mkdtemp()
self._clean_up_files.append(tmp_dir)
biom_fp = join(tmp_dir, 'otu_table.biom')
smr_dir = join(tmp_dir, 'sortmerna_picked_otus')
log_dir = join(smr_dir, 'seqs_otus.log')
tgz = join(tmp_dir, 'sortmerna_picked_otus.tgz')
with biom_open(biom_fp, 'w') as f:
et.to_hdf5(f, "test")
makedirs(smr_dir)
with open(log_dir, 'w') as f:
f.write('\n')
with open(tgz, 'w') as f:
f.write('\n')
files_biom = [(biom_fp, 'biom'), (smr_dir, 'directory'), (tgz, 'tgz')]
params = Parameters.from_default_params(
next(Command(3).default_parameter_sets), {'input_data': 1})
a = Artifact.create(files_biom,
"BIOM",
parents=[Artifact(2)],
processing_parameters=params)
for x in a.filepaths:
self._clean_up_files.append(x['fp'])
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 200)
exp = ('- 1256812 /protected/processed_data/'
'1_study_1001_closed_reference_otu_table.biom processed_data/'
'1_study_1001_closed_reference_otu_table.biom\n'
'- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
'mapping_files/4_mapping_file.txt\n'
'- 1256812 /protected/processed_data/'
'1_study_1001_closed_reference_otu_table.biom processed_data/'
'1_study_1001_closed_reference_otu_table.biom\n'
'- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
'mapping_files/5_mapping_file.txt\n'
'- 1256812 /protected/processed_data/1_study_1001_'
'closed_reference_otu_table_Silva.biom processed_data/'
'1_study_1001_closed_reference_otu_table_Silva.biom\n'
'- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
'mapping_files/6_mapping_file.txt\n'
'- 1093210 /protected/BIOM/7/biom_table.biom '
'BIOM/7/biom_table.biom\n'
'- [0-9]* /protected/templates/1_prep_2_[0-9]*-[0-9]*.txt '
'mapping_files/7_mapping_file.txt\n'
'- [0-9]* /protected/BIOM/{0}/otu_table.biom '
'BIOM/{0}/otu_table.biom\n'
'- 1 /protected/BIOM/10/sortmerna_picked_otus/seqs_otus.log '
'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n'
'- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
'mapping_files/{0}_mapping_file.txt\n'.format(a.id))
self.assertRegex(response.body.decode('ascii'), exp)
response = self.get('/download_study_bioms/200')
self.assertEqual(response.code, 405)
# changing user so we can test the failures
BaseHandler.get_current_user = Mock(
return_value=User("*****@*****.**"))
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 405)
a.visibility = 'public'
response = self.get('/download_study_bioms/1')
# returning visibility
a.visibility = 'private'
self.assertEqual(response.code, 200)
# we should have the same files than the previous test, except artifact
# and mapping file 7: position 6 and 7; thus removing 6 twice
exp = exp.split('\n')
exp.pop(6)
exp.pop(6)
exp = '\n'.join(exp)
self.assertRegex(response.body.decode('ascii'), exp)
def test_artifact_summary_get_request(self):
user = User('*****@*****.**')
main_buttons = (
'<button onclick="if (confirm('
"\'Are you sure you want to make "
"public artifact id: 1?')) { set_artifact_visibility('public', 1) "
'}" class="btn btn-primary btn-sm">Make public</button> <button '
'onclick="if (confirm('
"'Are you sure you want to revert to "
"sandbox artifact id: 1?')) { set_artifact_visibility('sandbox', 1"
') }" class="btn btn-primary btn-sm">Revert to sandbox</button> ')
private_download_button = (
'<button class="btn btn-primary btn-sm" type="button" '
'aria-expanded="false" aria-controls="privateDownloadLink" '
'onclick="generate_private_download_link(%s)">Generate Download '
'Link</button><div class="collapse" id="privateDownloadLink"><div '
'class="card card-body" id="privateDownloadText">Generating '
'Download Link...</div></div>')
# Artifact w/o summary
obs = artifact_summary_get_request(user, 1)
exp_files = [
(1, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)',
'2125826711', '58 Bytes'),
(2, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)',
'2125826711', '58 Bytes')
]
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'private',
'editable': True,
'buttons': main_buttons + private_download_button % 1,
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': None,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# Artifact with summary being generated
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 1}))
job._set_status('queued')
obs = artifact_summary_get_request(user, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'private',
'editable': True,
'buttons': main_buttons + private_download_button % 1,
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': None,
'job': [job.id, 'queued', None],
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# Artifact with summary
fd, fp = mkstemp(suffix=".html")
close(fd)
with open(fp, 'w') as f:
f.write('<b>HTML TEST - not important</b>\n')
a = Artifact(1)
a.set_html_summary(fp)
self._files_to_remove.extend([fp, a.html_summary_fp[1]])
exp_files.append((a.html_summary_fp[0],
'%s (html summary)' % basename(a.html_summary_fp[1]),
'1642196267', '33 Bytes'))
exp_summary_path = relpath(a.html_summary_fp[1],
qiita_config.base_data_dir)
obs = artifact_summary_get_request(user, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'private',
'editable': True,
'buttons': main_buttons + private_download_button % 1,
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': exp_summary_path,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# No access
demo_u = User('*****@*****.**')
with self.assertRaises(QiitaHTTPError):
obs = artifact_summary_get_request(demo_u, 1)
# A non-owner/share user can't see the files
a.visibility = 'public'
obs = artifact_summary_get_request(demo_u, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'public',
'editable': False,
'buttons': '',
'processing_info': {},
'files': [],
'is_from_analysis': False,
'summary': exp_summary_path,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# testing sandbox
a.visibility = 'sandbox'
obs = artifact_summary_get_request(user, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'artifact_type': 'FASTQ',
'artifact_timestamp': '2012-10-01 09:10',
'visibility': 'sandbox',
'editable': True,
'buttons': private_download_button % 1,
'processing_info': {},
'files': exp_files,
'is_from_analysis': False,
'summary': exp_summary_path,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# returnig to private
a.visibility = 'private'
# admin gets buttons
obs = artifact_summary_get_request(User('*****@*****.**'), 2)
exp_files = [(3, '1_seqs.fna (preprocessed fasta)', '', '0 Bytes'),
(4, '1_seqs.qual (preprocessed fastq)', '', '0 Bytes'),
(5, '1_seqs.demux (preprocessed demux)', '', '0 Bytes')]
exp = {
'name':
'Demultiplexed 1',
'artifact_id':
2,
'artifact_type':
'Demultiplexed',
'artifact_timestamp':
'2012-10-01 10:10',
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 2?\')) { '
'set_artifact_visibility(\'public\', 2) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'2?\')) { set_artifact_visibility(\'sandbox\', 2) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button> <a class="btn btn-primary '
'btn-sm" href="/ebi_submission/2"><span '
'class="glyphicon glyphicon-export"></span> '
'Submit to EBI</a> <a class="btn btn-primary '
'btn-sm" href="/vamps/2"><span class="glyphicon '
'glyphicon-export"></span> Submit to VAMPS</a> ' +
private_download_button % 2),
'processing_info': {
'command_active': True,
'software_deprecated': False,
'command': 'Split libraries FASTQ',
'processing_parameters': {
'max_barcode_errors': '1.5',
'sequence_max_n': '0',
'max_bad_run_length': '3',
'phred_offset': 'auto',
'rev_comp': 'False',
'phred_quality_threshold': '3',
'input_data': '1',
'rev_comp_barcode': 'False',
'rev_comp_mapping_barcodes': 'False',
'min_per_read_length_fraction': '0.75',
'barcode_type': 'golay_12'
},
'software_version': '1.9.1',
'software': 'QIIME'
},
'files':
exp_files,
'is_from_analysis':
False,
'summary':
None,
'job':
None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
# analysis artifact
obs = artifact_summary_get_request(user, 8)
exp = {
'name': 'noname',
'artifact_id': 8,
'artifact_type': 'BIOM',
# this value changes on build so copy from obs
'artifact_timestamp': obs['artifact_timestamp'],
'visibility': 'sandbox',
'editable': True,
'buttons': private_download_button % 8,
'processing_info': {},
'files': [(22, 'biom_table.biom (biom)', '1756512010', '1.1 MB')],
'is_from_analysis': True,
'summary': None,
'job': None,
'errored_summary_jobs': []
}
self.assertEqual(obs, exp)
def test_get_analysis_graph_handler(self):
response = self.get('/analysis/description/1/graph/')
self.assertEqual(response.code, 200)
# The job id is randomly generated in the test environment. Gather
# it here. There is only 1 job in the first artifact of the analysis
job_id = Analysis(1).artifacts[0].jobs()[0].id
obs = loads(response.body)
exp = {
'edges': [[8, job_id], [job_id, 9]],
'nodes': [['job', 'job', job_id, 'Single Rarefaction', 'success'],
['artifact', 'BIOM', 9, 'noname\n(BIOM)', 'artifact'],
['artifact', 'BIOM', 8, 'noname\n(BIOM)', 'artifact']],
'workflow':
None
}
self.assertItemsEqual(obs, exp)
self.assertItemsEqual(obs['edges'], exp['edges'])
self.assertItemsEqual(obs['nodes'], exp['nodes'])
self.assertIsNone(obs['workflow'])
# Create a new analysis with 2 starting BIOMs to be able to test
# the different if statements of the request
BaseHandler.get_current_user = Mock(
return_value=User('*****@*****.**'))
user = User('*****@*****.**')
dflt_analysis = user.default_analysis
dflt_analysis.add_samples({
4: ['1.SKB8.640193', '1.SKD8.640184', '1.SKB7.640196'],
6: ['1.SKB8.640193', '1.SKD8.640184', '1.SKB7.640196']
})
args = {'name': 'New Test Graph Analysis', 'description': 'Desc'}
response = self.post('/analysis/create/', args)
new_id = response.effective_url.split('/')[-2]
a = Analysis(new_id)
# Wait until all the jobs are done so the BIOM tables exist
for j in a.jobs:
wait_for_processing_job(j.id)
artifacts = a.artifacts
self.assertEqual(len(artifacts), 2)
# Create a new workflow starting on the first artifact
# Magic number 9 -> Summarize Taxa command
params = Parameters.load(Command(9),
values_dict={
'metadata_category': 'None',
'sort': 'False',
'biom_table': artifacts[0].id
})
wf = ProcessingWorkflow.from_scratch(user, params)
# There is only one job in the workflow
job_id = wf.graph.nodes()[0].id
response = self.get('/analysis/description/%s/graph/' % new_id)
self.assertEqual(response.code, 200)
obs = loads(response.body)
exp = {
'edges': [[artifacts[0].id, job_id],
[job_id, '%s:taxa_summary' % job_id]],
'nodes':
[['job', 'job', job_id, 'Summarize Taxa', 'in_construction'],
[
'artifact', 'BIOM', artifacts[0].id, 'noname\n(BIOM)',
'artifact'
],
[
'artifact', 'BIOM', artifacts[1].id, 'noname\n(BIOM)',
'artifact'
],
[
'type', 'taxa_summary',
'%s:taxa_summary' % job_id, 'taxa_summary\n(taxa_summary)',
'type'
]],
'workflow':
wf.id
}
# Check that the keys are the same
self.assertItemsEqual(obs, exp)
# Check the edges
self.assertItemsEqual(obs['edges'], exp['edges'])
# Check the edges
self.assertItemsEqual(obs['nodes'], exp['nodes'])
# Check the edges
self.assertEqual(obs['workflow'], exp['workflow'])
# Add a job to the second BIOM to make sure that the edges and nodes
# are respected. Magic number 12 -> Single Rarefaction
job2 = wf.add(DefaultParameters(16),
req_params={
'depth': '100',
'biom_table': artifacts[1].id
})
job_id_2 = job2.id
response = self.get('/analysis/description/%s/graph/' % new_id)
self.assertEqual(response.code, 200)
obs = loads(response.body)
exp = {
'edges': [[artifacts[0].id, job_id],
[job_id, '%s:taxa_summary' % job_id],
[artifacts[1].id, job_id_2],
[job_id_2, '%s:rarefied_table' % job_id_2]],
'nodes':
[['job', 'job', job_id, 'Summarize Taxa', 'in_construction'],
['job', 'job', job_id_2, 'Single Rarefaction', 'in_construction'],
[
'artifact', 'BIOM', artifacts[0].id, 'noname\n(BIOM)',
'artifact'
],
[
'artifact', 'BIOM', artifacts[1].id, 'noname\n(BIOM)',
'artifact'
],
[
'type', 'taxa_summary',
'%s:taxa_summary' % job_id, 'taxa_summary\n(taxa_summary)',
'type'
],
[
'type', 'BIOM',
'%s:rarefied_table' % job_id_2, 'rarefied_table\n(BIOM)',
'type'
]],
'workflow':
wf.id
}
# Check that the keys are the same
self.assertItemsEqual(obs, exp)
# Check the edges
self.assertItemsEqual(obs['edges'], exp['edges'])
# Check the edges
self.assertItemsEqual(obs['nodes'], exp['nodes'])
# Check the edges
self.assertEqual(obs['workflow'], exp['workflow'])
# Add a second Workflow to the second artifact to force the raise of
# the error. This situation should never happen when using
# the interface
wf.remove(job2)
params = Parameters.load(Command(9),
values_dict={
'metadata_category': 'None',
'sort': 'False',
'biom_table': artifacts[1].id
})
wf = ProcessingWorkflow.from_scratch(user, params)
response = self.get('/analysis/description/%s/graph/' % new_id)
self.assertEqual(response.code, 500)
def test_download_study(self):
tmp_dir = mkdtemp()
self._clean_up_files.append(tmp_dir)
biom_fp = join(tmp_dir, 'otu_table.biom')
smr_dir = join(tmp_dir, 'sortmerna_picked_otus')
log_dir = join(smr_dir, 'seqs_otus.log')
tgz = join(tmp_dir, 'sortmerna_picked_otus.tgz')
with biom_open(biom_fp, 'w') as f:
et.to_hdf5(f, "test")
makedirs(smr_dir)
with open(log_dir, 'w') as f:
f.write('\n')
with open(tgz, 'w') as f:
f.write('\n')
files_biom = [(biom_fp, 'biom'), (smr_dir, 'directory'), (tgz, 'tgz')]
params = Parameters.from_default_params(
Command(3).default_parameter_sets.next(), {'input_data': 1})
a = Artifact.create(files_biom,
"BIOM",
parents=[Artifact(2)],
processing_parameters=params)
for _, fp, _ in a.filepaths:
self._clean_up_files.append(fp)
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 200)
exp = (
'- 1256812 /protected/processed_data/1_study_1001_closed_'
'reference_otu_table.biom processed_data/1_study_1001_closed_'
'reference_otu_table.biom\n'
'- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-'
'[0-9]*.txt mapping_files/4_mapping_file.txt\n'
'- 1256812 /protected/processed_data/'
'1_study_1001_closed_reference_otu_table.biom processed_data/'
'1_study_1001_closed_reference_otu_table.biom\n'
'- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-'
'[0-9]*.txt mapping_files/5_mapping_file.txt\n'
'- 1256812 /protected/processed_data/'
'1_study_1001_closed_reference_otu_table_Silva.biom processed_data'
'/1_study_1001_closed_reference_otu_table_Silva.biom\n'
'- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-'
'[0-9]*.txt mapping_files/6_mapping_file.txt\n'
'- 36615 /protected/templates/1_prep_2_qiime_[0-9]*-'
'[0-9]*.txt mapping_files/7_mapping_file.txt\n'
'- 39752 /protected/BIOM/{0}/otu_table.biom '
'BIOM/{0}/otu_table.biom\n'
'- 1 /protected/BIOM/{0}/sortmerna_picked_otus/seqs_otus.log '
'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n'
'- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]*.'
'txt mapping_files/{0}_mapping_file.txt\n'.format(a.id))
self.assertRegexpMatches(response.body, exp)
response = self.get('/download_study_bioms/200')
self.assertEqual(response.code, 405)
# changing user so we can test the failures
BaseHandler.get_current_user = Mock(
return_value=User("*****@*****.**"))
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 405)
a.visibility = 'public'
response = self.get('/download_study_bioms/1')
self.assertEqual(response.code, 200)
exp = ('- 39752 /protected/BIOM/{0}/otu_table.biom '
'BIOM/{0}/otu_table.biom\n'
'- 1 /protected/BIOM/{0}/sortmerna_picked_otus/seqs_otus.log '
'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n'
'- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]*.'
'txt mapping_files/{0}_mapping_file.txt\n'.format(a.id))
self.assertRegexpMatches(response.body, exp)
def test_artifact_summary_get_request(self):
# Artifact w/o summary
obs = artifact_summary_get_request('*****@*****.**', 1)
exp_p_jobs = [[
'063e553b-327c-4818-ab4a-adfe58e49860', 'Split libraries FASTQ',
'queued', None, None
],
[
'bcc7ebcd-39c1-43e4-af2d-822e3589f14d',
'Split libraries', 'running', 'demultiplexing', None
]]
exp_files = [
(1L, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)'),
(2L, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)')
]
exp = {
'status':
'success',
'message':
'',
'name':
'Raw data 1',
'summary':
None,
'job':
None,
'processing_jobs':
exp_p_jobs,
'errored_jobs': [],
'visibility':
'private',
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'files':
exp_files,
'editable':
True,
'prep_id':
1,
'study_id':
1
}
self.assertEqual(obs, exp)
# Artifact with summary being generated
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 1}))
job._set_status('queued')
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {
'status':
'success',
'message':
'',
'name':
'Raw data 1',
'summary':
None,
'job': [job.id, 'queued', None],
'processing_jobs':
exp_p_jobs,
'errored_jobs': [],
'visibility':
'private',
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'files':
exp_files,
'editable':
True,
'prep_id':
1,
'study_id':
1
}
self.assertEqual(obs, exp)
# Artifact with summary
fd, fp = mkstemp(suffix=".html")
close(fd)
with open(fp, 'w') as f:
f.write('<b>HTML TEST - not important</b>\n')
a = Artifact(1)
a.html_summary_fp = fp
self._files_to_remove.extend([fp, a.html_summary_fp[1]])
exp_files.append(
(a.html_summary_fp[0],
'%s (html summary)' % basename(a.html_summary_fp[1])))
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {
'status':
'success',
'message':
'',
'name':
'Raw data 1',
'summary':
'<b>HTML TEST - not important</b>\n',
'job':
None,
'processing_jobs':
exp_p_jobs,
'errored_jobs': [],
'visibility':
'private',
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'files':
exp_files,
'editable':
True,
'prep_id':
1,
'study_id':
1
}
self.assertEqual(obs, exp)
# No access
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {
'status': 'error',
'message': 'User does not have access to study'
}
self.assertEqual(obs, exp)
# A non-owner/share user can't see the files
a.visibility = 'public'
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {
'status': 'success',
'message': '',
'name': 'Raw data 1',
'summary': '<b>HTML TEST - not important</b>\n',
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'public',
'buttons': '',
'files': [],
'editable': False,
'prep_id': 1,
'study_id': 1
}
self.assertEqual(obs, exp)
# returnig to private
a.visibility = 'sandbox'
# admin gets buttons
obs = artifact_summary_get_request('*****@*****.**', 2)
exp_p_jobs = [[
'd19f76ee-274e-4c1b-b3a2-a12d73507c55',
'Pick closed-reference OTUs', 'error', 'generating demux file',
'Error message'
]]
exp_files = [(3L, '1_seqs.fna (preprocessed fasta)'),
(4L, '1_seqs.qual (preprocessed fastq)'),
(5L, '1_seqs.demux (preprocessed demux)')]
exp = {
'status':
'success',
'files':
exp_files,
'errored_jobs': [],
'editable':
True,
'visibility':
'private',
'job':
None,
'message':
'',
'name':
'Demultiplexed 1',
'processing_jobs':
exp_p_jobs,
'summary':
None,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 2?\')) { '
'set_artifact_visibility(\'public\', 2) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'2?\')) { set_artifact_visibility(\'sandbox\', 2) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button> <a class="btn btn-primary '
'btn-sm" href="/vamps/2"><span class="glyphicon '
'glyphicon-export"></span> Submit to VAMPS</a>'),
'study_id':
1,
'prep_id':
1
}
self.assertEqual(obs, exp)
def test_artifact_summary_get_request(self):
user = User('*****@*****.**')
# Artifact w/o summary
obs = artifact_summary_get_request(user, 1)
exp_p_jobs = [[
'063e553b-327c-4818-ab4a-adfe58e49860', 'Split libraries FASTQ',
'queued', None, None
],
[
'bcc7ebcd-39c1-43e4-af2d-822e3589f14d',
'Split libraries', 'running', 'demultiplexing', None
]]
exp_files = [
(1L, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)'),
(2L, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)')
]
exp = {
'name':
'Raw data 1',
'artifact_id':
1,
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'processing_parameters': {},
'files':
exp_files,
'summary':
None,
'job':
None,
'processing_jobs':
exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# Artifact with summary being generated
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 1}))
job._set_status('queued')
obs = artifact_summary_get_request(user, 1)
exp = {
'name':
'Raw data 1',
'artifact_id':
1,
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'processing_parameters': {},
'files':
exp_files,
'summary':
None,
'job': [job.id, 'queued', None],
'processing_jobs':
exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# Artifact with summary
fd, fp = mkstemp(suffix=".html")
close(fd)
with open(fp, 'w') as f:
f.write('<b>HTML TEST - not important</b>\n')
a = Artifact(1)
a.set_html_summary(fp)
self._files_to_remove.extend([fp, a.html_summary_fp[1]])
exp_files.append(
(a.html_summary_fp[0],
'%s (html summary)' % basename(a.html_summary_fp[1])))
exp_summary_path = relpath(a.html_summary_fp[1],
qiita_config.base_data_dir)
obs = artifact_summary_get_request(user, 1)
exp = {
'name':
'Raw data 1',
'artifact_id':
1,
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'1?\')) { set_artifact_visibility(\'sandbox\', 1) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button>'),
'processing_parameters': {},
'files':
exp_files,
'summary':
exp_summary_path,
'job':
None,
'processing_jobs':
exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# No access
demo_u = User('*****@*****.**')
with self.assertRaises(QiitaHTTPError):
obs = artifact_summary_get_request(demo_u, 1)
# A non-owner/share user can't see the files
a.visibility = 'public'
obs = artifact_summary_get_request(demo_u, 1)
exp = {
'name': 'Raw data 1',
'artifact_id': 1,
'visibility': 'public',
'editable': False,
'buttons': '',
'processing_parameters': {},
'files': [],
'summary': exp_summary_path,
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# returnig to private
a.visibility = 'private'
# admin gets buttons
obs = artifact_summary_get_request(User('*****@*****.**'), 2)
exp_p_jobs = [[
'd19f76ee-274e-4c1b-b3a2-a12d73507c55',
'Pick closed-reference OTUs', 'error', 'generating demux file',
'Error message'
]]
exp_files = [(3L, '1_seqs.fna (preprocessed fasta)'),
(4L, '1_seqs.qual (preprocessed fastq)'),
(5L, '1_seqs.demux (preprocessed demux)')]
exp = {
'name':
'Demultiplexed 1',
'artifact_id':
2,
'visibility':
'private',
'editable':
True,
'buttons': ('<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 2?\')) { '
'set_artifact_visibility(\'public\', 2) }" '
'class="btn btn-primary btn-sm">Make public'
'</button> <button onclick="if (confirm(\'Are you '
'sure you want to revert to sandbox artifact id: '
'2?\')) { set_artifact_visibility(\'sandbox\', 2) '
'}" class="btn btn-primary btn-sm">Revert to '
'sandbox</button> <a class="btn btn-primary '
'btn-sm" href="/ebi_submission/2"><span '
'class="glyphicon glyphicon-export"></span> '
'Submit to EBI</a> <a class="btn btn-primary '
'btn-sm" href="/vamps/2"><span class="glyphicon '
'glyphicon-export"></span> Submit to VAMPS</a>'),
'processing_parameters': {
'max_barcode_errors': 1.5,
'sequence_max_n': 0,
'max_bad_run_length': 3,
'phred_offset': u'auto',
'rev_comp': False,
'phred_quality_threshold': 3,
'input_data': 1,
'rev_comp_barcode': False,
'rev_comp_mapping_barcodes': False,
'min_per_read_length_fraction': 0.75,
'barcode_type': u'golay_12'
},
'files':
exp_files,
'summary':
None,
'job':
None,
'processing_jobs':
exp_p_jobs,
'errored_jobs': []
}
self.assertEqual(obs, exp)
# analysis artifact
obs = artifact_summary_get_request(user, 8)
exp = {
'name': 'noname',
'artifact_id': 8,
'visibility': 'sandbox',
'editable': True,
'buttons': '',
'processing_parameters': {},
'files': [(27, 'biom_table.biom (biom)')],
'summary': None,
'job': None,
'processing_jobs': [],
'errored_jobs': []
}
self.assertEqual(obs, exp)
def render(self, study, prep_template, full_access, ena_terms,
user_defined_terms):
user = self.current_user
is_local_request = is_localhost(self.request.headers['host'])
template_fps = []
qiime_fps = []
# Unfortunately, both the prep template and the qiime mapping files
# have the sample type. The way to differentiate them is if we have
# the substring 'qiime' in the basename
for id_, fp in prep_template.get_filepaths():
if 'qiime' in basename(fp):
qiime_fps.append(
download_link_or_path(is_local_request, fp, id_,
'Qiime mapping'))
else:
template_fps.append(
download_link_or_path(is_local_request, fp, id_,
'Prep template'))
# Since get_filepaths returns the paths sorted from newest to oldest,
# the first in both list is the latest one
current_template_fp = template_fps[0]
current_qiime_fp = qiime_fps[0]
if len(template_fps) > 1:
show_old_templates = True
old_templates = template_fps[1:]
else:
show_old_templates = False
old_templates = None
if len(qiime_fps) > 1:
show_old_qiime_fps = True
old_qiime_fps = qiime_fps[1:]
else:
show_old_qiime_fps = False
old_qiime_fps = None
filetypes = sorted(((ft, ft_id, fp_type_by_ft[ft])
for ft, ft_id in viewitems(get_artifact_types())),
key=itemgetter(1))
files = [f for _, f in get_files_from_uploads_folders(str(study.id))]
other_studies_rd = sorted(viewitems(_get_accessible_raw_data(user)))
# A prep template can be modified if its status is sandbox
is_editable = prep_template.status == 'sandbox'
raw_data = prep_template.artifact
preprocess_options = []
preprocessed_data = None
show_preprocess_btn = True
no_preprocess_msg = None
preprocessing_status = 'Not processed'
preprocessing_status_msg = ""
if raw_data:
raw_data_ft = raw_data.artifact_type
# If the prep template has a raw data associated, it can be
# preprocessed. Retrieve the pre-processing parameters
# Hardcoding the command ids until the interface is refactored
if raw_data_ft in ('SFF', 'FASTA'):
param_iter = Command(2).default_parameter_sets
elif raw_data_ft == 'FASTQ':
param_iter = [
p for p in Command(1).default_parameter_sets
if p.values['barcode_type'] != 'not-barcoded'
]
elif raw_data_ft == 'per_sample_FASTQ':
param_iter = [
p for p in Command(1).default_parameter_sets
if p.values['barcode_type'] == 'not-barcoded'
]
else:
raise NotImplementedError(
"Pre-processing of %s files currently not supported." %
raw_data_ft)
preprocess_options = []
for param in param_iter:
text = ("<b>%s:</b> %s" % (k, v)
for k, v in viewitems(param.values))
preprocess_options.append(
(param.id, param.name, '<br>'.join(text)))
preprocessed_data = raw_data.children
# Check if the template have all the required columns for
# preprocessing
raw_data_files = raw_data.filepaths
if len(raw_data_files) == 0:
show_preprocess_btn = False
no_preprocess_msg = (
"Preprocessing disabled because there are no files "
"linked with the Raw Data")
else:
if prep_template.data_type() in TARGET_GENE_DATA_TYPES:
raw_forward_fps = [
fp for _, fp, ftype in raw_data_files
if ftype == 'raw_forward_seqs'
]
key = ('demultiplex_multiple'
if len(raw_forward_fps) > 1 else 'demultiplex')
missing_cols = prep_template.check_restrictions(
[PREP_TEMPLATE_COLUMNS_TARGET_GENE[key]])
if raw_data_ft == 'per_sample_FASTQ':
show_preprocess_btn = 'run_prefix' not in missing_cols
else:
show_preprocess_btn = len(missing_cols) == 0
no_preprocess_msg = None
if not show_preprocess_btn:
no_preprocess_msg = (
"Preprocessing disabled due to missing columns in "
"the prep template: %s" % ', '.join(missing_cols))
# Check the processing status
preprocessing_status, preprocessing_status_msg = \
get_artifact_processing_status(raw_data)
ebi_link = None
if prep_template.is_submitted_to_ebi:
ebi_link = EBI_LINKIFIER.format(study.ebi_study_accession)
return self.render_string(
"study_description_templates/prep_template_info_tab.html",
raw_data=raw_data,
current_template_fp=current_template_fp,
current_qiime_fp=current_qiime_fp,
show_old_templates=show_old_templates,
old_templates=old_templates,
show_old_qiime_fps=show_old_qiime_fps,
old_qiime_fps=old_qiime_fps,
filetypes=filetypes,
files=files,
other_studies_rd=other_studies_rd,
prep_template=prep_template,
study=study,
ena_terms=ena_terms,
user_defined_terms=user_defined_terms,
investigation_type=prep_template.investigation_type,
is_editable=is_editable,
preprocess_options=preprocess_options,
preprocessed_data=preprocessed_data,
preprocessing_status=preprocessing_status,
preprocessing_status_message=preprocessing_status_msg,
show_preprocess_btn=show_preprocess_btn,
no_preprocess_msg=no_preprocess_msg,
ebi_link=ebi_link)
def render(self, study_id, preprocessed_data):
user = self.current_user
ppd_id = preprocessed_data.id
vamps_status = preprocessed_data.is_submitted_to_vamps
filepaths = preprocessed_data.filepaths
is_local_request = is_localhost(self.request.headers['host'])
show_ebi_btn = user.level == "admin"
processing_status, processing_status_msg = \
get_artifact_processing_status(preprocessed_data)
processed_data = sorted([pd.id for pd in preprocessed_data.children])
# Get all the ENA terms for the investigation type
ontology = Ontology(convert_to_id('ENA', 'ontology'))
# make "Other" show at the bottom of the drop down menu
ena_terms = []
for v in sorted(ontology.terms):
if v != 'Other':
ena_terms.append('<option value="%s">%s</option>' % (v, v))
ena_terms.append('<option value="Other">Other</option>')
# New Type is for users to add a new user-defined investigation type
user_defined_terms = ontology.user_defined_terms + ['New Type']
# ppd can only have 1 prep template
prep_template = preprocessed_data.prep_templates[0]
# this block might seem wrong but is here due to a possible
# pathological case that we used to have in the system: preprocessed
# data without valid prep_templates
prep_templates = preprocessed_data.prep_templates
if len(prep_templates) == 1:
prep_template_id = prep_template.id
raw_data_id = prep_template.artifact.id
inv_type = prep_template.investigation_type or "None selected"
else:
prep_template_id = None
raw_data_id = None
inv_type = "None Selected"
process_params = {
param.id: (generate_param_str(param), param.name)
for param in Command(3).default_parameter_sets
}
# We just need to provide an ID for the default parameters,
# so we can initialize the interface
default_params = min(process_params.keys())
ebi_link = None
if preprocessed_data.is_submitted_to_ebi:
ebi_link = EBI_LINKIFIER.format(
Study(study_id).ebi_study_accession)
return self.render_string(
"study_description_templates/preprocessed_data_info_tab.html",
ppd_id=ppd_id,
show_ebi_btn=show_ebi_btn,
filepaths=filepaths,
is_local_request=is_local_request,
prep_template_id=prep_template_id,
raw_data_id=raw_data_id,
inv_type=inv_type,
ena_terms=ena_terms,
vamps_status=vamps_status,
user_defined_terms=user_defined_terms,
process_params=process_params,
default_params=default_params,
study_id=preprocessed_data.study.id,
processing_status=processing_status,
processing_status_msg=processing_status_msg,
processed_data=processed_data,
ebi_link=ebi_link)
def create_command(software, name, description, parameters, outputs=None,
analysis_only=False):
r"""Replicates the Command.create code at the time the patch was written"""
# Perform some sanity checks in the parameters dictionary
if not parameters:
raise QiitaDBError(
"Error creating command %s. At least one parameter should "
"be provided." % name)
sql_param_values = []
sql_artifact_params = []
for pname, vals in parameters.items():
if len(vals) != 2:
raise QiitaDBError(
"Malformed parameters dictionary, the format should be "
"{param_name: [parameter_type, default]}. Found: "
"%s for parameter name %s" % (vals, pname))
ptype, dflt = vals
# Check that the type is one of the supported types
supported_types = ['string', 'integer', 'float', 'reference',
'boolean', 'prep_template', 'analysis']
if ptype not in supported_types and not ptype.startswith(
('choice', 'mchoice', 'artifact')):
supported_types.extend(['choice', 'mchoice', 'artifact'])
raise QiitaDBError(
"Unsupported parameters type '%s' for parameter %s. "
"Supported types are: %s"
% (ptype, pname, ', '.join(supported_types)))
if ptype.startswith(('choice', 'mchoice')) and dflt is not None:
choices = set(loads(ptype.split(':')[1]))
dflt_val = dflt
if ptype.startswith('choice'):
# In the choice case, the dflt value is a single string,
# create a list with it the string on it to use the
# issuperset call below
dflt_val = [dflt_val]
else:
# jsonize the list to store it in the DB
dflt = dumps(dflt)
if not choices.issuperset(dflt_val):
raise QiitaDBError(
"The default value '%s' for the parameter %s is not "
"listed in the available choices: %s"
% (dflt, pname, ', '.join(choices)))
if ptype.startswith('artifact'):
atypes = loads(ptype.split(':')[1])
sql_artifact_params.append(
[pname, 'artifact', atypes])
else:
if dflt is not None:
sql_param_values.append([pname, ptype, False, dflt])
else:
sql_param_values.append([pname, ptype, True, None])
with TRN:
sql = """SELECT EXISTS(SELECT *
FROM qiita.software_command
WHERE software_id = %s AND name = %s)"""
TRN.add(sql, [software.id, name])
if TRN.execute_fetchlast():
raise QiitaDBDuplicateError(
"command", "software: %d, name: %s"
% (software.id, name))
# Add the command to the DB
sql = """INSERT INTO qiita.software_command
(name, software_id, description, is_analysis)
VALUES (%s, %s, %s, %s)
RETURNING command_id"""
sql_params = [name, software.id, description, analysis_only]
TRN.add(sql, sql_params)
c_id = TRN.execute_fetchlast()
# Add the parameters to the DB
sql = """INSERT INTO qiita.command_parameter
(command_id, parameter_name, parameter_type, required,
default_value)
VALUES (%s, %s, %s, %s, %s)
RETURNING command_parameter_id"""
sql_params = [[c_id, pname, p_type, reqd, default]
for pname, p_type, reqd, default in sql_param_values]
TRN.add(sql, sql_params, many=True)
TRN.execute()
# Add the artifact parameters
sql_type = """INSERT INTO qiita.parameter_artifact_type
(command_parameter_id, artifact_type_id)
VALUES (%s, %s)"""
supported_types = []
for pname, p_type, atypes in sql_artifact_params:
sql_params = [c_id, pname, p_type, True, None]
TRN.add(sql, sql_params)
pid = TRN.execute_fetchlast()
sql_params = [[pid, convert_to_id(at, 'artifact_type')]
for at in atypes]
TRN.add(sql_type, sql_params, many=True)
supported_types.extend([atid for _, atid in sql_params])
# If the software type is 'artifact definition', there are a couple
# of extra steps
if software.type == 'artifact definition':
# If supported types is not empty, link the software with these
# types
if supported_types:
sql = """INSERT INTO qiita.software_artifact_type
(software_id, artifact_type_id)
VALUES (%s, %s)"""
sql_params = [[software.id, atid]
for atid in supported_types]
TRN.add(sql, sql_params, many=True)
# If this is the validate command, we need to add the
# provenance and name parameters. These are used internally,
# that's why we are adding them here
if name == 'Validate':
sql = """INSERT INTO qiita.command_parameter
(command_id, parameter_name, parameter_type,
required, default_value)
VALUES (%s, 'name', 'string', 'False',
'dflt_name'),
(%s, 'provenance', 'string', 'False', NULL)
"""
TRN.add(sql, [c_id, c_id])
# Add the outputs to the command
if outputs:
sql = """INSERT INTO qiita.command_output
(name, command_id, artifact_type_id)
VALUES (%s, %s, %s)"""
sql_args = [[pname, c_id, convert_to_id(at, 'artifact_type')]
for pname, at in outputs.items()]
TRN.add(sql, sql_args, many=True)
TRN.execute()
return Command(c_id)
def test_artifact_summary_get_request(self):
# Artifact w/o summary
obs = artifact_summary_get_request('*****@*****.**', 1)
exp_p_jobs = [[
'063e553b-327c-4818-ab4a-adfe58e49860', 'Split libraries FASTQ',
'queued', None, None
],
[
'bcc7ebcd-39c1-43e4-af2d-822e3589f14d',
'Split libraries', 'running', 'demultiplexing', None
]]
exp_files = [
(1L, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)'),
(2L, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)')
]
exp = {
'status':
'success',
'message':
'',
'name':
'Raw data 1',
'summary':
None,
'job':
None,
'processing_jobs':
exp_p_jobs,
'errored_jobs': [],
'visibility':
'private',
'buttons':
'<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public</button>'
' <button onclick="if (confirm(\'Are you sure you '
'want to revert to sandbox artifact id: 1?\')) '
'{ set_artifact_visibility(\'sandbox\', 1) }" '
'class="btn btn-primary btn-sm">Revert to '
'sandbox</button>',
'files':
exp_files,
'editable':
True,
'prep_id':
1,
'study_id':
1
}
self.assertEqual(obs, exp)
# Artifact with summary being generated
job = ProcessingJob.create(
User('*****@*****.**'),
Parameters.load(Command(7), values_dict={'input_data': 1}))
job._set_status('queued')
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {
'status':
'success',
'message':
'',
'name':
'Raw data 1',
'summary':
None,
'job': [job.id, 'queued', None],
'processing_jobs':
exp_p_jobs,
'errored_jobs': [],
'visibility':
'private',
'buttons':
'<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public</button>'
' <button onclick="if (confirm(\'Are you sure you '
'want to revert to sandbox artifact id: 1?\')) { '
'set_artifact_visibility(\'sandbox\', 1) }" '
'class="btn btn-primary btn-sm">Revert to '
'sandbox</button>',
'files':
exp_files,
'editable':
True,
'prep_id':
1,
'study_id':
1
}
self.assertEqual(obs, exp)
# Artifact with summary
fd, fp = mkstemp(suffix=".html")
close(fd)
with open(fp, 'w') as f:
f.write('<b>HTML TEST - not important</b>\n')
a = Artifact(1)
a.html_summary_fp = fp
self._files_to_remove.extend([fp, a.html_summary_fp[1]])
exp_files.append(
(a.html_summary_fp[0],
'%s (html summary)' % basename(a.html_summary_fp[1])))
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {
'status':
'success',
'message':
'',
'name':
'Raw data 1',
'summary':
'<b>HTML TEST - not important</b>\n',
'job':
None,
'processing_jobs':
exp_p_jobs,
'errored_jobs': [],
'visibility':
'private',
'buttons':
'<button onclick="if (confirm(\'Are you sure you '
'want to make public artifact id: 1?\')) { '
'set_artifact_visibility(\'public\', 1) }" '
'class="btn btn-primary btn-sm">Make public</button>'
' <button onclick="if (confirm(\'Are you sure you '
'want to revert to sandbox artifact id: 1?\')) { '
'set_artifact_visibility(\'sandbox\', 1) }" '
'class="btn btn-primary btn-sm">Revert to '
'sandbox</button>',
'files':
exp_files,
'editable':
True,
'prep_id':
1,
'study_id':
1
}
self.assertEqual(obs, exp)
# No access
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {
'status': 'error',
'message': 'User does not have access to study'
}
self.assertEqual(obs, exp)
# A non-owner/share user can't see the files
a.visibility = 'public'
obs = artifact_summary_get_request('*****@*****.**', 1)
exp = {
'status': 'success',
'message': '',
'name': 'Raw data 1',
'summary': '<b>HTML TEST - not important</b>\n',
'job': None,
'processing_jobs': exp_p_jobs,
'errored_jobs': [],
'visibility': 'public',
'buttons': '',
'files': [],
'editable': False,
'prep_id': 1,
'study_id': 1
}
self.assertEqual(obs, exp)
