def delete_processed_data(self, study, user, callback):
"""Delete the selected processed data
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
"""
pd_id = int(self.get_argument('processed_data_id'))
try:
Artifact.delete(pd_id)
msg = ("Processed data %d has been deleted" % pd_id)
msg_level = "success"
pd_id = None
except Exception as e:
msg = ("Couldn't remove processed data %d: %s" % (pd_id, str(e)))
msg_level = "danger"
callback((msg, msg_level, 'processed_data_tab', pd_id, None))
def approve_study(self, study, user, callback):
"""Approves the current study if and only if the current user is admin
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
"""
if _approve(user.level):
pd_id = int(self.get_argument('pd_id'))
pd = Artifact(pd_id)
pd.visibility = 'private'
_propagate_visibility(pd)
msg = "Processed data approved"
msg_level = "success"
else:
msg = ("The current user does not have permission to approve "
"the processed data")
msg_level = "danger"
callback((msg, msg_level, "processed_data_tab", pd_id, None))
def approve_study(self, study, user, callback):
"""Approves the current study if and only if the current user is admin
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
"""
if _approve(user.level):
pd_id = int(self.get_argument('pd_id'))
pd = Artifact(pd_id)
pd.visibility = 'private'
_propagate_visibility(pd)
msg = "Processed data approved"
msg_level = "success"
else:
msg = ("The current user does not have permission to approve "
"the processed data")
msg_level = "danger"
callback((msg, msg_level, "processed_data_tab", pd_id, None))
def setUp(self):
uploads_path = get_mountpoint('uploads')[0][1]
# Create prep test file to point at
self.update_fp = join(uploads_path, '1', 'update.txt')
with open(self.update_fp, 'w') as f:
f.write("""sample_name\tnew_col\n1.SKD6.640190\tnew_value\n""")
self._files_to_remove = [self.update_fp]
self._files_to_remove = []
# creating temporal files and artifact
# NOTE: we don't need to remove the artifact created cause it's
# used to test the delete functionality
fd, fp = mkstemp(suffix='_seqs.fna')
close(fd)
with open(fp, 'w') as f:
f.write(">1.sid_r4_0 M02034:17:000000000-A5U18:1:1101:15370:1394 "
"1:N:0:1 orig_bc=CATGAGCT new_bc=CATGAGCT bc_diffs=0\n"
"GTGTGCCAGCAGCCGCGGTAATACGTAGGG\n")
# 4 Demultiplexed
filepaths_processed = [(fp, 4)]
# 1 for default parameters and input data
exp_params = Parameters.from_default_params(DefaultParameters(1),
{'input_data': 1})
self.artifact = Artifact.create(filepaths_processed, "Demultiplexed",
parents=[Artifact(1)],
processing_parameters=exp_params)
def delete_raw_data(self, study, user, callback):
"""Delete the selected raw data
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
"""
raw_data_id = int(self.get_argument('raw_data_id'))
prep_template_id = int(self.get_argument('prep_template_id'))
try:
Artifact.delete(raw_data_id)
msg = ("Raw data %d has been deleted from prep_template %d" %
(raw_data_id, prep_template_id))
msg_level = "success"
except Exception as e:
msg = "Couldn't remove raw data %d: %s" % (raw_data_id, str(e))
msg_level = "danger"
callback((msg, msg_level, "prep_template_tab", prep_template_id, None))
def test_prep_template_graph_get_req(self):
obs = prep_template_graph_get_req(1, '*****@*****.**')
exp = {'edge_list': [(1, 3), (1, 2), (2, 4), (2, 5), (2, 6)],
'node_labels': [(1, 'Raw data 1 - FASTQ'),
(2, 'Demultiplexed 1 - Demultiplexed'),
(3, 'Demultiplexed 2 - Demultiplexed'),
(4, 'BIOM - BIOM'),
(5, 'BIOM - BIOM'),
(6, 'BIOM - BIOM')],
'status': 'success',
'message': ''}
self.assertItemsEqual(obs['edge_list'], exp['edge_list'])
self.assertItemsEqual(obs['node_labels'], exp['node_labels'])
self.assertEqual(obs['status'], exp['status'])
self.assertEqual(obs['message'], exp['message'])
Artifact(4).visibility = "public"
obs = prep_template_graph_get_req(1, '*****@*****.**')
exp = {'edge_list': [(1, 2), (2, 4)],
'node_labels': [(1, 'Raw data 1 - FASTQ'),
(2, 'Demultiplexed 1 - Demultiplexed'),
(4, 'BIOM - BIOM')],
'status': 'success',
'message': ''}
self.assertItemsEqual(obs['edge_list'], exp['edge_list'])
self.assertItemsEqual(obs['node_labels'], exp['node_labels'])
self.assertEqual(obs['status'], exp['status'])
self.assertEqual(obs['message'], exp['message'])
# Reset visibility of the artifacts
for i in range(4, 0, -1):
Artifact(i).visibility = "private"
def post(self, preprocessed_data_id):
user = self.current_user
# make sure user is admin and can therefore actually submit to VAMPS
if user.level != 'admin':
raise HTTPError(403, "User %s cannot submit to VAMPS!" %
user.id)
msg = ''
msg_level = 'success'
plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = plugin.get_command('submit_to_VAMPS')
artifact = Artifact(preprocessed_data_id)
# Check if the artifact is already being submitted to VAMPS
is_being_submitted = any(
[j.status in ('queued', 'running')
for j in artifact.jobs(cmd=cmd)])
if is_being_submitted == 'submitting':
msg = "Cannot resubmit! Data is already being submitted"
msg_level = 'danger'
self.display_template(preprocessed_data_id, msg, msg_level)
else:
params = Parameters.load(
cmd, values_dict={'artifact': preprocessed_data_id})
job = ProcessingJob.create(user, params, True)
job.submit()
self.redirect('/study/description/%s' % artifact.study.study_id)
def delete_raw_data(self, study, user, callback):
"""Delete the selected raw data
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
"""
raw_data_id = int(self.get_argument('raw_data_id'))
prep_template_id = int(self.get_argument('prep_template_id'))
try:
Artifact.delete(raw_data_id)
msg = ("Raw data %d has been deleted from prep_template %d"
% (raw_data_id, prep_template_id))
msg_level = "success"
except Exception as e:
msg = "Couldn't remove raw data %d: %s" % (raw_data_id, str(e))
msg_level = "danger"
callback((msg, msg_level, "prep_template_tab", prep_template_id, None))
def post(self, preprocessed_data_id):
user = self.current_user
# make sure user is admin and can therefore actually submit to VAMPS
if user.level != 'admin':
raise HTTPError(403, "User %s cannot submit to VAMPS!" % user.id)
msg = ''
msg_level = 'success'
plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = plugin.get_command('submit_to_VAMPS')
artifact = Artifact(preprocessed_data_id)
# Check if the artifact is already being submitted to VAMPS
is_being_submitted = any([
j.status in ('queued', 'running') for j in artifact.jobs(cmd=cmd)
])
if is_being_submitted == 'submitting':
msg = "Cannot resubmit! Data is already being submitted"
msg_level = 'danger'
self.display_template(preprocessed_data_id, msg, msg_level)
else:
params = Parameters.load(
cmd, values_dict={'artifact': preprocessed_data_id})
job = ProcessingJob.create(user, params)
job.submit()
self.redirect('/study/description/%s' % artifact.study.study_id)
def post(self, preprocessed_data_id):
# make sure user is admin and can therefore actually submit to VAMPS
if self.current_user.level != "admin":
raise HTTPError(403, "User %s cannot submit to VAMPS!" % self.current_user.id)
msg = ""
msg_level = "success"
preprocessed_data = Artifact(preprocessed_data_id)
state = preprocessed_data.submitted_to_vamps_status()
demux = [path for _, path, ftype in preprocessed_data.get_filepaths() if ftype == "preprocessed_demux"]
demux_length = len(demux)
if state in ("submitting", "success"):
msg = "Cannot resubmit! Current state is: %s" % state
msg_level = "danger"
elif demux_length != 1:
msg = "The study doesn't have demux files or have too many" % state
msg_level = "danger"
else:
channel = self.current_user.id
job_id = submit(channel, submit_to_VAMPS, int(preprocessed_data_id))
self.render(
"compute_wait.html",
job_id=job_id,
title="VAMPS Submission",
completion_redirect="/compute_complete/%s" % job_id,
)
return
self.display_template(preprocessed_data_id, msg, msg_level)
def artifact_get_prep_req(user_id, artifact_ids):
"""Returns all prep info sample ids for the given artifact_ids
Parameters
----------
user_id : str
user making the request
artifact_ids : list of int
list of artifact ids
Returns
-------
dict of objects
A dictionary containing the artifact information
{'status': status,
'message': message,
'data': {artifact_id: [prep info sample ids]}
"""
samples = {}
for aid in sorted(artifact_ids):
artifact = Artifact(aid)
access_error = check_access(artifact.study.id, user_id)
if access_error:
return access_error
samples[aid] = list(
chain(*[sorted(pt.keys()) for pt in Artifact(aid).prep_templates]))
return {'status': 'success', 'msg': '', 'data': samples}
def post(self, preprocessed_data_id):
# make sure user is admin and can therefore actually submit to VAMPS
if self.current_user.level != 'admin':
raise HTTPError(
403, "User %s cannot submit to VAMPS!" % self.current_user.id)
msg = ''
msg_level = 'success'
preprocessed_data = Artifact(preprocessed_data_id)
state = preprocessed_data.submitted_to_vamps_status()
demux = [
path for _, path, ftype in preprocessed_data.get_filepaths()
if ftype == 'preprocessed_demux'
]
demux_length = len(demux)
if state in ('submitting', 'success'):
msg = "Cannot resubmit! Current state is: %s" % state
msg_level = 'danger'
elif demux_length != 1:
msg = "The study doesn't have demux files or have too many" % state
msg_level = 'danger'
else:
channel = self.current_user.id
job_id = submit(channel, submit_to_VAMPS,
int(preprocessed_data_id))
self.render('compute_wait.html',
job_id=job_id,
title='VAMPS Submission',
completion_redirect='/compute_complete/%s' % job_id)
return
self.display_template(preprocessed_data_id, msg, msg_level)
def artifact_delete_req(artifact_id, user_id):
"""Deletes the artifact
Parameters
----------
artifact_id : int
Artifact being acted on
user_id : str
The user requesting the action
Returns
-------
dict
Status of action, in the form {'status': status, 'message': msg}
status: status of the action, either success or error
message: Human readable message for status
"""
pd = Artifact(int(artifact_id))
access_error = check_access(pd.study.id, user_id)
if access_error:
return access_error
try:
Artifact.delete(int(artifact_id))
except QiitaDBArtifactDeletionError as e:
return {'status': 'error',
'message': str(e)}
return {'status': 'success',
'message': ''}
def delete_processed_data(self, study, user, callback):
"""Delete the selected processed data
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
"""
pd_id = int(self.get_argument('processed_data_id'))
try:
Artifact.delete(pd_id)
msg = ("Processed data %d has been deleted" % pd_id)
msg_level = "success"
pd_id = None
except Exception as e:
msg = ("Couldn't remove processed data %d: %s" %
(pd_id, str(e)))
msg_level = "danger"
callback((msg, msg_level, 'processed_data_tab', pd_id, None))
def write_demux_files(self, prep_template, generate_hdf5=True):
"""Writes a demux test file to avoid duplication of code"""
fna_fp = join(self.temp_dir, 'seqs.fna')
demux_fp = join(self.temp_dir, 'demux.seqs')
if generate_hdf5:
with open(fna_fp, 'w') as f:
f.write(FASTA_EXAMPLE)
with File(demux_fp, "w") as f:
to_hdf5(fna_fp, f)
else:
with open(demux_fp, 'w') as f:
f.write('')
if prep_template.artifact is None:
ppd = Artifact.create([(demux_fp, 6)],
"Demultiplexed",
prep_template=prep_template)
else:
params = Parameters.from_default_params(
DefaultParameters(1),
{'input_data': prep_template.artifact.id})
ppd = Artifact.create([(demux_fp, 6)],
"Demultiplexed",
parents=[prep_template.artifact],
processing_parameters=params)
return ppd
def copy_raw_data(prep_template, artifact_id):
"""Creates a new raw data by copying from artifact_id
Parameters
----------
prep_template : qiita_db.metadata_template.prep_template.PrepTemplate
The template to attach the artifact
artifact_id : int
The id of the artifact to duplicate
Returns
-------
dict of {str: str}
A dict of the form {'status': str, 'message': str}
"""
from qiita_db.artifact import Artifact
status = 'success'
msg = ''
try:
Artifact.copy(Artifact(artifact_id), prep_template)
except Exception as e:
# We should hit this exception rarely (that's why it is an
# exception) since at this point we have done multiple checks.
# However, it can occur in weird cases, so better let the GUI know
# that this failed
return {'status': 'danger',
'message': "Error creating artifact: %s" % str(e)}
return {'status': status, 'message': msg}
def test_check_artifact_access(self):
# "Study" artifact
a = Artifact(1)
# The user has access
u = User('*****@*****.**')
check_artifact_access(u, a)
# Admin has access to everything
admin = User('*****@*****.**')
check_artifact_access(admin, a)
# Demo user doesn't have access
demo_u = User('*****@*****.**')
with self.assertRaises(HTTPError):
check_artifact_access(demo_u, a)
# "Analysis" artifact
a = Artifact(8)
a.visibility = 'private'
check_artifact_access(u, a)
check_artifact_access(admin, a)
with self.assertRaises(HTTPError):
check_artifact_access(demo_u, a)
check_artifact_access(User('*****@*****.**'), a)
a.visibility = 'public'
check_artifact_access(demo_u, a)
def write_demux_files(self, prep_template, generate_hdf5=True):
"""Writes a demux test file to avoid duplication of code"""
fna_fp = join(self.temp_dir, 'seqs.fna')
demux_fp = join(self.temp_dir, 'demux.seqs')
if generate_hdf5:
with open(fna_fp, 'w') as f:
f.write(FASTA_EXAMPLE)
with File(demux_fp, "w") as f:
to_hdf5(fna_fp, f)
else:
with open(demux_fp, 'w') as f:
f.write('')
if prep_template.artifact is None:
ppd = Artifact.create(
[(demux_fp, 6)], "Demultiplexed", prep_template=prep_template,
can_be_submitted_to_ebi=True, can_be_submitted_to_vamps=True)
else:
params = Parameters.from_default_params(
DefaultParameters(1),
{'input_data': prep_template.artifact.id})
ppd = Artifact.create(
[(demux_fp, 6)], "Demultiplexed",
parents=[prep_template.artifact], processing_parameters=params,
can_be_submitted_to_ebi=True, can_be_submitted_to_vamps=True)
return ppd
def test_patch_artifact_ajax_handler(self):
a = Artifact(1)
self.assertEqual(a.name, 'Raw data 1')
arguments = {'op': 'replace', 'path': '/name/', 'value': 'NEW_NAME'}
response = self.patch('/artifact/1/', data=arguments)
self.assertEqual(response.code, 200)
self.assertEqual(a.name, 'NEW_NAME')
a.name = 'Raw data 1'
def test_patch_artifact_ajax_handler(self):
a = Artifact(1)
self.assertEqual(a.name, 'Raw data 1')
arguments = {'op': 'replace', 'path': '/name/', 'value': 'NEW_NAME'}
response = self.patch('/artifact/1/', data=arguments)
self.assertEqual(response.code, 200)
self.assertEqual(a.name, 'NEW_NAME')
a.name = 'Raw data 1'
def test_download(self):
# check success
response = self.get('/download/1')
self.assertEqual(response.code, 200)
self.assertEqual(response.body.decode('ascii'), (
"This installation of Qiita was not equipped with nginx, so it "
"is incapable of serving files. The file you attempted to "
"download is located at raw_data/1_s_G1_L001_sequences.fastq.gz"))
self.assertEqual(
response.headers['Content-Disposition'],
"attachment; filename=1_1_s_G1_L001_sequences.fastq.gz")
# other tests to validate the filename
response = self.get('/download/2')
self.assertEqual(
response.headers['Content-Disposition'],
"attachment; filename=1_1_s_G1_L001_sequences_barcodes.fastq.gz")
response = self.get('/download/3')
self.assertEqual(
response.headers['Content-Disposition'],
"attachment; filename=2_1_seqs.fna")
response = self.get('/download/18')
self.assertEqual(
response.headers['Content-Disposition'],
"attachment; filename=1_prep_1_19700101-000000.txt")
response = self.get('/download/22')
self.assertEqual(
response.headers['Content-Disposition'],
"attachment; filename=7_biom_table.biom")
# failure
response = self.get('/download/1000')
self.assertEqual(response.code, 403)
# directory
a = Artifact(1)
fd, fp = mkstemp(suffix='.html')
close(fd)
with open(fp, 'w') as f:
f.write('\n')
self._clean_up_files.append(fp)
dirpath = mkdtemp()
fd, fp2 = mkstemp(suffix='.txt', dir=dirpath)
close(fd)
with open(fp2, 'w') as f:
f.write('\n')
self._clean_up_files.append(dirpath)
a.set_html_summary(fp, support_dir=dirpath)
for x in a.filepaths:
if x['fp_type'] == 'html_summary_dir':
break
response = self.get('/download/%d' % x['fp_id'])
self.assertEqual(response.code, 200)
fp_name = basename(fp2)
dirname = basename(dirpath)
self.assertEqual(response.body.decode('ascii'),
"- 1 /protected/FASTQ/1/%s/%s FASTQ/1/%s/%s\n" % (
dirname, fp_name, dirname, fp_name))
def test_download(self):
# check success
response = self.get('/download/1')
self.assertEqual(response.code, 200)
self.assertEqual(response.body.decode('ascii'), (
"This installation of Qiita was not equipped with nginx, so it "
"is incapable of serving files. The file you attempted to "
"download is located at raw_data/1_s_G1_L001_sequences.fastq.gz"))
self.assertEqual(
response.headers['Content-Disposition'],
"attachment; filename=1_1_s_G1_L001_sequences.fastq.gz")
# other tests to validate the filename
response = self.get('/download/2')
self.assertEqual(
response.headers['Content-Disposition'],
"attachment; filename=1_1_s_G1_L001_sequences_barcodes.fastq.gz")
response = self.get('/download/3')
self.assertEqual(
response.headers['Content-Disposition'],
"attachment; filename=2_1_seqs.fna")
response = self.get('/download/18')
self.assertEqual(
response.headers['Content-Disposition'],
"attachment; filename=1_prep_1_19700101-000000.txt")
response = self.get('/download/22')
self.assertEqual(
response.headers['Content-Disposition'],
"attachment; filename=7_biom_table.biom")
# failure
response = self.get('/download/1000')
self.assertEqual(response.code, 403)
# directory
a = Artifact(1)
fd, fp = mkstemp(suffix='.html')
close(fd)
with open(fp, 'w') as f:
f.write('\n')
self._clean_up_files.append(fp)
dirpath = mkdtemp()
fd, fp2 = mkstemp(suffix='.txt', dir=dirpath)
close(fd)
with open(fp2, 'w') as f:
f.write('\n')
self._clean_up_files.append(dirpath)
a.set_html_summary(fp, support_dir=dirpath)
for x in a.filepaths:
if x['fp_type'] == 'html_summary_dir':
break
response = self.get('/download/%d' % x['fp_id'])
self.assertEqual(response.code, 200)
fp_name = basename(fp2)
dirname = basename(dirpath)
self.assertEqual(response.body.decode('ascii'),
"- 1 /protected/FASTQ/1/%s/%s FASTQ/1/%s/%s\n" % (
dirname, fp_name, dirname, fp_name))
def artifact_patch_request(user_id, req_op, req_path, req_value=None,
req_from=None):
"""Modifies an attribute of the artifact
Parameters
----------
user_id : str
The id of the user performing the patch operation
req_op : str
The operation to perform on the artifact
req_path : str
The prep information and attribute to patch
req_value : str, optional
The value that needs to be modified
req_from : str, optional
The original path of the element
Returns
-------
dict of {str, str}
A dictionary with the following keys:
- status: str, whether if the request is successful or not
- message: str, if the request is unsuccessful, a human readable error
"""
if req_op == 'replace':
req_path = [v for v in req_path.split('/') if v]
if len(req_path) != 2:
return {'status': 'error',
'message': 'Incorrect path parameter'}
artifact_id = req_path[0]
attribute = req_path[1]
# Check if the user actually has access to the artifact
artifact = Artifact(artifact_id)
access_error = check_access(artifact.study.id, user_id)
if access_error:
return access_error
if not req_value:
return {'status': 'error',
'message': 'A value is required'}
if attribute == 'name':
artifact.name = req_value
return {'status': 'success',
'message': ''}
else:
# We don't understand the attribute so return an error
return {'status': 'error',
'message': 'Attribute "%s" not found. '
'Please, check the path parameter' % attribute}
else:
return {'status': 'error',
'message': 'Operation "%s" not supported. '
'Current supported operations: replace' % req_op}
def test_artifact_patch_request(self):
a = Artifact(1)
test_user = User('*****@*****.**')
self.assertEqual(a.name, 'Raw data 1')
artifact_patch_request(test_user, 1, 'replace', '/name/',
req_value='NEW_NAME')
self.assertEqual(a.name, 'NEW_NAME')
# Reset the name
a.name = 'Raw data 1'
# No access
with self.assertRaises(QiitaHTTPError):
artifact_patch_request(User('*****@*****.**'), 1, 'replace',
'/name/', req_value='NEW_NAME')
# Incorrect path parameter
with self.assertRaises(QiitaHTTPError):
artifact_patch_request(test_user, 1, 'replace',
'/name/wrong/', req_value='NEW_NAME')
# Missing value
with self.assertRaises(QiitaHTTPError):
artifact_patch_request(test_user, 1, 'replace', '/name/')
# Wrong attribute
with self.assertRaises(QiitaHTTPError):
artifact_patch_request(test_user, 1, 'replace',
'/wrong/', req_value='NEW_NAME')
# Wrong operation
with self.assertRaises(QiitaHTTPError):
artifact_patch_request(test_user, 1, 'add', '/name/',
req_value='NEW_NAME')
# Changing visibility
self.assertEqual(a.visibility, 'private')
artifact_patch_request(test_user, 1, 'replace', '/visibility/',
req_value='sandbox')
self.assertEqual(a.visibility, 'sandbox')
# Admin can change to private
artifact_patch_request(User('*****@*****.**'), 1, 'replace',
'/visibility/', req_value='private')
self.assertEqual(a.visibility, 'private')
# Test user can't change to private
with self.assertRaises(QiitaHTTPError):
artifact_patch_request(test_user, 1, 'replace', '/visibility/',
req_value='private')
# Unkown req value
with self.assertRaises(QiitaHTTPError):
artifact_patch_request(test_user, 1, 'replace', '/visibility/',
req_value='wrong')
def list_options_handler_get_req(command_id, artifact_id=None):
"""Returns the available default parameters set for the given command
Parameters
----------
command_id : int
The command id
artifact_id : int, optional
The artifact id so to limit options based on how it has already been
processed
Returns
-------
dict of objects
A dictionary containing the commands information
{'status': str,
'message': str,
'options': list of dicts of {'id: str', 'name': str,
'values': dict of {str: str}}}
"""
def _helper_process_params(params):
return dumps({k: str(v).lower()
for k, v in params.items()},
sort_keys=True)
command = Command(command_id)
rparamers = command.required_parameters.keys()
eparams = []
if artifact_id is not None:
artifact = Artifact(artifact_id)
for job in artifact.jobs(cmd=command):
jstatus = job.status
outputs = job.outputs if job.status == 'success' else None
# this ignore any jobs that weren't successful or are in
# construction, or the results have been deleted [outputs == {}]
if jstatus not in {'success', 'in_construction'} or outputs == {}:
continue
params = job.parameters.values.copy()
for k in rparamers:
del params[k]
eparams.append(_helper_process_params(params))
options = [{
'id': p.id,
'name': p.name,
'values': p.values
} for p in command.default_parameter_sets
if _helper_process_params(p.values) not in eparams]
return {
'status': 'success',
'message': '',
'options': options,
'req_options': command.required_parameters,
'opt_options': command.optional_parameters
}
def test_artifact_post_req(self):
# Create new prep template to attach artifact to
pt = npt.assert_warns(QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {
'1.SKD6.640190': 1
}}), Study(1), '16S')
self._files_to_remove.extend([fp for _, fp in pt.get_filepaths()])
new_artifact_id = get_count('qiita.artifact') + 1
filepaths = {
'raw_forward_seqs': 'uploaded_file.txt',
'raw_barcodes': 'update.txt'
}
obs = artifact_post_req('*****@*****.**', filepaths, 'FASTQ',
'New Test Artifact', pt.id)
exp = {'status': 'success', 'message': ''}
self.assertEqual(obs, exp)
obs = r_client.get('prep_template_%d' % pt.id)
self.assertIsNotNone(obs)
redis_info = loads(r_client.get(loads(obs)['job_id']))
while redis_info['status_msg'] == 'Running':
sleep(0.05)
redis_info = loads(r_client.get(loads(obs)['job_id']))
# Instantiate the artifact to make sure it was made and
# to clean the environment
a = Artifact(new_artifact_id)
self._files_to_remove.extend([fp for _, fp, _ in a.filepaths])
# Test importing an artifact
# Create new prep template to attach artifact to
pt = npt.assert_warns(QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {
'1.SKD6.640190': 1
}}), Study(1), '16S')
self._files_to_remove.extend([fp for _, fp in pt.get_filepaths()])
new_artifact_id_2 = get_count('qiita.artifact') + 1
obs = artifact_post_req('*****@*****.**', {}, 'FASTQ',
'New Test Artifact 2', pt.id, new_artifact_id)
exp = {'status': 'success', 'message': ''}
self.assertEqual(obs, exp)
obs = r_client.get('prep_template_%d' % pt.id)
self.assertIsNotNone(obs)
redis_info = loads(r_client.get(loads(obs)['job_id']))
while redis_info['status_msg'] == 'Running':
sleep(0.05)
redis_info = loads(r_client.get(loads(obs)['job_id']))
# Instantiate the artifact to make sure it was made and
# to clean the environment
a = Artifact(new_artifact_id_2)
self._files_to_remove.extend([fp for _, fp, _ in a.filepaths])
def artifact_status_put_req(artifact_id, user_id, visibility):
"""Set the status of the artifact given
Parameters
----------
artifact_id : int
Artifact being acted on
user_id : str
The user requesting the action
visibility : {'sandbox', 'awaiting_approval', 'private', 'public'}
What to change the visibility to
Returns
-------
dict
Status of action, in the form {'status': status, 'message': msg}
status: status of the action, either success or error
message: Human readable message for status
"""
if visibility not in get_visibilities():
return {'status': 'error',
'message': 'Unknown visibility value: %s' % visibility}
pd = Artifact(int(artifact_id))
sid = pd.study.id
access_error = check_access(sid, user_id)
if access_error:
return access_error
user = User(str(user_id))
status = 'success'
msg = 'Artifact visibility changed to %s' % visibility
# Set the approval to private if needs approval and admin
if visibility == 'private':
if not qiita_config.require_approval:
pd.visibility = 'private'
# Set the approval to private if approval not required
elif user.level == 'admin':
pd.visibility = 'private'
# Trying to set approval without admin privileges
else:
status = 'error'
msg = 'User does not have permissions to approve change'
else:
pd.visibility = visibility
LogEntry.create('Warning', '%s changed artifact %s (study %d) to %s' % (
user_id, artifact_id, sid, visibility))
return {'status': status,
'message': msg}
def test_get_artifact_summary_handler(self):
a = Artifact(1)
# Add a summary to the artifact
fd, fp = mkstemp(suffix=".html")
close(fd)
with open(fp, 'w') as f:
f.write('<b>HTML TEST - not important</b>\n')
a = Artifact(1)
a.set_html_summary(fp)
self._files_to_remove.extend([fp, a.html_summary_fp[1]])
summary = relpath(a.html_summary_fp[1], qiita_config.base_data_dir)
response = self.get('/artifact/html_summary/%s' % summary)
self.assertEqual(response.code, 200)
self.assertEqual(response.body, '<b>HTML TEST - not important</b>\n')
def display_template(self, preprocessed_data_id, msg, msg_level):
"""Simple function to avoid duplication of code"""
preprocessed_data_id = int(preprocessed_data_id)
try:
preprocessed_data = Artifact(preprocessed_data_id)
except QiitaDBUnknownIDError:
raise HTTPError(
404, "Artifact %d does not exist!" % preprocessed_data_id)
else:
user = self.current_user
if user.level != 'admin':
raise HTTPError(
403, "No permissions of admin, "
"get/VAMPSSubmitHandler: %s!" % user.id)
prep_template = PrepTemplate(preprocessed_data.prep_template)
sample_template = SampleTemplate(preprocessed_data.study)
study = Study(preprocessed_data.study)
stats = [('Number of samples', len(prep_template)),
('Number of metadata headers',
len(sample_template.categories()))]
demux = [
path for _, path, ftype in preprocessed_data.get_filepaths()
if ftype == 'preprocessed_demux'
]
demux_length = len(demux)
if not demux_length:
msg = ("Study does not appear to have demultiplexed "
"sequences associated")
msg_level = 'danger'
elif demux_length > 1:
msg = ("Study appears to have multiple demultiplexed files!")
msg_level = 'danger'
elif demux_length == 1:
demux_file = demux[0]
demux_file_stats = demux_stats(demux_file)
stats.append(('Number of sequences', demux_file_stats.n))
msg_level = 'success'
self.render('vamps_submission.html',
study_title=study.title,
stats=stats,
message=msg,
study_id=study.id,
level=msg_level,
preprocessed_data_id=preprocessed_data_id)
def get(self, analysis_id):
analysis_id = int(analysis_id.split("/")[0])
analysis = Analysis(analysis_id)
check_analysis_access(self.current_user, analysis)
jobres = defaultdict(list)
for jobject in analysis.jobs:
results = []
for res in jobject.results:
name = basename(res)
if name.startswith('index'):
name = basename(dirname(res)).replace('_', ' ')
results.append((res, name))
jobres[jobject.datatype].append((jobject.command[0], results))
dropped_samples = analysis.dropped_samples
dropped = defaultdict(list)
for proc_data_id, samples in viewitems(dropped_samples):
if not samples:
continue
proc_data = Artifact(proc_data_id)
data_type = proc_data.data_type
dropped[data_type].append(
(proc_data.study.title, len(samples), ', '.join(samples)))
self.render("analysis_results.html",
analysis_id=analysis_id,
jobres=jobres,
aname=analysis.name,
dropped=dropped,
basefolder=get_db_files_base_dir())
def get(self):
# Format sel_data to get study IDs for the processed data
sel_data = defaultdict(dict)
proc_data_info = {}
sel_samps = self.current_user.default_analysis.samples
for aid, samples in viewitems(sel_samps):
a = Artifact(aid)
sel_data[a.study][aid] = samples
# Also get processed data info
processing_parameters = a.processing_parameters
if processing_parameters is None:
params = None
algorithm = None
else:
cmd = processing_parameters.command
params = processing_parameters.values
if 'reference' in params:
ref = Reference(params['reference'])
del params['reference']
params['reference_name'] = ref.name
params['reference_version'] = ref.version
algorithm = '%s (%s)' % (cmd.software.name, cmd.name)
proc_data_info[aid] = {
'processed_date': str(a.timestamp),
'algorithm': algorithm,
'data_type': a.data_type,
'params': params
}
self.render("analysis_selected.html", sel_data=sel_data,
proc_info=proc_data_info)
def post(self, study_id, prep_id):
study = self.safe_get_study(study_id)
if study is None:
return
prep_id = to_int(prep_id)
try:
p = PrepTemplate(prep_id)
except QiitaDBUnknownIDError:
self.fail('Preparation not found', 404)
return
if p.study_id != study.id:
self.fail('Preparation ID not associated with the study', 409)
return
artifact_deets = json_decode(self.request.body)
_, upload = get_mountpoint('uploads')[0]
base = os.path.join(upload, study_id)
filepaths = [(os.path.join(base, fp), fp_type)
for fp, fp_type in artifact_deets['filepaths']]
try:
art = Artifact.create(filepaths,
artifact_deets['artifact_type'],
artifact_deets['artifact_name'],
p)
except QiitaError as e:
self.fail(str(e), 406)
return
self.write({'id': art.id})
self.set_status(201)
self.finish()
def artifact_post_req(user, artifact_id):
"""Deletes the artifact
Parameters
----------
user : qiita_db.user.User
The user requesting the action
artifact_id : int
Id of the artifact being deleted
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
check_artifact_access(user, artifact)
analysis = artifact.analysis
if analysis:
# Do something when deleting in the analysis part to keep track of it
redis_key = "analysis_%s" % analysis.id
else:
pt_id = artifact.prep_templates[0].id
redis_key = PREP_TEMPLATE_KEY_FORMAT % pt_id
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('delete_artifact')
params = Parameters.load(cmd, values_dict={'artifact': artifact_id})
job = ProcessingJob.create(user, params)
r_client.set(redis_key, dumps({'job_id': job.id, 'is_qiita_job': True}))
job.submit()
def test_artifact_post_req(self):
# Create new prep template to attach artifact to
pt = npt.assert_warns(
QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {'1.SKD6.640190': 1}}), Study(1), '16S')
self._files_to_remove.extend([fp for _, fp in pt.get_filepaths()])
filepaths = {'raw_forward_seqs': 'uploaded_file.txt',
'raw_barcodes': 'update.txt'}
obs = artifact_post_req(
'*****@*****.**', filepaths, 'FASTQ', 'New Test Artifact', pt.id)
exp = {'status': 'success',
'message': ''}
self.assertEqual(obs, exp)
wait_for_prep_information_job(pt.id)
# Test importing an artifact
# Create new prep template to attach artifact to
pt = npt.assert_warns(
QiitaDBWarning, PrepTemplate.create,
pd.DataFrame({'new_col': {'1.SKD6.640190': 1}}), Study(1), '16S')
self._files_to_remove.extend([fp for _, fp in pt.get_filepaths()])
obs = artifact_post_req(
'*****@*****.**', {}, 'Demultiplexed', 'New Test Artifact 2',
pt.id, 3)
exp = {'status': 'success',
'message': ''}
self.assertEqual(obs, exp)
wait_for_prep_information_job(pt.id)
# Instantiate the artifact to make sure it was made and
# to clean the environment
a = Artifact(pt.artifact.id)
self._files_to_remove.extend([fp for _, fp, _ in a.filepaths])
def process_artifact_handler_get_req(artifact_id):
"""Returns the information for the process artifact handler
Parameters
----------
artifact_id : int
The artifact to be processed
Returns
-------
dict of str
A dictionary containing the artifact information
{'status': str,
'message': str,
'name': str,
'type': str}
"""
artifact = Artifact(artifact_id)
return {
'status': 'success',
'message': '',
'name': artifact.name,
'type': artifact.artifact_type,
'study_id': artifact.study.id
}
def post(self, preprocessed_data_id):
user = self.current_user
# make sure user is admin and can therefore actually submit to EBI
if user.level != 'admin':
raise HTTPError(403, reason="User %s cannot submit to EBI!" %
user.id)
submission_type = self.get_argument('submission_type')
if submission_type not in ['ADD', 'MODIFY']:
raise HTTPError(403, reason="User: %s, %s is not a recognized "
"submission type" % (user.id, submission_type))
study = Artifact(preprocessed_data_id).study
state = study.ebi_submission_status
if state == 'submitting':
message = "Cannot resubmit! Current state is: %s" % state
self.display_template(preprocessed_data_id, message, 'danger')
else:
qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
cmd = qiita_plugin.get_command('submit_to_EBI')
params = Parameters.load(
cmd, values_dict={'artifact': preprocessed_data_id,
'submission_type': submission_type})
job = ProcessingJob.create(user, params, True)
r_client.set('ebi_submission_%s' % preprocessed_data_id,
dumps({'job_id': job.id, 'is_qiita_job': True}))
job.submit()
level = 'success'
message = 'EBI submission started. Job id: %s' % job.id
self.redirect("%s/study/description/%d?level=%s&message=%s" % (
qiita_config.portal_dir, study.id, level, url_escape(message)))
def get(self):
# Format sel_data to get study IDs for the processed data
sel_data = defaultdict(dict)
proc_data_info = {}
sel_samps = self.current_user.default_analysis.samples
for pid, samps in viewitems(sel_samps):
proc_data = Artifact(pid)
sel_data[proc_data.study][pid] = samps
# Also get processed data info
parameters = proc_data.processing_parameters
reference = Reference(parameters.values['reference'])
proc_data_info[pid] = {
'processed_date': str(proc_data.timestamp),
'algorithm': parameters.command.name,
'reference_name': reference.name,
'reference_version': reference.version,
'sequence_filepath': reference.sequence_fp,
'taxonomy_filepath': reference.taxonomy_fp,
'tree_filepath': reference.tree_fp,
'data_type': proc_data.data_type
}
self.render("analysis_selected.html",
sel_data=sel_data,
proc_info=proc_data_info)
def setUp(self):
uploads_path = get_mountpoint('uploads')[0][1]
# Create prep test file to point at
self.update_fp = join(uploads_path, '1', 'update.txt')
with open(self.update_fp, 'w') as f:
f.write("""sample_name\tnew_col\n1.SKD6.640190\tnew_value\n""")
self._files_to_remove = [self.update_fp]
self._files_to_remove = []
# creating temporal files and artifact
# NOTE: we don't need to remove the artifact created cause it's
# used to test the delete functionality
fd, fp = mkstemp(suffix='_seqs.fna')
close(fd)
with open(fp, 'w') as f:
f.write(">1.sid_r4_0 M02034:17:000000000-A5U18:1:1101:15370:1394 "
"1:N:0:1 orig_bc=CATGAGCT new_bc=CATGAGCT bc_diffs=0\n"
"GTGTGCCAGCAGCCGCGGTAATACGTAGGG\n")
# 4 Demultiplexed
filepaths_processed = [(fp, 4)]
# 1 for default parameters and input data
exp_params = Parameters.from_default_params(DefaultParameters(1),
{'input_data': 1})
self.artifact = Artifact.create(filepaths_processed, "Demultiplexed",
parents=[Artifact(1)],
processing_parameters=exp_params)
def post(self, preprocessed_data_id):
user = self.current_user
# make sure user is admin and can therefore actually submit to VAMPS
if user.level != 'admin':
raise HTTPError(403, "User %s cannot submit to VAMPS!" % user.id)
msg = ''
msg_level = 'success'
study = Artifact(preprocessed_data_id).study
study_id = study.id
state = study.ebi_submission_status
if state == 'submitting':
msg = "Cannot resubmit! Current state is: %s" % state
msg_level = 'danger'
else:
channel = user.id
job_id = submit(channel, submit_to_VAMPS,
int(preprocessed_data_id))
self.render(
'compute_wait.html',
job_id=job_id,
title='VAMPS Submission',
completion_redirect=('/study/description/%s?top_tab='
'preprocessed_data_tab&sub_tab=%s' %
(study_id, preprocessed_data_id)))
return
self.display_template(preprocessed_data_id, msg, msg_level)
def test_download(self):
# check failures
response = self.get('/public_artifact_download/')
self.assertEqual(response.code, 422)
self.assertEqual(response.reason, 'You need to specify an artifact id')
response = self.get('/public_artifact_download/?artifact_id=10000')
self.assertEqual(response.code, 404)
self.assertEqual(response.reason, 'Artifact does not exist')
response = self.get('/public_artifact_download/?artifact_id=3')
self.assertEqual(response.code, 404)
self.assertEqual(
response.reason, 'Artifact is not public. If this is '
'a mistake contact: [email protected]')
# check success
Artifact(5).visibility = 'public'
response = self.get('/public_artifact_download/?artifact_id=5')
self.assertEqual(response.code, 200)
exp = ('- [0-9]* /protected/processed_data/'
'1_study_1001_closed_reference_otu_table.biom '
'processed_data/1_study_1001_closed_reference_otu_table.biom\n'
'- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
'mapping_files/5_mapping_file.txt')
self.assertRegex(response.body.decode('ascii'), exp)
def get(self):
artifact_id = self.get_argument("artifact_id", None)
if artifact_id is None:
raise HTTPError(422, reason='You need to specify an artifact id')
else:
try:
artifact = Artifact(artifact_id)
except QiitaDBUnknownIDError:
raise HTTPError(404, reason='Artifact does not exist')
else:
if artifact.visibility != 'public':
raise HTTPError(404, reason='Artifact is not public. If '
'this is a mistake contact: '
'*****@*****.**')
else:
to_download = self._list_artifact_files_nginx(artifact)
if not to_download:
raise HTTPError(422, reason='Nothing to download. If '
'this is a mistake contact: '
'*****@*****.**')
else:
self._write_nginx_file_list(to_download)
zip_fn = 'artifact_%s_%s.zip' % (
artifact_id, datetime.now().strftime(
'%m%d%y-%H%M%S'))
self._set_nginx_headers(zip_fn)
self.finish()
def artifact_delete_req(artifact_id, user_id):
"""Deletes the artifact
Parameters
----------
artifact_id : int
Artifact being acted on
user_id : str
The user requesting the action
Returns
-------
dict
Status of action, in the form {'status': status, 'message': msg}
status: status of the action, either success or error
message: Human readable message for status
"""
pd = Artifact(int(artifact_id))
pt_id = pd.prep_templates[0].id
access_error = check_access(pd.study.id, user_id)
if access_error:
return access_error
job_id = safe_submit(user_id, delete_artifact, artifact_id)
r_client.set(PREP_TEMPLATE_KEY_FORMAT % pt_id, dumps({'job_id': job_id}))
return {'status': 'success', 'message': ''}
def post(self, preprocessed_data_id):
user = self.current_user
# make sure user is admin and can therefore actually submit to EBI
if user.level != 'admin':
raise HTTPError(403, "User %s cannot submit to EBI!" %
user.id)
submission_type = self.get_argument('submission_type')
if submission_type not in ['ADD', 'MODIFY']:
raise HTTPError(403, "User: %s, %s is not a recognized submission "
"type" % (user.id, submission_type))
msg = ''
msg_level = 'success'
study = Artifact(preprocessed_data_id).study
study_id = study.id
state = study.ebi_submission_status
if state == 'submitting':
msg = "Cannot resubmit! Current state is: %s" % state
msg_level = 'danger'
else:
channel = user.id
job_id = submit(channel, submit_to_ebi, int(preprocessed_data_id),
submission_type)
self.render('compute_wait.html',
job_id=job_id, title='EBI Submission',
completion_redirect=('/study/description/%s?top_tab='
'preprocessed_data_tab&sub_tab=%s'
% (study_id,
preprocessed_data_id)))
return
self.display_template(preprocessed_data_id, msg, msg_level)
def artifact_post_req(user, artifact_id):
"""Deletes the artifact
Parameters
----------
user : qiita_db.user.User
The user requesting the action
artifact_id : int
Id of the artifact being deleted
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
check_artifact_access(user, artifact)
analysis = artifact.analysis
if analysis:
# Do something when deleting in the analysis part to keep track of it
redis_key = "analysis_%s" % analysis.id
else:
pt_id = artifact.prep_templates[0].id
redis_key = PREP_TEMPLATE_KEY_FORMAT % pt_id
job_id = safe_submit(user.id, delete_artifact, artifact_id)
r_client.set(redis_key, dumps({'job_id': job_id, 'is_qiita_job': False}))
def artifact_status_put_req(artifact_id, user_id, visibility):
"""Set the status of the artifact given
Parameters
----------
artifact_id : int
Artifact being acted on
user_id : str
The user requesting the action
visibility : {'sandbox', 'awaiting_approval', 'private', 'public'}
What to change the visibility to
Returns
-------
dict
Status of action, in the form {'status': status, 'message': msg}
status: status of the action, either success or error
message: Human readable message for status
"""
if visibility not in get_visibilities():
return {'status': 'error',
'message': 'Unknown visiblity value: %s' % visibility}
pd = Artifact(int(artifact_id))
access_error = check_access(pd.study.id, user_id)
if access_error:
return access_error
user = User(str(user_id))
status = 'success'
msg = 'Artifact visibility changed to %s' % visibility
# Set the approval to private if needs approval and admin
if visibility == 'private':
if not qiita_config.require_approval:
pd.visibility = 'private'
# Set the approval to private if approval not required
elif user.level == 'admin':
pd.visibility = 'private'
# Trying to set approval without admin privileges
else:
status = 'error'
msg = 'User does not have permissions to approve change'
else:
pd.visibility = visibility
return {'status': status,
'message': msg}
def artifact_summary_post_request(user_id, artifact_id):
"""Launches the HTML summary generation and returns the job information
Parameters
----------
user_id : str
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the artifact summary information
{'status': str,
'message': str,
'job': list of [str, str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
access_error = check_access(artifact.study.id, user_id)
if access_error:
return access_error
# Check if the summary is being generated or has been already generated
command = Command.get_html_generator(artifact.artifact_type)
jobs = artifact.jobs(cmd=command)
jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
if jobs:
# The HTML summary is either being generated or already generated.
# Return the information of that job so we only generate the HTML
# once
job = jobs[0]
else:
# Create a new job to generate the HTML summary and return the newly
# created job information
job = ProcessingJob.create(
User(user_id),
Parameters.load(command, values_dict={'input_data': artifact_id}))
job.submit()
return {'status': 'success',
'message': '',
'job': [job.id, job.status, job.step]}
def display_template(self, preprocessed_data_id, msg, msg_level):
"""Simple function to avoid duplication of code"""
preprocessed_data_id = int(preprocessed_data_id)
try:
preprocessed_data = Artifact(preprocessed_data_id)
except QiitaDBUnknownIDError:
raise HTTPError(404, "Artifact %d does not exist!" % preprocessed_data_id)
else:
user = self.current_user
if user.level != "admin":
raise HTTPError(403, "No permissions of admin, " "get/VAMPSSubmitHandler: %s!" % user.id)
prep_template = PrepTemplate(preprocessed_data.prep_template)
sample_template = SampleTemplate(preprocessed_data.study)
study = Study(preprocessed_data.study)
stats = [
("Number of samples", len(prep_template)),
("Number of metadata headers", len(sample_template.categories())),
]
demux = [path for _, path, ftype in preprocessed_data.get_filepaths() if ftype == "preprocessed_demux"]
demux_length = len(demux)
if not demux_length:
msg = "Study does not appear to have demultiplexed " "sequences associated"
msg_level = "danger"
elif demux_length > 1:
msg = "Study appears to have multiple demultiplexed files!"
msg_level = "danger"
elif demux_length == 1:
demux_file = demux[0]
demux_file_stats = demux_stats(demux_file)
stats.append(("Number of sequences", demux_file_stats.n))
msg_level = "success"
self.render(
"vamps_submission.html",
study_title=study.title,
stats=stats,
message=msg,
study_id=study.id,
level=msg_level,
preprocessed_data_id=preprocessed_data_id,
)
def artifact_graph_get_req(artifact_id, direction, user_id):
"""Creates graphs of ancestor or descendant artifacts from given one
Parameters
----------
artifact_id : int
Artifact ID to get graph for
direction : {'ancestors', 'descendants'}
What direction to get the graph in
Returns
-------
dict of lists of tuples
A dictionary containing the edge list representation of the graph,
and the node labels. Formatted as:
{'status': status,
'message': message,
'edge_list': [(0, 1), (0, 2)...],
'node_labels': [(0, 'label0'), (1, 'label1'), ...]}
Notes
-----
Nodes are identified by the corresponding Artifact ID.
"""
access_error = check_access(Artifact(artifact_id).study.id, user_id)
if access_error:
return access_error
if direction == 'descendants':
G = Artifact(int(artifact_id)).descendants
elif direction == 'ancestors':
G = Artifact(int(artifact_id)).ancestors
else:
return {
'status': 'error',
'message': 'Unknown directon %s' % direction
}
node_labels = [(n.id, ' - '.join([n.name, n.artifact_type]))
for n in G.nodes()]
return {'edge_list': [(n.id, m.id) for n, m in G.edges()],
'node_labels': node_labels,
'status': 'success',
'message': ''}
def test_propagate_visibility(self):
a = Artifact(4)
a.visibility = 'public'
_propagate_visibility(a)
self.assertEqual(Artifact(1).visibility, 'public')
self.assertEqual(Artifact(2).visibility, 'public')
self.assertEqual(Artifact(4).visibility, 'public')
a.visibility = 'private'
_propagate_visibility(a)
self.assertEqual(Artifact(1).visibility, 'private')
self.assertEqual(Artifact(2).visibility, 'private')
self.assertEqual(Artifact(4).visibility, 'private')
a = Artifact(2)
a.visibility = 'public'
_propagate_visibility(a)
self.assertEqual(Artifact(1).visibility, 'private')
self.assertEqual(Artifact(2).visibility, 'private')
self.assertEqual(Artifact(4).visibility, 'private')
def request_approval(self, study, user, callback):
"""Changes the status of the current study to "awaiting_approval"
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
"""
pd_id = int(self.get_argument('pd_id'))
pd = Artifact(pd_id)
pd.visibility = 'awaiting_approval'
_propagate_visibility(pd)
msg = "Processed data sent to admin for approval"
msg_level = "success"
callback((msg, msg_level, "processed_data_tab", pd_id, None))
def make_public(self, study, user, callback):
"""Makes the current study public
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
"""
pd_id = int(self.get_argument('pd_id'))
pd = Artifact(pd_id)
pd.visibility = 'public'
_propagate_visibility(pd)
msg = "Processed data set to public"
msg_level = "success"
callback((msg, msg_level, "processed_data_tab", pd_id, None))
def make_sandbox(self, study, user, callback):
"""Reverts the current study to the 'sandbox' status
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
"""
pd_id = int(self.get_argument('pd_id'))
pd = Artifact(pd_id)
pd.visibility = 'sandbox'
_propagate_visibility(pd)
msg = "Processed data reverted to sandbox"
msg_level = "success"
callback((msg, msg_level, "processed_data_tab", pd_id, None))
def artifact_summary_post_request(user, artifact_id):
"""Launches the HTML summary generation and returns the job information
Parameters
----------
user : qiita_db.user.User
The user making the request
artifact_id : int or str
The artifact id
Returns
-------
dict of objects
A dictionary containing the job summary information
{'job': [str, str, str]}
"""
artifact_id = int(artifact_id)
artifact = Artifact(artifact_id)
check_artifact_access(user, artifact)
# Check if the summary is being generated or has been already generated
command = Command.get_html_generator(artifact.artifact_type)
jobs = artifact.jobs(cmd=command)
jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
if jobs:
# The HTML summary is either being generated or already generated.
# Return the information of that job so we only generate the HTML
# once - Magic number 0 -> we are ensuring that there is only one
# job generating the summary, so we can use the index 0 to access to
# that job
job = jobs[0]
else:
# Create a new job to generate the HTML summary and return the newly
# created job information
job = ProcessingJob.create(user, Parameters.load(
command, values_dict={'input_data': artifact_id}), True)
job.submit()
return {'job': [job.id, job.status, job.step]}
def get(self):
user = self.current_user
if user.level != 'admin':
raise HTTPError(403, 'User %s is not admin' % self.current_user)
studies = defaultdict(list)
for artifact in Artifact.iter_by_visibility('awaiting_approval'):
studies[artifact.study].append(artifact.id)
parsed_studies = [(s.id, s.title, s.owner.email, pds)
for s, pds in viewitems(studies)]
self.render('admin_approval.html',
study_info=parsed_studies)
def test_download(self):
# check success
response = self.get('/download/1')
self.assertEqual(response.code, 200)
self.assertEqual(response.body, (
"This installation of Qiita was not equipped with nginx, so it "
"is incapable of serving files. The file you attempted to "
"download is located at raw_data/1_s_G1_L001_sequences.fastq.gz"))
# failure
response = self.get('/download/1000')
self.assertEqual(response.code, 403)
# directory
a = Artifact(1)
fd, fp = mkstemp(suffix='.html')
close(fd)
with open(fp, 'w') as f:
f.write('\n')
self._clean_up_files.append(fp)
dirpath = mkdtemp()
fd, fp2 = mkstemp(suffix='.txt', dir=dirpath)
close(fd)
with open(fp2, 'w') as f:
f.write('\n')
self._clean_up_files.append(dirpath)
a.set_html_summary(fp, support_dir=dirpath)
for fp_id, _, fp_type in a.filepaths:
if fp_type == 'html_summary_dir':
break
response = self.get('/download/%d' % fp_id)
self.assertEqual(response.code, 200)
fp_name = basename(fp2)
dirname = basename(dirpath)
self.assertEqual(
response.body, "- 1 /protected/FASTQ/1/%s/%s FASTQ/1/%s/%s\n"
% (dirname, fp_name, dirname, fp_name))
def create_raw_data(artifact_type, prep_template, filepaths, name=None):
"""Creates a new raw data
Needs to be dispachable because it moves large files
Parameters
----------
artifact_type: str
The artifact type
prep_template : qiita_db.metadata_template.prep_template.PrepTemplate
The template to attach the artifact
filepaths : list of (str, str)
The list with filepaths and their filepath types
name : str, optional
The name of the new artifact
Returns
-------
dict of {str: str}
A dict of the form {'status': str, 'message': str}
"""
from qiita_db.artifact import Artifact
status = 'success'
msg = ''
try:
Artifact.create(filepaths, artifact_type, name=name,
prep_template=prep_template)
except Exception as e:
# We should hit this exception rarely (that's why it is an
# exception) since at this point we have done multiple checks.
# However, it can occur in weird cases, so better let the GUI know
# that this failed
return {'status': 'danger',
'message': "Error creating artifact: %s" % str(e)}
return {'status': status, 'message': msg}
def delete_artifact(artifact_id):
"""Deletes an artifact from the system
Parameters
----------
artifact_id : int
The artifact to delete
Returns
-------
dict of {str: str}
A dict of the form {'status': str, 'message': str}
"""
from qiita_db.artifact import Artifact
status = 'success'
msg = ''
try:
Artifact.delete(artifact_id)
except Exception as e:
status = 'danger'
msg = str(e)
return {'status': status, 'message': msg}
def artifact_types_get_req():
"""Gets artifact types and descriptions available
Returns
-------
dict of objects
{'status': status,
'message': message,
'types': [[str, str], ...]}
types holds type and description of the artifact type, in the form
[[artifact_type, description], ...]
"""
return {'status': 'success',
'message': '',
'types': Artifact.types()}