def test_create(self):
"""Makes sure creation works as expected"""
# make first job
new = Job.create("18S", "Alpha Rarefaction", {"opt1": 4}, Analysis(1))
self.assertEqual(new.id, 4)
# make sure job inserted correctly
obs = self.conn_handler.execute_fetchall("SELECT * FROM qiita.job "
"WHERE job_id = 4")
exp = [[4, 2, 1, 3, '{"opt1":4}', None]]
self.assertEqual(obs, exp)
# make sure job added to analysis correctly
obs = self.conn_handler.execute_fetchall("SELECT * FROM "
"qiita.analysis_job WHERE "
"job_id = 4")
exp = [[1, 4]]
self.assertEqual(obs, exp)
# make second job with diff datatype and command to test column insert
new = Job.create("16S", "Beta Diversity", {"opt1": 4}, Analysis(1))
self.assertEqual(new.id, 5)
# make sure job inserted correctly
obs = self.conn_handler.execute_fetchall("SELECT * FROM qiita.job "
"WHERE job_id = 5")
exp = [[5, 1, 1, 2, '{"opt1":4}', None]]
self.assertEqual(obs, exp)
# make sure job added to analysis correctly
obs = self.conn_handler.execute_fetchall("SELECT * FROM "
"qiita.analysis_job WHERE "
"job_id = 5")
exp = [[1, 5]]
self.assertEqual(obs, exp)
def test_generate_analysis_tgz(self):
obs_sout, obs_serr, obs_return = _generate_analysis_tgz(Analysis(1))
# not testing obs_serr as it will change depending on the system's tar
# version
self.assertEqual(obs_sout, "")
self.assertEqual(obs_return, 0)
def get(self, analysis_id):
analysis_id = int(analysis_id.split("/")[0])
analysis = Analysis(analysis_id)
check_analysis_access(self.current_user, analysis)
jobres = defaultdict(list)
for job in analysis.jobs:
jobject = Job(job)
jobres[jobject.datatype].append(
(jobject.command[0], jobject.results))
dropped = {}
dropped_samples = analysis.dropped_samples
if dropped_samples:
for proc_data_id, samples in viewitems(dropped_samples):
proc_data = ProcessedData(proc_data_id)
key = "Data type %s, Study: %s" % (proc_data.data_type(),
proc_data.study)
dropped[key] = samples
self.render("analysis_results.html",
jobres=jobres,
aname=analysis.name,
dropped=dropped,
basefolder=get_db_files_base_dir())
def post(self, analysis_id):
user = self.current_user
analysis_id = int(analysis_id)
rarefaction_depth = self.get_argument('rarefaction-depth')
# convert to integer if rarefaction level given
if rarefaction_depth:
rarefaction_depth = int(rarefaction_depth)
else:
rarefaction_depth = None
check_analysis_access(User(user), analysis_id)
command_args = self.get_arguments("commands")
split = [x.split("#") for x in command_args]
commands = ["%s: %s" % (s[0], s[1]) for s in split]
analysis = Analysis(analysis_id)
self.render("analysis_waiting.html",
user=user,
aid=analysis_id,
aname=analysis.name,
commands=commands)
app = RunAnalysis()
app(user,
analysis,
split,
comm_opts={},
rarefaction_depth=rarefaction_depth)
def test_create_exists(self):
"""Makes sure creation doesn't duplicate a job"""
with self.assertRaises(QiitaDBDuplicateError):
Job.create("18S", "Beta Diversity", {
"--otu_table_fp": 1,
"--mapping_fp": 1
}, Analysis(1))
def get(self, analysis_id):
analysis_id = int(analysis_id.split("/")[0])
analysis = Analysis(analysis_id)
check_analysis_access(self.current_user, analysis)
jobres = defaultdict(list)
for jobject in analysis.jobs:
results = []
for res in jobject.results:
name = basename(res)
if name.startswith('index'):
name = basename(dirname(res)).replace('_', ' ')
results.append((res, name))
jobres[jobject.datatype].append((jobject.command[0], results))
dropped_samples = analysis.dropped_samples
dropped = defaultdict(list)
for proc_data_id, samples in viewitems(dropped_samples):
if not samples:
continue
proc_data = Artifact(proc_data_id)
data_type = proc_data.data_type
dropped[data_type].append(
(proc_data.study.title, len(samples), ', '.join(samples)))
self.render("analysis_results.html",
analysis_id=analysis_id,
jobres=jobres,
aname=analysis.name,
dropped=dropped,
basefolder=get_db_files_base_dir())
def post(self, analysis_id):
analysis_id = int(analysis_id.split("/")[0])
analysis_id_sent = int(self.get_argument('analysis_id'))
action = self.get_argument('action')
if analysis_id != analysis_id_sent or action != 'delete_analysis':
raise QiitaPetAuthorizationError(
self.current_user.id,
'analysis/results/%d-delete' % analysis_id)
analysis = Analysis(analysis_id)
analysis_name = analysis.name
check_analysis_access(self.current_user, analysis)
try:
Analysis.delete(analysis_id)
msg = ("Analysis <b><i>%s</i></b> has been deleted." %
(analysis_name))
level = "success"
except Exception as e:
e = str(e)
msg = ("Couldn't remove <b><i>%s</i></b> analysis: %s" %
(analysis_name, e))
level = "danger"
LogEntry.create(
'Runtime',
"Couldn't remove analysis ID %d: %s" % (analysis_id, e))
self.redirect(u"/analysis/show/?level=%s&message=%s" % (level, msg))
def test_analyisis_graph_handler_get_request(self):
obs = analyisis_graph_handler_get_request(1, User('*****@*****.**'))
# The job id is randomly generated in the test environment. Gather
# it here. There is only 1 job in the first artifact of the analysis
job_id = Analysis(1).artifacts[0].jobs()[0].id
exp = {
'edges': [(8, job_id), (job_id, 9)],
'nodes': [('job', 'job', job_id, 'Single Rarefaction', 'success'),
('artifact', 'BIOM', 9, 'noname\n(BIOM)', 'artifact'),
('artifact', 'BIOM', 8, 'noname\n(BIOM)', 'artifact')],
'workflow':
None
}
self.assertItemsEqual(obs, exp)
self.assertItemsEqual(obs['edges'], exp['edges'])
self.assertItemsEqual(obs['nodes'], exp['nodes'])
self.assertIsNone(obs['workflow'])
# An admin has full access to the analysis
obs = analyisis_graph_handler_get_request(1, User('*****@*****.**'))
self.assertItemsEqual(obs, exp)
self.assertItemsEqual(obs['edges'], exp['edges'])
self.assertItemsEqual(obs['nodes'], exp['nodes'])
# If the analysis is shared with the user he also has access
obs = analyisis_graph_handler_get_request(1, User('*****@*****.**'))
self.assertItemsEqual(obs, exp)
self.assertItemsEqual(obs['edges'], exp['edges'])
self.assertItemsEqual(obs['nodes'], exp['nodes'])
# The user doesn't have access to the analysis
with self.assertRaises(HTTPError):
analyisis_graph_handler_get_request(1, User('*****@*****.**'))
def test_get_no_access(self):
s = Analysis(2)
u = User('*****@*****.**')
args = {'selected': u.id, 'id': 2}
response = self.get('/analysis/sharing/', args)
self.assertEqual(response.code, 403)
self.assertEqual(s.shared_with, [])
def test_redis_comms(self):
"""Make sure redis communication happens"""
msgs = []
redis = Redis()
pubsub = redis.pubsub()
pubsub.subscribe("*****@*****.**")
app = RunAnalysis()
app("*****@*****.**", Analysis(2), [], rarefaction_depth=100)
for msg in pubsub.listen():
if msg['type'] == 'message':
msgs.append(msg['data'])
if "allcomplete" in msg['data']:
pubsub.unsubscribe("*****@*****.**")
break
self.assertEqual(
msgs,
['{"msg": "Running", "command": "18S: Beta Diversity", '
'"analysis": 2}',
'{"msg": "ERROR", "command": "18S: Beta Diversity", '
'"analysis": 2}',
'{"msg": "allcomplete", "analysis": 2}'])
log = self.conn_handler.execute_fetchall(
"SELECT * from qiita.logging")
self.assertEqual(1, len(log))
log = log[0]
self.assertEqual(1, log[0])
self.assertEqual(2, log[2])
self.assertTrue(len(log[3]) > 0)
self.assertTrue('[{"job": 3, "analysis": 2}]')
def test_set_options(self):
new = Job.create("18S", "Alpha Rarefaction", {"opt1": 4}, Analysis(1))
new.options = self.options
self.options['--output_dir'] = join(
get_db_files_base_dir(), 'job/4_alpha_rarefaction.'
'py_output_dir')
self.assertEqual(new.options, self.options)
def test_set_options(self):
new = Job.create("18S", "Alpha Rarefaction", {"opt1": 4}, Analysis(1))
new.options = self.options
self.options['--output_dir'] = join(self._job_folder,
'4_alpha_rarefaction.'
'py_output_dir')
self.assertEqual(new.options, self.options)
def test_delete_analysis(self):
# as samples have been submitted to EBI, this will fail
job = self._create_job('delete_analysis', {'analysis_id': 1})
private_task(job.id)
self.assertEqual(job.status, 'success')
with self.assertRaises(QiitaDBUnknownIDError):
Analysis(1)
def test_get(self):
a = Analysis(1)
u = User('*****@*****.**')
self.assertEqual(a.shared_with, [u])
# deselecting
args = {'deselected': u.id, 'id': a.id}
response = self.get('/analysis/sharing/', args)
self.assertEqual(response.code, 200)
exp = {'users': [], 'links': ''}
self.assertEqual(loads(response.body), exp)
self.assertEqual(a.shared_with, [])
# Make sure unshared message added to the system
self.assertEqual("Analysis 'SomeAnalysis' has been unshared with you.",
u.messages()[0][1])
# selecting
args = {'selected': u.id, 'id': a.id}
response = self.get('/analysis/sharing/', args)
self.assertEqual(response.code, 200)
exp = {
'users': ['*****@*****.**'],
'links':
('<a target="_blank" href="mailto:[email protected]">Shared</a>')
}
self.assertEqual(loads(response.body), exp)
self.assertEqual(a.shared_with, [u])
# Make sure shared message added to the system
self.assertEqual(
'Analysis <a href="/analysis/description/1">\'SomeAnalysis\'</a> '
'has been shared with you.',
u.messages()[0][1])
def analyisis_job_handler_get_request(analysis_id, user):
"""Returns the job information of the analysis
Parameters
----------
analysis_id: int
The analysis id
user : qiita_db.user.User
The user performing the request
Returns
-------
dict with the jobs information
"""
analysis = Analysis(analysis_id)
# Check if the user actually has access to the analysis
check_analysis_access(user, analysis)
return {
j.id: {
'status': j.status,
'step': j.step,
'error': j.log.msg if j.log else ""
}
for j in analysis.jobs
}
def test_set_step(self):
new = Analysis.create(User("*****@*****.**"), "newAnalysis",
"A New Analysis", Analysis(1))
new.step = 2
sql = "SELECT * FROM qiita.analysis_workflow WHERE analysis_id = 3"
obs = self.conn_handler.execute_fetchall(sql)
self.assertEqual(obs, [[3, 2]])
def test_post_create_analysis_handler(self):
user = User('*****@*****.**')
dflt_analysis = user.default_analysis
dflt_analysis.add_samples({
4: [
'1.SKB8.640193', '1.SKD8.640184', '1.SKB7.640196',
'1.SKM9.640192', '1.SKM4.640180'
]
})
args = {
'name': 'New Test Analysis',
'description': 'Test Analysis Description'
}
response = self.post('/analysis/create/', args)
self.assertRegexpMatches(
response.effective_url,
r"http://localhost:\d+/analysis/description/\d+/")
self.assertEqual(response.code, 200)
# The new analysis id is located at the -2 position (see regex above)
new_id = response.effective_url.split('/')[-2]
a = Analysis(new_id)
# Make sure that all jobs have completed before we exit this tests
for j in a.jobs:
wait_for_processing_job(j.id)
def get(self, analysis_id):
user = self.current_user
analysis_id = int(analysis_id)
check_analysis_access(User(user), analysis_id)
analysis = Analysis(analysis_id)
jobres = defaultdict(list)
for job in analysis.jobs:
jobject = Job(job)
jobres[jobject.datatype].append(
(jobject.command[0], jobject.results))
dropped = {}
for proc_data_id, samples in viewitems(analysis.dropped_samples):
proc_data = ProcessedData(proc_data_id)
key = "Data type %s, Study: %s" % (proc_data.data_type(),
proc_data.study)
dropped[key] = samples
self.render("analysis_results.html",
user=self.current_user,
jobres=jobres,
aname=analysis.name,
dropped=dropped,
basefolder=get_db_files_base_dir())
# wipe out cached messages for this analysis
r_server = Redis()
key = '%s:messages' % self.current_user
oldmessages = r_server.lrange(key, 0, -1)
if oldmessages is not None:
for message in oldmessages:
if '"analysis": %d' % analysis_id in message:
r_server.lrem(key, message, 1)
def test_exists_noexist_return_jobid(self):
"""tests that non-existant job with bad samples returns false"""
exists, jid = Job.exists(
"16S", "Beta Diversity",
{"--otu_table_fp": 1, "--mapping_fp": 27}, Analysis(1),
return_existing=True)
self.assertFalse(exists)
self.assertEqual(jid, None)
def run_analysis(analysis_id, commands, comm_opts=None,
rarefaction_depth=None, merge_duplicated_sample_ids=False,
**kwargs):
"""Run an analysis"""
analysis = Analysis(analysis_id)
ar = RunAnalysis(**kwargs)
return ar(analysis, commands, comm_opts, rarefaction_depth,
merge_duplicated_sample_ids)
def get(self):
user = self.current_user
analyses = [
Analysis(a) for a in user.shared_analyses + user.private_analyses
]
self.render("show_analyses.html", analyses=analyses)
def post(self, analysis_id):
analysis = Analysis(analysis_id)
check_analysis_access(self.current_user, analysis)
message = ''
try:
Analysis(analysis_id).make_public()
except Exception as e:
message = str(e)
res = analysis_description_handler_get_request(
analysis_id, self.current_user)
if message:
# this will display the error message in the main banner
res['level'] = 'danger'
res['message'] = message
self.render("analysis_description.html", **res)
def run_analysis(analysis_id,
commands,
comm_opts=None,
rarefaction_depth=None,
**kwargs):
"""Run an analysis"""
analysis = Analysis(analysis_id)
ar = RunAnalysis(**kwargs)
return ar(analysis, commands, comm_opts, rarefaction_depth)
def test_post(self):
new_aid = get_count('qiita.analysis') + 1
post_args = {'name': 'post-test', 'description': "test of posting"}
response = self.post('/analysis/3', post_args)
# Make sure page response loaded sucessfully
self.assertEqual(response.code, 200)
# make sure analysis created
analysis = Analysis(new_aid)
self.assertEqual(analysis.name, 'post-test')
def test_delete_analysis(self):
# adding extra filepaths to make sure the delete works as expected, we
# basically want 8 -> 9 -> 10 -> 12 -> 14
# -> 11 -> 13
fd, fp10 = mkstemp(suffix='_table.biom')
close(fd)
fd, fp11 = mkstemp(suffix='_table.biom')
close(fd)
fd, fp12 = mkstemp(suffix='_table.biom')
close(fd)
fd, fp13 = mkstemp(suffix='_table.biom')
close(fd)
fd, fp14 = mkstemp(suffix='_table.biom')
close(fd)
with biom_open(fp10, 'w') as f:
et.to_hdf5(f, "test")
with biom_open(fp11, 'w') as f:
et.to_hdf5(f, "test")
with biom_open(fp12, 'w') as f:
et.to_hdf5(f, "test")
with biom_open(fp13, 'w') as f:
et.to_hdf5(f, "test")
with biom_open(fp14, 'w') as f:
et.to_hdf5(f, "test")
self._clean_up_files.extend([fp10, fp11, fp12, fp13, fp14])
# copying some processing parameters
a9 = Artifact(9)
pp = a9.processing_parameters
# 7: BIOM
a10 = Artifact.create([(fp10, 7)],
"BIOM",
parents=[a9],
processing_parameters=pp)
a11 = Artifact.create([(fp11, 7)],
"BIOM",
parents=[a9],
processing_parameters=pp)
a12 = Artifact.create([(fp12, 7)],
"BIOM",
parents=[a10],
processing_parameters=pp)
Artifact.create([(fp13, 7)],
"BIOM",
parents=[a11],
processing_parameters=pp)
Artifact.create([(fp14, 7)],
"BIOM",
parents=[a12],
processing_parameters=pp)
job = self._create_job('delete_analysis', {'analysis_id': 1})
private_task(job.id)
self.assertEqual(job.status, 'success')
with self.assertRaises(QiitaDBUnknownIDError):
Analysis(1)
def post(self):
analysis = Analysis(int(self.get_argument('analysis-id')))
# set to third step since this page is third step in workflow
analysis.step = SELECT_COMMANDS
data_types = analysis.data_types
commands = Command.get_commands_by_datatype()
self.render('select_commands.html',
commands=commands,
data_types=data_types,
aid=analysis.id)
def get(self):
analysis_id = int(self.get_argument('aid'))
analysis = Analysis(analysis_id)
check_analysis_access(self.current_user, analysis)
data_types = analysis.data_types
commands = Command.get_commands_by_datatype()
self.render('select_commands.html',
commands=commands, data_types=data_types, aid=analysis.id)
def get(self, analysis_id):
analysis_id = int(analysis_id)
try:
analysis = Analysis(analysis_id)
except QiitaDBUnknownIDError:
raise HTTPError(404, "Analysis %d does not exist" % analysis_id)
else:
check_analysis_access(self.current_user, analysis)
group_id = r_client.hget('analyis-map', analysis_id)
self.render("analysis_waiting.html",
group_id=group_id, aname=analysis.name)
def analyisis_graph_handler_get_request(analysis_id, user):
"""Returns the graph information of the analysis
Parameters
----------
analysis_id : int
The analysis id
user : qiita_db.user.User
The user performing the request
Returns
-------
dict with the graph information
"""
analysis = Analysis(analysis_id)
# Check if the user actually has access to the analysis
check_analysis_access(user, analysis)
# A user has full access to the analysis if it is one of its private
# analyses, the analysis has been shared with the user or the user is a
# superuser or admin
full_access = (analysis in (user.private_analyses | user.shared_analyses)
or user.level in {'superuser', 'admin'})
nodes = set()
edges = set()
# Loop through all the initial artifacts of the analysis
for a in analysis.artifacts:
g = a.descendants_with_jobs
# Loop through all the nodes in artifact descendants graph
for n in g.nodes():
# Get if the object is an artifact or a job
obj_type = n[0]
# Get the actual object
obj = n[1]
if obj_type == 'job':
name = obj.command.name
elif not full_access and not obj.visibility == 'public':
# The object is an artifact, it is not public and the user
# doesn't have full access, so we don't include it in the
# graph
continue
else:
name = '%s - %s' % (obj.name, obj.artifact_type)
nodes.add((obj_type, obj.id, name))
edges.update({(s[1].id, t[1].id) for s, t in g.edges()})
# Nodes and Edges are sets, but the set object can't be serialized using
# JSON. Transforming them to lists so when this is returned to the GUI
# over HTTP can be JSONized.
return {'edges': list(edges), 'nodes': list(nodes)}
def test_create_exists_return_existing(self):
"""Makes sure creation doesn't duplicate a job by returning existing"""
Analysis.create(User("*****@*****.**"), "new", "desc")
self.conn_handler.execute(
"INSERT INTO qiita.analysis_sample "
"(analysis_id, processed_data_id, sample_id) VALUES "
"(3, 1, '1.SKB8.640193'), (3, 1, '1.SKD8.640184'), "
"(3, 1, '1.SKB7.640196'), (3, 1, '1.SKM9.640192'), "
"(3, 1, '1.SKM4.640180')")
new = Job.create("18S", "Beta Diversity",
{"--otu_table_fp": 1, "--mapping_fp": 1},
Analysis(3), return_existing=True)
self.assertEqual(new.id, 2)
