def test_create(self): """Makes sure creation works as expected""" # make first job new = Job.create("18S", "Alpha Rarefaction", {"opt1": 4}, Analysis(1)) self.assertEqual(new.id, 4) # make sure job inserted correctly obs = self.conn_handler.execute_fetchall("SELECT * FROM qiita.job " "WHERE job_id = 4") exp = [[4, 2, 1, 3, '{"opt1":4}', None]] self.assertEqual(obs, exp) # make sure job added to analysis correctly obs = self.conn_handler.execute_fetchall("SELECT * FROM " "qiita.analysis_job WHERE " "job_id = 4") exp = [[1, 4]] self.assertEqual(obs, exp) # make second job with diff datatype and command to test column insert new = Job.create("16S", "Beta Diversity", {"opt1": 4}, Analysis(1)) self.assertEqual(new.id, 5) # make sure job inserted correctly obs = self.conn_handler.execute_fetchall("SELECT * FROM qiita.job " "WHERE job_id = 5") exp = [[5, 1, 1, 2, '{"opt1":4}', None]] self.assertEqual(obs, exp) # make sure job added to analysis correctly obs = self.conn_handler.execute_fetchall("SELECT * FROM " "qiita.analysis_job WHERE " "job_id = 5") exp = [[1, 5]] self.assertEqual(obs, exp)
def test_generate_analysis_tgz(self): obs_sout, obs_serr, obs_return = _generate_analysis_tgz(Analysis(1)) # not testing obs_serr as it will change depending on the system's tar # version self.assertEqual(obs_sout, "") self.assertEqual(obs_return, 0)
def get(self, analysis_id): analysis_id = int(analysis_id.split("/")[0]) analysis = Analysis(analysis_id) check_analysis_access(self.current_user, analysis) jobres = defaultdict(list) for job in analysis.jobs: jobject = Job(job) jobres[jobject.datatype].append( (jobject.command[0], jobject.results)) dropped = {} dropped_samples = analysis.dropped_samples if dropped_samples: for proc_data_id, samples in viewitems(dropped_samples): proc_data = ProcessedData(proc_data_id) key = "Data type %s, Study: %s" % (proc_data.data_type(), proc_data.study) dropped[key] = samples self.render("analysis_results.html", jobres=jobres, aname=analysis.name, dropped=dropped, basefolder=get_db_files_base_dir())
def post(self, analysis_id): user = self.current_user analysis_id = int(analysis_id) rarefaction_depth = self.get_argument('rarefaction-depth') # convert to integer if rarefaction level given if rarefaction_depth: rarefaction_depth = int(rarefaction_depth) else: rarefaction_depth = None check_analysis_access(User(user), analysis_id) command_args = self.get_arguments("commands") split = [x.split("#") for x in command_args] commands = ["%s: %s" % (s[0], s[1]) for s in split] analysis = Analysis(analysis_id) self.render("analysis_waiting.html", user=user, aid=analysis_id, aname=analysis.name, commands=commands) app = RunAnalysis() app(user, analysis, split, comm_opts={}, rarefaction_depth=rarefaction_depth)
def test_create_exists(self): """Makes sure creation doesn't duplicate a job""" with self.assertRaises(QiitaDBDuplicateError): Job.create("18S", "Beta Diversity", { "--otu_table_fp": 1, "--mapping_fp": 1 }, Analysis(1))
def get(self, analysis_id): analysis_id = int(analysis_id.split("/")[0]) analysis = Analysis(analysis_id) check_analysis_access(self.current_user, analysis) jobres = defaultdict(list) for jobject in analysis.jobs: results = [] for res in jobject.results: name = basename(res) if name.startswith('index'): name = basename(dirname(res)).replace('_', ' ') results.append((res, name)) jobres[jobject.datatype].append((jobject.command[0], results)) dropped_samples = analysis.dropped_samples dropped = defaultdict(list) for proc_data_id, samples in viewitems(dropped_samples): if not samples: continue proc_data = Artifact(proc_data_id) data_type = proc_data.data_type dropped[data_type].append( (proc_data.study.title, len(samples), ', '.join(samples))) self.render("analysis_results.html", analysis_id=analysis_id, jobres=jobres, aname=analysis.name, dropped=dropped, basefolder=get_db_files_base_dir())
def post(self, analysis_id): analysis_id = int(analysis_id.split("/")[0]) analysis_id_sent = int(self.get_argument('analysis_id')) action = self.get_argument('action') if analysis_id != analysis_id_sent or action != 'delete_analysis': raise QiitaPetAuthorizationError( self.current_user.id, 'analysis/results/%d-delete' % analysis_id) analysis = Analysis(analysis_id) analysis_name = analysis.name check_analysis_access(self.current_user, analysis) try: Analysis.delete(analysis_id) msg = ("Analysis <b><i>%s</i></b> has been deleted." % (analysis_name)) level = "success" except Exception as e: e = str(e) msg = ("Couldn't remove <b><i>%s</i></b> analysis: %s" % (analysis_name, e)) level = "danger" LogEntry.create( 'Runtime', "Couldn't remove analysis ID %d: %s" % (analysis_id, e)) self.redirect(u"/analysis/show/?level=%s&message=%s" % (level, msg))
def test_analyisis_graph_handler_get_request(self): obs = analyisis_graph_handler_get_request(1, User('*****@*****.**')) # The job id is randomly generated in the test environment. Gather # it here. There is only 1 job in the first artifact of the analysis job_id = Analysis(1).artifacts[0].jobs()[0].id exp = { 'edges': [(8, job_id), (job_id, 9)], 'nodes': [('job', 'job', job_id, 'Single Rarefaction', 'success'), ('artifact', 'BIOM', 9, 'noname\n(BIOM)', 'artifact'), ('artifact', 'BIOM', 8, 'noname\n(BIOM)', 'artifact')], 'workflow': None } self.assertItemsEqual(obs, exp) self.assertItemsEqual(obs['edges'], exp['edges']) self.assertItemsEqual(obs['nodes'], exp['nodes']) self.assertIsNone(obs['workflow']) # An admin has full access to the analysis obs = analyisis_graph_handler_get_request(1, User('*****@*****.**')) self.assertItemsEqual(obs, exp) self.assertItemsEqual(obs['edges'], exp['edges']) self.assertItemsEqual(obs['nodes'], exp['nodes']) # If the analysis is shared with the user he also has access obs = analyisis_graph_handler_get_request(1, User('*****@*****.**')) self.assertItemsEqual(obs, exp) self.assertItemsEqual(obs['edges'], exp['edges']) self.assertItemsEqual(obs['nodes'], exp['nodes']) # The user doesn't have access to the analysis with self.assertRaises(HTTPError): analyisis_graph_handler_get_request(1, User('*****@*****.**'))
def test_get_no_access(self): s = Analysis(2) u = User('*****@*****.**') args = {'selected': u.id, 'id': 2} response = self.get('/analysis/sharing/', args) self.assertEqual(response.code, 403) self.assertEqual(s.shared_with, [])
def test_redis_comms(self): """Make sure redis communication happens""" msgs = [] redis = Redis() pubsub = redis.pubsub() pubsub.subscribe("*****@*****.**") app = RunAnalysis() app("*****@*****.**", Analysis(2), [], rarefaction_depth=100) for msg in pubsub.listen(): if msg['type'] == 'message': msgs.append(msg['data']) if "allcomplete" in msg['data']: pubsub.unsubscribe("*****@*****.**") break self.assertEqual( msgs, ['{"msg": "Running", "command": "18S: Beta Diversity", ' '"analysis": 2}', '{"msg": "ERROR", "command": "18S: Beta Diversity", ' '"analysis": 2}', '{"msg": "allcomplete", "analysis": 2}']) log = self.conn_handler.execute_fetchall( "SELECT * from qiita.logging") self.assertEqual(1, len(log)) log = log[0] self.assertEqual(1, log[0]) self.assertEqual(2, log[2]) self.assertTrue(len(log[3]) > 0) self.assertTrue('[{"job": 3, "analysis": 2}]')
def test_set_options(self): new = Job.create("18S", "Alpha Rarefaction", {"opt1": 4}, Analysis(1)) new.options = self.options self.options['--output_dir'] = join( get_db_files_base_dir(), 'job/4_alpha_rarefaction.' 'py_output_dir') self.assertEqual(new.options, self.options)
def test_set_options(self): new = Job.create("18S", "Alpha Rarefaction", {"opt1": 4}, Analysis(1)) new.options = self.options self.options['--output_dir'] = join(self._job_folder, '4_alpha_rarefaction.' 'py_output_dir') self.assertEqual(new.options, self.options)
def test_delete_analysis(self): # as samples have been submitted to EBI, this will fail job = self._create_job('delete_analysis', {'analysis_id': 1}) private_task(job.id) self.assertEqual(job.status, 'success') with self.assertRaises(QiitaDBUnknownIDError): Analysis(1)
def test_get(self): a = Analysis(1) u = User('*****@*****.**') self.assertEqual(a.shared_with, [u]) # deselecting args = {'deselected': u.id, 'id': a.id} response = self.get('/analysis/sharing/', args) self.assertEqual(response.code, 200) exp = {'users': [], 'links': ''} self.assertEqual(loads(response.body), exp) self.assertEqual(a.shared_with, []) # Make sure unshared message added to the system self.assertEqual("Analysis 'SomeAnalysis' has been unshared with you.", u.messages()[0][1]) # selecting args = {'selected': u.id, 'id': a.id} response = self.get('/analysis/sharing/', args) self.assertEqual(response.code, 200) exp = { 'users': ['*****@*****.**'], 'links': ('<a target="_blank" href="mailto:[email protected]">Shared</a>') } self.assertEqual(loads(response.body), exp) self.assertEqual(a.shared_with, [u]) # Make sure shared message added to the system self.assertEqual( 'Analysis <a href="/analysis/description/1">\'SomeAnalysis\'</a> ' 'has been shared with you.', u.messages()[0][1])
def analyisis_job_handler_get_request(analysis_id, user): """Returns the job information of the analysis Parameters ---------- analysis_id: int The analysis id user : qiita_db.user.User The user performing the request Returns ------- dict with the jobs information """ analysis = Analysis(analysis_id) # Check if the user actually has access to the analysis check_analysis_access(user, analysis) return { j.id: { 'status': j.status, 'step': j.step, 'error': j.log.msg if j.log else "" } for j in analysis.jobs }
def test_set_step(self): new = Analysis.create(User("*****@*****.**"), "newAnalysis", "A New Analysis", Analysis(1)) new.step = 2 sql = "SELECT * FROM qiita.analysis_workflow WHERE analysis_id = 3" obs = self.conn_handler.execute_fetchall(sql) self.assertEqual(obs, [[3, 2]])
def test_post_create_analysis_handler(self): user = User('*****@*****.**') dflt_analysis = user.default_analysis dflt_analysis.add_samples({ 4: [ '1.SKB8.640193', '1.SKD8.640184', '1.SKB7.640196', '1.SKM9.640192', '1.SKM4.640180' ] }) args = { 'name': 'New Test Analysis', 'description': 'Test Analysis Description' } response = self.post('/analysis/create/', args) self.assertRegexpMatches( response.effective_url, r"http://localhost:\d+/analysis/description/\d+/") self.assertEqual(response.code, 200) # The new analysis id is located at the -2 position (see regex above) new_id = response.effective_url.split('/')[-2] a = Analysis(new_id) # Make sure that all jobs have completed before we exit this tests for j in a.jobs: wait_for_processing_job(j.id)
def get(self, analysis_id): user = self.current_user analysis_id = int(analysis_id) check_analysis_access(User(user), analysis_id) analysis = Analysis(analysis_id) jobres = defaultdict(list) for job in analysis.jobs: jobject = Job(job) jobres[jobject.datatype].append( (jobject.command[0], jobject.results)) dropped = {} for proc_data_id, samples in viewitems(analysis.dropped_samples): proc_data = ProcessedData(proc_data_id) key = "Data type %s, Study: %s" % (proc_data.data_type(), proc_data.study) dropped[key] = samples self.render("analysis_results.html", user=self.current_user, jobres=jobres, aname=analysis.name, dropped=dropped, basefolder=get_db_files_base_dir()) # wipe out cached messages for this analysis r_server = Redis() key = '%s:messages' % self.current_user oldmessages = r_server.lrange(key, 0, -1) if oldmessages is not None: for message in oldmessages: if '"analysis": %d' % analysis_id in message: r_server.lrem(key, message, 1)
def test_exists_noexist_return_jobid(self): """tests that non-existant job with bad samples returns false""" exists, jid = Job.exists( "16S", "Beta Diversity", {"--otu_table_fp": 1, "--mapping_fp": 27}, Analysis(1), return_existing=True) self.assertFalse(exists) self.assertEqual(jid, None)
def run_analysis(analysis_id, commands, comm_opts=None, rarefaction_depth=None, merge_duplicated_sample_ids=False, **kwargs): """Run an analysis""" analysis = Analysis(analysis_id) ar = RunAnalysis(**kwargs) return ar(analysis, commands, comm_opts, rarefaction_depth, merge_duplicated_sample_ids)
def get(self): user = self.current_user analyses = [ Analysis(a) for a in user.shared_analyses + user.private_analyses ] self.render("show_analyses.html", analyses=analyses)
def post(self, analysis_id): analysis = Analysis(analysis_id) check_analysis_access(self.current_user, analysis) message = '' try: Analysis(analysis_id).make_public() except Exception as e: message = str(e) res = analysis_description_handler_get_request( analysis_id, self.current_user) if message: # this will display the error message in the main banner res['level'] = 'danger' res['message'] = message self.render("analysis_description.html", **res)
def run_analysis(analysis_id, commands, comm_opts=None, rarefaction_depth=None, **kwargs): """Run an analysis""" analysis = Analysis(analysis_id) ar = RunAnalysis(**kwargs) return ar(analysis, commands, comm_opts, rarefaction_depth)
def test_post(self): new_aid = get_count('qiita.analysis') + 1 post_args = {'name': 'post-test', 'description': "test of posting"} response = self.post('/analysis/3', post_args) # Make sure page response loaded sucessfully self.assertEqual(response.code, 200) # make sure analysis created analysis = Analysis(new_aid) self.assertEqual(analysis.name, 'post-test')
def test_delete_analysis(self): # adding extra filepaths to make sure the delete works as expected, we # basically want 8 -> 9 -> 10 -> 12 -> 14 # -> 11 -> 13 fd, fp10 = mkstemp(suffix='_table.biom') close(fd) fd, fp11 = mkstemp(suffix='_table.biom') close(fd) fd, fp12 = mkstemp(suffix='_table.biom') close(fd) fd, fp13 = mkstemp(suffix='_table.biom') close(fd) fd, fp14 = mkstemp(suffix='_table.biom') close(fd) with biom_open(fp10, 'w') as f: et.to_hdf5(f, "test") with biom_open(fp11, 'w') as f: et.to_hdf5(f, "test") with biom_open(fp12, 'w') as f: et.to_hdf5(f, "test") with biom_open(fp13, 'w') as f: et.to_hdf5(f, "test") with biom_open(fp14, 'w') as f: et.to_hdf5(f, "test") self._clean_up_files.extend([fp10, fp11, fp12, fp13, fp14]) # copying some processing parameters a9 = Artifact(9) pp = a9.processing_parameters # 7: BIOM a10 = Artifact.create([(fp10, 7)], "BIOM", parents=[a9], processing_parameters=pp) a11 = Artifact.create([(fp11, 7)], "BIOM", parents=[a9], processing_parameters=pp) a12 = Artifact.create([(fp12, 7)], "BIOM", parents=[a10], processing_parameters=pp) Artifact.create([(fp13, 7)], "BIOM", parents=[a11], processing_parameters=pp) Artifact.create([(fp14, 7)], "BIOM", parents=[a12], processing_parameters=pp) job = self._create_job('delete_analysis', {'analysis_id': 1}) private_task(job.id) self.assertEqual(job.status, 'success') with self.assertRaises(QiitaDBUnknownIDError): Analysis(1)
def post(self): analysis = Analysis(int(self.get_argument('analysis-id'))) # set to third step since this page is third step in workflow analysis.step = SELECT_COMMANDS data_types = analysis.data_types commands = Command.get_commands_by_datatype() self.render('select_commands.html', commands=commands, data_types=data_types, aid=analysis.id)
def get(self): analysis_id = int(self.get_argument('aid')) analysis = Analysis(analysis_id) check_analysis_access(self.current_user, analysis) data_types = analysis.data_types commands = Command.get_commands_by_datatype() self.render('select_commands.html', commands=commands, data_types=data_types, aid=analysis.id)
def get(self, analysis_id): analysis_id = int(analysis_id) try: analysis = Analysis(analysis_id) except QiitaDBUnknownIDError: raise HTTPError(404, "Analysis %d does not exist" % analysis_id) else: check_analysis_access(self.current_user, analysis) group_id = r_client.hget('analyis-map', analysis_id) self.render("analysis_waiting.html", group_id=group_id, aname=analysis.name)
def analyisis_graph_handler_get_request(analysis_id, user): """Returns the graph information of the analysis Parameters ---------- analysis_id : int The analysis id user : qiita_db.user.User The user performing the request Returns ------- dict with the graph information """ analysis = Analysis(analysis_id) # Check if the user actually has access to the analysis check_analysis_access(user, analysis) # A user has full access to the analysis if it is one of its private # analyses, the analysis has been shared with the user or the user is a # superuser or admin full_access = (analysis in (user.private_analyses | user.shared_analyses) or user.level in {'superuser', 'admin'}) nodes = set() edges = set() # Loop through all the initial artifacts of the analysis for a in analysis.artifacts: g = a.descendants_with_jobs # Loop through all the nodes in artifact descendants graph for n in g.nodes(): # Get if the object is an artifact or a job obj_type = n[0] # Get the actual object obj = n[1] if obj_type == 'job': name = obj.command.name elif not full_access and not obj.visibility == 'public': # The object is an artifact, it is not public and the user # doesn't have full access, so we don't include it in the # graph continue else: name = '%s - %s' % (obj.name, obj.artifact_type) nodes.add((obj_type, obj.id, name)) edges.update({(s[1].id, t[1].id) for s, t in g.edges()}) # Nodes and Edges are sets, but the set object can't be serialized using # JSON. Transforming them to lists so when this is returned to the GUI # over HTTP can be JSONized. return {'edges': list(edges), 'nodes': list(nodes)}
def test_create_exists_return_existing(self): """Makes sure creation doesn't duplicate a job by returning existing""" Analysis.create(User("*****@*****.**"), "new", "desc") self.conn_handler.execute( "INSERT INTO qiita.analysis_sample " "(analysis_id, processed_data_id, sample_id) VALUES " "(3, 1, '1.SKB8.640193'), (3, 1, '1.SKD8.640184'), " "(3, 1, '1.SKB7.640196'), (3, 1, '1.SKM9.640192'), " "(3, 1, '1.SKM4.640180')") new = Job.create("18S", "Beta Diversity", {"--otu_table_fp": 1, "--mapping_fp": 1}, Analysis(3), return_existing=True) self.assertEqual(new.id, 2)