def testStrip(self): id = helper_functions.get_sha1sum(self.QA_test_fasta)+'-test' keyfile = open(self.QA_keyfile).readlines() self.server({'request':'exec_analysis', 'data_file_sha1sum':id, 'data_file_path':self.QA_test_fasta, 'seperator':'_', 'job_description':'QA test', 'analysis_type':'rdp', 'codes_primers':keyfile, "qa_mode":fasta.STRIP_BARCODES_PRIMERS, 'return_when_done': True, }) i = self.server({'request':'info', 'resource':'analysis', 'analysis_id':id}) while(True): dat = open(os.path.join(c.analyses_dir,id,'log')) try: dat.read().index("Filling job description") dat.close() break except: dat.close() correct = helper_functions.get_sha1sum(self.QA_result) framework = helper_functions.get_sha1sum(self.QA_test_fasta)
def setUp(self): J = lambda s: os.path.join(c.base_dir,'../','unittests/test_files',s) self.test_fasta_file = J('example_fasta') self.split_analysis = [J(s) for s in ['split1.fna','split2.fna']] self.confidence = J('confidence_levels.fas') self.QA_test_fasta = J("QA_test_fasta") self.QA_keyfile = J('QA_keyfile') self.QA_result = J('QA_stripped') self.analysis_id = helper_functions.get_sha1sum(self.test_fasta_file) + "-test" self.QA_id = helper_functions.get_sha1sum(self.QA_test_fasta) + "-test" self.rdp_analysis_dir = os.path.join(c.analyses_dir, self.analysis_id) self.QA_analysis_dir = os.path.join(c.analyses_dir, self.QA_id) self.test_files = [self.test_fasta_file, self.confidence, self.QA_test_fasta]+self.split_analysis #create db: self.db_name = "gg_coreset_test" blastdb_path = os.path.join(c.base_dir,'../', 'unittests/test_files', self.db_name) self.server({'request':'new_blast_db', 'data_file_path':blastdb_path, 'db_name':self.db_name}) #check db exists self.assertTrue(os.path.exists(os.path.join(c.blastdb_dir, self.db_name)))
def testLowConfidence(self): count_seqs = lambda f: sum(1 for l in f if l.startswith('>')) id = helper_functions.get_sha1sum(self.confidence)+'-test' #also you can check the samples_dict to count how many were classified self.server({'request':'exec_analysis','data_file_sha1sum':id,'data_file_path':self.confidence,'seperator':'_','job_description':'confidence test','analysis_type':'rdp','return_when_done': True,'threshold':0.8}) i = self.server({'request':'info','resource':'analysis','analysis_id':id}) n_low = count_seqs(open(os.path.join(c.analyses_dir,id,c.low_confidence_seqs_file_name))) path = self.server({'request':'get_samples_dict_path','analysis_id':id})['samples_dict_path'] samples_dict = cPickle.load(open(path)) total_classified = 0 for _sample in samples_dict.values(): sample = _sample['genus'] total_classified += sum([sample[genus] for genus in sample if ':' not in genus]) self.assertTrue(n_low == 10) self.assertTrue(n_low + total_classified == 20) self.server({'request': 'remove_analysis', 'analysis_id': id}) self.server({'request':'exec_analysis','data_file_sha1sum':id,'data_file_path':self.confidence,'seperator':'_','job_description':'confidence test','analysis_type':'rdp','return_when_done': True,'threshold':0.9999}) i = self.server({'request':'info','resource':'analysis','analysis_id':id}) n_low = count_seqs(open(os.path.join(c.analyses_dir,id,c.low_confidence_seqs_file_name))) print 'should be 11: '+str(n_low) self.assertTrue(n_low == 11)#expect occasional random failure self.server({'request': 'remove_analysis', 'analysis_id': id})
def update_analysis_append_samples(request, analysis_id): if request.method == 'POST': form = webforms.DataUploadForm(request.POST, request.FILES) if form.is_valid(): time_stamp = helper_functions.get_time_stamp() data_file_path = helper_functions.save_uploaded_file(form.cleaned_data['data_file'], time_stamp, output_file_name = constants.data_file_name) data_file_sha1sum = helper_functions.get_sha1sum(data_file_path) # just send the data file server_request = {'request': 'append_samples_to_analysis', 'analysis_id': analysis_id, 'data_file_path': data_file_path} server_response = server(server_request) if server_response['response'] == 'OK': return HttpResponse(get_template("simple.tmpl").render(Context({'content': "Your request is being processed."}))) else: return HttpResponse(get_template("simple.tmpl").render(Context({'content': "something happened. has no clue. seriously :("}))) else: return HttpResponse(get_template("error.tmpl").render(Context({'content': "Form wasn't valid :( Please click 'back' button of your browser and re-submit the form."}))) else: form = webforms.DataUploadForm() tmpl = "data_upload_form.tmpl" return HttpResponse(get_template(tmpl).render(Context({'form': form, 'analysis_id': analysis_id})))
def setUp(self): J = lambda s: os.path.join(c.base_dir,'../','unittests/test_files',s) self.test_fasta_file = J('example_fasta') self.split_analysis = [J(s) for s in ['split1.fna','split2.fna']] self.confidence = J('confidence_levels.fas') self.QA_test_fasta = J("QA_test_fasta") self.QA_keyfile = J('QA_keyfile') self.QA_result = J('QA_stripped') self.analysis_id = helper_functions.get_sha1sum(self.test_fasta_file) + "-test" self.rdp_analysis_dir = os.path.join(c.analyses_dir, self.analysis_id) self.test_files = [self.test_fasta_file, self.confidence, self.QA_test_fasta]+self.split_analysis
def tearDown(self): for t in self.test_files: self.server({'request': 'remove_analysis', 'analysis_id': helper_functions.get_sha1sum(t)+'-test'}) #remove db self.db_name = "gg_coreset_test" self.server({"request":"delete", "resource":"blastdb", "id":self.db_name}) #check db was deleted self.assertFalse(os.path.exists(os.path.join(c.blastdb_dir, self.db_name)))
def testAppend(self): initial = self.split_analysis[0] append = self.split_analysis[1] id = helper_functions.get_sha1sum(initial)+'-test' #create the analysis: self.server({'request':'exec_analysis','data_file_sha1sum':id,'data_file_path':initial,'seperator':':','job_description':'cat-test','analysis_type':'rdp','return_when_done': True}) i = self.server({'request':'info','resource':'analysis','analysis_id':id}) self.assertTrue(len(i['info']['all_unique_samples_list']) == 1) #append samples: self.server({'request':'append_samples_to_analysis','analysis_id':id,'data_file_path':append,'return_when_done':True}) i = self.server({'request':'info','resource':'analysis','analysis_id':id}) self.assertTrue(len(i['info']['all_unique_samples_list']) == 2) #clean up: self.server({'request': 'remove_analysis', 'analysis_id': id})
def exec_env(request): if request.method == 'POST': form = webforms.EnvUploadForm(request.POST, request.FILES) if form.is_valid(): job_description = form.cleaned_data['job_description'] time_stamp = helper_functions.get_time_stamp() data_file_path = helper_functions.save_uploaded_file(form.cleaned_data['env_file'], time_stamp, output_file_name = constants.data_file_name) data_file_sha1sum = helper_functions.get_sha1sum(data_file_path) server_request = {'request': 'status'} server_response = server(server_request) if data_file_sha1sum in server_response['running_analyses']: #error we already have it lan tmpl = get_template("simple.tmpl") return HttpResponse(tmpl.render(Context({'content': "Your ENV file is still being analyzed"}))) elif data_file_sha1sum in server_response['done_analyses']: tmpl = get_template("simple.tmpl") return HttpResponse(tmpl.render(Context({'content': "It seems we already have this ENV file analyzed"}))) # TODO: check validity of the file here.. server_request = {'request': 'exec_env', 'job_description': job_description, 'data_file_path': data_file_path, 'data_file_sha1sum': data_file_sha1sum} server_response = server(server_request) if server_response['response'] == 'OK': return HttpResponse(get_template("simple.tmpl").render(Context({'content': "Your ENV file is being processed. This is your process id: %s" % server_response['process_id']}))) else: return HttpResponse(get_template("simple.tmpl").render(Context({'content': "something happened"}))) else: form = webforms.EnvUploadForm() return HttpResponse(get_template("env_upload_form.tmpl").render(Context({'form': form})))
def tearDown(self): for t in self.test_files: self.server({'request': 'remove_analysis', 'analysis_id': helper_functions.get_sha1sum(t)+'-test'})
def new_analysis(request, analysis_type): if request.method == 'POST': #import pdb; pdb.set_trace() if analysis_type == "rdp": form = webforms.FastaUploadForm(request.POST, request.FILES) if analysis_type == "qpcr": form = webforms.QpcrUploadForm(request.POST, request.FILES) if analysis_type == "env": form = webforms.EnvUploadForm(request.POST, request.FILES) if analysis_type == "blast": form = webforms.BlastUploadForm(request.POST, request.FILES) if analysis_type == "vamps": form = webforms.VampsUploadForm(request.POST, request.FILES) if form.is_valid(): job_description = form.cleaned_data['job_description'] time_stamp = helper_functions.get_time_stamp() if len(request.FILES) == 1: data_file_path = helper_functions.save_uploaded_file( form.cleaned_data['data_file'], time_stamp, output_file_name = constants.data_file_name) elif len(request.FILES) > 1: files = [request.FILES[f] for f in request.FILES.keys() if f.startswith("data_file")] data_file_path = cat(files) data_file_sha1sum = helper_functions.get_sha1sum(data_file_path) server_request = {'request': 'status'} server_response = server(server_request) if data_file_sha1sum in server_response['running_analyses']: return HttpResponse(get_template("simple.tmpl").render(Context({'content': "This data file is still being analyzed"}))) elif data_file_sha1sum in server_response['done_analyses']: return HttpResponse(get_template("simple.tmpl").render(Context({'content': "This data file has already been analyzed. The analysis id is "+str(data_file_sha1sum)}))) # TODO: check validity of the file here.. #standard request dict for all analyses. server_request = {'request': 'exec_analysis', 'analysis_type': analysis_type, 'job_description': job_description, 'data_file_path': data_file_path, 'data_file_sha1sum': data_file_sha1sum} #update server request with special options for certain analysis types #if your module needs information not contained in a standard request, #(say phase of the moon or something), add it here in a tuple of #(name, function) where name is the name attribute in the form, and #the function converts the raw value or cleaned_data into #whatever it needs to be special_options = [("seperator",str), ("qa_mode", int), ("db_name",str), ("threshold",float), ("codes_primers",list), ("homopolymer_length",int), ("threshold_dict",dict)] for opt in special_options: if form.cleaned_data.get(opt[0]): server_request[opt[0]] = opt[1](form.cleaned_data[opt[0]]) server_response = server(server_request) if server_response['response'] == 'OK': return HttpResponse(get_template("simple.tmpl").render(Context({'content': "Your request is being processed. This is your process id: %s" % server_response['process_id']}))) else: return HttpResponse(get_template("simple.tmpl").render(Context({'content': "Server error:"+str(server_response.get("exception"))}))) else: return HttpResponse(get_template(templates[analysis_type]).render( Context({"form":form}))) else: if analysis_type == "rdp": form = webforms.FastaUploadForm() tmpl = "fasta_upload_form.tmpl" elif analysis_type == "qpcr": form = webforms.QpcrUploadForm() tmpl = "qpcr_upload_form.tmpl" elif analysis_type == "env": form = webforms.EnvUploadForm() tmpl = "env_upload_form.tmpl" elif analysis_type == 'blast': form = webforms.BlastUploadForm() tmpl = "blast_upload_form.tmpl" elif analysis_type == "vamps": form = webforms.EnvUploadForm() tmpl = "vamps_upload_form.tmpl" return HttpResponse(get_template(tmpl).render(Context({'form': form})))
def setUp(self): self.socket_name = c.socket_name self.test_fasta_file = os.path.join(c.base_dir, "../", 'unittests/test_files/example_fasta') self.analysis_id = helper_functions.get_sha1sum(self.test_fasta_file) + "-test" self.rdp_analysis_dir = os.path.join(c.analyses_dir, self.analysis_id)