def testStrip(self):
id = helper_functions.get_sha1sum(self.QA_test_fasta)+'-test'
keyfile = open(self.QA_keyfile).readlines()
self.server({'request':'exec_analysis',
'data_file_sha1sum':id,
'data_file_path':self.QA_test_fasta,
'seperator':'_',
'job_description':'QA test',
'analysis_type':'rdp',
'codes_primers':keyfile,
"qa_mode":fasta.STRIP_BARCODES_PRIMERS,
'return_when_done': True,
})
i = self.server({'request':'info',
'resource':'analysis',
'analysis_id':id})
while(True):
dat = open(os.path.join(c.analyses_dir,id,'log'))
try:
dat.read().index("Filling job description")
dat.close()
break
except:
dat.close()
correct = helper_functions.get_sha1sum(self.QA_result)
framework = helper_functions.get_sha1sum(self.QA_test_fasta)
def setUp(self):
J = lambda s: os.path.join(c.base_dir,'../','unittests/test_files',s)
self.test_fasta_file = J('example_fasta')
self.split_analysis = [J(s) for s in ['split1.fna','split2.fna']]
self.confidence = J('confidence_levels.fas')
self.QA_test_fasta = J("QA_test_fasta")
self.QA_keyfile = J('QA_keyfile')
self.QA_result = J('QA_stripped')
self.analysis_id = helper_functions.get_sha1sum(self.test_fasta_file) + "-test"
self.QA_id = helper_functions.get_sha1sum(self.QA_test_fasta) + "-test"
self.rdp_analysis_dir = os.path.join(c.analyses_dir, self.analysis_id)
self.QA_analysis_dir = os.path.join(c.analyses_dir, self.QA_id)
self.test_files = [self.test_fasta_file,
self.confidence,
self.QA_test_fasta]+self.split_analysis
#create db:
self.db_name = "gg_coreset_test"
blastdb_path = os.path.join(c.base_dir,'../',
'unittests/test_files',
self.db_name)
self.server({'request':'new_blast_db',
'data_file_path':blastdb_path,
'db_name':self.db_name})
#check db exists
self.assertTrue(os.path.exists(os.path.join(c.blastdb_dir, self.db_name)))
def testLowConfidence(self):
count_seqs = lambda f: sum(1 for l in f if l.startswith('>'))
id = helper_functions.get_sha1sum(self.confidence)+'-test'
#also you can check the samples_dict to count how many were classified
self.server({'request':'exec_analysis','data_file_sha1sum':id,'data_file_path':self.confidence,'seperator':'_','job_description':'confidence test','analysis_type':'rdp','return_when_done': True,'threshold':0.8})
i = self.server({'request':'info','resource':'analysis','analysis_id':id})
n_low = count_seqs(open(os.path.join(c.analyses_dir,id,c.low_confidence_seqs_file_name)))
path = self.server({'request':'get_samples_dict_path','analysis_id':id})['samples_dict_path']
samples_dict = cPickle.load(open(path))
total_classified = 0
for _sample in samples_dict.values():
sample = _sample['genus']
total_classified += sum([sample[genus] for genus in sample if ':' not in genus])
self.assertTrue(n_low == 10)
self.assertTrue(n_low + total_classified == 20)
self.server({'request': 'remove_analysis',
'analysis_id': id})
self.server({'request':'exec_analysis','data_file_sha1sum':id,'data_file_path':self.confidence,'seperator':'_','job_description':'confidence test','analysis_type':'rdp','return_when_done': True,'threshold':0.9999})
i = self.server({'request':'info','resource':'analysis','analysis_id':id})
n_low = count_seqs(open(os.path.join(c.analyses_dir,id,c.low_confidence_seqs_file_name)))
print 'should be 11: '+str(n_low)
self.assertTrue(n_low == 11)#expect occasional random failure
self.server({'request': 'remove_analysis',
'analysis_id': id})
def update_analysis_append_samples(request, analysis_id):
if request.method == 'POST':
form = webforms.DataUploadForm(request.POST, request.FILES)
if form.is_valid():
time_stamp = helper_functions.get_time_stamp()
data_file_path = helper_functions.save_uploaded_file(form.cleaned_data['data_file'], time_stamp, output_file_name = constants.data_file_name)
data_file_sha1sum = helper_functions.get_sha1sum(data_file_path)
# just send the data file
server_request = {'request': 'append_samples_to_analysis',
'analysis_id': analysis_id,
'data_file_path': data_file_path}
server_response = server(server_request)
if server_response['response'] == 'OK':
return HttpResponse(get_template("simple.tmpl").render(Context({'content': "Your request is being processed."})))
else:
return HttpResponse(get_template("simple.tmpl").render(Context({'content': "something happened. has no clue. seriously :("})))
else:
return HttpResponse(get_template("error.tmpl").render(Context({'content': "Form wasn't valid :( Please click 'back' button of your browser and re-submit the form."})))
else:
form = webforms.DataUploadForm()
tmpl = "data_upload_form.tmpl"
return HttpResponse(get_template(tmpl).render(Context({'form': form, 'analysis_id': analysis_id})))
def setUp(self):
J = lambda s: os.path.join(c.base_dir,'../','unittests/test_files',s)
self.test_fasta_file = J('example_fasta')
self.split_analysis = [J(s) for s in ['split1.fna','split2.fna']]
self.confidence = J('confidence_levels.fas')
self.QA_test_fasta = J("QA_test_fasta")
self.QA_keyfile = J('QA_keyfile')
self.QA_result = J('QA_stripped')
self.analysis_id = helper_functions.get_sha1sum(self.test_fasta_file) + "-test"
self.rdp_analysis_dir = os.path.join(c.analyses_dir, self.analysis_id)
self.test_files = [self.test_fasta_file,
self.confidence,
self.QA_test_fasta]+self.split_analysis
def tearDown(self):
for t in self.test_files:
self.server({'request': 'remove_analysis',
'analysis_id': helper_functions.get_sha1sum(t)+'-test'})
#remove db
self.db_name = "gg_coreset_test"
self.server({"request":"delete",
"resource":"blastdb",
"id":self.db_name})
#check db was deleted
self.assertFalse(os.path.exists(os.path.join(c.blastdb_dir, self.db_name)))
def testAppend(self):
initial = self.split_analysis[0]
append = self.split_analysis[1]
id = helper_functions.get_sha1sum(initial)+'-test'
#create the analysis:
self.server({'request':'exec_analysis','data_file_sha1sum':id,'data_file_path':initial,'seperator':':','job_description':'cat-test','analysis_type':'rdp','return_when_done': True})
i = self.server({'request':'info','resource':'analysis','analysis_id':id})
self.assertTrue(len(i['info']['all_unique_samples_list']) == 1)
#append samples:
self.server({'request':'append_samples_to_analysis','analysis_id':id,'data_file_path':append,'return_when_done':True})
i = self.server({'request':'info','resource':'analysis','analysis_id':id})
self.assertTrue(len(i['info']['all_unique_samples_list']) == 2)
#clean up:
self.server({'request': 'remove_analysis',
'analysis_id': id})
def exec_env(request):
if request.method == 'POST':
form = webforms.EnvUploadForm(request.POST, request.FILES)
if form.is_valid():
job_description = form.cleaned_data['job_description']
time_stamp = helper_functions.get_time_stamp()
data_file_path = helper_functions.save_uploaded_file(form.cleaned_data['env_file'], time_stamp, output_file_name = constants.data_file_name)
data_file_sha1sum = helper_functions.get_sha1sum(data_file_path)
server_request = {'request': 'status'}
server_response = server(server_request)
if data_file_sha1sum in server_response['running_analyses']:
#error we already have it lan
tmpl = get_template("simple.tmpl")
return HttpResponse(tmpl.render(Context({'content': "Your ENV file is still being analyzed"})))
elif data_file_sha1sum in server_response['done_analyses']:
tmpl = get_template("simple.tmpl")
return HttpResponse(tmpl.render(Context({'content': "It seems we already have this ENV file analyzed"})))
# TODO: check validity of the file here..
server_request = {'request': 'exec_env',
'job_description': job_description,
'data_file_path': data_file_path,
'data_file_sha1sum': data_file_sha1sum}
server_response = server(server_request)
if server_response['response'] == 'OK':
return HttpResponse(get_template("simple.tmpl").render(Context({'content': "Your ENV file is being processed. This is your process id: %s" % server_response['process_id']})))
else:
return HttpResponse(get_template("simple.tmpl").render(Context({'content': "something happened"})))
else:
form = webforms.EnvUploadForm()
return HttpResponse(get_template("env_upload_form.tmpl").render(Context({'form': form})))
def tearDown(self):
for t in self.test_files:
self.server({'request': 'remove_analysis',
'analysis_id': helper_functions.get_sha1sum(t)+'-test'})
def new_analysis(request, analysis_type):
if request.method == 'POST':
#import pdb; pdb.set_trace()
if analysis_type == "rdp":
form = webforms.FastaUploadForm(request.POST, request.FILES)
if analysis_type == "qpcr":
form = webforms.QpcrUploadForm(request.POST, request.FILES)
if analysis_type == "env":
form = webforms.EnvUploadForm(request.POST, request.FILES)
if analysis_type == "blast":
form = webforms.BlastUploadForm(request.POST, request.FILES)
if analysis_type == "vamps":
form = webforms.VampsUploadForm(request.POST, request.FILES)
if form.is_valid():
job_description = form.cleaned_data['job_description']
time_stamp = helper_functions.get_time_stamp()
if len(request.FILES) == 1:
data_file_path = helper_functions.save_uploaded_file(
form.cleaned_data['data_file'],
time_stamp, output_file_name = constants.data_file_name)
elif len(request.FILES) > 1:
files = [request.FILES[f] for f in request.FILES.keys() if
f.startswith("data_file")]
data_file_path = cat(files)
data_file_sha1sum = helper_functions.get_sha1sum(data_file_path)
server_request = {'request': 'status'}
server_response = server(server_request)
if data_file_sha1sum in server_response['running_analyses']:
return HttpResponse(get_template("simple.tmpl").render(Context({'content': "This data file is still being analyzed"})))
elif data_file_sha1sum in server_response['done_analyses']:
return HttpResponse(get_template("simple.tmpl").render(Context({'content': "This data file has already been analyzed. The analysis id is "+str(data_file_sha1sum)})))
# TODO: check validity of the file here..
#standard request dict for all analyses.
server_request = {'request': 'exec_analysis',
'analysis_type': analysis_type,
'job_description': job_description,
'data_file_path': data_file_path,
'data_file_sha1sum': data_file_sha1sum}
#update server request with special options for certain analysis types
#if your module needs information not contained in a standard request,
#(say phase of the moon or something), add it here in a tuple of
#(name, function) where name is the name attribute in the form, and
#the function converts the raw value or cleaned_data into
#whatever it needs to be
special_options = [("seperator",str),
("qa_mode", int),
("db_name",str),
("threshold",float),
("codes_primers",list),
("homopolymer_length",int),
("threshold_dict",dict)]
for opt in special_options:
if form.cleaned_data.get(opt[0]):
server_request[opt[0]] = opt[1](form.cleaned_data[opt[0]])
server_response = server(server_request)
if server_response['response'] == 'OK':
return HttpResponse(get_template("simple.tmpl").render(Context({'content': "Your request is being processed. This is your process id: %s" % server_response['process_id']})))
else:
return HttpResponse(get_template("simple.tmpl").render(Context({'content': "Server error:"+str(server_response.get("exception"))})))
else:
return HttpResponse(get_template(templates[analysis_type]).render(
Context({"form":form})))
else:
if analysis_type == "rdp":
form = webforms.FastaUploadForm()
tmpl = "fasta_upload_form.tmpl"
elif analysis_type == "qpcr":
form = webforms.QpcrUploadForm()
tmpl = "qpcr_upload_form.tmpl"
elif analysis_type == "env":
form = webforms.EnvUploadForm()
tmpl = "env_upload_form.tmpl"
elif analysis_type == 'blast':
form = webforms.BlastUploadForm()
tmpl = "blast_upload_form.tmpl"
elif analysis_type == "vamps":
form = webforms.EnvUploadForm()
tmpl = "vamps_upload_form.tmpl"
return HttpResponse(get_template(tmpl).render(Context({'form': form})))
def setUp(self):
self.socket_name = c.socket_name
self.test_fasta_file = os.path.join(c.base_dir, "../", 'unittests/test_files/example_fasta')
self.analysis_id = helper_functions.get_sha1sum(self.test_fasta_file) + "-test"
self.rdp_analysis_dir = os.path.join(c.analyses_dir,
self.analysis_id)