Beispiel #1
0
    def testStrip(self):
        id = helper_functions.get_sha1sum(self.QA_test_fasta)+'-test'
        keyfile = open(self.QA_keyfile).readlines()
        
        
        self.server({'request':'exec_analysis',
                     'data_file_sha1sum':id,
                     'data_file_path':self.QA_test_fasta,
                     'seperator':'_',
                     'job_description':'QA test',
                     'analysis_type':'rdp',
                     'codes_primers':keyfile,
                     "qa_mode":fasta.STRIP_BARCODES_PRIMERS,
                     'return_when_done': True,
                     })
        i = self.server({'request':'info',
                         'resource':'analysis',
                         'analysis_id':id})
        while(True):
            dat = open(os.path.join(c.analyses_dir,id,'log'))
            try:                
                dat.read().index("Filling job description")
                dat.close()
                break
            except:
                dat.close()

        correct = helper_functions.get_sha1sum(self.QA_result)
        framework = helper_functions.get_sha1sum(self.QA_test_fasta)
Beispiel #2
0
    def setUp(self):
        J = lambda s: os.path.join(c.base_dir,'../','unittests/test_files',s)
        self.test_fasta_file = J('example_fasta')
        self.split_analysis = [J(s) for s in ['split1.fna','split2.fna']]
        self.confidence = J('confidence_levels.fas')
        self.QA_test_fasta = J("QA_test_fasta")
        self.QA_keyfile = J('QA_keyfile')
        self.QA_result = J('QA_stripped')
        self.analysis_id = helper_functions.get_sha1sum(self.test_fasta_file) + "-test"
        self.QA_id = helper_functions.get_sha1sum(self.QA_test_fasta) + "-test"
        self.rdp_analysis_dir = os.path.join(c.analyses_dir, self.analysis_id)
        self.QA_analysis_dir = os.path.join(c.analyses_dir, self.QA_id)

        self.test_files = [self.test_fasta_file,
                           self.confidence,
                           self.QA_test_fasta]+self.split_analysis

        #create db:
        self.db_name = "gg_coreset_test"
        blastdb_path = os.path.join(c.base_dir,'../',
                                    'unittests/test_files',
                                    self.db_name)
        
        self.server({'request':'new_blast_db',
                'data_file_path':blastdb_path,
                'db_name':self.db_name})

        #check db exists
        self.assertTrue(os.path.exists(os.path.join(c.blastdb_dir, self.db_name)))    
Beispiel #3
0
    def testLowConfidence(self):
        count_seqs = lambda f: sum(1 for l in f if l.startswith('>'))
        id = helper_functions.get_sha1sum(self.confidence)+'-test'

        #also you can check the samples_dict to count how many were classified
        self.server({'request':'exec_analysis','data_file_sha1sum':id,'data_file_path':self.confidence,'seperator':'_','job_description':'confidence test','analysis_type':'rdp','return_when_done': True,'threshold':0.8})
        i = self.server({'request':'info','resource':'analysis','analysis_id':id})
        n_low = count_seqs(open(os.path.join(c.analyses_dir,id,c.low_confidence_seqs_file_name)))
        path = self.server({'request':'get_samples_dict_path','analysis_id':id})['samples_dict_path']
        samples_dict = cPickle.load(open(path))
        total_classified = 0
        for _sample in samples_dict.values():
            sample = _sample['genus']
            total_classified += sum([sample[genus] for genus in sample if ':' not in genus]) 
        self.assertTrue(n_low == 10)
        self.assertTrue(n_low + total_classified == 20)
        self.server({'request': 'remove_analysis',
                     'analysis_id': id})

        
        self.server({'request':'exec_analysis','data_file_sha1sum':id,'data_file_path':self.confidence,'seperator':'_','job_description':'confidence test','analysis_type':'rdp','return_when_done': True,'threshold':0.9999})
        i = self.server({'request':'info','resource':'analysis','analysis_id':id})
        n_low = count_seqs(open(os.path.join(c.analyses_dir,id,c.low_confidence_seqs_file_name)))
        print 'should be 11: '+str(n_low)
        self.assertTrue(n_low == 11)#expect occasional random failure
        self.server({'request': 'remove_analysis',
                     'analysis_id': id})
Beispiel #4
0
def update_analysis_append_samples(request, analysis_id):

    if request.method == 'POST':

        form = webforms.DataUploadForm(request.POST, request.FILES)

        if form.is_valid():
            time_stamp = helper_functions.get_time_stamp()
            data_file_path = helper_functions.save_uploaded_file(form.cleaned_data['data_file'], time_stamp, output_file_name = constants.data_file_name)

            data_file_sha1sum = helper_functions.get_sha1sum(data_file_path)

            # just send the data file
            server_request = {'request': 'append_samples_to_analysis',
                              'analysis_id': analysis_id,
                              'data_file_path': data_file_path}

            server_response = server(server_request)

            if server_response['response'] == 'OK':
                return HttpResponse(get_template("simple.tmpl").render(Context({'content': "Your request is being processed."})))
            else:
                return HttpResponse(get_template("simple.tmpl").render(Context({'content': "something happened. has no clue. seriously :("})))
        else:
            return HttpResponse(get_template("error.tmpl").render(Context({'content': "Form wasn't valid :( Please click 'back' button of your browser and re-submit the form."})))

    else:
        form = webforms.DataUploadForm()
        tmpl = "data_upload_form.tmpl"

        return HttpResponse(get_template(tmpl).render(Context({'form': form, 'analysis_id': analysis_id})))
Beispiel #5
0
    def setUp(self):
        J = lambda s: os.path.join(c.base_dir,'../','unittests/test_files',s)
        self.test_fasta_file = J('example_fasta')
        self.split_analysis = [J(s) for s in ['split1.fna','split2.fna']]
        self.confidence = J('confidence_levels.fas')
        self.QA_test_fasta = J("QA_test_fasta")
        self.QA_keyfile = J('QA_keyfile')
        self.QA_result = J('QA_stripped')
        self.analysis_id = helper_functions.get_sha1sum(self.test_fasta_file) + "-test"
        self.rdp_analysis_dir = os.path.join(c.analyses_dir, self.analysis_id)

        self.test_files = [self.test_fasta_file,
                           self.confidence,
                           self.QA_test_fasta]+self.split_analysis
Beispiel #6
0
    def tearDown(self):
        
        for t in self.test_files:
            self.server({'request': 'remove_analysis',
                     'analysis_id': helper_functions.get_sha1sum(t)+'-test'})

        #remove db
        self.db_name = "gg_coreset_test"
        
        self.server({"request":"delete",
                "resource":"blastdb",
                "id":self.db_name})
        #check db was deleted
        self.assertFalse(os.path.exists(os.path.join(c.blastdb_dir, self.db_name))) 
Beispiel #7
0
 def testAppend(self):
     initial = self.split_analysis[0]
     append = self.split_analysis[1]
     id = helper_functions.get_sha1sum(initial)+'-test'
     
         #create the analysis:
     self.server({'request':'exec_analysis','data_file_sha1sum':id,'data_file_path':initial,'seperator':':','job_description':'cat-test','analysis_type':'rdp','return_when_done': True})
     i = self.server({'request':'info','resource':'analysis','analysis_id':id})
     self.assertTrue(len(i['info']['all_unique_samples_list']) == 1)
     
         #append samples:
     self.server({'request':'append_samples_to_analysis','analysis_id':id,'data_file_path':append,'return_when_done':True})
     i = self.server({'request':'info','resource':'analysis','analysis_id':id})
     self.assertTrue(len(i['info']['all_unique_samples_list']) == 2)
     
         #clean up:
     self.server({'request': 'remove_analysis',
                  'analysis_id': id})
Beispiel #8
0
def exec_env(request):
    if request.method == 'POST':
        form = webforms.EnvUploadForm(request.POST, request.FILES)
        if form.is_valid():
            job_description = form.cleaned_data['job_description']
            time_stamp = helper_functions.get_time_stamp()
            data_file_path = helper_functions.save_uploaded_file(form.cleaned_data['env_file'], time_stamp, output_file_name = constants.data_file_name)

            data_file_sha1sum = helper_functions.get_sha1sum(data_file_path)

            server_request = {'request': 'status'}
            server_response = server(server_request)

            if data_file_sha1sum in server_response['running_analyses']:
                #error we already have it lan
                tmpl = get_template("simple.tmpl")
                return HttpResponse(tmpl.render(Context({'content': "Your ENV file is still being analyzed"})))
            elif data_file_sha1sum in server_response['done_analyses']:
                tmpl = get_template("simple.tmpl")
                return HttpResponse(tmpl.render(Context({'content': "It seems we already have this ENV file analyzed"})))
            # TODO: check validity of the file here..

            server_request = {'request': 'exec_env',
                              'job_description': job_description,
                              'data_file_path': data_file_path,
                              'data_file_sha1sum': data_file_sha1sum}

            server_response = server(server_request)

            if server_response['response'] == 'OK':
                return HttpResponse(get_template("simple.tmpl").render(Context({'content': "Your ENV file is being processed. This is your process id: %s" % server_response['process_id']})))
            else:
                return HttpResponse(get_template("simple.tmpl").render(Context({'content': "something happened"})))
    else:
        form = webforms.EnvUploadForm()

    return HttpResponse(get_template("env_upload_form.tmpl").render(Context({'form': form})))
Beispiel #9
0
 def tearDown(self):
     
     for t in self.test_files:
         self.server({'request': 'remove_analysis',
                  'analysis_id': helper_functions.get_sha1sum(t)+'-test'})
Beispiel #10
0
def new_analysis(request, analysis_type):
    if request.method == 'POST':

        #import pdb; pdb.set_trace()
        if analysis_type == "rdp":
            form = webforms.FastaUploadForm(request.POST, request.FILES)
        if analysis_type == "qpcr":
            form = webforms.QpcrUploadForm(request.POST, request.FILES)
        if analysis_type == "env":
            form = webforms.EnvUploadForm(request.POST, request.FILES)
        if analysis_type == "blast":
            form = webforms.BlastUploadForm(request.POST, request.FILES)
        if analysis_type == "vamps":
            form = webforms.VampsUploadForm(request.POST, request.FILES)

        if form.is_valid():
            job_description = form.cleaned_data['job_description']
            time_stamp = helper_functions.get_time_stamp()
            if len(request.FILES) == 1:
                data_file_path = helper_functions.save_uploaded_file(
                    form.cleaned_data['data_file'],
                    time_stamp, output_file_name = constants.data_file_name)
            elif len(request.FILES) > 1:
                files = [request.FILES[f] for f in request.FILES.keys() if
                         f.startswith("data_file")]
                data_file_path = cat(files)

            data_file_sha1sum = helper_functions.get_sha1sum(data_file_path)

            server_request = {'request': 'status'}
            server_response = server(server_request)

            if data_file_sha1sum in server_response['running_analyses']:
                return HttpResponse(get_template("simple.tmpl").render(Context({'content': "This data file is still being analyzed"})))
            elif data_file_sha1sum in server_response['done_analyses']:
                return HttpResponse(get_template("simple.tmpl").render(Context({'content': "This data file has already been analyzed. The analysis id is "+str(data_file_sha1sum)})))

            # TODO: check validity of the file here..

            #standard request dict for all analyses.
            server_request = {'request': 'exec_analysis',
                              'analysis_type': analysis_type,
                              'job_description': job_description,
                              'data_file_path': data_file_path,
                              'data_file_sha1sum': data_file_sha1sum}

            #update server request with special options for certain analysis types
            #if your module needs information not contained in a standard request,
            #(say phase of the moon or something), add it here in a tuple of
            #(name, function) where name is the name attribute in the form, and
            #the function converts the raw value or cleaned_data into
            #whatever it needs to be
            special_options = [("seperator",str),
                               ("qa_mode", int),
                               ("db_name",str),
                               ("threshold",float),
                               ("codes_primers",list),
                               ("homopolymer_length",int),
                               ("threshold_dict",dict)]
            for opt in special_options:
                if form.cleaned_data.get(opt[0]):
                    server_request[opt[0]] = opt[1](form.cleaned_data[opt[0]])
                    
             

            server_response = server(server_request)

            if server_response['response'] == 'OK':
                return HttpResponse(get_template("simple.tmpl").render(Context({'content': "Your request is being processed. This is your process id: %s" % server_response['process_id']})))
            else:
                return HttpResponse(get_template("simple.tmpl").render(Context({'content': "Server error:"+str(server_response.get("exception"))})))
        else:
            return HttpResponse(get_template(templates[analysis_type]).render(
                Context({"form":form})))
    else:
        if analysis_type == "rdp":
            form = webforms.FastaUploadForm()
            tmpl = "fasta_upload_form.tmpl"
        elif analysis_type == "qpcr":
            form = webforms.QpcrUploadForm()
            tmpl = "qpcr_upload_form.tmpl"
        elif analysis_type == "env":
            form = webforms.EnvUploadForm()
            tmpl = "env_upload_form.tmpl"
        elif analysis_type == 'blast':
            form = webforms.BlastUploadForm()
            tmpl = "blast_upload_form.tmpl"
        elif analysis_type == "vamps":
            form = webforms.EnvUploadForm()
            tmpl = "vamps_upload_form.tmpl"

        return HttpResponse(get_template(tmpl).render(Context({'form': form})))
Beispiel #11
0
 def setUp(self):
     self.socket_name = c.socket_name
     self.test_fasta_file = os.path.join(c.base_dir, "../", 'unittests/test_files/example_fasta')
     self.analysis_id = helper_functions.get_sha1sum(self.test_fasta_file) + "-test"
     self.rdp_analysis_dir = os.path.join(c.analyses_dir,
                                          self.analysis_id)