def viewTimecourseInTable(request): params = {} params['queries'] = internal.ListQueries( request, {'projectID': [request.session['projectID']]}) params['attributes'] = sorted([ x.name for x in internal.ListAttributes( request, {'projectID': [request.session['projectID']]}) ]) if request.method == "POST": name = request.POST.get('query', None) attr = request.POST.get('attribute', None) if name and attr: query = Query.objects.get(project=request.session['projectID'], name=name) samples = query.expandsamples svm = SubjectMap.objects.filter(sample__in=samples) subjects = svm.values_list('subject', flat=True) params['subjects'] = {} for subj in subjects: params['subjects'][subj] = [] visits = svm.filter(subject=subj).values_list( 'visit', flat=True).distinct() data = {} data['samples'] = [] for v in visits: data['id'] = v data['samples'] += svm.filter( subject=subj, visit=v).values_list('sample', flat=True).distinct() params['subjects'][subj].append(data) return render(request, 'viewTimecourseInTable.html', params)
def comparison(request): params = {} params['queries'] = internal.ListQueries( request, {'projectID': [request.session['projectID']]}) params['attributes'] = sorted([ x.name for x in internal.ListAttributes( request, {'projectID': [request.session['projectID']]}) ]) return render(request, 'visualComparison.html', params)
def main(request): params = {} params['queries'] = internal.ListQueries( request, {'projectID': [request.session['projectID']]}) kwargs = dict(request.GET) kwargs['projectID'] = [request.session['projectID']] params['attributes'] = sorted( [x.name for x in internal.ListAttributes(request, kwargs)]) if request.method == 'POST': queryname = request.POST.get('query') or None attribute = request.POST.get('attribute') or 'Race' if not queryname: return render(request, 'home.html', params) params['feedback'] = 'Query selected: "%s"' % queryname query = internal.ListQueries(request, { 'projectID': [request.session['projectID']], 'full': [True] }) if not query or not len(query): return render(request, 'home.html', params) else: query = query[0] samplelist = myutils.fieldexpand(query['results']) dataset = internal.ShowDistribution( request, { 'queryname': [queryname], 'projectID': [query['project_id']], 'attribute': [ attribute, ], }) jsondata = {'key': attribute, 'values': []} for d in sorted(dataset[1:], key=lambda x: x[0]): tmp = {'label': d[0], 'value': d[1]} jsondata['values'].append(tmp) params['data'] = dataset params['json'] = json.dumps([jsondata]) params['selectedquery'] = queryname params['selectedattribute'] = attribute if request.POST.get('format') == 'json': return HttpResponse(params['json'], content_type="application/json") #params['data'] = sorted(dataset, key=lambda x: x[1]) #params['data'] = sorted(params['data'], key=lambda x: x[5]) return render(request, 'attributes.html', params)
def timeseries(request): params = {} params['queries'] = internal.ListQueries( request, {'projectID': [request.session['projectID']]}) kwargs = {} kwargs['projectID'] = [request.session['projectID']] params['attributes'] = sorted( [x.name for x in internal.ListAttributes(request, kwargs)]) if request.method == 'POST': queryname = request.POST.get('query') or None attribute = request.POST.get('attribute') or 'Race' if not queryname: return render(request, 'home.html', params) query = Query.objects.get(project_id=request.session['projectID'], name=queryname) samplelist = query.expandsamples subjects = Attributes.objects.filter( project_id=request.session['projectID'], sample__in=samplelist, field='tssubject.').values_list('value', flat=True) allsamples = Attributes.objects.filter( project_id=request.session['projectID'], field='tssubject.', value__in=subjects).values('sample', 'value') subjectmap = {} for s in allsamples: subjectmap.setdefault(s['sample'], {}) subjectmap[s['sample']]['subject'] = s['value'] timepoints = Attributes.objects.filter( project_id=request.session['projectID'], field='tstimepoint.', sample__in=subjectmap.keys()).values('sample', 'value') for s in timepoints: subjectmap[s['sample']]['timepoint'] = float(s['value']) attributedata = Attributes.objects.filter( project_id=request.session['projectID'], sample__in=subjectmap.keys(), field=attribute).values('sample', 'value') for a in attributedata: subjectmap[a['sample']]['feature'] = a['value'] resp = [] for sample in subjectmap: try: resp.append({ 'subject': subjectmap[sample]['subject'], 'timepoint': subjectmap[sample]['timepoint'], 'feature': subjectmap[sample]['feature'], 'sample': sample }) except: continue return HttpResponse(json.dumps(resp), content_type="application/json") return render(request, 'metadataTimeline.html', params)
def getRareTaxa(request): params = dict() params['queries'] = internal.ListQueries( request, {'projectID': [request.session['projectID']]}) if request.method == 'POST': try: queryname = request.POST.get('query', None) datasetname = request.POST.get('dataset', None) method = request.POST.get('method', None) category = request.POST.get('category', None) minthresh = float(request.POST.get('minthresh', 0)) maxthresh = float(request.POST.get('maxthresh', 5)) minSamplePerc = float(request.POST.get('minSamplePerc', 5)) maxSamplePerc = float(request.POST.get('maxSamplePerc', 10)) query = Query.objects.get(project=request.session['projectID'], name=queryname) querysamples = query.expandsamples numSamples = len(querysamples) minSampleCount = numSamples * (minSamplePerc / 100.0) maxSampleCount = numSamples * (maxSamplePerc / 100.0) queryset = Analysis.objects.filter( project=request.session['projectID'], method=method, category=category, sample__in=querysamples) rareCandidates = queryset.values('entity').annotate( maxprofile=Max('profile')).annotate( minprofile=Min('profile')).annotate( mycount=Count('entity')).filter( maxprofile__lte=maxthresh, minprofile__gte=minthresh, mycount__gte=minSampleCount, mycount__lte=maxSampleCount).values_list( 'entity', 'minprofile', 'maxprofile', 'mycount').order_by('-mycount') resp = dict() resp['minthresh'] = minthresh resp['maxthresh'] = maxthresh resp['minSamplePerc'] = minSamplePerc resp['maxSamplePerc'] = maxSamplePerc resp['minSampleCount'] = minSampleCount resp['maxSampleCount'] = maxSampleCount resp['rareCandidates'] = len(rareCandidates) resp['numSamples'] = numSamples resp['rareTaxa'] = list(rareCandidates) return HttpResponse(json.dumps(resp), content_type='application/json') except: msg = "%s" % traceback.format_exc() messages.add_message(request, messages.ERROR, msg) return render(request, 'taxaInvestigator.html', params)
def manager(request): params = dict() params['queries'] = internal.ListQueries(request, {'projectID': [request.session['projectID']]}) if request.method == 'POST': resp = dict() queryname = request.POST.get('query', None) dataset = request.POST.get('dataset', None) method = request.POST.get('method', None) category=request.POST.get('category', None) count=request.POST.get('count', 20) if not queryname or not dataset or not method or not category: return HttpResponse(json.dumps('Please check that the input form is complete.'), content_type='application/json') query = Query.objects.get(project=request.session['projectID'], name=queryname) from rpy2 import robjects import rpy2.robjects.numpy2ri as rpyn # fetch Analysis data of interest heatmapScript = SCRIPTPATH+'r/heatmapDataCreator.R' robjects.r.source(heatmapScript) heatmapCommand = robjects.r['heatmapDataCreator'] profiles1 = Analysis.objects.filter(project=request.session['projectID'], dataset=dataset, method=method, category=category, sample__in=query.expandsamples).values_list('sample', 'entity', 'profile') # format analysis data and load into R profiles = zip(*profiles1) profileRdata = robjects.DataFrame({'samples': robjects.StrVector(profiles[0]), 'entity': robjects.StrVector(profiles[1]), 'profile': robjects.FloatVector(profiles[2])}) processedMatrix = heatmapCommand(profileRdata, count) vector=rpyn.ri2numpy(processedMatrix) resp['rows'] = list(processedMatrix.rownames) resp['cols'] = list(processedMatrix.colnames) resp['maxVal'] = numpy.amax(vector) resp['minVal'] = numpy.amin(vector) resp['data'] = vector.tolist() return HttpResponse(json.dumps(resp), content_type='application/json') return render(request, 'heatmap.html', params)
def alpha2(request): params = {} params['queries'] = internal.ListQueries( request, {'projectID': [request.session['projectID']]}) querynames = request.GET.getlist('query') or None inputdataset = request.GET.get('dataset') or None method = request.GET.get('method') or None category = request.GET.get('category') or None if querynames: if not querynames or not inputdataset or not method or not category: return render(request, 'alpha2.html', params) datapoints = [] outlierpoints = [] count = 0 for query in querynames: sampleslist = Query.objects.get( project=request.session['projectID'], name=query).expandsamples querydata = list( Calculation.objects.filter( project=request.session['projectID'], calculation='Alpha Diversity', dataset=inputdataset, method=method, category=category, sample__in=sampleslist).values_list('value', flat=True)) querydata.sort() myarr = np.array(querydata) median = np.percentile(myarr, 50) lowerq = np.percentile(myarr, 25) upperq = np.percentile(myarr, 75) iqr = 1.5 * (upperq - lowerq) outliers = [[count, x] for x in querydata if x < lowerq - iqr or x > iqr + upperq] lowerx = [x for x in querydata if x > lowerq - iqr][0] upperx = [x for x in querydata if x < upperq + iqr][-1] datapoints.append([lowerx, lowerq, median, upperq, upperx]) outlierpoints.append(outliers) count += 1 finaldata = [querynames, datapoints, outlierpoints] return HttpResponse(json.dumps(finaldata), content_type="application/json") return render(request, 'alpha2.html', params)
def main(request): params = {} params['queries'] = [] params['pubqueries'] = [] queries = internal.ListQueries(request, { 'projectID': [request.session['projectID']], 'full': [True] }) for q in queries: tmp = {} tmp['name'] = q['name'] tmp['numsamples'] = q['number of samples'] tmp['numsubjects'] = q['number of subjects'] tmp['numvisits'] = q['number of visits'] tmp['share'] = bool(q['share']) tmp['user'] = User.objects.get(pk=q['user_id']).username tmp['project'] = q['project_id'] tmp['description'] = q['description'] tmp['sqlstring'] = q['sqlstring'] if request.user.username == tmp['user']: params['queries'].append(tmp) else: params['pubqueries'].append(tmp) return render(request, 'managequeries.html', params)
def get(self, request, *args, **kwargs): kwargs = dict(request.GET) #kwargs['projectID'] = Project.objects.filter(name__in=kwargs['projectID']).values_list('pk', flat=True) return Response(internal.ListQueries(request, kwargs))
def main(request): params = {} params['queries'] = internal.ListQueries(request, {'projectID': [request.session['projectID']]}) querynames = request.GET.getlist('query') or None inputdataset = request.GET.get('dataset') or None method = request.GET.get('method') or None category = request.GET.get('category') or None if querynames: if not querynames or not inputdataset or not method or not category: return render(request, 'alpha.html', params) entities = [] samples = [] datahash = {} filename = "/tmp/%s-%d.txt" %(request.user.username, int(time.time())) for query in querynames: dataset = internal.GetDataset(request, params={'queryname': [query], 'projectID': [request.session['projectID']], 'dataset': [inputdataset], 'category': [category], 'method': [method]}) for d in dataset: if d.entity not in entities: entities.append(d.entity) if d.sample not in samples: samples.append(d.sample) datahash[d.sample] = {'query' : [query]} datahash[d.sample][d.entity] = d.profile if query not in datahash[d.sample]['query']: datahash[d.sample]['query'].append(query) with open(filename, 'w') as f: f.write('Rank,Taxa,') for sample in samples: f.write(str(sample)+',') f.write('\n') for entity in entities: mycontent = category+','+entity+',' for sample in samples: if entity in datahash[sample]: mycontent+=str(datahash[sample][entity]) else: mycontent += str('0') mycontent+=',' mycontent+='\n' f.write(mycontent) pca = myutils.runRscript('pca.R', filename) pca = pca.split('\n') pca.pop(0) pca.pop() finaldata = [{'name' : querynames[0], 'color': '#ff0000', 'data': []}, {'name': querynames[1], 'color': '#0000ff', 'data': []}, {'name': 'Both', 'color': '#00ff00', 'data' : []}] i = 0 for i in range(len(pca)): row = pca[i] if row.startswith('-----'): break cols = row.split(',') sample = cols[0] if sample[0] == 'X' and sample not in datahash: sample = sample[1:] xy = [float(cols[1]), float(cols[2])] if len(datahash[sample]['query']) > 1: finaldata[2]['data'].append(xy) elif querynames[0] in datahash[sample]['query']: finaldata[0]['data'].append(xy) else: finaldata[1]['data'].append(xy) i+=1 cutoff = i variances = [] for i in range(cutoff, len(pca)): row = pca[i] if row.startswith('-----'): break cols = row.split(',') variances.append(cols) i+=1 cutoff = i finaldata.append(variances) keytaxa = [] for i in range(cutoff, len(pca)): row = pca[i] cols = row.split(',') keytaxa.append(cols) finaldata.append(keytaxa) #os.remove(filename) return HttpResponse(json.dumps(finaldata), content_type="application/json") return render(request, 'pca.html', params)
def main(request): params = {} params['time'] = [] params['queries'] = internal.ListQueries( request, {'projectID': [request.session['projectID']]}) if request.method == 'POST': start = time.time() queryname = request.POST.get('query') or None dataset = request.POST.get('dataset') or None method = request.POST.get('method') or None category = request.POST.get('category') or None if not queryname or not dataset or not method or not category: return render(request, 'profile.html', params) params['feedback'] = 'Query selected: "%s"' % queryname query = internal.ListQueries( request, { 'projectID': [request.session['projectID']], 'full': [True], 'queryname': [queryname] }) if not query or not len(query): return render(request, 'profile.html', params) else: query = query[0] samplelist = myutils.fieldexpand(query['results']) mark = time.time() dataset = internal.GetDataset(request, params={ 'queryname': [queryname], 'projectID': [query['project_id']], 'dataset': [dataset], 'category': [category], 'method': [method] }) jsondata = [] magichash = {} maxhash = {} sorterhash = {} entityorder = [] first = True for d in dataset: if first: params['time'].append( 'parsed input and fetched API data: %.02f' % (time.time() - start)) first = False if d.profile < 1: continue if d.sample not in maxhash or d.profile > maxhash[d.sample]['val']: maxhash[d.sample] = {'entity': d.entity, 'val': d.profile} if d.entity in magichash: magichash[d.entity][d.sample] = d.profile else: magichash[d.entity] = {d.sample: d.profile} params['time'].append('computed max profile values: %.02f ' % (time.time() - start)) for sample in sorted(maxhash, key=lambda x: maxhash[x]['val'], reverse=True): if maxhash[sample]['val'] < 5: continue if maxhash[sample]['entity'] not in sorterhash: sorterhash[maxhash[sample]['entity']] = [sample] entityorder.append(maxhash[sample]['entity']) else: sorterhash[maxhash[sample]['entity']].append(sample) params['time'].append('sorted taxa: %.02f' % (time.time() - start)) samplelist = [] for element in entityorder: for x in sorterhash[element]: samplelist.append(x) params['time'].append('sorted samples: %.02f' % (time.time() - start)) for m in magichash: tmp = {'name': m, 'data': []} for i, s in enumerate(samplelist): if s in magichash[m]: tmp['data'].append(magichash[m][s]) else: tmp['data'].append(0) jsondata.append(tmp) params['time'].append('formatted data for JSON conversion: %.02f' % (time.time() - start)) params['json'] = json.dumps([samplelist, jsondata, params['time']]) return HttpResponse(params['json'], content_type="application/json") return render(request, 'profile.html', params)
def main(request): CUTOFF = 20 params = {} params['queries'] = internal.ListQueries( request, {'projectID': [request.session['projectID']]}) querynames = request.GET.getlist('query') or None querynames = request.GET.getlist('query') or None inputdataset = request.GET.get('dataset') or None category = request.GET.get('category') or None method = request.GET.get('method') or None print querynames if querynames: if not querynames or not inputdataset or not method or not category: return render(request, 'alpha.html', params) #entities = [] entities = myutils.topTaxa(request.session['projectID'], querynames, inputdataset, method, category, CUTOFF) elementOfGlory = {en: {q: {} for q in querynames} for en in entities} for query in querynames: filename = "/tmp/%s-%d.txt" % (request.user.pk, int(time.time())) datahash = {} for ent in entities: datahash[ent] = [] longest_list = -1 print "Query:", query dataset = internal.GetDataset(request, params={ 'queryname': [query], 'projectID': [request.session['projectID']], 'dataset': [inputdataset], 'category': [category], 'method': [method], 'entity': entities }) for d in dataset: taxa = d.entity datahash[taxa].append(d.profile) if len(datahash[taxa]) > longest_list: longest_list = len(datahash[taxa]) with open(filename, 'a') as f: for taxa in datahash: f.write(str(taxa)) count = 0 for profile in datahash[taxa]: count += 1 f.write(',' + str(profile)) elementOfGlory[taxa][query]['n'] = str(count) while count < longest_list: f.write(',' + str('0')) count += 1 f.write('\n') boxplot = myutils.runRscript('16sProfilesBoxplot.R', filename) os.remove(filename) tmp = boxplot.split('\n') boxplot = [] for bp in tmp: if bp == "": continue tmp2 = bp.split(',') tmp2 = [float(s) if isfloat(s) else s for s in tmp2] if tmp2[0] in elementOfGlory: elementOfGlory[tmp2[0]][query]['stats'] = tmp2 else: elementOfGlory[tmp2[0]] = {query: {'stats': tmp2}} finaldata = [[], [], [], len(querynames)] i = 0 for en in entities: for query in querynames: bp = elementOfGlory[en][query]['stats'] bp.pop(0) if len(bp[5:]): [ finaldata[2].append([i, float(x)]) for x in bp[5:] if x != '' ] finaldata[0].append(en + ' (' + elementOfGlory[en][query].get('n', '?') + ')') finaldata[1].append(bp[:5]) i += 1 return HttpResponse(json.dumps(finaldata), content_type="application/json") return render(request, '16sProfileBoxplot.html', params)
def manager(request): params = dict() params['queries'] = internal.ListQueries( request, {'projectID': [request.session['projectID']]}) params['attributes'] = internal.ListAttributes( request, {'projectID': [request.session['projectID']]}) if request.method == 'POST': queryname = request.POST.get('query', None) dataset = request.POST.get('dataset', None) method = request.POST.get('method', None) category = request.POST.get('category', None) metadatafield = request.POST.get('attribute', None) if not queryname or not dataset or not method or not category or not metadatafield: return HttpResponse( json.dumps('Please check that the input form is complete.'), content_type='application/json') query = Query.objects.get(project=request.session['projectID'], name=queryname) filename = md5(request.session['projectName'] + query.name + dataset + method + category + metadatafield).hexdigest() + '.pdf' outputpath = mroot + filename outputurl = murl + filename if not os.path.isfile(outputpath): from rpy2 import robjects from api.models import Analysis, Attributes # fetch Analysis data of interest clusterScript = SCRIPTPATH + 'r/hierarchicalClustering.R' robjects.r.source(clusterScript) clusterCommand = robjects.r['clusterSamples'] profiles1 = Analysis.objects.filter( project=request.session['projectID'], dataset=dataset, method=method, category=category, sample__in=query.expandsamples).values_list( 'sample', 'entity', 'profile') # format analysis data and load into R profiles = zip(*profiles1) profileRdata = robjects.DataFrame({ 'samples': robjects.StrVector(profiles[0]), 'entity': robjects.StrVector(profiles[1]), 'profile': robjects.FloatVector(profiles[2]) }) metadata = Attributes.objects.filter( project=request.session['projectID'], field=metadatafield, sample__in=query.expandsamples).values_list('sample', 'value') metadata = zip(*metadata) # load metadata into R and merge with ordered samples metadataRdata = robjects.DataFrame({ 'samples': robjects.StrVector(metadata[0]), metadatafield: robjects.StrVector(metadata[1]) }) clusterCommand(profileRdata, metadataRdata, outputpath) return HttpResponse(json.dumps(outputurl), content_type='application/json') return render(request, 'clustering.html', params)
def idea(request): from api.models import * from django.db.models import Max params = {} params['queries'] = internal.ListQueries( request, {'projectID': [request.session['projectID']]}) if request.method == 'POST': queryname = request.POST.get('query') or None datasetname = request.POST.get('dataset') or None method = request.POST.get('method') or None category = request.POST.get('category') or None featurecount = request.POST.get('amount', 10) featurecount = int(featurecount) - 1 if not queryname or not datasetname or not method or not category: return render(request, 'profile.html', params) query = internal.ListQueries( request, { 'projectID': [request.session['projectID']], 'full': [True], 'queryname': [queryname] }) if not query or not len(query): return render(request, 'profile.html', params) else: query = query[0] filename = md5('stackedbars.%d.%d.%s.%s.%s.%d' % (query['id'], query['project_id'], datasetname, method, category, featurecount)).hexdigest() + '.json' furl = murl + filename fpath = mroot + filename if not os.path.isfile(fpath): samplelist = myutils.fieldexpand(query['results']) toptaxa = Analysis.objects.filter( project=request.session['projectID'], dataset=datasetname, method=method, category=category, sample__in=samplelist).values('entity').annotate( maxprofile=Max('profile')).distinct().order_by( '-maxprofile')[:featurecount] taxadict = {} resp = [{'name': 'other', 'data': []}] for i, t in enumerate(toptaxa, start=1): taxadict[t['entity']] = {'loc': i, 'samples': []} resp.append({'name': t['entity'], 'data': []}) rows = Analysis.objects.filter( project=request.session['projectID'], dataset=datasetname, method=method, category=category, sample__in=samplelist, entity__in=taxadict.keys()).order_by('-profile') sampleloc = dict() sampleorder = [] for row in rows: if row.sample not in sampleloc: sampleloc[row.sample] = { 'group': row.entity, 'loc': len(taxadict[row.entity]['samples']), 'other': 100.0 } taxadict[row.entity]['samples'].append([{ 'sample': row.sample, 'entity': row.entity, 'y': row.profile }]) else: taxadict[sampleloc[row.sample]['group']]['samples'][ sampleloc[row.sample]['loc']].append({ 'sample': row.sample, 'entity': row.entity, 'y': row.profile }) sampleloc[row.sample]['other'] = sampleloc[ row.sample]['other'] - row.profile ctr = 0 for entitydict in toptaxa: entity = entitydict['entity'] for samplelist in taxadict[entity]['samples']: resp[0]['data'].append({ 'x': ctr, 'y': sampleloc[samplelist[0]['sample']]['other'] }) sampleorder.append(samplelist[0]['sample']) for sample in samplelist: sample.pop("sample", None) cur_entity = sample.pop("entity", None) sample['x'] = ctr resp[taxadict[cur_entity]['loc']]['data'].append( sample) ctr += 1 with open(fpath, 'w') as outfile: json.dump([sampleorder, resp], outfile) return HttpResponse(json.dumps(furl), content_type="application/json") return render(request, 'ideastackedbars.html', params)
def main(request): params = {} params['queries'] = internal.ListQueries( request, {'projectID': [request.session['projectID']]}) if request.method == 'POST': start = time.time() queryname = request.POST.get('query') or None datasetname = request.POST.get('dataset') or None method = request.POST.get('method') or None category = request.POST.get('category') or None if not queryname or not datasetname or not method or not category: return render(request, 'profile.html', params) params['feedback'] = 'Query selected: "%s"' % queryname query = internal.ListQueries( request, { 'projectID': [request.session['projectID']], 'full': [True], 'queryname': [queryname] }) if not query or not len(query): return render(request, 'profile.html', params) else: query = query[0] filename = 'stackedbars.%d.%d.%s.%s.%s.csv' % ( query['id'], query['project_id'], datasetname, method, category) furl = murl + filename fpath = mroot + filename if not os.path.isfile(fpath): samplelist = myutils.fieldexpand(query['results']) dataset = internal.GetDataset(request, params={ 'queryname': [queryname], 'projectID': [query['project_id']], 'dataset': [datasetname], 'category': [category], 'method': [method] }) with open(fpath, 'w') as f: f.write('sample,taxa,profile\n') #for d in dataset.order_by('sample'): # if d.profile > 0.1: f.write("%s,%s,%f\n" % ( d.sample, d.entity, d.profile )) taxahash = dict() samplemax = dict() datahash = dict() for d in dataset: if d.profile < 0.1: continue if d.entity not in taxahash or d.profile > taxahash[ d.entity]: taxahash[d.entity] = d.profile if d.entity not in datahash: datahash[d.entity] = dict() if d.sample not in samplemax: samplemax[d.sample] = {'e': d.entity, 'p': d.profile} datahash[d.entity][d.sample] = {d.entity: d.profile} elif d.profile > samplemax[d.sample]['p']: current = samplemax[d.sample] samplemax[d.sample] = {'e': d.entity, 'p': d.profile} sampledata = datahash[current['e']][d.sample] del datahash[current['e']][d.sample] sampledata[d.entity] = d.profile datahash[samplemax[d.sample]['e']][ d.sample] = sampledata else: datahash[samplemax[d.sample]['e']][d.sample][ d.entity] = d.profile for t, tmax in sorted(taxahash.items(), key=lambda x: -x[1]): for sample, data in sorted(datahash[t].items(), key=lambda x: -x[1][t]): for entity, profile in sorted(data.items(), key=lambda x: x[1]): f.write("%s,%s,%s\n" % (sample, entity, profile)) return HttpResponse(json.dumps(furl), content_type="application/json") return render(request, 'stackedbars.html', params)
def alpha(request): params = {} params['queries'] = internal.ListQueries( request, {'projectID': [request.session['projectID']]}) querynames = request.GET.getlist('query') or None inputdataset = request.GET.get('dataset') or None method = request.GET.get('method') or None category = request.GET.get('category') or None if querynames: if not querynames or not inputdataset or not method or not category: return render(request, 'alpha.html', params) datapoints = [] outlierpoints = [] count = 0 for query in querynames: entities = [] datahash = {} dataset = internal.GetDataset(request, params={ 'queryname': [query], 'projectID': [request.session['projectID']], 'dataset': [inputdataset], 'category': [category], 'method': [method] }) for d in dataset: if d.entity not in entities: entities.append(d.entity) if d.sample not in datahash: datahash[d.sample] = {} datahash[d.sample][d.entity] = d.numreads filename = "/tmp/%s-%d.txt" % (request.user.username, int(time.time())) with open(filename, 'w') as f: for taxa in entities: f.write(',' + str(taxa)) f.write('\n') for sample in datahash: mycontent = sample for taxa in entities: mycontent += ',' if taxa in datahash[sample]: mycontent += str(datahash[sample][taxa]) else: mycontent += str('0') mycontent += '\n' f.write(mycontent) boxplot = myutils.runRscript('alphaDiversity.R', filename) boxplot, outliers = boxplot.split('\n')[:2] boxplot = boxplot.split(',') outliers = outliers.split(',') if len(boxplot): boxplot = [float(x) for x in boxplot] if len(outliers): outliers = [[count, float(x)] for x in outliers if x != ''] os.remove(filename) datapoints.append(boxplot) outlierpoints.append(outliers) count += 1 finaldata = [querynames, datapoints, outlierpoints] return HttpResponse(json.dumps(finaldata), content_type="application/json") return render(request, 'alpha.html', params)
def mostAbundantOverTime(request): params = {} params['queries'] = internal.ListQueries( request, {'projectID': [request.session['projectID']]}) if request.method == 'POST': query = request.POST.get('query', None) dataset = request.POST.get('dataset', None) method = request.POST.get('method', None) category = request.POST.get('category', None) if not query: return HttpResponseBadRequest( json.dumps("query is a required argument"), content_type='application/json') if not dataset: return HttpResponseBadRequest( json.dumps("dataset is a required argument"), content_type='application/json') if not method: return HttpResponseBadRequest( json.dumps("method is a required argument"), content_type='application/json') if not category: return HttpResponseBadRequest( json.dumps("category is a required argument"), content_type='application/json') queryobj = Query.objects.get(project_id=request.session['projectID'], name=query) samplelist = queryobj.expandsamples subjects = Attributes.objects.filter( project_id=request.session['projectID'], sample__in=samplelist, field='tssubject.').values_list('value', flat=True) if not subjects: return HttpResponseBadRequest( json.dumps("project does not have timeseries information"), content_type='application/json') allsamples = Attributes.objects.filter( project_id=request.session['projectID'], field='tssubject.', value__in=subjects).values('sample', 'value') subjectmap = {} for s in allsamples: subjectmap.setdefault(s['sample'], {}) subjectmap[s['sample']]['subject'] = s['value'] timepoints = Attributes.objects.filter( project_id=request.session['projectID'], field='tstimepoint.', sample__in=subjectmap.keys()).values('sample', 'value') for s in timepoints: subjectmap[s['sample']]['timepoint'] = float(s['value']) mostAbundant = {} featureCount = Counter() for sample in subjectmap.keys(): try: dataObjs = Analysis.objects.filter( project_id=request.session['projectID'], dataset=dataset, method=method, category=category, sample=sample).order_by('-profile')[:3] except: continue for rank, dataObj in enumerate(dataObjs, start=1): mostAbundant.setdefault(sample, []) mostAbundant[sample].append( (dataObj.entity, dataObj.profile, rank)) featureCount[dataObj.entity] += 1 topFeatures = {} for x, y in featureCount.most_common(9): topFeatures[x] = y resp = [] for sample in mostAbundant: for entryrank in mostAbundant[sample]: entity, abundance, rank = entryrank if entity not in topFeatures: entity = 'Other' resp.append({ 'subject': subjectmap[sample]['subject'], 'feature': entity, 'value': abundance, 'timepoint': subjectmap[sample]['timepoint'], 'sample': sample, 'rank': rank }) return HttpResponse(json.dumps(resp), content_type='application/json') return render(request, 'mostAbundantOverTime.html', params)
def main(request): params = {} params['queries'] = internal.ListQueries( request, {'projectID': [request.session['projectID']]}) params['attributes'] = internal.ListAttributes( request, {'projectID': [request.session['projectID']]}) queryname = request.GET.get('query') or None inputdataset = request.GET.get('dataset') or None method = request.GET.get('method') or None category = request.GET.get('category') or None attribute = request.GET.get('attribute') or None mydata = {} sa = {} if queryname: if not queryname or not inputdataset or not method or not category: return HttpResponse( "required GET parameters: query, dataset, method, category", content_type='text/plain') dataset = internal.GetDataset(request, params={ 'queryname': [queryname], 'projectID': [request.session['projectID']], 'dataset': [inputdataset], 'category': [category], 'method': [method] }) attributes = internal.GetData( request, { 'queryname': [queryname], 'projectID': [request.session['projectID']], 'attribute': [ attribute, ], }) attributes.pop(0) for a in attributes: sa[a[0]] = a[1] samples = list(set(dataset.values_list('sample', flat=True))) for row in dataset: if row.entity in mydata: mydata[row.entity][row.sample] = row.profile / 100 else: mydata[row.entity] = {row.sample: row.profile / 100} inputfile = "/tmp/%d-%d-%s.txt" % (request.user.pk, int( time.time()), attribute.replace(' ', '_')) formatfile = inputfile + ".format" resultfile = inputfile + ".result" with open(inputfile, 'w') as f: f.write(attribute) for s in samples: if s in sa: f.write("\t" + str(sa[s]).lower()) else: f.write("\tNA") f.write("\nsubject_id") for s in samples: f.write("\t" + str(s)) for e in sorted(mydata): f.write("\n" + e) for s in samples: if s in mydata[e]: f.write("\t" + str(mydata[e][s])) else: f.write("\t0") formatresp = myutils.runPython("lefse/format_input.py", inputfile, [formatfile, "-u2", "-o1000000"]) lefseresp = myutils.runPython("lefse/run_lefse.py", formatfile, [resultfile]).strip() lefseresp = lefseresp.replace('\n', '<br />') lefseresp = "<strong>Query:</strong> " + queryname + "<br><strong>Attribute:</strong> " + attribute + "<hr>" + lefseresp #lefseresp = myutils.runCmd("python /home/orean/OREAN/scripts/misc/lefse/run_lefse.py "+ formatfile+" "+resultfile) rows = "" with open(resultfile, 'r') as f: rows = f.read() os.remove(inputfile) os.remove(formatfile) os.remove(resultfile) return HttpResponse(json.dumps({ 'msg': lefseresp, 'data': rows }), content_type='application/json') return render(request, "lefse.html", params)
def main(request): params = {} params['queries'] = internal.ListQueries(request, {'projectID': [request.session['projectID']]}) params['attributes'] = internal.ListAttributes(request, {'projectID': [request.session['projectID']]}) querynamelist = request.GET.getlist('query') or None inputdataset = request.GET.get('dataset') or None method = request.GET.get('method') or None category = request.GET.get('category') or None bothcount = 0 mydata = {} sa = {} if querynamelist: if not len(querynamelist) == 2 or not inputdataset or not method or not category: return HttpResponse("required GET parameters: query, dataset, method, category, query2" , content_type='text/plain') queryname, queryname2 = querynamelist query1dataset = internal.GetDataset(request, params={'queryname': [queryname], 'projectID': [request.session['projectID']], 'dataset': [inputdataset], 'category': [category], 'method': [method]}) query2dataset = internal.GetDataset(request, params={'queryname': [queryname2], 'projectID': [request.session['projectID']], 'dataset': [inputdataset], 'category': [category], 'method': [method]}) #query1dataset = internal.GetDataset(request, params={'queryname': [queryname], 'projectID': [request.session['projectID']], 'dataset': [inputdataset], 'method': [method]}) #query2dataset = internal.GetDataset(request, params={'queryname': [queryname2], 'projectID': [request.session['projectID']], 'dataset': [inputdataset], 'method': [method]}) query1samples = internal.ListQueries(request, {'projectID': [request.session['projectID']], 'full': [True], 'queryname': [queryname]})[0]['results'].split(',') query2samples = internal.ListQueries(request, {'projectID': [request.session['projectID']], 'full': [True], 'queryname': [queryname2]})[0]['results'].split(',') totalsamples = list(set(query1samples+query2samples)) dataset = query1dataset | query2dataset for s in totalsamples: if s in query1samples and s in query2samples: sa[s] = "Both" bothcount+=1 elif s in query1samples and s not in query2samples: sa[s] = queryname elif s not in query1samples and s in query2samples: sa[s] = queryname2 else: return HttpResponse(json.dumps({'msg': "Could not determine how to group sample '%s'. Aborting LefSE analysis." % s, 'data': False}), content_type='application/json') if bothcount == len(totalsamples): return HttpResponse(json.dumps({'msg': "Queries have the same sample composition. Cannot perform an analysis", 'data': False}), content_type='application/json') samples = list(set(dataset.values_list('sample', flat=True))) for row in dataset: #if row.entity in mydata: mydata[row.entity][row.sample] = row.profile/100 #else: mydata[row.entity] = {row.sample: row.profile/100} taxaName = row.entity if row.taxatree is not None: taxaName = row.taxatree.full_tree if taxaName in mydata: mydata[taxaName][row.sample] = row.profile/100 else: mydata[taxaName] = {row.sample: row.profile/100} inputfile = "/tmp/%d-%d-lefse.txt" %(request.user.pk, int(time.time())) formatfile = inputfile+".format" resultfile = inputfile+".result" with open(inputfile, 'w') as f: f.write("QueryStatus") for s in samples: if s in sa: f.write("\t"+str(sa[s]).lower()) else: f.write("\tNA") f.write("\nsubject_id") for s in samples: f.write("\t"+str(s)) for e in sorted(mydata): f.write("\n"+e) for s in samples: if s in mydata[e]: f.write("\t"+str(mydata[e][s])) else: f.write("\t0") formatresp = myutils.runPython("lefse/format_input.py", inputfile, [formatfile, "-u2", "-o1000000"]) lefseresp = myutils.runPython("lefse/run_lefse.py", formatfile, [resultfile]).strip() lefseresp = lefseresp.replace('\n', '<br />') lefseresp = "<strong>Query 1:</strong> "+queryname+"<br><strong>Query 2:</strong> "+queryname2+"<hr>"+lefseresp #lefseresp = myutils.runCmd("python /home/orean/OREAN/scripts/misc/lefse/run_lefse.py "+ formatfile+" "+resultfile) rows = "" with open(resultfile, 'r') as f: rows = f.read() #os.remove(inputfile) #os.remove(formatfile) #os.remove(resultfile) return HttpResponse(json.dumps({'msg': lefseresp, 'data': rows}), content_type='application/json') return render(request, "lefse2.html", params)