def viewTimecourseInTable(request):
params = {}
params['queries'] = internal.ListQueries(
request, {'projectID': [request.session['projectID']]})
params['attributes'] = sorted([
x.name for x in internal.ListAttributes(
request, {'projectID': [request.session['projectID']]})
])
if request.method == "POST":
name = request.POST.get('query', None)
attr = request.POST.get('attribute', None)
if name and attr:
query = Query.objects.get(project=request.session['projectID'],
name=name)
samples = query.expandsamples
svm = SubjectMap.objects.filter(sample__in=samples)
subjects = svm.values_list('subject', flat=True)
params['subjects'] = {}
for subj in subjects:
params['subjects'][subj] = []
visits = svm.filter(subject=subj).values_list(
'visit', flat=True).distinct()
data = {}
data['samples'] = []
for v in visits:
data['id'] = v
data['samples'] += svm.filter(
subject=subj,
visit=v).values_list('sample', flat=True).distinct()
params['subjects'][subj].append(data)
return render(request, 'viewTimecourseInTable.html', params)
def comparison(request):
params = {}
params['queries'] = internal.ListQueries(
request, {'projectID': [request.session['projectID']]})
params['attributes'] = sorted([
x.name for x in internal.ListAttributes(
request, {'projectID': [request.session['projectID']]})
])
return render(request, 'visualComparison.html', params)
def main(request):
params = {}
params['queries'] = internal.ListQueries(
request, {'projectID': [request.session['projectID']]})
kwargs = dict(request.GET)
kwargs['projectID'] = [request.session['projectID']]
params['attributes'] = sorted(
[x.name for x in internal.ListAttributes(request, kwargs)])
if request.method == 'POST':
queryname = request.POST.get('query') or None
attribute = request.POST.get('attribute') or 'Race'
if not queryname: return render(request, 'home.html', params)
params['feedback'] = 'Query selected: "%s"' % queryname
query = internal.ListQueries(request, {
'projectID': [request.session['projectID']],
'full': [True]
})
if not query or not len(query):
return render(request, 'home.html', params)
else:
query = query[0]
samplelist = myutils.fieldexpand(query['results'])
dataset = internal.ShowDistribution(
request, {
'queryname': [queryname],
'projectID': [query['project_id']],
'attribute': [
attribute,
],
})
jsondata = {'key': attribute, 'values': []}
for d in sorted(dataset[1:], key=lambda x: x[0]):
tmp = {'label': d[0], 'value': d[1]}
jsondata['values'].append(tmp)
params['data'] = dataset
params['json'] = json.dumps([jsondata])
params['selectedquery'] = queryname
params['selectedattribute'] = attribute
if request.POST.get('format') == 'json':
return HttpResponse(params['json'],
content_type="application/json")
#params['data'] = sorted(dataset, key=lambda x: x[1])
#params['data'] = sorted(params['data'], key=lambda x: x[5])
return render(request, 'attributes.html', params)
def timeseries(request):
params = {}
params['queries'] = internal.ListQueries(
request, {'projectID': [request.session['projectID']]})
kwargs = {}
kwargs['projectID'] = [request.session['projectID']]
params['attributes'] = sorted(
[x.name for x in internal.ListAttributes(request, kwargs)])
if request.method == 'POST':
queryname = request.POST.get('query') or None
attribute = request.POST.get('attribute') or 'Race'
if not queryname: return render(request, 'home.html', params)
query = Query.objects.get(project_id=request.session['projectID'],
name=queryname)
samplelist = query.expandsamples
subjects = Attributes.objects.filter(
project_id=request.session['projectID'],
sample__in=samplelist,
field='tssubject.').values_list('value', flat=True)
allsamples = Attributes.objects.filter(
project_id=request.session['projectID'],
field='tssubject.',
value__in=subjects).values('sample', 'value')
subjectmap = {}
for s in allsamples:
subjectmap.setdefault(s['sample'], {})
subjectmap[s['sample']]['subject'] = s['value']
timepoints = Attributes.objects.filter(
project_id=request.session['projectID'],
field='tstimepoint.',
sample__in=subjectmap.keys()).values('sample', 'value')
for s in timepoints:
subjectmap[s['sample']]['timepoint'] = float(s['value'])
attributedata = Attributes.objects.filter(
project_id=request.session['projectID'],
sample__in=subjectmap.keys(),
field=attribute).values('sample', 'value')
for a in attributedata:
subjectmap[a['sample']]['feature'] = a['value']
resp = []
for sample in subjectmap:
try:
resp.append({
'subject': subjectmap[sample]['subject'],
'timepoint': subjectmap[sample]['timepoint'],
'feature': subjectmap[sample]['feature'],
'sample': sample
})
except:
continue
return HttpResponse(json.dumps(resp), content_type="application/json")
return render(request, 'metadataTimeline.html', params)
def getRareTaxa(request):
params = dict()
params['queries'] = internal.ListQueries(
request, {'projectID': [request.session['projectID']]})
if request.method == 'POST':
try:
queryname = request.POST.get('query', None)
datasetname = request.POST.get('dataset', None)
method = request.POST.get('method', None)
category = request.POST.get('category', None)
minthresh = float(request.POST.get('minthresh', 0))
maxthresh = float(request.POST.get('maxthresh', 5))
minSamplePerc = float(request.POST.get('minSamplePerc', 5))
maxSamplePerc = float(request.POST.get('maxSamplePerc', 10))
query = Query.objects.get(project=request.session['projectID'],
name=queryname)
querysamples = query.expandsamples
numSamples = len(querysamples)
minSampleCount = numSamples * (minSamplePerc / 100.0)
maxSampleCount = numSamples * (maxSamplePerc / 100.0)
queryset = Analysis.objects.filter(
project=request.session['projectID'],
method=method,
category=category,
sample__in=querysamples)
rareCandidates = queryset.values('entity').annotate(
maxprofile=Max('profile')).annotate(
minprofile=Min('profile')).annotate(
mycount=Count('entity')).filter(
maxprofile__lte=maxthresh,
minprofile__gte=minthresh,
mycount__gte=minSampleCount,
mycount__lte=maxSampleCount).values_list(
'entity', 'minprofile', 'maxprofile',
'mycount').order_by('-mycount')
resp = dict()
resp['minthresh'] = minthresh
resp['maxthresh'] = maxthresh
resp['minSamplePerc'] = minSamplePerc
resp['maxSamplePerc'] = maxSamplePerc
resp['minSampleCount'] = minSampleCount
resp['maxSampleCount'] = maxSampleCount
resp['rareCandidates'] = len(rareCandidates)
resp['numSamples'] = numSamples
resp['rareTaxa'] = list(rareCandidates)
return HttpResponse(json.dumps(resp),
content_type='application/json')
except:
msg = "%s" % traceback.format_exc()
messages.add_message(request, messages.ERROR, msg)
return render(request, 'taxaInvestigator.html', params)
def manager(request):
params = dict()
params['queries'] = internal.ListQueries(request, {'projectID': [request.session['projectID']]})
if request.method == 'POST':
resp = dict()
queryname = request.POST.get('query', None)
dataset = request.POST.get('dataset', None)
method = request.POST.get('method', None)
category=request.POST.get('category', None)
count=request.POST.get('count', 20)
if not queryname or not dataset or not method or not category:
return HttpResponse(json.dumps('Please check that the input form is complete.'), content_type='application/json')
query = Query.objects.get(project=request.session['projectID'], name=queryname)
from rpy2 import robjects
import rpy2.robjects.numpy2ri as rpyn
# fetch Analysis data of interest
heatmapScript = SCRIPTPATH+'r/heatmapDataCreator.R'
robjects.r.source(heatmapScript)
heatmapCommand = robjects.r['heatmapDataCreator']
profiles1 = Analysis.objects.filter(project=request.session['projectID'],
dataset=dataset,
method=method,
category=category,
sample__in=query.expandsamples).values_list('sample',
'entity',
'profile')
# format analysis data and load into R
profiles = zip(*profiles1)
profileRdata = robjects.DataFrame({'samples': robjects.StrVector(profiles[0]),
'entity': robjects.StrVector(profiles[1]),
'profile': robjects.FloatVector(profiles[2])})
processedMatrix = heatmapCommand(profileRdata, count)
vector=rpyn.ri2numpy(processedMatrix)
resp['rows'] = list(processedMatrix.rownames)
resp['cols'] = list(processedMatrix.colnames)
resp['maxVal'] = numpy.amax(vector)
resp['minVal'] = numpy.amin(vector)
resp['data'] = vector.tolist()
return HttpResponse(json.dumps(resp), content_type='application/json')
return render(request, 'heatmap.html', params)
def alpha2(request):
params = {}
params['queries'] = internal.ListQueries(
request, {'projectID': [request.session['projectID']]})
querynames = request.GET.getlist('query') or None
inputdataset = request.GET.get('dataset') or None
method = request.GET.get('method') or None
category = request.GET.get('category') or None
if querynames:
if not querynames or not inputdataset or not method or not category:
return render(request, 'alpha2.html', params)
datapoints = []
outlierpoints = []
count = 0
for query in querynames:
sampleslist = Query.objects.get(
project=request.session['projectID'], name=query).expandsamples
querydata = list(
Calculation.objects.filter(
project=request.session['projectID'],
calculation='Alpha Diversity',
dataset=inputdataset,
method=method,
category=category,
sample__in=sampleslist).values_list('value', flat=True))
querydata.sort()
myarr = np.array(querydata)
median = np.percentile(myarr, 50)
lowerq = np.percentile(myarr, 25)
upperq = np.percentile(myarr, 75)
iqr = 1.5 * (upperq - lowerq)
outliers = [[count, x] for x in querydata
if x < lowerq - iqr or x > iqr + upperq]
lowerx = [x for x in querydata if x > lowerq - iqr][0]
upperx = [x for x in querydata if x < upperq + iqr][-1]
datapoints.append([lowerx, lowerq, median, upperq, upperx])
outlierpoints.append(outliers)
count += 1
finaldata = [querynames, datapoints, outlierpoints]
return HttpResponse(json.dumps(finaldata),
content_type="application/json")
return render(request, 'alpha2.html', params)
def main(request):
params = {}
params['queries'] = []
params['pubqueries'] = []
queries = internal.ListQueries(request, {
'projectID': [request.session['projectID']],
'full': [True]
})
for q in queries:
tmp = {}
tmp['name'] = q['name']
tmp['numsamples'] = q['number of samples']
tmp['numsubjects'] = q['number of subjects']
tmp['numvisits'] = q['number of visits']
tmp['share'] = bool(q['share'])
tmp['user'] = User.objects.get(pk=q['user_id']).username
tmp['project'] = q['project_id']
tmp['description'] = q['description']
tmp['sqlstring'] = q['sqlstring']
if request.user.username == tmp['user']: params['queries'].append(tmp)
else: params['pubqueries'].append(tmp)
return render(request, 'managequeries.html', params)
def get(self, request, *args, **kwargs):
kwargs = dict(request.GET)
#kwargs['projectID'] = Project.objects.filter(name__in=kwargs['projectID']).values_list('pk', flat=True)
return Response(internal.ListQueries(request, kwargs))
def main(request):
params = {}
params['queries'] = internal.ListQueries(request, {'projectID': [request.session['projectID']]})
querynames = request.GET.getlist('query') or None
inputdataset = request.GET.get('dataset') or None
method = request.GET.get('method') or None
category = request.GET.get('category') or None
if querynames:
if not querynames or not inputdataset or not method or not category: return render(request, 'alpha.html', params)
entities = []
samples = []
datahash = {}
filename = "/tmp/%s-%d.txt" %(request.user.username, int(time.time()))
for query in querynames:
dataset = internal.GetDataset(request, params={'queryname': [query], 'projectID': [request.session['projectID']], 'dataset': [inputdataset], 'category': [category], 'method': [method]})
for d in dataset:
if d.entity not in entities:
entities.append(d.entity)
if d.sample not in samples:
samples.append(d.sample)
datahash[d.sample] = {'query' : [query]}
datahash[d.sample][d.entity] = d.profile
if query not in datahash[d.sample]['query']: datahash[d.sample]['query'].append(query)
with open(filename, 'w') as f:
f.write('Rank,Taxa,')
for sample in samples: f.write(str(sample)+',')
f.write('\n')
for entity in entities:
mycontent = category+','+entity+','
for sample in samples:
if entity in datahash[sample]: mycontent+=str(datahash[sample][entity])
else: mycontent += str('0')
mycontent+=','
mycontent+='\n'
f.write(mycontent)
pca = myutils.runRscript('pca.R', filename)
pca = pca.split('\n')
pca.pop(0)
pca.pop()
finaldata = [{'name' : querynames[0], 'color': '#ff0000', 'data': []}, {'name': querynames[1], 'color': '#0000ff', 'data': []}, {'name': 'Both', 'color': '#00ff00', 'data' : []}]
i = 0
for i in range(len(pca)):
row = pca[i]
if row.startswith('-----'): break
cols = row.split(',')
sample = cols[0]
if sample[0] == 'X' and sample not in datahash: sample = sample[1:]
xy = [float(cols[1]), float(cols[2])]
if len(datahash[sample]['query']) > 1: finaldata[2]['data'].append(xy)
elif querynames[0] in datahash[sample]['query']: finaldata[0]['data'].append(xy)
else: finaldata[1]['data'].append(xy)
i+=1
cutoff = i
variances = []
for i in range(cutoff, len(pca)):
row = pca[i]
if row.startswith('-----'): break
cols = row.split(',')
variances.append(cols)
i+=1
cutoff = i
finaldata.append(variances)
keytaxa = []
for i in range(cutoff, len(pca)):
row = pca[i]
cols = row.split(',')
keytaxa.append(cols)
finaldata.append(keytaxa)
#os.remove(filename)
return HttpResponse(json.dumps(finaldata), content_type="application/json")
return render(request, 'pca.html', params)
def main(request):
params = {}
params['time'] = []
params['queries'] = internal.ListQueries(
request, {'projectID': [request.session['projectID']]})
if request.method == 'POST':
start = time.time()
queryname = request.POST.get('query') or None
dataset = request.POST.get('dataset') or None
method = request.POST.get('method') or None
category = request.POST.get('category') or None
if not queryname or not dataset or not method or not category:
return render(request, 'profile.html', params)
params['feedback'] = 'Query selected: "%s"' % queryname
query = internal.ListQueries(
request, {
'projectID': [request.session['projectID']],
'full': [True],
'queryname': [queryname]
})
if not query or not len(query):
return render(request, 'profile.html', params)
else:
query = query[0]
samplelist = myutils.fieldexpand(query['results'])
mark = time.time()
dataset = internal.GetDataset(request,
params={
'queryname': [queryname],
'projectID': [query['project_id']],
'dataset': [dataset],
'category': [category],
'method': [method]
})
jsondata = []
magichash = {}
maxhash = {}
sorterhash = {}
entityorder = []
first = True
for d in dataset:
if first:
params['time'].append(
'parsed input and fetched API data: %.02f' %
(time.time() - start))
first = False
if d.profile < 1: continue
if d.sample not in maxhash or d.profile > maxhash[d.sample]['val']:
maxhash[d.sample] = {'entity': d.entity, 'val': d.profile}
if d.entity in magichash: magichash[d.entity][d.sample] = d.profile
else: magichash[d.entity] = {d.sample: d.profile}
params['time'].append('computed max profile values: %.02f ' %
(time.time() - start))
for sample in sorted(maxhash,
key=lambda x: maxhash[x]['val'],
reverse=True):
if maxhash[sample]['val'] < 5: continue
if maxhash[sample]['entity'] not in sorterhash:
sorterhash[maxhash[sample]['entity']] = [sample]
entityorder.append(maxhash[sample]['entity'])
else:
sorterhash[maxhash[sample]['entity']].append(sample)
params['time'].append('sorted taxa: %.02f' % (time.time() - start))
samplelist = []
for element in entityorder:
for x in sorterhash[element]:
samplelist.append(x)
params['time'].append('sorted samples: %.02f' % (time.time() - start))
for m in magichash:
tmp = {'name': m, 'data': []}
for i, s in enumerate(samplelist):
if s in magichash[m]: tmp['data'].append(magichash[m][s])
else: tmp['data'].append(0)
jsondata.append(tmp)
params['time'].append('formatted data for JSON conversion: %.02f' %
(time.time() - start))
params['json'] = json.dumps([samplelist, jsondata, params['time']])
return HttpResponse(params['json'], content_type="application/json")
return render(request, 'profile.html', params)
def main(request):
CUTOFF = 20
params = {}
params['queries'] = internal.ListQueries(
request, {'projectID': [request.session['projectID']]})
querynames = request.GET.getlist('query') or None
querynames = request.GET.getlist('query') or None
inputdataset = request.GET.get('dataset') or None
category = request.GET.get('category') or None
method = request.GET.get('method') or None
print querynames
if querynames:
if not querynames or not inputdataset or not method or not category:
return render(request, 'alpha.html', params)
#entities = []
entities = myutils.topTaxa(request.session['projectID'], querynames,
inputdataset, method, category, CUTOFF)
elementOfGlory = {en: {q: {} for q in querynames} for en in entities}
for query in querynames:
filename = "/tmp/%s-%d.txt" % (request.user.pk, int(time.time()))
datahash = {}
for ent in entities:
datahash[ent] = []
longest_list = -1
print "Query:", query
dataset = internal.GetDataset(request,
params={
'queryname': [query],
'projectID':
[request.session['projectID']],
'dataset': [inputdataset],
'category': [category],
'method': [method],
'entity':
entities
})
for d in dataset:
taxa = d.entity
datahash[taxa].append(d.profile)
if len(datahash[taxa]) > longest_list:
longest_list = len(datahash[taxa])
with open(filename, 'a') as f:
for taxa in datahash:
f.write(str(taxa))
count = 0
for profile in datahash[taxa]:
count += 1
f.write(',' + str(profile))
elementOfGlory[taxa][query]['n'] = str(count)
while count < longest_list:
f.write(',' + str('0'))
count += 1
f.write('\n')
boxplot = myutils.runRscript('16sProfilesBoxplot.R', filename)
os.remove(filename)
tmp = boxplot.split('\n')
boxplot = []
for bp in tmp:
if bp == "": continue
tmp2 = bp.split(',')
tmp2 = [float(s) if isfloat(s) else s for s in tmp2]
if tmp2[0] in elementOfGlory:
elementOfGlory[tmp2[0]][query]['stats'] = tmp2
else:
elementOfGlory[tmp2[0]] = {query: {'stats': tmp2}}
finaldata = [[], [], [], len(querynames)]
i = 0
for en in entities:
for query in querynames:
bp = elementOfGlory[en][query]['stats']
bp.pop(0)
if len(bp[5:]):
[
finaldata[2].append([i, float(x)]) for x in bp[5:]
if x != ''
]
finaldata[0].append(en + ' (' +
elementOfGlory[en][query].get('n', '?') +
')')
finaldata[1].append(bp[:5])
i += 1
return HttpResponse(json.dumps(finaldata),
content_type="application/json")
return render(request, '16sProfileBoxplot.html', params)
def manager(request):
params = dict()
params['queries'] = internal.ListQueries(
request, {'projectID': [request.session['projectID']]})
params['attributes'] = internal.ListAttributes(
request, {'projectID': [request.session['projectID']]})
if request.method == 'POST':
queryname = request.POST.get('query', None)
dataset = request.POST.get('dataset', None)
method = request.POST.get('method', None)
category = request.POST.get('category', None)
metadatafield = request.POST.get('attribute', None)
if not queryname or not dataset or not method or not category or not metadatafield:
return HttpResponse(
json.dumps('Please check that the input form is complete.'),
content_type='application/json')
query = Query.objects.get(project=request.session['projectID'],
name=queryname)
filename = md5(request.session['projectName'] + query.name + dataset +
method + category + metadatafield).hexdigest() + '.pdf'
outputpath = mroot + filename
outputurl = murl + filename
if not os.path.isfile(outputpath):
from rpy2 import robjects
from api.models import Analysis, Attributes
# fetch Analysis data of interest
clusterScript = SCRIPTPATH + 'r/hierarchicalClustering.R'
robjects.r.source(clusterScript)
clusterCommand = robjects.r['clusterSamples']
profiles1 = Analysis.objects.filter(
project=request.session['projectID'],
dataset=dataset,
method=method,
category=category,
sample__in=query.expandsamples).values_list(
'sample', 'entity', 'profile')
# format analysis data and load into R
profiles = zip(*profiles1)
profileRdata = robjects.DataFrame({
'samples':
robjects.StrVector(profiles[0]),
'entity':
robjects.StrVector(profiles[1]),
'profile':
robjects.FloatVector(profiles[2])
})
metadata = Attributes.objects.filter(
project=request.session['projectID'],
field=metadatafield,
sample__in=query.expandsamples).values_list('sample', 'value')
metadata = zip(*metadata)
# load metadata into R and merge with ordered samples
metadataRdata = robjects.DataFrame({
'samples':
robjects.StrVector(metadata[0]),
metadatafield:
robjects.StrVector(metadata[1])
})
clusterCommand(profileRdata, metadataRdata, outputpath)
return HttpResponse(json.dumps(outputurl),
content_type='application/json')
return render(request, 'clustering.html', params)
def idea(request):
from api.models import *
from django.db.models import Max
params = {}
params['queries'] = internal.ListQueries(
request, {'projectID': [request.session['projectID']]})
if request.method == 'POST':
queryname = request.POST.get('query') or None
datasetname = request.POST.get('dataset') or None
method = request.POST.get('method') or None
category = request.POST.get('category') or None
featurecount = request.POST.get('amount', 10)
featurecount = int(featurecount) - 1
if not queryname or not datasetname or not method or not category:
return render(request, 'profile.html', params)
query = internal.ListQueries(
request, {
'projectID': [request.session['projectID']],
'full': [True],
'queryname': [queryname]
})
if not query or not len(query):
return render(request, 'profile.html', params)
else:
query = query[0]
filename = md5('stackedbars.%d.%d.%s.%s.%s.%d' %
(query['id'], query['project_id'], datasetname, method,
category, featurecount)).hexdigest() + '.json'
furl = murl + filename
fpath = mroot + filename
if not os.path.isfile(fpath):
samplelist = myutils.fieldexpand(query['results'])
toptaxa = Analysis.objects.filter(
project=request.session['projectID'],
dataset=datasetname,
method=method,
category=category,
sample__in=samplelist).values('entity').annotate(
maxprofile=Max('profile')).distinct().order_by(
'-maxprofile')[:featurecount]
taxadict = {}
resp = [{'name': 'other', 'data': []}]
for i, t in enumerate(toptaxa, start=1):
taxadict[t['entity']] = {'loc': i, 'samples': []}
resp.append({'name': t['entity'], 'data': []})
rows = Analysis.objects.filter(
project=request.session['projectID'],
dataset=datasetname,
method=method,
category=category,
sample__in=samplelist,
entity__in=taxadict.keys()).order_by('-profile')
sampleloc = dict()
sampleorder = []
for row in rows:
if row.sample not in sampleloc:
sampleloc[row.sample] = {
'group': row.entity,
'loc': len(taxadict[row.entity]['samples']),
'other': 100.0
}
taxadict[row.entity]['samples'].append([{
'sample': row.sample,
'entity': row.entity,
'y': row.profile
}])
else:
taxadict[sampleloc[row.sample]['group']]['samples'][
sampleloc[row.sample]['loc']].append({
'sample': row.sample,
'entity': row.entity,
'y': row.profile
})
sampleloc[row.sample]['other'] = sampleloc[
row.sample]['other'] - row.profile
ctr = 0
for entitydict in toptaxa:
entity = entitydict['entity']
for samplelist in taxadict[entity]['samples']:
resp[0]['data'].append({
'x':
ctr,
'y':
sampleloc[samplelist[0]['sample']]['other']
})
sampleorder.append(samplelist[0]['sample'])
for sample in samplelist:
sample.pop("sample", None)
cur_entity = sample.pop("entity", None)
sample['x'] = ctr
resp[taxadict[cur_entity]['loc']]['data'].append(
sample)
ctr += 1
with open(fpath, 'w') as outfile:
json.dump([sampleorder, resp], outfile)
return HttpResponse(json.dumps(furl), content_type="application/json")
return render(request, 'ideastackedbars.html', params)
def main(request):
params = {}
params['queries'] = internal.ListQueries(
request, {'projectID': [request.session['projectID']]})
if request.method == 'POST':
start = time.time()
queryname = request.POST.get('query') or None
datasetname = request.POST.get('dataset') or None
method = request.POST.get('method') or None
category = request.POST.get('category') or None
if not queryname or not datasetname or not method or not category:
return render(request, 'profile.html', params)
params['feedback'] = 'Query selected: "%s"' % queryname
query = internal.ListQueries(
request, {
'projectID': [request.session['projectID']],
'full': [True],
'queryname': [queryname]
})
if not query or not len(query):
return render(request, 'profile.html', params)
else:
query = query[0]
filename = 'stackedbars.%d.%d.%s.%s.%s.csv' % (
query['id'], query['project_id'], datasetname, method, category)
furl = murl + filename
fpath = mroot + filename
if not os.path.isfile(fpath):
samplelist = myutils.fieldexpand(query['results'])
dataset = internal.GetDataset(request,
params={
'queryname': [queryname],
'projectID':
[query['project_id']],
'dataset': [datasetname],
'category': [category],
'method': [method]
})
with open(fpath, 'w') as f:
f.write('sample,taxa,profile\n')
#for d in dataset.order_by('sample'):
# if d.profile > 0.1: f.write("%s,%s,%f\n" % ( d.sample, d.entity, d.profile ))
taxahash = dict()
samplemax = dict()
datahash = dict()
for d in dataset:
if d.profile < 0.1: continue
if d.entity not in taxahash or d.profile > taxahash[
d.entity]:
taxahash[d.entity] = d.profile
if d.entity not in datahash: datahash[d.entity] = dict()
if d.sample not in samplemax:
samplemax[d.sample] = {'e': d.entity, 'p': d.profile}
datahash[d.entity][d.sample] = {d.entity: d.profile}
elif d.profile > samplemax[d.sample]['p']:
current = samplemax[d.sample]
samplemax[d.sample] = {'e': d.entity, 'p': d.profile}
sampledata = datahash[current['e']][d.sample]
del datahash[current['e']][d.sample]
sampledata[d.entity] = d.profile
datahash[samplemax[d.sample]['e']][
d.sample] = sampledata
else:
datahash[samplemax[d.sample]['e']][d.sample][
d.entity] = d.profile
for t, tmax in sorted(taxahash.items(), key=lambda x: -x[1]):
for sample, data in sorted(datahash[t].items(),
key=lambda x: -x[1][t]):
for entity, profile in sorted(data.items(),
key=lambda x: x[1]):
f.write("%s,%s,%s\n" % (sample, entity, profile))
return HttpResponse(json.dumps(furl), content_type="application/json")
return render(request, 'stackedbars.html', params)
def alpha(request):
params = {}
params['queries'] = internal.ListQueries(
request, {'projectID': [request.session['projectID']]})
querynames = request.GET.getlist('query') or None
inputdataset = request.GET.get('dataset') or None
method = request.GET.get('method') or None
category = request.GET.get('category') or None
if querynames:
if not querynames or not inputdataset or not method or not category:
return render(request, 'alpha.html', params)
datapoints = []
outlierpoints = []
count = 0
for query in querynames:
entities = []
datahash = {}
dataset = internal.GetDataset(request,
params={
'queryname': [query],
'projectID':
[request.session['projectID']],
'dataset': [inputdataset],
'category': [category],
'method': [method]
})
for d in dataset:
if d.entity not in entities:
entities.append(d.entity)
if d.sample not in datahash: datahash[d.sample] = {}
datahash[d.sample][d.entity] = d.numreads
filename = "/tmp/%s-%d.txt" % (request.user.username,
int(time.time()))
with open(filename, 'w') as f:
for taxa in entities:
f.write(',' + str(taxa))
f.write('\n')
for sample in datahash:
mycontent = sample
for taxa in entities:
mycontent += ','
if taxa in datahash[sample]:
mycontent += str(datahash[sample][taxa])
else:
mycontent += str('0')
mycontent += '\n'
f.write(mycontent)
boxplot = myutils.runRscript('alphaDiversity.R', filename)
boxplot, outliers = boxplot.split('\n')[:2]
boxplot = boxplot.split(',')
outliers = outliers.split(',')
if len(boxplot): boxplot = [float(x) for x in boxplot]
if len(outliers):
outliers = [[count, float(x)] for x in outliers if x != '']
os.remove(filename)
datapoints.append(boxplot)
outlierpoints.append(outliers)
count += 1
finaldata = [querynames, datapoints, outlierpoints]
return HttpResponse(json.dumps(finaldata),
content_type="application/json")
return render(request, 'alpha.html', params)
def mostAbundantOverTime(request):
params = {}
params['queries'] = internal.ListQueries(
request, {'projectID': [request.session['projectID']]})
if request.method == 'POST':
query = request.POST.get('query', None)
dataset = request.POST.get('dataset', None)
method = request.POST.get('method', None)
category = request.POST.get('category', None)
if not query:
return HttpResponseBadRequest(
json.dumps("query is a required argument"),
content_type='application/json')
if not dataset:
return HttpResponseBadRequest(
json.dumps("dataset is a required argument"),
content_type='application/json')
if not method:
return HttpResponseBadRequest(
json.dumps("method is a required argument"),
content_type='application/json')
if not category:
return HttpResponseBadRequest(
json.dumps("category is a required argument"),
content_type='application/json')
queryobj = Query.objects.get(project_id=request.session['projectID'],
name=query)
samplelist = queryobj.expandsamples
subjects = Attributes.objects.filter(
project_id=request.session['projectID'],
sample__in=samplelist,
field='tssubject.').values_list('value', flat=True)
if not subjects:
return HttpResponseBadRequest(
json.dumps("project does not have timeseries information"),
content_type='application/json')
allsamples = Attributes.objects.filter(
project_id=request.session['projectID'],
field='tssubject.',
value__in=subjects).values('sample', 'value')
subjectmap = {}
for s in allsamples:
subjectmap.setdefault(s['sample'], {})
subjectmap[s['sample']]['subject'] = s['value']
timepoints = Attributes.objects.filter(
project_id=request.session['projectID'],
field='tstimepoint.',
sample__in=subjectmap.keys()).values('sample', 'value')
for s in timepoints:
subjectmap[s['sample']]['timepoint'] = float(s['value'])
mostAbundant = {}
featureCount = Counter()
for sample in subjectmap.keys():
try:
dataObjs = Analysis.objects.filter(
project_id=request.session['projectID'],
dataset=dataset,
method=method,
category=category,
sample=sample).order_by('-profile')[:3]
except:
continue
for rank, dataObj in enumerate(dataObjs, start=1):
mostAbundant.setdefault(sample, [])
mostAbundant[sample].append(
(dataObj.entity, dataObj.profile, rank))
featureCount[dataObj.entity] += 1
topFeatures = {}
for x, y in featureCount.most_common(9):
topFeatures[x] = y
resp = []
for sample in mostAbundant:
for entryrank in mostAbundant[sample]:
entity, abundance, rank = entryrank
if entity not in topFeatures:
entity = 'Other'
resp.append({
'subject': subjectmap[sample]['subject'],
'feature': entity,
'value': abundance,
'timepoint': subjectmap[sample]['timepoint'],
'sample': sample,
'rank': rank
})
return HttpResponse(json.dumps(resp), content_type='application/json')
return render(request, 'mostAbundantOverTime.html', params)
def main(request):
params = {}
params['queries'] = internal.ListQueries(
request, {'projectID': [request.session['projectID']]})
params['attributes'] = internal.ListAttributes(
request, {'projectID': [request.session['projectID']]})
queryname = request.GET.get('query') or None
inputdataset = request.GET.get('dataset') or None
method = request.GET.get('method') or None
category = request.GET.get('category') or None
attribute = request.GET.get('attribute') or None
mydata = {}
sa = {}
if queryname:
if not queryname or not inputdataset or not method or not category:
return HttpResponse(
"required GET parameters: query, dataset, method, category",
content_type='text/plain')
dataset = internal.GetDataset(request,
params={
'queryname': [queryname],
'projectID':
[request.session['projectID']],
'dataset': [inputdataset],
'category': [category],
'method': [method]
})
attributes = internal.GetData(
request, {
'queryname': [queryname],
'projectID': [request.session['projectID']],
'attribute': [
attribute,
],
})
attributes.pop(0)
for a in attributes:
sa[a[0]] = a[1]
samples = list(set(dataset.values_list('sample', flat=True)))
for row in dataset:
if row.entity in mydata:
mydata[row.entity][row.sample] = row.profile / 100
else:
mydata[row.entity] = {row.sample: row.profile / 100}
inputfile = "/tmp/%d-%d-%s.txt" % (request.user.pk, int(
time.time()), attribute.replace(' ', '_'))
formatfile = inputfile + ".format"
resultfile = inputfile + ".result"
with open(inputfile, 'w') as f:
f.write(attribute)
for s in samples:
if s in sa: f.write("\t" + str(sa[s]).lower())
else: f.write("\tNA")
f.write("\nsubject_id")
for s in samples:
f.write("\t" + str(s))
for e in sorted(mydata):
f.write("\n" + e)
for s in samples:
if s in mydata[e]: f.write("\t" + str(mydata[e][s]))
else: f.write("\t0")
formatresp = myutils.runPython("lefse/format_input.py", inputfile,
[formatfile, "-u2", "-o1000000"])
lefseresp = myutils.runPython("lefse/run_lefse.py", formatfile,
[resultfile]).strip()
lefseresp = lefseresp.replace('\n', '<br />')
lefseresp = "<strong>Query:</strong> " + queryname + "<br><strong>Attribute:</strong> " + attribute + "<hr>" + lefseresp
#lefseresp = myutils.runCmd("python /home/orean/OREAN/scripts/misc/lefse/run_lefse.py "+ formatfile+" "+resultfile)
rows = ""
with open(resultfile, 'r') as f:
rows = f.read()
os.remove(inputfile)
os.remove(formatfile)
os.remove(resultfile)
return HttpResponse(json.dumps({
'msg': lefseresp,
'data': rows
}),
content_type='application/json')
return render(request, "lefse.html", params)
def main(request):
params = {}
params['queries'] = internal.ListQueries(request, {'projectID': [request.session['projectID']]})
params['attributes'] = internal.ListAttributes(request, {'projectID': [request.session['projectID']]})
querynamelist = request.GET.getlist('query') or None
inputdataset = request.GET.get('dataset') or None
method = request.GET.get('method') or None
category = request.GET.get('category') or None
bothcount = 0
mydata = {}
sa = {}
if querynamelist:
if not len(querynamelist) == 2 or not inputdataset or not method or not category: return HttpResponse("required GET parameters: query, dataset, method, category, query2" , content_type='text/plain')
queryname, queryname2 = querynamelist
query1dataset = internal.GetDataset(request, params={'queryname': [queryname], 'projectID': [request.session['projectID']], 'dataset': [inputdataset], 'category': [category], 'method': [method]})
query2dataset = internal.GetDataset(request, params={'queryname': [queryname2], 'projectID': [request.session['projectID']], 'dataset': [inputdataset], 'category': [category], 'method': [method]})
#query1dataset = internal.GetDataset(request, params={'queryname': [queryname], 'projectID': [request.session['projectID']], 'dataset': [inputdataset], 'method': [method]})
#query2dataset = internal.GetDataset(request, params={'queryname': [queryname2], 'projectID': [request.session['projectID']], 'dataset': [inputdataset], 'method': [method]})
query1samples = internal.ListQueries(request, {'projectID': [request.session['projectID']], 'full': [True], 'queryname': [queryname]})[0]['results'].split(',')
query2samples = internal.ListQueries(request, {'projectID': [request.session['projectID']], 'full': [True], 'queryname': [queryname2]})[0]['results'].split(',')
totalsamples = list(set(query1samples+query2samples))
dataset = query1dataset | query2dataset
for s in totalsamples:
if s in query1samples and s in query2samples:
sa[s] = "Both"
bothcount+=1
elif s in query1samples and s not in query2samples:
sa[s] = queryname
elif s not in query1samples and s in query2samples:
sa[s] = queryname2
else:
return HttpResponse(json.dumps({'msg': "Could not determine how to group sample '%s'. Aborting LefSE analysis." % s, 'data': False}), content_type='application/json')
if bothcount == len(totalsamples): return HttpResponse(json.dumps({'msg': "Queries have the same sample composition. Cannot perform an analysis", 'data': False}), content_type='application/json')
samples = list(set(dataset.values_list('sample', flat=True)))
for row in dataset:
#if row.entity in mydata: mydata[row.entity][row.sample] = row.profile/100
#else: mydata[row.entity] = {row.sample: row.profile/100}
taxaName = row.entity
if row.taxatree is not None: taxaName = row.taxatree.full_tree
if taxaName in mydata: mydata[taxaName][row.sample] = row.profile/100
else: mydata[taxaName] = {row.sample: row.profile/100}
inputfile = "/tmp/%d-%d-lefse.txt" %(request.user.pk, int(time.time()))
formatfile = inputfile+".format"
resultfile = inputfile+".result"
with open(inputfile, 'w') as f:
f.write("QueryStatus")
for s in samples:
if s in sa: f.write("\t"+str(sa[s]).lower())
else: f.write("\tNA")
f.write("\nsubject_id")
for s in samples: f.write("\t"+str(s))
for e in sorted(mydata):
f.write("\n"+e)
for s in samples:
if s in mydata[e]: f.write("\t"+str(mydata[e][s]))
else: f.write("\t0")
formatresp = myutils.runPython("lefse/format_input.py", inputfile, [formatfile, "-u2", "-o1000000"])
lefseresp = myutils.runPython("lefse/run_lefse.py", formatfile, [resultfile]).strip()
lefseresp = lefseresp.replace('\n', '<br />')
lefseresp = "<strong>Query 1:</strong> "+queryname+"<br><strong>Query 2:</strong> "+queryname2+"<hr>"+lefseresp
#lefseresp = myutils.runCmd("python /home/orean/OREAN/scripts/misc/lefse/run_lefse.py "+ formatfile+" "+resultfile)
rows = ""
with open(resultfile, 'r') as f:
rows = f.read()
#os.remove(inputfile)
#os.remove(formatfile)
#os.remove(resultfile)
return HttpResponse(json.dumps({'msg': lefseresp, 'data': rows}), content_type='application/json')
return render(request, "lefse2.html", params)