def fieldtobact(expdat,field,bactname='',meanreads=1000,cutoff=0):
"""
convert values in a map file field to a new bacteria (to facilitate numeric analysis)
input:
expdat : Experiment
field : string
name of the field to convert
bactname : string
name of the new bacteria (empty to have similar to field name)
meanreads : int
the mean number of reads for the new field bacteria
cutoff : int
the minimal value of the field per sample (otherwise replace with meanreads)
output:
newexp : Experiment
with added bacteria with the field vals as reads
"""
params=locals()
if len(bactname)==0:
bactname=field
fv=hs.getfieldvals(expdat,field)
vals=np.array(hs.tofloat(fv))
okpos=np.where(vals>=cutoff)[0]
badpos=np.where(vals<cutoff)[0]
scalefactor=np.mean(vals[okpos])
vals[okpos]=(vals[okpos]/scalefactor)*meanreads
vals[badpos]=meanreads
newexp=hs.copyexp(expdat)
hs.insertbacteria(newexp,vals,bactname,bactname,logit=False)
newexp.filters.append('add bacteria from map field %s' % field)
hs.addcommand(newexp,"fieldtobact",params=params,replaceparams={'expdat':expdat})
return(newexp)
def sortsamples(exp,field,numeric=False,logit=True):
"""
sort samples according to field
input:
exp : Experiment
field : string
name of the field to sort by
numeric : bool
True for numeric values in field, false for text
output:
newexp : Experiment
the sorted experiment
"""
params=locals()
fvals=hs.getfieldvals(exp,field)
if numeric:
fvals=hs.tofloat(fvals)
svals,sidx=hs.isort(fvals)
newexp=hs.reordersamples(exp,sidx)
if logit:
hs.addcommand(newexp,"sortsamples",params=params,replaceparams={'exp':exp})
newexp.filters.append('sorted samples by field %s' % field)
return newexp
def loaddistmat(expdat,dmfilename):
"""
load a distance matrix (from qiime) for analysis
input:
expdat : Experiment
dmfilename : string
name of the qiime distance matrix file
output:
distmat : numpy 2d array
the distance matrix
dsamp : dict
the mapping to position in the experiment for each distmat entry
"""
fl=open(dmfilename,'rU')
# get the column ids
head=fl.readline().strip('\n')
ids=head.split('\t')
ids=ids[1:]
dist=np.array([])
snames={}
for idx,cline in enumerate(fl):
cline=cline.strip("\n")
vals=cline.split('\t')
dist=np.vstack((dist,hs.tofloat(vals[1:]))) if dist.size else np.array(hs.tofloat(vals[1:]))
if not vals[0]==ids[idx]:
hs.Debug(9,"strange! line %d row head %s but col head %s" % (idx,vals[0],ids[idx]))
snames[vals[0]]=idx
fl.close()
expkeep=[]
distorder=[]
dsamp={}
for idx,csamp in enumerate(expdat.samples):
if csamp in snames:
distorder.append(snames[csamp])
expkeep.append(idx)
dsamp[csamp]=snames[csamp]
hs.Debug(6,"%d samples in dist mat, %d samples in experiment" % (len(ids),len(expdat.samples)))
hs.Debug(6,"%d samples to keep from dist mat, %d samples to keep from experiment" % (len(distorder),len(expkeep)))
return dist,dsamp
def plotseqfreq(expdat,seqs,toaxis=False,xfield=False,normalizey=False):
"""
plot the frequency of sequences in seq as a function of the sortfield
input:
expdat : Experiment
seqs : list of sequence strings
a list of sequnces (acgt) to plot
toaxis : matplotlib axis
if not empty - the axis to plot to, False plot a new figure
xfield : string
if not empty - space the points on the x axis according to (numeric) value of in xfield, False - just according to the sorted order
normalizey : bool
True: normalize all y values to 0-1, False: no normalization
"""
if xfield:
xdat=hs.tofloat(getfieldvals(expdat,xfield))
else:
xdat=range(len(expdat.samples))
sv,si=hs.isort(xdat)
if not toaxis:
figure()
toaxis=plt.gca()
labels=[]
for cseq in seqs:
if cseq not in expdat.seqdict:
continue
spos=expdat.seqdict[cseq]
cdat=expdat.data[spos,si]
if normalizey:
cdat=cdat/sum(cdat)
toaxis.plot(sv,cdat)
labels.append(str(expdat.sids[spos])+'-'+expdat.tax[spos])
labels=hs.clipstrings(labels,20,reverse=True)
toaxis.legend(labels,prop={'size':6})
toaxis.set_xticks(xdat)
def plotexp(exp,sortby=False,numeric=False,minreads=4,rangeall=False,seqdb=None,cdb=None,showline=True,ontofig=False,usegui=True,showxall=False,showcolorbar=False,ptitle=False,lowcutoff=1,uselog=True,showxlabel=True,colormap=False,colorrange=False,linewidth=2,subline='',showhline=True,newfig=True,fixfont=False,fontsize=None,nosort=False,zeroisnone=False,xlabelrotation=45,showtaxnames=False):
"""
Plot an experiment
input:
exp - from load()
sortby - name of mapping file field to sort by or Flase to not sort
numeric - True if the field is numeric
minreads - minimum number of reads per bacteria in order to show it or 0 to show all
rangeall - True to show all frequencies in image scale, false to saturate at 10%
seqdb - the SRBactDB database (from bactdb.load)
cdb - the cool sequences database (from cooldb.load), or None (default) to use the heatsequer loaded cdb
showline - if True plot lines between category values
ontofig - name of ontology to plot for bactdb or false to no plot
usegui - True use a gui for otu summary, False just print
showxall - True to show all sample names when not sorting, False to show no more than 10
showcolorbar - True to plot the colorbar. False to not plot
ptitle : str (optional)
'' to show o show processing history as name, None to not show title, or str of name of the figure
lowcutoff - minimal value for read (for 0 log transform) - the minimal resolution - could be 10000*2/origreads
showxlabel : bool
True to show the x label (default), False to hide it
colormap : string or False
name of colormap or False (default) to use mpl default colormap
colorrange : [min,max] or False
[min,max] to set the colormap range, False to use data min,max (default) as specified in rangeall
subline : str
Name of category for subline plotting or '' (Default) for no sublines
showhline : bool
True (default) to plot the horizontal lines listed in exp.hlines. False to not plot them
newfig : bool
True (default) to open figure in new window, False to use current
fixfont : bool (optional)
False (default) to use fixedfont, True to use fixed width font
fontsize : int or None (optional)
None (default) to use default font size, number to use that font size
nosort : bool (optional)
False (default) to sort by the sort field, True to skip the sorting
zeroisnone : bool (optional)
False (default) to plot zeros as 0, True to assign None (white color)
xlabelrotation : int (optional)
the rotation of the xtick labels
showtaxnames : book (optional)
False (default) to not show tax names (need to press 'h' to show)
True to show the taxonomy names
output:
newexp - the plotted experiment (sorted and filtered)
ax - the plot axis
"""
hs.Debug(1,"Plot experiment %s" % exp.studyname)
hs.Debug(1,"Commands:")
for ccommand in exp.commands:
hs.Debug(1,"%s" % ccommand)
if exp.sparse:
hs.Debug(9,'Sparse matrix - converting to dense')
exp=hs.copyexp(exp,todense=True)
vals=[]
if cdb is None:
cdb=hs.cdb
if seqdb is None:
seqdb=hs.bdb
if sortby:
if not nosort:
hs.Debug(1,"Sorting by field %s" % sortby)
for csamp in exp.samples:
vals.append(exp.smap[csamp][sortby])
if numeric:
hs.Debug(1,"(numeric sort)")
vals=hs.tofloat(vals)
svals,sidx=hs.isort(vals)
newexp=hs.reordersamples(exp,sidx)
else:
hs.Debug(1,"no sorting but showing columns")
svals=hs.getfieldvals(exp,sortby)
newexp=hs.copyexp(exp)
else:
hs.Debug(1,"No sample sorting")
svals=hs.getfieldvals(exp,'#SampleID')
newexp=hs.copyexp(exp)
hs.Debug(1,"Filtering min reads. original bacteria - %d" % len(newexp.seqs))
if minreads>0:
newexp=hs.filterminreads(newexp,minreads,logit=uselog)
hs.Debug(1,"New number of bacteria %d" % len(newexp.seqs))
newexp.seqdb=seqdb
newexp.cdb=cdb
newexp.scdb=hs.scdb
# if usegui:
# hs.Debug(1,"Using the GUI window")
# import heatsequer.plots.plotwingui
# from PyQt4 import QtGui
# app = QtGui.QApplication(sys.argv)
# guiwin = heatsequer.plots.plotwingui.PlotGUIWindow(newexp)
# ldat=ldat[:,sidx]
ldat=newexp.data
if zeroisnone:
ldat[ldat==0]=None
if uselog:
hs.Debug(1,"Using log, cutoff at %f" % lowcutoff)
ldat[np.where(ldat<lowcutoff)]=lowcutoff
ldat=np.log2(ldat)
oldparams=plt.rcParams
mpl.rc('keymap',back='c, backspace')
mpl.rc('keymap',forward='v')
mpl.rc('keymap',all_axes='A')
if newfig:
f=plt.figure(tight_layout=True)
else:
f=plt.gcf()
# set the colormap to default if not supplied
if not colormap:
colormap=plt.rcParams['image.cmap']
# plot the image
if colorrange:
hs.Debug(1,"colormap range is 0,10")
iax=plt.imshow(ldat,interpolation='nearest',aspect='auto',clim=colorrange,cmap=plt.get_cmap(colormap))
elif rangeall:
hs.Debug(1,"colormap range is all")
iax=plt.imshow(ldat,interpolation='nearest',aspect='auto',cmap=plt.get_cmap(colormap))
else:
hs.Debug(1,"colormap range is 0,10")
iax=plt.imshow(ldat,interpolation='nearest',aspect='auto',clim=[0,10],cmap=plt.get_cmap(colormap))
if ptitle is not None:
if not ptitle:
hs.Debug(1,"Showing filters in title")
if (len(newexp.filters))>4:
cfilters=[newexp.filters[0],'...',newexp.filters[-2],newexp.filters[-1]]
else:
cfilters=newexp.filters
cfilters=hs.clipstrings(cfilters,30)
ptitle='\n'.join(cfilters)
plt.title(ptitle,fontsize=10)
ax=iax.get_axes()
ax.autoscale(False)
# plot the sublines (smaller category lines)
if subline:
slval=hs.getfieldvals(newexp,subline)
prevval=slval[0]
for idx,cval in enumerate(slval):
if cval!=prevval:
xpos=idx-0.5
plt.plot([xpos,xpos],[-0.5,np.size(ldat,0)-0.5],'w:')
prevval=cval
if showline:
hs.Debug(1,"Showing lines")
labs=[]
labpos=[]
linepos=[]
minpos=0
svals.append('end')
for idx,cval in enumerate(svals[:-1]):
if cval==svals[idx+1]:
continue
labpos.append(minpos-0.5+float(idx+1-minpos)/2)
minpos=idx+1
linepos.append(idx+0.5)
labs.append(cval)
hs.Debug(1,"number of lines is %d" % len(linepos))
if showxlabel:
ax.set_xticks(labpos)
ax.set_xticklabels(labs,rotation=xlabelrotation,ha='right')
for cx in linepos:
plt.plot([cx,cx],[-0.5,np.size(ldat,0)-0.5],'k',linewidth=linewidth)
plt.plot([cx,cx],[-0.5,np.size(ldat,0)-0.5],'w:',linewidth=linewidth)
else:
hs.Debug(1,"Not showing lines")
if showxall or len(newexp.samples)<=10:
hs.Debug(1,"less than 10 samples, showing all sample names")
ax.set_xticklabels(svals,rotation=90)
ax.set_xticks(range(len(newexp.samples)))
# f.tight_layout()
ax.set_ylim(-0.5,np.size(ldat,0)-0.5)
if fixfont:
fontProperties = {'family':'monospace'}
ax.set_yticklabels(ax.get_yticks(), fontProperties)
if showcolorbar:
hs.Debug(1,"Showing colorbar")
cb=plt.colorbar(ticks=list(np.log2([2,10,100,500,1000])))
cb.ax.set_yticklabels(['<0.02%','0.1%','1%','5%','>10%'])
# create the plot
ax.expdat=newexp
ax.lastselect=-1
ax.sampline=''
ax.ofig=f
ax.labelson=False
ax.labelnames=[]
f.canvas.mpl_connect('button_press_event', onplotmouseclick)
f.canvas.mpl_connect('key_press_event', onplotkeyclick)
# show()
plt.rcParams=oldparams
# if want the ontology analysis for a given category:
if ontofig:
hs.Debug(1,"Ontofig is set")
newexp.ontofigname=ontofig
else:
newexp.ontofigname=False
# if we want gui, open it
if usegui:
hs.Debug(1,"Using the GUI window")
import heatsequer.plots.plotwingui
# from PyQt4 import QtGui
# app = QtGui.QApplication(sys.argv)
guiwin = heatsequer.plots.plotwingui.PlotGUIWindow(newexp)
from heatsequer.plots import plotwingui
guiwin = plotwingui.PlotGUIWindow(newexp)
ax.guiwin=guiwin
guiwin.plotfig=f
guiwin.plotax=ax
guiwin.show()
else:
ax.guiwin=False
hs.Debug(7,'Not using gui')
ax.plot_labelsize=fontsize
if newexp.plotmetadata:
hs.Debug(1,"Experiment has metadata attached for plotting (%d points)" % len(newexp.plotmetadata))
for cmet in newexp.plotmetadata:
addplotmetadata(newexp,field=cmet[0],value=cmet[1],color=cmet[2],inverse=cmet[3],beforesample=cmet[4])
if showhline:
if newexp.hlines:
for cpos in newexp.hlines:
plt.plot([0,np.shape(newexp.data)[1]],[cpos-0.5,cpos-0.5],'g')
plt.show()
if showtaxnames:
showtaxonomies(newexp,ax,showdb=False,showcontam=False)
# if usegui:
# app.exec_()
return newexp,ax
def plotexp(exp,sortby=False,numeric=False,minreads=4,rangeall=False,seqdb=None,cdb=None,showline=True,ontofig=False,usegui=True,showxall=False,showcolorbar=False,ptitle=False,lowcutoff=1,uselog=True,showxlabel=True,colormap=False,colorrange=False):
"""
Plot an experiment
input:
exp - from load()
sortby - name of mapping file field to sort by or Flase to not sort
numeric - True if the field is numeric
minreads - minimum number of reads per bacteria in order to show it or 0 to show all
rangeall - True to show all frequencies in image scale, false to saturate at 10%
seqdb - the SRBactDB database (from bactdb.load)
cdb - the cool sequences database (from cooldb.load)
showline - if True plot lines between category values
ontofig - name of ontology to plot for bactdb or false to no plot
usegui - True use a gui for otu summary, False just print
showxall - True to show all sample names when not sorting, False to show no more than 10
showcolorbar - True to plot the colorbar. False to not plot
ptitle - name of the figure or False to show processing history as name
lowcutoff - minimal value for read (for 0 log transform) - the minimal resolution - could be 10000*2/origreads
showxlabel : bool
True to show the x label (default), False to hide it
colormap : string or False
name of colormap or False (default) to use mpl default colormap
colorrange : [min,max] or False
[min,max] to set the colormap range, False to use data min,max (default) as specified in rangeall
output:
newexp - the plotted experiment (sorted and filtered)
ax - the plot axis
"""
hs.Debug(1,"Plot experiment %s" % exp.studyname)
hs.Debug(1,"Commands:")
for ccommand in exp.commands:
hs.Debug(1,"%s" % ccommand)
vals=[]
if sortby:
hs.Debug(1,"Sorting by field %s" % sortby)
for csamp in exp.samples:
vals.append(exp.smap[csamp][sortby])
if numeric:
hs.Debug(1,"(numeric sort)")
vals=hs.tofloat(vals)
svals,sidx=hs.isort(vals)
newexp=hs.reordersamples(exp,sidx)
else:
hs.Debug(1,"No sample sorting")
svals=hs.getfieldvals(exp,'#SampleID')
newexp=hs.copyexp(exp)
hs.Debug(1,"Filtering min reads. original bacteria - %d" % len(newexp.seqs))
if minreads>0:
newexp=hs.filterminreads(newexp,minreads,logit=uselog)
hs.Debug(1,"New number of bacteria %d" % len(newexp.seqs))
newexp.seqdb=seqdb
newexp.cdb=cdb
# ldat=ldat[:,sidx]
ldat=newexp.data
if uselog:
hs.Debug(1,"Using log, cutoff at %f" % lowcutoff)
ldat[np.where(ldat<lowcutoff)]=lowcutoff
ldat=np.log2(ldat)
oldparams=plt.rcParams
mpl.rc('keymap',back='c, backspace')
mpl.rc('keymap',forward='v')
mpl.rc('keymap',all_axes='A')
f=figure()
# set the colormap to default if not supplied
if not colormap:
colormap=plt.rcParams['image.cmap']
# plot the image
if colorrange:
hs.Debug(1,"colormap range is 0,10")
iax=imshow(ldat,interpolation='nearest',aspect='auto',clim=colorrange,cmap=plt.get_cmap(colormap))
elif rangeall:
hs.Debug(1,"colormap range is all")
iax=imshow(ldat,interpolation='nearest',aspect='auto',cmap=plt.get_cmap(colormap))
else:
hs.Debug(1,"colormap range is 0,10")
iax=imshow(ldat,interpolation='nearest',aspect='auto',clim=[0,10],cmap=plt.get_cmap(colormap))
if not ptitle:
hs.Debug(1,"Showing filters in title")
if (len(newexp.filters))>4:
cfilters=[newexp.filters[0],'...',newexp.filters[-2],newexp.filters[-1]]
else:
cfilters=newexp.filters
cfilters=hs.clipstrings(cfilters,30)
ptitle='\n'.join(cfilters)
title(ptitle,fontsize=10)
ax=iax.get_axes()
ax.autoscale(False)
if showline:
hs.Debug(1,"Showing lines")
labs=[]
labpos=[]
linepos=[]
minpos=0
svals.append('end')
for idx,cval in enumerate(svals[:-1]):
if cval==svals[idx+1]:
continue
labpos.append(minpos-0.5+float(idx+1-minpos)/2)
minpos=idx+1
linepos.append(idx+0.5)
labs.append(cval)
hs.Debug(1,"number of lines is %d" % len(linepos))
if showxlabel:
ax.set_xticks(labpos)
ax.set_xticklabels(labs,rotation=45,ha='right')
for cx in linepos:
plot([cx,cx],[-0.5,np.size(ldat,0)-0.5],'k',linewidth=2)
else:
hs.Debug(1,"Not showing lines")
if showxall or len(newexp.samples)<=10:
hs.Debug(1,"less than 10 samples, showing all sample names")
ax.set_xticklabels(svals,rotation=90)
ax.set_xticks(range(len(newexp.samples)))
tight_layout()
ax.set_ylim(-0.5,np.size(ldat,0)+0.5)
if showcolorbar:
hs.Debug(1,"Showing colorbar")
cb=colorbar(ticks=list(np.log2([2,10,100,500,1000])))
cb.ax.set_yticklabels(['<0.02%','0.1%','1%','5%','>10%'])
# create the plot
ax.expdat=newexp
ax.lastselect=-1
ax.sampline=''
ax.ofig=f
ax.labelson=False
ax.labelnames=[]
f.canvas.mpl_connect('button_press_event', onplotmouseclick)
f.canvas.mpl_connect('key_press_event', onplotkeyclick)
# show()
plt.rcParams=oldparams
# if want the ontology analysis for a given category:
if ontofig:
hs.Debug(1,"Ontofig is set")
newexp.ontofigname=ontofig
else:
newexp.ontofigname=False
# if we want gui, open it
if usegui:
hs.Debug(1,"Using the GUI window")
import heatsequer.plots.plotwingui
guiwin = heatsequer.plots.plotwingui.PlotGUIWindow(newexp)
# from heatsequer.plots import plotwingui
# guiwin = plotwingui.PlotGUIWindow(newexp)
ax.guiwin=guiwin
guiwin.plotfig=f
guiwin.plotax=ax
guiwin.show()
else:
ax.guiwin=False
hs.Debug(7,'Not using gui')
if newexp.plotmetadata:
hs.Debug(1,"Experiment has metadata attached for plotting (%d points)" % len(newexp.plotmetadata))
for cmet in newexp.plotmetadata:
addplotmetadata(newexp,field=cmet[0],value=cmet[1],color=cmet[2],inverse=cmet[3],beforesample=cmet[4])
show()
return newexp,ax