def getTrackView(self, region):
#print 'get tv for reg: ',region
#print str(type(self._origRegion)) + " and " + str(type(region))
assert (not isIter(self._origRegion) and self._origRegion == region) or (isIter(self._origRegion) and region in self._origRegion)
#if self._cachedTV is None:
rawData = RawDataStat(region, self._origTrack, self._trackFormatReq)
origTV = rawData.getResult()
self._checkTrackFormat(origTV)
assert(not origTV.allowOverlaps)
assert(origTV.borderHandling == 'crop')
assert region == origTV.genomeAnchor
starts, ends, vals, strands, ids, edges, weights, extras = \
self._createRandomizedNumpyArrays(len(origTV.genomeAnchor), origTV.startsAsNumpyArray(), \
origTV.endsAsNumpyArray(), origTV.valsAsNumpyArray(), \
origTV.strandsAsNumpyArray(), origTV.idsAsNumpyArray(), \
origTV.edgesAsNumpyArray(), origTV.weightsAsNumpyArray(), \
origTV.allExtrasAsDictOfNumpyArrays(), origTV.trackFormat, region)
from gold.util.CommonFunctions import getClassName
self._cachedTV = TrackView(origTV.genomeAnchor, \
(starts + origTV.genomeAnchor.start if starts is not None else None), \
(ends + origTV.genomeAnchor.start if ends is not None else None), \
vals, strands, ids, edges, weights, origTV.borderHandling, origTV.allowOverlaps, extraLists=extras)
assert self._trackFormatReq.isCompatibleWith(self._cachedTV.trackFormat), 'Incompatible track-format: '\
+ str(self._trackFormatReq) + ' VS ' + str(self._cachedTV.trackFormat)
return self._cachedTV
def getPresCollectionType(self):
#print 'self.getGlobalResult(): ',self.getGlobalResult()
if self.getStatClassName() == 'DataComparisonStat': #a bit ad hoc criterion. Also should check plotType...
presCollectionType = 'scatter'
elif 'BinScaled' in self.getStatClassName(): #a bit ad hoc criterion. Also should check plotType...
presCollectionType = 'binscaled'
elif self.getGlobalResult() not in [None,{}] and len(self.getGlobalResult())>0 and \
isinstance(self.getGlobalResult().values()[0], LinePlotResultType):
assert len(self.getGlobalResult().values()) == 1
presCollectionType = 'lineplot'
elif (self.getGlobalResult() not in [None,{}] and isinstance(self.getGlobalResult().values()[0], dict)):
if self.getGlobalResult().values()[0].get('Matrix') is not None:
presCollectionType = 'matrix'
else:
presCollectionType = 'dictofdicts'
elif self.getGlobalResult() not in [None,{}] and len(self.getGlobalResult())>0 and \
isinstance(self.getGlobalResult().values()[0], GlobalVisualizationResultType):
assert len(self.getGlobalResult().values()) == 1, 'Should currently be one if this is visualization result'
presCollectionType = 'visualization'
elif self.getGlobalResult() not in [None,{}] and len(self.getGlobalResult())>0 and \
isinstance(self.getGlobalResult().values()[0], RawVisualizationResultType):
#assert len(self.getGlobalResult().values()) == 1, 'Should currently be one if this is visualization result'
presCollectionType = 'rawDataVisualization' #elif self.getGlobalResult() not in [None,{}] and len(self.getGlobalResult())==1 and \
# hasattr(self.getGlobalResult().values()[0], '__iter__'):
elif self.getGlobalResult() not in [None,{}] and \
isIter(self.getGlobalResult().values()[0]):#and len(self.getGlobalResult().values()[0])>1:
#or type(self.getGlobalResult().values()[0])==numpy.ndarray):
#print 'TYPE: ',type(self.getGlobalResult().values()[0])
presCollectionType = 'distribution'
else:
presCollectionType = 'standard'
#print 'presCollectionType: ',presCollectionType
#print isinstance(self.getGlobalResult(), GlobalVisualizationResultType), type(self.getGlobalResult()), str(type(self.getGlobalResult().values()[0])).replace('<','')
return presCollectionType
def __init__(self, region, track, track2, tail='more', globalSource='chrs', minimal=False, **kwArgs):
if isIter(region):
raise SplittableStatNotAvailableError()
self._globalSource = GenericRelativeToGlobalStatUnsplittable.getGlobalSource(globalSource, region.genome, minimal)
#if minimal == True:
# self._globalSource = MinimalBinSource(region.genome)
#elif globalSource == 'test':
# self._globalSource = UserBinSource('TestGenome:chr21:10000000-15000000','1000000')
#elif globalSource == 'chrs':
# self._globalSource = GenomeInfo.getChrRegs(region.genome)
#elif globalSource == 'chrarms':
# self._globalSource = GenomeInfo.getChrArmRegs(region.genome)
#elif globalSource == 'ensembl':
# self._globalSource = GenomeInfo.getStdGeneRegs(region.genome)
#elif globalSource == 'userbins':
# from gold.application.StatRunner import StatJob
# assert StatJob.USER_BIN_SOURCE is not None
# self._globalSource = StatJob.USER_BIN_SOURCE
# #self._globalSource = kwArgs['userBins']
#else:
# raise ShouldNotOccurError('globalSource not recognized')
# #self._globalSource = GlobalBinSource(region.genome)
assert tail in ['more', 'less', 'different']
self._tail = tail
super(self.__class__, self).__init__(region, track, track2, tail=tail, globalSource=globalSource, minimal=minimal, **kwArgs)
def _getSubCls(origCls, region):
#print "with class: ",origCls.__name__,'and region: ',region,"<br>"
if isIter(region) or CompBinManager.canBeSplitted(region):
try:
splittableClass = MagicStatFactory._getClass(origCls.__name__, 'Splittable')
#print "FOUND SPLITTABLE: ", <splittableClass
if isIter(region):
#Always use splittableClass if a global region
return splittableClass
else:
#Use only if splittableClass also accepts splitting of userbins
if not issubclass(splittableClass, OnlyGloballySplittable):
return splittableClass
#except (KeyError, SplittableStatNotAvailableError), e:
except KeyError, e:
if DebugConfig.VERBOSE:
logException(e, message="In MagicStatFactory._getSubCls: ")
def constructUniqueKey(cls, region, track, track2=None, *args, **kwArgs):
#If only one track is provided to a stat, other params may turn up as track2 above. Then set to None..
if track2!=None and not isinstance(track2, Track):
track2 = None
reg = id(region) if isIter(region) else region
#logMessage('%s, %s, %s, %s, %s' % (str(cls), Statistic._constructConfigKey(kwArgs), (str([str(x) for x in reg]) if hasattr(reg, '__iter__') else str(reg)), tuple(track.trackName), tuple(track2.trackName) if track2 != None else ''))
return (hash(str(cls)), Statistic._constructConfigKey(kwArgs), hash(reg), tuple(track.trackName), tuple(track2.trackName) if track2 != None else '')
def _createExtraLists(target, dtype, size):
if type(target) == bool:
if target == True:
return OrderedDict([('extra1', getRandExtraList(size)), ('extra2', getRandExtraList(size))])
else:
return OrderedDict()
elif isinstance(target, dict):
return OrderedDict([(name, array(content, dtype=dtype)) for name, content in target.items()])
elif isIter(target):
return OrderedDict([(name, getRandExtraList(size)) for name in target])
else:
raise ShouldNotOccurError
def __init__(self, region, track, track2, tail='more', globalSource='', minimal=False, **kwArgs):
if isIter(region):
raise SplittableStatNotAvailableError()
#if minimal == True:
# self._globalSource = MinimalBinSource(region.genome)
#elif globalSource == 'test':
# self._globalSource = UserBinSource('TestGenome:chr21:10000000-15000000','1000000')
#else:
# self._globalSource = GlobalBinSource(region.genome)
self._globalSource = GenericRelativeToGlobalStatUnsplittable.getGlobalSource(globalSource, region.genome, minimal)
assert tail in ['more', 'less', 'different']
self._tail = tail
super(self.__class__, self).__init__(region, track, track2, tail=tail, globalSource=globalSource, minimal=minimal, **kwArgs)
def _getNewContentsOfPrev(self, x, y, prefix):
if x is not None and y is not None:
if isIter(x):
if x != y:
if len(x) != len(y):
return []
else:
return [self._getMissingValue(prefix) for i,item in enumerate(x)]
else:
valDataType = self._geSource.getValDataType()
if prefix == 'val' and valDataType[0] == 'S' and valDataType != 'S1':
return self._getConcatContent(x, y, prefix, sorted=True)
elif prefix in ['id'] + self._prevEl.orderedExtraKeys:
return self._getConcatContent(x, y, prefix, sorted=False)
elif x != y:
return self._getMissingValue(prefix)
def _init(self, globalSource=None, weightThreshold=None, **kwArgs):
if isIter(self._region):
raise SplittableStatNotAvailableError()
assert globalSource is not None
assert weightThreshold is not None
self._weightThreshold = float(weightThreshold)
def __init__(self, segments=None, starts=True, ends=True, vals=True, strands=False, ids=False, edges=False, weights=False, \
extras=False, anchor=None, numElements=None, valDType='float64', borderHandling='crop', allowOverlaps=False):
if type(starts) != bool and ends == True:
ends = False
if type(ends) != bool and starts == True:
starts = False
assert not (starts==False and ends==False)
assert segments!=False and segments!=True
assert starts!=None and ends!=None and vals!=None and strands!=None
assert segments==None or (starts==True and ends==True)
assert not (isIter(weights) and not isIter(edges))
assert (any( type(x) not in [bool,type(None)] for x in [segments,starts,ends,vals,strands,ids,edges,weights,extras]) and numElements==None) \
or numElements!=None
#assert(( (type(segments)!=bool or type(starts)!=bool or type(ends)!=bool or \
# type(vals)!=bool or type(strands)!=bool) and numElements==None )\
# or numElements!=None)
#
if anchor==None:
anchor = [10,1000]
if segments != None:
starts = []
ends = []
for seg in segments:
starts.append(seg[0])
ends.append(seg[1])
if isIter(edges):
maxNumEdges = self._findMaxNumEls(edges)
edges = self._appendEmptyToEnd(edges, '', maxNumEdges)
if isIter(weights):
weights = self._appendEmptyToEnd(weights, numpy.nan, maxNumEdges)
[starts, ends, vals, strands, ids, edges, weights] + ([x for x in extras.values()] if isinstance(extras, dict) else [])
for list in [starts, ends, vals, strands, ids, edges, weights] + ([x for x in extras.values()] if isinstance(extras, dict) else []):
if type(list) != bool and numElements == None:
numElements = len(list)
assert(type(list) == bool or len(list) == numElements)
for coordList in [starts, ends]:
if type(coordList) != bool:
for j in range(len(coordList)):
coordList[j] += anchor[0]
randSegmentLists = getRandSegments(numElements, anchor[0], anchor[1])
starts = self._createList(starts, randSegmentLists[0], 'int32')
ends = self._createList(ends, randSegmentLists[1], 'int32')
vals = self._createList(vals, getRandValList(numElements, valDType), valDType)
strands = self._createList(strands, getRandStrandList(numElements), 'bool8')
randIds, randEdges, randWeights = getRandGraphLists(numElements)
ids = self._createList(ids, randIds, randIds.dtype)
edges = self._createList(edges, randEdges, randEdges.dtype)
weights = self._createList(weights, randWeights, 'float64')
if weights is not None and len(weights.shape) == 1:
weights = weights.reshape(weights.shape + (0,))
extras = self._createExtraLists(extras, 'S', numElements)
if starts == None:
if ends[0] != 0:
ends = numpy.append([anchor[0]], ends)
if vals != None:
vals = numpy.append([nan], vals)
if strands != None:
strands = numpy.append([True], strands)
if ends[-1] != anchor[1]:
ends[-1] = anchor[1]
# print (starts, ends, vals, strands, anchor)
TrackView.__init__(self, GenomeRegion('TestGenome', 'chr21', anchor[0], anchor[1]), starts, ends, vals, \
strands, ids, edges, weights, borderHandling, allowOverlaps, extraLists=extras)
def __init__(self,
segments=None,
starts=True,
ends=True,
vals=False,
strands=False,
ids=False,
edges=False,
weights=False,
extras=False,
anchor=None,
numElements=None,
valDType='float64',
borderHandling='crop',
allowOverlaps=False):
if type(starts) != bool and ends == True:
ends = False
if type(ends) != bool and starts == True:
starts = False
assert not (starts == False and ends == False)
assert segments != False and segments != True
assert starts is not None and ends is not None and vals is not None and strands is not None
assert segments is None or (starts == True and ends == True)
assert not (isIter(weights) and not isIter(edges))
assert (any( type(x) not in [bool,type(None)] for x in [segments,starts,ends,vals,strands,ids,edges,weights,extras]) and numElements==None) \
or numElements is not None
#assert(( (type(segments)!=bool or type(starts)!=bool or type(ends)!=bool or \
# type(vals)!=bool or type(strands)!=bool) and numElements==None )\
# or numElements!=None)
#
if anchor is None:
anchor = [10, 1000]
if segments is not None:
starts = []
ends = []
for seg in segments:
starts.append(seg[0])
ends.append(seg[1])
if isIter(edges):
maxNumEdges = self._findMaxNumEls(edges)
edges = self._appendEmptyToEnd(edges, '', maxNumEdges)
if isIter(weights):
weights = self._appendEmptyToEnd(weights, numpy.nan,
maxNumEdges)
[starts, ends, vals, strands, ids, edges, weights
] + ([x for x in extras.values()] if isinstance(extras, dict) else [])
for list in [starts, ends, vals, strands, ids, edges, weights] + (
[x for x in extras.values()] if isinstance(extras, dict) else []):
if type(list) != bool and numElements is None:
numElements = len(list)
assert (type(list) == bool or len(list) == numElements)
for coordList in [starts, ends]:
if type(coordList) != bool:
for j in range(len(coordList)):
coordList[j] += anchor[0]
randSegmentLists = getRandSegments(numElements, anchor[0], anchor[1])
starts = self._createList(starts, randSegmentLists[0], 'int32')
ends = self._createList(ends, randSegmentLists[1], 'int32')
vals = self._createList(vals, getRandValList(numElements, valDType),
valDType)
strands = self._createList(strands, getRandStrandList(numElements),
'bool8')
randIds, randEdges, randWeights = getRandGraphLists(numElements)
ids = self._createList(ids, randIds, randIds.dtype)
edges = self._createList(edges, randEdges, randEdges.dtype)
weights = self._createList(weights, randWeights, 'float64')
if weights is not None and len(weights.shape) == 1:
weights = weights.reshape(weights.shape + (0, ))
extras = self._createExtraLists(extras, 'S', numElements)
if starts is None:
if ends[0] != 0:
ends = numpy.append([anchor[0]], ends)
if vals is not None:
vals = numpy.append([nan], vals)
if strands is not None:
strands = numpy.append([True], strands)
if ends[-1] != anchor[1]:
ends[-1] = anchor[1]
# print (starts, ends, vals, strands, anchor)
TrackView.__init__(self, GenomeRegion('TestGenome', 'chr21', anchor[0], anchor[1]), starts, ends, vals, \
strands, ids, edges, weights, borderHandling, allowOverlaps, extraLists=extras)
def _init(self, rawStatistic, globalSource='', minimal=False, **kwArgs):
if isIter(self._region):
raise SplittableStatNotAvailableError()
self._rawStatistic = self.getRawStatisticClass(rawStatistic)
self._globalSource = GenericRelativeToGlobalStatUnsplittable.getGlobalSource(globalSource, self._region.genome, minimal)
def _splitRegion(self):
if isIter(self._region):
#return self._region.__iter__()
return self._region.__iter__()
else:
return CompBinManager.splitUserBin(self._region).__iter__()
def _splitRegion(self):
if isIter(self._region):
#return self._region.__iter__()
return self._region.__iter__()
else:
return CompBinManager.splitUserBin(self._region).__iter__()
def _init(self, globalSource=None, weightThreshold=None, **kwArgs):
if isIter(self._region):
raise SplittableStatNotAvailableError()
assert globalSource is not None
assert weightThreshold is not None
self._weightThreshold = float(weightThreshold)
def _compute(self):
if not isIter(self._region):
#return {'Result':{'Matrix': None}}
return None
childRes = self._children[0].getResult()
if childRes is None:
return None
childRes = childRes['Result']
matrix = numpy.array(copy(childRes.get('Matrix')), dtype='float64')
rows, cols, counts, pvals, significance = [numpy.array(copy(childRes.get(member))) if member in childRes else None \
for member in ['Rows', 'Cols', 'Counts', 'Pvals', 'Significance']]
if not self._complete:
matrix, rows, cols, counts ,pvals, significance = self._removeNanLines(matrix, rows, cols, counts, pvals, significance)
rowClust = self._getRowCluster(matrix)
colClust = self._getColCluster(matrix)
rowOrder, colOrder = self._getRowAndColOrders(rowClust, colClust)
rows = self._cleanUpNameArray(rows, rowOrder)
cols = self._cleanUpNameArray(cols, colOrder)
if all(x == 1 for x in [self._numClustersRows, self._numClustersCols]):
ret = {'Result': OrderedDict([('Matrix', matrix), ('Rows', rows), ('Cols', cols),\
('RowClust', rowClust), ('ColClust', colClust),\
('RowOrder', rowOrder), ('ColOrder', colOrder),\
('Counts', counts)])}
if pvals is not None:
ret['Result']['Pvals'] = pvals
if significance is not None:
ret['Result']['Significance'] = significance
return ret
else:
rowGroups = self._getGroups(rowClust, self._numClustersRows)
colGroups = self._getGroups(colClust, self._numClustersCols)
ret = {}
for rowNo in xrange(rowGroups.max() + 1):
for colNo in xrange(colGroups.max() + 1):
#print matrix, rows, cols, rowGroups, colGroups, rowNo, colNo
#print rowGroups == rowNo, colGroups ==colNo
newMatrix = copy(matrix[numpy.ix_(rowGroups==rowNo, colGroups==colNo)])
newCounts = copy(counts[numpy.ix_(rowGroups==rowNo, colGroups==colNo)])
if pvals is not None:
newPvals = copy(pvals[numpy.ix_(rowGroups==rowNo, colGroups==colNo)])
if significance is not None:
newSignificance = copy(significance[numpy.ix_(rowGroups==rowNo, colGroups==colNo)])
newRows = copy(rows[rowGroups==rowNo])
newCols = copy(cols[colGroups==colNo])
#if isIter(self._region):
rowClust = self._getRowCluster(matrix[rowGroups==rowNo])
colClust = self._getColCluster(matrix[:,colGroups==colNo])
rowOrder, colOrder = self._getRowAndColOrders(rowClust, colClust)
#else:
#rowClust, colClust = None, None
#print {'Matrix': newMatrix, 'Rows': newRows, 'Cols': newCols, 'RowClust': rowClust, 'ColClust': colClust}
label = 'Part_' + str(rowNo+1) + '_' + str(colNo+1)
ret[label] = {'Matrix': newMatrix, 'Rows': newRows, 'Cols': newCols, 'Counts': newCounts}
if pvals is not None:
ret[label]['Pvals'] = newPvals
if significance is not None:
ret[label]['Significance'] = newSignificance
if rowClust is not None:
ret[label]['RowClust'] = rowClust
ret[label]['RowOrder'] = rowOrder
if colClust is not None:
ret[label]['ColClust'] = colClust
ret[label]['ColOrder'] = colOrder
return ret
def __init__(self, region, track, **kwArgs):
if isIter(region):
raise SplittableStatNotAvailableError()
#super(self.__class__, self).__init__(region, track, trackFormatReq=trackFormatReq, **kwArgs)
super(self.__class__, self).__init__(region, track, **kwArgs)
def _compute(self):
if not isIter(self._region):
#return {'Result':{'Matrix': None}}
return None
childRes = self._children[0].getResult()
if childRes is None:
return None
childRes = childRes['Result']
matrix = numpy.array(copy(childRes.get('Matrix')), dtype='float64')
rows, cols, counts, pvals, significance = [numpy.array(copy(childRes.get(member))) if member in childRes else None \
for member in ['Rows', 'Cols', 'Counts', 'Pvals', 'Significance']]
if not self._complete:
matrix, rows, cols, counts, pvals, significance = \
self._removeNanLines(matrix, rows, cols, counts, pvals, significance)
rowClust = self._getRowCluster(matrix)
colClust = self._getColCluster(matrix)
rowOrder, colOrder = self._getRowAndColOrders(rowClust, colClust)
rows = self._cleanUpNameArray(rows, rowOrder)
cols = self._cleanUpNameArray(cols, colOrder)
if all(x == 1 for x in [self._numClustersRows, self._numClustersCols]):
ret = {'Result': OrderedDict([('Matrix', matrix), ('Rows', rows), ('Cols', cols),\
('RowClust', rowClust), ('ColClust', colClust),\
('RowOrder', rowOrder), ('ColOrder', colOrder),\
('Counts', counts)])}
if pvals is not None:
ret['Result']['Pvals'] = pvals
if significance is not None:
ret['Result']['Significance'] = significance
return ret
else:
rowGroups = self._getGroups(rowClust, self._numClustersRows)
colGroups = self._getGroups(colClust, self._numClustersCols)
ret = {}
for rowNo in xrange(rowGroups.max() + 1):
for colNo in xrange(colGroups.max() + 1):
#print matrix, rows, cols, rowGroups, colGroups, rowNo, colNo
#print rowGroups == rowNo, colGroups ==colNo
newMatrix = copy(matrix[numpy.ix_(rowGroups == rowNo,
colGroups == colNo)])
newCounts = copy(counts[numpy.ix_(rowGroups == rowNo,
colGroups == colNo)])
if pvals is not None:
newPvals = copy(pvals[numpy.ix_(
rowGroups == rowNo, colGroups == colNo)])
if significance is not None:
newSignificance = copy(significance[numpy.ix_(
rowGroups == rowNo, colGroups == colNo)])
newRows = copy(rows[rowGroups == rowNo])
newCols = copy(cols[colGroups == colNo])
# if isIter(self._region):
rowClust = self._getRowCluster(newMatrix)
colClust = self._getColCluster(newMatrix)
rowOrder, colOrder = self._getRowAndColOrders(
rowClust, colClust)
# print {'Matrix': newMatrix, 'Rows': newRows,
# 'Cols': newCols, 'RowClust': rowClust,
# 'ColClust': colClust}
label = 'Part_' + str(rowNo + 1) + '_' + str(colNo + 1)
ret[label] = {
'Matrix': newMatrix,
'Rows': newRows,
'Cols': newCols,
'Counts': newCounts
}
if pvals is not None:
ret[label]['Pvals'] = newPvals
if significance is not None:
ret[label]['Significance'] = newSignificance
if rowClust is not None:
ret[label]['RowClust'] = rowClust
ret[label]['RowOrder'] = rowOrder
if colClust is not None:
ret[label]['ColClust'] = colClust
ret[label]['ColOrder'] = colOrder
return ret
def __init__(self, region, track, **kwArgs):
if isIter(region):
raise SplittableStatNotAvailableError()
#super(self.__class__, self).__init__(region, track, trackFormatReq=trackFormatReq, **kwArgs)
super(self.__class__, self).__init__(region, track, **kwArgs)