def doSaddle(filename, eig, gen):
c = cooler.Cooler(filename)
gen = Genome("/home/magus/HiC2011/data/" + gen, readChrms=["#", "X"])
gen.setResolution(getResolution(filename))
saddles = []
for chrom in range(gen.chrmCount):
saddle = np.zeros((5,5), dtype = float)
st = gen.chrmStartsBinCont[chrom]
end = gen.chrmEndsBinCont[chrom]
cur = c.matrix(balance=False).fetch(gen.idx2label[chrom])
cur = observedOverExpected(cur)
mask = np.sum(cur , axis=0) > 0
cur = cur [mask]
cur = cur [:, mask]
GC = eig[st:end]
GC = GC[mask]
if len(GC) > 5:
for i in range(5):
for j in range(5):
G1, G2 = np.percentile(GC, [20 * i, 20 * i + 20])
mask1 = (GC > G1) * (GC < G2)
G1, G2 = np.percentile(GC, [20 * j, 20 * j + 20])
mask2 = (GC > G1) * (GC < G2)
saddle[i, j] += cur[np.ix_(mask1, mask2)].mean()
saddles.append(saddle)
return saddles
def coverage(s1, s2, TR):
genome = Genome()
genome.createMapping(resolution)
label = genome.chromosomeStarts[TR.ufragments / TR.fragIDmult - 1] + (
TR.ufragments % TR.fragIDmult) / resolution
counts = np.bincount(label, weights=s1)
counts2 = np.bincount(label, weights=s2)
data = cPickle.load(open("GC1M", 'rb'))
eigenvector = np.zeros(genome.chromosomeEnds[-1], float)
inds = np.argsort(counts)
mask = inds[int(0.02 * len(inds)):]
for chrom in range(1, 24):
eigenvector[genome.chromosomeStarts[chrom - 1]:genome.
chromosomeStarts[chrom - 1] +
len(data[chrom - 1])] = data[chrom - 1]
eigenvector[eigenvector < 35] = 35
plt.scatter(counts[mask],
counts2[mask],
c=eigenvector[mask],
s=6,
linewidth=0)
print stats.spearmanr(counts[mask], counts2[mask])
plt.xlabel("Coverage from all reads")
plt.xticks([0, 5000, 10000, 15000])
plt.ylabel("Coverage from RBs")
b = plt.colorbar()
b.ax.set_xlabel("GC content")
def __init__(self, genome, resolution, storageFile="inMemory", mode="w"):
"""
Initializes the high-resolution Hi-C data storage.
Parameters
----------
genome : folder or Genome object
matching Genome object or folder to load it form
resolution : int
Resolution (number of bp per bin)
storageFile : str (optional)
File to store the h5dict.
File will be created.
By default stores in memory
mode : "w", "w-" "r+" or "a", optional
Access mode to h5dict (see h5dict manual)
"""
inMemory = (storageFile == "inMemory")
self._h5dict = h5dict(storageFile, mode=mode, in_memory=inMemory)
if type(genome) == str:
genome = Genome(genome, readChrms=["#", "X"])
assert isinstance(genome, Genome)
self.genome = genome
self.resolution = resolution
self.genome.setResolution(resolution)
if self.genome.numBins < 7000:
print "Total number of bins in the genome is just %d" % self.genome.numBins
warnings.warn(
"For low-resolution analysis use binnedData, as it provides"
"more analysis tools")
M = self.genome.chrmCount
self.cisKeys = [(i, i) for i in xrange(M)]
self.transKeys = [(i, j) for i in range(M) for j in range(M) if j > i]
self.allKeys = self.cisKeys + self.transKeys
self.data = {}
self._initChromosomes()
def doSaddleError(filename, eig, gen, correct=False):
gen = Genome("/home/magus/HiC2011/data/" + gen, readChrms=["#", "X"])
cur = 0
data = h5dict(filename,'r')["heatmap"]
if correct:
data = completeIC(data)
gen.setResolution(getResolution(filename))
if eig == "GC":
eig = np.concatenate(gen.GCBin)
saddles = []
permutted = []
saddle = np.zeros((5,5), dtype = float)
for i in range(100):
permutted.append(np.zeros((5,5), dtype = float))
for chrom in range(gen.chrmCount):
st = gen.chrmStartsBinCont[chrom]
end = gen.chrmEndsBinCont[chrom]
cur = data[st:end, st:end]
cur = observedOverExpected(cur)
mask = np.sum(cur , axis=0) > 0
cur = cur [mask]
cur = cur [:, mask]
GC = eig[st:end]
GC = GC[mask]
if len(GC) > 5:
for i in range(5):
for j in range(5):
G1, G2 = np.percentile(GC, [20 * i, 20 * i + 20])
mask1 = (GC > G1) * (GC < G2)
G1, G2 = np.percentile(GC, [20 * j, 20 * j + 20])
mask2 = (GC > G1) * (GC < G2)
addition = cur[np.ix_(mask1, mask2)]
addition = np.reshape(addition, (-1))
for k in range(100):
resampled = np.random.choice(addition, len(addition), replace=True)
permutted[k][i,j] += resampled.mean()
saddle[i, j] += addition.mean()
return saddle, permutted
def doEigenvector(filename, genome):
if filename == "GC":
gen = Genome("/home/magus/HiC2011/data/" + genome, readChrms=["#","X"])
gen.setResolution(1000000)
GC = np.concatenate(gen.GCBin)
return GC
resolution = getResolution(filename)
BD = binnedData.binnedData(resolution, "/home/magus/HiC2011/data/" + genome, ["#","X"])
BD.simpleLoad(filename, "bla")
BD.removeDiagonal()
BD.removeBySequencedCount(0.5)
BD.removeCis()
BD.truncTrans(high=0.0005)
BD.removePoorRegions(cutoff=1)
BD.fakeCis()
BD.removeZeros()
BD.doEig(numPCs=2)
BD.restoreZeros(value=0)
return BD.EigDict["bla"][0]
def byChrEig(filename,
genome,
chromosomes="all",
resolution="auto",
byArm=True,
doSmooth=False):
from mirnylib.genome import Genome
if resolution == "auto":
resolution = getResolution(filename)
if type(genome) == str:
genome = Genome(genome)
assert isinstance(genome, Genome)
genome.setResolution(resolution)
mydict = mirnylib.h5dict.h5dict(filename)
if chromosomes == "all":
chromosomes = list(range(genome.chrmCount))
chromosomes = [i for i in chromosomes if "{0} {0}".format(i) in mydict]
if len(chromosomes) == 0:
raise ValueError("No chromosomes left. Check h5dict file.")
result = []
for chrom in chromosomes:
data = mydict["{0} {0}".format(chrom)]
if not byArm:
result.append(
completeEig(data, genome.GCBin[chrom], doSmooth=doSmooth))
else:
GC = genome.GCBin[chrom]
result.append(np.zeros(len(GC), dtype=float))
cent = genome.cntrMids[chrom] / resolution
result[-1][:cent] = completeEig(data[:cent, :cent],
genome.GCBin[chrom][:cent],
doSmooth=doSmooth)
result[-1][cent:] = completeEig(data[cent:, cent:],
genome.GCBin[chrom][cent:],
doSmooth=doSmooth)
return result
def refine_paper(filename, create=True):
"""filename[0] is a list of filenames of incoming files
filename[1] is a folder for outgoing file"""
if create == True:
for onename in filename[0]:
#Parsing individual files
if not os.path.exists(onename):
raise StandardError("path not found: %s" % onename)
TR = HiCdataset("bla",
genome=genomeFolder,
enzymeName="HindIII",
maximumMoleculeLength=500,
inMemory=True)
print "\nTesting loading new data without rsite information "
TR.parseInputData(dictLike=onename, enzymeToFillRsites="HindIII")
#assert len(TR.DS) == 856143
#assert len(TR.ufragments) == 634572
TR.save(onename + "_parsed.frag")
#Merging files alltogether, applying filters
TR = HiCdataset(filename[1] + "_merged.frag",
enzymeName="HindIII",
genome=genomeFolder,
mode="w")
TR.merge([i + "_parsed.frag" for i in filename[0]])
TR = HiCdataset("refined",
genome=genomeFolder,
enzymeName="HindIII",
mode="w",
inMemory=True)
print "\nTesting chunking during all tests"
TR.chunksize = 30000
#because we do many operations, we disable autoFlush here
TR.load(filename[1] + "_merged.frag")
print "\nTesting Rsite filter"
TR.filterRsiteStart(offset=5)
#assert len(TR.DS) == 832110
print "\nTesting duplicate filter"
TR.filterDuplicates(chunkSize=30000)
#assert len(TR.DS) == 830275
print "\nTesting small/large and extreme fragment filter"
TR.filterLarge()
#assert len(TR.DS) == 825442
TR.filterExtreme(cutH=0.005, cutL=0)
TR.writeFilteringStats()
#assert len(TR.DS) == 803845
#-------------------------------------------
TR.printMetadata(saveTo="metadata")
import cPickle
stop = False
mdata = cPickle.load(open("sampleMetadata"))
for i in sorted(mdata.keys()):
if TR.metadata[i] != mdata[i]:
print "Key {0} is not consistent: should be {1}, is {2}".format(
i, mdata[i], TR.metadata[i])
stop = True
if stop == True:
print("""------------_ERROR_--------------
Inconsistent metadata: see above
----------------------------------------""")
raise ValueError("Inconsistent Metadata")
print "Testing allxall and by-chromosome heatmap counting diagonal twice"
print "----> saving allxall heatmap"
TR.saveHeatmap(filename[1] + "-1M.hm", 1000000, countDiagonalReads="twice")
a = h5dict(filename[1] + "-1M.hm")
st, end = TR.genome.chrmStartsBinCont[1], TR.genome.chrmEndsBinCont[1]
st2, end2 = TR.genome.chrmStartsBinCont[2], TR.genome.chrmEndsBinCont[2]
chrom1 = a["heatmap"][st:end, st:end]
chrom12 = a["heatmap"][st:end, st2:end2]
setExceptionHook()
print "----> saving by chromosome heatmap"
TR.saveByChromosomeHeatmap(filename[1] + "-1M.hm",
resolution=1000000,
includeTrans=True,
countDiagonalReads="twice")
b = h5dict(filename[1] + "-1M.hm")["1 1"]
bb = h5dict(filename[1] + "-1M.hm")["1 2"]
assert (b - chrom1).sum() == 0
print "Cis heatmap consistent"
assert (bb - chrom12).sum() == 0
print 'Trans heatmap consistent'
print a["heatmap"][::10, ::10].sum()
#assert a["heatmap"][::10, ::10].sum() == 21800
print "Heatmap sum correct\n"
#---------------------------------
print "Testing allxall and by-chromosome heatmap counting diagonal once"
TR.saveHeatmap(filename[1] + "-1M.hm", 1000000, countDiagonalReads="once")
Ta = h5dict(filename[1] + "-1M.hm")
st, end = TR.genome.chrmStartsBinCont[1], TR.genome.chrmEndsBinCont[1]
st2, end2 = TR.genome.chrmStartsBinCont[2], TR.genome.chrmEndsBinCont[2]
chrom1 = Ta["heatmap"][st:end, st:end]
chrom12 = Ta["heatmap"][st:end, st2:end2]
setExceptionHook()
print "----> saving by chromosome heatmap"
TR.saveByChromosomeHeatmap(filename[1] + "-1M-byChr.hm",
resolution=1000000,
includeTrans=True,
countDiagonalReads="once")
TR.saveHiResHeatmapWithOverlaps(filename[1] + "-1M-highRes.hm",
resolution=50000,
countDiagonalReads="twice")
TR.saveSuperHighResMapWithOverlaps(filename[1] + "-5k-SuperHighRes.hm",
resolution=5000,
chromosomes=[14],
countDiagonalReads="twice")
Tb = h5dict(filename[1] + "-1M-byChr.hm")["1 1"]
Tbb = h5dict(filename[1] + "-1M-byChr.hm")["1 2"]
assert ((Tb - chrom1) == 0).all()
assert ((Tbb - chrom12) == 0).all()
assert ((Tb + np.diag(np.diag(Tb))) == b).all()
print "Diagonal counting methods are consistent\n"
newchrom1 = chrom1.copy()
for i in xrange(len(newchrom1)):
newchrom1[i, i] = 2 * newchrom1[i, i]
Tb = h5dict(filename[1] + "-1M-highRes.hm")["1 1"]
assert np.abs(Tb.sum() - newchrom1.sum()) < 1
assert np.sum(np.abs(coarsegrain(Tb, 20, True) - newchrom1)) < 500
#------------------------------
print "Testing updateGenome method"
from mirnylib.genome import Genome
removeChromIDs = np.array([0, 1, 1, 1, 1] + [0] * 17 + [1] + [0])
#print ((removeChromIDs[TR.chrms1] == 1) + (removeChromIDs[TR.chrms2] == 1) ).sum()
t = ((removeChromIDs[TR.chrms1] == 1) *
(removeChromIDs[TR.chrms2] == 1)).sum() + (
(removeChromIDs[TR.chrms1] == 1) * (TR.chrms2 == -1)).sum()
newGenome = Genome(genomePath=genomeFolder,
readChrms=["2", "3", "4", "5", "X"])
TR.updateGenome(newGenome)
assert TR.N == t
a = h5dict(filename[1] + "-1M.hm")["heatmap"]
def getGenome():
return Genome("../data/caul", chrmFileTemplate="%s.fa", readChrms=[])
from mirnylib.h5dict import h5dict
import matplotlib.pyplot as plt
import os
import sys
import numpy as np
from mirnylib.systemutils import setExceptionHook
from mirnylib.numutils import coarsegrain
from mirnylib.genome import Genome
setExceptionHook()
workingGenome = "hg19"
genomeFolder = sys.argv[1]
if not os.path.exists(genomeFolder):
raise Exception("Please provide hg19 Genome folder in the code or as a first argument")
mygenome = Genome(genomeFolder, readChrms = ["#","X"])
def source(ID):
return os.path.join("%s-%s.hdf5" % (ID, workingGenome)) # determines path to the parsed file by ID
N = 2000000
chrms1 = np.random.randint(0,22,N)
chrms2 = chrms1.copy()
mask = np.random.random(N) < 0.5
chrms2[mask] = np.random.randint(0,22,mask.sum())
pos1 = np.array(np.array((0.1 + 0.8 * np.random.random(N)) * mygenome.chrmLens[chrms1]), dtype=int)
offset1 = np.exp(3 + np.random.random(N) * (np.log(1000000) - 3)) * (2 * (np.random.random(N)>0.5) - 1 )
pos2 = np.array(pos1 + offset1, dtype=int)
strands1 = np.random.random(N) > 0.5
def parseInputData(self, dictLike, commandArgs, **kwargs):
"""
Added Parameters
----------------
commandArgs : NameSpace
A NameSpace object defined by argparse.
"""
## Necessary Modules
import numexpr
if not os.path.exists(dictLike):
raise IOError('File not found: %s' % dictLike)
dictLike = h5dict(dictLike, 'r')
self.chrms1 = dictLike['chrms1']
self.chrms2 = dictLike['chrms2']
self.cuts1 = dictLike['cuts1']
self.cuts2 = dictLike['cuts2']
self.strands1 = dictLike['strands1']
self.strands2 = dictLike['strands2']
self.dists1 = np.abs(dictLike['rsites1'] - self.cuts1)
self.dists2 = np.abs(dictLike['rsites2'] - self.cuts2)
self.mids1 = (dictLike['uprsites1'] + dictLike['downrsites1']) / 2
self.mids2 = (dictLike['uprsites2'] + dictLike['downrsites2']) / 2
self.fraglens1 = np.abs(
(dictLike['uprsites1'] - dictLike['downrsites1']))
self.fraglens2 = np.abs(
(dictLike['uprsites2'] - dictLike['downrsites2']))
self.fragids1 = self.mids1 + np.array(self.chrms1,
dtype='int64') * self.fragIDmult
self.fragids2 = self.mids2 + np.array(self.chrms2,
dtype='int64') * self.fragIDmult
distances = np.abs(self.mids1 - self.mids2)
distances[self.chrms1 != self.chrms2] = -1
self.distances = distances # Distances between restriction fragments
del distances
# Total Reads
self.N = len(self.chrms1)
self.metadata["100_TotalReads"] = self.N
try:
dictLike['misc']['genome']['idx2label']
self.updateGenome(
self.genome,
oldGenome=dictLike["misc"]["genome"]["idx2label"],
putMetadata=True)
except KeyError:
assumedGenome = Genome(self.genome.genomePath)
self.updateGenome(self.genome,
oldGenome=assumedGenome,
putMetadata=True)
DSmask = (self.chrms1 >= 0) * (self.chrms2 >= 0)
self.metadata["200_totalDSReads"] = DSmask.sum()
self.metadata["201_DS+SS"] = len(DSmask)
self.metadata["202_SSReadsRemoved"] = len(DSmask) - DSmask.sum()
mask = DSmask
## Information based on restriction fragments
sameFragMask = self.evaluate("a = (fragids1 == fragids2)",
["fragids1", "fragids2"]) * DSmask
cutDifs = self.cuts2[sameFragMask] > self.cuts1[sameFragMask]
s1 = self.strands1[sameFragMask]
s2 = self.strands2[sameFragMask]
SSDE = (s1 != s2)
SS = SSDE * (cutDifs == s2)
Dangling = SSDE & (~SS)
SS_N = SS.sum()
SSDE_N = SSDE.sum()
sameFrag_N = sameFragMask.sum()
dist = self.evaluate(
"a = - cuts1 * (2 * strands1 -1) - "
"cuts2 * (2 * strands2 - 1)",
["cuts1", "cuts2", "strands1", "strands2"])
Dangling_L = dist[sameFragMask][Dangling]
library_L = int(np.ceil((np.percentile(Dangling_L, 95))))
self.maximumMoleculeLength = library_L
readsMolecules = self.evaluate(
"a = numexpr.evaluate('(chrms1 == chrms2) & (strands1 != strands2) & (dist >=0) &"
" (dist <= maximumMoleculeLength)')",
internalVariables=["chrms1", "chrms2", "strands1", "strands2"],
externalVariables={"dist": dist},
constants={
"maximumMoleculeLength": self.maximumMoleculeLength,
"numexpr": numexpr
})
if commandArgs.sameFragments:
mask *= (-sameFragMask)
noSameFrag = mask.sum()
self.metadata["210_sameFragmentReadsRemoved"] = sameFrag_N
self.metadata["212_Self-Circles"] = SS_N
self.metadata["214_DandlingEnds"] = SSDE_N - SS_N
self.metadata["216_error"] = sameFrag_N - SSDE_N
mask *= (readsMolecules == False)
extraDE = mask.sum()
self.metadata[
"220_extraDandlingEndsRemoved"] = -extraDE + noSameFrag
if commandArgs.RandomBreaks:
ini_N = extraDE
mask *= ((self.dists1 + self.dists2) <= library_L)
rb_N = ini_N - mask.sum()
self.metadata["330_removeRandomBreaks"] = rb_N
if mask.sum() == 0:
raise Exception(
'No reads left after filtering. Please, check the input data')
del DSmask, sameFragMask
del dist, readsMolecules
self.metadata["300_ValidPairs"] = self.N
self.maskFilter(mask)
from hiclib.hicShared import byChrEig
from mirnylib.genome import Genome
import matplotlib.pyplot as plt
from mirnylib.systemutils import setExceptionHook
from mirnylib.plotting import nicePlot, maximumContrastList
setExceptionHook()
gen = Genome('../../../../hg19', readChrms=["#", "X"])
mychroms = [0, 2, 5, 13, 20]
eigs = byChrEig("../fragmentHiC/test-1M-byChr.hm", gen, chromosomes=mychroms)
for j, chrom in enumerate(mychroms):
plt.scatter(eigs[j],
gen.GCBin[chrom],
color=maximumContrastList[j],
label="Chr {0}".format(chrom + 1))
plt.xlabel("eigenvector")
plt.ylabel("GC content")
nicePlot()
genomeFolder = "../../../data/hg18"
readChrms = [
"#", # read all numbered chromosomes
"X"
] # add X chromosome
for inDataset in inDatasets.values():
if not os.path.exists(inDataset):
raise IOError("Raw heatmap file does not exist: {}".format(inDataset))
if not os.path.isdir(genomeFolder):
raise IOError("Genome folder does not exist")
# When you do this, be sure that readChrms used to save heatmap matches
# readChrms that you define here!
genome = Genome(genomeFolder, readChrms=readChrms)
# Read resolution from one of the datasets
sampleDataset = h5dict(inDatasets.values()[0], mode="r") # random dataset
resolution = int(sampleDataset["resolution"])
# Define the binnedData object, load data
BD = binnedData(resolution, genome, readChrms)
for name, filename in inDatasets.items():
BD.simpleLoad(filename, name)
BD.removeDiagonal()
# Remove bins with less than half of a bin sequenced
BD.removeBySequencedCount(0.5)
def parseInputData(self, dictLike, **kwargs):
import numexpr
if not os.path.exists(dictLike):
raise IOError('File not found: %s' % dictLike)
dictLike = h5dict(dictLike, 'r')
self.chrms1 = dictLike['chrms1']
self.chrms2 = dictLike['chrms2']
self.cuts1 = dictLike['cuts1']
self.cuts2 = dictLike['cuts2']
self.strands1 = dictLike['strands1']
self.strands2 = dictLike['strands2']
self.dists1 = np.abs(dictLike['rsites1'] - self.cuts1)
self.dists2 = np.abs(dictLike['rsites2'] - self.cuts2)
self.mids1 = (dictLike['uprsites1'] + dictLike['downrsites1']) / 2
self.mids2 = (dictLike['uprsites2'] + dictLike['downrsites2']) / 2
self.fraglens1 = np.abs(
(dictLike['uprsites1'] - dictLike['downrsites1']))
self.fraglens2 = np.abs(
(dictLike['uprsites2'] - dictLike['downrsites2']))
self.fragids1 = self.mids1 + np.array(self.chrms1,
dtype='int64') * self.fragIDmult
self.fragids2 = self.mids2 + np.array(self.chrms2,
dtype='int64') * self.fragIDmult
distances = np.abs(self.mids1 - self.mids2)
distances[self.chrms1 != self.chrms2] = -1
self.distances = distances # Distances between restriction fragments
del distances
self.N = len(self.chrms1)
try:
dictLike['misc']['genome']['idx2label']
self.updateGenome(self.genome,
oldGenome = dictLike["misc"]["genome"]["idx2label"])
except KeyError:
assumedGenome = Genome(self.genome.genomePath)
self.updateGenome(self.genome, oldGenome = assumedGenome)
# Discard dangling ends and self-circles
DSmask = (self.chrms1 >= 0) * (self.chrms2 >= 0)
self.metadata['100_NormalPairs'] = DSmask.sum()
sameFragMask = self.evaluate("a = (fragids1 == fragids2)",
["fragids1", "fragids2"]) * DSmask
cutDifs = self.cuts2[sameFragMask] > self.cuts1[sameFragMask]
s1 = self.strands1[sameFragMask]
s2 = self.strands2[sameFragMask]
SSDE = (s1 != s2)
SS = SSDE * (cutDifs == s2)
SS_N = SS.sum()
SSDE_N = SSDE.sum()
sameFrag_N = sameFragMask.sum()
self.metadata['120_SameFragmentReads'] = sameFrag_N
self.metadata['122_SelfLigationReads'] = SS_N
self.metadata['124_DanglingReads'] = SSDE_N - SS_N
self.metadata['126_UnknownMechanism'] = sameFrag_N - SSDE_N
mask = DSmask * (-sameFragMask)
del DSmask, sameFragMask
noSameFrag = mask.sum()
# distance between sites facing each other
dist = self.evaluate("a = numexpr.evaluate('- cuts1 * (2 * strands1 -1) - "
"cuts2 * (2 * strands2 - 1)')",
["cuts1", "cuts2", "strands1", "strands2"],
constants={"numexpr":numexpr})
readsMolecules = self.evaluate(
"a = numexpr.evaluate('(chrms1 == chrms2) & (strands1 != strands2) & (dist >=0) &"
" (dist <= maximumMoleculeLength)')",
internalVariables=["chrms1", "chrms2", "strands1", "strands2"],
externalVariables={"dist": dist},
constants={"maximumMoleculeLength": self.maximumMoleculeLength, "numexpr": numexpr})
mask *= (readsMolecules == False)
extraDE = mask.sum()
self.metadata['210_ExtraDanglingReads'] = -extraDE + noSameFrag
if mask.sum() == 0:
raise Exception('No reads left after filtering. Please, check the input data')
del dist, readsMolecules
self.maskFilter(mask)
c2 = hd["chrms2"]
p1 = hd["cuts1"]
p2 = hd["cuts2"]
mask = c1 == c2
cis = mask.sum()
more20kb = (np.abs(p1[mask] - p2[mask]) > 20000).sum()
return more20kb / cis
dfs = []
for genome in ["mm9", 'hg19']:
genomeObject = Genome("/home/magus/HiC2011/data/{0}".format(genome), readChrms = ["#", "X"])
filenames = [i.replace(".1000000.cool", "") for i in os.listdir(genome) if
".1000000.cool" in i and ("DpnII" in i or "combined" in i or "sperm" in i.lower() or "ad" in i.lower() )]
sampleDict = pd.read_csv("samples.csv", index_col=0)
#nsnDict = sampleDict["Stage"]
### Bug fix to read files only contained within sampleDict ###
# RUN THIS THE FIRST TIME only
#fnames = [];
#for f in list(sampleDict.index):
# thisFile = [file for file in filenames if str(f) in file]
# if len(thisFile)>0:
# fnames.append(thisFile[0])
#filenames = fnames
overhead, and therefore having 23*11 chromosome pairs would slow it down a bit.
Note the trick of converting a global heatmap map to a smaller map.
It tests only iterative correction now.
"""
from hiclib.highResBinnedData import HiResHiC
from mirnylib.genome import Genome
from hiclib.binnedData import binnedData
from mirnylib.h5dict import h5dict
import numpy as np
import sys
import os
genome = Genome(sys.argv[1], readChrms=["1", "2", "3", "4", "5"])
a = HiResHiC(genome, 1000000, "hiResDict", mode='w')
a.loadData(dictLike="../fragmentHiC/test-1M-byChr.hm")
a.removeDiagonal()
a.removePoorRegions(2)
a.iterativeCorrection(1e-10)
b = binnedData(1000000, genome)
data = {"heatmap": h5dict("../fragmentHiC/test-1M.hm")["heatmap"]}
lim = b.genome.chrmEndsBinCont[-1]
data["heatmap"] = data["heatmap"][:lim, :lim]
b.simpleLoad(data, "data")
b.removeDiagonal()
from mirnylib.genome import Genome
import pandas as pd
from hiclib import hicShared
from mirnylib.h5dict import h5dict
from mirnylib.numutils import coarsegrain, observedOverExpected, completeIC
import joblib
import pickle
import cooler
from mirnylib.numutils import zoomArray
from hiclib.hicShared import getResolution
from hiclib import binnedData
from mirnylib.systemutils import setExceptionHook
# defining genomes used here
mygenM = Genome("../../data/mm9", readChrms=["#", "X"])
mygenH = Genome("../../data/hg19", readChrms=["#", "X"])
genomDict = {"mm9":mygenM, "hg19":mygenH}
mem = joblib.Memory(".")
def getFrags():
"""
A method that calculates a set of restriction fragments covered in all hg19, or in all mm9 datasets
Used for the correct calculation of scalings
"""
mouse = "/home/magus/HiC2011/DrosophilaSingleCell2015/alternativeFiltering/mm9/oocyte_combined_refined.frag"
human = "/home/magus/HiC2011/DrosophilaSingleCell2015/alternativeFiltering/hg19/K562_combined_refined.frag"
and might be in fact pretty fast.
General recommendation: if you have 16+GB of RAM, and .sra (.fastq.gz) files were less than 30GB, then you should be fine with parsing things in memory.
"""
from hiclib.fragmentHiC import HiCdataset
from mirnylib.systemutils import fmap,setExceptionHook
from mirnylib.genome import Genome
import numpy as np
import os
import sys
#from defineGenome import getGenome
genomeName = "galGal4_1MB"
genome_db = Genome("/mnt/storage/home/vsfishman/HiC/fasta/galGal4/"+genomeName,readChrms=[],chrmFileTemplate="chr%s.fa",gapFile="galGal4.gap.txt")
def getGenome(workingGenome):
if workingGenome != genomeName: raise
return genome_db
setExceptionHook()
def ensure(f):
d = os.path.dirname(f)
if os.path.isdir(d):
return f
else:
try:
os.makedirs(d)
except:
raise ValueError("Cannot create directory")
return f
try:
import numpy as np
except:
sys.path = ["/usr/lib64/python2.7/site-packages"] + sys.path
import numpy as np
print "Numpy inported!"
from hiclib.fragmentHiC import HiCdataset
from mirnylib.systemutils import fmap, setExceptionHook
from mirnylib.genome import Genome
import os
#from defineGenome import getGenome
genomeName = "mm10"
genome_db = Genome("/mnt/storage/home/vsfishman/HiC/fasta/" + genomeName,
readChrms=["#", "X", "Y"])
data_folder = "/mnt/storage/home/vsfishman/HiC/data/mESC/mapped-mm10/mm10/"
fdataset_fname = "mESC-all-HindIII_refined.frag"
setExceptionHook()
print "Loading HiCdataset"
TR = HiCdataset(data_folder + fdataset_fname,
enzymeName="HindIII",
mode='r',
genome=genome_db)
#print "Saving heatmap"
#TR.saveHeatmap(data_folder + fdataset_fname + "1000k.hm", 1000000)
#print "Saving heatmap"
#TR.saveHeatmap(data_folder + fdataset_fname + "40k.hm", 40000)
print "Saving heatmap"
def get_chrom_arms(c, gen_name):
genome = Genome('/net/levsha/share/lab/genomes/'+gen_name)
converter.create_agp_dict()
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import os
sys.path.append("/mnt/storage/home/vsfishman/HiC/tutorial_Fishman/mESC")
from getIntraChrHeatmaps import get_chromosomes,extractResolutionFromFileName
sys.path.append("/mnt/storage/home/vsfishman/HiC/tutorial_Fishman/mESC")
genome_db_contig = Genome("/mnt/storage/home/vsfishman/HiC/fasta/GalGal5/GCF_000002315.4_Gallus_gallus-5.0_assembly_structure/Primary_Assembly/galGal5_all_contigs.filtered/",
readChrms=[],
chrmFileTemplate="N%s.fa")
genome_db_chrmLevel = Genome("/mnt/storage/home/vsfishman/HiC/fasta/GalGal5/GCF_000002315.4_Gallus_gallus-5.0_assembly_structure/Primary_Assembly/assembled_chromosomes/FASTA/",
readChrms=[],
chrmFileTemplate="%s.fna")
genome_db_contig = genome_db_chrmLevel
#hm_file = "/mnt/storage/home/vsfishman/HiC/tutorial_Fishman/chick/mapped-GalGal5filtered/GalGal5filtered/ChEF-all-HindIII-100k.hm.IC"
#second_hm_file = "/mnt/storage/home/vsfishman/HiC/tutorial_Fishman/chick/mapped-GalGal5filtered/GalGal5filtered/Blood-all-HindIII-100k.hm.IC"
########################WRITE YOUR HEATMAP HERE########################
hm_file = "/mnt/storage/home/vsfishman/HiC/tutorial_Fishman/chick/mapped-GalGal5filtered/GalGal5filteredChrmLevel/ChEF-all-HindIII-40k.hm.IC"
domains_files_Arm = "mapped-GalGal5filtered/GalGal5filteredChrmLevel/ChEF-all-HindIII-40k.hm.gzipped_matrix/ChEF-all-HindIII-40k.hm.gzipped_matrix.jucebox_domains.annotation"
domains_files_Dix = "/mnt/storage/home/vsfishman/HiC/data/chick/DixonDomainsChEF_all_HindIII_40k.hm.IC_domains_40KB/DixonDomainsChEF_all_HindIII_40k.hm.IC_domains_40KB.jucebox_domains.annotation"
second_hm_file = "/mnt/storage/home/vsfishman/HiC/tutorial_Fishman/chick/mapped-GalGal5filtered/GalGal5filteredChrmLevel/Blood-all-HindIII-40k.hm.IC"
help=
'domains file to use for annotation,coud be several files separated with ","'
)
parser.add_argument(
"--colors",
default="",
help=
'colors for domains,should be same number of colors as number of domains')
parser.add_argument("--out",
default="heatmap_plots",
help='output dir for plots')
args = parser.parse_args()
if args.level == "contig":
genome_db = Genome(
"/mnt/storage/home/vsfishman/HiC/fasta/GalGal5/GCF_000002315.4_Gallus_gallus-5.0_assembly_structure/Primary_Assembly/galGal5_all_contigs.filtered/",
readChrms=[],
chrmFileTemplate="N%s.fa")
elif args.level == "chr":
#genome_db = Genome("/mnt/storage/home/vsfishman/HiC/fasta/GalGal5/GCF_000002315.4_Gallus_gallus-5.0_assembly_structure/Primary_Assembly/assembled_chromosomes/FASTA/",
# readChrms=[],
# chrmFileTemplate="%s.fna")
genome_db = Genome('/mnt/storage/home/vsfishman/HiC/fasta/mm9',
readChrms=['#', 'X'])
hm_file = args.hmap
figure_path = args.out + "/" + hm_file.split("/")[-1]
if args.domains != "":
domains = args.domains.split(",")
colors = args.colors.split(",")
else:
domains = []