def getPrFileName(self, pop, stat, cut, supp=None):
if stat == "iHS":
fName = "iHS-%1.2f.top" % (cut / 100.0)
inds = self.study.getStatIndivs("iHS", pop)
md5 = MEGA.getHash(inds)
elif stat == "xpEHH":
fName = "xpEHH-%s-%1.2f.top" % (supp, cut / 100.0)
inds = self.study.getStatIndivs("xpEHH", pop)
md5 = MEGA.getHash(inds)
return md5, fName
def doxpEHH(study, force, pop, supp):
logging.info("Starting xpEHH %s %s", pop, supp)
shutil.rmtree("xpEHH", True)
os.mkdir("xpEHH")
os.chdir("xpEHH")
setIndivs = study.getStatIndivs("xpEHH", pop)
suppSetIndivs = study.getStatIndivs("xpEHH", supp)
logging.info("numIndivs: %d %d", len(setIndivs), len(suppSetIndivs))
hash = MEGA.getHash(setIndivs)
hashSupp = MEGA.getHash(suppSetIndivs)
myDir = MEGA.cacheDB + "/sets/" + karyo.karyotype + "/" + hash
logging.info("xpEHH: %s hash: %s phase: %s", pop, hash,
study.getPhasePop("xpEHH", pop))
if not force:
if os.path.isfile(myDir + "/%s-22.uxpEHH" % (supp,)):
logging.info("Already done and not forced")
os.chdir("..")
return
name = "study/%s/xpEHH/%s" % (study.name, pop)
sql.addId(name, hash)
suppName = "study/%s/xpEHH/%s-support" % (study.name, supp)
sql.addId(suppName, hashSupp)
stats.doBasicInfoSet(setIndivs, force)
stats.doBasicInfoSet(suppSetIndivs, force)
w = open(tmpDir + "/xpEHH/inds", "w")
for famId, sampId in setIndivs:
w.write("%s\t%s\n" % (famId, sampId))
w.close()
w = open(tmpDir + "/xpEHH/suppinds", "w")
for famId, sampId in suppSetIndivs:
w.write("%s\t%s\n" % (famId, sampId))
w.close()
os.system("python3 %s/doHaploStats.py prepareData %s %s suppinds %s %s xpEHH" %
(MEGA.haploScripts, karyo.karyotype, study.name,
study.xpEHHConf["source"],
study.getPhasePop("xpEHH", supp)))
for k in range(1, 23 + 1):
shutil.copyfile("%d.hap" % (k,),
"s%d.hap" % (k,))
os.system("python3 %s/doHaploStats.py prepareData %s %s inds %s %s xpEHH" % (
MEGA.haploScripts, karyo.karyotype, study.name,
study.xpEHHConf["source"], study.getPhasePop("xpEHH", pop),))
os.system("python3 %s/doHaploStats.py XPEHH %s" % (
MEGA.haploScripts, karyo.karyotype))
for k in range(1, 23 + 1):
shutil.copyfile("%d.xpEHH" % (k,),
myDir + "/%s-%d.uxpEHH" % (supp, k,))
os.chdir("..")
logging.info("xpEHH Done")
return hash
def getOvFileNames(self, name):
for stat, cut, pop in self.getOvComponents(name):
if stat == "iHS":
fName = "iHS-%1.2f.top" % (cut / 100.0)
inds = self.study.getStatIndivs("iHS", pop)
md5 = MEGA.getHash(inds)
yield md5, fName
def doChro(myHash, chro):
studyCase = getCase(open("%s/sets/%s/%d.uiHS" % (MEGA.cacheDB, myHash, chro) ), True)
refCases = {}
for ref in refPops:
myHash = MEGA.getHash(study.getStatIndivs("iHS", ref))
refCases[ref] = getCase(open("%s/sets/%s/%d.uiHS" % (MEGA.cacheDB, myHash,
chro) ))
for rs, pos, f0, i1, i2 in studyCase:
val = i1 if f0>0.5 else i2
comparisons = []
for refCase in list(refCases.values()):
try:
rf0, ri1, ri2 = refCase[rs]
except KeyError:
continue
if rf0>0.5:
comparisons.append(ri1)
else:
comparisons.append(ri2)
if len(comparisons)==0:
continue
comparisons.sort()
refVal = comparisons[len(comparisons)/2]
if val== 0 or refVal == 0:
continue
stat = math.log(val/refVal)
stat = val/refVal
print(chro, rs, pos, val, refVal, stat, f0)
def doPop(study, force, pop):
setIndivs = study.pops.getIndivs(pop)
hash = MEGA.getHash(setIndivs)
if addPopHash(study.cacheDir + "/pops", pop, hash) or force:
name = "study/%s/%s" % (study.name, pop)
sql.addId(name, hash)
stats.doBasicInfoSet(setIndivs, force)
def getDfFileNames(self, name):
mdNames = []
for stat, cut, pop in self.getDfComponents(name):
if stat == "iHS":
fName = "iHS-%1.2f.top" % (cut / 100.0)
inds = self.study.getStatIndivs("iHS", pop)
md5 = MEGA.getHash(inds)
mdNames.append((md5, fName))
return mdNames
def getPopURL(pop, karyo, hash=None, onlyPop=False, onlyKaryo=False):
if not hash:
hash = MEGA.getHash(MEGA.getIndivs(pop))
if onlyPop:
return '<a href="pop.py?md5=%s&k=%s">%s<a>' % (hash, karyo, pop)
elif onlyKaryo:
return '<a href="pop.py?md5=%s&k=%s">%s<a>' % (hash, karyo, karyo)
else:
return '<a href="pop.py?md5=%s&k=%s">%s (%s)<a>' % (hash, karyo, pop, karyo)
def doBasicInfoSet(indivs, force=True):
popHash = MEGA.getHash(indivs)
size = len(indivs)
popDir = os.sep.join([MEGA.cacheDB, "sets", karyo.karyotype, popHash])
try:
os.makedirs(popDir)
except OSError:
pass # Already exists, OK
if (not force) and path.exists(popDir + os.sep + "basic"):
return
w = open(popDir + os.sep + "basic", "w")
w.write(str(size) + "\n")
w.close()
w = open(popDir + os.sep + "indivs", "w")
for fam, ind in indivs:
w.write("\t".join([fam, ind]) + "\n")
w.close()
return popHash
def doIBD(studyName):
myStudy = study.Study(studyName)
w = open("ibd.problems", "w")
for pop in myStudy.pops.pops:
md5 = MEGA.getHash(myStudy.pops.getIndivs(pop))
try:
for rec in plink.parseGenome(open(MEGA.cacheDB + "/sets/" +
args.karyo + "/" +
md5 + "/genome")):
if rec["PI_HAT"] >= 0.125:
w.write("\t".join([rec["FID1"], rec["IID1"], rec["FID2"],
rec["IID2"], rec["RT"], rec["EZ"],
str(rec["Z0"]), str(rec["Z1"]),
str(rec["Z2"]), str(rec["PI_HAT"]),
str(rec["PHE"]), str(rec["DST"]),
str(rec["PPC"]), str(rec["RATIO"])]) +
"\n")
except IOError:
pass
w.close()
def doiHS(study, force, pop):
logging.info("Starting iHS")
shutil.rmtree("iHS", True)
os.mkdir("iHS")
os.chdir("iHS")
setIndivs = study.getStatIndivs("iHS", pop)
logging.info("numIndivs: %d", len(setIndivs))
hash = MEGA.getHash(setIndivs)
myDir = MEGA.cacheDB + "/sets/" + karyo.karyotype + "/" + hash
logging.info("iHS: %s hash: %s phase: %s",
pop, hash, study.getPhasePop("iHS", pop))
if not force:
if os.path.isfile(myDir + "/22.uiHS"):
logging.info("Already done and not forced")
os.chdir("..")
return
name = "study/%s/iHS/%s" % (study.name, pop)
sql.addId(name, hash)
stats.doBasicInfoSet(setIndivs, force)
w = open(tmpDir + "/iHS/inds", "w")
for famId, sampId in setIndivs:
w.write("%s\t%s\n" % (famId, sampId))
w.close()
os.system("python3 %s/doHaploStats.py prepareData %s %s inds %s %s" %
(MEGA.haploScripts, karyo.karyotype, study.name,
study.iHSConf["source"], study.getPhasePop("iHS", pop)))
os.system("python3 %s/doHaploStats.py iHS %s %s" % (MEGA.haploScripts,
karyo.karyotype, study.name))
for k in range(1, 23 + 1):
try:
shutil.copyfile("%d.uiHS" % k, myDir + "/%d.uiHS" % k)
except IOError:
pass
os.chdir("..")
logging.info("iHS Done")
return hash
def getIndivsPop(self, pop, cut, withPop=True, doDistant=False):
indivs = self.study.pops.getIndivs(pop)
popDir = MEGA.getHash(indivs)
distantIndivs = []
if doDistant:
f = open(MEGA.cacheDB + "/sets/" + karyo.karyotype + "/" +
popDir + "/distant." + str(cut))
for l in f:
famId, sampId = tuple(l.rstrip().split("\t"))
distantIndivs.append((famId, sampId))
f.close()
f = open(MEGA.cacheDB + "/sets/" + karyo.karyotype + "/" +
popDir + "/ibd.retain." + str(cut))
indivs = []
for l in f:
famId, sampId, pop = tuple(l.rstrip().split(" ")[:3])
if (famId, sampId) not in distantIndivs:
if withPop:
indivs.append((famId, sampId, pop))
else:
indivs.append((famId, sampId))
f.close()
return indivs
def getCase(f):
x = []
y = []
for l in f:
toks = [x for x in l.rstrip().split("\t") if x != ""]
rs = toks[0]
pos = int(toks[1])
uihs = float(toks[2])
x.append(pos)
y.append(uihs)
return x, y
study = MEGA.study.Study(studyName)
myHash = MEGA.getHash(study.getStatIndivs("iHS", studyPop))
def doChro(myHash, chro):
return getCase(open("%s/sets/%s/%d.iHS" % (MEGA.cacheDB, myHash, chro)))
x = []
y = []
for i in chros:
cx, cy = doChro(myHash, i)
print(len(cx))
x.extend(cx)
y.extend(cy)
print(1)
for path in paths:
sys.path.insert(0, os.path.expanduser(path))
import MEGA
from MEGA import web
web.sendPreamble(None, None, "Populations")
print("<h2>Populations</h2>")
pops = MEGA.pops
karyos = os.listdir(MEGA.cacheDB + "/sets/")
print('<table border="1">')
curr = 1
print("<tr>")
for pop in pops:
indivs = MEGA.getIndivs(pop)
hash = MEGA.getHash(indivs)
print('<td>')
print(pop)
start = False
for myKaryo in karyos:
if os.path.isdir('%s/sets/%s/%s' % (MEGA.cacheDB, myKaryo, hash)):
print("%s" % web.getPopURL(pop, myKaryo, hash, onlyKaryo=True))
print('</td>')
if curr % 6 == 0:
print("</tr><tr>")
curr += 1
print("</tr>")
print("</table>")
def doZoom(study, force, name):
logging.info("Starting Zoom - %s", name)
#XXX FORCE
try:
os.mkdir("zoom")
except OSError:
pass # Already exists
os.chdir("zoom")
chro, start, end, pop, ibd = study.zoom.general[name]
pops = pop.split("+")
indivs = []
for myPop in pops:
if ibd:
indivs.extend(study.ibd.getIndivsPop(myPop, ibd, False))
else:
indivs.extend(study.pops.getIndivs(myPop))
hash = MEGA.getHash(indivs)
posAlls = ensembl.getSNPs(chro)
poses = {}
snps = []
snpAtPos = {}
for rs, content in list(posAlls.items()):
pos = content[0]
if pos >= start and pos <= end:
snps.append(rs)
poses[rs] = pos
snpAtPos[pos] = rs
ancAlls = ensembl.getAncs(chro)
ancs = {}
for snp in snps:
ancs[snp] = ancAlls.get(snp, None)
gPoses = {}
f = open(MEGA.geneticMapDB + "/37-%d.map" % chro)
f.readline()
for l in f:
toks = l.rstrip().split("\t")
pos = int(toks[1])
if pos < start:
continue
if pos > end:
break
rs = snpAtPos.get(pos, "")
if rs == "":
continue
gPos = float(toks[3])
gPoses[rs] = gPos
f.close()
#assuming ihs phase.conf exists
source = study.iHSConf["source"]
refPop = study.getPhasePop("iHS", pop)
if refPop != "shapeIt":
phasedFile = "%s/%s/%s-%d.gz" % (MEGA.phaseDB, source, refPop, chro)
else:
phasedFile = "%s/%s/%d.gz" % (MEGA.phaseDB, source, chro)
inds = [x[1] for x in indivs]
project_beagle_phase(gzip.open("%d.gz" % chro, "w"),
gzip.open(phasedFile),
ind_retain=inds, snp_retain=snps,
want_phased=True, is_phased=True)
realSNPs = set()
f = gzip.open("%d.gz" % chro)
f.readline()
f.readline()
for l in f:
toks = l.rstrip().split("\t")
realSNPs.add(toks[1])
f.close()
stats = study.zoom.stats[name]
vals = {}
for stat, params in stats:
if stat == "iHS":
vals["iHS"] = {}
myHash = MEGA.getHash(study.getStatIndivs("iHS", pop))
f = open(MEGA.cacheDB + "/sets/" + karyo.karyotype + "/" +
myHash + "/" + str(chro) + ".iHS")
for l in f:
toks = l.rstrip().split("\t")
if toks[0] in realSNPs:
vals["iHS"][toks[0]] = float(toks[2])
elif stat == "xpEHH":
vals["xpEHH"] = {}
myHash = MEGA.getHash(study.getStatIndivs("xpEHH", pop))
f = open(MEGA.cacheDB + "/sets/" + karyo.karyotype + "/" +
myHash + "/" + params[0] + "-" + str(chro) + ".xpEHH")
for l in f:
toks = l.rstrip().split("\t")
if toks[0] in realSNPs:
vals["xpEHH"][toks[0]] = float(toks[2])
elif stat == "EHH":
rsId = params[0]
allele = params[1]
vals["EHH"] = calcEHH(chro, poses, rsId, allele)
statNames = list(vals.keys())
statNames.sort()
w = open("zoom.txt", "w")
f = gzip.open("%d.gz" % chro)
f.readline()
header = f.readline().rstrip().split("\t")[2:]
w.write("RS\tpos\tgPos\tanc\t")
for name in statNames:
w.write(name + "\t")
w.write("\t".join(header))
w.write("\n")
for l in f:
toks = l.rstrip().split("\t")
rs = toks[1]
haplos = toks[2:]
w.write("%s\t%d\t%s\t%s\t" % (rs, poses.get(rs, 0),
str(gPoses.get(rs, "")), ancs.get(rs, "")))
for name in statNames:
w.write(str(vals[name].get(rs, "")) + "\t")
w.write("\t".join(haplos))
w.write("\n")
w.close()
f.close()
shutil.copyfile("zoom.txt", MEGA.cacheDB + "/sets/" + karyo.karyotype +
"/" + hash + "/" +
"%s-%d-%d-%d.zoom" % (study.name, chro, start, end))
def getRefiHS(study, name):
md5 = MEGA.getHash(study.getStatIndivs("iHS", name))
return '<a href="%s/pop.py?md5=%s">iHS %s</a><br>' % (MEGA.webRoot,
md5, name)
def doIBDs(ibdlist):
logging.info("Starting IBD")
doSomething = False
w = open("lst", "w")
popsToDo = {}
for study, force, pop in ibdlist:
indivs = study.pops.getIndivs(pop)
#this is OK, but it is really study.ibd.pops
hash = MEGA.getHash(indivs)
if not force:
doThis = False
for cut in study.ibd.cuts:
if not os.path.isfile(MEGA.cacheDB + "/sets/" +
karyo.karyotype + "/" + hash +
"/ibd.retain." + str(cut)):
doThis = True
if not doThis:
continue
doSomething = True
popsToDo[pop] = hash
for famId, sampId in indivs:
w.write("%s\t%s\t%s\n" % (famId, sampId, pop))
w.close()
if doSomething:
logging.debug("Running doRelated split")
os.system("cat lst |python3 %s/doRelated split" % (MEGA.inbreedingScripts))
logging.debug("Running doRelated IBD")
os.system("cat lst |python3 %s/doRelated IBD" % (MEGA.inbreedingScripts))
logging.debug("Running doRelated statIBD")
os.system("cat lst |python3 %s/doRelated statIBD > ibdata/statIBD" %
(MEGA.inbreedingScripts))
for pop in popsToDo:
pop_ = pop.replace(" ", "_")
hash = popsToDo[pop]
logging.debug("copy %s %s",
"ibdata/%s.genome" % (pop_,),
MEGA.cacheDB + "/sets/" + karyo.karyotype + "/" +
hash + "/genome")
shutil.copyfile("ibdata/%s.genome" % (pop_,),
MEGA.cacheDB + "/sets/" + karyo.karyotype + "/" +
hash + "/genome")
for cut in study.ibd.cuts:
logging.debug("Running doRelated removeCloserIBD %f", cut)
os.system("cat lst |python3 %s/doRelated removeCloserIBD %f" %
(MEGA.inbreedingScripts, cut))
ibds = {}
sIBD = open("ibdata/statIBD")
for l in sIBD:
toks = l.strip().rstrip().split("\t")
ibds[toks[0]] = "\t".join([x.strip().rstrip() for x in toks[1:]])
sIBD.close()
for pop in popsToDo:
if pop not in ibds:
continue
hash = popsToDo[pop]
myDir = MEGA.cacheDB + "/sets/" + karyo.karyotype + "/" + hash
w = open(myDir + "/IBDstat." + str(cut), "w")
w.write(ibds[pop] + "\n")
w.close()
pop_ = pop.replace(" ", "_")
logging.debug("copy %s %s",
"ibdata/%s.distant" % (pop_,),
MEGA.cacheDB + "/sets/" + karyo.karyotype +
"/" + hash + "/distant." + str(cut))
shutil.copyfile("ibdata/%s.distant" % (pop_,),
MEGA.cacheDB + "/sets/" + karyo.karyotype +
"/" + hash + "/distant." + str(cut))
shutil.copyfile("ibdata/%s_ibd.txt" % (pop_,),
MEGA.cacheDB + "/sets/" + karyo.karyotype +
"/" + hash + "/ibd.retain." + str(cut))
logging.info("IBD Done")
def getRefPop(indivs, karyo, popName=None):
md5 = MEGA.getHash(indivs)
if not popName:
popName = MEGA.getNameFromMD5(md5)
return '<a href="%s/pop.py?md5=%s&k=%s">%s</a><br>' % (MEGA.webRoot,
md5, karyo, popName)
def copy(self):
for pop in self.pops:
indivs = self.study.pops.getIndivs(pop)
hash = MEGA.getHash(indivs)
shutil.copyfile('hwe-' + pop, '%s/sets/%s/%s/hwe.bz2' %
(MEGA.cacheDB, karyo.karyotype, hash))
print("<h1>%s</h1>" % (studyName,))
print(study.desc)
print("<br>")
print(study.summary)
print("<h2>Map</h2>")
web.addMap(study.pops.pops)
print("<h2>Populations</h2>")
cuts = study.ibd.cuts
print('<table border="1">')
print('<tr><td>Population</td><td>#</td>')
for cut in cuts:
print('<td>', cut, '</td>')
print('</tr>')
for pop in study.pops.pops:
md5 = MEGA.getHash(study.pops.getIndivs(pop))
print("<tr>")
print("<td>", web.getPopURL(pop, karyo.karyotype, md5), "</td>")
print("<td>", len(study.pops.getIndivs(pop)), "</td>")
for cut in cuts:
try:
print('<td>', len(study.ibd.getIndivsPop(pop, cut)), '</td>')
except IOError:
print('<td></td>')
print("</tr>")
print("</td>")
print("</tr></table>")
print("<br>")
print(web.linkFile(studyName, "ibd.problems", "IBD issues (>12.5)", True))
print('<h2>PCA</h2>')
def doChro(myHash, chro, refs):
studyCase = getCase(open("%s/sets/%s/%d.uiHS" % (MEGA.cacheDB, myHash, chro) ))
for f0, i1, i2, uihs in studyCase:
if f0<0.5:
refs.setdefault(f0,[]).append(uihs)
else:
refs.setdefault(f0,[]).append(uihs)
return refs
for ref in refPops:
vals = {}
for i in range(maxChro):
k = i+1
myHash = MEGA.getHash(study.getStatIndivs("iHS", ref))
doChro(myHash, k, vals)
fs = list(vals.keys())
fs.sort()
x = []
ymax = []
y20 = []
ymed = []
y80 = []
yvar = []
ycnt = []
for f in fs:
x.append(f)
yvar.append(numpy.std(vals[f]))
y20.append(numpy.percentile(vals[f],20))
def getHash(popStudy):
return MEGA.getHash(study.pops.getIndivs(popStudy))
