def simImage(self, oImg, tImg):
os.system("rm -rf %s/*"%(self.tmpDir))
os.system("cp %s/%s %s/%s"%(self.srcDir, oImg, self.tmpDir, self.objectImg))
os.system("cp %s/%s %s/%s"%(self.srcDir, tImg, self.tmpDir, self.templateImg))
self.objectImgCat = self.runSextractor(self.objectImg)
self.templateImgCat = self.runSextractor(self.templateImg)
mchFile, nmhFile = self.runSelfMatch(self.objectImgCat, self.r16)
self.osn16 = nmhFile
mchFile, nmhFile = self.runSelfMatch(self.objectImgCat, self.r10)
osn10 = nmhFile
mchFile, nmhFile = self.runSelfMatch(self.templateImgCat, self.r10)
tsn10 = nmhFile
osn16s = selectTempOTs(self.osn16, self.tmpDir)
osn16sf = filtOTs(osn16s, self.tmpDir)
imgSimClass = ImageSimulation()
simFile, simPosFile = imgSimClass.simulateImage2(self.objectImg, osn16sf, self.objectImg)
simTmpResi = self.runHotpants(simFile, self.templateImg)
simTmpResiCat = self.runSextractor(simTmpResi)
mchFile, nmhFile = self.runSelfMatch(simTmpResiCat, self.r10)
str_sn10 = nmhFile
str_sn10f = filtOTs(str_sn10, self.tmpDir)
osn10f = filtOTs(osn10, self.tmpDir)
tsn10f = filtOTs(tsn10, self.tmpDir)
mchFile1, nmhFile1, mchPair1 = self.runCrossMatch(str_sn10f, simPosFile, self.r5)
str_sn10f_spf_cn5 = nmhFile1
mchFile2, nmhFile2, mchPair2 = self.runCrossMatch(osn10f, tsn10f, self.r5)
tIdx1 = np.loadtxt("%s/%s"%(self.tmpDir, mchPair1)).astype(np.int)
tIdx2 = np.loadtxt("%s/%s"%(self.tmpDir, mchPair2)).astype(np.int)
tIdx1 = tIdx1 - 1
tIdx2 = tIdx2 - 1
tdata11 = np.loadtxt("%s/%s"%(self.tmpDir, str_sn10f))
tdata12 = np.loadtxt("%s/%s"%(self.tmpDir, simPosFile))
tdata21 = np.loadtxt("%s/%s"%(self.tmpDir, osn10f))
tdata22 = np.loadtxt("%s/%s"%(self.tmpDir, tsn10f))
print(str_sn10f_spf_cn5)
print(mchPair1)
print(mchPair2)
print(str_sn10f)
print(simPosFile)
print(osn10f)
print(tsn10f)
def simImage(self, oImg):
outpre = oImg.split(".")[0]
os.system("rm -rf %s/*" % (self.tmpDir))
os.system("cp %s/%s %s/%s" %
(self.srcDir, oImg, self.tmpDir, self.objectImg))
self.removeHeader(self.objectImg)
self.objectImgCat = self.runSextractor(self.objectImg)
mchFile, nmhFile = self.runSelfMatch(self.objectImgCat, self.r16)
self.osn16 = nmhFile
osn16s = selectTempOTs(self.osn16, self.tmpDir)
tdata = np.loadtxt("%s/%s" % (self.tmpDir, osn16s))
if len(tdata.shape) < 2 or tdata.shape[0] < 100:
print("%s has too little stars, break this run" % (oImg))
return
osn16sf = filtOTs(osn16s, self.tmpDir)
tdata = np.loadtxt("%s/%s" % (self.tmpDir, osn16sf))
if len(tdata.shape) < 2 or tdata.shape[0] < 45:
print("%s has too little stars, break this run" % (oImg))
return
imgSimClass = ImageSimulation()
otImgs = imgSimClass.getTmpOtImgs(osn16sf, self.objectImg)
psfView = genPSFView(otImgs)
thumbnail = getThumbnail(self.srcDir,
oImg,
stampSize=(100, 100),
grid=(5, 5),
innerSpace=1)
thumbnail = scipy.ndimage.zoom(thumbnail, 4, order=0)
'''
plt.clf()
plt.figure(figsize=(20,20))
plt.imshow(psfView, interpolation = "nearest", cmap='gray')
plt.show()
plt.clf()
plt.figure(figsize=(20,20))
plt.imshow(thumbnail, interpolation = "nearest", cmap='gray')
plt.show()
'''
tpath11 = "%s_view" % (self.srcDir)
if not os.path.exists(tpath11):
os.system("mkdir %s" % (tpath11))
dpath1 = "%s/%s_psf.jpg" % (tpath11, outpre)
dpath2 = "%s/%s_thb.jpg" % (tpath11, outpre)
Image.fromarray(psfView).save(dpath1)
Image.fromarray(thumbnail).save(dpath2)
def simTOT(self, oImg, tImg, subImgNum=1000):
mchFile, nmhFile = self.runSelfMatch(self.objectImgCat, 24)
self.osn32 = nmhFile
osn16s = selectTempOTs(self.osn16, self.tmpDir)
tdata = np.loadtxt("%s/%s" % (self.tmpDir, osn16s))
if len(tdata.shape) < 2 or tdata.shape[0] < 100:
print("%s has too little stars, break this run" % (oImg))
return np.array([]), np.array([])
osn16sf = filtOTs(osn16s, self.tmpDir)
tdata = np.loadtxt("%s/%s" % (self.tmpDir, osn16sf))
if len(tdata.shape) < 2 or tdata.shape[0] < 45:
print("%s has too little stars, break this run" % (oImg))
return np.array([]), np.array([])
osn32f = filtOTs(self.osn32, self.tmpDir)
mchFile, nmhFile, mchPair = self.runCrossMatch(osn32f, self.tsn16, 5)
osn32s_tsn5_cm5 = mchFile
osn32s_tsn5_cm5_pair = mchPair
totalTOT = subImgNum
subImgBuffer = []
objS2NBuffer = []
tnum = 0
imgSimClass = ImageSimulation()
ii = 1
sexConf = [
'-DETECT_MINAREA', '3', '-DETECT_THRESH', '2.5',
'-ANALYSIS_THRESH', '2.5'
]
while tnum < totalTOT:
simFile, simPosFile, simDeltaXYA, psfImgs = imgSimClass.simulateImage1(
osn32f, self.objectImg, osn16sf, self.objectImg)
self.objectImgSim = simFile
self.objectImgSimAdd = simPosFile
self.runSextractor(self.objectImgSim)
self.simTmpResi = self.runHotpants(self.objectImgSim,
self.templateImg)
self.simTmpResiCat = self.runSextractor(self.simTmpResi, sexConf)
simTmpResiCatEf = filtByEllipticity(self.simTmpResiCat,
self.tmpDir,
maxEllip=0.5)
mchFile, nmhFile = self.runSelfMatch(simTmpResiCatEf, 16)
simTmpResiCatEf_sn32 = nmhFile
#simTmpResiCatEf_sn32 = simTmpResiCatEf
mchFile, nmhFile, mchPair = self.runCrossMatch(
self.objectImgSimAdd, simTmpResiCatEf_sn32, self.r5)
str_oisa_cm5 = mchFile
str_oisa_cm5_pair = mchPair
tIdx1 = np.loadtxt(
"%s/%s" % (self.tmpDir, osn32s_tsn5_cm5_pair)).astype(np.int)
tIdx2 = np.loadtxt("%s/%s" %
(self.tmpDir, str_oisa_cm5_pair)).astype(np.int)
tIdx1 = tIdx1 - 1
tIdx2 = tIdx2 - 1
self.log.debug(
"objectCat matched data %d, ResiCat matched data %d" %
(tIdx1.shape[0], tIdx2.shape[0]))
tnames1 = ['objId', 'tmpId']
tnames2 = ['objId', 'resiId']
#unionIdx = np.intersect1d(tIdx1[:,0], tIdx2[:,0]) #union1d
#self.log.debug("intersect objectCat and templateCat matched data: %d"%(unionIdx.shape[0]))
df1 = pd.DataFrame(data=tIdx1, columns=tnames1)
df2 = pd.DataFrame(data=tIdx2, columns=tnames2)
unionIdx = pd.merge(df1, df2, how='inner', on=['objId'])
self.log.debug(
"innerjoin objectCat and templateCat matched data %d" %
(unionIdx.shape[0]))
tdata1 = np.loadtxt("%s/%s" % (self.tmpDir, self.objectImgSimAdd))
tdata2 = np.loadtxt("%s/%s" % (self.tmpDir, self.tsn16))
tdata3 = np.loadtxt("%s/%s" % (self.tmpDir, simTmpResiCatEf_sn32))
simDeltaXYA = np.array(simDeltaXYA)
tdeltaXYA = simDeltaXYA[unionIdx["objId"].values]
tdata11 = tdata1[unionIdx["objId"].values]
tdata12 = tdata2[unionIdx["tmpId"].values]
tdata22 = tdata3[unionIdx["resiId"].values]
poslist = np.concatenate(([tdata11[:, 0]], [tdata11[:, 1]], [
tdata12[:, 3] + tdeltaXYA[:, 0]
], [tdata12[:, 4] + tdeltaXYA[:, 1]], [tdata22[:, 0]
], [tdata22[:, 1]]),
axis=0).transpose()
#print(poslist)
#genFinalOTDs9Reg('tot', self.tmpDir, poslist)
#size = self.subImgSize
size = 100
subImgs = self.getWindowImgs(self.objectImgSim, self.templateImg,
self.simTmpResi, poslist, size)
#subImgs = self.getWindowImgs(self.objectImgSimSubBkg, self.templateImgSubBkg, self.simTmpResi, poslist, size)
tnum = tnum + len(subImgs)
subImgBuffer.extend(subImgs)
objS2NBuffer.extend(tdata22[:, 3].tolist())
self.log.info("\n******************")
self.log.info("run %d, total sub image %d" % (ii, tnum))
if ii > 6:
break
ii = ii + 1
#break
subImgs = np.array(subImgBuffer)
objS2NBuffer = np.array(objS2NBuffer)
print(subImgs.shape)
print(objS2NBuffer.shape)
return subImgs, objS2NBuffer
def simTOT(self, oImg, tImg, subImgNum=1000):
mchFile, nmhFile = self.runSelfMatch(self.objectImgCatNew, 24)
self.osn32 = nmhFile
osn16s = selectTempOTs(self.osn16, self.tmpDir)
tdata = np.loadtxt("%s/%s" % (self.tmpDir, osn16s))
if len(tdata.shape) < 2 or tdata.shape[0] < 100:
print("%s has too little stars, break this run" % (oImg))
return np.array([]), np.array([])
osn16sf = filtOTs(osn16s, self.tmpDir)
tdata = np.loadtxt("%s/%s" % (self.tmpDir, osn16sf))
if len(tdata.shape) < 2 or tdata.shape[0] < 45:
print("%s has too little stars, break this run" % (oImg))
return np.array([]), np.array([])
osn32f = filtOTs(self.osn32, self.tmpDir)
totalTOT = subImgNum
subImgBuffer = []
objS2NBuffer = []
tnum = 0
imgSimClass = ImageSimulation()
ii = 1
sexConf = [
'-DETECT_MINAREA', '3', '-DETECT_THRESH', '2.5',
'-ANALYSIS_THRESH', '2.5'
]
while tnum < totalTOT:
simFile, simPosFile, simDeltaXYA, tmpOtImgs = imgSimClass.simulateImage1(
osn32f, self.newImageName, osn16sf, self.newImageName)
self.objectImgSim = simFile
self.objectImgSimAdd = simPosFile
self.runSextractor(self.objectImgSim)
self.simTmpResi = self.runHotpants(self.objectImgSim,
self.templateImg)
self.simTmpResiCat = self.runSextractor(self.simTmpResi, sexConf)
simTmpResiCatEf = filtByEllipticity(self.simTmpResiCat,
self.tmpDir,
maxEllip=0.5)
mchFile, nmhFile = self.runSelfMatch(simTmpResiCatEf, 16)
simTmpResiCatEf_sn32 = nmhFile
#simTmpResiCatEf_sn32 = simTmpResiCatEf
mchFile, nmhFile, mchPair = self.runCrossMatch(
simTmpResiCatEf_sn32, self.objectImgSimAdd, 1)
tdata1 = np.loadtxt("%s/%s" % (self.tmpDir, mchFile))
poslist = tdata1[:, 0:2]
size = 100
subImgs = self.getWindowImgs(self.objectImgSim, self.templateImg,
self.simTmpResi, poslist, size)
tnum = tnum + len(subImgs)
ts2n = tdata1[:, 3]
subImgBuffer.extend(subImgs)
objS2NBuffer.extend(ts2n.tolist())
self.log.info("\n******************")
self.log.info("run %d, total sub image %d" % (ii, tnum))
if ii > 6:
break
ii = ii + 1
#break
subImgs = np.array(subImgBuffer)
objS2NBuffer = np.array(objS2NBuffer)
print(subImgs.shape)
print(objS2NBuffer.shape)
return subImgs, objS2NBuffer
def imgSimulate(self, srcFitDir, destFitDir, destCatAddDir):
try:
tfiles0 = os.listdir(srcFitDir)
tfiles0.sort()
tfiles = []
for tfile in tfiles0:
if tfile.find('mon_objt_190116') > 0:
tfiles.append(tfile[:33])
tmpImgPath = "%s/%s" % (destFitDir, tfiles[0])
if not os.path.exists(tmpImgPath):
os.system("cp %s/%s %s" % (srcFitDir, tfiles[0], tmpImgPath))
os.system(
"cp %s/%s %s/%s" %
(srcFitDir, tfiles[0], self.templateDir, self.templateImg))
sexConf = [
'-DETECT_MINAREA', '5', '-DETECT_THRESH', '3',
'-ANALYSIS_THRESH', '3'
]
fpar = 'sex_diff.par'
imgSimClass = ImageSimulation(self.tmpDir)
for i, imgName in enumerate(tfiles[1:]):
#if i<460:
# continue
starttime = datetime.now()
self.log.info("\n\n**********\nsimulate %d: %s" % (i, imgName))
os.system("rm -rf %s/*" % (self.tmpDir))
imgpre = imgName.split(".")[0]
tobjFitsFullPath = "%s/%s.fit" % (srcFitDir, imgpre)
if not os.path.exists(tobjFitsFullPath):
self.log.error("%s.fit not exist, stop" % (imgpre))
break
if os.path.exists("%s/%s.fit" % (destFitDir, imgpre)):
self.log.info("%s.fit already simulated, skip" % (imgpre))
continue
os.system("cp %s/%s.fit %s/%s" %
(srcFitDir, imgpre, self.tmpDir, self.objectImg))
#sexConf=['-DETECT_MINAREA','10','-DETECT_THRESH','5','-ANALYSIS_THRESH','5']
sexConf = [
'-DETECT_MINAREA', '5', '-DETECT_THRESH', '3',
'-ANALYSIS_THRESH', '3'
]
self.objectImgCat = self.tools.runSextractor(self.objectImg,
self.tmpDir,
self.tmpDir,
fpar,
sexConf=sexConf)
mchFile16, nmhFile16 = self.tools.runSelfMatch(
self.tmpDir, self.objectImgCat, 16)
self.osn16 = nmhFile16
objCali = self.magCali(self.osn16)
osn16s = selectTempOTs(objCali, self.tmpDir)
tdata = np.loadtxt("%s/%s" % (self.tmpDir, osn16s))
if len(tdata.shape) < 2 or tdata.shape[0] < 100:
self.log.error("%s has too little stars, break this run" %
(self.objectImg))
continue
osn16sf = filtOTs(osn16s, self.tmpDir)
tdata = np.loadtxt("%s/%s" % (self.tmpDir, osn16sf))
if len(tdata.shape) < 2 or tdata.shape[0] < 45:
self.log.error("%s has too little stars, break this run" %
(self.objectImg))
continue
mchFile, nmhFile = self.tools.runSelfMatch(
self.tmpDir, self.objectImgCat, 24)
self.osn32 = nmhFile
osn32f = filtOTs(self.osn32, self.tmpDir)
posmagFile = "simAdd190324.cat"
if i == 0 or (not os.path.exists(
"%s/%s" % (self.templateDir, posmagFile))):
tposmagFile = ""
else:
tposmagFile = posmagFile
os.system("cp %s/%s %s/%s" % (self.templateDir, posmagFile,
self.tmpDir, posmagFile))
simFile, simPosFile, tmpOtImgs = imgSimClass.simulateImage1(
osn32f,
self.objectImg,
osn16sf,
self.objectImg,
posmagFile=tposmagFile)
self.objectImgSim = simFile
self.objectImgSimCatAdd = simPosFile
#destFitDir, destCatAddDir
os.system("cp %s/%s %s/%s.fit" %
(self.tmpDir, self.objectImgSim, destFitDir, imgpre))
os.system("cp %s/%s %s/%s.cat" %
(self.tmpDir, self.objectImgSimCatAdd, destCatAddDir,
imgpre))
if i == 0 or (not os.path.exists(
"%s/%s" % (self.templateDir, posmagFile))):
os.system("cp %s/%s %s/%s" %
(self.tmpDir, self.objectImgSimCatAdd,
self.templateDir, posmagFile))
if i % 50 == 1 or i >= len(tfiles) - 3:
os.system("cp %s/%s %s/%s" %
(self.templateDir, self.templateImg, self.tmpDir,
self.templateImg))
self.simTmpResi, runSuccess = self.tools.runHotpants(
self.objectImgSim, self.templateImg, self.tmpDir)
if not runSuccess:
self.log.info("%s.fit sim diff error..." % (imgpre))
continue
sexConf = [
'-DETECT_MINAREA', '3', '-DETECT_THRESH', '2.5',
'-ANALYSIS_THRESH', '2.5'
]
#self.objectImgSimCat = self.tools.runSextractor(self.objectImgSim, self.tmpDir, self.tmpDir, sexConf=sexConf)
self.simTmpResiCat = self.tools.runSextractor(
self.simTmpResi,
self.tmpDir,
self.tmpDir,
fpar,
sexConf=sexConf)
#simTmpResiCatEf = filtByEllipticity(self.simTmpResiCat, self.tmpDir, maxEllip=0.5)
#mchFile, nmhFile = self.tools.runSelfMatch(self.tmpDir, simTmpResiCatEf, 16)
#simTmpResiCatEf_sn32 = nmhFile
#simTmpResiCatEf_sn32 = simTmpResiCatEf
#mchFile, nmhFile, mchPair = self.tools.runCrossMatch(self.tmpDir, self.objectImgSimCatAdd, simTmpResiCatEf, 5)
mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
self.tmpDir, self.objectImgSimCatAdd,
self.simTmpResiCat, 5)
tdata1 = np.loadtxt("%s/%s" %
(self.tmpDir, self.objectImgSimCatAdd))
tdata2 = np.loadtxt("%s/%s" %
(self.tmpDir, self.simTmpResiCat))
tdata3 = np.loadtxt("%s/%s" % (self.tmpDir, mchFile))
self.log.info(
"sim add %d, diff find %d, sim match diff %d, find percentage %.2f%%"
% (tdata1.shape[0], tdata2.shape[0], tdata3.shape[0],
tdata3.shape[0] * 100.0 / tdata1.shape[0]))
os.system(
"cp %s/%s %s/%s_simResi.fit" %
(self.tmpDir, self.simTmpResi, destFitDir, imgpre))
os.system("cp %s/%s %s/%s_simResi.cat" %
(self.tmpDir, self.simTmpResiCat, destCatAddDir,
imgpre))
tgrid = 4
tsize = 500
tzoom = 2
timg = getThumbnail(self.tmpDir,
self.simTmpResi,
stampSize=(tsize, tsize),
grid=(tgrid, tgrid),
innerSpace=1)
timg = scipy.ndimage.zoom(timg, tzoom, order=0)
preViewPath = "%s/%s_resi.jpg" % (self.preViewSimResiDir,
imgpre)
Image.fromarray(timg).save(preViewPath)
endtime = datetime.now()
runTime = (endtime - starttime).seconds
self.log.info("sim: %s use %d seconds" % (imgName, runTime))
#break
except Exception as e:
print(str(e))
tstr = traceback.format_exc()
print(tstr)
def simTOT(self, oImg, tImg, subImgNum=100):
mchFile, nmhFile = self.runSelfMatch(self.objectImgCat, 24)
self.osn32 = nmhFile
osn16s = selectTempOTs(self.osn16, self.tmpDir)
osn16sf = filtOTs(osn16s, self.tmpDir)
osn32f = filtOTs(self.osn32, self.tmpDir)
mchFile, nmhFile, mchPair = self.runCrossMatch(osn32f, self.tsn5, self.r5)
osn32s_tsn5_cm5 = mchFile
osn32s_tsn5_cm5_pair = mchPair
totalTOT = subImgNum
subImgBuffer = []
tnum = 0
imgSimClass = ImageSimulation()
ii = 1
sexConf=['-DETECT_MINAREA','3','-DETECT_THRESH','2.5','-ANALYSIS_THRESH','2.5']
while tnum<totalTOT:
simFile, simPosFile, simDeltaXYA = imgSimClass.simulateImage1(osn32f, self.objectImg, osn16sf, self.objectImg)
self.objectImgSim = simFile
self.objectImgSimAdd = simPosFile
self.simTmpResi = self.runHotpants(self.objectImgSim, self.templateImg)
objectImgSimCat = self.runSextractor(self.simTmpResi, sexConf)
tdata0 = np.loadtxt("%s/%s"%(self.tmpDir, objectImgSimCat))
print(tdata0.shape)
simTmpResiCatEf = filtByEllipticity(objectImgSimCat, self.tmpDir, maxEllip=0.5)
mchFile, nmhFile = self.runSelfMatch(simTmpResiCatEf, self.r5)
simTmpResiCatEf_sn32 = nmhFile
#simTmpResiCatEf_sn32 = simTmpResiCatEf
mchFile, nmhFile, mchPair = self.runCrossMatch(self.objectImgSimAdd, simTmpResiCatEf_sn32, self.r5)
str_oisa_cm5 = mchFile
str_oisa_cm5_pair = mchPair
tIdx1 = np.loadtxt("%s/%s"%(self.tmpDir, osn32s_tsn5_cm5_pair)).astype(np.int)
tIdx2 = np.loadtxt("%s/%s"%(self.tmpDir, str_oisa_cm5_pair)).astype(np.int)
tIdx1 = tIdx1 - 1
tIdx2 = tIdx2 - 1
self.log.debug("objectCat matched data %d, templateCat matched data %d"%(tIdx1.shape[0], tIdx2.shape[0]))
tnames1 = ['objId', 'tmpId']
tnames2 = ['objId', 'resiId']
#unionIdx = np.intersect1d(tIdx1[:,0], tIdx2[:,0]) #union1d
#self.log.debug("intersect objectCat and templateCat matched data: %d"%(unionIdx.shape[0]))
df1 = pd.DataFrame(data=tIdx1, columns=tnames1)
df2 = pd.DataFrame(data=tIdx2, columns=tnames2)
unionIdx=pd.merge(df1, df2, how='inner', on=['objId'])
self.log.debug("innerjoin objectCat and templateCat matched data %d"%(unionIdx.shape[0]))
tdata1 = np.loadtxt("%s/%s"%(self.tmpDir, self.objectImgSimAdd))
tdata2 = np.loadtxt("%s/%s"%(self.tmpDir, self.tsn5))
tdata3 = np.loadtxt("%s/%s"%(self.tmpDir, simTmpResiCatEf_sn32))
simDeltaXYA = np.array(simDeltaXYA)
tdeltaXYA = simDeltaXYA[unionIdx["objId"].values]
tdata11 = tdata1[unionIdx["objId"].values]
tdata12 = tdata2[unionIdx["tmpId"].values]
tdata22 = tdata3[unionIdx["resiId"].values]
print(tdata22.shape)
print("sim star %d, matched %d"%(tdata1.shape[0],tIdx2.shape[0]))
magerr = tdata22[:,3]
print(magerr[:10])
from matplotlib.ticker import MultipleLocator, FormatStrFormatter
xmajorLocator = MultipleLocator(2)
xminorLocator = MultipleLocator(1)
plt.figure(figsize = (16, 8))
ax = plt.subplot(111)
plt.hist(magerr, bins=20, range=(0,20), normed=False,
weights=None, cumulative=False, bottom=None,
histtype=u'bar', align=u'left', orientation=u'vertical',
rwidth=0.8, log=False, color=None, label=None, stacked=False,
hold=None)
ax.xaxis.set_major_locator(xmajorLocator)
ax.xaxis.set_minor_locator(xminorLocator)
plt.show()
self.log.info("\n******************")
self.log.info("run %d, total sub image %d"%(ii, tnum))
if ii>0:
break
ii = ii + 1
rsts = np.array(subImgBuffer)
print(rsts.shape)
return rsts