def simImage(self, oImg):
outpre = oImg.split(".")[0]
os.system("rm -rf %s/*" % (self.tmpDir))
os.system("cp %s/%s %s/%s" %
(self.srcDir, oImg, self.tmpDir, self.objectImg))
self.removeHeader(self.objectImg)
self.objectImgCat = self.runSextractor(self.objectImg)
mchFile, nmhFile = self.runSelfMatch(self.objectImgCat, self.r16)
self.osn16 = nmhFile
osn16s = selectTempOTs(self.osn16, self.tmpDir)
tdata = np.loadtxt("%s/%s" % (self.tmpDir, osn16s))
if len(tdata.shape) < 2 or tdata.shape[0] < 100:
print("%s has too little stars, break this run" % (oImg))
return
osn16sf = filtOTs(osn16s, self.tmpDir)
tdata = np.loadtxt("%s/%s" % (self.tmpDir, osn16sf))
if len(tdata.shape) < 2 or tdata.shape[0] < 45:
print("%s has too little stars, break this run" % (oImg))
return
imgSimClass = ImageSimulation()
otImgs = imgSimClass.getTmpOtImgs(osn16sf, self.objectImg)
psfView = genPSFView(otImgs)
thumbnail = getThumbnail(self.srcDir,
oImg,
stampSize=(100, 100),
grid=(5, 5),
innerSpace=1)
thumbnail = scipy.ndimage.zoom(thumbnail, 4, order=0)
'''
plt.clf()
plt.figure(figsize=(20,20))
plt.imshow(psfView, interpolation = "nearest", cmap='gray')
plt.show()
plt.clf()
plt.figure(figsize=(20,20))
plt.imshow(thumbnail, interpolation = "nearest", cmap='gray')
plt.show()
'''
tpath11 = "%s_view" % (self.srcDir)
if not os.path.exists(tpath11):
os.system("mkdir %s" % (tpath11))
dpath1 = "%s/%s_psf.jpg" % (tpath11, outpre)
dpath2 = "%s/%s_thb.jpg" % (tpath11, outpre)
Image.fromarray(psfView).save(dpath1)
Image.fromarray(thumbnail).save(dpath2)
def diffImage(self):
starttime = datetime.datetime.now()
resultFlag = True
oImgPre = self.origObjectImg[:self.origObjectImg.index(".")]
os.system("cp %s/%s %s/%s" % (self.templateDir, self.templateImg,
self.tmpDir, self.templateImg))
os.system(
"cp %s/%s %s/%s" %
(self.templateDir, self.badPixCat, self.tmpDir, self.badPixCat))
os.system("cp %s/%s %s/%s" % (self.templateDir, self.templateImgCat,
self.tmpDir, self.templateImgCat))
#self.newImageName = self.tools.imageAlign(self.tmpDir, self.objectImg, self.transHG)
self.objTmpResi = self.tools.runHotpants(self.newImageName,
self.templateImg, self.tmpDir)
fpar = 'sex_diff.par'
sexConf = [
'-DETECT_MINAREA', '3', '-DETECT_THRESH', '2.5',
'-ANALYSIS_THRESH', '2.5'
]
resiCat = self.tools.runSextractor(self.objTmpResi, self.tmpDir,
self.tmpDir, fpar, sexConf)
'''
self.tools.runSelfMatch(self.tmpDir, resiCat, 1) #debug: get ds9 reg file
tdata = np.loadtxt("%s/%s"%(self.tmpDir, resiCat))
self.log.info("resi image star %d"%(tdata.shape[0]))
'''
'''
mchRadius = 10
mchFile, nmhFile, mchPair = self.tools.runCrossMatch(self.tmpDir, resiCat, self.templateImgCat, mchRadius)
tdata = np.loadtxt("%s/%s"%(self.tmpDir, mchFile))
print("resi star not match template %d"%(tdata.shape[0]))
'''
mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
self.tmpDir, resiCat, self.badPixCat, 1) #1 and 5
os.system("cp %s/%s %s/%s" %
(self.tmpDir, nmhFile, self.resiCatDir, "%s.cat" %
(oImgPre)))
tdata = np.loadtxt("%s/%s" % (self.tmpDir, nmhFile))
self.log.info("resi star not match template and remove badpix %d" %
(tdata.shape[0]))
if tdata.shape[0] < 3000:
size = self.subImgSize
fSubImgs, fparms = self.tools.getWindowImgs(
self.tmpDir, self.newImageName, self.templateImg,
self.objTmpResi, tdata, size)
tXY = fparms[:, 0:2]
tRaDec = self.wcs.all_pix2world(tXY, 1)
fparms = np.concatenate((fparms, tRaDec), axis=1)
fotpath = '%s/%s_otimg.npz' % (self.destDir, oImgPre)
np.savez_compressed(fotpath, fot=fSubImgs, parms=fparms)
resiImgs = []
for timg in fSubImgs:
resiImgs.append(timg[2])
preViewPath = "%s/%s_psf.jpg" % (self.preViewDir, oImgPre)
#if not os.path.exists(preViewPath):
psfView = genPSFView(resiImgs)
Image.fromarray(psfView).save(preViewPath)
resultFlag = True
else:
tmsgStr = "%s.fit resi image has %d objects, maybe wrong" % (
oImgPre, tdata.shape[0])
self.log.error(tmsgStr)
self.tools.sendTriggerMsg(tmsgStr)
resultFlag = False
tgrid = 4
tsize = 500
tzoom = 1
timg = getThumbnail(self.tmpDir,
self.objTmpResi,
stampSize=(tsize, tsize),
grid=(tgrid, tgrid),
innerSpace=1)
timg = scipy.ndimage.zoom(timg, tzoom, order=0)
preViewPath = "%s/%s_resi.jpg" % (self.origPreViewDir, oImgPre)
Image.fromarray(timg).save(preViewPath)
timg = getThumbnail(self.tmpDir,
self.newImageName,
stampSize=(tsize, tsize),
grid=(tgrid, tgrid),
innerSpace=1)
timg = scipy.ndimage.zoom(timg, tzoom, order=0)
preViewPath = "%s/%s_obj.jpg" % (self.origPreViewDir, oImgPre)
Image.fromarray(timg).save(preViewPath)
timg = getThumbnail(self.tmpDir,
self.templateImg,
stampSize=(tsize, tsize),
grid=(tgrid, tgrid),
innerSpace=1)
timg = scipy.ndimage.zoom(timg, tzoom, order=0)
preViewPath = "%s/%s_tmp.jpg" % (self.origPreViewDir, oImgPre)
Image.fromarray(timg).save(preViewPath)
endtime = datetime.datetime.now()
runTime = (endtime - starttime).seconds
self.log.info("********** image diff total use %d seconds" % (runTime))
return resultFlag
''' '''
def diffImage(self, imgName, tmplParms):
starttime = datetime.now()
resultFlag = True
oImgPre = imgName.split(".")[0]
dateStr = imgName.split('_')[3][:6]
os.system("rm -rf %s/*" % (self.diff))
status = tmplParms[0]
#tmplImgName = tmplParms[1][-1][0]
tmplImgName = tmplParms[1][0][0]
tmplImgPre = tmplImgName.split(".")[0]
if status == '1':
ttmplPath = self.tmplAlignDir
else:
ttmplPath = self.tmplDiffDir
ofitPath = "%s/%s.fit" % (self.doDiffTmpl, tmplImgPre)
if not os.path.exists(ofitPath):
os.system("rm -rf %s/*" % (self.doDiffTmpl))
os.system("cp %s/%s.fit %s/%s.fit" %
(ttmplPath, tmplImgPre, self.doDiffTmpl, tmplImgPre))
os.system("cp %s/%s.cat %s/%s.cat" %
(ttmplPath, tmplImgPre, self.doDiffTmpl, tmplImgPre))
os.system("cp %s/%s_badpix.cat %s/%s_badpix.cat" %
(ttmplPath, tmplImgPre, self.doDiffTmpl, tmplImgPre))
os.system("cp %s/%s.fit %s/ti.fit" %
(self.doDiffTmpl, tmplImgPre, self.diff))
os.system("cp %s/%s.cat %s/ti.cat" %
(self.doDiffTmpl, tmplImgPre, self.diff))
os.system("cp %s/%s_badpix.cat %s/ti_badpix.cat" %
(self.doDiffTmpl, tmplImgPre, self.diff))
os.system("cp %s/%s_align.fit %s/oi.fit" %
(self.alignDir, oImgPre, self.diff))
os.system("cp %s/%s_align.cat %s/oi.cat" %
(self.alignDir, oImgPre, self.diff))
theader = fits.getheader("%s/oi.fit" % (self.diff))
dateObs = theader['DATE-OBS']
timeObs = theader['TIME-OBS']
dtStr = "%sT%s" % (dateObs, timeObs)
objTmpResi, runSuccess = self.tools.runHotpants(
'oi.fit', 'ti.fit', self.diff)
if not runSuccess:
self.log.error("diffImage failure: %s" % (imgName))
return False
fpar = 'sex_diff.par'
#sexConf=['-DETECT_MINAREA','3','-DETECT_THRESH','2.5','-ANALYSIS_THRESH','2.5']
sexConf = [
'-DETECT_MINAREA', '5', '-DETECT_THRESH', '2.5',
'-ANALYSIS_THRESH', '2.5'
]
resiCat, isSuccess = self.tools.runSextractor(objTmpResi, self.diff,
self.diff, fpar, sexConf)
if not isSuccess:
self.log.error("diffImage runSextractor failure")
return isSuccess
mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
self.diff, resiCat, 'oi.cat', 1) #1 and 5
badPix2Path = "%s/%s" % (self.diff, nmhFile)
if os.path.exists(badPix2Path):
badPixProps2 = np.loadtxt(badPix2Path)
else:
badPixProps2 = np.array([])
mchRadius = 10 #15 10
mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
self.diff, mchFile, 'ti.cat', mchRadius)
fotPath = "%s/%s" % (self.diff, mchFile)
if os.path.exists(fotPath):
fotProps = np.loadtxt(fotPath)
else:
fotProps = np.array([])
mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
self.diff, nmhFile, 'ti_badpix.cat', 1) #1 and 5
os.system("cp %s/%s %s/%s_tot.cat" %
(self.diff, nmhFile, self.diffCatDir, oImgPre))
totPath = "%s/%s" % (self.diff, nmhFile)
if os.path.exists(totPath):
totProps = np.loadtxt(totPath)
else:
totProps = np.array([])
badPixPath = "%s/ti_badpix.cat" % (self.diff)
if os.path.exists(badPixPath):
badPixProps = np.loadtxt(badPixPath)
else:
badPixProps = np.array([])
#badPixProps = np.array([])
tstr = "orgBadPix %d, nmBad %d, match %d, noMatch %d" % (
badPixProps.shape[0], badPixProps2.shape[0], fotProps.shape[0],
totProps.shape[0])
self.log.info(tstr)
print(tstr)
#size = self.subImgSize
size = 68
if totProps.shape[0] < 500 and totProps.shape[0] > 0:
wcsPath = "%s/%s.wcs" % (ttmplPath, tmplImgPre)
wcs = WCS(wcsPath)
searchRadius = 0.05
mpQuery = QueryMinorPlanet()
mpQuery.connDb()
totSubImgs, totParms = getWindowImgs(self.diff, 'oi.fit', 'ti.fit',
objTmpResi, totProps, size)
if totParms.shape[0] > 0:
tXY = totParms[:, 0:2]
#tdates = np.repeat(dtStr,tXY.shape[0]).reshape((tXY.shape[0],1))
tRaDec = wcs.all_pix2world(tXY, 1)
totParms = np.concatenate((totParms, tRaDec), axis=1)
mchDis = []
for tpos in tRaDec:
ra = tpos[0]
dec = tpos[1]
tdis, tmag = mpQuery.matchMP(ra, dec, dateStr, dtStr,
searchRadius + 0.001)
mchDis.append([tdis, tmag])
mchDis = np.array(mchDis)
totParms = np.concatenate((totParms, mchDis), axis=1)
totSubImgs1 = totSubImgs[mchDis[:, 0] >= searchRadius]
totParms1 = totParms[mchDis[:, 0] >= searchRadius]
mpSubImgs1 = totSubImgs[mchDis[:, 0] < searchRadius]
mpParms1 = totParms[mchDis[:, 0] < searchRadius]
tstr0 = "%s has %d tot, contain %d minorplanet." % (
imgName, totParms.shape[0], mpParms1.shape[0])
print(tstr0)
self.log.info(tstr0)
if totSubImgs1.shape[0] > 0:
totpath = '%s/%s_totimg.npz' % (self.destDir, oImgPre)
np.savez_compressed(totpath,
imgs=totSubImgs1,
parms=totParms1,
obsUtc=dtStr)
resiImgs = []
for timg in totSubImgs1:
resiImgs.append(timg[2])
preViewPath = "%s/%s_tot.jpg" % (self.subImgViewDir,
oImgPre)
#if not os.path.exists(preViewPath):
psfView = genPSFView(resiImgs)
Image.fromarray(psfView).save(preViewPath)
if mpSubImgs1.shape[0] > 0:
self.sendMsg(tstr0)
totpath = '%s/%s_mpimgt.npz' % (self.destDir, oImgPre)
np.savez_compressed(totpath,
imgs=mpSubImgs1,
parms=mpParms1,
obsUtc=dtStr)
resiImgs = []
for timg in mpSubImgs1:
resiImgs.append(timg[2])
preViewPath = "%s/%s_mpt.jpg" % (self.subImgViewDir,
oImgPre)
#if not os.path.exists(preViewPath):
psfView = genPSFView(resiImgs)
Image.fromarray(psfView).save(preViewPath)
os.system(
"cp %s/%s %s/%s_resi.fit" %
(self.diff, objTmpResi, self.diffImgDir, oImgPre))
os.system("cp %s/%s %s/%s_resi.cat" %
(self.diff, resiCat, self.diffCatDir, oImgPre))
else:
alignImgName = "%s/%s_align.fit" % (self.alignDir, oImgPre)
os.system("rm -rf %s" % (alignImgName))
alignImgName = "%s/%s_align.cat" % (self.alignDir, oImgPre)
os.system("rm -rf %s" % (alignImgName))
alignImgName = "%s/%s.cat" % (self.catDir, oImgPre)
os.system("rm -rf %s" % (alignImgName))
if fotProps.shape[0] > 0 and fotProps.shape[0] < 500:
fotSubImgs, fotParms = getWindowImgs(self.diff, 'oi.fit',
'ti.fit', objTmpResi,
fotProps, size)
if fotParms.shape[0] > 0:
tXY = fotParms[:, 0:2]
#tdates = np.repeat(dtStr,tXY.shape[0]).reshape((tXY.shape[0],1))
tRaDec = wcs.all_pix2world(tXY, 1)
fotParms = np.concatenate((fotParms, tRaDec), axis=1)
if fotSubImgs.shape[0] > 0:
totpath = '%s/%s_fotimg.npz' % (self.destDir, oImgPre)
np.savez_compressed(totpath,
imgs=fotSubImgs,
parms=fotParms,
obsUtc=dtStr)
mpQuery.closeDb()
resultFlag = True
else:
tmsgStr = "%s.fit resi image has %d tot objects, maybe wrong" % (
oImgPre, totProps.shape[0])
self.log.error(tmsgStr)
#self.sendMsg(tmsgStr)
resultFlag = False
endtime = datetime.now()
runTime = (endtime - starttime).seconds
self.log.info("********** image diff total use %d seconds" % (runTime))
return resultFlag
def simImage(self, oImg, tImg):
starttime = datetime.datetime.now()
self.objectImgOrig = oImg
self.templateImgOrig = tImg
os.system("rm -rf %s/*" % (self.tmpDir))
oImgfz = "%s/%s.fz" % (self.srcDir, oImg)
tImgfz = "%s/%s.fz" % (self.srcDir, tImg)
if (not os.path.exists(oImgfz)) or (not os.path.exists(tImgfz)):
print("%s or %s not exist" % (oImgfz, tImgfz))
return
os.system("cp %s/%s.fz %s/%s.fz" %
(self.srcDir, oImg, self.tmpDir, self.objectImg))
os.system("cp %s/%s.fz %s/%s.fz" %
(self.srcDir, tImg, self.tmpDir, self.templateImg))
os.system("%s %s/%s.fz" %
(self.funpackProgram, self.tmpDir, self.objectImg))
os.system("%s %s/%s.fz" %
(self.funpackProgram, self.tmpDir, self.templateImg))
self.removeHeaderAndOverScan(self.objectImg)
self.removeHeaderAndOverScan(self.templateImg)
sexConf = [
'-DETECT_MINAREA', '7', '-DETECT_THRESH', '5', '-ANALYSIS_THRESH',
'5'
]
fpar = 'sex_diff_fot.par'
self.objectImgCat = self.runSextractor(self.objectImg, fpar, sexConf)
self.templateImgCat = self.runSextractor(self.templateImg, fpar,
sexConf)
tdata = np.loadtxt("%s/%s" % (self.tmpDir, self.objectImgCat))
print("objImg extract star %d" % (tdata.shape[0]))
if len(tdata.shape) < 2 or tdata.shape[0] < 5000:
print("%s has too little stars, break this run" % (oImg))
return
tdata = np.loadtxt("%s/%s" % (self.tmpDir, self.templateImgCat))
print("tempImg extract star %d" % (tdata.shape[0]))
if len(tdata.shape) < 2 or tdata.shape[0] < 5000:
print("%s has too little stars, break this run" % (tImg))
return
badPixCat = self.processBadPix()
''' '''
mchFile, nmhFile = self.runSelfMatch(self.objectImgCat, self.r16)
self.osn16 = nmhFile
mchFile, nmhFile = self.runSelfMatch(self.templateImgCat, self.r16)
self.tsn16 = nmhFile
mchFile, nmhFile, mchPair = self.runCrossMatch(self.osn16, self.tsn16,
10)
osn16_tsn16_cm5 = mchFile
osn16_tsn16_cm5_pair = mchPair
self.gridStatistic(osn16_tsn16_cm5, gridNum=4)
newimage, h = self.getMatchPos(self.osn16, self.tsn16,
osn16_tsn16_cm5_pair)
newName = "new.fit"
newPath = "%s/%s" % (self.tmpDir, newName)
if os.path.exists(newPath):
os.remove(newPath)
hdu = fits.PrimaryHDU(newimage)
hdul = fits.HDUList([hdu])
hdul.writeto(newPath)
'''
tdata = np.loadtxt("%s/%s"%(self.tmpDir, self.objectImgCat))
tdata = np.array([tdata])
tdata2 = cv2.perspectiveTransform(tdata, h)
tdata2 = tdata2[0]
oiTransName = "%s_trans.cat"%(self.objectImgCat[:self.objectImgCat.index(".")])
oiTransPath = "%s/%s"%(self.tmpDir, oiTransName)
with open(oiTransPath, 'w') as fp1:
for tobj in tdata2:
fp1.write("%.3f %.3f\n"%(tobj[0], tobj[1]))
'''
self.objTmpResi = self.runHotpants(newName, self.templateImg)
tgrid = 4
tsize = 1000
tzoom = 1
oImgPre = oImg[:oImg.index(".")]
timg = getThumbnail(self.tmpDir,
self.objTmpResi,
stampSize=(tsize, tsize),
grid=(tgrid, tgrid),
innerSpace=1)
timg = scipy.ndimage.zoom(timg, tzoom, order=0)
preViewPath = "%s/%s_resi.jpg" % (self.origPreViewDir, oImgPre)
Image.fromarray(timg).save(preViewPath)
timg = getThumbnail(self.tmpDir,
self.objectImg,
stampSize=(tsize, tsize),
grid=(tgrid, tgrid),
innerSpace=1)
timg = scipy.ndimage.zoom(timg, tzoom, order=0)
preViewPath = "%s/%s_obj.jpg" % (self.origPreViewDir, oImgPre)
Image.fromarray(timg).save(preViewPath)
timg = getThumbnail(self.tmpDir,
self.templateImg,
stampSize=(tsize, tsize),
grid=(tgrid, tgrid),
innerSpace=1)
timg = scipy.ndimage.zoom(timg, tzoom, order=0)
preViewPath = "%s/%s_tmp.jpg" % (self.origPreViewDir, oImgPre)
Image.fromarray(timg).save(preViewPath)
fpar = 'sex_diff.par'
sexConf = [
'-DETECT_MINAREA', '3', '-DETECT_THRESH', '2.5',
'-ANALYSIS_THRESH', '2.5'
]
resiCat = self.runSextractor(self.objTmpResi, fpar, sexConf)
mchFile, nmhFile = self.runSelfMatch(resiCat, 1)
tdata = np.loadtxt("%s/%s" % (self.tmpDir, resiCat))
print("resi image star %d" % (tdata.shape[0]))
mchRadius = 10
mchFile, nmhFile, mchPair = self.runCrossMatch(resiCat,
self.templateImgCat,
mchRadius)
tdata = np.loadtxt("%s/%s" % (self.tmpDir, mchFile))
print("resi star match template %d" % (tdata.shape[0]))
mchFile, nmhFile, mchPair = self.runCrossMatch(mchFile, badPixCat, 5)
tdata = np.loadtxt("%s/%s" % (self.tmpDir, nmhFile))
print("resi star match template and remove badpix %d" %
(tdata.shape[0]))
size = self.subImgSize
fSubImgs, fparms = self.getWindowImgs(newName, self.templateImg,
self.objTmpResi, tdata, size)
oImgPre = oImg[:oImg.index(".")]
fotpath = '%s/%s_otimg_fot.npz' % (self.destDir, oImgPre)
np.savez_compressed(fotpath, fot=fSubImgs, parms=fparms)
self.log.info("\n******************")
self.log.info("simulation False OT, total sub image %d" %
(len(fSubImgs)))
resiImgs = []
for timg in fSubImgs:
resiImgs.append(timg[2])
preViewPath = "%s/%s_psf.jpg" % (self.preViewDir, oImgPre)
#if not os.path.exists(preViewPath):
psfView = genPSFView(resiImgs)
Image.fromarray(psfView).save(preViewPath)
endtime = datetime.datetime.now()
runTime = (endtime - starttime).seconds
self.log.debug("image diff total use %d seconds" % (runTime))
def simTOT(self, oImg, tImg, subImgNum=100):
mchFile, nmhFile = self.runSelfMatch(self.objectImgCat, 24)
self.osn32 = nmhFile
osn16s = selectTempOTs(self.osn16, self.tmpDir)
tdata = np.loadtxt("%s/%s"%(self.tmpDir, osn16s))
if len(tdata.shape)<2 or tdata.shape[0]<100:
print("%s has too little stars, break this run"%(oImg))
return np.array([]), np.array([])
osn16sf = filtOTs(osn16s, self.tmpDir)
tdata = np.loadtxt("%s/%s"%(self.tmpDir, osn16sf))
if len(tdata.shape)<2 or tdata.shape[0]<45:
print("%s has too little stars, break this run"%(oImg))
return np.array([]), np.array([])
osn32f = filtOTs(self.osn32, self.tmpDir)
mchFile, nmhFile, mchPair = self.runCrossMatch(osn32f, self.tsn5, self.r5)
osn32s_tsn5_cm5 = mchFile
osn32s_tsn5_cm5_pair = mchPair
totalTOT = subImgNum
subImgBuffer = []
objS2NBuffer = []
tnum = 0
imgSimClass = ImageSimulation()
ii = 1
sexConf=['-DETECT_MINAREA','3','-DETECT_THRESH','2.5','-ANALYSIS_THRESH','2.5']
while tnum<totalTOT:
simFile, simPosFile, simDeltaXYA, tmpOtImgs = imgSimClass.simulateImage1(osn32f, self.objectImg, osn16sf, self.objectImg)
outpre= self.objectImgOrig.split(".")[0]
preViewPath = "%s/%s_psf.jpg"%(self.preViewDir, outpre)
if not os.path.exists(preViewPath):
psfView = genPSFView(tmpOtImgs)
Image.fromarray(psfView).save(preViewPath)
self.objectImgSim = simFile
self.objectImgSimAdd = simPosFile
self.simTmpResi = self.runHotpants(self.objectImgSim, self.templateImg)
self.simTmpResiCat = self.runSextractor(self.simTmpResi, sexConf)
simTmpResiCatEf = filtByEllipticity(self.simTmpResiCat, self.tmpDir, maxEllip=0.5)
mchFile, nmhFile = self.runSelfMatch(simTmpResiCatEf, self.r5)
simTmpResiCatEf_sn32 = nmhFile
#simTmpResiCatEf_sn32 = simTmpResiCatEf
mchFile, nmhFile, mchPair = self.runCrossMatch(self.objectImgSimAdd, simTmpResiCatEf_sn32, self.r5)
str_oisa_cm5 = mchFile
str_oisa_cm5_pair = mchPair
tIdx1 = np.loadtxt("%s/%s"%(self.tmpDir, osn32s_tsn5_cm5_pair)).astype(np.int)
tIdx2 = np.loadtxt("%s/%s"%(self.tmpDir, str_oisa_cm5_pair)).astype(np.int)
tIdx1 = tIdx1 - 1
tIdx2 = tIdx2 - 1
self.log.debug("objectCat matched data %d, ResiCat matched data %d"%(tIdx1.shape[0], tIdx2.shape[0]))
tnames1 = ['objId', 'tmpId']
tnames2 = ['objId', 'resiId']
#unionIdx = np.intersect1d(tIdx1[:,0], tIdx2[:,0]) #union1d
#self.log.debug("intersect objectCat and templateCat matched data: %d"%(unionIdx.shape[0]))
df1 = pd.DataFrame(data=tIdx1, columns=tnames1)
df2 = pd.DataFrame(data=tIdx2, columns=tnames2)
unionIdx=pd.merge(df1, df2, how='inner', on=['objId'])
self.log.debug("innerjoin objectCat and templateCat matched data %d"%(unionIdx.shape[0]))
tdata1 = np.loadtxt("%s/%s"%(self.tmpDir, self.objectImgSimAdd))
tdata2 = np.loadtxt("%s/%s"%(self.tmpDir, self.tsn5))
tdata3 = np.loadtxt("%s/%s"%(self.tmpDir, simTmpResiCatEf_sn32))
simDeltaXYA = np.array(simDeltaXYA)
tdeltaXYA = simDeltaXYA[unionIdx["objId"].values]
tdata11 = tdata1[unionIdx["objId"].values]
tdata12 = tdata2[unionIdx["tmpId"].values]
tdata22 = tdata3[unionIdx["resiId"].values]
poslist = np.concatenate(([tdata11[:,0]], [tdata11[:,1]],
[tdata12[:,3]+tdeltaXYA[:,0]], [tdata12[:,4]+tdeltaXYA[:,1]],
[tdata22[:,0]], [tdata22[:,1]]), axis=0).transpose()
#print(poslist)
#genFinalOTDs9Reg('tot', self.tmpDir, poslist)
size = self.subImgSize
subImgs = self.getWindowImgs(self.objectImgSim, self.templateImg, self.simTmpResi, poslist, size)
tnum = tnum + len(subImgs)
subImgBuffer.extend(subImgs)
objS2NBuffer.extend(tdata22[:,3].tolist())
self.log.info("\n******************")
self.log.info("run %d, total sub image %d"%(ii, tnum))
if ii>6:
break
ii = ii + 1
#break
subImgs = np.array(subImgBuffer)
objS2NBuffer = np.array(objS2NBuffer)
print(subImgs.shape)
print(objS2NBuffer.shape)
return subImgs, objS2NBuffer
def diffImage(self):
starttime = datetime.now()
resultFlag = True
oImgPre = self.origObjectImg[:self.origObjectImg.index(".")]
os.system("cp %s/%s %s/%s" % (self.templateDir, self.templateImg,
self.tmpDir, self.templateImg))
os.system(
"cp %s/%s %s/%s" %
(self.templateDir, self.badPixCat, self.tmpDir, self.badPixCat))
os.system("cp %s/%s %s/%s" % (self.templateDir, self.templateImgCat,
self.tmpDir, self.templateImgCat))
#self.newImageName = self.tools.imageAlign(self.tmpDir, self.objectImg, self.transHG)
self.objTmpResi, runSuccess = self.tools.runHotpants(
self.newImageName, self.templateImg, self.tmpDir)
if not runSuccess:
return False
os.system("cp %s/%s %s/%s.fit" %
(self.tmpDir, self.objTmpResi, self.resiFitsDir, oImgPre))
tgrid = 1
tsize = 4096
tzoom = 1
timg = getThumbnail(self.tmpDir,
self.objTmpResi,
stampSize=(tsize, tsize),
grid=(tgrid, tgrid),
innerSpace=1)
#timg = scipy.ndimage.zoom(timg, tzoom, order=0)
preViewPath = "%s/%s_resi.jpg" % (self.origPreViewDir, oImgPre)
Image.fromarray(timg).save(preViewPath)
'''
return
'''
fpar = 'sex_diff.par'
sexConf = [
'-DETECT_MINAREA', '3', '-DETECT_THRESH', '2.5',
'-ANALYSIS_THRESH', '2.5'
]
resiCat = self.tools.runSextractor(self.objTmpResi, self.tmpDir,
self.tmpDir, fpar, sexConf)
mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
self.tmpDir, resiCat, self.objectImgCatTrans, 1) #1 and 5
badPixProps2 = np.loadtxt("%s/%s" % (self.tmpDir, nmhFile))
tdata1 = np.loadtxt("%s/%s" % (self.tmpDir, mchFile))
tdata2 = np.loadtxt("%s/%s" % (self.tmpDir, self.objectImgCatTrans))
tIdx1 = np.loadtxt("%s/%s" % (self.tmpDir, mchPair)).astype(np.int)
tIdx1 = tIdx1 - 1
origData = tdata2[tIdx1[:, 1]]
if origData.shape[0] == tdata1.shape[0]:
outCatName = "%s_orgpos.cat" % (mchFile[:mchFile.index(".")])
outCatPath = "%s/%s" % (self.tmpDir, outCatName)
tstr = ""
i = 0
for td in tdata1:
tstr += "%.4f %.4f %.2f %.2f %.2f %.3f %.3f %.3f %.2f %.2f %d %.4f %.4f %.4f %.4f\n"%\
(td[0],td[1],td[2],td[3],td[4],td[5],td[6],td[7],td[8],td[9],td[10],origData[i][11], origData[i][12], origData[i][13], origData[i][14])
i = i + 1
fp0 = open(outCatPath, 'w')
fp0.write(tstr)
fp0.close()
mchFile = outCatName
else:
self.log.error("add orig pos error")
'''
self.tools.runSelfMatch(self.tmpDir, resiCat, 1) #debug: get ds9 reg file
tdata = np.loadtxt("%s/%s"%(self.tmpDir, resiCat))
self.log.info("resi image star %d"%(tdata.shape[0]))
'''
''' '''
mchRadius = 15 #15 10
mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
self.tmpDir, mchFile, self.templateImgCat, mchRadius)
fotProps = np.loadtxt("%s/%s" % (self.tmpDir, mchFile))
mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
self.tmpDir, nmhFile, self.badPixCat, 1) #1 and 5
os.system("cp %s/%s %s/%s" %
(self.tmpDir, nmhFile, self.resiCatDir, "%s.cat" %
(oImgPre)))
totProps = np.loadtxt("%s/%s" % (self.tmpDir, nmhFile))
#badPixProps = np.loadtxt("%s/%s"%(self.tmpDir, self.badPixCat))
badPixProps = np.array([])
tstr = "orgBadPix %d, nmBad %d, match %d, noMatch %d" % (
badPixProps.shape[0], badPixProps2.shape[0], fotProps.shape[0],
totProps.shape[0])
self.log.info(tstr)
#size = self.subImgSize
size = 68
if totProps.shape[0] < 500 and totProps.shape[0] > 0:
totSubImgs, totParms = getWindowImgs(self.tmpDir,
self.newImageName,
self.templateImg,
self.objTmpResi, totProps,
size)
if totParms.shape[0] > 0:
tXY = totParms[:, 0:2]
tRaDec = self.wcs.all_pix2world(tXY, 1)
totParms = np.concatenate((totParms, tRaDec), axis=1)
fotpath = '%s/%s_totimg.npz' % (self.destDir, oImgPre)
np.savez_compressed(fotpath, imgs=totSubImgs, parms=totParms)
resiImgs = []
for timg in totSubImgs:
resiImgs.append(timg[2])
preViewPath = "%s/%s_tot.jpg" % (self.preViewDir, oImgPre)
#if not os.path.exists(preViewPath):
psfView = genPSFView(resiImgs)
Image.fromarray(psfView).save(preViewPath)
if fotProps.shape[0] > 0 and fotProps.shape[0] < 2000:
fotSubImgs, fotParms = getWindowImgs(self.tmpDir,
self.newImageName,
self.templateImg,
self.objTmpResi, fotProps,
size)
if fotParms.shape[0] > 0:
tXY = fotParms[:, 0:2]
tRaDec = self.wcs.all_pix2world(tXY, 1)
fotParms = np.concatenate((fotParms, tRaDec), axis=1)
fotpath = '%s/%s_fotimg.npz' % (self.destDir, oImgPre)
np.savez_compressed(fotpath,
imgs=fotSubImgs,
parms=fotParms)
'''
if badPixProps.shape[0]>0:
badSubImgs, badParms = getWindowImgs(self.tmpDir, self.newImageName, self.templateImg, self.objTmpResi, badPixProps, size)
if badParms.shape[0]>0:
fotpath = '%s/%s_badimg.npz'%(self.destDir, oImgPre)
np.savez_compressed(fotpath, imgs=badSubImgs, parms=badParms)
if badPixProps2.shape[0]>0:
badSubImgs, badParms = getWindowImgs(self.tmpDir, self.newImageName, self.templateImg, self.objTmpResi, badPixProps2, size)
if badParms.shape[0]>0:
fotpath = '%s/%s_badimg2.npz'%(self.destDir, oImgPre)
np.savez_compressed(fotpath, imgs=badSubImgs, parms=badParms)
'''
resultFlag = True
else:
tmsgStr = "%s.fit resi image has %d tot objects, maybe wrong" % (
oImgPre, totProps.shape[0])
self.log.error(tmsgStr)
#self.sendMsg(tmsgStr)
resultFlag = False
'''
tgrid = 4
tsize = 500
tzoom = 1
timg = getThumbnail(self.tmpDir, self.objTmpResi, stampSize=(tsize,tsize), grid=(tgrid, tgrid), innerSpace = 1)
#timg = scipy.ndimage.zoom(timg, tzoom, order=0)
preViewPath = "%s/%s_resi.jpg"%(self.origPreViewDir, oImgPre)
Image.fromarray(timg).save(preViewPath)
'''
'''
timg = getThumbnail(self.tmpDir, self.newImageName, stampSize=(tsize,tsize), grid=(tgrid, tgrid), innerSpace = 1)
timg = scipy.ndimage.zoom(timg, tzoom, order=0)
preViewPath = "%s/%s_obj.jpg"%(self.origPreViewDir, oImgPre)
Image.fromarray(timg).save(preViewPath)
timg = getThumbnail(self.tmpDir, self.templateImg, stampSize=(tsize,tsize), grid=(tgrid, tgrid), innerSpace = 1)
timg = scipy.ndimage.zoom(timg, tzoom, order=0)
preViewPath = "%s/%s_tmp.jpg"%(self.origPreViewDir, oImgPre)
Image.fromarray(timg).save(preViewPath)
'''
endtime = datetime.now()
runTime = (endtime - starttime).seconds
self.log.info("********** image diff total use %d seconds" % (runTime))
return resultFlag
def diffImage(self, imgName, tmplParms):
starttime = datetime.now()
resultFlag = True
oImgPre = imgName.split(".")[0]
os.system("rm -rf %s/*" % (self.diff))
status = tmplParms[0]
#tmplImgName = tmplParms[1][-1][0]
tmplImgName = tmplParms[1][0][0]
tmplImgPre = tmplImgName.split(".")[0]
if status == '1':
ttmplPath = self.tmplAlignDir
else:
ttmplPath = self.tmplDiffDir
ofitPath = "%s/%s.fit" % (self.doDiffTmpl, tmplImgPre)
if not os.path.exists(ofitPath):
os.system("rm -rf %s/*" % (self.doDiffTmpl))
os.system("cp %s/%s.fit %s/%s.fit" %
(ttmplPath, tmplImgPre, self.doDiffTmpl, tmplImgPre))
os.system("cp %s/%s.cat %s/%s.cat" %
(ttmplPath, tmplImgPre, self.doDiffTmpl, tmplImgPre))
os.system("cp %s/%s_badpix.cat %s/%s_badpix.cat" %
(ttmplPath, tmplImgPre, self.doDiffTmpl, tmplImgPre))
os.system("cp %s/%s.fit %s/ti.fit" %
(self.doDiffTmpl, tmplImgPre, self.diff))
os.system("cp %s/%s.cat %s/ti.cat" %
(self.doDiffTmpl, tmplImgPre, self.diff))
os.system("cp %s/%s_badpix.cat %s/ti_badpix.cat" %
(self.doDiffTmpl, tmplImgPre, self.diff))
os.system("cp %s/%s.fit %s/oi.fit" % (self.cmbDir, oImgPre, self.diff))
os.system("cp %s/%s.cat %s/oi.cat" %
(self.cmbCatDir, oImgPre, self.diff))
theader = fits.getheader("%s/oi.fit" % (self.diff))
dateObs = theader['DATE-OBS']
timeObs = theader['TIME-OBS']
dtStr = "%sT%s" % (dateObs, timeObs)
objTmpResi, runSuccess = self.tools.runHotpants(
'oi.fit', 'ti.fit', self.diff)
if not runSuccess:
self.log.error("diffImage failure: %s" % (imgName))
return False
fpar = 'sex_diff.par'
#sexConf=['-DETECT_MINAREA','3','-DETECT_THRESH','2.5','-ANALYSIS_THRESH','2.5']
sexConf = [
'-DETECT_MINAREA', '5', '-DETECT_THRESH', '2.5',
'-ANALYSIS_THRESH', '2.5'
]
resiCat, isSuccess = self.tools.runSextractor(objTmpResi, self.diff,
self.diff, fpar, sexConf)
if not isSuccess:
self.log.error("diffImage runSextractor failure")
return isSuccess
os.system("cp %s/%s %s/%s_resi.fit" %
(self.diff, objTmpResi, self.diffImgDir, oImgPre))
os.system("cp %s/%s %s/%s_resi.cat" %
(self.diff, resiCat, self.diffCatDir, oImgPre))
mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
self.diff, resiCat, 'oi.cat', 1) #1 and 5
badPix2Path = "%s/%s" % (self.diff, nmhFile)
if os.path.exists(badPix2Path):
badPixProps2 = np.loadtxt(badPix2Path)
else:
badPixProps2 = np.array([])
mchRadius = 15 #15 10
mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
self.diff, mchFile, 'ti.cat', mchRadius)
fotPath = "%s/%s" % (self.diff, mchFile)
if os.path.exists(fotPath):
fotProps = np.loadtxt(fotPath)
else:
fotProps = np.array([])
mchFile, nmhFile, mchPair = self.tools.runCrossMatch(
self.diff, nmhFile, 'ti_badpix.cat', 1) #1 and 5
os.system("cp %s/%s %s/%s_tot.cat" %
(self.diff, nmhFile, self.diffCatDir, oImgPre))
totPath = "%s/%s" % (self.diff, nmhFile)
if os.path.exists(totPath):
totProps = np.loadtxt(totPath)
else:
totProps = np.array([])
badPixPath = "%s/ti_badpix.cat" % (self.diff)
if os.path.exists(badPixPath):
badPixProps = np.loadtxt(badPixPath)
else:
badPixProps = np.array([])
#badPixProps = np.array([])
tstr = "orgBadPix %d, nmBad %d, match %d, noMatch %d" % (
badPixProps.shape[0], badPixProps2.shape[0], fotProps.shape[0],
totProps.shape[0])
self.log.info(tstr)
print(tstr)
#size = self.subImgSize
size = 68
if totProps.shape[0] < 500 and totProps.shape[0] > 0:
wcsPath = "%s/%s.wcs" % (ttmplPath, tmplImgPre)
wcs = WCS(wcsPath)
totSubImgs, totParms = getWindowImgs(self.diff, 'oi.fit', 'ti.fit',
objTmpResi, totProps, size)
if totParms.shape[0] > 0:
tXY = totParms[:, 0:2]
#tdates = np.repeat(dtStr,tXY.shape[0]).reshape((tXY.shape[0],1))
tRaDec = wcs.all_pix2world(tXY, 1)
totParms = np.concatenate((totParms, tRaDec), axis=1)
fotpath = '%s/%s_totimg.npz' % (self.destDir, oImgPre)
np.savez_compressed(fotpath,
imgs=totSubImgs,
parms=totParms,
obsUtc=dtStr)
resiImgs = []
for timg in totSubImgs:
resiImgs.append(timg[2])
preViewPath = "%s/%s_tot.jpg" % (self.subImgViewDir, oImgPre)
#if not os.path.exists(preViewPath):
psfView = genPSFView(resiImgs)
Image.fromarray(psfView).save(preViewPath)
if fotProps.shape[0] > 0 and fotProps.shape[0] < 3000:
fotSubImgs, fotParms = getWindowImgs(self.diff, 'oi.fit',
'ti.fit', objTmpResi,
fotProps, size)
if fotParms.shape[0] > 0:
tXY = fotParms[:, 0:2]
#tdates = np.repeat(dtStr,tXY.shape[0]).reshape((tXY.shape[0],1))
tRaDec = wcs.all_pix2world(tXY, 1)
fotParms = np.concatenate((fotParms, tRaDec), axis=1)
fotpath = '%s/%s_fotimg.npz' % (self.destDir, oImgPre)
np.savez_compressed(fotpath,
imgs=fotSubImgs,
parms=fotParms,
obsUtc=dtStr)
'''
if badPixProps.shape[0]>0:
badSubImgs, badParms = getWindowImgs(self.diff, 'oi.fit', 'ti.fit', objTmpResi, badPixProps, size)
if badParms.shape[0]>0:
fotpath = '%s/%s_badimg.npz'%(self.destDir, oImgPre)
np.savez_compressed(fotpath, imgs=badSubImgs, parms=badParms)
if badPixProps2.shape[0]>0:
badSubImgs, badParms = getWindowImgs(self.diff, 'oi.fit', 'ti.fit', objTmpResi, badPixProps2, size)
if badParms.shape[0]>0:
tXY = badParms[:,0:2]
tRaDec = wcs.all_pix2world(tXY, 1)
badParms = np.concatenate((badParms, tRaDec), axis=1)
fotpath = '%s/%s_badimg2.npz'%(self.destDir, oImgPre)
np.savez_compressed(fotpath, imgs=badSubImgs, parms=badParms, obsUtc=dtStr)
'''
resultFlag = True
else:
tmsgStr = "%s.fit resi image has %d tot objects, maybe wrong" % (
oImgPre, totProps.shape[0])
self.log.error(tmsgStr)
#self.sendMsg(tmsgStr)
resultFlag = False
''' '''
tgrid = 4
tsize = 500
tzoom = 1
timg = getThumbnail(self.diff,
objTmpResi,
stampSize=(tsize, tsize),
grid=(tgrid, tgrid),
innerSpace=1)
timg = scipy.ndimage.zoom(timg, tzoom, order=0)
preViewPath = "%s/%s_resi.jpg" % (self.origPreViewDir, oImgPre)
Image.fromarray(timg).save(preViewPath)
'''
'''
timg = getThumbnail(self.diff,
'oi.fit',
stampSize=(tsize, tsize),
grid=(tgrid, tgrid),
innerSpace=1)
timg = scipy.ndimage.zoom(timg, tzoom, order=0)
preViewPath = "%s/%s_obj.jpg" % (self.origPreViewDir, oImgPre)
Image.fromarray(timg).save(preViewPath)
timg = getThumbnail(self.diff,
'ti.fit',
stampSize=(tsize, tsize),
grid=(tgrid, tgrid),
innerSpace=1)
timg = scipy.ndimage.zoom(timg, tzoom, order=0)
preViewPath = "%s/%s_tmp.jpg" % (self.origPreViewDir, oImgPre)
Image.fromarray(timg).save(preViewPath)
endtime = datetime.now()
runTime = (endtime - starttime).seconds
self.log.info("********** image diff total use %d seconds" % (runTime))
return resultFlag