def diffImage(self, imgName, tmplParms): starttime = datetime.now() resultFlag = True oImgPre = imgName.split(".")[0] dateStr = imgName.split('_')[3][:6] os.system("rm -rf %s/*" % (self.diff)) status = tmplParms[0] #tmplImgName = tmplParms[1][-1][0] tmplImgName = tmplParms[1][0][0] tmplImgPre = tmplImgName.split(".")[0] if status == '1': ttmplPath = self.tmplAlignDir else: ttmplPath = self.tmplDiffDir ofitPath = "%s/%s.fit" % (self.doDiffTmpl, tmplImgPre) if not os.path.exists(ofitPath): os.system("rm -rf %s/*" % (self.doDiffTmpl)) os.system("cp %s/%s.fit %s/%s.fit" % (ttmplPath, tmplImgPre, self.doDiffTmpl, tmplImgPre)) os.system("cp %s/%s.cat %s/%s.cat" % (ttmplPath, tmplImgPre, self.doDiffTmpl, tmplImgPre)) os.system("cp %s/%s_badpix.cat %s/%s_badpix.cat" % (ttmplPath, tmplImgPre, self.doDiffTmpl, tmplImgPre)) os.system("cp %s/%s.fit %s/ti.fit" % (self.doDiffTmpl, tmplImgPre, self.diff)) os.system("cp %s/%s.cat %s/ti.cat" % (self.doDiffTmpl, tmplImgPre, self.diff)) os.system("cp %s/%s_badpix.cat %s/ti_badpix.cat" % (self.doDiffTmpl, tmplImgPre, self.diff)) os.system("cp %s/%s_align.fit %s/oi.fit" % (self.alignDir, oImgPre, self.diff)) os.system("cp %s/%s_align.cat %s/oi.cat" % (self.alignDir, oImgPre, self.diff)) theader = fits.getheader("%s/oi.fit" % (self.diff)) dateObs = theader['DATE-OBS'] timeObs = theader['TIME-OBS'] dtStr = "%sT%s" % (dateObs, timeObs) objTmpResi, runSuccess = self.tools.runHotpants( 'oi.fit', 'ti.fit', self.diff) if not runSuccess: self.log.error("diffImage failure: %s" % (imgName)) return False fpar = 'sex_diff.par' #sexConf=['-DETECT_MINAREA','3','-DETECT_THRESH','2.5','-ANALYSIS_THRESH','2.5'] sexConf = [ '-DETECT_MINAREA', '5', '-DETECT_THRESH', '2.5', '-ANALYSIS_THRESH', '2.5' ] resiCat, isSuccess = self.tools.runSextractor(objTmpResi, self.diff, self.diff, fpar, sexConf) if not isSuccess: self.log.error("diffImage runSextractor failure") return isSuccess mchFile, nmhFile, mchPair = self.tools.runCrossMatch( self.diff, resiCat, 'oi.cat', 1) #1 and 5 badPix2Path = "%s/%s" % (self.diff, nmhFile) if os.path.exists(badPix2Path): badPixProps2 = np.loadtxt(badPix2Path) else: badPixProps2 = np.array([]) mchRadius = 10 #15 10 mchFile, nmhFile, mchPair = self.tools.runCrossMatch( self.diff, mchFile, 'ti.cat', mchRadius) fotPath = "%s/%s" % (self.diff, mchFile) if os.path.exists(fotPath): fotProps = np.loadtxt(fotPath) else: fotProps = np.array([]) mchFile, nmhFile, mchPair = self.tools.runCrossMatch( self.diff, nmhFile, 'ti_badpix.cat', 1) #1 and 5 os.system("cp %s/%s %s/%s_tot.cat" % (self.diff, nmhFile, self.diffCatDir, oImgPre)) totPath = "%s/%s" % (self.diff, nmhFile) if os.path.exists(totPath): totProps = np.loadtxt(totPath) else: totProps = np.array([]) badPixPath = "%s/ti_badpix.cat" % (self.diff) if os.path.exists(badPixPath): badPixProps = np.loadtxt(badPixPath) else: badPixProps = np.array([]) #badPixProps = np.array([]) tstr = "orgBadPix %d, nmBad %d, match %d, noMatch %d" % ( badPixProps.shape[0], badPixProps2.shape[0], fotProps.shape[0], totProps.shape[0]) self.log.info(tstr) print(tstr) #size = self.subImgSize size = 68 if totProps.shape[0] < 500 and totProps.shape[0] > 0: wcsPath = "%s/%s.wcs" % (ttmplPath, tmplImgPre) wcs = WCS(wcsPath) searchRadius = 0.05 mpQuery = QueryMinorPlanet() mpQuery.connDb() totSubImgs, totParms = getWindowImgs(self.diff, 'oi.fit', 'ti.fit', objTmpResi, totProps, size) if totParms.shape[0] > 0: tXY = totParms[:, 0:2] #tdates = np.repeat(dtStr,tXY.shape[0]).reshape((tXY.shape[0],1)) tRaDec = wcs.all_pix2world(tXY, 1) totParms = np.concatenate((totParms, tRaDec), axis=1) mchDis = [] for tpos in tRaDec: ra = tpos[0] dec = tpos[1] tdis, tmag = mpQuery.matchMP(ra, dec, dateStr, dtStr, searchRadius + 0.001) mchDis.append([tdis, tmag]) mchDis = np.array(mchDis) totParms = np.concatenate((totParms, mchDis), axis=1) totSubImgs1 = totSubImgs[mchDis[:, 0] >= searchRadius] totParms1 = totParms[mchDis[:, 0] >= searchRadius] mpSubImgs1 = totSubImgs[mchDis[:, 0] < searchRadius] mpParms1 = totParms[mchDis[:, 0] < searchRadius] tstr0 = "%s has %d tot, contain %d minorplanet." % ( imgName, totParms.shape[0], mpParms1.shape[0]) print(tstr0) self.log.info(tstr0) if totSubImgs1.shape[0] > 0: totpath = '%s/%s_totimg.npz' % (self.destDir, oImgPre) np.savez_compressed(totpath, imgs=totSubImgs1, parms=totParms1, obsUtc=dtStr) resiImgs = [] for timg in totSubImgs1: resiImgs.append(timg[2]) preViewPath = "%s/%s_tot.jpg" % (self.subImgViewDir, oImgPre) #if not os.path.exists(preViewPath): psfView = genPSFView(resiImgs) Image.fromarray(psfView).save(preViewPath) if mpSubImgs1.shape[0] > 0: self.sendMsg(tstr0) totpath = '%s/%s_mpimgt.npz' % (self.destDir, oImgPre) np.savez_compressed(totpath, imgs=mpSubImgs1, parms=mpParms1, obsUtc=dtStr) resiImgs = [] for timg in mpSubImgs1: resiImgs.append(timg[2]) preViewPath = "%s/%s_mpt.jpg" % (self.subImgViewDir, oImgPre) #if not os.path.exists(preViewPath): psfView = genPSFView(resiImgs) Image.fromarray(psfView).save(preViewPath) os.system( "cp %s/%s %s/%s_resi.fit" % (self.diff, objTmpResi, self.diffImgDir, oImgPre)) os.system("cp %s/%s %s/%s_resi.cat" % (self.diff, resiCat, self.diffCatDir, oImgPre)) else: alignImgName = "%s/%s_align.fit" % (self.alignDir, oImgPre) os.system("rm -rf %s" % (alignImgName)) alignImgName = "%s/%s_align.cat" % (self.alignDir, oImgPre) os.system("rm -rf %s" % (alignImgName)) alignImgName = "%s/%s.cat" % (self.catDir, oImgPre) os.system("rm -rf %s" % (alignImgName)) if fotProps.shape[0] > 0 and fotProps.shape[0] < 500: fotSubImgs, fotParms = getWindowImgs(self.diff, 'oi.fit', 'ti.fit', objTmpResi, fotProps, size) if fotParms.shape[0] > 0: tXY = fotParms[:, 0:2] #tdates = np.repeat(dtStr,tXY.shape[0]).reshape((tXY.shape[0],1)) tRaDec = wcs.all_pix2world(tXY, 1) fotParms = np.concatenate((fotParms, tRaDec), axis=1) if fotSubImgs.shape[0] > 0: totpath = '%s/%s_fotimg.npz' % (self.destDir, oImgPre) np.savez_compressed(totpath, imgs=fotSubImgs, parms=fotParms, obsUtc=dtStr) mpQuery.closeDb() resultFlag = True else: tmsgStr = "%s.fit resi image has %d tot objects, maybe wrong" % ( oImgPre, totProps.shape[0]) self.log.error(tmsgStr) #self.sendMsg(tmsgStr) resultFlag = False endtime = datetime.now() runTime = (endtime - starttime).seconds self.log.info("********** image diff total use %d seconds" % (runTime)) return resultFlag
def doClassifyAndUpload(self, subImgPath, totFile, fotFile, fullImgPath, newImg, tmpImg, resImg, origName, serverIP, prob=0.01, maxNEllip=0.6, maxMEllip=0.5, reverse=False): self.log.debug("start new thread classifyAndUpload %s" % (origName)) starttime = datetime.now() self.pbb_threshold = prob try: nameBase = origName[:origName.index(".")] tParms1 = self.doClassifyFile(subImgPath, totFile) if tParms1.shape[0] > 0: tParms1 = tParms1[tParms1[:, 6] < maxNEllip] #tParms1 = tParms1[(tParms1[:,6]<maxMEllip) & (tParms1[:,17]>=prob)] if tParms1.shape[0] > 0: tflags1 = np.ones((tParms1.shape[0], 1)) #OT FLAG tParms1 = np.concatenate((tParms1, tflags1), axis=1) tParms2 = self.doClassifyFile(subImgPath, fotFile) if tParms2.shape[0] > 0: tParms2 = tParms2[(tParms2[:, 6] < maxMEllip) & (tParms2[:, 17] >= prob)] if tParms2.shape[0] > 0: tflags2 = np.zeros((tParms2.shape[0], 1)) #OT FLAG tParms2 = np.concatenate((tParms2, tflags2), axis=1) if tParms1.shape[0] > 0 and tParms2.shape[0] > 0 and tParms2.shape[ 0] < 50: tParms = np.concatenate((tParms1, tParms2), axis=0) elif tParms1.shape[0] > 0: tParms = tParms1 elif tParms2.shape[0] > 0 and tParms2.shape[0] < 25: tParms = tParms2 else: tParms = np.array([]) if tParms.shape[0] > 0: tSubImgs, tParms = getWindowImgs(fullImgPath, newImg, tmpImg, resImg, tParms, 100) if tParms.shape[0] > 0: self.log.info( "after classified, %s total get %d sub images" % (origName, tSubImgs.shape[0])) i = 1 timgNames = [] for timg in tSubImgs: objWid, tmpWid, resiWid = timg[0], timg[1], timg[2] objWidz = zscale_image(objWid) tmpWidz = zscale_image(tmpWid) resiWidz = zscale_image(resiWid) objWidz = scipy.ndimage.zoom(objWidz, 2, order=0) tmpWidz = scipy.ndimage.zoom(tmpWidz, 2, order=0) resiWidz = scipy.ndimage.zoom(resiWidz, 2, order=0) objWidz = objWidz.reshape(objWidz.shape[0], objWidz.shape[1], 1).repeat(3, 2) tmpWidz = tmpWidz.reshape(tmpWidz.shape[0], tmpWidz.shape[1], 1).repeat(3, 2) resiWidz = resiWidz.reshape(resiWidz.shape[0], resiWidz.shape[1], 1).repeat(3, 2) shift00 = 3 cv2.circle(objWidz, (int(objWidz.shape[0] / 2) - shift00, int(objWidz.shape[1] / 2) - shift00), 10, (0, 255, 0), thickness=2) cv2.circle(tmpWidz, (int(tmpWidz.shape[0] / 2) - shift00, int(tmpWidz.shape[1] / 2) - shift00), 10, (0, 255, 0), thickness=2) cv2.circle(resiWidz, (int(resiWidz.shape[0] / 2) - shift00, int(resiWidz.shape[1] / 2) - shift00), 10, (0, 255, 0), thickness=2) spaceLine = np.ones( (objWidz.shape[0], 5), dtype=np.uint8) * 255 spaceLine = spaceLine.reshape(spaceLine.shape[0], spaceLine.shape[1], 1).repeat(3, 2) sub2Con = np.concatenate( (objWidz, spaceLine, tmpWidz, spaceLine, resiWidz), axis=1) tImgName = "%s_%05d.jpg" % (nameBase, i) timgNames.append(tImgName) savePath = "%s/%s" % (fullImgPath, tImgName) Image.fromarray(sub2Con).save(savePath) i = i + 1 catName = tImgName = "%s.cat" % (nameBase) catPath = "%s/%s" % (fullImgPath, catName) fp0 = open(catPath, 'w') #fp0.write(self.theader) i = 0 for td in tParms: tstr = "%.4f,%.4f,%.2f,%.2f,%.2f,%.3f,%.3f,%.3f,%.2f,%.2f,%d,%.4f,%.4f,%.4f,%.4f,%f,%f,%.3f,%d,%s\n"%\ (td[0],td[1],td[2],td[3],td[4],td[5],td[6],td[7],td[8],td[9],td[10],td[11],td[12],td[13], td[14],td[15],td[16],td[17],td[18], timgNames[i]) fp0.write(tstr) i = i + 1 fp0.close() self.doUpload(fullImgPath, [catName], 'diffot1', serverIP) self.doUpload(fullImgPath, timgNames, 'diffot1img', serverIP) if tParms.shape[0] == 0: self.log.info("after classified, no OT candidate left") if tParms2.shape[0] >= 50: self.log.error( "too more matched OT candidate, skip upload matched to db: after classified, %s total get %d matchend sub images" % (origName, tParms2.shape[0])) os.system("rm -rf %s" % (fullImgPath)) except Exception as e: self.log.error('classifyAndUpload error') self.log.error(str(e)) tstr = traceback.format_exc() self.log.error(tstr) endtime = datetime.now() runTime = (endtime - starttime).seconds self.log.info("********** classifyAndUpload %s use %d seconds" % (origName, runTime))
def doClassifyAndUpload(self, subImgPath, totFile, fotFile, fullImgPath, newImg, tmpImg, resImg, origName, serverIP, runName, skyName, tcatParm, prob=0.0000001, maxNEllip=0.6, maxMEllip=0.5, reverse=False): self.log.info("start new thread classifyAndUpload %s" % (origName)) print("start new thread classifyAndUpload %s" % (origName)) minFwhm = 1.5 maxFwhm = 3.0 starttime = datetime.now() self.pbb_threshold = prob try: nameBase = origName[:origName.index(".")] dateStr = nameBase.split('_')[3][:6] camName = nameBase[:4] cmbNum = 5 #tidx = nameBase.index('_c')+2 #cmbNum = nameBase[tidx:tidx+3] #'G021_tom_objt_190109T13531492_c005.fit' crossTaskName = "%s_%s_%s_C%03d" % (dateStr, camName, skyName, cmbNum) if runName != 'p1': crossTaskName = "%s_%s" % (crossTaskName, runName) #self.log.info("crossTaskName %s"%(crossTaskName)) #print("crossTaskName %s"%(crossTaskName)) tParms1t = np.array([]) tParms2t = np.array([]) tParms1, obsUtc1 = self.doClassifyFile(subImgPath, totFile) #print("doClassifyAndUpload 001") if tParms1.shape[0] > 0: tParms1t = tParms1[(tParms1[:, 6] < maxNEllip) & (tParms1[:, 9] < maxFwhm) & (tParms1[:, 9] > minFwhm)] #tParms1t = tParms1[(tParms1[:,6]<maxMEllip) & (tParms1[:,15]>=prob)] if tParms1t.shape[0] > 0: tflags1t = np.ones((tParms1t.shape[0], 1)) #OT FLAG tParms1t = np.concatenate((tParms1t, tflags1t), axis=1) #print("doClassifyAndUpload 002") tParms2, obsUtc2 = self.doClassifyFile(subImgPath, fotFile) if tParms2.shape[0] > 0: #tParms2t = tParms2[(tParms2[:,6]<maxMEllip) & (tParms2[:,15]>=prob)&(tParms2[:,9]<maxFwhm)&(tParms2[:,9]>minFwhm)] tParms2t = tParms2[(tParms2[:, 6] < maxMEllip) & (tParms2[:, 9] < maxFwhm) & (tParms2[:, 9] > minFwhm)] if tParms2t.shape[0] > 0: tflags2t = np.zeros((tParms2t.shape[0], 1)) #OT FLAG tParms2t = np.concatenate((tParms2t, tflags2t), axis=1) ''' tParms3t = np.array([]) badotFile = fotFile.replace('fotimg', 'badimg2') tParms3, obsUtc3 = self.doClassifyFile(subImgPath, badotFile) if tParms3.shape[0]>0: tParms3t = tParms3[(tParms3[:,6]<maxMEllip) & (tParms3[:,15]>=prob)] if tParms3t.shape[0]>0: tflags3t = np.zeros((tParms3t.shape[0],1)) #OT FLAG tParms3t = np.concatenate((tParms3t, tflags3t), axis=1) ''' print("classify result: %d tot(%d), %d fot(%d)" % (tParms1t.shape[0], tParms1.shape[0], tParms2t.shape[0], tParms2.shape[0])) #print("doClassifyAndUpload 003") if tParms1t.shape[0] > 0 and tParms1t.shape[ 0] < 100 and tParms2t.shape[0] > 0 and tParms2t.shape[ 0] < 100: tParms = np.concatenate((tParms1t, tParms2t), axis=0) elif tParms1t.shape[0] > 0 and tParms1t.shape[0] < 100: tParms = tParms1t elif tParms2t.shape[0] > 0 and tParms2t.shape[0] < 100: tParms = tParms2t else: tParms = np.array([]) #print("doClassifyAndUpload 004") if tParms.shape[0] == 0: #print("doClassifyAndUpload 005") self.log.info("after classified, no OT candidate left") elif tParms.shape[0] > 0: #print("doClassifyAndUpload 006") tSubImgs, tParms = getWindowImgs(fullImgPath, newImg, tmpImg, resImg, tParms, 100) if tParms.shape[0] > 0: self.log.info( "after classified, %s total get %d sub images" % (origName, tSubImgs.shape[0])) #print("after classified, %s total get %d sub images"%(origName, tSubImgs.shape[0])) i = 1 timgNames = [] for timg in tSubImgs: objWid, tmpWid, resiWid = timg[0], timg[1], timg[2] objWidz = zscale_image(objWid) tmpWidz = zscale_image(tmpWid) resiWidz = zscale_image(resiWid) objWidz = scipy.ndimage.zoom(objWidz, 2, order=0) tmpWidz = scipy.ndimage.zoom(tmpWidz, 2, order=0) resiWidz = scipy.ndimage.zoom(resiWidz, 2, order=0) objWidz = objWidz.reshape(objWidz.shape[0], objWidz.shape[1], 1).repeat(3, 2) tmpWidz = tmpWidz.reshape(tmpWidz.shape[0], tmpWidz.shape[1], 1).repeat(3, 2) resiWidz = resiWidz.reshape(resiWidz.shape[0], resiWidz.shape[1], 1).repeat(3, 2) shift00 = 3 cv2.circle(objWidz, (int(objWidz.shape[0] / 2) - shift00, int(objWidz.shape[1] / 2) - shift00), 10, (0, 255, 0), thickness=2) cv2.circle(tmpWidz, (int(tmpWidz.shape[0] / 2) - shift00, int(tmpWidz.shape[1] / 2) - shift00), 10, (0, 255, 0), thickness=2) cv2.circle(resiWidz, (int(resiWidz.shape[0] / 2) - shift00, int(resiWidz.shape[1] / 2) - shift00), 10, (0, 255, 0), thickness=2) spaceLine = np.ones( (objWidz.shape[0], 5), dtype=np.uint8) * 255 spaceLine = spaceLine.reshape(spaceLine.shape[0], spaceLine.shape[1], 1).repeat(3, 2) sub2Con = np.concatenate( (objWidz, spaceLine, tmpWidz, spaceLine, resiWidz), axis=1) tstrs = tmpImg.split( '_') #G044_mon_objt_190128T10264248.fit if len(tstrs) == 4: tmpDateTime = tstrs[3] cv2.putText( sub2Con, #numpy array on which text is written tmpDateTime[:tmpDateTime.index('.')], #text (objWidz.shape[1] + 25, objWidz.shape[0] - 10), #position at which writing has to start cv2.FONT_HERSHEY_SIMPLEX, #font family 0.5, #font size (0, 255, 0), #font color 2) tImgName = "%s_%05d.jpg" % (nameBase, i) timgNames.append(tImgName) savePath = "%s/%s" % (fullImgPath, tImgName) Image.fromarray(sub2Con).save(savePath) i = i + 1 catName = tImgName = "%s.cat" % (nameBase) catPath = "%s/%s" % (fullImgPath, catName) fp0 = open(catPath, 'w') fp0.write(self.theader2) t2oX = tcatParm[-2] t2oY = tcatParm[-1] tx = tParms[:, 0] ty = tParms[:, 1] ox = t2oX(tx, ty) oy = t2oY(tx, ty) i = 0 for td in tParms: #print(td) #print(td[15]) tstr = self.catFormate%\ (ox[i],oy[i],td[2],td[3],td[4],td[5],td[6],td[7],td[8],td[9],td[10],td[11],td[12],td[13], td[14],td[15],td[16], timgNames[i], obsUtc1, td[0],td[1]) fp0.write(tstr) i = i + 1 fp0.close() self.doUpload(fullImgPath, [catName], 'crossOTList', serverIP, crossTaskName) self.doUpload(fullImgPath, timgNames, 'crossOTStamp', serverIP, crossTaskName) if tParms1.shape[0] >= 100 or tParms2.shape[0] >= 100: self.log.error( "too more unmatched or matched OT candidate, skip upload matched to db: after classified, %s total get %d matchend sub images" % (origName, tParms2.shape[0])) os.system("rm -rf %s" % (fullImgPath)) except Exception as e: self.log.error('classifyAndUpload error') self.log.error(str(e)) tstr = traceback.format_exc() self.log.error(tstr) endtime = datetime.now() runTime = (endtime - starttime).seconds self.log.info("********** classifyAndUpload %s use %d seconds" % (origName, runTime))
def diff(self, srcDir, destDir, tmpFit, objFits, reverse=False): try: if not os.path.exists(destDir): os.system("mkdir -p %s" % (destDir)) self.regImg2DB(srcDir, tmpFit) os.system("rm -rf %s/*" % (self.templateDir)) os.system("cp %s/%s %s/%s" % (srcDir, tmpFit, self.templateDir, self.templateImg)) self.makeTemplate() for i, imgName in enumerate(objFits): starttime = datetime.now() self.log.info("diff %d: %s" % (i, imgName)) self.regImg2DB(srcDir, imgName) os.system("rm -rf %s/*" % (self.tmpDir)) imgpre = imgName.split(".")[0] tobjFitsFullPath = "%s/%s.fit" % (srcDir, imgpre) if not os.path.exists(tobjFitsFullPath): self.log.error("%s.fit not exist, stop" % (imgpre)) break if os.path.exists("%s/diffResi/%s.fit" % (destDir, imgpre)): self.log.info("%s.fit already diffed, skip" % (imgpre)) continue os.system("cp %s/%s.fit %s/%s" % (srcDir, imgpre, self.tmpDir, self.objectImg)) os.system("cp %s/%s %s/%s" % (self.templateDir, self.templateImg, self.tmpDir, self.templateImg)) os.system("cp %s/%s %s/%s" % (self.templateDir, self.badPixCat, self.tmpDir, self.badPixCat)) self.objTmpResi, runSuccess = self.tools.runHotpants( self.objectImg, self.templateImg, self.tmpDir) if not runSuccess: self.log.info("%s.fit diff error..." % (imgpre)) continue if reverse: os.system("cp %s/%s %s/diffResi/%s_r.fit" % (self.tmpDir, self.objTmpResi, destDir, imgpre)) else: os.system("cp %s/%s %s/diffResi/%s.fit" % (self.tmpDir, self.objTmpResi, destDir, imgpre)) tgrid = 4 tsize = 500 tzoom = 2 timg = getThumbnail(self.tmpDir, self.objTmpResi, stampSize=(tsize, tsize), grid=(tgrid, tgrid), innerSpace=1) timg = scipy.ndimage.zoom(timg, tzoom, order=0) if reverse: preViewPath = "%s/preview/%s_resi_r.jpg" % (destDir, imgpre) else: preViewPath = "%s/preview/%s_resi.jpg" % (destDir, imgpre) Image.fromarray(timg).save(preViewPath) fpar = 'sex_diff.par' sexConf = [ '-DETECT_MINAREA', '3', '-DETECT_THRESH', '2.5', '-ANALYSIS_THRESH', '2.5' ] self.objectImgCat = self.tools.runSextractor( self.objectImg, self.tmpDir, self.tmpDir, fpar, sexConf) resiCat = self.tools.runSextractor(self.objTmpResi, self.tmpDir, self.tmpDir, fpar, sexConf) mchFile, nmhFile, mchPair = self.tools.runCrossMatch( self.tmpDir, resiCat, self.objectImgCat, 1) #1 and 5 badPixProps2 = np.loadtxt("%s/%s" % (self.tmpDir, nmhFile)) mchFile, nmhFile, mchPair = self.tools.runCrossMatch( self.tmpDir, mchFile, self.badPixCat, 1) #1 and 5 badPixProps = np.loadtxt("%s/%s" % (self.tmpDir, self.badPixCat)) resiCatTrans = self.getRaDec(nmhFile) objProps = np.loadtxt("%s/%s" % (self.tmpDir, resiCatTrans)) tstr = "%s, resi objs %d, orgBadPix %d, nmBad %d" % ( imgName, objProps.shape[0], badPixProps.shape[0], badPixProps2.shape[0]) self.log.info(tstr) #size = self.subImgSize size = 68 if objProps.shape[0] < 20000 and objProps.shape[0] > 0: totSubImgs, totParms = getWindowImgs( self.tmpDir, self.objectImg, self.templateImg, self.objTmpResi, objProps, size) if totParms.shape[0] > 0: if reverse: fotpath = '%s/subImgs/%s_r.npz' % (destDir, imgpre) else: fotpath = '%s/subImgs/%s.npz' % (destDir, imgpre) np.savez_compressed(fotpath, imgs=totSubImgs, parms=totParms) self.classifyAndUpload(destDir, imgName, reverse=reverse) endtime = datetime.now() runTime = (endtime - starttime).seconds self.log.info("diff: %s use %d seconds" % (imgName, runTime)) #break except Exception as e: print(str(e)) tstr = traceback.format_exc() print(tstr)
def diffImage(self): starttime = datetime.now() resultFlag = True oImgPre = self.origObjectImg[:self.origObjectImg.index(".")] os.system("cp %s/%s %s/%s" % (self.templateDir, self.templateImg, self.tmpDir, self.templateImg)) os.system( "cp %s/%s %s/%s" % (self.templateDir, self.badPixCat, self.tmpDir, self.badPixCat)) os.system("cp %s/%s %s/%s" % (self.templateDir, self.templateImgCat, self.tmpDir, self.templateImgCat)) #self.newImageName = self.tools.imageAlign(self.tmpDir, self.objectImg, self.transHG) self.objTmpResi, runSuccess = self.tools.runHotpants( self.newImageName, self.templateImg, self.tmpDir) if not runSuccess: return False os.system("cp %s/%s %s/%s.fit" % (self.tmpDir, self.objTmpResi, self.resiFitsDir, oImgPre)) tgrid = 1 tsize = 4096 tzoom = 1 timg = getThumbnail(self.tmpDir, self.objTmpResi, stampSize=(tsize, tsize), grid=(tgrid, tgrid), innerSpace=1) #timg = scipy.ndimage.zoom(timg, tzoom, order=0) preViewPath = "%s/%s_resi.jpg" % (self.origPreViewDir, oImgPre) Image.fromarray(timg).save(preViewPath) ''' return ''' fpar = 'sex_diff.par' sexConf = [ '-DETECT_MINAREA', '3', '-DETECT_THRESH', '2.5', '-ANALYSIS_THRESH', '2.5' ] resiCat = self.tools.runSextractor(self.objTmpResi, self.tmpDir, self.tmpDir, fpar, sexConf) mchFile, nmhFile, mchPair = self.tools.runCrossMatch( self.tmpDir, resiCat, self.objectImgCatTrans, 1) #1 and 5 badPixProps2 = np.loadtxt("%s/%s" % (self.tmpDir, nmhFile)) tdata1 = np.loadtxt("%s/%s" % (self.tmpDir, mchFile)) tdata2 = np.loadtxt("%s/%s" % (self.tmpDir, self.objectImgCatTrans)) tIdx1 = np.loadtxt("%s/%s" % (self.tmpDir, mchPair)).astype(np.int) tIdx1 = tIdx1 - 1 origData = tdata2[tIdx1[:, 1]] if origData.shape[0] == tdata1.shape[0]: outCatName = "%s_orgpos.cat" % (mchFile[:mchFile.index(".")]) outCatPath = "%s/%s" % (self.tmpDir, outCatName) tstr = "" i = 0 for td in tdata1: tstr += "%.4f %.4f %.2f %.2f %.2f %.3f %.3f %.3f %.2f %.2f %d %.4f %.4f %.4f %.4f\n"%\ (td[0],td[1],td[2],td[3],td[4],td[5],td[6],td[7],td[8],td[9],td[10],origData[i][11], origData[i][12], origData[i][13], origData[i][14]) i = i + 1 fp0 = open(outCatPath, 'w') fp0.write(tstr) fp0.close() mchFile = outCatName else: self.log.error("add orig pos error") ''' self.tools.runSelfMatch(self.tmpDir, resiCat, 1) #debug: get ds9 reg file tdata = np.loadtxt("%s/%s"%(self.tmpDir, resiCat)) self.log.info("resi image star %d"%(tdata.shape[0])) ''' ''' ''' mchRadius = 15 #15 10 mchFile, nmhFile, mchPair = self.tools.runCrossMatch( self.tmpDir, mchFile, self.templateImgCat, mchRadius) fotProps = np.loadtxt("%s/%s" % (self.tmpDir, mchFile)) mchFile, nmhFile, mchPair = self.tools.runCrossMatch( self.tmpDir, nmhFile, self.badPixCat, 1) #1 and 5 os.system("cp %s/%s %s/%s" % (self.tmpDir, nmhFile, self.resiCatDir, "%s.cat" % (oImgPre))) totProps = np.loadtxt("%s/%s" % (self.tmpDir, nmhFile)) #badPixProps = np.loadtxt("%s/%s"%(self.tmpDir, self.badPixCat)) badPixProps = np.array([]) tstr = "orgBadPix %d, nmBad %d, match %d, noMatch %d" % ( badPixProps.shape[0], badPixProps2.shape[0], fotProps.shape[0], totProps.shape[0]) self.log.info(tstr) #size = self.subImgSize size = 68 if totProps.shape[0] < 500 and totProps.shape[0] > 0: totSubImgs, totParms = getWindowImgs(self.tmpDir, self.newImageName, self.templateImg, self.objTmpResi, totProps, size) if totParms.shape[0] > 0: tXY = totParms[:, 0:2] tRaDec = self.wcs.all_pix2world(tXY, 1) totParms = np.concatenate((totParms, tRaDec), axis=1) fotpath = '%s/%s_totimg.npz' % (self.destDir, oImgPre) np.savez_compressed(fotpath, imgs=totSubImgs, parms=totParms) resiImgs = [] for timg in totSubImgs: resiImgs.append(timg[2]) preViewPath = "%s/%s_tot.jpg" % (self.preViewDir, oImgPre) #if not os.path.exists(preViewPath): psfView = genPSFView(resiImgs) Image.fromarray(psfView).save(preViewPath) if fotProps.shape[0] > 0 and fotProps.shape[0] < 2000: fotSubImgs, fotParms = getWindowImgs(self.tmpDir, self.newImageName, self.templateImg, self.objTmpResi, fotProps, size) if fotParms.shape[0] > 0: tXY = fotParms[:, 0:2] tRaDec = self.wcs.all_pix2world(tXY, 1) fotParms = np.concatenate((fotParms, tRaDec), axis=1) fotpath = '%s/%s_fotimg.npz' % (self.destDir, oImgPre) np.savez_compressed(fotpath, imgs=fotSubImgs, parms=fotParms) ''' if badPixProps.shape[0]>0: badSubImgs, badParms = getWindowImgs(self.tmpDir, self.newImageName, self.templateImg, self.objTmpResi, badPixProps, size) if badParms.shape[0]>0: fotpath = '%s/%s_badimg.npz'%(self.destDir, oImgPre) np.savez_compressed(fotpath, imgs=badSubImgs, parms=badParms) if badPixProps2.shape[0]>0: badSubImgs, badParms = getWindowImgs(self.tmpDir, self.newImageName, self.templateImg, self.objTmpResi, badPixProps2, size) if badParms.shape[0]>0: fotpath = '%s/%s_badimg2.npz'%(self.destDir, oImgPre) np.savez_compressed(fotpath, imgs=badSubImgs, parms=badParms) ''' resultFlag = True else: tmsgStr = "%s.fit resi image has %d tot objects, maybe wrong" % ( oImgPre, totProps.shape[0]) self.log.error(tmsgStr) #self.sendMsg(tmsgStr) resultFlag = False ''' tgrid = 4 tsize = 500 tzoom = 1 timg = getThumbnail(self.tmpDir, self.objTmpResi, stampSize=(tsize,tsize), grid=(tgrid, tgrid), innerSpace = 1) #timg = scipy.ndimage.zoom(timg, tzoom, order=0) preViewPath = "%s/%s_resi.jpg"%(self.origPreViewDir, oImgPre) Image.fromarray(timg).save(preViewPath) ''' ''' timg = getThumbnail(self.tmpDir, self.newImageName, stampSize=(tsize,tsize), grid=(tgrid, tgrid), innerSpace = 1) timg = scipy.ndimage.zoom(timg, tzoom, order=0) preViewPath = "%s/%s_obj.jpg"%(self.origPreViewDir, oImgPre) Image.fromarray(timg).save(preViewPath) timg = getThumbnail(self.tmpDir, self.templateImg, stampSize=(tsize,tsize), grid=(tgrid, tgrid), innerSpace = 1) timg = scipy.ndimage.zoom(timg, tzoom, order=0) preViewPath = "%s/%s_tmp.jpg"%(self.origPreViewDir, oImgPre) Image.fromarray(timg).save(preViewPath) ''' endtime = datetime.now() runTime = (endtime - starttime).seconds self.log.info("********** image diff total use %d seconds" % (runTime)) return resultFlag
def diffImage(self, imgName, tmplParms): starttime = datetime.now() resultFlag = True oImgPre = imgName.split(".")[0] os.system("rm -rf %s/*" % (self.diff)) status = tmplParms[0] #tmplImgName = tmplParms[1][-1][0] tmplImgName = tmplParms[1][0][0] tmplImgPre = tmplImgName.split(".")[0] if status == '1': ttmplPath = self.tmplAlignDir else: ttmplPath = self.tmplDiffDir ofitPath = "%s/%s.fit" % (self.doDiffTmpl, tmplImgPre) if not os.path.exists(ofitPath): os.system("rm -rf %s/*" % (self.doDiffTmpl)) os.system("cp %s/%s.fit %s/%s.fit" % (ttmplPath, tmplImgPre, self.doDiffTmpl, tmplImgPre)) os.system("cp %s/%s.cat %s/%s.cat" % (ttmplPath, tmplImgPre, self.doDiffTmpl, tmplImgPre)) os.system("cp %s/%s_badpix.cat %s/%s_badpix.cat" % (ttmplPath, tmplImgPre, self.doDiffTmpl, tmplImgPre)) os.system("cp %s/%s.fit %s/ti.fit" % (self.doDiffTmpl, tmplImgPre, self.diff)) os.system("cp %s/%s.cat %s/ti.cat" % (self.doDiffTmpl, tmplImgPre, self.diff)) os.system("cp %s/%s_badpix.cat %s/ti_badpix.cat" % (self.doDiffTmpl, tmplImgPre, self.diff)) os.system("cp %s/%s.fit %s/oi.fit" % (self.cmbDir, oImgPre, self.diff)) os.system("cp %s/%s.cat %s/oi.cat" % (self.cmbCatDir, oImgPre, self.diff)) theader = fits.getheader("%s/oi.fit" % (self.diff)) dateObs = theader['DATE-OBS'] timeObs = theader['TIME-OBS'] dtStr = "%sT%s" % (dateObs, timeObs) objTmpResi, runSuccess = self.tools.runHotpants( 'oi.fit', 'ti.fit', self.diff) if not runSuccess: self.log.error("diffImage failure: %s" % (imgName)) return False fpar = 'sex_diff.par' #sexConf=['-DETECT_MINAREA','3','-DETECT_THRESH','2.5','-ANALYSIS_THRESH','2.5'] sexConf = [ '-DETECT_MINAREA', '5', '-DETECT_THRESH', '2.5', '-ANALYSIS_THRESH', '2.5' ] resiCat, isSuccess = self.tools.runSextractor(objTmpResi, self.diff, self.diff, fpar, sexConf) if not isSuccess: self.log.error("diffImage runSextractor failure") return isSuccess os.system("cp %s/%s %s/%s_resi.fit" % (self.diff, objTmpResi, self.diffImgDir, oImgPre)) os.system("cp %s/%s %s/%s_resi.cat" % (self.diff, resiCat, self.diffCatDir, oImgPre)) mchFile, nmhFile, mchPair = self.tools.runCrossMatch( self.diff, resiCat, 'oi.cat', 1) #1 and 5 badPix2Path = "%s/%s" % (self.diff, nmhFile) if os.path.exists(badPix2Path): badPixProps2 = np.loadtxt(badPix2Path) else: badPixProps2 = np.array([]) mchRadius = 15 #15 10 mchFile, nmhFile, mchPair = self.tools.runCrossMatch( self.diff, mchFile, 'ti.cat', mchRadius) fotPath = "%s/%s" % (self.diff, mchFile) if os.path.exists(fotPath): fotProps = np.loadtxt(fotPath) else: fotProps = np.array([]) mchFile, nmhFile, mchPair = self.tools.runCrossMatch( self.diff, nmhFile, 'ti_badpix.cat', 1) #1 and 5 os.system("cp %s/%s %s/%s_tot.cat" % (self.diff, nmhFile, self.diffCatDir, oImgPre)) totPath = "%s/%s" % (self.diff, nmhFile) if os.path.exists(totPath): totProps = np.loadtxt(totPath) else: totProps = np.array([]) badPixPath = "%s/ti_badpix.cat" % (self.diff) if os.path.exists(badPixPath): badPixProps = np.loadtxt(badPixPath) else: badPixProps = np.array([]) #badPixProps = np.array([]) tstr = "orgBadPix %d, nmBad %d, match %d, noMatch %d" % ( badPixProps.shape[0], badPixProps2.shape[0], fotProps.shape[0], totProps.shape[0]) self.log.info(tstr) print(tstr) #size = self.subImgSize size = 68 if totProps.shape[0] < 500 and totProps.shape[0] > 0: wcsPath = "%s/%s.wcs" % (ttmplPath, tmplImgPre) wcs = WCS(wcsPath) totSubImgs, totParms = getWindowImgs(self.diff, 'oi.fit', 'ti.fit', objTmpResi, totProps, size) if totParms.shape[0] > 0: tXY = totParms[:, 0:2] #tdates = np.repeat(dtStr,tXY.shape[0]).reshape((tXY.shape[0],1)) tRaDec = wcs.all_pix2world(tXY, 1) totParms = np.concatenate((totParms, tRaDec), axis=1) fotpath = '%s/%s_totimg.npz' % (self.destDir, oImgPre) np.savez_compressed(fotpath, imgs=totSubImgs, parms=totParms, obsUtc=dtStr) resiImgs = [] for timg in totSubImgs: resiImgs.append(timg[2]) preViewPath = "%s/%s_tot.jpg" % (self.subImgViewDir, oImgPre) #if not os.path.exists(preViewPath): psfView = genPSFView(resiImgs) Image.fromarray(psfView).save(preViewPath) if fotProps.shape[0] > 0 and fotProps.shape[0] < 3000: fotSubImgs, fotParms = getWindowImgs(self.diff, 'oi.fit', 'ti.fit', objTmpResi, fotProps, size) if fotParms.shape[0] > 0: tXY = fotParms[:, 0:2] #tdates = np.repeat(dtStr,tXY.shape[0]).reshape((tXY.shape[0],1)) tRaDec = wcs.all_pix2world(tXY, 1) fotParms = np.concatenate((fotParms, tRaDec), axis=1) fotpath = '%s/%s_fotimg.npz' % (self.destDir, oImgPre) np.savez_compressed(fotpath, imgs=fotSubImgs, parms=fotParms, obsUtc=dtStr) ''' if badPixProps.shape[0]>0: badSubImgs, badParms = getWindowImgs(self.diff, 'oi.fit', 'ti.fit', objTmpResi, badPixProps, size) if badParms.shape[0]>0: fotpath = '%s/%s_badimg.npz'%(self.destDir, oImgPre) np.savez_compressed(fotpath, imgs=badSubImgs, parms=badParms) if badPixProps2.shape[0]>0: badSubImgs, badParms = getWindowImgs(self.diff, 'oi.fit', 'ti.fit', objTmpResi, badPixProps2, size) if badParms.shape[0]>0: tXY = badParms[:,0:2] tRaDec = wcs.all_pix2world(tXY, 1) badParms = np.concatenate((badParms, tRaDec), axis=1) fotpath = '%s/%s_badimg2.npz'%(self.destDir, oImgPre) np.savez_compressed(fotpath, imgs=badSubImgs, parms=badParms, obsUtc=dtStr) ''' resultFlag = True else: tmsgStr = "%s.fit resi image has %d tot objects, maybe wrong" % ( oImgPre, totProps.shape[0]) self.log.error(tmsgStr) #self.sendMsg(tmsgStr) resultFlag = False ''' ''' tgrid = 4 tsize = 500 tzoom = 1 timg = getThumbnail(self.diff, objTmpResi, stampSize=(tsize, tsize), grid=(tgrid, tgrid), innerSpace=1) timg = scipy.ndimage.zoom(timg, tzoom, order=0) preViewPath = "%s/%s_resi.jpg" % (self.origPreViewDir, oImgPre) Image.fromarray(timg).save(preViewPath) ''' ''' timg = getThumbnail(self.diff, 'oi.fit', stampSize=(tsize, tsize), grid=(tgrid, tgrid), innerSpace=1) timg = scipy.ndimage.zoom(timg, tzoom, order=0) preViewPath = "%s/%s_obj.jpg" % (self.origPreViewDir, oImgPre) Image.fromarray(timg).save(preViewPath) timg = getThumbnail(self.diff, 'ti.fit', stampSize=(tsize, tsize), grid=(tgrid, tgrid), innerSpace=1) timg = scipy.ndimage.zoom(timg, tzoom, order=0) preViewPath = "%s/%s_tmp.jpg" % (self.origPreViewDir, oImgPre) Image.fromarray(timg).save(preViewPath) endtime = datetime.now() runTime = (endtime - starttime).seconds self.log.info("********** image diff total use %d seconds" % (runTime)) return resultFlag
def diffImage(self): starttime = datetime.now() resultFlag = True oImgPre = self.origObjectImg[:self.origObjectImg.index(".")] os.system("cp %s/%s %s/%s" % (self.templateDir, self.templateImg, self.tmpDir, self.templateImg)) os.system( "cp %s/%s %s/%s" % (self.templateDir, self.badPixCat, self.tmpDir, self.badPixCat)) os.system("cp %s/%s %s/%s" % (self.templateDir, self.templateImgCat, self.tmpDir, self.templateImgCat)) #self.newImageName = self.tools.imageAlign(self.tmpDir, self.objectImg, self.transHG) self.objTmpResi = self.tools.runHotpants(self.newImageName, self.templateImg, self.tmpDir) fpar = 'sex_diff.par' sexConf = [ '-DETECT_MINAREA', '3', '-DETECT_THRESH', '2.5', '-ANALYSIS_THRESH', '2.5' ] resiCat = self.tools.runSextractor(self.objTmpResi, self.tmpDir, self.tmpDir, fpar, sexConf) ''' self.tools.runSelfMatch(self.tmpDir, resiCat, 1) #debug: get ds9 reg file tdata = np.loadtxt("%s/%s"%(self.tmpDir, resiCat)) self.log.info("resi image star %d"%(tdata.shape[0])) ''' ''' ''' mchRadius = 15 #15 10 mchFile, nmhFile, mchPair = self.tools.runCrossMatch( self.tmpDir, resiCat, self.templateImgCat, mchRadius) fotProps = np.loadtxt("%s/%s" % (self.tmpDir, mchFile)) mchFile, nmhFile, mchPair = self.tools.runCrossMatch( self.tmpDir, nmhFile, self.badPixCat, 1) #1 and 5 os.system("cp %s/%s %s/%s" % (self.tmpDir, nmhFile, self.resiCatDir, "%s.cat" % (oImgPre))) totProps = np.loadtxt("%s/%s" % (self.tmpDir, nmhFile)) #badPixProps = np.loadtxt("%s/%s"%(self.tmpDir, mchFile)) badPixProps = np.loadtxt("%s/%s" % (self.tmpDir, self.badPixCat)) tstr = "badPix %d, match %d, noMatch %d" % ( badPixProps.shape[0], fotProps.shape[0], totProps.shape[0]) self.log.info(tstr) size = self.subImgSize if totProps.shape[0] < 500 and totProps.shape[0] > 0: totSubImgs, totParms = getWindowImgs(self.tmpDir, self.newImageName, self.templateImg, self.objTmpResi, totProps, size) tXY = totParms[:, 0:2] tRaDec = self.wcs.all_pix2world(tXY, 1) totParms = np.concatenate((totParms, tRaDec), axis=1) fotpath = '%s/%s_totimg.npz' % (self.destDir, oImgPre) np.savez_compressed(fotpath, imgs=totSubImgs, parms=totParms) resiImgs = [] for timg in totSubImgs: resiImgs.append(timg[2]) preViewPath = "%s/%s_tot.jpg" % (self.preViewDir, oImgPre) #if not os.path.exists(preViewPath): psfView = genPSFView(resiImgs) Image.fromarray(psfView).save(preViewPath) if fotProps.shape[0] > 0 and fotProps.shape[0] < 2000: fotSubImgs, fotParms = getWindowImgs(self.tmpDir, self.newImageName, self.templateImg, self.objTmpResi, fotProps, size) tXY = fotParms[:, 0:2] tRaDec = self.wcs.all_pix2world(tXY, 1) fotParms = np.concatenate((fotParms, tRaDec), axis=1) fotpath = '%s/%s_fotimg.npz' % (self.destDir, oImgPre) np.savez_compressed(fotpath, imgs=fotSubImgs, parms=fotParms) if badPixProps.shape[0] > 0: badSubImgs, badParms = getWindowImgs(self.tmpDir, self.newImageName, self.templateImg, self.objTmpResi, badPixProps, size) fotpath = '%s/%s_badimg.npz' % (self.destDir, oImgPre) np.savez_compressed(fotpath, imgs=badSubImgs, parms=badParms) resultFlag = True else: tmsgStr = "%s.fit resi image has %d tot objects, maybe wrong" % ( oImgPre, totProps.shape[0]) self.log.error(tmsgStr) self.tools.sendTriggerMsg(tmsgStr) resultFlag = False tgrid = 4 tsize = 500 tzoom = 1 timg = getThumbnail(self.tmpDir, self.objTmpResi, stampSize=(tsize, tsize), grid=(tgrid, tgrid), innerSpace=1) #timg = scipy.ndimage.zoom(timg, tzoom, order=0) preViewPath = "%s/%s_resi.jpg" % (self.origPreViewDir, oImgPre) Image.fromarray(timg).save(preViewPath) ''' timg = getThumbnail(self.tmpDir, self.newImageName, stampSize=(tsize,tsize), grid=(tgrid, tgrid), innerSpace = 1) timg = scipy.ndimage.zoom(timg, tzoom, order=0) preViewPath = "%s/%s_obj.jpg"%(self.origPreViewDir, oImgPre) Image.fromarray(timg).save(preViewPath) timg = getThumbnail(self.tmpDir, self.templateImg, stampSize=(tsize,tsize), grid=(tgrid, tgrid), innerSpace = 1) timg = scipy.ndimage.zoom(timg, tzoom, order=0) preViewPath = "%s/%s_tmp.jpg"%(self.origPreViewDir, oImgPre) Image.fromarray(timg).save(preViewPath) ''' endtime = datetime.now() runTime = (endtime - starttime).seconds self.log.info("********** image diff total use %d seconds" % (runTime)) return resultFlag