def srcExtract(self, camName, catList, imgDiff, logDestDir, runName): self.catRunning = 1 query = QueryData() tfiles = query.getFileList(camName, self.curFFId) print("secExtract: get %d images" % (len(tfiles))) for tfile in tfiles: try: curFfId = tfile[0] ffNumber = tfile[1] curSkyId = tfile[2] timgName = tfile[3] #G021_tom_objt_190109T13531492.fit tpath = tfile[ 4] #/data3/G002_021_190109/G021_tom_objt_190109T13531492.fit #print("srcExtract: %s"%(tpath)) srcDir = tpath[:(tpath.find(camName) - 1)] #/data3/G002_021_190109 dateStr = srcDir[srcDir.find('G'):] #G002_021_190109 logfName0 = '%s/%s_%s.log' % (logDestDir, dateStr, runName) if self.curFFId == 0: if os.path.exists( logfName0) and os.stat(logfName0).st_size > 0: tlastLine = getLastLine(logfName0) if len(tlastLine) > 2: self.curFFId = int(tlastLine.strip()) if curFfId > self.curFFId: self.curFFId = curFfId logfile0 = open(logfName0, 'a') logfile0.write("\n\n%d\n" % (self.curFFId)) logfile0.close() tpathfz = "%s.fz" % (tpath) if os.path.exists(tpath) or os.path.exists(tpathfz): isSuccess, skyName, starNum, fwhmMean, bgMean = imgDiff.getCat( srcDir, timgName, imgDiff.catDir) if isSuccess: #we can add more filter condition, to remove bad images catList.append([ isSuccess, self.curFFId, timgName, starNum, skyName, fwhmMean, bgMean, curSkyId, srcDir ]) print("srcExtract %d: %s success." % (len(catList), tpath)) else: print("srcExtract %d: %s failure." % (len(catList), tpath)) else: print("getCat: %s not exist" % (tpath)) except Exception as e: tstr = traceback.format_exc() imgDiff.log.error(tstr) self.catRunning = 0
def srcExtract(self, camName, catList, tmplMap, imgDiff, logDestDir, runName): self.catRunning = 1 if self.curFFId==0: tdirs = os.listdir(logDestDir) tdirs.sort() for logfile in tdirs: tlogpath="%s/%s"%(logDestDir, logfile) if os.path.exists(tlogpath) and os.stat(tlogpath).st_size > 0: tlastLine = getLastLine(tlogpath) if len(tlastLine)>2: tffid = int(tlastLine.strip()) if self.curFFId<tffid: self.curFFId=tffid query = QueryData() tfiles = query.getFileList(camName, self.curFFId) tnum = len(tfiles) tstr = "%s, secExtract: get %d images, ffId=%d, %s"%(camName, tnum, tfiles[0][0], tfiles[0][3]) print(tstr) if tnum>0: if self.loopNum%40==1: imgDiff.sendMsg(tstr) for tfile in tfiles: try: curFfId = tfile[0] ffNumber = tfile[1] curSkyId = tfile[2] timgName = tfile[3] #G021_tom_objt_190109T13531492.fit tpath = tfile[4] #/data3/G002_021_190109/G021_tom_objt_190109T13531492.fit skyName = tfile[5] print("srcExtract: %s"%(tpath)) srcDir= tpath[:(tpath.find(camName)-1)] #/data3/G002_021_190109 dateStr = srcDir[srcDir.find('G'):] #G002_021_190109 logfName0 = '%s/%s_%s.log'%(logDestDir, dateStr, runName) if curFfId>self.curFFId: self.curFFId=curFfId logfile0 = open(logfName0, 'a') logfile0.write("\n\n%d\n"%(self.curFFId)) logfile0.close() if skyName in tmplMap: tparms = tmplMap[skyName] #['2', [],1] if tparms[0]!='1': print("srcExtract: %s, no history template, skip"%(tpath)) continue tpathfz = "%s.fz"%(tpath) if os.path.exists(tpath) or os.path.exists(tpathfz): isSuccess, skyName, starNum, fwhmMean, bgMean = imgDiff.getCat(srcDir, timgName, imgDiff.catDir) if isSuccess: #we can add more filter condition, to remove bad images catList.append([isSuccess, self.curFFId, timgName, starNum, skyName, fwhmMean, bgMean, curSkyId, srcDir]) print("srcExtract %d: %s success."%(len(catList), tpath)) else: print("srcExtract %d: %s failure."%(len(catList), tpath)) else: print("getCat: %s not exist"%(tpath)) except Exception as e: tstr = traceback.format_exc() imgDiff.log.error(tstr) self.catRunning = 0
def srcExtract(camName, catList, imgDiff, logDestDir, isRunning): isRunning.value = 1 catNum = len(catList) if catNum > 0: lastCat = catList[catNum - 1] ffId = lastCat[1] else: ffId = 0 query = QueryData() tfiles = query.getFileList(camName, ffId) #print(tfiles) for tfile in tfiles: try: curFfId = tfile[0] ffNumber = tfile[1] curSkyId = tfile[2] timgName = tfile[3] #G021_tom_objt_190109T13531492.fit tpath = tfile[ 4] #/data3/G002_021_190109/G021_tom_objt_190109T13531492.fit print("srcExtract: %s" % (tpath)) srcDir = tpath[:(tpath.find(camName) - 1)] #/data3/G002_021_190109 dateStr = srcDir[srcDir.find('G'):] #G002_021_190109 logfName0 = '%s/%s.log' % (logDestDir, dateStr) if ffId == 0: if os.path.exists( logfName0) and os.stat(logfName0).st_size > 0: tlastLine = getLastLine(logfName0) if len(tlastLine) > 2: ffId = int(tlastLine.strip()) if curFfId > ffId: ffId = curFfId logfile0 = open(logfName0, 'a') logfile0.write("\n\n%d\n" % (ffId)) logfile0.close() tpathfz = "%s.fz" % (tpath) if os.path.exists(tpath) or os.path.exists(tpathfz): starttime = datetime.now() isSuccess, skyName, starNum, fwhmMean, bgMean = imgDiff.getCat( srcDir, timgName, imgDiff.catDir) if isSuccess: #we can add more filter condition, to remove bad images catList.append([ isSuccess, ffId, timgName, starNum, skyName, fwhmMean, bgMean, curSkyId, srcDir ]) endtime = datetime.now() runTime = (endtime - starttime).seconds imgDiff.log.info( "totalTime %d seconds, sky:%d, ffNum:%d, %s" % (runTime, curSkyId, ffNumber, timgName)) else: imgDiff.log.error("getCat: %s not exist" % (tpath)) except Exception as e: tstr = traceback.format_exc() imgDiff.log.error(tstr) isRunning.value = 0
def srcExtractLocalDir(tpath, camName, catList, imgDiff, logDestDir, isRunning): isRunning.value = 1 #print("srcExtractLocalDir: catNum=%d"%(len(catList))) ffId = 0 tfiles = os.listdir(tpath) tfiles.sort() #print("total read %d files"%(len(tfiles))) runIdx = 1 for i, tfile in enumerate(tfiles): if runIdx > 10: break try: curFfId = i + 1 ffNumber = 100 curSkyId = 0 timgName = tfile #G021_tom_objt_190109T13531492.fit srcDir = tpath #/data3/G002_021_190109 /home/xy/work/imgDiffTest2/fits/190128_G004_044 dateStr = srcDir[srcDir.find('G'):] #G002_021_190109 logfName0 = '%s/%s.log' % (logDestDir, dateStr) if ffId == 0: if os.path.exists( logfName0) and os.stat(logfName0).st_size > 0: tlastLine = getLastLine(logfName0) #print("tlastLine %s"%(tlastLine)) if len(tlastLine) >= 2: ffId = int(tlastLine.strip()) #print("read ffId %d"%(ffId)) if curFfId > ffId: runIdx = runIdx + 1 ffId = curFfId logfile0 = open(logfName0, 'a') logfile0.write("\n\n%d\n" % (ffId)) logfile0.close() tpathfz = "%s.fz" % (tpath) if os.path.exists(tpath) or os.path.exists(tpathfz): starttime = datetime.now() isSuccess, skyName, starNum, fwhmMean, bgMean = imgDiff.getCat( srcDir, timgName, imgDiff.catDir) if isSuccess: #we can add more filter condition, to remove bad images catList.append([ isSuccess, ffId, timgName, starNum, skyName, fwhmMean, bgMean, curSkyId, srcDir ]) endtime = datetime.now() runTime = (endtime - starttime).seconds imgDiff.log.info( "srcExtractLocalDir, totalTime %d seconds, sky:%d, ffNum:%d, %s" % (runTime, curSkyId, ffNumber, timgName)) else: imgDiff.log.error("getCat: %s not exist" % (tpath)) except Exception as e: tstr = traceback.format_exc() imgDiff.log.error(tstr) isRunning.value = 0
def run1(camName): #toolPath = os.getcwd() toolPath = '/home/gwac/img_diff_xy/image_diff' tools = AstroTools(toolPath) query = QueryData() dataDest0 = "/data/gwac_diff_xy/data" logDest0 = "/data/gwac_diff_xy/log" if not os.path.exists(dataDest0): os.system("mkdir -p %s"%(dataDest0)) if not os.path.exists(logDest0): os.system("mkdir -p %s"%(logDest0)) startProcess = False dayRun = 0 nigRun = 0 skyId = 0 ffId = 0 tfiles = [] while True: curUtcDateTime = datetime.utcnow() tDateTime = datetime.utcnow() startDateTime = tDateTime.replace(hour=9, minute=30, second=0) #9=17 1=9 endDateTime = tDateTime.replace(hour=22, minute=30, second=0) #22=6 8=16 remainSeconds1 = (startDateTime - curUtcDateTime).total_seconds() remainSeconds2 = (endDateTime - curUtcDateTime).total_seconds() if remainSeconds1<0 and remainSeconds2>0: dayRun = 0 try: tfiles = query.getFileList(camName, ffId) #print(tfiles) for tfile in tfiles: curFfId = tfile[0] ffNumber = tfile[1] curSkyId = tfile[2] timgName = tfile[3] #G021_tom_objt_190109T13531492.fit tpath = tfile[4] #/data3/G002_021_190109/G021_tom_objt_190109T13531492.fit imgDate = timgName[14:20] pathDate = tpath[16:22] if imgDate!=pathDate: ffId=curFfId startProcess = False continue elif not startProcess: ffId=0 startProcess = True srcDir= tpath[:(tpath.find(camName)-1)] #/data3/G002_021_190109 dateStr = srcDir[srcDir.find('G'):] #G002_021_190109 logfName0 = '%s/%s.log'%(logDest0, dateStr) if ffId==0: if os.path.exists(logfName0) and os.stat(logfName0).st_size > 0: tlastLine = getLastLine(logfName0) if len(tlastLine)>2: ffId=int(tlastLine.strip()) if skyId!=curSkyId and curFfId>ffId: dstDir='%s/%s'%(dataDest0, dateStr) tdiff = BatchImageDiff(srcDir, dstDir, tools, camName, curSkyId) tStr = "start diff: %s"%(timgName) tdiff.log.info(tStr) tdiff.sendMsg(tStr) if curFfId>ffId: skyId=curSkyId ffId=curFfId logfile0 = open(logfName0, 'a') logfile0.write("\n\n%d\n"%(ffId)) logfile0.close() tpathfz = "%s.fz"%(tpath) if os.path.exists(tpath) or os.path.exists(tpathfz): starttime = datetime.now() tdiff.processImg(srcDir, timgName, ffNumber) endtime = datetime.now() runTime = (endtime - starttime).seconds tdiff.log.info("totalTime %d seconds, sky:%d, ffNum:%d, %s"%(runTime, curSkyId, ffNumber, timgName)) else: print("%s not exist"%(tpath)) #if curFfId>ffId: # break except Exception as e: print(str(e)) tstr = traceback.format_exc() print(tstr) try: if 'tdiff' in locals(): tStr = "diff error" tdiff.log.info(tStr) tdiff.sendMsg(tStr) except Exception as e1: print(str(e1)) tstr = traceback.format_exc() print(tstr) if len(tfiles)==0: time.sleep(5) nigRun = nigRun+1 #if nigRun>=1: # break else: # day temp file clean try: if ('tdiff' in locals()) and (dayRun==0): tdiff.initReg(0) except Exception as e1: print(str(e1)) tstr = traceback.format_exc() print(tstr) nigRun = 0 dayRun = dayRun+1 skyId = 0 ffId = 0 startProcess = False if dayRun%6==1: print("day %d wait"%(dayRun)) time.sleep(10*60)