def srcExtract(self, camName, catList, imgDiff, logDestDir, runName):
self.catRunning = 1
query = QueryData()
tfiles = query.getFileList(camName, self.curFFId)
print("secExtract: get %d images" % (len(tfiles)))
for tfile in tfiles:
try:
curFfId = tfile[0]
ffNumber = tfile[1]
curSkyId = tfile[2]
timgName = tfile[3] #G021_tom_objt_190109T13531492.fit
tpath = tfile[
4] #/data3/G002_021_190109/G021_tom_objt_190109T13531492.fit
#print("srcExtract: %s"%(tpath))
srcDir = tpath[:(tpath.find(camName) -
1)] #/data3/G002_021_190109
dateStr = srcDir[srcDir.find('G'):] #G002_021_190109
logfName0 = '%s/%s_%s.log' % (logDestDir, dateStr, runName)
if self.curFFId == 0:
if os.path.exists(
logfName0) and os.stat(logfName0).st_size > 0:
tlastLine = getLastLine(logfName0)
if len(tlastLine) > 2:
self.curFFId = int(tlastLine.strip())
if curFfId > self.curFFId:
self.curFFId = curFfId
logfile0 = open(logfName0, 'a')
logfile0.write("\n\n%d\n" % (self.curFFId))
logfile0.close()
tpathfz = "%s.fz" % (tpath)
if os.path.exists(tpath) or os.path.exists(tpathfz):
isSuccess, skyName, starNum, fwhmMean, bgMean = imgDiff.getCat(
srcDir, timgName, imgDiff.catDir)
if isSuccess: #we can add more filter condition, to remove bad images
catList.append([
isSuccess, self.curFFId, timgName, starNum,
skyName, fwhmMean, bgMean, curSkyId, srcDir
])
print("srcExtract %d: %s success." %
(len(catList), tpath))
else:
print("srcExtract %d: %s failure." %
(len(catList), tpath))
else:
print("getCat: %s not exist" % (tpath))
except Exception as e:
tstr = traceback.format_exc()
imgDiff.log.error(tstr)
self.catRunning = 0
def srcExtract(self, camName, catList, tmplMap, imgDiff, logDestDir, runName):
self.catRunning = 1
if self.curFFId==0:
tdirs = os.listdir(logDestDir)
tdirs.sort()
for logfile in tdirs:
tlogpath="%s/%s"%(logDestDir, logfile)
if os.path.exists(tlogpath) and os.stat(tlogpath).st_size > 0:
tlastLine = getLastLine(tlogpath)
if len(tlastLine)>2:
tffid = int(tlastLine.strip())
if self.curFFId<tffid:
self.curFFId=tffid
query = QueryData()
tfiles = query.getFileList(camName, self.curFFId)
tnum = len(tfiles)
tstr = "%s, secExtract: get %d images, ffId=%d, %s"%(camName, tnum, tfiles[0][0], tfiles[0][3])
print(tstr)
if tnum>0:
if self.loopNum%40==1:
imgDiff.sendMsg(tstr)
for tfile in tfiles:
try:
curFfId = tfile[0]
ffNumber = tfile[1]
curSkyId = tfile[2]
timgName = tfile[3] #G021_tom_objt_190109T13531492.fit
tpath = tfile[4] #/data3/G002_021_190109/G021_tom_objt_190109T13531492.fit
skyName = tfile[5]
print("srcExtract: %s"%(tpath))
srcDir= tpath[:(tpath.find(camName)-1)] #/data3/G002_021_190109
dateStr = srcDir[srcDir.find('G'):] #G002_021_190109
logfName0 = '%s/%s_%s.log'%(logDestDir, dateStr, runName)
if curFfId>self.curFFId:
self.curFFId=curFfId
logfile0 = open(logfName0, 'a')
logfile0.write("\n\n%d\n"%(self.curFFId))
logfile0.close()
if skyName in tmplMap:
tparms = tmplMap[skyName] #['2', [],1]
if tparms[0]!='1':
print("srcExtract: %s, no history template, skip"%(tpath))
continue
tpathfz = "%s.fz"%(tpath)
if os.path.exists(tpath) or os.path.exists(tpathfz):
isSuccess, skyName, starNum, fwhmMean, bgMean = imgDiff.getCat(srcDir, timgName, imgDiff.catDir)
if isSuccess: #we can add more filter condition, to remove bad images
catList.append([isSuccess, self.curFFId, timgName, starNum, skyName, fwhmMean, bgMean, curSkyId, srcDir])
print("srcExtract %d: %s success."%(len(catList), tpath))
else:
print("srcExtract %d: %s failure."%(len(catList), tpath))
else:
print("getCat: %s not exist"%(tpath))
except Exception as e:
tstr = traceback.format_exc()
imgDiff.log.error(tstr)
self.catRunning = 0
def srcExtract(camName, catList, imgDiff, logDestDir, isRunning):
isRunning.value = 1
catNum = len(catList)
if catNum > 0:
lastCat = catList[catNum - 1]
ffId = lastCat[1]
else:
ffId = 0
query = QueryData()
tfiles = query.getFileList(camName, ffId)
#print(tfiles)
for tfile in tfiles:
try:
curFfId = tfile[0]
ffNumber = tfile[1]
curSkyId = tfile[2]
timgName = tfile[3] #G021_tom_objt_190109T13531492.fit
tpath = tfile[
4] #/data3/G002_021_190109/G021_tom_objt_190109T13531492.fit
print("srcExtract: %s" % (tpath))
srcDir = tpath[:(tpath.find(camName) - 1)] #/data3/G002_021_190109
dateStr = srcDir[srcDir.find('G'):] #G002_021_190109
logfName0 = '%s/%s.log' % (logDestDir, dateStr)
if ffId == 0:
if os.path.exists(
logfName0) and os.stat(logfName0).st_size > 0:
tlastLine = getLastLine(logfName0)
if len(tlastLine) > 2:
ffId = int(tlastLine.strip())
if curFfId > ffId:
ffId = curFfId
logfile0 = open(logfName0, 'a')
logfile0.write("\n\n%d\n" % (ffId))
logfile0.close()
tpathfz = "%s.fz" % (tpath)
if os.path.exists(tpath) or os.path.exists(tpathfz):
starttime = datetime.now()
isSuccess, skyName, starNum, fwhmMean, bgMean = imgDiff.getCat(
srcDir, timgName, imgDiff.catDir)
if isSuccess: #we can add more filter condition, to remove bad images
catList.append([
isSuccess, ffId, timgName, starNum, skyName,
fwhmMean, bgMean, curSkyId, srcDir
])
endtime = datetime.now()
runTime = (endtime - starttime).seconds
imgDiff.log.info(
"totalTime %d seconds, sky:%d, ffNum:%d, %s" %
(runTime, curSkyId, ffNumber, timgName))
else:
imgDiff.log.error("getCat: %s not exist" % (tpath))
except Exception as e:
tstr = traceback.format_exc()
imgDiff.log.error(tstr)
isRunning.value = 0
def srcExtractLocalDir(tpath, camName, catList, imgDiff, logDestDir,
isRunning):
isRunning.value = 1
#print("srcExtractLocalDir: catNum=%d"%(len(catList)))
ffId = 0
tfiles = os.listdir(tpath)
tfiles.sort()
#print("total read %d files"%(len(tfiles)))
runIdx = 1
for i, tfile in enumerate(tfiles):
if runIdx > 10:
break
try:
curFfId = i + 1
ffNumber = 100
curSkyId = 0
timgName = tfile #G021_tom_objt_190109T13531492.fit
srcDir = tpath #/data3/G002_021_190109 /home/xy/work/imgDiffTest2/fits/190128_G004_044
dateStr = srcDir[srcDir.find('G'):] #G002_021_190109
logfName0 = '%s/%s.log' % (logDestDir, dateStr)
if ffId == 0:
if os.path.exists(
logfName0) and os.stat(logfName0).st_size > 0:
tlastLine = getLastLine(logfName0)
#print("tlastLine %s"%(tlastLine))
if len(tlastLine) >= 2:
ffId = int(tlastLine.strip())
#print("read ffId %d"%(ffId))
if curFfId > ffId:
runIdx = runIdx + 1
ffId = curFfId
logfile0 = open(logfName0, 'a')
logfile0.write("\n\n%d\n" % (ffId))
logfile0.close()
tpathfz = "%s.fz" % (tpath)
if os.path.exists(tpath) or os.path.exists(tpathfz):
starttime = datetime.now()
isSuccess, skyName, starNum, fwhmMean, bgMean = imgDiff.getCat(
srcDir, timgName, imgDiff.catDir)
if isSuccess: #we can add more filter condition, to remove bad images
catList.append([
isSuccess, ffId, timgName, starNum, skyName,
fwhmMean, bgMean, curSkyId, srcDir
])
endtime = datetime.now()
runTime = (endtime - starttime).seconds
imgDiff.log.info(
"srcExtractLocalDir, totalTime %d seconds, sky:%d, ffNum:%d, %s"
% (runTime, curSkyId, ffNumber, timgName))
else:
imgDiff.log.error("getCat: %s not exist" % (tpath))
except Exception as e:
tstr = traceback.format_exc()
imgDiff.log.error(tstr)
isRunning.value = 0
def run1(camName):
#toolPath = os.getcwd()
toolPath = '/home/gwac/img_diff_xy/image_diff'
tools = AstroTools(toolPath)
query = QueryData()
dataDest0 = "/data/gwac_diff_xy/data"
logDest0 = "/data/gwac_diff_xy/log"
if not os.path.exists(dataDest0):
os.system("mkdir -p %s"%(dataDest0))
if not os.path.exists(logDest0):
os.system("mkdir -p %s"%(logDest0))
startProcess = False
dayRun = 0
nigRun = 0
skyId = 0
ffId = 0
tfiles = []
while True:
curUtcDateTime = datetime.utcnow()
tDateTime = datetime.utcnow()
startDateTime = tDateTime.replace(hour=9, minute=30, second=0) #9=17 1=9
endDateTime = tDateTime.replace(hour=22, minute=30, second=0) #22=6 8=16
remainSeconds1 = (startDateTime - curUtcDateTime).total_seconds()
remainSeconds2 = (endDateTime - curUtcDateTime).total_seconds()
if remainSeconds1<0 and remainSeconds2>0:
dayRun = 0
try:
tfiles = query.getFileList(camName, ffId)
#print(tfiles)
for tfile in tfiles:
curFfId = tfile[0]
ffNumber = tfile[1]
curSkyId = tfile[2]
timgName = tfile[3] #G021_tom_objt_190109T13531492.fit
tpath = tfile[4] #/data3/G002_021_190109/G021_tom_objt_190109T13531492.fit
imgDate = timgName[14:20]
pathDate = tpath[16:22]
if imgDate!=pathDate:
ffId=curFfId
startProcess = False
continue
elif not startProcess:
ffId=0
startProcess = True
srcDir= tpath[:(tpath.find(camName)-1)] #/data3/G002_021_190109
dateStr = srcDir[srcDir.find('G'):] #G002_021_190109
logfName0 = '%s/%s.log'%(logDest0, dateStr)
if ffId==0:
if os.path.exists(logfName0) and os.stat(logfName0).st_size > 0:
tlastLine = getLastLine(logfName0)
if len(tlastLine)>2:
ffId=int(tlastLine.strip())
if skyId!=curSkyId and curFfId>ffId:
dstDir='%s/%s'%(dataDest0, dateStr)
tdiff = BatchImageDiff(srcDir, dstDir, tools, camName, curSkyId)
tStr = "start diff: %s"%(timgName)
tdiff.log.info(tStr)
tdiff.sendMsg(tStr)
if curFfId>ffId:
skyId=curSkyId
ffId=curFfId
logfile0 = open(logfName0, 'a')
logfile0.write("\n\n%d\n"%(ffId))
logfile0.close()
tpathfz = "%s.fz"%(tpath)
if os.path.exists(tpath) or os.path.exists(tpathfz):
starttime = datetime.now()
tdiff.processImg(srcDir, timgName, ffNumber)
endtime = datetime.now()
runTime = (endtime - starttime).seconds
tdiff.log.info("totalTime %d seconds, sky:%d, ffNum:%d, %s"%(runTime, curSkyId, ffNumber, timgName))
else:
print("%s not exist"%(tpath))
#if curFfId>ffId:
# break
except Exception as e:
print(str(e))
tstr = traceback.format_exc()
print(tstr)
try:
if 'tdiff' in locals():
tStr = "diff error"
tdiff.log.info(tStr)
tdiff.sendMsg(tStr)
except Exception as e1:
print(str(e1))
tstr = traceback.format_exc()
print(tstr)
if len(tfiles)==0:
time.sleep(5)
nigRun = nigRun+1
#if nigRun>=1:
# break
else:
# day temp file clean
try:
if ('tdiff' in locals()) and (dayRun==0):
tdiff.initReg(0)
except Exception as e1:
print(str(e1))
tstr = traceback.format_exc()
print(tstr)
nigRun = 0
dayRun = dayRun+1
skyId = 0
ffId = 0
startProcess = False
if dayRun%6==1:
print("day %d wait"%(dayRun))
time.sleep(10*60)