def run1():
#toolPath = os.getcwd()
toolPath = '/home/gwac/img_diff_xy/image_diff'
tools = AstroTools(toolPath)
dataDest0 = "/data3/simulationTest"
if not os.path.exists(dataDest0):
os.system("mkdir -p %s" % (dataDest0))
srcPath00 = '/data2/G003_031_190116/f_20620425'
dateStr = '20190325'
camName = 'G031'
curSkyId = '123'
dataDest = '%s/%s' % (dataDest0, dateStr)
otRcgDir = "%s/otrcg" % (dataDest)
diffCatDir = "%s/cmbDiffCat" % (dataDest)
diffDir = "%s/cmbDiff" % (dataDest)
objDir = "%s/cmbFits" % (dataDest)
if not os.path.exists(otRcgDir):
os.system("mkdir -p %s" % (otRcgDir))
tsim = BatchImageSim(srcPath00, dataDest, tools, camName, curSkyId)
tsim.log.info("\n\n***************\nstart recognition diff OTs..\n")
tsim.batchRecognition(objDir, diffDir, diffCatDir, otRcgDir)
tsim.genRegFile(otRcgDir)
def run3():
toolPath = os.getcwd()
tools = AstroTools(toolPath)
dataRoot = "/data/gwac_data/gwac_orig_fits"
dataDest = "/data/gwac_data/gwac_simot/data_1231"
#fileList = [(2575,14,'181208'),(2573,19,'181208'),(2574,9,'181206'),(2576,16,'181206')]
fileList = [(2573, 19, '181208'), (2574, 9, '181206')]
query = QueryData()
print(fileList)
for tflist in fileList:
files = query.getFileList(tflist[0], tflist[1], tflist[2])
total = len(files)
#continue
ccd = files[0][1]
#G004_041
ccdDir = "G0%s_%s" % (ccd[:2], ccd)
srcDir = "%s/%s/%s" % (dataRoot, tflist[2], ccdDir)
dstDir = "%s/%s/G%s" % (dataDest, tflist[2], ccd)
tdiff = BatchImageDiff(srcDir, dstDir, tools)
tStr = "start imageDiff, %s,skyId:%d, camId:%d, imgNum:%d" % (
tflist[2], tflist[0], tflist[1], total)
tdiff.log.info(tStr)
tdiff.tools.sendTriggerMsg(tStr)
tdiff.batchSim3(files)
tStr = "end imageDiff, %s,skyId:%d, camId:%d, imgNum:%d" % (
tflist[2], tflist[0], tflist[1], total)
tdiff.log.info(tStr)
tdiff.tools.sendTriggerMsg(tStr)
def run1():
#toolPath = os.getcwd()
toolPath = '/home/gwac/img_diff_xy/image_diff'
tools = AstroTools(toolPath)
dataDest0 = "/data3/simulationTest"
if not os.path.exists(dataDest0):
os.system("mkdir -p %s" % (dataDest0))
srcPath00 = '/data2/G003_031_190116/f_20620425'
dateStr = '20190324'
camName = 'G031'
curSkyId = '123'
dataDest = '%s/%s' % (dataDest0, dateStr)
dCatDir = "%s/dcats" % (dataDest)
simFitsDir = "%s/simFits" % (dataDest)
alignFitsDir = "%s/alignFits" % (dataDest)
alignCatsDir = "%s/alignCats" % (dataDest)
resiFitDir = "%s/resiFit" % (dataDest)
resiCatDir = "%s/resiCat" % (dataDest)
preViewDir = "%s/preview" % (dataDest)
preViewSimResiDir = "%s/previewSimResi" % (dataDest)
simCatAddDir = "%s/simCatAdd" % (dataDest)
if not os.path.exists(dCatDir):
os.system("mkdir -p %s" % (dCatDir))
if not os.path.exists(simFitsDir):
os.system("mkdir -p %s" % (simFitsDir))
if not os.path.exists(alignFitsDir):
os.system("mkdir -p %s" % (alignFitsDir))
if not os.path.exists(alignCatsDir):
os.system("mkdir -p %s" % (alignCatsDir))
if not os.path.exists(resiFitDir):
os.system("mkdir -p %s" % (resiFitDir))
if not os.path.exists(resiCatDir):
os.system("mkdir -p %s" % (resiCatDir))
if not os.path.exists(preViewDir):
os.system("mkdir -p %s" % (preViewDir))
if not os.path.exists(preViewSimResiDir):
os.system("mkdir -p %s" % (preViewSimResiDir))
if not os.path.exists(simCatAddDir):
os.system("mkdir -p %s" % (simCatAddDir))
tsim = BatchImageSim(srcPath00, dataDest, tools, camName, curSkyId)
tsim.log.info("\n\n***************\nstart run Sextractor..\n")
tsim.getCats(srcPath00, dCatDir)
tsim.log.info("\n\n***************\nstart align image..\n")
tsim.imgAlign()
tsim.log.info("\n\n***************\nstart sim image..\n")
tsim.imgSimulate(alignFitsDir, simFitsDir, simCatAddDir)
tsim.log.info("\n\n***************\nstart diff image..\n")
tsim.imgDiff(alignFitsDir, resiFitDir)
def __init__(self):
self.conn = False
self.imgSize = (4136, 4096)
self.templateImg = 'ti.fit'
self.tmpRoot = "/dev/shm/gwacWCS"
self.templateDir = "%s/tmpl" % (self.tmpRoot)
if not os.path.exists(self.templateDir):
os.system("mkdir -p %s" % (self.templateDir))
self.toolPath = '/data/work/program/image_diff'
self.funpackProgram = "%s/tools/cfitsio/funpack" % (self.toolPath)
logPath = os.getcwd()
self.tools = AstroTools(self.toolPath, logPath)
self.log = self.tools.log
def getImgCenter(tpath, imgName, x, y):
tools = AstroTools('/home/xy/Downloads/myresource/deep_data2/image_diff')
fieldId, ra, dec = tools.getRaDec(tpath, imgName)
fpar = 'sex_diff.par'
sexConf = [
'-DETECT_MINAREA', '10', '-DETECT_THRESH', '5', '-ANALYSIS_THRESH',
'5', '-CATALOG_TYPE', 'FITS_LDAC'
]
tmplCat, isSuccess = tools.runSextractor(imgName,
tpath,
tpath,
fpar,
sexConf,
outSuffix='_ldac.fit')
if not isSuccess:
print("getDiffTemplate runSextractor failure2")
return isSuccess, 0, 0
tools.ldac2fits('%s/%s' % (tpath, tmplCat), '%s/ti_cat.fit' % (tpath))
runSuccess = tools.runWCS(tpath, 'ti_cat.fit', ra, dec)
if runSuccess:
wcs = WCS('%s/ti_cat.wcs' % (tpath))
ra, dec = wcs.all_pix2world(x, y, 1)
return runSuccess, ra, dec
def wcsRemap():
#toolPath = os.getcwd()
toolPath = '/home/xy/Downloads/myresource/deep_data2/image_diff'
tools = AstroTools(toolPath)
dataDest0 = "tmp/data"
logDest0 = "tmp/log"
if not os.path.exists(dataDest0):
os.system("mkdir -p %s" % (dataDest0))
if not os.path.exists(logDest0):
os.system("mkdir -p %s" % (logDest0))
startProcess = False
dayRun = 0
nigRun = 0
skyId = 0
ffId = 0
tfiles = []
srcPath00 = '/home/xy/test7'
dateStr = '190124'
camName = 'G041'
curSkyId = '123'
dstDir = '%s/%s' % (dataDest0, dateStr)
tdiff = BatchImageDiff(srcPath00, dstDir, tools, camName, curSkyId)
try:
tfiles0 = os.listdir(srcPath00)
tfiles0.sort()
ra0, dec0 = -1000, -1000
for timgName in tfiles0:
if timgName.find('fit') > 0:
wcsfile, ra_center, dec_center = tdiff.getWCS(
srcPath00, timgName, ra0, dec0)
ra0, dec0 = ra_center, dec_center
print("%f, %f" % (ra0, dec0))
os.system(
"cp %s/%s %s/%s.wcs" %
(tdiff.tmpDir, wcsfile, srcPath00, timgName.split('.')[0]))
except Exception as e:
print(str(e))
tstr = traceback.format_exc()
print(tstr)
try:
if 'tdiff' in locals():
tStr = "diff error"
tdiff.log.info(tStr)
tdiff.sendMsg(tStr)
except Exception as e1:
print(str(e1))
tstr = traceback.format_exc()
print(tstr)
def run1():
toolPath = os.getcwd()
tools = AstroTools(toolPath)
dataRoot = "/data/gwac_data/gwac_orig_fits"
dataDest = "/data/gwac_data/gwac_simot/data_1231"
tdiff = BatchImageDiff(dataRoot, dataDest, tools)
tdiff.batchSim()
def run2():
toolPath = os.getcwd()
tools = AstroTools(toolPath)
dataRoot = "/home/xy/Downloads/myresource/deep_data2/G180216/17320495.0"
dataDest = "/home/xy/Downloads/myresource/deep_data2/simot/data_190101"
tdiff = BatchImageDiff(dataRoot, dataDest, tools)
tdiff.batchSim2()
def test():
toolPath = '/home/xy/Downloads/myresource/deep_data2/image_diff' #os.getcwd()
tools = AstroTools(toolPath)
file1 = 'G034_mon_objt_190211T12192603.cat'
file2 = 'G034_mon_objt_190227T12304321.cat'
file3 = 'G034_mon_objt_190324T12222033.cat'
file4 = 'G044_mon_objt_190305T13070793.cat'
posReg(tools, file1, file2)
posReg(tools, file1, file3)
posReg(tools, file1, file4)
def run1():
#storePath = '/home/xy/work/imgDiffTest2'
storePath = '/home/xy/work/imgDiffTest2/combine2'
#toolPath = os.getcwd()
toolPath = '/home/xy/Downloads/myresource/deep_data2/image_diff'
tools = AstroTools(toolPath)
dataDest0 = "/home/xy/work/imgDiffTest2/combineCat"
logDest0 = "/home/xy/work/gwac_diff_xy/log"
if not os.path.exists(dataDest0):
os.system("mkdir -p %s" % (dataDest0))
if not os.path.exists(logDest0):
os.system("mkdir -p %s" % (logDest0))
tdates = os.listdir(storePath)
print("total has %d date&ccd" % (len(tdates)))
for i, tdate in enumerate(tdates):
dstDir = '%s/%s' % (dataDest0, tdate)
tdiff = BatchImageDiff(storePath, dstDir, tools)
tpath1 = '%s/%s' % (storePath, tdate)
timgs = os.listdir(tpath1)
timgs.sort()
print("total has %d images" % (len(timgs)))
for j, tname in enumerate(timgs):
try:
tpath = "%s/%s" % (tpath1, tname)
if not os.path.exists(tpath):
print("%s not exist" % (tpath))
continue
print("process %d %s" % (j, tname))
tdiff.getCat(tpath1, tname)
except Exception as e:
print(str(e))
tstr = traceback.format_exc()
print(tstr)
def mainControl(self,
camName,
runName='p1',
destDir='/data/gwac_diff_xy',
toolPath='/home/gwac/img_diff_xy/image_diff'):
tools = AstroTools(toolPath)
logDest0 = "%s/log" % (destDir)
if not os.path.exists(logDest0):
os.system("mkdir -p %s" % (logDest0))
dateStr = datetime.strftime(datetime.utcnow(), "%Y%m%d")
dataDest0 = "%s/data/%s_%s" % (destDir, dateStr, runName)
if not os.path.exists(dataDest0):
os.system("mkdir -p %s" % (dataDest0))
imgDiff = GWACDiff(camName, dataDest0, tools)
tStr = "%s: start combine diff..." % (camName)
imgDiff.log.info(tStr)
imgDiff.sendMsg(tStr)
dataPath = '/home/xy/Downloads/myresource/SuperNova20190113/stampImage/190101_G004_041_test2'
try:
self.srcExtract(dataPath, camName, self.catList, imgDiff, logDest0,
runName)
self.getAlignTemplate(camName, self.catList, self.alignTmplMap,
imgDiff)
self.doAlign(camName, self.catList, self.alignTmplMap,
self.alignList, imgDiff)
#self.doCombine(camName, self.alignList, self.alignTmplMap, imgDiff, self.cmbImgList, self.crossTaskParms, runName, 5)
self.srcExtractCombine(camName, self.alignList, self.cmbCatList,
imgDiff)
self.getDiffTemplate(camName, self.cmbCatList, self.alignTmplMap,
self.diffTmplMap, imgDiff)
self.doDiff(camName, self.cmbCatList, self.diffTmplMap, imgDiff,
self.diffImgList)
#self.doRecognitionAndUpload(camName, self.diffImgList, self.diffTmplMap, imgDiff, runName)
except Exception as e:
tstr = traceback.format_exc()
print("Scheduling main error....")
print(tstr)
time.sleep(10)
def run2():
toolPath = os.getcwd()
tools = AstroTools(toolPath)
dataRoot = "/home/xy/Downloads/myresource/deep_data2/G180216/17320495.0"
dataDest = "/home/xy/Downloads/myresource/deep_data2/simot/data_190107"
flist = os.listdir(dataRoot)
flist.sort()
imgs = []
for tfilename in flist:
if tfilename.find("fit") > -1:
imgs.append(tfilename)
print("total image %d" % (len(imgs)))
tdiff = BatchImageDiff(dataRoot, dataDest, tools)
tdiff.batchSim(imgs)
def run1():
#toolPath = os.getcwd()
toolPath = '/home/gwac/img_diff_xy/image_diff'
tools = AstroTools(toolPath)
dateStr = '20190413'
camName = 'G031'
curSkyId = '123'
srcPath00 = '/data/gwac_diff_xy/combine/%s' % (dateStr)
dataDest0 = "/data/gwac_diff_xy/combineRst/%s" % (dateStr)
if not os.path.exists(dataDest0):
os.system("mkdir -p %s" % (dataDest0))
tsim = BatchImageSim(srcPath00, dataDest0, tools, camName, curSkyId)
tsim.log.info("\n\n***************\nstart diff simCombine image..\n")
tsim.process(srcPath00, dataDest0)
def run1():
toolPath = os.getcwd()
tools = AstroTools(toolPath)
dateStr = datetime.strftime(datetime.now(), "%Y%m%d")
dataRoot = "/data/gwac_data/gwac_orig_fits"
#dataDest = "/data/gwac_data/gwac_simot/imgdiff_%s"%(dateStr)
dataDest = "/data/gwac_data/gwac_simot/imgdiff"
query = QueryData()
#filesNum = query.queryFilesNum()
filesNum = query.queryFilesNumHis()
for tnum in filesNum:
if tnum[3] > 100 and tnum[2] % 5 > 0:
#files = query.getFileList(tnum[1], tnum[2], tnum[0])
files = query.getFileListHis(tnum[1], tnum[2], tnum[0])
total = len(files)
ccd = files[0][1]
#G004_041
ccdDir = "G0%s_%s" % (ccd[:2], ccd)
tflist = (tnum[1], tnum[2], tnum[0])
srcDir = "%s/%s/%s" % (dataRoot, tflist[2], ccdDir)
dstDir = "%s/%s/G%s" % (dataDest, tflist[2], ccd)
tdiff = BatchImageDiff(srcDir, dstDir, tools)
tStr = "start imageDiff, %s,skyId:%d, camId:%d, imgNum:%d" % (
tflist[2], tflist[0], tflist[1], total)
tdiff.log.info(tStr)
tdiff.tools.sendTriggerMsg(tStr)
tdiff.batchSim(files)
tStr = "end imageDiff, %s,skyId:%d, camId:%d, imgNum:%d" % (
tflist[2], tflist[0], tflist[1], total)
tdiff.log.info(tStr)
tdiff.tools.sendTriggerMsg(tStr)
def run1():
storePath = '/data/gwac_data/gwac_orig_fits'
#toolPath = os.getcwd()
toolPath = '/data/work/program/image_diff'
tools = AstroTools(toolPath)
dataDest0 = "/data/gwac_diff_xy/data"
logDest0 = "/data/gwac_diff_xy/log"
if not os.path.exists(dataDest0):
os.system("mkdir -p %s"%(dataDest0))
if not os.path.exists(logDest0):
os.system("mkdir -p %s"%(logDest0))
tfiles = []
#srcPath00='/data1/G004_041_190124'
dateStr='190620'
camName='G041'
curSkyId='123'
dstDir='%s/%s'%(dataDest0, dateStr)
tdiff = BatchImageDiff(storePath, dstDir, tools, camName, curSkyId)
skyId = 2577
sql1 = "select date_str, cam_id from file_number where sky_id=%d ORDER BY date_str, cam_id"%(skyId)
print(sql1)
tquery = QueryData()
tdates = tquery.query(sql1)
print("total has %d date&ccd"%(len(tdates)))
for i, tdate in enumerate(tdates):
try:
dateStr = tdate[0][2:8]
camId = tdate[1]
mountNum = int(camId/5) + 1
ccdNum = camId%5
if ccdNum==0:
continue
camName = '0%d%d'%(mountNum, ccdNum)
tpath1 = '%s/%s/G00%d_%s'%(storePath, dateStr, mountNum,camName)
sql1 = "select img_name " \
"from fits_file2_his " \
"where sky_id=%s and cam_id=%s and substr(img_path, 17, 6)='%s' " \
"order by img_name"%(skyId, camId, dateStr)
print(sql1)
tquery = QueryData()
tfiles = tquery.query(sql1)
print("total has %d images"%(len(tfiles)))
for j, tname in enumerate(tfiles):
tname00 = tname[0]
tpath = "%s/%s"%(tpath1, tname00)
tpathz = "%s/%s.fz"%(tpath1, tname00)
if os.path.exists(tpath):
timgName = tname00
elif os.path.exists(tpathz):
timgName = tname00 + '.fz'
else:
print("%s not exist"%(tpath))
continue
if j%25==0:
print("process %d %s"%(j, timgName))
tdiff.getCat(tpath1, timgName)
except Exception as e:
print(str(e))
tstr = traceback.format_exc()
print(tstr)
def test():
#toolPath = '/home/gwac/img_diff_xy/image_diff'
toolPath = '/home/xy/Downloads/myresource/deep_data2/image_diff'
tools = AstroTools(toolPath)
dataDest0 = "/home/xy/gwac_diff_xy/data"
logDest0 = "/home/xy/gwac_diff_xy/log"
if not os.path.exists(dataDest0):
os.system("mkdir -p %s" % (dataDest0))
if not os.path.exists(logDest0):
os.system("mkdir -p %s" % (logDest0))
startProcess = False
dayRun = 0
nigRun = 0
skyId = 0
ffId = 0
tfiles = []
#srcPath00='/data1/G004_041_190124'
srcPath00 = '/home/xy/Downloads/myresource/matchTest'
dateStr = '190124'
camName = 'G041'
curSkyId = '123'
dstDir = '%s/%s' % (dataDest0, dateStr)
tdiff = BatchImageDiff(srcPath00, dstDir, tools, camName, curSkyId)
#tpath1 = '/data2/G003_034_190211'
#tpath2 = '/data2/G003_034_190227'
tpath1 = '/home/xy/Downloads/myresource/matchTest'
tpath2 = '/home/xy/Downloads/myresource/matchTest'
fname1 = 'G034_mon_objt_190211T12192603.fit'
fname2 = 'G044_mon_objt_190305T13070793.fit'
ra0, dec0 = -1000, -1000
wcsfile1, ra_center1, dec_center1 = tdiff.getWCS(tpath1, fname1, ra0, dec0)
wcs1 = WCS('%s/%s' % (tdiff.tmpDir, wcsfile1))
wcsfile2, ra_center2, dec_center2 = tdiff.getWCS(tpath2, fname2, ra0, dec0)
wcs2 = WCS('%s/%s' % (tdiff.tmpDir, wcsfile2))
os.system("cp %s/%s %s/%s" % (tpath1, fname1, tdiff.tmpDir, fname1))
os.system("%s %s/%s" % (tdiff.funpackProgram, tdiff.tmpDir, fname1))
os.system("cp %s/%s %s/%s" % (tpath2, fname2, tdiff.tmpDir, fname2))
os.system("%s %s/%s" % (tdiff.funpackProgram, tdiff.tmpDir, fname2))
fpar = 'sex_diff.par'
#sexConf=['-DETECT_MINAREA','3','-DETECT_THRESH','2.5','-ANALYSIS_THRESH','2.5']
sexConf = [
'-DETECT_MINAREA', '10', '-DETECT_THRESH', '5', '-ANALYSIS_THRESH', '5'
]
tcat1 = tools.runSextractor('G034_mon_objt_190211T12192603.fit',
tdiff.tmpDir, tdiff.tmpDir, fpar, sexConf)
tcat2 = tools.runSextractor('G044_mon_objt_190305T13070793.fit',
tdiff.tmpDir, tdiff.tmpDir, fpar, sexConf)
tdata2 = np.loadtxt("%s/%s" % (tdiff.tmpDir, tcat2))
tXY = tdata2[:, 0:2]
print(tXY[:3])
tRaDec = wcs2.all_pix2world(tXY, 1)
print(tRaDec[:3])
tXY2 = wcs1.all_world2pix(tRaDec, 1)
print(tXY2[:3])
tdata2[:, 0:2] = tXY2
saveName = "%s_trans.cat" % (fname2.split(".")[0])
savePath = "%s/%s" % (tdiff.tmpDir, saveName)
np.savetxt(savePath, tdata2, fmt='%.4f')
mchFile, nmhFile, mchPair = tools.runCrossMatch(tdiff.tmpDir, saveName,
tcat1, 1)
evaluatePos(tdiff.tmpDir, saveName, tcat1, mchPair)
evaluatePos(tdiff.tmpDir, tcat2, tcat1, mchPair)
def mainControl(self,
camName,
runName='p1',
destDir='/data/gwac_diff_xy',
toolPath='/home/gwac/img_diff_xy/image_diff'):
#os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152
#os.environ["CUDA_VISIBLE_DEVICES"] = ""
tools = AstroTools(toolPath)
logDest0 = "%s/log" % (destDir)
if not os.path.exists(logDest0):
os.system("mkdir -p %s" % (logDest0))
dateStr = datetime.strftime(datetime.utcnow(), "%Y%m%d")
dataDest0 = "%s/data/%s_%s" % (destDir, dateStr, runName)
if not os.path.exists(dataDest0):
os.system("mkdir -p %s" % (dataDest0))
imgDiff = GWACDiff(camName, dataDest0, tools)
tStr = "%s: start combine diff..." % (camName)
imgDiff.log.info(tStr)
imgDiff.sendMsg(tStr)
#dataPath = '/home/xy/work/imgDiffTest2/fits/190128_G004_044'
#
#processRecord = '/home/xy/work/imgDiffTest3/log/G004_044.log'
#os.system("rm -rf %s"%(processRecord))
#sexLock=Lock()
self.loopNum = 1
while True:
try:
curUtcDateTime = datetime.utcnow()
tDateTime = datetime.utcnow()
startDateTime = tDateTime.replace(hour=9, minute=30,
second=0) #9=17 1=9
endDateTime = tDateTime.replace(hour=9, minute=30,
second=10) #22=6 8=16
remainSeconds1 = (startDateTime -
curUtcDateTime).total_seconds()
remainSeconds2 = (endDateTime - curUtcDateTime).total_seconds()
if remainSeconds1 < 0 and remainSeconds2 > 0:
if os.path.exists(dataDest0):
os.system("rm -rf %s" % (dataDest0))
self.initData()
dateStr = datetime.strftime(datetime.utcnow(), "%Y%m%d")
dataDest0 = "%s/data/%s_%s" % (destDir, dateStr, runName)
if not os.path.exists(dataDest0):
os.system("mkdir -p %s" % (dataDest0))
imgDiff.reInitDataDir(dataDest0)
tStr = "%s: combine diff reInit data" % (camName)
print(tStr)
imgDiff.log.info(tStr)
imgDiff.sendMsg(tStr)
time.sleep(10)
except Exception as e:
tstr = traceback.format_exc()
print("combine diff reInit data error....")
print(tstr)
time.sleep(10)
try:
print("\n\nmain loop %d****************" % (self.loopNum))
tIdx1 = self.loopNum % 5
#tIdx1 = self.loopNum%1
''' '''
if tIdx1 == 0 and self.catRunning == 0: #cat
self.srcExtract(camName, self.catList, imgDiff, logDest0,
runName)
#tIdx2 = self.loopNum%15
tIdx2 = self.loopNum % 2
if tIdx2 == 0 and self.alignTmplRunning == 0: #template getAlignTemplateLocal
self.getAlignTemplate(camName, self.catList,
self.alignTmplMap, imgDiff)
tIdx3 = self.loopNum % 2
if tIdx3 == 0 and self.alignRunning == 0: #align
self.doAlign(camName, self.catList, self.alignTmplMap,
self.alignList, imgDiff)
tIdx4 = self.loopNum % 2
if tIdx4 == 0 and self.cmbRunning == 0: #combine5
self.doCombine(camName, self.alignList, self.alignTmplMap,
imgDiff, self.cmbImgList,
self.crossTaskParms, runName, 5)
tIdx5 = self.loopNum % 2
if tIdx5 == 0 and self.cmbCatRunning == 0: #cat of combine5
self.srcExtractCombine(camName, self.cmbImgList,
self.cmbCatList, imgDiff)
#else:
# print("%d cmbCatJob is running"%(self.loopNum))
#tIdx6 = self.loopNum%15
tIdx6 = self.loopNum % 2
if tIdx6 == 0 and self.diffTmplRunning == 0: #cat of combine5
self.getDiffTemplate(camName, self.cmbCatList,
self.alignTmplMap, self.diffTmplMap,
imgDiff)
#tIdx7 = self.loopNum%5
tIdx7 = self.loopNum % 2
if tIdx7 == 0:
if self.diffRunning == 0: #diff
self.doDiff(camName, self.cmbCatList, self.diffTmplMap,
imgDiff, self.diffImgList)
#tIdx8 = self.loopNum%2
tIdx8 = self.loopNum % 2
if tIdx8 == 0:
if self.recgRunning == 0: #recognition
#print("%d doRecognitionAndUpload start run"%(self.loopNum))
self.doRecognitionAndUpload(camName, self.diffImgList,
self.diffTmplMap, imgDiff,
runName)
#else:
# print("%d doRecognitionAndUpload is running"%(self.loopNum))
self.loopNum = self.loopNum + 1
time.sleep(1)
except Exception as e:
tstr = traceback.format_exc()
print("Scheduling main error....")
print(tstr)
time.sleep(10)
class GWACWCSIndex:
orgImgRoot = '/data/gwac_data/gwac_orig_fits'
wcsIdxRoot = '/data/gwac_data/gwac_wcs_idx'
webServerIP1 = '172.28.8.28:8080'
webServerIP2 = '10.0.10.236:9995'
connParam = {
"host": "190.168.1.27",
"port": "5432",
"dbname": "gwac2",
"user": "******",
"password": "******"
}
connParam2 = {
"host": "172.28.8.28",
"port": "5432",
"dbname": "gwac2",
"user": "******",
"password": "******"
}
connParam3 = {
"host": "10.0.3.62",
"port": "5433",
"dbname": "gwac2",
"user": "******",
"password": "******"
}
webUser = '******'
webPassword = '******'
webIp = '172.28.8.8'
def __init__(self):
self.conn = False
self.imgSize = (4136, 4096)
self.templateImg = 'ti.fit'
self.tmpRoot = "/dev/shm/gwacWCS"
self.templateDir = "%s/tmpl" % (self.tmpRoot)
if not os.path.exists(self.templateDir):
os.system("mkdir -p %s" % (self.templateDir))
#/home/gwac/img_diff_xy/image_diff
#/data/work/program/image_diff
self.toolPath = '/home/gwac/img_diff_xy/image_diff'
self.funpackProgram = "%s/tools/cfitsio/funpack" % (self.toolPath)
#logPath = os.getcwd()
self.tools = AstroTools(self.toolPath, "gwac_wcs")
self.log = self.tools.log
def makeTemplate(self, tparm):
try:
starttime = datetime.now()
#print(tparm)
tRaDec = np.array([])
os.system("rm -rf %s/*" % (self.templateDir))
imgName = tparm['img_name'].decode("utf-8")
fwhm = tparm['fwhm']
ffNumber = tparm['ff_number']
objNum = tparm['obj_num']
imgPath = tparm['img_path'].decode("utf-8")
tmsgStr = "select %s as template, min fwhm %.2f" % (imgName, fwhm)
print(tmsgStr)
self.log.info(tmsgStr)
self.tools.sendTriggerMsg(tmsgStr)
os.system("rm -rf %s/*" % (self.templateDir))
#oImgf = "%s/%s"%(imgPath,imgName)
oImgf = imgPath
print(oImgf)
oImgfz = "%s.fz" % (imgPath)
if os.path.exists(oImgfz):
os.system("cp %s %s/%s.fz" %
(oImgfz, self.templateDir, self.templateImg))
os.system(
"%s %s/%s.fz" %
(self.funpackProgram, self.templateDir, self.templateImg))
elif os.path.exists(oImgf):
os.system("cp %s %s/%s" %
(oImgf, self.templateDir, self.templateImg))
else:
self.log.warning("%s not exist" % (oImgf))
return
fieldId, ra, dec = self.tools.removeHeaderAndOverScan(
self.templateDir, self.templateImg)
sexConf = [
'-DETECT_MINAREA', '10', '-DETECT_THRESH', '5',
'-ANALYSIS_THRESH', '5'
]
fpar = 'sex_diff.par'
tmplCat, isSuccess = self.tools.runSextractor(self.templateImg,
self.templateDir,
self.templateDir,
fpar,
sexConf,
cmdStatus=0)
starNum, fwhmMean, fwhmRms, bgMean, bgRms = self.tools.basicStatistic(
self.templateDir, tmplCat)
sexConf = [
'-DETECT_MINAREA', '10', '-DETECT_THRESH', '5',
'-ANALYSIS_THRESH', '5', '-CATALOG_TYPE', 'FITS_LDAC'
]
tmplCat, isSuccess = self.tools.runSextractor(
self.templateImg,
self.templateDir,
self.templateDir,
fpar,
sexConf,
cmdStatus=0,
outSuffix='_ldac.fit')
self.tools.ldac2fits('%s/%s' % (self.templateDir, tmplCat),
'%s/ti_cat.fit' % (self.templateDir))
tpath = "%s/%s" % (self.templateDir, tmplCat)
runSuccess = self.tools.runWCS(self.templateDir, 'ti_cat.fit', ra,
dec)
#ccdName = self.origTmplImgName[:4]
#runSuccess = self.tools.runWCSRemotePC780(self.templateDir,'ti_cat.fit', ra, dec, ccdName)
if runSuccess:
twcs = WCS('%s/ti_cat.wcs' % (self.templateDir))
height = self.imgSize[0]
width = self.imgSize[1]
tXYs = []
tXYs.append((width / 2, height / 2))
tXYs.append((0, 0))
tXYs.append((0, height - 1))
tXYs.append((width - 1, height - 1))
tXYs.append((width - 1, 0))
tXYs = np.array(tXYs)
try:
tRaDec = twcs.all_pix2world(tXYs, 1)
self.log.info(
'read_ra_dec:(%.5f, %.5f), real_dec_center:(%.5f, %.5f)'
% (ra, dec, tRaDec[0][0], tRaDec[0][1]))
except Exception as e:
self.log.error(e)
runSuccess = False
tstr = traceback.format_exc()
self.log.error(tstr)
self.log.error('make template %s, xy to radec error' %
(imgName))
objName = 'ti.fit'
bkgName = 'ti_bkg.fit'
badPixCat = self.tools.processBadPix(objName, bkgName,
self.templateDir,
self.templateDir)
imgPre = imgName.split('.')[0]
tpaths = imgPath.split(
'/'
) #/data2/G004_041_191015/G041_Mon_objt_191015T20282227.fit
tpath0 = tpaths[-2]
tcamName = tpath0[:8]
tdateStr = tpath0[9:]
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(self.webIp,
username=self.webUser,
password=self.webPassword)
#/data/gwac_data/gwac_wcs_idx/12854925/G003_031/180210
storePath = "%s/%s/%s/%s" % (self.wcsIdxRoot, fieldId,
tcamName, tdateStr)
tcmd = "mkdir -p %s;" % (storePath)
ssh.exec_command(tcmd)
time.sleep(10)
fitImg = "%s/%s" % (self.templateDir, self.templateImg)
cat = "%s/ti.cat" % (self.templateDir)
catFit = "%s/ti_cat.fit" % (self.templateDir)
wcs = '%s/ti_cat.wcs' % (self.templateDir)
badPix = '%s/%s' % (self.templateDir, badPixCat)
if os.path.exists(fitImg) and os.path.exists(cat) and os.path.exists(catFit) \
and os.path.exists(wcs) and os.path.exists(badPix):
print("start copy to remote dir %s" % (storePath))
ftp = ssh.open_sftp()
ftp.put(fitImg, "%s/%s.fit" % (storePath, imgPre))
ftp.put(cat, "%s/%s.cat" % (storePath, imgPre))
ftp.put(catFit, "%s/%s_cat.fit" % (storePath, imgPre))
ftp.put(wcs, "%s/%s.wcs" % (storePath, imgPre))
ftp.put(badPix,
"%s/%s_badpix.cat" % (storePath, imgPre))
print("copy success")
elif not os.path.exists(fitImg):
runSuccess = False
print("%s is not exist" % (fitImg))
elif not os.path.exists(cat):
runSuccess = False
print("%s is not exist" % (cat))
elif not os.path.exists(catFit):
runSuccess = False
print("%s is not exist" % (catFit))
elif not os.path.exists(wcs):
runSuccess = False
print("%s is not exist" % (wcs))
elif not os.path.exists(badPix):
runSuccess = False
print("%s is not exist" % (badPix))
except paramiko.AuthenticationException:
self.log.error("Authentication Failed!")
runSuccess = False
tstr = traceback.format_exc()
self.log.error(tstr)
except paramiko.SSHException:
self.log.error("Issues with SSH service!")
runSuccess = False
tstr = traceback.format_exc()
self.log.error(tstr)
except Exception as e:
self.log.error(
'make template %s, copy data to dest error' %
(imgName))
self.log.error(str(e))
runSuccess = False
tstr = traceback.format_exc()
self.log.error(tstr)
else:
self.log.error('make template %s, get wcs error' % (imgName))
endtime = datetime.now()
runTime = (endtime - starttime).seconds
self.log.info("********** make template %s use %d seconds" %
(imgName, runTime))
except Exception as e:
runSuccess = False
tRaDec = np.array([])
starNum = 0
fwhmMean = 0
bgMean = 0
self.log.error(e)
tstr = traceback.format_exc()
self.log.error(tstr)
tmsgStr = "%s make template error" % (imgName)
self.tools.sendTriggerMsg(tmsgStr)
return runSuccess, tRaDec, starNum, fwhmMean, bgMean
def connDb(self):
self.conn = psycopg2.connect(**self.connParam2)
def closeDb(self):
self.conn.close()
def getDataFromDB(self, sql):
tsql = sql
#self.log.debug(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
rows = cur.fetchall()
cur.close()
self.closeDb()
except Exception as err:
rows = []
self.log.error(" query data error ")
self.log.error(err)
return np.array(rows)
def queryObs(self, camName):
tsql = "select ors_id, date_str, sky_id, cam_id, img_num "\
"from observation_record_statistic ors "\
"INNER JOIN camera cam on cam.camera_id=ors.cam_id "\
"where do_wcs=false and cam.name='%s' "\
"ORDER BY ors_id "%(camName)
#print(tsql)
return self.getDataFromDB(tsql)
def queryImgParm(self, obs, isHis='_his'):
tsql = "SELECT ff.img_name, isp.fwhm, ff.ff_number, isp.obj_num, isp.time_obs_ut, ff.ff_id, ff.img_path "\
"from image_status_parameter%s isp "\
"INNER JOIN fits_file2%s ff on isp.ff_id=ff.ff_id "\
"where isp.fwhm>1 and ff.sky_id=%s and ff.cam_id=%s and to_char(ff.gen_time, 'YYMMDD')='%s' "\
"ORDER BY ff_number"%(isHis, isHis, obs[2], obs[3], obs[1])
trst = self.getDataFromDB(tsql)
return trst
def queryImgParm2(self, obs, isHis='_his'):
tsql = "SELECT ff.img_name, 0, ff.ff_number, 0, ff.gen_time, ff.ff_id, ff.img_path "\
"from fits_file2%s ff "\
"where ff.sky_id=%s and ff.cam_id=%s and to_char(ff.gen_time, 'YYMMDD')='%s' "\
"ORDER BY ff_number"%(isHis, obs[2], obs[3], obs[1])
trst = self.getDataFromDB(tsql)
return trst
def updateHasWCS(self, orsId, imgNum, minTime, maxTime, hasWCS='true'):
startTime = datetime.strftime(minTime, "%Y-%m-%d %H:%M:%S")
endTime = datetime.strftime(maxTime, "%Y-%m-%d %H:%M:%S")
tsql = "update observation_record_statistic set has_wcs=%s, real_img_num=%d, " \
"start_obs_time='%s', end_obs_time='%s' where ors_id=%s"%(hasWCS, imgNum, startTime, endTime, orsId)
#print(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
self.conn.commit()
cur.close()
self.closeDb()
except Exception as err:
self.log.error(" update science_object status error ")
self.log.error(err)
def updateDoWCS(self, orsId, doWCS='true'):
tsql = "update observation_record_statistic set do_wcs=%s where ors_id=%s" % (
doWCS, orsId)
#print(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
self.conn.commit()
cur.close()
self.closeDb()
except Exception as err:
self.log.error(" update science_object status error ")
self.log.error(err)
def updateWCSCoors(self, orsId, tRaDecs):
rd = tRaDecs
tsql = "update observation_record_statistic set center_ra=%f, center_dec=%f, " \
"left_top_ra=%f, left_top_dec=%f, left_bottom_ra=%f, left_bottom_dec=%f, "\
"right_top_ra=%f, right_top_dec=%f, right_bottom_ra=%f, right_bottom_dec=%f where ors_id=%s" \
%(rd[0][0], rd[0][1], rd[1][0], rd[1][1], rd[2][0], rd[2][1], rd[3][0], rd[3][1],rd[4][0], rd[4][1], orsId)
#print(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
self.conn.commit()
cur.close()
self.closeDb()
except Exception as err:
self.log.error(" update science_object status error ")
self.log.error(err)
def insertObsWcs(self,
orsId,
tparms,
starNum,
fwhm,
background,
getWCS='true'):
#print(orsId)
#print(tparms)
#tsql = "insert into observation_record_statistic_wcs(ff_id,fwhm,star_num,ors_id,get_wcs)"\
# "values(%d,%f,%d,%s,%s)"%(tparms[5],tparms[1],tparms[3],orsId,getWCS)
tsql = "insert into observation_record_statistic_wcs(ff_id,fwhm,star_num,ors_id,get_wcs, background)"\
"values(%d,%f,%d,%s,%s, %f)"%(tparms[5],fwhm,starNum,orsId,getWCS, background)
#print(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
self.conn.commit()
cur.close()
self.closeDb()
except Exception as err:
self.log.error(" update science_object status error ")
self.log.error(err)
def checkImg(self, obs, imgParms):
dateStr = obs[1]
skyId = int(obs[2])
camId = int(obs[3])
mountId = int(camId / 5) + 1
camId2 = camId % 5
if camId2 == 0:
mountId = mountId - 1
camId2 = 5
camName = "G%03d_%02d%d" % (mountId, mountId, camId2)
#print(dateStr,skyId, camName)
#fullPath = "%s/%s/%s"%(self.orgImgRoot, dateStr, camName) #/data2/G002_023_191015 /data/gwac_data/gwac_orig_fits/191015/G004_043
tparms = []
for tparm in imgParms:
imgPath = tparm[6]
#imgPath = "%s/%s"%(fullPath, tparm[0])
imgPathfz = "%s.fz" % (imgPath)
#print(imgPathfz)
if os.path.exists(imgPathfz) or os.path.exists(imgPath):
tparms.append((tparm[0], tparm[1], tparm[2], tparm[3],
tparm[6], tparm[5]))
dtype = [('img_name', 'S40'), ('fwhm', float), ('ff_number', int),
('obj_num', int), ('img_path', 'S100'), ('ff_id', int)]
trst = np.array(tparms, dtype=dtype)
return trst
def doAstrometry(self, orsId, tparms, hasParms=True):
doSuccess = False
if hasParms:
sortParms = np.sort(tparms, order='fwhm')
starIdx = 0
else:
tnum = tparms.shape[0]
starIdx = int(tnum * 2.0 / 3)
sortParms = tparms
#print(sortParms)
for i in range(3):
tIdx = starIdx + i
runSuccess, tRaDec, starNum, fwhmMean, bgMean = self.makeTemplate(
sortParms[tIdx])
if runSuccess:
doSuccess = True
self.updateWCSCoors(orsId, tRaDec)
self.insertObsWcs(orsId, sortParms[tIdx], starNum, fwhmMean,
bgMean)
break
else:
print("%dth run failure, try next image" % (i + 1))
#break
return doSuccess
def createWCS(self, camName, minNum=50):
tobs = self.queryObs(camName)
print("query %d sky..." % (tobs.shape[0]))
if tobs.shape[0] > 0:
for obs in tobs:
print(obs)
orsId = obs[0]
dateStr = obs[1]
imgNum = int(obs[4])
self.updateDoWCS(orsId)
camId = int(obs[3])
if camId % 5 == 0:
continue
imgParms = self.queryImgParm(obs)
if imgParms.shape[0] == 0:
print(
"cannot image parameters form history table, refind from current table "
)
imgParms = self.queryImgParm(obs, " ")
if imgParms.shape[0] > minNum:
timeObsUt = imgParms[:, 4]
minTime = np.min(timeObsUt)
maxTime = np.max(timeObsUt)
#self.log.debug(imgParms.shape[0])
tparms = self.checkImg(obs, imgParms)
tnum = tparms.shape[0]
tstr = "%s,%s,%s observe %s img, %d has parameter, %d still in local path."\
%(obs[1], obs[2], obs[3], obs[4], imgParms.shape[0], tnum)
self.log.debug(tstr)
print(tstr)
if tnum < minNum:
print(
"%s,%s,%s has %d images, small than minimum %d number, skip"
% (obs[1], obs[2], obs[3], tnum, minNum))
self.updateHasWCS(orsId, tnum, minTime, maxTime,
'false')
else:
if tnum > 100:
tparms = tparms[25:-25]
doSuccess = self.doAstrometry(orsId, tparms)
else:
doSuccess = self.doAstrometry(orsId, tparms)
if doSuccess:
self.updateHasWCS(orsId, tnum, minTime, maxTime,
'true')
else:
self.updateHasWCS(orsId, tnum, minTime, maxTime,
'false')
#break
else:
print(
"image parameter number is %d less than min number %d"
% (imgParms.shape[0], minNum))
if imgParms.shape[0] == 0 and imgNum > minNum:
print("directly build wcs from orig image.")
#ff.img_name, isp.fwhm, ff.ff_number, isp.obj_num, isp.time_obs_ut, ff.ff_id, ff.img_path
imgParms = self.queryImgParm2(obs)
if imgParms.shape[0] == 0:
print(
"cannot image parameters form history table, refind from current table "
)
imgParms = self.queryImgParm2(obs, " ")
if imgParms.shape[0] > minNum:
#self.log.debug(imgParms.shape[0])
timeObsUt = imgParms[:, 4]
minTime = np.min(timeObsUt)
maxTime = np.max(timeObsUt)
tparms = self.checkImg(obs, imgParms)
tnum = tparms.shape[0]
tstr = "%s,%s,%s observe %s img, 0 has parameter, %d still in local path."\
%(obs[1], obs[2], obs[3], obs[4], tnum)
self.log.debug(tstr)
print(tstr)
if tnum < minNum:
print(
"%s,%s,%s has %d images, small than minimum %d number, skip"
% (obs[1], obs[2], obs[3], tnum, minNum))
self.updateHasWCS(orsId, tnum, minTime,
maxTime, 'false')
else:
if tnum > 100:
tparms = tparms[25:-25]
doSuccess = self.doAstrometry(
orsId, tparms, False)
else:
doSuccess = self.doAstrometry(
orsId, tparms, False)
if doSuccess:
self.updateHasWCS(orsId, tnum, minTime,
maxTime, 'true')
else:
self.updateHasWCS(orsId, tnum, minTime,
maxTime, 'false')
else:
minTime = datetime.now()
maxTime = minTime
tnum = 0
self.updateHasWCS(orsId, tnum, minTime, maxTime,
'false')
else:
try:
timeObsUt = imgParms[:, 4]
minTime = np.min(timeObsUt)
maxTime = np.max(timeObsUt)
except Exception as err:
minTime = datetime.now()
maxTime = minTime
self.log.error(err)
tnum = 0
self.updateHasWCS(orsId, tnum, minTime, maxTime,
'false')
#break
else:
print("already completely built all wcs")
def scheduling(self, camName):
while True:
try:
curUtcDateTime = datetime.now()
tDateTime = datetime.now()
startDateTime = tDateTime.replace(hour=7, minute=10, second=0)
endDateTime = tDateTime.replace(hour=8, minute=10, second=10)
remainSeconds1 = (startDateTime -
curUtcDateTime).total_seconds()
remainSeconds2 = (endDateTime - curUtcDateTime).total_seconds()
if remainSeconds1 < 0 and remainSeconds2 > 0:
self.createWCS(camName)
time.sleep(15 * 60)
else:
time.sleep(60)
except Exception as e:
tstr = traceback.format_exc()
print("everyday create WCS error....")
print(tstr)
time.sleep(10 * 60)
class GWACWCSIndex:
orgImgRoot = '/data/gwac_data/gwac_orig_fits'
wcsIdxRoot = '/data/gwac_data/gwac_wcs_idx'
webServerIP1 = '172.28.8.28:8080'
webServerIP2 = '10.0.10.236:9995'
connParam = {
"host": "190.168.1.27",
"port": "5432",
"dbname": "gwac2",
"user": "******",
"password": "******"
}
connParam2 = {
"host": "172.28.8.28",
"port": "5432",
"dbname": "gwac2",
"user": "******",
"password": "******"
}
connParam3 = {
"host": "10.0.3.62",
"port": "5433",
"dbname": "gwac2",
"user": "******",
"password": "******"
}
def __init__(self):
self.conn = False
self.imgSize = (4136, 4096)
self.templateImg = 'ti.fit'
self.tmpRoot = "/dev/shm/gwacWCS"
self.templateDir = "%s/tmpl" % (self.tmpRoot)
if not os.path.exists(self.templateDir):
os.system("mkdir -p %s" % (self.templateDir))
self.toolPath = '/data/work/program/image_diff'
self.funpackProgram = "%s/tools/cfitsio/funpack" % (self.toolPath)
logPath = os.getcwd()
self.tools = AstroTools(self.toolPath, logPath)
self.log = self.tools.log
def makeTemplate(self, tparm):
try:
starttime = datetime.now()
print(tparm)
tRaDec = np.array([])
os.system("rm -rf %s/*" % (self.templateDir))
imgName = tparm['img_name'].decode("utf-8")
fwhm = tparm['fwhm']
ffNumber = tparm['ff_number']
objNum = tparm['obj_num']
imgPath = tparm['img_path'].decode("utf-8")
tmsgStr = "select %s as template, min fwhm %.2f" % (imgName, fwhm)
self.log.info(tmsgStr)
self.tools.sendTriggerMsg(tmsgStr)
os.system("rm -rf %s/*" % (self.templateDir))
oImgf = "%s/%s" % (imgPath, imgName)
print(oImgf)
oImgfz = "%s/%s.fz" % (imgPath, imgName)
if os.path.exists(oImgfz):
os.system("cp %s %s/%s.fz" %
(oImgfz, self.templateDir, self.templateImg))
os.system(
"%s %s/%s.fz" %
(self.funpackProgram, self.templateDir, self.templateImg))
elif os.path.exists(oImgf):
os.system("cp %s %s/%s" %
(oImgf, self.templateDir, self.templateImg))
else:
self.log.warning("%s not exist" % (oImgf))
return
fieldId, ra, dec = self.tools.removeHeaderAndOverScan(
self.templateDir, self.templateImg)
sexConf = [
'-DETECT_MINAREA', '10', '-DETECT_THRESH', '5',
'-ANALYSIS_THRESH', '5'
]
fpar = 'sex_diff.par'
tmplCat = self.tools.runSextractor(self.templateImg,
self.templateDir,
self.templateDir,
fpar,
sexConf,
cmdStatus=0)
sexConf = [
'-DETECT_MINAREA', '10', '-DETECT_THRESH', '5',
'-ANALYSIS_THRESH', '5', '-CATALOG_TYPE', 'FITS_LDAC'
]
tmplCat = self.tools.runSextractor(self.templateImg,
self.templateDir,
self.templateDir,
fpar,
sexConf,
cmdStatus=0,
outSuffix='_ldac.fit')
self.tools.ldac2fits('%s/%s' % (self.templateDir, tmplCat),
'%s/ti_cat.fit' % (self.templateDir))
tpath = "%s/%s" % (self.templateDir, tmplCat)
runSuccess = self.tools.runWCS(self.templateDir, 'ti_cat.fit', ra,
dec)
#ccdName = self.origTmplImgName[:4]
#runSuccess = self.tools.runWCSRemotePC780(self.templateDir,'ti_cat.fit', ra, dec, ccdName)
if runSuccess:
twcs = WCS('%s/ti_cat.wcs' % (self.templateDir))
height = self.imgSize[0]
width = self.imgSize[1]
tXYs = []
tXYs.append((width / 2, height / 2))
tXYs.append((0, 0))
tXYs.append((0, height - 1))
tXYs.append((width - 1, height - 1))
tXYs.append((width - 1, 0))
tXYs = np.array(tXYs)
try:
tRaDec = twcs.all_pix2world(tXYs, 1)
self.log.info(
'read_ra_dec:(%.5f, %.5f), real_dec_center:(%.5f, %.5f)'
% (ra, dec, tRaDec[0][0], tRaDec[0][1]))
except Exception as e:
self.log.error(e)
runSuccess = False
tstr = traceback.format_exc()
self.log.error(tstr)
self.log.error('make template %s, xy to radec error' %
(imgName))
objName = 'ti.fit'
bkgName = 'ti_bkg.fit'
badPixCat = self.tools.processBadPix(objName, bkgName,
self.templateDir,
self.templateDir)
imgPre = imgName.split('.')[0]
tpaths = imgPath.split('/')
storePath = "%s/%s/%s/%s" % (self.wcsIdxRoot, fieldId,
tpaths[-1], tpaths[-2])
if not os.path.exists(storePath):
os.system("mkdir -p %s" % (storePath))
fitImg = "%s/%s" % (self.templateDir, self.templateImg)
cat = "%s/ti.cat" % (self.templateDir)
catFit = "%s/ti_cat.fit" % (self.templateDir)
wcs = '%s/ti_cat.wcs' % (self.templateDir)
badPix = '%s/%s' % (self.templateDir, badPixCat)
try:
os.system("cp %s %s/%s.fit" % (fitImg, storePath, imgPre))
os.system("cp %s %s/%s.cat" % (cat, storePath, imgPre))
os.system("cp %s %s/%s_cat.fit" %
(catFit, storePath, imgPre))
os.system("cp %s %s/%s.wcs" % (wcs, storePath, imgPre))
os.system("cp %s %s/%s_badpix.cat" %
(badPix, storePath, imgPre))
except Exception as e:
self.log.error(e)
runSuccess = False
tstr = traceback.format_exc()
self.log.error(tstr)
self.log.error(
'make template %s, copy data to dest error' %
(imgName))
else:
self.log.error('make template %s, get wcs error' % (imgName))
endtime = datetime.now()
runTime = (endtime - starttime).seconds
self.log.info("********** make template %s use %d seconds" %
(imgName, runTime))
except Exception as e:
runSuccess = False
self.log.error(e)
tstr = traceback.format_exc()
self.log.error(tstr)
tmsgStr = "%s make template error" % (imgName)
self.tools.sendTriggerMsg(tmsgStr)
return runSuccess, tRaDec
def connDb(self):
self.conn = psycopg2.connect(**self.connParam2)
def closeDb(self):
self.conn.close()
def getDataFromDB(self, sql):
tsql = sql
#self.log.debug(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
rows = cur.fetchall()
cur.close()
self.closeDb()
except Exception as err:
rows = []
self.log.error(" query data error ")
self.log.error(err)
return np.array(rows)
def queryObs(self, size=10):
#"where do_wcs=false and date_str<'190924'"\
tsql = "select ors_id, date_str, sky_id, cam_id, img_num "\
"from observation_record_statistic "\
"where do_wcs=false "\
"ORDER BY ors_id limit %d"%(size)
print(tsql)
return self.getDataFromDB(tsql)
def queryImgParm(self, obs):
tsql = "SELECT ff.img_name, isp.fwhm, ff.ff_number, isp.obj_num, isp.time_obs_ut, ff.ff_id "\
"from image_status_parameter_his isp "\
"INNER JOIN fits_file2_his ff on isp.ff_id=ff.ff_id "\
"where isp.fwhm>1 and ff.sky_id=%s and ff.cam_id=%s and substr(ff.img_name, 15 , 6)='%s' "\
"ORDER BY ff_number"%(obs[2], obs[3], obs[1])
print(tsql)
return self.getDataFromDB(tsql)
def updateHasWCS(self, orsId, imgNum, minTime, maxTime, hasWCS='true'):
startTime = datetime.strftime(minTime, "%Y-%m-%d %H:%M:%S")
endTime = datetime.strftime(maxTime, "%Y-%m-%d %H:%M:%S")
tsql = "update observation_record_statistic set has_wcs=%s, real_img_num=%d, " \
"start_obs_time='%s', end_obs_time='%s' where ors_id=%s"%(hasWCS, imgNum, startTime, endTime, orsId)
print(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
self.conn.commit()
cur.close()
self.closeDb()
except Exception as err:
self.log.error(" update science_object status error ")
self.log.error(err)
def updateDoWCS(self, orsId, doWCS='true'):
tsql = "update observation_record_statistic set do_wcs=%s where ors_id=%s" % (
doWCS, orsId)
print(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
self.conn.commit()
cur.close()
self.closeDb()
except Exception as err:
self.log.error(" update science_object status error ")
self.log.error(err)
def updateWCSCoors(self, orsId, tRaDecs):
rd = tRaDecs
tsql = "update observation_record_statistic set center_ra=%f, center_dec=%f, " \
"left_top_ra=%f, left_top_dec=%f, left_bottom_ra=%f, left_bottom_dec=%f, "\
"right_top_ra=%f, right_top_dec=%f, right_bottom_ra=%f, right_bottom_dec=%f where ors_id=%s" \
%(rd[0][0], rd[0][1], rd[1][0], rd[1][1], rd[2][0], rd[2][1], rd[3][0], rd[3][1],rd[4][0], rd[4][1], orsId)
print(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
self.conn.commit()
cur.close()
self.closeDb()
except Exception as err:
self.log.error(" update science_object status error ")
self.log.error(err)
def insertObsWcs(self, orsId, tparms, getWCS='true'):
print(orsId)
print(tparms)
tsql = "insert into observation_record_statistic_wcs(ff_id,fwhm,star_num,ors_id,get_wcs)"\
"values(%d,%f,%d,%s,%s)"%(tparms[5],tparms[1],tparms[3],orsId,getWCS)
print(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
self.conn.commit()
cur.close()
self.closeDb()
except Exception as err:
self.log.error(" update science_object status error ")
self.log.error(err)
def checkImg(self, obs, imgParms):
dateStr = obs[1]
skyId = int(obs[2])
camId = int(obs[3])
mountId = int(camId / 5) + 1
camId2 = camId % 5
if camId2 == 0:
mountId = mountId - 1
camId2 = 5
camName = "G%03d_%02d%d" % (mountId, mountId, camId2)
#print(dateStr,skyId, camName)
fullPath = "%s/%s/%s" % (self.orgImgRoot, dateStr, camName)
tparms = []
for tparm in imgParms:
imgPath = "%s/%s" % (fullPath, tparm[0])
imgPathfz = "%s.fz" % (imgPath)
if os.path.exists(imgPathfz) or os.path.exists(imgPath):
tparms.append((tparm[0], tparm[1], tparm[2], tparm[3],
fullPath, tparm[5]))
dtype = [('img_name', 'S40'), ('fwhm', float), ('ff_number', int),
('obj_num', int), ('img_path', 'S100'), ('ff_id', int)]
trst = np.array(tparms, dtype=dtype)
return trst
def doAstrometry(self, orsId, tparms):
doSuccess = False
sortParms = np.sort(tparms, order='fwhm')
for i in range(3):
runSuccess, tRaDec = self.makeTemplate(sortParms[i])
if runSuccess:
doSuccess = True
self.updateWCSCoors(orsId, tRaDec)
self.insertObsWcs(orsId, sortParms[i])
break
return doSuccess
def createWCS(self, startQueryNum=7, minNum=50):
totalNum = int(15038 / 10) + 1
stopFlag = False
while True:
time.sleep(10)
tobs = self.queryObs()
if tobs.shape[0] == 0:
break
for obs in tobs:
print(obs)
orsId = obs[0]
self.updateDoWCS(orsId)
camId = int(obs[3])
if int(orsId) < 80 or camId % 5 == 0:
continue
imgParms = self.queryImgParm(obs)
if imgParms.shape[0] > minNum:
timeObsUt = imgParms[:, 4]
minTime = np.min(timeObsUt)
maxTime = np.max(timeObsUt)
#self.log.debug(imgParms.shape[0])
tparms = self.checkImg(obs, imgParms)
tnum = tparms.shape[0]
tstr = "%s,%s,%s observe %s img, %d has parameter, %d backup to web server."\
%(obs[1], obs[2], obs[3], obs[4], imgParms.shape[0], tnum)
self.log.debug(tstr)
if tnum < minNum:
self.updateHasWCS(orsId, tnum, minTime, maxTime,
'false')
else:
if tnum > 100:
tparms = tparms[25:-25]
doSuccess = self.doAstrometry(orsId, tparms)
else:
doSuccess = self.doAstrometry(orsId, tparms)
if doSuccess:
self.updateHasWCS(orsId, tnum, minTime, maxTime,
'true')
def mainControl(self, camName, runName = 'p1', destDir='/data/gwac_diff_xy', toolPath='/home/gwac/img_diff_xy/image_diff'):
#os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152
#os.environ["CUDA_VISIBLE_DEVICES"] = ""
tools = AstroTools(toolPath)
logDest0 = "%s/log"%(destDir)
if not os.path.exists(logDest0):
os.system("mkdir -p %s"%(logDest0))
dateStr = datetime.strftime(datetime.utcnow(), "%Y%m%d")
dataDest0 = "%s/data/%s_%s"%(destDir, dateStr, runName)
if not os.path.exists(dataDest0):
os.system("mkdir -p %s"%(dataDest0))
imgDiff = GWACDiff(camName, dataDest0, tools)
tStr = "%s: start combine diff..."%(camName)
imgDiff.log.info(tStr)
imgDiff.sendMsg(tStr)
tdirs00 = "%s/data"%(destDir)
tdirs = os.listdir(tdirs00)
tdirs.sort()
for tdir in tdirs:
tproot00="%s/%s"%(tdirs00, tdir)
imgDiff.cleanData(tproot00)
#dataPath = '/home/xy/work/imgDiffTest2/fits/190128_G004_044'
#
#processRecord = '/home/xy/work/imgDiffTest3/log/G004_044.log'
#os.system("rm -rf %s"%(processRecord))
#sexLock=Lock()
self.loopNum = 1
while True:
try:
curUtcDateTime = datetime.utcnow()
tDateTime = datetime.utcnow()
startDateTime = tDateTime.replace(hour=9, minute=30, second=0) #9=17 1=9
endDateTime = tDateTime.replace(hour=9, minute=30, second=10) #22=6 8=16
remainSeconds1 = (startDateTime - curUtcDateTime).total_seconds()
remainSeconds2 = (endDateTime - curUtcDateTime).total_seconds()
if remainSeconds1<0 and remainSeconds2>0:
#if os.path.exists(dataDest0):
# os.system("rm -rf %s"%(dataDest0))
self.initData()
dateStr = datetime.strftime(datetime.utcnow(), "%Y%m%d")
dataDest0 = "%s/data/%s_%s"%(destDir, dateStr, runName)
if not os.path.exists(dataDest0):
os.system("mkdir -p %s"%(dataDest0))
imgDiff.reInitDataDir(dataDest0)
tStr = "%s: combine diff reInit data"%(camName)
print(tStr)
imgDiff.log.info(tStr)
imgDiff.sendMsg(tStr)
time.sleep(10)
except Exception as e:
tstr = traceback.format_exc()
print("combine diff reInit data error....")
print(tstr)
time.sleep(10)
try:
print("\n\nmain loop %d****************"%(self.loopNum))
self.srcExtract(camName, self.catList, self.alignTmplMap, imgDiff, logDest0, runName)
self.getAlignTemplate(camName, self.catList, self.alignTmplMap, imgDiff)
self.doAlign(camName, self.catList, self.alignTmplMap, self.alignList, imgDiff)
self.doDiff(camName, self.alignList, self.alignTmplMap, imgDiff, self.diffImgList)
self.loopNum = self.loopNum + 1
time.sleep(1)
except Exception as e:
tstr = traceback.format_exc()
print("Scheduling main error....")
print(tstr)
time.sleep(10)
def wcsAlign(srcPath00, dateStr, camName, curSkyId):
#toolPath = os.getcwd()
toolPath = '/data/work/program/image_diff'
tools = AstroTools(toolPath)
dataDest0 = "/data/gwac_diff_xy/data"
logDest0 = "/data/gwac_diff_xy/log"
if not os.path.exists(dataDest0):
os.system("mkdir -p %s" % (dataDest0))
if not os.path.exists(logDest0):
os.system("mkdir -p %s" % (logDest0))
startProcess = False
dayRun = 0
nigRun = 0
skyId = 0
ffId = 0
tfiles = []
#srcPath00='/data1/G004_041_190124'
#dateStr='190124'
#camName='G041'
#curSkyId='123'
dstDir = '%s/%s' % (dataDest0, dateStr)
tdiff = BatchImageDiff(srcPath00, dstDir, tools, camName, curSkyId)
try:
tfiles0 = os.listdir(tdiff.wcsDir)
tfiles0.sort()
print("total has %d images" % (len(tfiles0)))
tmpWCSIdx = int(len(tfiles0) / 2)
tmpWCSFile = tfiles0[tmpWCSIdx]
tmpWCS = WCS('%s/%s' % (tdiff.wcsDir, tmpWCSFile))
print("select %d %s as tmp WCS" % (tmpWCSIdx, tmpWCSFile))
for i, tname in enumerate(tfiles0):
if i == tmpWCSIdx:
tname00 = tname.split('.')[0] + '.fit'
tpath = "%s/%s" % (srcPath00, tname00)
tpathz = "%s/%s.fz" % (srcPath00, tname00)
if os.path.exists(tpath):
timgName = tname00
os.system("cp %s/%s %s/%s.wcs" %
(srcPath00, timgName, tdiff.remapDir, timgName))
elif os.path.exists(tpathz):
timgName = tname00 + '.fz'
os.system("cp %s/%s %s/%s" %
(srcPath00, timgName, tdiff.remapDir, timgName))
os.system("%s %s/%s" %
(tools.funpackProgram, tdiff.remapDir, timgName))
os.system("rm -rf %s/%s" % (tdiff.remapDir, timgName))
else:
print("%s not exist" % (tpath))
continue
else:
starttime = datetime.now()
print("remap %s" % (tname))
tdiff.reMapImg(tname, tmpWCS, srcPath00)
endtime = datetime.now()
runTime = (endtime - starttime).seconds
tdiff.log.info("totalTime %d seconds, %s" % (runTime, tname))
except Exception as e:
print(str(e))
tstr = traceback.format_exc()
print(tstr)
try:
if 'tdiff' in locals():
tStr = "diff error"
tdiff.log.info(tStr)
tdiff.sendMsg(tStr)
except Exception as e1:
print(str(e1))
tstr = traceback.format_exc()
print(tstr)
def run1():
#toolPath = os.getcwd()
toolPath = '/home/gwac/img_diff_xy/image_diff'
tools = AstroTools(toolPath)
dataDest0 = "/data/gwac_diff_xy/data"
logDest0 = "/data/gwac_diff_xy/log"
if not os.path.exists(dataDest0):
os.system("mkdir -p %s" % (dataDest0))
if not os.path.exists(logDest0):
os.system("mkdir -p %s" % (logDest0))
startProcess = False
dayRun = 0
nigRun = 0
skyId = 0
ffId = 0
tfiles = []
srcPath00 = '/data1/G004_041_190124'
dateStr = '190124'
camName = 'G041'
curSkyId = '123'
dstDir = '%s/%s' % (dataDest0, dateStr)
tdiff = BatchImageDiff(srcPath00, dstDir, tools, camName, curSkyId)
try:
tfiles0 = os.listdir(srcPath00)
tfiles0.sort()
for tfile in tfiles0:
if tfile.find(
'mon_objt') > 0: #G041_mon_objt_190124T10274817.fit.fz
tfiles.append(tfile[:33])
print("total has %d images" % (len(tfiles)))
ra0, dec0 = -1000, -1000
for i, timgName in enumerate(tfiles):
tpath = "%s/%s.fz" % (srcPath00, timgName)
if os.path.exists(tpath):
tStr = "start diff: %s" % (timgName)
tdiff.log.info(tStr)
starttime = datetime.now()
print("process %s" % (timgName))
wcsfile, ra_center, dec_center = tdiff.getWCS(
srcPath00, timgName, ra0, dec0)
ra0, dec0 = ra_center, dec_center
#os.system("cp %s/%s %s/%s"%(self.tmpCat, wcsfile, self.wcsDir, wcsfile))
endtime = datetime.now()
runTime = (endtime - starttime).seconds
tdiff.log.info("totalTime %d seconds, %s" %
(runTime, timgName))
else:
print("%s not exist" % (tpath))
if i > 5:
break
except Exception as e:
print(str(e))
tstr = traceback.format_exc()
print(tstr)
try:
if 'tdiff' in locals():
tStr = "diff error"
tdiff.log.info(tStr)
tdiff.sendMsg(tStr)
except Exception as e1:
print(str(e1))
tstr = traceback.format_exc()
print(tstr)
def wcsAlign2():
#toolPath = os.getcwd()
toolPath = '/home/xy/Downloads/myresource/deep_data2/image_diff'
tools = AstroTools(toolPath)
dataDest0 = "tmp/data"
logDest0 = "tmp/log"
if not os.path.exists(dataDest0):
os.system("mkdir -p %s" % (dataDest0))
if not os.path.exists(logDest0):
os.system("mkdir -p %s" % (logDest0))
startProcess = False
dayRun = 0
nigRun = 0
skyId = 0
ffId = 0
tfiles = []
srcPath00 = '/home/xy/test7'
dateStr = '190124'
camName = 'G041'
curSkyId = '123'
dstDir = '%s/%s' % (dataDest0, dateStr)
tdiff = BatchImageDiff(srcPath00, dstDir, tools, camName, curSkyId)
tdiff.wcsDir = srcPath00
try:
tfiles0 = os.listdir(srcPath00)
tfiles0.sort()
tmpWCSFile = 'G031_mon_objt_190116T21321726.wcs'
tmpWCS = WCS('%s/%s' % (tdiff.wcsDir, tmpWCSFile))
print("select %s as tmp WCS" % (tmpWCSFile))
for i, tname in enumerate(tfiles0):
if tname.find('wcs') > 0 and tname.find(
tmpWCSFile.split('.')[0]) < 0:
starttime = datetime.now()
print("remap %s" % (tname))
tdiff.reMapImg(tname, tmpWCS, srcPath00)
endtime = datetime.now()
runTime = (endtime - starttime).seconds
tdiff.log.info("totalTime %d seconds, %s" % (runTime, tname))
except Exception as e:
print(str(e))
tstr = traceback.format_exc()
print(tstr)
try:
if 'tdiff' in locals():
tStr = "diff error"
tdiff.log.info(tStr)
tdiff.sendMsg(tStr)
except Exception as e1:
print(str(e1))
tstr = traceback.format_exc()
print(tstr)
def run1():
#toolPath = os.getcwd()
toolPath = '/home/gwac/img_diff_xy/image_diff'
tools = AstroTools(toolPath)
query = QueryData()
dataDest0 = "/data/gwac_diff_xy/data"
logDest0 = "/data/gwac_diff_xy/log"
if not os.path.exists(dataDest0):
os.system("mkdir -p %s" % (dataDest0))
if not os.path.exists(logDest0):
os.system("mkdir -p %s" % (logDest0))
startProcess = False
dayRun = 0
nigRun = 0
skyId = 0
ffId = 0
tfiles = []
srcPath00 = '/data2/G003_031_190201'
dateStr = '190301'
camName = 'G031'
curSkyId = '123'
dstDir = '%s/%s' % (dataDest0, dateStr)
tdiff = BatchImageDiff(srcPath00, dstDir, tools, camName, curSkyId)
try:
tfiles0 = os.listdir(srcPath00)
tfiles0.sort()
for tfile in tfiles0:
if tfile >= 'G031_mon_objt_190201T17001303.fit.fz' and tfile <= 'G031_mon_objt_190201T18595803.fit.fz':
tfiles.append(tfile[:33])
print("total has %d images" % (len(tfiles)))
for i, timgName in enumerate(tfiles):
tpath = "%s/%s.fz" % (srcPath00, timgName)
if os.path.exists(tpath):
tStr = "start diff: %s" % (timgName)
tdiff.log.info(tStr)
starttime = datetime.now()
tdiff.processImg(srcPath00, timgName, i + 1)
endtime = datetime.now()
runTime = (endtime - starttime).seconds
tdiff.log.info("totalTime %d seconds, %s" %
(runTime, timgName))
else:
print("%s not exist" % (tpath))
except Exception as e:
print(str(e))
tstr = traceback.format_exc()
print(tstr)
try:
if 'tdiff' in locals():
tStr = "diff error"
tdiff.log.info(tStr)
tdiff.sendMsg(tStr)
except Exception as e1:
print(str(e1))
tstr = traceback.format_exc()
print(tstr)
def mainControl(self,
camName,
runName='p1',
destDir='/data/gwac_diff_xy',
toolPath='/home/gwac/img_diff_xy/image_diff'):
#os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152
#os.environ["CUDA_VISIBLE_DEVICES"] = ""
dateStr = datetime.strftime(datetime.utcnow(), "%Y%m%d")
dataDest0 = "%s/data/%s_%s" % (destDir, dateStr, runName)
logDest0 = "%s/log_%s" % (destDir, runName)
if not os.path.exists(logDest0):
os.system("mkdir -p %s" % (logDest0))
if not os.path.exists(dataDest0):
os.system("mkdir -p %s" % (dataDest0))
tools = AstroTools(toolPath)
imgDiff = GWACDiff(camName, dataDest0, tools)
tStr = "start diff..."
imgDiff.log.info(tStr)
imgDiff.sendMsg(tStr)
#dataPath = '/home/xy/work/imgDiffTest2/fits/190128_G004_044'
#
#processRecord = '/home/xy/work/imgDiffTest3/log/G004_044.log'
#os.system("rm -rf %s"%(processRecord))
#sexLock=Lock()
loopNum = 1
while True:
try:
tIdx1 = loopNum % 5
''' '''
if tIdx1 == 1 and self.catRunning.value == 0: #cat
#catJob = Process(target=srcExtractLocalDir, args=(dataPath, camName, self.catList, imgDiff, logDest0, self.catRunning))
catJob = Process(target=srcExtract,
args=(camName, self.catList, imgDiff,
logDest0, self.catRunning))
catJob.start()
tIdx2 = loopNum % 15
#tIdx2 = loopNum%5
if tIdx2 == 0 and self.alignTmplRunning.value == 0: #template getAlignTemplateLocal
alignTmplJob = Process(target=getAlignTemplate,
args=(camName, self.catList,
self.alignTmplMap,
self.alignTmplIdx, imgDiff,
self.alignTmplRunning))
alignTmplJob.start()
if self.alignRunning.value == 0: #align
alignJob = Process(target=doAlign,
args=(camName, self.catList,
self.alignTmplMap, self.alignIdx,
self.alignList, imgDiff,
self.alignRunning))
alignJob.start()
if self.cmbRunning.value == 0: #combine5
#print("%d cmbJob is not running, catNum=%d, alignNum=%d, cmbIdx=%d"%(loopNum, len(self.catList), len(self.alignList), self.cmbIdx.value))
cmbJob = Process(
target=doCombine,
args=(camName, self.alignList, self.alignTmplMap,
self.cmbIdx, imgDiff, self.cmbImgList,
self.cmbRunning, self.crossTaskParms, 5))
cmbJob.start()
#else:
# print("%d cmbJob is running"%(loopNum))
if self.cmbCatRunning.value == 0: #cat of combine5
cmbCatJob = Process(target=srcExtractCombine,
args=(camName, self.cmbImgList,
self.cmbCatList, self.cmbCatIdx,
imgDiff, self.cmbCatRunning))
cmbCatJob.start()
#else:
# print("%d cmbCatJob is running"%(loopNum))
tIdx6 = loopNum % 15
#tIdx6 = loopNum%1
if tIdx6 == 0 and self.diffTmplRunning.value == 0: #cat of combine5
diffTmplJob = Process(
target=getDiffTemplate,
args=(camName, self.cmbCatList, self.alignTmplMap,
self.diffTmplMap, self.diffTmplIdx, imgDiff,
self.diffTmplRunning))
diffTmplJob.start()
#else:
# print("%d diffTmplJob is running"%(loopNum))
tIdx7 = loopNum % 5
if tIdx7 == 0:
if self.diffRunning.value == 0: #diff
#print("%d doDiff start run"%(loopNum))
#diffJob = Process(target=doDiff, args=(
# camName, self.cmbImgList, self.diffTmplMap, self.diffIdx, imgDiff, self.diffImgList, self.diffRunning))
diffJob = Process(target=doDiff,
args=(camName, self.cmbCatList,
self.diffTmplMap, self.diffIdx,
imgDiff, self.diffImgList,
self.diffRunning))
diffJob.start()
#else:
# print("%d doDiff is running"%(loopNum))
tIdx8 = loopNum % 2
if tIdx8 == 0:
if self.recgRunning.value == 0: #recognition
print("%d doRecognitionAndUpload start run" %
(loopNum))
#recgJob = Process(target=doRecognitionAndUpload, args=(
# camName, self.diffImgList, self.diffTmplMap, self.recgIdx, imgDiff, self.recgRunning))
#recgJob.start()
doRecognitionAndUpload(camName, self.diffImgList,
self.diffTmplMap, self.recgIdx,
imgDiff, self.recgRunning)
else:
print("%d doRecognitionAndUpload is running" %
(loopNum))
loopNum = loopNum + 1
#if loopNum>4000:
# break
time.sleep(1)
'''
try:
catJob.join()
alignJob.join()
cmbJob.join()
except NameError as e:
tstr = traceback.format_exc()
print("catJob is not defined")
print(tstr)
'''
except Exception as e:
tstr = traceback.format_exc()
print("Scheduling main error....")
print(tstr)
time.sleep(10)
def run1(srcPath00, dateStr, camName, curSkyId):
#toolPath = os.getcwd()
toolPath = '/home/gwac/img_diff_xy/image_diff'
tools = AstroTools(toolPath)
dataDest0 = "/data/gwac_diff_xy/data"
logDest0 = "/data/gwac_diff_xy/log"
if not os.path.exists(dataDest0):
os.system("mkdir -p %s" % (dataDest0))
if not os.path.exists(logDest0):
os.system("mkdir -p %s" % (logDest0))
startProcess = False
dayRun = 0
nigRun = 0
skyId = 0
ffId = 0
tfiles = []
#srcPath00='/data1/G004_041_190124'
#dateStr='190124'
#camName='G041'
#curSkyId='123'
dstDir = '%s/%s' % (dataDest0, dateStr)
tdiff = BatchImageDiff(srcPath00, dstDir, tools, camName, curSkyId)
try:
camId = (int(camName[1]) - 1) * 5 + int(camName[2])
sql1 = "select img_name " \
"from fits_file2_his " \
"where sky_id=%s and cam_id=%s and substr(img_path, 17, 6)='%s' " \
"order by img_name"%(curSkyId, camId, dateStr)
print(sql1)
tquery = QueryData()
tfiles = tquery.query(sql1)
print("total has %d images" % (len(tfiles)))
ra0, dec0 = -1000, -1000
for i, tname in enumerate(tfiles):
tname00 = tname[0]
tpath = "%s/%s" % (srcPath00, tname00)
tpathz = "%s/%s.fz" % (srcPath00, tname00)
if os.path.exists(tpath):
timgName = tname00
elif os.path.exists(tpathz):
timgName = tname00 + '.fz'
else:
print("%s not exist" % (tpath))
continue
tStr = "start diff: %s" % (timgName)
tdiff.log.info(tStr)
starttime = datetime.now()
print("process %s" % (timgName))
wcsfile, ra_center, dec_center = tdiff.getWCS(
srcPath00, timgName, ra0, dec0)
ra0, dec0 = ra_center, dec_center
os.system(
"cp %s/%s %s/%s.wcs" %
(tdiff.tmpDir, wcsfile, tdiff.wcsDir, timgName.split('.')[0]))
endtime = datetime.now()
runTime = (endtime - starttime).seconds
tdiff.log.info("totalTime %d seconds, %s" % (runTime, timgName))
except Exception as e:
print(str(e))
tstr = traceback.format_exc()
print(tstr)
try:
if 'tdiff' in locals():
tStr = "diff error"
tdiff.log.info(tStr)
tdiff.sendMsg(tStr)
except Exception as e1:
print(str(e1))
tstr = traceback.format_exc()
print(tstr)
#subImg = getWindowImgs(tpath1, fname, x, y, size)
subImg = getWindowImgs(storePath, fname, x, y, size)
subImgz = zscale_image(subImg)
subImgz = scipy.ndimage.zoom(subImgz, 4, order=0)
imgpre = fname.split(".")[0]
tImgName = "%s.jpg" % (imgpre)
savePath = "%s/%s" % (workPath, tImgName)
Image.fromarray(subImgz).save(savePath)
if __name__ == "__main__":
toolPath = '/data/work/program/image_diff'
tools = AstroTools(toolPath)
tpath = '/data3/G004_044_190305'
fname = 'G044_mon_objt_190305T13393793.fit'
x = 1644.03
y = 1700.2
ra0 = 203.81125
dec0 = 44.76603
otName = 'G190116_C19152'
#runSuccess, ot2ReRa, ot2ReDec = reAstrometry(tools, tpath, fname, x, y, ra0, dec0, otName)
startDate = '2018-12-17'
endDate = '2019-01-17'
skyListFile = getSkyList(tools, otName, ra0, dec0, startDate, endDate)
def run1(camName):
#toolPath = os.getcwd()
toolPath = '/home/gwac/img_diff_xy/image_diff'
tools = AstroTools(toolPath)
query = QueryData()
dataDest0 = "/data/gwac_diff_xy/data"
logDest0 = "/data/gwac_diff_xy/log"
if not os.path.exists(dataDest0):
os.system("mkdir -p %s"%(dataDest0))
if not os.path.exists(logDest0):
os.system("mkdir -p %s"%(logDest0))
startProcess = False
dayRun = 0
nigRun = 0
skyId = 0
ffId = 0
tfiles = []
while True:
curUtcDateTime = datetime.utcnow()
tDateTime = datetime.utcnow()
startDateTime = tDateTime.replace(hour=9, minute=30, second=0) #9=17 1=9
endDateTime = tDateTime.replace(hour=22, minute=30, second=0) #22=6 8=16
remainSeconds1 = (startDateTime - curUtcDateTime).total_seconds()
remainSeconds2 = (endDateTime - curUtcDateTime).total_seconds()
if remainSeconds1<0 and remainSeconds2>0:
dayRun = 0
try:
tfiles = query.getFileList(camName, ffId)
#print(tfiles)
for tfile in tfiles:
curFfId = tfile[0]
ffNumber = tfile[1]
curSkyId = tfile[2]
timgName = tfile[3] #G021_tom_objt_190109T13531492.fit
tpath = tfile[4] #/data3/G002_021_190109/G021_tom_objt_190109T13531492.fit
imgDate = timgName[14:20]
pathDate = tpath[16:22]
if imgDate!=pathDate:
ffId=curFfId
startProcess = False
continue
elif not startProcess:
ffId=0
startProcess = True
srcDir= tpath[:(tpath.find(camName)-1)] #/data3/G002_021_190109
dateStr = srcDir[srcDir.find('G'):] #G002_021_190109
logfName0 = '%s/%s.log'%(logDest0, dateStr)
if ffId==0:
if os.path.exists(logfName0) and os.stat(logfName0).st_size > 0:
tlastLine = getLastLine(logfName0)
if len(tlastLine)>2:
ffId=int(tlastLine.strip())
if skyId!=curSkyId and curFfId>ffId:
dstDir='%s/%s'%(dataDest0, dateStr)
tdiff = BatchImageDiff(srcDir, dstDir, tools, camName, curSkyId)
tStr = "start diff: %s"%(timgName)
tdiff.log.info(tStr)
tdiff.sendMsg(tStr)
if curFfId>ffId:
skyId=curSkyId
ffId=curFfId
logfile0 = open(logfName0, 'a')
logfile0.write("\n\n%d\n"%(ffId))
logfile0.close()
tpathfz = "%s.fz"%(tpath)
if os.path.exists(tpath) or os.path.exists(tpathfz):
starttime = datetime.now()
tdiff.processImg(srcDir, timgName, ffNumber)
endtime = datetime.now()
runTime = (endtime - starttime).seconds
tdiff.log.info("totalTime %d seconds, sky:%d, ffNum:%d, %s"%(runTime, curSkyId, ffNumber, timgName))
else:
print("%s not exist"%(tpath))
#if curFfId>ffId:
# break
except Exception as e:
print(str(e))
tstr = traceback.format_exc()
print(tstr)
try:
if 'tdiff' in locals():
tStr = "diff error"
tdiff.log.info(tStr)
tdiff.sendMsg(tStr)
except Exception as e1:
print(str(e1))
tstr = traceback.format_exc()
print(tstr)
if len(tfiles)==0:
time.sleep(5)
nigRun = nigRun+1
#if nigRun>=1:
# break
else:
# day temp file clean
try:
if ('tdiff' in locals()) and (dayRun==0):
tdiff.initReg(0)
except Exception as e1:
print(str(e1))
tstr = traceback.format_exc()
print(tstr)
nigRun = 0
dayRun = dayRun+1
skyId = 0
ffId = 0
startProcess = False
if dayRun%6==1:
print("day %d wait"%(dayRun))
time.sleep(10*60)