def run1(): #toolPath = os.getcwd() toolPath = '/home/gwac/img_diff_xy/image_diff' tools = AstroTools(toolPath) dataDest0 = "/data3/simulationTest" if not os.path.exists(dataDest0): os.system("mkdir -p %s" % (dataDest0)) srcPath00 = '/data2/G003_031_190116/f_20620425' dateStr = '20190325' camName = 'G031' curSkyId = '123' dataDest = '%s/%s' % (dataDest0, dateStr) otRcgDir = "%s/otrcg" % (dataDest) diffCatDir = "%s/cmbDiffCat" % (dataDest) diffDir = "%s/cmbDiff" % (dataDest) objDir = "%s/cmbFits" % (dataDest) if not os.path.exists(otRcgDir): os.system("mkdir -p %s" % (otRcgDir)) tsim = BatchImageSim(srcPath00, dataDest, tools, camName, curSkyId) tsim.log.info("\n\n***************\nstart recognition diff OTs..\n") tsim.batchRecognition(objDir, diffDir, diffCatDir, otRcgDir) tsim.genRegFile(otRcgDir)
def run1(): #toolPath = os.getcwd() toolPath = '/home/gwac/img_diff_xy/image_diff' tools = AstroTools(toolPath) dataDest0 = "/data3/simulationTest" if not os.path.exists(dataDest0): os.system("mkdir -p %s" % (dataDest0)) srcPath00 = '/data2/G003_031_190116/f_20620425' dateStr = '20190324' camName = 'G031' curSkyId = '123' dataDest = '%s/%s' % (dataDest0, dateStr) dCatDir = "%s/dcats" % (dataDest) simFitsDir = "%s/simFits" % (dataDest) alignFitsDir = "%s/alignFits" % (dataDest) alignCatsDir = "%s/alignCats" % (dataDest) resiFitDir = "%s/resiFit" % (dataDest) resiCatDir = "%s/resiCat" % (dataDest) preViewDir = "%s/preview" % (dataDest) preViewSimResiDir = "%s/previewSimResi" % (dataDest) simCatAddDir = "%s/simCatAdd" % (dataDest) if not os.path.exists(dCatDir): os.system("mkdir -p %s" % (dCatDir)) if not os.path.exists(simFitsDir): os.system("mkdir -p %s" % (simFitsDir)) if not os.path.exists(alignFitsDir): os.system("mkdir -p %s" % (alignFitsDir)) if not os.path.exists(alignCatsDir): os.system("mkdir -p %s" % (alignCatsDir)) if not os.path.exists(resiFitDir): os.system("mkdir -p %s" % (resiFitDir)) if not os.path.exists(resiCatDir): os.system("mkdir -p %s" % (resiCatDir)) if not os.path.exists(preViewDir): os.system("mkdir -p %s" % (preViewDir)) if not os.path.exists(preViewSimResiDir): os.system("mkdir -p %s" % (preViewSimResiDir)) if not os.path.exists(simCatAddDir): os.system("mkdir -p %s" % (simCatAddDir)) tsim = BatchImageSim(srcPath00, dataDest, tools, camName, curSkyId) tsim.log.info("\n\n***************\nstart run Sextractor..\n") tsim.getCats(srcPath00, dCatDir) tsim.log.info("\n\n***************\nstart align image..\n") tsim.imgAlign() tsim.log.info("\n\n***************\nstart sim image..\n") tsim.imgSimulate(alignFitsDir, simFitsDir, simCatAddDir) tsim.log.info("\n\n***************\nstart diff image..\n") tsim.imgDiff(alignFitsDir, resiFitDir)
def run3(): toolPath = os.getcwd() tools = AstroTools(toolPath) dataRoot = "/data/gwac_data/gwac_orig_fits" dataDest = "/data/gwac_data/gwac_simot/data_1231" #fileList = [(2575,14,'181208'),(2573,19,'181208'),(2574,9,'181206'),(2576,16,'181206')] fileList = [(2573, 19, '181208'), (2574, 9, '181206')] query = QueryData() print(fileList) for tflist in fileList: files = query.getFileList(tflist[0], tflist[1], tflist[2]) total = len(files) #continue ccd = files[0][1] #G004_041 ccdDir = "G0%s_%s" % (ccd[:2], ccd) srcDir = "%s/%s/%s" % (dataRoot, tflist[2], ccdDir) dstDir = "%s/%s/G%s" % (dataDest, tflist[2], ccd) tdiff = BatchImageDiff(srcDir, dstDir, tools) tStr = "start imageDiff, %s,skyId:%d, camId:%d, imgNum:%d" % ( tflist[2], tflist[0], tflist[1], total) tdiff.log.info(tStr) tdiff.tools.sendTriggerMsg(tStr) tdiff.batchSim3(files) tStr = "end imageDiff, %s,skyId:%d, camId:%d, imgNum:%d" % ( tflist[2], tflist[0], tflist[1], total) tdiff.log.info(tStr) tdiff.tools.sendTriggerMsg(tStr)
def getImgCenter(tpath, imgName, x, y): tools = AstroTools('/home/xy/Downloads/myresource/deep_data2/image_diff') fieldId, ra, dec = tools.getRaDec(tpath, imgName) fpar = 'sex_diff.par' sexConf = [ '-DETECT_MINAREA', '10', '-DETECT_THRESH', '5', '-ANALYSIS_THRESH', '5', '-CATALOG_TYPE', 'FITS_LDAC' ] tmplCat, isSuccess = tools.runSextractor(imgName, tpath, tpath, fpar, sexConf, outSuffix='_ldac.fit') if not isSuccess: print("getDiffTemplate runSextractor failure2") return isSuccess, 0, 0 tools.ldac2fits('%s/%s' % (tpath, tmplCat), '%s/ti_cat.fit' % (tpath)) runSuccess = tools.runWCS(tpath, 'ti_cat.fit', ra, dec) if runSuccess: wcs = WCS('%s/ti_cat.wcs' % (tpath)) ra, dec = wcs.all_pix2world(x, y, 1) return runSuccess, ra, dec
def wcsRemap(): #toolPath = os.getcwd() toolPath = '/home/xy/Downloads/myresource/deep_data2/image_diff' tools = AstroTools(toolPath) dataDest0 = "tmp/data" logDest0 = "tmp/log" if not os.path.exists(dataDest0): os.system("mkdir -p %s" % (dataDest0)) if not os.path.exists(logDest0): os.system("mkdir -p %s" % (logDest0)) startProcess = False dayRun = 0 nigRun = 0 skyId = 0 ffId = 0 tfiles = [] srcPath00 = '/home/xy/test7' dateStr = '190124' camName = 'G041' curSkyId = '123' dstDir = '%s/%s' % (dataDest0, dateStr) tdiff = BatchImageDiff(srcPath00, dstDir, tools, camName, curSkyId) try: tfiles0 = os.listdir(srcPath00) tfiles0.sort() ra0, dec0 = -1000, -1000 for timgName in tfiles0: if timgName.find('fit') > 0: wcsfile, ra_center, dec_center = tdiff.getWCS( srcPath00, timgName, ra0, dec0) ra0, dec0 = ra_center, dec_center print("%f, %f" % (ra0, dec0)) os.system( "cp %s/%s %s/%s.wcs" % (tdiff.tmpDir, wcsfile, srcPath00, timgName.split('.')[0])) except Exception as e: print(str(e)) tstr = traceback.format_exc() print(tstr) try: if 'tdiff' in locals(): tStr = "diff error" tdiff.log.info(tStr) tdiff.sendMsg(tStr) except Exception as e1: print(str(e1)) tstr = traceback.format_exc() print(tstr)
def run1(): toolPath = os.getcwd() tools = AstroTools(toolPath) dataRoot = "/data/gwac_data/gwac_orig_fits" dataDest = "/data/gwac_data/gwac_simot/data_1231" tdiff = BatchImageDiff(dataRoot, dataDest, tools) tdiff.batchSim()
def run2(): toolPath = os.getcwd() tools = AstroTools(toolPath) dataRoot = "/home/xy/Downloads/myresource/deep_data2/G180216/17320495.0" dataDest = "/home/xy/Downloads/myresource/deep_data2/simot/data_190101" tdiff = BatchImageDiff(dataRoot, dataDest, tools) tdiff.batchSim2()
def test(): toolPath = '/home/xy/Downloads/myresource/deep_data2/image_diff' #os.getcwd() tools = AstroTools(toolPath) file1 = 'G034_mon_objt_190211T12192603.cat' file2 = 'G034_mon_objt_190227T12304321.cat' file3 = 'G034_mon_objt_190324T12222033.cat' file4 = 'G044_mon_objt_190305T13070793.cat' posReg(tools, file1, file2) posReg(tools, file1, file3) posReg(tools, file1, file4)
def __init__(self): self.conn = False self.imgSize = (4136, 4096) self.templateImg = 'ti.fit' self.tmpRoot = "/dev/shm/gwacWCS" self.templateDir = "%s/tmpl" % (self.tmpRoot) if not os.path.exists(self.templateDir): os.system("mkdir -p %s" % (self.templateDir)) self.toolPath = '/data/work/program/image_diff' self.funpackProgram = "%s/tools/cfitsio/funpack" % (self.toolPath) logPath = os.getcwd() self.tools = AstroTools(self.toolPath, logPath) self.log = self.tools.log
def run1(): #storePath = '/home/xy/work/imgDiffTest2' storePath = '/home/xy/work/imgDiffTest2/combine2' #toolPath = os.getcwd() toolPath = '/home/xy/Downloads/myresource/deep_data2/image_diff' tools = AstroTools(toolPath) dataDest0 = "/home/xy/work/imgDiffTest2/combineCat" logDest0 = "/home/xy/work/gwac_diff_xy/log" if not os.path.exists(dataDest0): os.system("mkdir -p %s" % (dataDest0)) if not os.path.exists(logDest0): os.system("mkdir -p %s" % (logDest0)) tdates = os.listdir(storePath) print("total has %d date&ccd" % (len(tdates))) for i, tdate in enumerate(tdates): dstDir = '%s/%s' % (dataDest0, tdate) tdiff = BatchImageDiff(storePath, dstDir, tools) tpath1 = '%s/%s' % (storePath, tdate) timgs = os.listdir(tpath1) timgs.sort() print("total has %d images" % (len(timgs))) for j, tname in enumerate(timgs): try: tpath = "%s/%s" % (tpath1, tname) if not os.path.exists(tpath): print("%s not exist" % (tpath)) continue print("process %d %s" % (j, tname)) tdiff.getCat(tpath1, tname) except Exception as e: print(str(e)) tstr = traceback.format_exc() print(tstr)
def mainControl(self, camName, runName='p1', destDir='/data/gwac_diff_xy', toolPath='/home/gwac/img_diff_xy/image_diff'): tools = AstroTools(toolPath) logDest0 = "%s/log" % (destDir) if not os.path.exists(logDest0): os.system("mkdir -p %s" % (logDest0)) dateStr = datetime.strftime(datetime.utcnow(), "%Y%m%d") dataDest0 = "%s/data/%s_%s" % (destDir, dateStr, runName) if not os.path.exists(dataDest0): os.system("mkdir -p %s" % (dataDest0)) imgDiff = GWACDiff(camName, dataDest0, tools) tStr = "%s: start combine diff..." % (camName) imgDiff.log.info(tStr) imgDiff.sendMsg(tStr) dataPath = '/home/xy/Downloads/myresource/SuperNova20190113/stampImage/190101_G004_041_test2' try: self.srcExtract(dataPath, camName, self.catList, imgDiff, logDest0, runName) self.getAlignTemplate(camName, self.catList, self.alignTmplMap, imgDiff) self.doAlign(camName, self.catList, self.alignTmplMap, self.alignList, imgDiff) #self.doCombine(camName, self.alignList, self.alignTmplMap, imgDiff, self.cmbImgList, self.crossTaskParms, runName, 5) self.srcExtractCombine(camName, self.alignList, self.cmbCatList, imgDiff) self.getDiffTemplate(camName, self.cmbCatList, self.alignTmplMap, self.diffTmplMap, imgDiff) self.doDiff(camName, self.cmbCatList, self.diffTmplMap, imgDiff, self.diffImgList) #self.doRecognitionAndUpload(camName, self.diffImgList, self.diffTmplMap, imgDiff, runName) except Exception as e: tstr = traceback.format_exc() print("Scheduling main error....") print(tstr) time.sleep(10)
def run2(): toolPath = os.getcwd() tools = AstroTools(toolPath) dataRoot = "/home/xy/Downloads/myresource/deep_data2/G180216/17320495.0" dataDest = "/home/xy/Downloads/myresource/deep_data2/simot/data_190107" flist = os.listdir(dataRoot) flist.sort() imgs = [] for tfilename in flist: if tfilename.find("fit") > -1: imgs.append(tfilename) print("total image %d" % (len(imgs))) tdiff = BatchImageDiff(dataRoot, dataDest, tools) tdiff.batchSim(imgs)
def run1(): #toolPath = os.getcwd() toolPath = '/home/gwac/img_diff_xy/image_diff' tools = AstroTools(toolPath) dateStr = '20190413' camName = 'G031' curSkyId = '123' srcPath00 = '/data/gwac_diff_xy/combine/%s' % (dateStr) dataDest0 = "/data/gwac_diff_xy/combineRst/%s" % (dateStr) if not os.path.exists(dataDest0): os.system("mkdir -p %s" % (dataDest0)) tsim = BatchImageSim(srcPath00, dataDest0, tools, camName, curSkyId) tsim.log.info("\n\n***************\nstart diff simCombine image..\n") tsim.process(srcPath00, dataDest0)
def run1(): toolPath = os.getcwd() tools = AstroTools(toolPath) dateStr = datetime.strftime(datetime.now(), "%Y%m%d") dataRoot = "/data/gwac_data/gwac_orig_fits" #dataDest = "/data/gwac_data/gwac_simot/imgdiff_%s"%(dateStr) dataDest = "/data/gwac_data/gwac_simot/imgdiff" query = QueryData() #filesNum = query.queryFilesNum() filesNum = query.queryFilesNumHis() for tnum in filesNum: if tnum[3] > 100 and tnum[2] % 5 > 0: #files = query.getFileList(tnum[1], tnum[2], tnum[0]) files = query.getFileListHis(tnum[1], tnum[2], tnum[0]) total = len(files) ccd = files[0][1] #G004_041 ccdDir = "G0%s_%s" % (ccd[:2], ccd) tflist = (tnum[1], tnum[2], tnum[0]) srcDir = "%s/%s/%s" % (dataRoot, tflist[2], ccdDir) dstDir = "%s/%s/G%s" % (dataDest, tflist[2], ccd) tdiff = BatchImageDiff(srcDir, dstDir, tools) tStr = "start imageDiff, %s,skyId:%d, camId:%d, imgNum:%d" % ( tflist[2], tflist[0], tflist[1], total) tdiff.log.info(tStr) tdiff.tools.sendTriggerMsg(tStr) tdiff.batchSim(files) tStr = "end imageDiff, %s,skyId:%d, camId:%d, imgNum:%d" % ( tflist[2], tflist[0], tflist[1], total) tdiff.log.info(tStr) tdiff.tools.sendTriggerMsg(tStr)
def run1(): #toolPath = os.getcwd() toolPath = '/home/gwac/img_diff_xy/image_diff' tools = AstroTools(toolPath) query = QueryData() dataDest0 = "/data/gwac_diff_xy/data" logDest0 = "/data/gwac_diff_xy/log" if not os.path.exists(dataDest0): os.system("mkdir -p %s" % (dataDest0)) if not os.path.exists(logDest0): os.system("mkdir -p %s" % (logDest0)) startProcess = False dayRun = 0 nigRun = 0 skyId = 0 ffId = 0 tfiles = [] srcPath00 = '/data2/G003_031_190201' dateStr = '190301' camName = 'G031' curSkyId = '123' dstDir = '%s/%s' % (dataDest0, dateStr) tdiff = BatchImageDiff(srcPath00, dstDir, tools, camName, curSkyId) try: tfiles0 = os.listdir(srcPath00) tfiles0.sort() for tfile in tfiles0: if tfile >= 'G031_mon_objt_190201T17001303.fit.fz' and tfile <= 'G031_mon_objt_190201T18595803.fit.fz': tfiles.append(tfile[:33]) print("total has %d images" % (len(tfiles))) for i, timgName in enumerate(tfiles): tpath = "%s/%s.fz" % (srcPath00, timgName) if os.path.exists(tpath): tStr = "start diff: %s" % (timgName) tdiff.log.info(tStr) starttime = datetime.now() tdiff.processImg(srcPath00, timgName, i + 1) endtime = datetime.now() runTime = (endtime - starttime).seconds tdiff.log.info("totalTime %d seconds, %s" % (runTime, timgName)) else: print("%s not exist" % (tpath)) except Exception as e: print(str(e)) tstr = traceback.format_exc() print(tstr) try: if 'tdiff' in locals(): tStr = "diff error" tdiff.log.info(tStr) tdiff.sendMsg(tStr) except Exception as e1: print(str(e1)) tstr = traceback.format_exc() print(tstr)
def run1(): #toolPath = os.getcwd() toolPath = '/home/gwac/img_diff_xy/image_diff' tools = AstroTools(toolPath) dataDest0 = "/data/gwac_diff_xy/data" logDest0 = "/data/gwac_diff_xy/log" if not os.path.exists(dataDest0): os.system("mkdir -p %s" % (dataDest0)) if not os.path.exists(logDest0): os.system("mkdir -p %s" % (logDest0)) startProcess = False dayRun = 0 nigRun = 0 skyId = 0 ffId = 0 tfiles = [] srcPath00 = '/data1/G004_041_190124' dateStr = '190124' camName = 'G041' curSkyId = '123' dstDir = '%s/%s' % (dataDest0, dateStr) tdiff = BatchImageDiff(srcPath00, dstDir, tools, camName, curSkyId) try: tfiles0 = os.listdir(srcPath00) tfiles0.sort() for tfile in tfiles0: if tfile.find( 'mon_objt') > 0: #G041_mon_objt_190124T10274817.fit.fz tfiles.append(tfile[:33]) print("total has %d images" % (len(tfiles))) ra0, dec0 = -1000, -1000 for i, timgName in enumerate(tfiles): tpath = "%s/%s.fz" % (srcPath00, timgName) if os.path.exists(tpath): tStr = "start diff: %s" % (timgName) tdiff.log.info(tStr) starttime = datetime.now() print("process %s" % (timgName)) wcsfile, ra_center, dec_center = tdiff.getWCS( srcPath00, timgName, ra0, dec0) ra0, dec0 = ra_center, dec_center #os.system("cp %s/%s %s/%s"%(self.tmpCat, wcsfile, self.wcsDir, wcsfile)) endtime = datetime.now() runTime = (endtime - starttime).seconds tdiff.log.info("totalTime %d seconds, %s" % (runTime, timgName)) else: print("%s not exist" % (tpath)) if i > 5: break except Exception as e: print(str(e)) tstr = traceback.format_exc() print(tstr) try: if 'tdiff' in locals(): tStr = "diff error" tdiff.log.info(tStr) tdiff.sendMsg(tStr) except Exception as e1: print(str(e1)) tstr = traceback.format_exc() print(tstr)
def run1(): storePath = '/data/gwac_data/gwac_orig_fits' #toolPath = os.getcwd() toolPath = '/data/work/program/image_diff' tools = AstroTools(toolPath) dataDest0 = "/data/gwac_diff_xy/data" logDest0 = "/data/gwac_diff_xy/log" if not os.path.exists(dataDest0): os.system("mkdir -p %s"%(dataDest0)) if not os.path.exists(logDest0): os.system("mkdir -p %s"%(logDest0)) tfiles = [] #srcPath00='/data1/G004_041_190124' dateStr='190620' camName='G041' curSkyId='123' dstDir='%s/%s'%(dataDest0, dateStr) tdiff = BatchImageDiff(storePath, dstDir, tools, camName, curSkyId) skyId = 2577 sql1 = "select date_str, cam_id from file_number where sky_id=%d ORDER BY date_str, cam_id"%(skyId) print(sql1) tquery = QueryData() tdates = tquery.query(sql1) print("total has %d date&ccd"%(len(tdates))) for i, tdate in enumerate(tdates): try: dateStr = tdate[0][2:8] camId = tdate[1] mountNum = int(camId/5) + 1 ccdNum = camId%5 if ccdNum==0: continue camName = '0%d%d'%(mountNum, ccdNum) tpath1 = '%s/%s/G00%d_%s'%(storePath, dateStr, mountNum,camName) sql1 = "select img_name " \ "from fits_file2_his " \ "where sky_id=%s and cam_id=%s and substr(img_path, 17, 6)='%s' " \ "order by img_name"%(skyId, camId, dateStr) print(sql1) tquery = QueryData() tfiles = tquery.query(sql1) print("total has %d images"%(len(tfiles))) for j, tname in enumerate(tfiles): tname00 = tname[0] tpath = "%s/%s"%(tpath1, tname00) tpathz = "%s/%s.fz"%(tpath1, tname00) if os.path.exists(tpath): timgName = tname00 elif os.path.exists(tpathz): timgName = tname00 + '.fz' else: print("%s not exist"%(tpath)) continue if j%25==0: print("process %d %s"%(j, timgName)) tdiff.getCat(tpath1, timgName) except Exception as e: print(str(e)) tstr = traceback.format_exc() print(tstr)
def test(): #toolPath = '/home/gwac/img_diff_xy/image_diff' toolPath = '/home/xy/Downloads/myresource/deep_data2/image_diff' tools = AstroTools(toolPath) dataDest0 = "/home/xy/gwac_diff_xy/data" logDest0 = "/home/xy/gwac_diff_xy/log" if not os.path.exists(dataDest0): os.system("mkdir -p %s" % (dataDest0)) if not os.path.exists(logDest0): os.system("mkdir -p %s" % (logDest0)) startProcess = False dayRun = 0 nigRun = 0 skyId = 0 ffId = 0 tfiles = [] #srcPath00='/data1/G004_041_190124' srcPath00 = '/home/xy/Downloads/myresource/matchTest' dateStr = '190124' camName = 'G041' curSkyId = '123' dstDir = '%s/%s' % (dataDest0, dateStr) tdiff = BatchImageDiff(srcPath00, dstDir, tools, camName, curSkyId) #tpath1 = '/data2/G003_034_190211' #tpath2 = '/data2/G003_034_190227' tpath1 = '/home/xy/Downloads/myresource/matchTest' tpath2 = '/home/xy/Downloads/myresource/matchTest' fname1 = 'G034_mon_objt_190211T12192603.fit' fname2 = 'G044_mon_objt_190305T13070793.fit' ra0, dec0 = -1000, -1000 wcsfile1, ra_center1, dec_center1 = tdiff.getWCS(tpath1, fname1, ra0, dec0) wcs1 = WCS('%s/%s' % (tdiff.tmpDir, wcsfile1)) wcsfile2, ra_center2, dec_center2 = tdiff.getWCS(tpath2, fname2, ra0, dec0) wcs2 = WCS('%s/%s' % (tdiff.tmpDir, wcsfile2)) os.system("cp %s/%s %s/%s" % (tpath1, fname1, tdiff.tmpDir, fname1)) os.system("%s %s/%s" % (tdiff.funpackProgram, tdiff.tmpDir, fname1)) os.system("cp %s/%s %s/%s" % (tpath2, fname2, tdiff.tmpDir, fname2)) os.system("%s %s/%s" % (tdiff.funpackProgram, tdiff.tmpDir, fname2)) fpar = 'sex_diff.par' #sexConf=['-DETECT_MINAREA','3','-DETECT_THRESH','2.5','-ANALYSIS_THRESH','2.5'] sexConf = [ '-DETECT_MINAREA', '10', '-DETECT_THRESH', '5', '-ANALYSIS_THRESH', '5' ] tcat1 = tools.runSextractor('G034_mon_objt_190211T12192603.fit', tdiff.tmpDir, tdiff.tmpDir, fpar, sexConf) tcat2 = tools.runSextractor('G044_mon_objt_190305T13070793.fit', tdiff.tmpDir, tdiff.tmpDir, fpar, sexConf) tdata2 = np.loadtxt("%s/%s" % (tdiff.tmpDir, tcat2)) tXY = tdata2[:, 0:2] print(tXY[:3]) tRaDec = wcs2.all_pix2world(tXY, 1) print(tRaDec[:3]) tXY2 = wcs1.all_world2pix(tRaDec, 1) print(tXY2[:3]) tdata2[:, 0:2] = tXY2 saveName = "%s_trans.cat" % (fname2.split(".")[0]) savePath = "%s/%s" % (tdiff.tmpDir, saveName) np.savetxt(savePath, tdata2, fmt='%.4f') mchFile, nmhFile, mchPair = tools.runCrossMatch(tdiff.tmpDir, saveName, tcat1, 1) evaluatePos(tdiff.tmpDir, saveName, tcat1, mchPair) evaluatePos(tdiff.tmpDir, tcat2, tcat1, mchPair)
def mainControl(self, camName, runName='p1', destDir='/data/gwac_diff_xy', toolPath='/home/gwac/img_diff_xy/image_diff'): #os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152 #os.environ["CUDA_VISIBLE_DEVICES"] = "" tools = AstroTools(toolPath) logDest0 = "%s/log" % (destDir) if not os.path.exists(logDest0): os.system("mkdir -p %s" % (logDest0)) dateStr = datetime.strftime(datetime.utcnow(), "%Y%m%d") dataDest0 = "%s/data/%s_%s" % (destDir, dateStr, runName) if not os.path.exists(dataDest0): os.system("mkdir -p %s" % (dataDest0)) imgDiff = GWACDiff(camName, dataDest0, tools) tStr = "%s: start combine diff..." % (camName) imgDiff.log.info(tStr) imgDiff.sendMsg(tStr) #dataPath = '/home/xy/work/imgDiffTest2/fits/190128_G004_044' # #processRecord = '/home/xy/work/imgDiffTest3/log/G004_044.log' #os.system("rm -rf %s"%(processRecord)) #sexLock=Lock() self.loopNum = 1 while True: try: curUtcDateTime = datetime.utcnow() tDateTime = datetime.utcnow() startDateTime = tDateTime.replace(hour=9, minute=30, second=0) #9=17 1=9 endDateTime = tDateTime.replace(hour=9, minute=30, second=10) #22=6 8=16 remainSeconds1 = (startDateTime - curUtcDateTime).total_seconds() remainSeconds2 = (endDateTime - curUtcDateTime).total_seconds() if remainSeconds1 < 0 and remainSeconds2 > 0: if os.path.exists(dataDest0): os.system("rm -rf %s" % (dataDest0)) self.initData() dateStr = datetime.strftime(datetime.utcnow(), "%Y%m%d") dataDest0 = "%s/data/%s_%s" % (destDir, dateStr, runName) if not os.path.exists(dataDest0): os.system("mkdir -p %s" % (dataDest0)) imgDiff.reInitDataDir(dataDest0) tStr = "%s: combine diff reInit data" % (camName) print(tStr) imgDiff.log.info(tStr) imgDiff.sendMsg(tStr) time.sleep(10) except Exception as e: tstr = traceback.format_exc() print("combine diff reInit data error....") print(tstr) time.sleep(10) try: print("\n\nmain loop %d****************" % (self.loopNum)) tIdx1 = self.loopNum % 5 #tIdx1 = self.loopNum%1 ''' ''' if tIdx1 == 0 and self.catRunning == 0: #cat self.srcExtract(camName, self.catList, imgDiff, logDest0, runName) #tIdx2 = self.loopNum%15 tIdx2 = self.loopNum % 2 if tIdx2 == 0 and self.alignTmplRunning == 0: #template getAlignTemplateLocal self.getAlignTemplate(camName, self.catList, self.alignTmplMap, imgDiff) tIdx3 = self.loopNum % 2 if tIdx3 == 0 and self.alignRunning == 0: #align self.doAlign(camName, self.catList, self.alignTmplMap, self.alignList, imgDiff) tIdx4 = self.loopNum % 2 if tIdx4 == 0 and self.cmbRunning == 0: #combine5 self.doCombine(camName, self.alignList, self.alignTmplMap, imgDiff, self.cmbImgList, self.crossTaskParms, runName, 5) tIdx5 = self.loopNum % 2 if tIdx5 == 0 and self.cmbCatRunning == 0: #cat of combine5 self.srcExtractCombine(camName, self.cmbImgList, self.cmbCatList, imgDiff) #else: # print("%d cmbCatJob is running"%(self.loopNum)) #tIdx6 = self.loopNum%15 tIdx6 = self.loopNum % 2 if tIdx6 == 0 and self.diffTmplRunning == 0: #cat of combine5 self.getDiffTemplate(camName, self.cmbCatList, self.alignTmplMap, self.diffTmplMap, imgDiff) #tIdx7 = self.loopNum%5 tIdx7 = self.loopNum % 2 if tIdx7 == 0: if self.diffRunning == 0: #diff self.doDiff(camName, self.cmbCatList, self.diffTmplMap, imgDiff, self.diffImgList) #tIdx8 = self.loopNum%2 tIdx8 = self.loopNum % 2 if tIdx8 == 0: if self.recgRunning == 0: #recognition #print("%d doRecognitionAndUpload start run"%(self.loopNum)) self.doRecognitionAndUpload(camName, self.diffImgList, self.diffTmplMap, imgDiff, runName) #else: # print("%d doRecognitionAndUpload is running"%(self.loopNum)) self.loopNum = self.loopNum + 1 time.sleep(1) except Exception as e: tstr = traceback.format_exc() print("Scheduling main error....") print(tstr) time.sleep(10)
def wcsAlign(srcPath00, dateStr, camName, curSkyId): #toolPath = os.getcwd() toolPath = '/data/work/program/image_diff' tools = AstroTools(toolPath) dataDest0 = "/data/gwac_diff_xy/data" logDest0 = "/data/gwac_diff_xy/log" if not os.path.exists(dataDest0): os.system("mkdir -p %s" % (dataDest0)) if not os.path.exists(logDest0): os.system("mkdir -p %s" % (logDest0)) startProcess = False dayRun = 0 nigRun = 0 skyId = 0 ffId = 0 tfiles = [] #srcPath00='/data1/G004_041_190124' #dateStr='190124' #camName='G041' #curSkyId='123' dstDir = '%s/%s' % (dataDest0, dateStr) tdiff = BatchImageDiff(srcPath00, dstDir, tools, camName, curSkyId) try: tfiles0 = os.listdir(tdiff.wcsDir) tfiles0.sort() print("total has %d images" % (len(tfiles0))) tmpWCSIdx = int(len(tfiles0) / 2) tmpWCSFile = tfiles0[tmpWCSIdx] tmpWCS = WCS('%s/%s' % (tdiff.wcsDir, tmpWCSFile)) print("select %d %s as tmp WCS" % (tmpWCSIdx, tmpWCSFile)) for i, tname in enumerate(tfiles0): if i == tmpWCSIdx: tname00 = tname.split('.')[0] + '.fit' tpath = "%s/%s" % (srcPath00, tname00) tpathz = "%s/%s.fz" % (srcPath00, tname00) if os.path.exists(tpath): timgName = tname00 os.system("cp %s/%s %s/%s.wcs" % (srcPath00, timgName, tdiff.remapDir, timgName)) elif os.path.exists(tpathz): timgName = tname00 + '.fz' os.system("cp %s/%s %s/%s" % (srcPath00, timgName, tdiff.remapDir, timgName)) os.system("%s %s/%s" % (tools.funpackProgram, tdiff.remapDir, timgName)) os.system("rm -rf %s/%s" % (tdiff.remapDir, timgName)) else: print("%s not exist" % (tpath)) continue else: starttime = datetime.now() print("remap %s" % (tname)) tdiff.reMapImg(tname, tmpWCS, srcPath00) endtime = datetime.now() runTime = (endtime - starttime).seconds tdiff.log.info("totalTime %d seconds, %s" % (runTime, tname)) except Exception as e: print(str(e)) tstr = traceback.format_exc() print(tstr) try: if 'tdiff' in locals(): tStr = "diff error" tdiff.log.info(tStr) tdiff.sendMsg(tStr) except Exception as e1: print(str(e1)) tstr = traceback.format_exc() print(tstr)
def mainControl(self, camName, runName = 'p1', destDir='/data/gwac_diff_xy', toolPath='/home/gwac/img_diff_xy/image_diff'): #os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152 #os.environ["CUDA_VISIBLE_DEVICES"] = "" tools = AstroTools(toolPath) logDest0 = "%s/log"%(destDir) if not os.path.exists(logDest0): os.system("mkdir -p %s"%(logDest0)) dateStr = datetime.strftime(datetime.utcnow(), "%Y%m%d") dataDest0 = "%s/data/%s_%s"%(destDir, dateStr, runName) if not os.path.exists(dataDest0): os.system("mkdir -p %s"%(dataDest0)) imgDiff = GWACDiff(camName, dataDest0, tools) tStr = "%s: start combine diff..."%(camName) imgDiff.log.info(tStr) imgDiff.sendMsg(tStr) tdirs00 = "%s/data"%(destDir) tdirs = os.listdir(tdirs00) tdirs.sort() for tdir in tdirs: tproot00="%s/%s"%(tdirs00, tdir) imgDiff.cleanData(tproot00) #dataPath = '/home/xy/work/imgDiffTest2/fits/190128_G004_044' # #processRecord = '/home/xy/work/imgDiffTest3/log/G004_044.log' #os.system("rm -rf %s"%(processRecord)) #sexLock=Lock() self.loopNum = 1 while True: try: curUtcDateTime = datetime.utcnow() tDateTime = datetime.utcnow() startDateTime = tDateTime.replace(hour=9, minute=30, second=0) #9=17 1=9 endDateTime = tDateTime.replace(hour=9, minute=30, second=10) #22=6 8=16 remainSeconds1 = (startDateTime - curUtcDateTime).total_seconds() remainSeconds2 = (endDateTime - curUtcDateTime).total_seconds() if remainSeconds1<0 and remainSeconds2>0: #if os.path.exists(dataDest0): # os.system("rm -rf %s"%(dataDest0)) self.initData() dateStr = datetime.strftime(datetime.utcnow(), "%Y%m%d") dataDest0 = "%s/data/%s_%s"%(destDir, dateStr, runName) if not os.path.exists(dataDest0): os.system("mkdir -p %s"%(dataDest0)) imgDiff.reInitDataDir(dataDest0) tStr = "%s: combine diff reInit data"%(camName) print(tStr) imgDiff.log.info(tStr) imgDiff.sendMsg(tStr) time.sleep(10) except Exception as e: tstr = traceback.format_exc() print("combine diff reInit data error....") print(tstr) time.sleep(10) try: print("\n\nmain loop %d****************"%(self.loopNum)) self.srcExtract(camName, self.catList, self.alignTmplMap, imgDiff, logDest0, runName) self.getAlignTemplate(camName, self.catList, self.alignTmplMap, imgDiff) self.doAlign(camName, self.catList, self.alignTmplMap, self.alignList, imgDiff) self.doDiff(camName, self.alignList, self.alignTmplMap, imgDiff, self.diffImgList) self.loopNum = self.loopNum + 1 time.sleep(1) except Exception as e: tstr = traceback.format_exc() print("Scheduling main error....") print(tstr) time.sleep(10)
def mainControl(self, camName, runName='p1', destDir='/data/gwac_diff_xy', toolPath='/home/gwac/img_diff_xy/image_diff'): #os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152 #os.environ["CUDA_VISIBLE_DEVICES"] = "" dateStr = datetime.strftime(datetime.utcnow(), "%Y%m%d") dataDest0 = "%s/data/%s_%s" % (destDir, dateStr, runName) logDest0 = "%s/log_%s" % (destDir, runName) if not os.path.exists(logDest0): os.system("mkdir -p %s" % (logDest0)) if not os.path.exists(dataDest0): os.system("mkdir -p %s" % (dataDest0)) tools = AstroTools(toolPath) imgDiff = GWACDiff(camName, dataDest0, tools) tStr = "start diff..." imgDiff.log.info(tStr) imgDiff.sendMsg(tStr) #dataPath = '/home/xy/work/imgDiffTest2/fits/190128_G004_044' # #processRecord = '/home/xy/work/imgDiffTest3/log/G004_044.log' #os.system("rm -rf %s"%(processRecord)) #sexLock=Lock() loopNum = 1 while True: try: tIdx1 = loopNum % 5 ''' ''' if tIdx1 == 1 and self.catRunning.value == 0: #cat #catJob = Process(target=srcExtractLocalDir, args=(dataPath, camName, self.catList, imgDiff, logDest0, self.catRunning)) catJob = Process(target=srcExtract, args=(camName, self.catList, imgDiff, logDest0, self.catRunning)) catJob.start() tIdx2 = loopNum % 15 #tIdx2 = loopNum%5 if tIdx2 == 0 and self.alignTmplRunning.value == 0: #template getAlignTemplateLocal alignTmplJob = Process(target=getAlignTemplate, args=(camName, self.catList, self.alignTmplMap, self.alignTmplIdx, imgDiff, self.alignTmplRunning)) alignTmplJob.start() if self.alignRunning.value == 0: #align alignJob = Process(target=doAlign, args=(camName, self.catList, self.alignTmplMap, self.alignIdx, self.alignList, imgDiff, self.alignRunning)) alignJob.start() if self.cmbRunning.value == 0: #combine5 #print("%d cmbJob is not running, catNum=%d, alignNum=%d, cmbIdx=%d"%(loopNum, len(self.catList), len(self.alignList), self.cmbIdx.value)) cmbJob = Process( target=doCombine, args=(camName, self.alignList, self.alignTmplMap, self.cmbIdx, imgDiff, self.cmbImgList, self.cmbRunning, self.crossTaskParms, 5)) cmbJob.start() #else: # print("%d cmbJob is running"%(loopNum)) if self.cmbCatRunning.value == 0: #cat of combine5 cmbCatJob = Process(target=srcExtractCombine, args=(camName, self.cmbImgList, self.cmbCatList, self.cmbCatIdx, imgDiff, self.cmbCatRunning)) cmbCatJob.start() #else: # print("%d cmbCatJob is running"%(loopNum)) tIdx6 = loopNum % 15 #tIdx6 = loopNum%1 if tIdx6 == 0 and self.diffTmplRunning.value == 0: #cat of combine5 diffTmplJob = Process( target=getDiffTemplate, args=(camName, self.cmbCatList, self.alignTmplMap, self.diffTmplMap, self.diffTmplIdx, imgDiff, self.diffTmplRunning)) diffTmplJob.start() #else: # print("%d diffTmplJob is running"%(loopNum)) tIdx7 = loopNum % 5 if tIdx7 == 0: if self.diffRunning.value == 0: #diff #print("%d doDiff start run"%(loopNum)) #diffJob = Process(target=doDiff, args=( # camName, self.cmbImgList, self.diffTmplMap, self.diffIdx, imgDiff, self.diffImgList, self.diffRunning)) diffJob = Process(target=doDiff, args=(camName, self.cmbCatList, self.diffTmplMap, self.diffIdx, imgDiff, self.diffImgList, self.diffRunning)) diffJob.start() #else: # print("%d doDiff is running"%(loopNum)) tIdx8 = loopNum % 2 if tIdx8 == 0: if self.recgRunning.value == 0: #recognition print("%d doRecognitionAndUpload start run" % (loopNum)) #recgJob = Process(target=doRecognitionAndUpload, args=( # camName, self.diffImgList, self.diffTmplMap, self.recgIdx, imgDiff, self.recgRunning)) #recgJob.start() doRecognitionAndUpload(camName, self.diffImgList, self.diffTmplMap, self.recgIdx, imgDiff, self.recgRunning) else: print("%d doRecognitionAndUpload is running" % (loopNum)) loopNum = loopNum + 1 #if loopNum>4000: # break time.sleep(1) ''' try: catJob.join() alignJob.join() cmbJob.join() except NameError as e: tstr = traceback.format_exc() print("catJob is not defined") print(tstr) ''' except Exception as e: tstr = traceback.format_exc() print("Scheduling main error....") print(tstr) time.sleep(10)
def run1(srcPath00, dateStr, camName, curSkyId): #toolPath = os.getcwd() toolPath = '/home/gwac/img_diff_xy/image_diff' tools = AstroTools(toolPath) dataDest0 = "/data/gwac_diff_xy/data" logDest0 = "/data/gwac_diff_xy/log" if not os.path.exists(dataDest0): os.system("mkdir -p %s" % (dataDest0)) if not os.path.exists(logDest0): os.system("mkdir -p %s" % (logDest0)) startProcess = False dayRun = 0 nigRun = 0 skyId = 0 ffId = 0 tfiles = [] #srcPath00='/data1/G004_041_190124' #dateStr='190124' #camName='G041' #curSkyId='123' dstDir = '%s/%s' % (dataDest0, dateStr) tdiff = BatchImageDiff(srcPath00, dstDir, tools, camName, curSkyId) try: camId = (int(camName[1]) - 1) * 5 + int(camName[2]) sql1 = "select img_name " \ "from fits_file2_his " \ "where sky_id=%s and cam_id=%s and substr(img_path, 17, 6)='%s' " \ "order by img_name"%(curSkyId, camId, dateStr) print(sql1) tquery = QueryData() tfiles = tquery.query(sql1) print("total has %d images" % (len(tfiles))) ra0, dec0 = -1000, -1000 for i, tname in enumerate(tfiles): tname00 = tname[0] tpath = "%s/%s" % (srcPath00, tname00) tpathz = "%s/%s.fz" % (srcPath00, tname00) if os.path.exists(tpath): timgName = tname00 elif os.path.exists(tpathz): timgName = tname00 + '.fz' else: print("%s not exist" % (tpath)) continue tStr = "start diff: %s" % (timgName) tdiff.log.info(tStr) starttime = datetime.now() print("process %s" % (timgName)) wcsfile, ra_center, dec_center = tdiff.getWCS( srcPath00, timgName, ra0, dec0) ra0, dec0 = ra_center, dec_center os.system( "cp %s/%s %s/%s.wcs" % (tdiff.tmpDir, wcsfile, tdiff.wcsDir, timgName.split('.')[0])) endtime = datetime.now() runTime = (endtime - starttime).seconds tdiff.log.info("totalTime %d seconds, %s" % (runTime, timgName)) except Exception as e: print(str(e)) tstr = traceback.format_exc() print(tstr) try: if 'tdiff' in locals(): tStr = "diff error" tdiff.log.info(tStr) tdiff.sendMsg(tStr) except Exception as e1: print(str(e1)) tstr = traceback.format_exc() print(tstr)
def wcsAlign2(): #toolPath = os.getcwd() toolPath = '/home/xy/Downloads/myresource/deep_data2/image_diff' tools = AstroTools(toolPath) dataDest0 = "tmp/data" logDest0 = "tmp/log" if not os.path.exists(dataDest0): os.system("mkdir -p %s" % (dataDest0)) if not os.path.exists(logDest0): os.system("mkdir -p %s" % (logDest0)) startProcess = False dayRun = 0 nigRun = 0 skyId = 0 ffId = 0 tfiles = [] srcPath00 = '/home/xy/test7' dateStr = '190124' camName = 'G041' curSkyId = '123' dstDir = '%s/%s' % (dataDest0, dateStr) tdiff = BatchImageDiff(srcPath00, dstDir, tools, camName, curSkyId) tdiff.wcsDir = srcPath00 try: tfiles0 = os.listdir(srcPath00) tfiles0.sort() tmpWCSFile = 'G031_mon_objt_190116T21321726.wcs' tmpWCS = WCS('%s/%s' % (tdiff.wcsDir, tmpWCSFile)) print("select %s as tmp WCS" % (tmpWCSFile)) for i, tname in enumerate(tfiles0): if tname.find('wcs') > 0 and tname.find( tmpWCSFile.split('.')[0]) < 0: starttime = datetime.now() print("remap %s" % (tname)) tdiff.reMapImg(tname, tmpWCS, srcPath00) endtime = datetime.now() runTime = (endtime - starttime).seconds tdiff.log.info("totalTime %d seconds, %s" % (runTime, tname)) except Exception as e: print(str(e)) tstr = traceback.format_exc() print(tstr) try: if 'tdiff' in locals(): tStr = "diff error" tdiff.log.info(tStr) tdiff.sendMsg(tStr) except Exception as e1: print(str(e1)) tstr = traceback.format_exc() print(tstr)
#subImg = getWindowImgs(tpath1, fname, x, y, size) subImg = getWindowImgs(storePath, fname, x, y, size) subImgz = zscale_image(subImg) subImgz = scipy.ndimage.zoom(subImgz, 4, order=0) imgpre = fname.split(".")[0] tImgName = "%s.jpg" % (imgpre) savePath = "%s/%s" % (workPath, tImgName) Image.fromarray(subImgz).save(savePath) if __name__ == "__main__": toolPath = '/data/work/program/image_diff' tools = AstroTools(toolPath) tpath = '/data3/G004_044_190305' fname = 'G044_mon_objt_190305T13393793.fit' x = 1644.03 y = 1700.2 ra0 = 203.81125 dec0 = 44.76603 otName = 'G190116_C19152' #runSuccess, ot2ReRa, ot2ReDec = reAstrometry(tools, tpath, fname, x, y, ra0, dec0, otName) startDate = '2018-12-17' endDate = '2019-01-17' skyListFile = getSkyList(tools, otName, ra0, dec0, startDate, endDate)
def run1(camName): #toolPath = os.getcwd() toolPath = '/home/gwac/img_diff_xy/image_diff' tools = AstroTools(toolPath) query = QueryData() dataDest0 = "/data/gwac_diff_xy/data" logDest0 = "/data/gwac_diff_xy/log" if not os.path.exists(dataDest0): os.system("mkdir -p %s"%(dataDest0)) if not os.path.exists(logDest0): os.system("mkdir -p %s"%(logDest0)) startProcess = False dayRun = 0 nigRun = 0 skyId = 0 ffId = 0 tfiles = [] while True: curUtcDateTime = datetime.utcnow() tDateTime = datetime.utcnow() startDateTime = tDateTime.replace(hour=9, minute=30, second=0) #9=17 1=9 endDateTime = tDateTime.replace(hour=22, minute=30, second=0) #22=6 8=16 remainSeconds1 = (startDateTime - curUtcDateTime).total_seconds() remainSeconds2 = (endDateTime - curUtcDateTime).total_seconds() if remainSeconds1<0 and remainSeconds2>0: dayRun = 0 try: tfiles = query.getFileList(camName, ffId) #print(tfiles) for tfile in tfiles: curFfId = tfile[0] ffNumber = tfile[1] curSkyId = tfile[2] timgName = tfile[3] #G021_tom_objt_190109T13531492.fit tpath = tfile[4] #/data3/G002_021_190109/G021_tom_objt_190109T13531492.fit imgDate = timgName[14:20] pathDate = tpath[16:22] if imgDate!=pathDate: ffId=curFfId startProcess = False continue elif not startProcess: ffId=0 startProcess = True srcDir= tpath[:(tpath.find(camName)-1)] #/data3/G002_021_190109 dateStr = srcDir[srcDir.find('G'):] #G002_021_190109 logfName0 = '%s/%s.log'%(logDest0, dateStr) if ffId==0: if os.path.exists(logfName0) and os.stat(logfName0).st_size > 0: tlastLine = getLastLine(logfName0) if len(tlastLine)>2: ffId=int(tlastLine.strip()) if skyId!=curSkyId and curFfId>ffId: dstDir='%s/%s'%(dataDest0, dateStr) tdiff = BatchImageDiff(srcDir, dstDir, tools, camName, curSkyId) tStr = "start diff: %s"%(timgName) tdiff.log.info(tStr) tdiff.sendMsg(tStr) if curFfId>ffId: skyId=curSkyId ffId=curFfId logfile0 = open(logfName0, 'a') logfile0.write("\n\n%d\n"%(ffId)) logfile0.close() tpathfz = "%s.fz"%(tpath) if os.path.exists(tpath) or os.path.exists(tpathfz): starttime = datetime.now() tdiff.processImg(srcDir, timgName, ffNumber) endtime = datetime.now() runTime = (endtime - starttime).seconds tdiff.log.info("totalTime %d seconds, sky:%d, ffNum:%d, %s"%(runTime, curSkyId, ffNumber, timgName)) else: print("%s not exist"%(tpath)) #if curFfId>ffId: # break except Exception as e: print(str(e)) tstr = traceback.format_exc() print(tstr) try: if 'tdiff' in locals(): tStr = "diff error" tdiff.log.info(tStr) tdiff.sendMsg(tStr) except Exception as e1: print(str(e1)) tstr = traceback.format_exc() print(tstr) if len(tfiles)==0: time.sleep(5) nigRun = nigRun+1 #if nigRun>=1: # break else: # day temp file clean try: if ('tdiff' in locals()) and (dayRun==0): tdiff.initReg(0) except Exception as e1: print(str(e1)) tstr = traceback.format_exc() print(tstr) nigRun = 0 dayRun = dayRun+1 skyId = 0 ffId = 0 startProcess = False if dayRun%6==1: print("day %d wait"%(dayRun)) time.sleep(10*60)