def getImgCenter(tpath, imgName, x, y):
tools = AstroTools('/home/xy/Downloads/myresource/deep_data2/image_diff')
fieldId, ra, dec = tools.getRaDec(tpath, imgName)
fpar = 'sex_diff.par'
sexConf = [
'-DETECT_MINAREA', '10', '-DETECT_THRESH', '5', '-ANALYSIS_THRESH',
'5', '-CATALOG_TYPE', 'FITS_LDAC'
]
tmplCat, isSuccess = tools.runSextractor(imgName,
tpath,
tpath,
fpar,
sexConf,
outSuffix='_ldac.fit')
if not isSuccess:
print("getDiffTemplate runSextractor failure2")
return isSuccess, 0, 0
tools.ldac2fits('%s/%s' % (tpath, tmplCat), '%s/ti_cat.fit' % (tpath))
runSuccess = tools.runWCS(tpath, 'ti_cat.fit', ra, dec)
if runSuccess:
wcs = WCS('%s/ti_cat.wcs' % (tpath))
ra, dec = wcs.all_pix2world(x, y, 1)
return runSuccess, ra, dec
class GWACWCSIndex:
orgImgRoot = '/data/gwac_data/gwac_orig_fits'
wcsIdxRoot = '/data/gwac_data/gwac_wcs_idx'
webServerIP1 = '172.28.8.28:8080'
webServerIP2 = '10.0.10.236:9995'
connParam = {
"host": "190.168.1.27",
"port": "5432",
"dbname": "gwac2",
"user": "******",
"password": "******"
}
connParam2 = {
"host": "172.28.8.28",
"port": "5432",
"dbname": "gwac2",
"user": "******",
"password": "******"
}
connParam3 = {
"host": "10.0.3.62",
"port": "5433",
"dbname": "gwac2",
"user": "******",
"password": "******"
}
webUser = '******'
webPassword = '******'
webIp = '172.28.8.8'
def __init__(self):
self.conn = False
self.imgSize = (4136, 4096)
self.templateImg = 'ti.fit'
self.tmpRoot = "/dev/shm/gwacWCS"
self.templateDir = "%s/tmpl" % (self.tmpRoot)
if not os.path.exists(self.templateDir):
os.system("mkdir -p %s" % (self.templateDir))
#/home/gwac/img_diff_xy/image_diff
#/data/work/program/image_diff
self.toolPath = '/home/gwac/img_diff_xy/image_diff'
self.funpackProgram = "%s/tools/cfitsio/funpack" % (self.toolPath)
#logPath = os.getcwd()
self.tools = AstroTools(self.toolPath, "gwac_wcs")
self.log = self.tools.log
def makeTemplate(self, tparm):
try:
starttime = datetime.now()
#print(tparm)
tRaDec = np.array([])
os.system("rm -rf %s/*" % (self.templateDir))
imgName = tparm['img_name'].decode("utf-8")
fwhm = tparm['fwhm']
ffNumber = tparm['ff_number']
objNum = tparm['obj_num']
imgPath = tparm['img_path'].decode("utf-8")
tmsgStr = "select %s as template, min fwhm %.2f" % (imgName, fwhm)
print(tmsgStr)
self.log.info(tmsgStr)
self.tools.sendTriggerMsg(tmsgStr)
os.system("rm -rf %s/*" % (self.templateDir))
#oImgf = "%s/%s"%(imgPath,imgName)
oImgf = imgPath
print(oImgf)
oImgfz = "%s.fz" % (imgPath)
if os.path.exists(oImgfz):
os.system("cp %s %s/%s.fz" %
(oImgfz, self.templateDir, self.templateImg))
os.system(
"%s %s/%s.fz" %
(self.funpackProgram, self.templateDir, self.templateImg))
elif os.path.exists(oImgf):
os.system("cp %s %s/%s" %
(oImgf, self.templateDir, self.templateImg))
else:
self.log.warning("%s not exist" % (oImgf))
return
fieldId, ra, dec = self.tools.removeHeaderAndOverScan(
self.templateDir, self.templateImg)
sexConf = [
'-DETECT_MINAREA', '10', '-DETECT_THRESH', '5',
'-ANALYSIS_THRESH', '5'
]
fpar = 'sex_diff.par'
tmplCat, isSuccess = self.tools.runSextractor(self.templateImg,
self.templateDir,
self.templateDir,
fpar,
sexConf,
cmdStatus=0)
starNum, fwhmMean, fwhmRms, bgMean, bgRms = self.tools.basicStatistic(
self.templateDir, tmplCat)
sexConf = [
'-DETECT_MINAREA', '10', '-DETECT_THRESH', '5',
'-ANALYSIS_THRESH', '5', '-CATALOG_TYPE', 'FITS_LDAC'
]
tmplCat, isSuccess = self.tools.runSextractor(
self.templateImg,
self.templateDir,
self.templateDir,
fpar,
sexConf,
cmdStatus=0,
outSuffix='_ldac.fit')
self.tools.ldac2fits('%s/%s' % (self.templateDir, tmplCat),
'%s/ti_cat.fit' % (self.templateDir))
tpath = "%s/%s" % (self.templateDir, tmplCat)
runSuccess = self.tools.runWCS(self.templateDir, 'ti_cat.fit', ra,
dec)
#ccdName = self.origTmplImgName[:4]
#runSuccess = self.tools.runWCSRemotePC780(self.templateDir,'ti_cat.fit', ra, dec, ccdName)
if runSuccess:
twcs = WCS('%s/ti_cat.wcs' % (self.templateDir))
height = self.imgSize[0]
width = self.imgSize[1]
tXYs = []
tXYs.append((width / 2, height / 2))
tXYs.append((0, 0))
tXYs.append((0, height - 1))
tXYs.append((width - 1, height - 1))
tXYs.append((width - 1, 0))
tXYs = np.array(tXYs)
try:
tRaDec = twcs.all_pix2world(tXYs, 1)
self.log.info(
'read_ra_dec:(%.5f, %.5f), real_dec_center:(%.5f, %.5f)'
% (ra, dec, tRaDec[0][0], tRaDec[0][1]))
except Exception as e:
self.log.error(e)
runSuccess = False
tstr = traceback.format_exc()
self.log.error(tstr)
self.log.error('make template %s, xy to radec error' %
(imgName))
objName = 'ti.fit'
bkgName = 'ti_bkg.fit'
badPixCat = self.tools.processBadPix(objName, bkgName,
self.templateDir,
self.templateDir)
imgPre = imgName.split('.')[0]
tpaths = imgPath.split(
'/'
) #/data2/G004_041_191015/G041_Mon_objt_191015T20282227.fit
tpath0 = tpaths[-2]
tcamName = tpath0[:8]
tdateStr = tpath0[9:]
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(self.webIp,
username=self.webUser,
password=self.webPassword)
#/data/gwac_data/gwac_wcs_idx/12854925/G003_031/180210
storePath = "%s/%s/%s/%s" % (self.wcsIdxRoot, fieldId,
tcamName, tdateStr)
tcmd = "mkdir -p %s;" % (storePath)
ssh.exec_command(tcmd)
time.sleep(10)
fitImg = "%s/%s" % (self.templateDir, self.templateImg)
cat = "%s/ti.cat" % (self.templateDir)
catFit = "%s/ti_cat.fit" % (self.templateDir)
wcs = '%s/ti_cat.wcs' % (self.templateDir)
badPix = '%s/%s' % (self.templateDir, badPixCat)
if os.path.exists(fitImg) and os.path.exists(cat) and os.path.exists(catFit) \
and os.path.exists(wcs) and os.path.exists(badPix):
print("start copy to remote dir %s" % (storePath))
ftp = ssh.open_sftp()
ftp.put(fitImg, "%s/%s.fit" % (storePath, imgPre))
ftp.put(cat, "%s/%s.cat" % (storePath, imgPre))
ftp.put(catFit, "%s/%s_cat.fit" % (storePath, imgPre))
ftp.put(wcs, "%s/%s.wcs" % (storePath, imgPre))
ftp.put(badPix,
"%s/%s_badpix.cat" % (storePath, imgPre))
print("copy success")
elif not os.path.exists(fitImg):
runSuccess = False
print("%s is not exist" % (fitImg))
elif not os.path.exists(cat):
runSuccess = False
print("%s is not exist" % (cat))
elif not os.path.exists(catFit):
runSuccess = False
print("%s is not exist" % (catFit))
elif not os.path.exists(wcs):
runSuccess = False
print("%s is not exist" % (wcs))
elif not os.path.exists(badPix):
runSuccess = False
print("%s is not exist" % (badPix))
except paramiko.AuthenticationException:
self.log.error("Authentication Failed!")
runSuccess = False
tstr = traceback.format_exc()
self.log.error(tstr)
except paramiko.SSHException:
self.log.error("Issues with SSH service!")
runSuccess = False
tstr = traceback.format_exc()
self.log.error(tstr)
except Exception as e:
self.log.error(
'make template %s, copy data to dest error' %
(imgName))
self.log.error(str(e))
runSuccess = False
tstr = traceback.format_exc()
self.log.error(tstr)
else:
self.log.error('make template %s, get wcs error' % (imgName))
endtime = datetime.now()
runTime = (endtime - starttime).seconds
self.log.info("********** make template %s use %d seconds" %
(imgName, runTime))
except Exception as e:
runSuccess = False
tRaDec = np.array([])
starNum = 0
fwhmMean = 0
bgMean = 0
self.log.error(e)
tstr = traceback.format_exc()
self.log.error(tstr)
tmsgStr = "%s make template error" % (imgName)
self.tools.sendTriggerMsg(tmsgStr)
return runSuccess, tRaDec, starNum, fwhmMean, bgMean
def connDb(self):
self.conn = psycopg2.connect(**self.connParam2)
def closeDb(self):
self.conn.close()
def getDataFromDB(self, sql):
tsql = sql
#self.log.debug(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
rows = cur.fetchall()
cur.close()
self.closeDb()
except Exception as err:
rows = []
self.log.error(" query data error ")
self.log.error(err)
return np.array(rows)
def queryObs(self, camName):
tsql = "select ors_id, date_str, sky_id, cam_id, img_num "\
"from observation_record_statistic ors "\
"INNER JOIN camera cam on cam.camera_id=ors.cam_id "\
"where do_wcs=false and cam.name='%s' "\
"ORDER BY ors_id "%(camName)
#print(tsql)
return self.getDataFromDB(tsql)
def queryImgParm(self, obs, isHis='_his'):
tsql = "SELECT ff.img_name, isp.fwhm, ff.ff_number, isp.obj_num, isp.time_obs_ut, ff.ff_id, ff.img_path "\
"from image_status_parameter%s isp "\
"INNER JOIN fits_file2%s ff on isp.ff_id=ff.ff_id "\
"where isp.fwhm>1 and ff.sky_id=%s and ff.cam_id=%s and to_char(ff.gen_time, 'YYMMDD')='%s' "\
"ORDER BY ff_number"%(isHis, isHis, obs[2], obs[3], obs[1])
trst = self.getDataFromDB(tsql)
return trst
def queryImgParm2(self, obs, isHis='_his'):
tsql = "SELECT ff.img_name, 0, ff.ff_number, 0, ff.gen_time, ff.ff_id, ff.img_path "\
"from fits_file2%s ff "\
"where ff.sky_id=%s and ff.cam_id=%s and to_char(ff.gen_time, 'YYMMDD')='%s' "\
"ORDER BY ff_number"%(isHis, obs[2], obs[3], obs[1])
trst = self.getDataFromDB(tsql)
return trst
def updateHasWCS(self, orsId, imgNum, minTime, maxTime, hasWCS='true'):
startTime = datetime.strftime(minTime, "%Y-%m-%d %H:%M:%S")
endTime = datetime.strftime(maxTime, "%Y-%m-%d %H:%M:%S")
tsql = "update observation_record_statistic set has_wcs=%s, real_img_num=%d, " \
"start_obs_time='%s', end_obs_time='%s' where ors_id=%s"%(hasWCS, imgNum, startTime, endTime, orsId)
#print(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
self.conn.commit()
cur.close()
self.closeDb()
except Exception as err:
self.log.error(" update science_object status error ")
self.log.error(err)
def updateDoWCS(self, orsId, doWCS='true'):
tsql = "update observation_record_statistic set do_wcs=%s where ors_id=%s" % (
doWCS, orsId)
#print(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
self.conn.commit()
cur.close()
self.closeDb()
except Exception as err:
self.log.error(" update science_object status error ")
self.log.error(err)
def updateWCSCoors(self, orsId, tRaDecs):
rd = tRaDecs
tsql = "update observation_record_statistic set center_ra=%f, center_dec=%f, " \
"left_top_ra=%f, left_top_dec=%f, left_bottom_ra=%f, left_bottom_dec=%f, "\
"right_top_ra=%f, right_top_dec=%f, right_bottom_ra=%f, right_bottom_dec=%f where ors_id=%s" \
%(rd[0][0], rd[0][1], rd[1][0], rd[1][1], rd[2][0], rd[2][1], rd[3][0], rd[3][1],rd[4][0], rd[4][1], orsId)
#print(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
self.conn.commit()
cur.close()
self.closeDb()
except Exception as err:
self.log.error(" update science_object status error ")
self.log.error(err)
def insertObsWcs(self,
orsId,
tparms,
starNum,
fwhm,
background,
getWCS='true'):
#print(orsId)
#print(tparms)
#tsql = "insert into observation_record_statistic_wcs(ff_id,fwhm,star_num,ors_id,get_wcs)"\
# "values(%d,%f,%d,%s,%s)"%(tparms[5],tparms[1],tparms[3],orsId,getWCS)
tsql = "insert into observation_record_statistic_wcs(ff_id,fwhm,star_num,ors_id,get_wcs, background)"\
"values(%d,%f,%d,%s,%s, %f)"%(tparms[5],fwhm,starNum,orsId,getWCS, background)
#print(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
self.conn.commit()
cur.close()
self.closeDb()
except Exception as err:
self.log.error(" update science_object status error ")
self.log.error(err)
def checkImg(self, obs, imgParms):
dateStr = obs[1]
skyId = int(obs[2])
camId = int(obs[3])
mountId = int(camId / 5) + 1
camId2 = camId % 5
if camId2 == 0:
mountId = mountId - 1
camId2 = 5
camName = "G%03d_%02d%d" % (mountId, mountId, camId2)
#print(dateStr,skyId, camName)
#fullPath = "%s/%s/%s"%(self.orgImgRoot, dateStr, camName) #/data2/G002_023_191015 /data/gwac_data/gwac_orig_fits/191015/G004_043
tparms = []
for tparm in imgParms:
imgPath = tparm[6]
#imgPath = "%s/%s"%(fullPath, tparm[0])
imgPathfz = "%s.fz" % (imgPath)
#print(imgPathfz)
if os.path.exists(imgPathfz) or os.path.exists(imgPath):
tparms.append((tparm[0], tparm[1], tparm[2], tparm[3],
tparm[6], tparm[5]))
dtype = [('img_name', 'S40'), ('fwhm', float), ('ff_number', int),
('obj_num', int), ('img_path', 'S100'), ('ff_id', int)]
trst = np.array(tparms, dtype=dtype)
return trst
def doAstrometry(self, orsId, tparms, hasParms=True):
doSuccess = False
if hasParms:
sortParms = np.sort(tparms, order='fwhm')
starIdx = 0
else:
tnum = tparms.shape[0]
starIdx = int(tnum * 2.0 / 3)
sortParms = tparms
#print(sortParms)
for i in range(3):
tIdx = starIdx + i
runSuccess, tRaDec, starNum, fwhmMean, bgMean = self.makeTemplate(
sortParms[tIdx])
if runSuccess:
doSuccess = True
self.updateWCSCoors(orsId, tRaDec)
self.insertObsWcs(orsId, sortParms[tIdx], starNum, fwhmMean,
bgMean)
break
else:
print("%dth run failure, try next image" % (i + 1))
#break
return doSuccess
def createWCS(self, camName, minNum=50):
tobs = self.queryObs(camName)
print("query %d sky..." % (tobs.shape[0]))
if tobs.shape[0] > 0:
for obs in tobs:
print(obs)
orsId = obs[0]
dateStr = obs[1]
imgNum = int(obs[4])
self.updateDoWCS(orsId)
camId = int(obs[3])
if camId % 5 == 0:
continue
imgParms = self.queryImgParm(obs)
if imgParms.shape[0] == 0:
print(
"cannot image parameters form history table, refind from current table "
)
imgParms = self.queryImgParm(obs, " ")
if imgParms.shape[0] > minNum:
timeObsUt = imgParms[:, 4]
minTime = np.min(timeObsUt)
maxTime = np.max(timeObsUt)
#self.log.debug(imgParms.shape[0])
tparms = self.checkImg(obs, imgParms)
tnum = tparms.shape[0]
tstr = "%s,%s,%s observe %s img, %d has parameter, %d still in local path."\
%(obs[1], obs[2], obs[3], obs[4], imgParms.shape[0], tnum)
self.log.debug(tstr)
print(tstr)
if tnum < minNum:
print(
"%s,%s,%s has %d images, small than minimum %d number, skip"
% (obs[1], obs[2], obs[3], tnum, minNum))
self.updateHasWCS(orsId, tnum, minTime, maxTime,
'false')
else:
if tnum > 100:
tparms = tparms[25:-25]
doSuccess = self.doAstrometry(orsId, tparms)
else:
doSuccess = self.doAstrometry(orsId, tparms)
if doSuccess:
self.updateHasWCS(orsId, tnum, minTime, maxTime,
'true')
else:
self.updateHasWCS(orsId, tnum, minTime, maxTime,
'false')
#break
else:
print(
"image parameter number is %d less than min number %d"
% (imgParms.shape[0], minNum))
if imgParms.shape[0] == 0 and imgNum > minNum:
print("directly build wcs from orig image.")
#ff.img_name, isp.fwhm, ff.ff_number, isp.obj_num, isp.time_obs_ut, ff.ff_id, ff.img_path
imgParms = self.queryImgParm2(obs)
if imgParms.shape[0] == 0:
print(
"cannot image parameters form history table, refind from current table "
)
imgParms = self.queryImgParm2(obs, " ")
if imgParms.shape[0] > minNum:
#self.log.debug(imgParms.shape[0])
timeObsUt = imgParms[:, 4]
minTime = np.min(timeObsUt)
maxTime = np.max(timeObsUt)
tparms = self.checkImg(obs, imgParms)
tnum = tparms.shape[0]
tstr = "%s,%s,%s observe %s img, 0 has parameter, %d still in local path."\
%(obs[1], obs[2], obs[3], obs[4], tnum)
self.log.debug(tstr)
print(tstr)
if tnum < minNum:
print(
"%s,%s,%s has %d images, small than minimum %d number, skip"
% (obs[1], obs[2], obs[3], tnum, minNum))
self.updateHasWCS(orsId, tnum, minTime,
maxTime, 'false')
else:
if tnum > 100:
tparms = tparms[25:-25]
doSuccess = self.doAstrometry(
orsId, tparms, False)
else:
doSuccess = self.doAstrometry(
orsId, tparms, False)
if doSuccess:
self.updateHasWCS(orsId, tnum, minTime,
maxTime, 'true')
else:
self.updateHasWCS(orsId, tnum, minTime,
maxTime, 'false')
else:
minTime = datetime.now()
maxTime = minTime
tnum = 0
self.updateHasWCS(orsId, tnum, minTime, maxTime,
'false')
else:
try:
timeObsUt = imgParms[:, 4]
minTime = np.min(timeObsUt)
maxTime = np.max(timeObsUt)
except Exception as err:
minTime = datetime.now()
maxTime = minTime
self.log.error(err)
tnum = 0
self.updateHasWCS(orsId, tnum, minTime, maxTime,
'false')
#break
else:
print("already completely built all wcs")
def scheduling(self, camName):
while True:
try:
curUtcDateTime = datetime.now()
tDateTime = datetime.now()
startDateTime = tDateTime.replace(hour=7, minute=10, second=0)
endDateTime = tDateTime.replace(hour=8, minute=10, second=10)
remainSeconds1 = (startDateTime -
curUtcDateTime).total_seconds()
remainSeconds2 = (endDateTime - curUtcDateTime).total_seconds()
if remainSeconds1 < 0 and remainSeconds2 > 0:
self.createWCS(camName)
time.sleep(15 * 60)
else:
time.sleep(60)
except Exception as e:
tstr = traceback.format_exc()
print("everyday create WCS error....")
print(tstr)
time.sleep(10 * 60)
class GWACWCSIndex:
orgImgRoot = '/data/gwac_data/gwac_orig_fits'
wcsIdxRoot = '/data/gwac_data/gwac_wcs_idx'
webServerIP1 = '172.28.8.28:8080'
webServerIP2 = '10.0.10.236:9995'
connParam = {
"host": "190.168.1.27",
"port": "5432",
"dbname": "gwac2",
"user": "******",
"password": "******"
}
connParam2 = {
"host": "172.28.8.28",
"port": "5432",
"dbname": "gwac2",
"user": "******",
"password": "******"
}
connParam3 = {
"host": "10.0.3.62",
"port": "5433",
"dbname": "gwac2",
"user": "******",
"password": "******"
}
def __init__(self):
self.conn = False
self.imgSize = (4136, 4096)
self.templateImg = 'ti.fit'
self.tmpRoot = "/dev/shm/gwacWCS"
self.templateDir = "%s/tmpl" % (self.tmpRoot)
if not os.path.exists(self.templateDir):
os.system("mkdir -p %s" % (self.templateDir))
self.toolPath = '/data/work/program/image_diff'
self.funpackProgram = "%s/tools/cfitsio/funpack" % (self.toolPath)
logPath = os.getcwd()
self.tools = AstroTools(self.toolPath, logPath)
self.log = self.tools.log
def makeTemplate(self, tparm):
try:
starttime = datetime.now()
print(tparm)
tRaDec = np.array([])
os.system("rm -rf %s/*" % (self.templateDir))
imgName = tparm['img_name'].decode("utf-8")
fwhm = tparm['fwhm']
ffNumber = tparm['ff_number']
objNum = tparm['obj_num']
imgPath = tparm['img_path'].decode("utf-8")
tmsgStr = "select %s as template, min fwhm %.2f" % (imgName, fwhm)
self.log.info(tmsgStr)
self.tools.sendTriggerMsg(tmsgStr)
os.system("rm -rf %s/*" % (self.templateDir))
oImgf = "%s/%s" % (imgPath, imgName)
print(oImgf)
oImgfz = "%s/%s.fz" % (imgPath, imgName)
if os.path.exists(oImgfz):
os.system("cp %s %s/%s.fz" %
(oImgfz, self.templateDir, self.templateImg))
os.system(
"%s %s/%s.fz" %
(self.funpackProgram, self.templateDir, self.templateImg))
elif os.path.exists(oImgf):
os.system("cp %s %s/%s" %
(oImgf, self.templateDir, self.templateImg))
else:
self.log.warning("%s not exist" % (oImgf))
return
fieldId, ra, dec = self.tools.removeHeaderAndOverScan(
self.templateDir, self.templateImg)
sexConf = [
'-DETECT_MINAREA', '10', '-DETECT_THRESH', '5',
'-ANALYSIS_THRESH', '5'
]
fpar = 'sex_diff.par'
tmplCat = self.tools.runSextractor(self.templateImg,
self.templateDir,
self.templateDir,
fpar,
sexConf,
cmdStatus=0)
sexConf = [
'-DETECT_MINAREA', '10', '-DETECT_THRESH', '5',
'-ANALYSIS_THRESH', '5', '-CATALOG_TYPE', 'FITS_LDAC'
]
tmplCat = self.tools.runSextractor(self.templateImg,
self.templateDir,
self.templateDir,
fpar,
sexConf,
cmdStatus=0,
outSuffix='_ldac.fit')
self.tools.ldac2fits('%s/%s' % (self.templateDir, tmplCat),
'%s/ti_cat.fit' % (self.templateDir))
tpath = "%s/%s" % (self.templateDir, tmplCat)
runSuccess = self.tools.runWCS(self.templateDir, 'ti_cat.fit', ra,
dec)
#ccdName = self.origTmplImgName[:4]
#runSuccess = self.tools.runWCSRemotePC780(self.templateDir,'ti_cat.fit', ra, dec, ccdName)
if runSuccess:
twcs = WCS('%s/ti_cat.wcs' % (self.templateDir))
height = self.imgSize[0]
width = self.imgSize[1]
tXYs = []
tXYs.append((width / 2, height / 2))
tXYs.append((0, 0))
tXYs.append((0, height - 1))
tXYs.append((width - 1, height - 1))
tXYs.append((width - 1, 0))
tXYs = np.array(tXYs)
try:
tRaDec = twcs.all_pix2world(tXYs, 1)
self.log.info(
'read_ra_dec:(%.5f, %.5f), real_dec_center:(%.5f, %.5f)'
% (ra, dec, tRaDec[0][0], tRaDec[0][1]))
except Exception as e:
self.log.error(e)
runSuccess = False
tstr = traceback.format_exc()
self.log.error(tstr)
self.log.error('make template %s, xy to radec error' %
(imgName))
objName = 'ti.fit'
bkgName = 'ti_bkg.fit'
badPixCat = self.tools.processBadPix(objName, bkgName,
self.templateDir,
self.templateDir)
imgPre = imgName.split('.')[0]
tpaths = imgPath.split('/')
storePath = "%s/%s/%s/%s" % (self.wcsIdxRoot, fieldId,
tpaths[-1], tpaths[-2])
if not os.path.exists(storePath):
os.system("mkdir -p %s" % (storePath))
fitImg = "%s/%s" % (self.templateDir, self.templateImg)
cat = "%s/ti.cat" % (self.templateDir)
catFit = "%s/ti_cat.fit" % (self.templateDir)
wcs = '%s/ti_cat.wcs' % (self.templateDir)
badPix = '%s/%s' % (self.templateDir, badPixCat)
try:
os.system("cp %s %s/%s.fit" % (fitImg, storePath, imgPre))
os.system("cp %s %s/%s.cat" % (cat, storePath, imgPre))
os.system("cp %s %s/%s_cat.fit" %
(catFit, storePath, imgPre))
os.system("cp %s %s/%s.wcs" % (wcs, storePath, imgPre))
os.system("cp %s %s/%s_badpix.cat" %
(badPix, storePath, imgPre))
except Exception as e:
self.log.error(e)
runSuccess = False
tstr = traceback.format_exc()
self.log.error(tstr)
self.log.error(
'make template %s, copy data to dest error' %
(imgName))
else:
self.log.error('make template %s, get wcs error' % (imgName))
endtime = datetime.now()
runTime = (endtime - starttime).seconds
self.log.info("********** make template %s use %d seconds" %
(imgName, runTime))
except Exception as e:
runSuccess = False
self.log.error(e)
tstr = traceback.format_exc()
self.log.error(tstr)
tmsgStr = "%s make template error" % (imgName)
self.tools.sendTriggerMsg(tmsgStr)
return runSuccess, tRaDec
def connDb(self):
self.conn = psycopg2.connect(**self.connParam2)
def closeDb(self):
self.conn.close()
def getDataFromDB(self, sql):
tsql = sql
#self.log.debug(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
rows = cur.fetchall()
cur.close()
self.closeDb()
except Exception as err:
rows = []
self.log.error(" query data error ")
self.log.error(err)
return np.array(rows)
def queryObs(self, size=10):
#"where do_wcs=false and date_str<'190924'"\
tsql = "select ors_id, date_str, sky_id, cam_id, img_num "\
"from observation_record_statistic "\
"where do_wcs=false "\
"ORDER BY ors_id limit %d"%(size)
print(tsql)
return self.getDataFromDB(tsql)
def queryImgParm(self, obs):
tsql = "SELECT ff.img_name, isp.fwhm, ff.ff_number, isp.obj_num, isp.time_obs_ut, ff.ff_id "\
"from image_status_parameter_his isp "\
"INNER JOIN fits_file2_his ff on isp.ff_id=ff.ff_id "\
"where isp.fwhm>1 and ff.sky_id=%s and ff.cam_id=%s and substr(ff.img_name, 15 , 6)='%s' "\
"ORDER BY ff_number"%(obs[2], obs[3], obs[1])
print(tsql)
return self.getDataFromDB(tsql)
def updateHasWCS(self, orsId, imgNum, minTime, maxTime, hasWCS='true'):
startTime = datetime.strftime(minTime, "%Y-%m-%d %H:%M:%S")
endTime = datetime.strftime(maxTime, "%Y-%m-%d %H:%M:%S")
tsql = "update observation_record_statistic set has_wcs=%s, real_img_num=%d, " \
"start_obs_time='%s', end_obs_time='%s' where ors_id=%s"%(hasWCS, imgNum, startTime, endTime, orsId)
print(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
self.conn.commit()
cur.close()
self.closeDb()
except Exception as err:
self.log.error(" update science_object status error ")
self.log.error(err)
def updateDoWCS(self, orsId, doWCS='true'):
tsql = "update observation_record_statistic set do_wcs=%s where ors_id=%s" % (
doWCS, orsId)
print(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
self.conn.commit()
cur.close()
self.closeDb()
except Exception as err:
self.log.error(" update science_object status error ")
self.log.error(err)
def updateWCSCoors(self, orsId, tRaDecs):
rd = tRaDecs
tsql = "update observation_record_statistic set center_ra=%f, center_dec=%f, " \
"left_top_ra=%f, left_top_dec=%f, left_bottom_ra=%f, left_bottom_dec=%f, "\
"right_top_ra=%f, right_top_dec=%f, right_bottom_ra=%f, right_bottom_dec=%f where ors_id=%s" \
%(rd[0][0], rd[0][1], rd[1][0], rd[1][1], rd[2][0], rd[2][1], rd[3][0], rd[3][1],rd[4][0], rd[4][1], orsId)
print(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
self.conn.commit()
cur.close()
self.closeDb()
except Exception as err:
self.log.error(" update science_object status error ")
self.log.error(err)
def insertObsWcs(self, orsId, tparms, getWCS='true'):
print(orsId)
print(tparms)
tsql = "insert into observation_record_statistic_wcs(ff_id,fwhm,star_num,ors_id,get_wcs)"\
"values(%d,%f,%d,%s,%s)"%(tparms[5],tparms[1],tparms[3],orsId,getWCS)
print(tsql)
try:
self.connDb()
cur = self.conn.cursor()
cur.execute(tsql)
self.conn.commit()
cur.close()
self.closeDb()
except Exception as err:
self.log.error(" update science_object status error ")
self.log.error(err)
def checkImg(self, obs, imgParms):
dateStr = obs[1]
skyId = int(obs[2])
camId = int(obs[3])
mountId = int(camId / 5) + 1
camId2 = camId % 5
if camId2 == 0:
mountId = mountId - 1
camId2 = 5
camName = "G%03d_%02d%d" % (mountId, mountId, camId2)
#print(dateStr,skyId, camName)
fullPath = "%s/%s/%s" % (self.orgImgRoot, dateStr, camName)
tparms = []
for tparm in imgParms:
imgPath = "%s/%s" % (fullPath, tparm[0])
imgPathfz = "%s.fz" % (imgPath)
if os.path.exists(imgPathfz) or os.path.exists(imgPath):
tparms.append((tparm[0], tparm[1], tparm[2], tparm[3],
fullPath, tparm[5]))
dtype = [('img_name', 'S40'), ('fwhm', float), ('ff_number', int),
('obj_num', int), ('img_path', 'S100'), ('ff_id', int)]
trst = np.array(tparms, dtype=dtype)
return trst
def doAstrometry(self, orsId, tparms):
doSuccess = False
sortParms = np.sort(tparms, order='fwhm')
for i in range(3):
runSuccess, tRaDec = self.makeTemplate(sortParms[i])
if runSuccess:
doSuccess = True
self.updateWCSCoors(orsId, tRaDec)
self.insertObsWcs(orsId, sortParms[i])
break
return doSuccess
def createWCS(self, startQueryNum=7, minNum=50):
totalNum = int(15038 / 10) + 1
stopFlag = False
while True:
time.sleep(10)
tobs = self.queryObs()
if tobs.shape[0] == 0:
break
for obs in tobs:
print(obs)
orsId = obs[0]
self.updateDoWCS(orsId)
camId = int(obs[3])
if int(orsId) < 80 or camId % 5 == 0:
continue
imgParms = self.queryImgParm(obs)
if imgParms.shape[0] > minNum:
timeObsUt = imgParms[:, 4]
minTime = np.min(timeObsUt)
maxTime = np.max(timeObsUt)
#self.log.debug(imgParms.shape[0])
tparms = self.checkImg(obs, imgParms)
tnum = tparms.shape[0]
tstr = "%s,%s,%s observe %s img, %d has parameter, %d backup to web server."\
%(obs[1], obs[2], obs[3], obs[4], imgParms.shape[0], tnum)
self.log.debug(tstr)
if tnum < minNum:
self.updateHasWCS(orsId, tnum, minTime, maxTime,
'false')
else:
if tnum > 100:
tparms = tparms[25:-25]
doSuccess = self.doAstrometry(orsId, tparms)
else:
doSuccess = self.doAstrometry(orsId, tparms)
if doSuccess:
self.updateHasWCS(orsId, tnum, minTime, maxTime,
'true')
def test():
#toolPath = '/home/gwac/img_diff_xy/image_diff'
toolPath = '/home/xy/Downloads/myresource/deep_data2/image_diff'
tools = AstroTools(toolPath)
dataDest0 = "/home/xy/gwac_diff_xy/data"
logDest0 = "/home/xy/gwac_diff_xy/log"
if not os.path.exists(dataDest0):
os.system("mkdir -p %s" % (dataDest0))
if not os.path.exists(logDest0):
os.system("mkdir -p %s" % (logDest0))
startProcess = False
dayRun = 0
nigRun = 0
skyId = 0
ffId = 0
tfiles = []
#srcPath00='/data1/G004_041_190124'
srcPath00 = '/home/xy/Downloads/myresource/matchTest'
dateStr = '190124'
camName = 'G041'
curSkyId = '123'
dstDir = '%s/%s' % (dataDest0, dateStr)
tdiff = BatchImageDiff(srcPath00, dstDir, tools, camName, curSkyId)
#tpath1 = '/data2/G003_034_190211'
#tpath2 = '/data2/G003_034_190227'
tpath1 = '/home/xy/Downloads/myresource/matchTest'
tpath2 = '/home/xy/Downloads/myresource/matchTest'
fname1 = 'G034_mon_objt_190211T12192603.fit'
fname2 = 'G044_mon_objt_190305T13070793.fit'
ra0, dec0 = -1000, -1000
wcsfile1, ra_center1, dec_center1 = tdiff.getWCS(tpath1, fname1, ra0, dec0)
wcs1 = WCS('%s/%s' % (tdiff.tmpDir, wcsfile1))
wcsfile2, ra_center2, dec_center2 = tdiff.getWCS(tpath2, fname2, ra0, dec0)
wcs2 = WCS('%s/%s' % (tdiff.tmpDir, wcsfile2))
os.system("cp %s/%s %s/%s" % (tpath1, fname1, tdiff.tmpDir, fname1))
os.system("%s %s/%s" % (tdiff.funpackProgram, tdiff.tmpDir, fname1))
os.system("cp %s/%s %s/%s" % (tpath2, fname2, tdiff.tmpDir, fname2))
os.system("%s %s/%s" % (tdiff.funpackProgram, tdiff.tmpDir, fname2))
fpar = 'sex_diff.par'
#sexConf=['-DETECT_MINAREA','3','-DETECT_THRESH','2.5','-ANALYSIS_THRESH','2.5']
sexConf = [
'-DETECT_MINAREA', '10', '-DETECT_THRESH', '5', '-ANALYSIS_THRESH', '5'
]
tcat1 = tools.runSextractor('G034_mon_objt_190211T12192603.fit',
tdiff.tmpDir, tdiff.tmpDir, fpar, sexConf)
tcat2 = tools.runSextractor('G044_mon_objt_190305T13070793.fit',
tdiff.tmpDir, tdiff.tmpDir, fpar, sexConf)
tdata2 = np.loadtxt("%s/%s" % (tdiff.tmpDir, tcat2))
tXY = tdata2[:, 0:2]
print(tXY[:3])
tRaDec = wcs2.all_pix2world(tXY, 1)
print(tRaDec[:3])
tXY2 = wcs1.all_world2pix(tRaDec, 1)
print(tXY2[:3])
tdata2[:, 0:2] = tXY2
saveName = "%s_trans.cat" % (fname2.split(".")[0])
savePath = "%s/%s" % (tdiff.tmpDir, saveName)
np.savetxt(savePath, tdata2, fmt='%.4f')
mchFile, nmhFile, mchPair = tools.runCrossMatch(tdiff.tmpDir, saveName,
tcat1, 1)
evaluatePos(tdiff.tmpDir, saveName, tcat1, mchPair)
evaluatePos(tdiff.tmpDir, tcat2, tcat1, mchPair)