def handle(self, fitsInfo):
keywords = fitsInfo[0].header
classType = astroUtil.getKeyword(keywords, "CLASS", "1D_CLASS")
if classType == "SKY":
return
ra = astroUtil.getKeyword(keywords, 'RA')
dec = astroUtil.getKeyword(keywords, 'DEC')
subClass = astroUtil.getKeyword(keywords, "SUBCLASS", "1D_SUBCL")
mainClass = astroUtil.getMainClass(classType, subClass)
date = astroUtil.getKeyword(keywords, "DATE-OBS")[0:10]
planID = astroUtil.getKeyword(keywords, "PLANID")
z = astroUtil.getKeyword(keywords, "Z", "1D_Z")
z_err = astroUtil.getKeyword(keywords, "1D_Z_ERR", "Z_ERR")
version_idl = astroUtil.getKeyword(keywords, "VERSIDL")
version_ppl = astroUtil.getKeyword(keywords, "VERS2D", "VERSPIPE")
SN_U = astroUtil.getKeyword(keywords, "SN_U")
SN_G = astroUtil.getKeyword(keywords, "SN_G")
SN_R = astroUtil.getKeyword(keywords, "SN_R")
SN_I = astroUtil.getKeyword(keywords, "SN_I")
SN_Z = astroUtil.getKeyword(keywords, "SN_Z")
sqlRecord = [planID, version_idl, version_ppl, date, ra, dec,
classType, mainClass, subClass, z, z_err, SN_U, SN_G, SN_R, SN_I, SN_Z]
sqlHelper.execWithValues(self.conn, self.sql_insert, sqlRecord)
self.count += 1
def insertCoordinate(sql_select, sql_select_count, insertTable):
coordinateDic = getCoordinateDic(sql_select, sql_select_count)
conn = sqlHelper.getMySQLConn()
sql_insert = sqlHelper.genInsertSQL(insertTable, 3)
for x in coordinateDic.keys():
for y in coordinateDic[x].keys():
count = coordinateDic[x][y]
sqlHelper.execWithValues(conn, sql_insert, [x, y, count])
sqlHelper.closeConn(conn)
def PanstarrsCrossUpdate(specid, ra, dec, conn, cache):
data = getCatchData(ra, dec, cache)
if data is None:
print "failed [file not exist]:" + specid
return False
for row in data:
ra_panstarrs = row[0]
dec_panstarrs = row[1]
if astroUtil.checkDuplicate(ra, dec, ra_panstarrs, dec_panstarrs):
# mags = row[6]
# values = [mags[0], mags[1], mags[2], mags[3], mags[4], specid]
values = [row[2], row[3], row[4], row[5], row[6], specid]
sqlHelper.execWithValues(conn, sql_update, values)
print "succeed :" + specid
return True
print "failed :" + specid
return False
def migrationSDSS(filePath, tableName, colCount, blankColCount=0):
sql_insert = sqlHelper.genInsertSQL(tableName, colCount)
data = open(filePath)
conn = sqlHelper.getMySQLConn()
count = 0
for row in data:
cols = row.split(",")
if blankColCount > 0:
for i in range(blankColCount):
cols.append("")
sqlHelper.execWithValues(conn, sql_insert, cols)
count += 1
if count % 10000 == 0:
print(count)
sqlHelper.closeConn(conn)
def run(self):
crossedCount = 0
minDis = 100
minID = 0
for row in self.data_sdss:
ra_sdss = row[1]
dec_sdss = row[2]
distance = getDistance(self.ra, self.dec, ra_sdss, dec_sdss)
if distance == 100:
continue
if distance < minDis:
minDis = distance
minID = row[0]
crossedCount += 1
if crossedCount == 0:
print "failed :" + specid
else:
sqlHelper.execWithValues(self.conn, sql_insert, [self.specid, minID])
print "%s - succeed [%s] : %s" % (self.totalCount, str(crossedCount), specid)
def importLamostData(sql_select, sql_insert):
conn = sqlHelper.getMySQLConn()
conn45 = sqlHelper.getMySQL45Conn()
count = 0
data = sqlHelper.getFetchManyByConn(conn45, sql_select)
for info in data:
sqlRecord = []
colIndex = 1
try:
for col in info:
if colIndex == 13:
classType = info[10]
subClass = info[11]
mainClass = astroUtil.getMainClass(classType, subClass)
sqlRecord.append(mainClass)
sqlRecord.append(col)
colIndex += 1
sqlHelper.execWithValues(conn, sql_insert, sqlRecord)
count += 1
except Exception, ex:
exUtil.printMessage(ex, "Migration")
def dereplicationByCoordinate(tableName, idField="objID", raField="ra", decField="dec", orderbyField="objID"):
sql_select = "select %s,%s,`%s` from %s order by %s desc" % (idField, raField, decField, tableName, orderbyField)
sql_delete = "delete from " + tableName + " where " + idField + " = %s"
data = sqlHelper.getFetchAll(sql_select)
data_saved = []
conn = sqlHelper.getMySQLConn()
count = 0
derepCount = 0
for row in data:
objID = row[0]
ra = row[1]
dec = row[2]
if checkReplication(ra, dec, data_saved):
sqlHelper.execWithValues(conn, sql_delete, [objID])
derepCount += 1
print str(derepCount) + ":" + str(objID)
else:
data_saved.insert(0, [ra, dec])
count += 1
if count % 1000 == 0:
print "handled count : " + str(count)
print "total : %s , derep : %s" % (count, derepCount)
sqlHelper.closeConn(conn)