def handle(self, fitsInfo): keywords = fitsInfo[0].header classType = astroUtil.getKeyword(keywords, "CLASS", "1D_CLASS") if classType == "SKY": return ra = astroUtil.getKeyword(keywords, 'RA') dec = astroUtil.getKeyword(keywords, 'DEC') subClass = astroUtil.getKeyword(keywords, "SUBCLASS", "1D_SUBCL") mainClass = astroUtil.getMainClass(classType, subClass) date = astroUtil.getKeyword(keywords, "DATE-OBS")[0:10] planID = astroUtil.getKeyword(keywords, "PLANID") z = astroUtil.getKeyword(keywords, "Z", "1D_Z") z_err = astroUtil.getKeyword(keywords, "1D_Z_ERR", "Z_ERR") version_idl = astroUtil.getKeyword(keywords, "VERSIDL") version_ppl = astroUtil.getKeyword(keywords, "VERS2D", "VERSPIPE") SN_U = astroUtil.getKeyword(keywords, "SN_U") SN_G = astroUtil.getKeyword(keywords, "SN_G") SN_R = astroUtil.getKeyword(keywords, "SN_R") SN_I = astroUtil.getKeyword(keywords, "SN_I") SN_Z = astroUtil.getKeyword(keywords, "SN_Z") sqlRecord = [planID, version_idl, version_ppl, date, ra, dec, classType, mainClass, subClass, z, z_err, SN_U, SN_G, SN_R, SN_I, SN_Z] sqlHelper.execWithValues(self.conn, self.sql_insert, sqlRecord) self.count += 1
def insertCoordinate(sql_select, sql_select_count, insertTable): coordinateDic = getCoordinateDic(sql_select, sql_select_count) conn = sqlHelper.getMySQLConn() sql_insert = sqlHelper.genInsertSQL(insertTable, 3) for x in coordinateDic.keys(): for y in coordinateDic[x].keys(): count = coordinateDic[x][y] sqlHelper.execWithValues(conn, sql_insert, [x, y, count]) sqlHelper.closeConn(conn)
def PanstarrsCrossUpdate(specid, ra, dec, conn, cache): data = getCatchData(ra, dec, cache) if data is None: print "failed [file not exist]:" + specid return False for row in data: ra_panstarrs = row[0] dec_panstarrs = row[1] if astroUtil.checkDuplicate(ra, dec, ra_panstarrs, dec_panstarrs): # mags = row[6] # values = [mags[0], mags[1], mags[2], mags[3], mags[4], specid] values = [row[2], row[3], row[4], row[5], row[6], specid] sqlHelper.execWithValues(conn, sql_update, values) print "succeed :" + specid return True print "failed :" + specid return False
def migrationSDSS(filePath, tableName, colCount, blankColCount=0): sql_insert = sqlHelper.genInsertSQL(tableName, colCount) data = open(filePath) conn = sqlHelper.getMySQLConn() count = 0 for row in data: cols = row.split(",") if blankColCount > 0: for i in range(blankColCount): cols.append("") sqlHelper.execWithValues(conn, sql_insert, cols) count += 1 if count % 10000 == 0: print(count) sqlHelper.closeConn(conn)
def run(self): crossedCount = 0 minDis = 100 minID = 0 for row in self.data_sdss: ra_sdss = row[1] dec_sdss = row[2] distance = getDistance(self.ra, self.dec, ra_sdss, dec_sdss) if distance == 100: continue if distance < minDis: minDis = distance minID = row[0] crossedCount += 1 if crossedCount == 0: print "failed :" + specid else: sqlHelper.execWithValues(self.conn, sql_insert, [self.specid, minID]) print "%s - succeed [%s] : %s" % (self.totalCount, str(crossedCount), specid)
def importLamostData(sql_select, sql_insert): conn = sqlHelper.getMySQLConn() conn45 = sqlHelper.getMySQL45Conn() count = 0 data = sqlHelper.getFetchManyByConn(conn45, sql_select) for info in data: sqlRecord = [] colIndex = 1 try: for col in info: if colIndex == 13: classType = info[10] subClass = info[11] mainClass = astroUtil.getMainClass(classType, subClass) sqlRecord.append(mainClass) sqlRecord.append(col) colIndex += 1 sqlHelper.execWithValues(conn, sql_insert, sqlRecord) count += 1 except Exception, ex: exUtil.printMessage(ex, "Migration")
def dereplicationByCoordinate(tableName, idField="objID", raField="ra", decField="dec", orderbyField="objID"): sql_select = "select %s,%s,`%s` from %s order by %s desc" % (idField, raField, decField, tableName, orderbyField) sql_delete = "delete from " + tableName + " where " + idField + " = %s" data = sqlHelper.getFetchAll(sql_select) data_saved = [] conn = sqlHelper.getMySQLConn() count = 0 derepCount = 0 for row in data: objID = row[0] ra = row[1] dec = row[2] if checkReplication(ra, dec, data_saved): sqlHelper.execWithValues(conn, sql_delete, [objID]) derepCount += 1 print str(derepCount) + ":" + str(objID) else: data_saved.insert(0, [ra, dec]) count += 1 if count % 1000 == 0: print "handled count : " + str(count) print "total : %s , derep : %s" % (count, derepCount) sqlHelper.closeConn(conn)