def getInputData(prefixes, table, sqlFilter): fieldList = genMagByPrefix(prefixes) fieldList.append("`type`") fieldList.append("mainclass") sql_select = sqlHelper.genSelectSQLWithFieldList(fieldList, table, sqlFilter) outFile = "data_input.txt" outFile_tag = "data_input_tag.txt" output = open(outFile, "w") output_tag = open(outFile_tag, "w") count = 0 res = [] tags = [] data = sqlHelper.getFetchAll(sql_select) for item in data: record = [] colCount = len(item) for i in range(colCount - 1): record.append(item[i]) res.append(record) tags.append(item[colCount - 1]) output.write(util.listToString(record)) output.write("\n") output_tag.write(item[colCount - 1]) output_tag.write("\n") count += 1 print "input data count : %s" % count output.close() output_tag.close() return np.mat(res), np.mat(tags).T
def dereplicationByCoordinate(tableName, idField="objID", raField="ra", decField="dec", orderbyField="objID"): sql_select = "select %s,%s,`%s` from %s order by %s desc" % (idField, raField, decField, tableName, orderbyField) sql_delete = "delete from " + tableName + " where " + idField + " = %s" data = sqlHelper.getFetchAll(sql_select) data_saved = [] conn = sqlHelper.getMySQLConn() count = 0 derepCount = 0 for row in data: objID = row[0] ra = row[1] dec = row[2] if checkReplication(ra, dec, data_saved): sqlHelper.execWithValues(conn, sql_delete, [objID]) derepCount += 1 print str(derepCount) + ":" + str(objID) else: data_saved.insert(0, [ra, dec]) count += 1 if count % 1000 == 0: print "handled count : " + str(count) print "total : %s , derep : %s" % (count, derepCount) sqlHelper.closeConn(conn)
def getCoordinateDic(tableName): sql_select = "SELECT * FROM " + tableName data = sqlHelper.getFetchAll(sql_select) return astroUtil.getCoordinateCountMap(data)
if distance == 100: continue if distance < minDis: minDis = distance minID = row[0] crossedCount += 1 if crossedCount == 0: print "failed :" + specid else: sqlHelper.execWithValues(self.conn, sql_insert, [self.specid, minID]) print "%s - succeed [%s] : %s" % (self.totalCount, str(crossedCount), specid) data1 = sqlHelper.getFetchAll(sql_select_lamost) data2 = sqlHelper.getFetchAll(sql_select_sdss) data_lamost = [] data_sdss = [] for row in data1: data_lamost.append([row[0], float(row[1]), float(row[2])]) for row in data2: data_sdss.append([row[0], float(row[1]), float(row[2])]) conn = sqlHelper.getMySQLConn() totalCount = 0 for row_lamost in data_lamost: specid = row_lamost[0]
# coding=utf-8 __author__ = 'zdwdong' """ 画 teff/logg散点图 """ import CodeLib.Database.MySQLHepler as sqlHelper import matplotlib.pyplot as plt import CodeLib.Util.PlotUtil as plotUtil sql_select = "select teff,logg from dr3.M31_ELL where teff>0 and logg>0" data = sqlHelper.getFetchAll(sql_select) teff = [] logg = [] for dataRow in data: teff.append(float(dataRow[0])) logg.append(float(dataRow[1])) plt.title("Lamost Logg_teff") plt.plot(teff, logg, '.') plt.xlabel('teff') plt.ylabel('logg') plotUtil.plotGrid(plt) plotUtil.invertX(plt) plt.show()