def get_wxnum(wxname): sql = "select wxnum from pb_wxnum where wxname='%s'" % (wxname) mysqlhelper = MysqlUtil() data = mysqlhelper.select(sql) if data[0] == 0: print u'数据库中没有找到微信号' return None else: return data[1]
def insSql(conls): insertSql = """insert into csg_info.src_table_dic_test ( domain ,company_code ,sys_name ,table_schema ,table_name ,table_name_zh ,column_position ,column_name ,pk ,data_type ,data_length ,data_precision ,data_scale ,data_default ,nullable ,table_comment ,column_comment ,remark ,table_name_des ) values ( '%s','%s','%s' ,'%s' ,'%s', '%s','%s','%s' ,'%s' ,'%s', '%s','%s','%s' ,'%s' ,'%s', '%s','%s','%s' ,'%s' ) """ % ( conls.get_domain() , conls.get_company_code() , conls.get_sys_name() , conls.get_table_schema() , conls.get_table_name() , conls.get_table_name_zh() , conls.get_column_position() , conls.get_column_name() , conls.get_pk() , conls.get_data_type() , conls.get_data_length() , conls.get_data_precision() , conls.get_data_scale() , conls.get_data_default() , conls.get_nullable() , conls.get_table_comment() , conls.get_column_comment() , conls.get_remark() , conls.get_table_name_des() ) MysqlUtil.upsert(insertSql)
def sourDataGetTableCommet(schema, tablename): tableList = MysqlUtil.query( " SELECT * FROM information_schema.tables WHERE table_schema ='%s' AND table_name = '%s' " % ( schema, tablename)) if (tableList): # for table in tableList: # print(table) print(tableList[0]) print(tableList[0][20])
def conGet(): connLinkList = MysqlUtil.query( "SELECT domain,company_code,conn_command FROM `src_connect_info` ORDER BY company_code ") conDic = {} if (connLinkList): for con in connLinkList: conDic[con[0] + "_" + con[1]] = con[2] print(conDic)
def sourDataGet(schema, tablename): # 返回对象list colslistRes = list(); colsList = MysqlUtil.query( " SELECT * FROM information_schema.columns WHERE table_schema ='%s' AND table_name = '%s' " % ( schema, tablename)) if (colsList): for conlItem in colsList: # print(conlItem[0]) colslistRes.append(conlItem) return colslistRes
def begin_process(wxnum, s_date, e_date): wxnum = str(wxnum) sql = "select wxbiz from pb_wxnum where wxnum='%s'" % (wxnum) mysqlhelper = MysqlUtil() data = mysqlhelper.select(sql) if data[0] == 0: print u'数据库中没有找到biz,改为Sogou查找' else: print data[0] print data[1][0][0] wx_link = 'http://mp.weixin.qq.com/mp/getmasssendmsg?__biz=%s#wechat_redirect' % ( data[1][0][0]) job = Job((wxnum, wx_link)) job.do_work() time.sleep(2) redis_client = RedisUtil() url = redis_client.get_out(wxnum) if url is None: print u'队列中没有此键值' raise Exception('队列中没有地址') # print url articles = crawler_wxnum_articles(url, s_date, e_date) # print articles[0][2][2] pdf_files = [] for target_list in articles: for target_sub in target_list: # print target_sub[2] # print html_parser.unescape(target_sub[3]) pdf_files.append(urltopdf(target_sub[3], target_sub[2])) pass return pdf_files
def conGetWhere(): company_code = '00'; domain = 'xt' connLinkList = MysqlUtil.query( "SELECT host,username,password,port,db_name FROM `src_connect_info` where db_type= 'mysql' and company_code= '%s' and domain = '%s' " % ( company_code, domain )) conDic = {} # if (connLinkList): # for con in connLinkList: # conDic[con[0] + "_" + con[1]] = con[2] if (connLinkList): print(connLinkList[0])
def update_stock_list(): logging.info("begin to update table stock list info") # 1. truncate table stock_list MysqlUtil.truncate_table(stock_list) # 2. 重新获取数据并写入表中 stock_list_db.get_stock_basic_and_insert()
def update_trade_calendar(): logging.info("begin to update table trade calendar info") # 1. truncate table trade_calendar MysqlUtil.truncate_table(trade_calendar) # 2. 重新获取数据并写入表中 trade_cal_db.get_trade_calendar_and_insert()
if __name__ == '__main__': worker = IdWorker(1, 1) # 查询mysql的系统信息,获得需要查询表结构的所有表名 # where db_type = 'mysql' and s.company_code = 99 result = MysqlUtil.query(""" SELECT s.table_id, s.company_code, s.domain, s.table_schema, s.table_name, s.db_type, d.`table_name` AS table_name_des FROM src_table_extra_info s JOIN info_des_table_view d ON s.`table_id` = d.`table_id` WHERE s.db_type = 'mysql' AND d.`pk_type`=1 ORDER BY s.company_code, s.domain, s.table_name """) # result = range(0, 1) if (result): for item in result: # item = list() # item.append('11111')
import os from util import StockUtil, MysqlUtil import shutil import sys import time def move_directory(source, destination): for file_name in os.listdir(source): if file_name.endswith(".csv"): shutil.move(os.path.join(source, file_name), destination) if __name__ == "__main__": stock_util = StockUtil() mysql_util = MysqlUtil(stock_util.conn, stock_util.cursor) stock_util.test = 0 # stock_util.stagnate = 1 if len(sys.argv) >= 2 and sys.argv[1] == '--stagnate' else 0 stock_util.stagnate = 0 stock_util.get_new_stocks = 0 if stock_util.get_new_stocks: if os.path.exists('GICS/backup'): shutil.rmtree('GICS/backup') os.makedirs('GICS/backup') move_directory('GICS', 'GICS/backup') stock_util.renew_categories_index_volume() shutil.rmtree('GICS/finished') sys.exit(0)