def get_wxnum(wxname):
sql = "select wxnum from pb_wxnum where wxname='%s'" % (wxname)
mysqlhelper = MysqlUtil()
data = mysqlhelper.select(sql)
if data[0] == 0:
print u'数据库中没有找到微信号'
return None
else:
return data[1]
def insSql(conls):
insertSql = """insert into csg_info.src_table_dic_test (
domain
,company_code
,sys_name
,table_schema
,table_name
,table_name_zh
,column_position
,column_name
,pk
,data_type
,data_length
,data_precision
,data_scale
,data_default
,nullable
,table_comment
,column_comment
,remark
,table_name_des
)
values ( '%s','%s','%s' ,'%s' ,'%s',
'%s','%s','%s' ,'%s' ,'%s',
'%s','%s','%s' ,'%s' ,'%s',
'%s','%s','%s' ,'%s'
) """ % (
conls.get_domain()
, conls.get_company_code()
, conls.get_sys_name()
, conls.get_table_schema()
, conls.get_table_name()
, conls.get_table_name_zh()
, conls.get_column_position()
, conls.get_column_name()
, conls.get_pk()
, conls.get_data_type()
, conls.get_data_length()
, conls.get_data_precision()
, conls.get_data_scale()
, conls.get_data_default()
, conls.get_nullable()
, conls.get_table_comment()
, conls.get_column_comment()
, conls.get_remark()
, conls.get_table_name_des()
)
MysqlUtil.upsert(insertSql)
def sourDataGetTableCommet(schema, tablename):
tableList = MysqlUtil.query(
" SELECT * FROM information_schema.tables WHERE table_schema ='%s' AND table_name = '%s' " % (
schema, tablename))
if (tableList):
# for table in tableList:
# print(table)
print(tableList[0])
print(tableList[0][20])
def conGet():
connLinkList = MysqlUtil.query(
"SELECT domain,company_code,conn_command FROM `src_connect_info` ORDER BY company_code ")
conDic = {}
if (connLinkList):
for con in connLinkList:
conDic[con[0] + "_" + con[1]] = con[2]
print(conDic)
def sourDataGet(schema, tablename):
# 返回对象list
colslistRes = list();
colsList = MysqlUtil.query(
" SELECT * FROM information_schema.columns WHERE table_schema ='%s' AND table_name = '%s' " % (
schema, tablename))
if (colsList):
for conlItem in colsList:
# print(conlItem[0])
colslistRes.append(conlItem)
return colslistRes
def begin_process(wxnum, s_date, e_date):
wxnum = str(wxnum)
sql = "select wxbiz from pb_wxnum where wxnum='%s'" % (wxnum)
mysqlhelper = MysqlUtil()
data = mysqlhelper.select(sql)
if data[0] == 0:
print u'数据库中没有找到biz,改为Sogou查找'
else:
print data[0]
print data[1][0][0]
wx_link = 'http://mp.weixin.qq.com/mp/getmasssendmsg?__biz=%s#wechat_redirect' % (
data[1][0][0])
job = Job((wxnum, wx_link))
job.do_work()
time.sleep(2)
redis_client = RedisUtil()
url = redis_client.get_out(wxnum)
if url is None:
print u'队列中没有此键值'
raise Exception('队列中没有地址')
# print url
articles = crawler_wxnum_articles(url, s_date, e_date)
# print articles[0][2][2]
pdf_files = []
for target_list in articles:
for target_sub in target_list:
# print target_sub[2]
# print html_parser.unescape(target_sub[3])
pdf_files.append(urltopdf(target_sub[3], target_sub[2]))
pass
return pdf_files
def conGetWhere():
company_code = '00';
domain = 'xt'
connLinkList = MysqlUtil.query(
"SELECT host,username,password,port,db_name FROM `src_connect_info` where db_type= 'mysql' and company_code= '%s' and domain = '%s' " % (
company_code, domain
))
conDic = {}
# if (connLinkList):
# for con in connLinkList:
# conDic[con[0] + "_" + con[1]] = con[2]
if (connLinkList):
print(connLinkList[0])
def update_stock_list():
logging.info("begin to update table stock list info")
# 1. truncate table stock_list
MysqlUtil.truncate_table(stock_list)
# 2. 重新获取数据并写入表中
stock_list_db.get_stock_basic_and_insert()
def update_trade_calendar():
logging.info("begin to update table trade calendar info")
# 1. truncate table trade_calendar
MysqlUtil.truncate_table(trade_calendar)
# 2. 重新获取数据并写入表中
trade_cal_db.get_trade_calendar_and_insert()
if __name__ == '__main__':
worker = IdWorker(1, 1)
# 查询mysql的系统信息，获得需要查询表结构的所有表名
# where db_type = 'mysql' and s.company_code = 99
result = MysqlUtil.query(""" SELECT
s.table_id,
s.company_code,
s.domain,
s.table_schema,
s.table_name,
s.db_type,
d.`table_name` AS table_name_des
FROM
src_table_extra_info s
JOIN info_des_table_view d ON s.`table_id` = d.`table_id`
WHERE s.db_type = 'mysql' AND d.`pk_type`=1
ORDER BY
s.company_code,
s.domain,
s.table_name """)
# result = range(0, 1)
if (result):
for item in result:
# item = list()
# item.append('11111')
import os
from util import StockUtil, MysqlUtil
import shutil
import sys
import time
def move_directory(source, destination):
for file_name in os.listdir(source):
if file_name.endswith(".csv"):
shutil.move(os.path.join(source, file_name), destination)
if __name__ == "__main__":
stock_util = StockUtil()
mysql_util = MysqlUtil(stock_util.conn, stock_util.cursor)
stock_util.test = 0
# stock_util.stagnate = 1 if len(sys.argv) >= 2 and sys.argv[1] == '--stagnate' else 0
stock_util.stagnate = 0
stock_util.get_new_stocks = 0
if stock_util.get_new_stocks:
if os.path.exists('GICS/backup'):
shutil.rmtree('GICS/backup')
os.makedirs('GICS/backup')
move_directory('GICS', 'GICS/backup')
stock_util.renew_categories_index_volume()
shutil.rmtree('GICS/finished')
sys.exit(0)