def fetchdata(table):
if table not in LOGGERMAP:
logger = tul.TuLog('fetch_' + table + '_x', '/log', True).getlog()
LOGGERMAP[table] = logger
logger = LOGGERMAP.get(table)
force = FORCEMAP.get(table, FORCE)
fav = FAVMAP.get(table, FAV)
batch = BATCHMAP.get(table, BATCH)
conn = tuh.getMysqlConn()
stktcs = tuh.getstktcs(fav)
stkmtdm = tuh.getstkmtdm(table)
tumethod = TUMETHODMAP[table]
logger.debug(
'====S====> table:%s tumethod:%s ===>force:%s fav:%s batch:%s' %
(table, tumethod, force, fav, batch))
cursor = conn.cursor()
isql = ''
for stktcm in stktcs:
stktc = stktcm['ts_code']
sdate = cd.ymd2date(tuh.getsdate(stkmtdm, stktc, force))
edate = cd.preday()
logger.debug('====SS====> %s ==== init ===>sd:%s ed:%s' %
(stktc, sdate, edate))
fetchstk(cursor, logger, table, tumethod, isql, stktc, sdate, edate,
batch)
cursor.close()
conn.close()
logger.debug('====E====> %s ============================' % (table, ))
def fetchdataone(liststs='L'):
logger = tul.TuLog('fetch_stk_basic_' + liststs + 'x', '/log',
True).getlog()
conn = tuh.getMysqlConn()
cursor = conn.cursor(pymysql.cursors.DictCursor)
cursor.execute(
"select t.ts_code,t.delist_date,t.list_status,t.`name` from stk_basic t order by t.ts_code;"
)
stkbd = cursor.fetchall()
stkbm = tuh.listToDict(stkbd, 'ts_code', '_all_')
cursor.close()
tuapi = tuh.tuApi
isql = ''
usql = ''
cursor = conn.cursor()
df = fetchtudata(tuapi, liststs=liststs)
if len(df) > 0:
cols = df.columns
isql = tuh.genInsSql(cols, 'stk_basic', isql)
rowvs = []
urowvs = []
for index, row in df.iterrows():
rowv = []
if row['ts_code'] in stkbm:
if checkdatachg(row, stkbm[row['ts_code']]):
usql = tuh.genupdsql(cols, 'stk_basic', ('ts_code', ), usql)
for col in cols:
rowv.append(row[col])
rowv.append(row['ts_code'])
if len(rowv) > 0:
urowvs.append(rowv)
continue
for col in cols:
rowv.append(row[col])
if len(rowv) > 0:
rowvs.append(rowv)
logger.info('LS: %s rowvs == == > %d urowvs == == > %d' %
(liststs, len(rowvs), len(urowvs)))
emsgs = tuh.saveorupdate({'sql': isql, 'vals': rowvs})
for m in emsgs:
logger.error(m)
emsgs = tuh.saveorupdate({'sql': usql, 'vals': urowvs})
for m in emsgs:
logger.error(m)
cursor.close()
conn.close()
# TU TRY TIMEStumaxexcnt
TUTRYTIMES = 3
# cut df
CUTMAXLEN = 5000
# stk sp method
OCOLS = ('close', 'open', 'high', 'low', 'pre_close', 'change', 'pct_chg')
STKSPOCOLMAP = {}
for ocol in OCOLS:
STKSPOCOLMAP[ocol] = 'ori_' + ocol
PRICE_COLS = ['open', 'close', 'high', 'low', 'pre_close']
FORMAT = lambda x: '%.2f' % x
GLOGGER = tul.TuLog('tuhelper', '/log', True).getlog()
def getMysqlConn():
conn = pymysql.connect(dbc.DBHOST, dbc.DBUNAME, dbc.DBPWD, dbc.DBSCHEMA)
return conn
def closeMysqlConn(conn):
try:
conn.close()
except:
pass
# 检查table是否存在,不存在新建
import time
import pandas as pd
import util.caldate as cd
import util.tuhelper as tuh
import util.tulog as tul
FORCE = False
FAV = 2
BATCH = True
TABLE = 'stk_daily_basic'
LOGGER = tul.TuLog('fetch_stk_daily_basic_x', '/log', True).getlog()
def fetchdataone():
logger = LOGGER
conn = tuh.getMysqlConn()
stktcs = tuh.getstktcs(FAV)
stkmtdm = tuh.getstkmtdm('stk_daily_basic')
tuapi = tuh.tuApi
isql = ''
cursor = conn.cursor()
for stktcm in stktcs:
stktc = stktcm['ts_code']
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
__author__ = 'Erik YU'
import conf.tables as tbsc
import util.tuhelper as tuh
import util.tulog as tul
GLOGGER = tul.TuLog('fbasicdata', '/log', True).getlog()
def fhdatas(tabn, tabc):
tuapi = tabc['tuapi']
params = tabc['params']
kfield = tabc['kfield']
ufields = tabc['ufields']
GLOGGER.debug('==S==> table:%s tuapi:%s params:%s' % (tabn, tuapi, params))
bdmap = tuh.getbdmap(tabn, kfield, tabc['ufields'])
rdf = None
for parma in params:
df = tuh.gettudf(tuapi, tabc['fields'], parma, spm=tabc['spm'])
GLOGGER.debug('====PARAM DATAS==> param:%s dflen:%s' %
(parma, len(df)))
if len(df) > 0:
if rdf is None:
rdf = df
else:
rdf = rdf.append(df)
conn = tuh.getMysqlConn()
cursor = conn.cursor()
emsgs = tuh.savebasicdf(tabn, rdf, bdmap, kfield, ufields, GLOGGER)
import pandas as pd
import pymysql
import util.caldate as cd
import util.tuhelper as tuh
import util.tulog as tul
__force = False
__fav = 2
__batch = True
__mwdx = ('daily', 'weekly', 'monthly')
__mwd = ('daily', )
__logmap = {
'mlog': tul.TuLog('fetch_stk_m_x', '/log', True).getlog(),
'wlog': tul.TuLog('fetch_stk_w_x', '/log', True).getlog(),
'dlog': tul.TuLog('fetch_stk_d_x', '/log', True).getlog()
}
__qdfcmap = {'m': [], 'w': [], 'd': []}
__ocols = ('close', 'open', 'high', 'low', 'pre_close', 'change', 'pct_chg')
__ocolmap = {}
for ocol in __ocols:
__ocolmap[ocol] = 'ori_' + ocol
PRICE_COLS = ['open', 'close', 'high', 'low', 'pre_close']
FORMAT = lambda x: '%.2f' % x
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
__author__ = 'Erik YU'
import pandas as pd
import conf.tables as tbsc
import util.caldate as cd
import util.tuhelper as tuh
import util.tulog as tul
GLOGGER = tul.TuLog('fstkdailydata', '/log', True).getlog()
TABLOGMAP = {}
TABMAXVAL = {}
BASICTABDATA = {}
# 1.每日新增,有开始结束时间 loop=true
# 2.每日新增,有开始结束时间,需要裁剪 loop=true,cut=true
# 3.每日新增,无需时间 loop=false
def fhdatas(kf, kfv, tabn, tabc):
tuapi = tabc['tuapi']
dfield = tabc['dfield']
GLOGGER.debug('==S==> table:%s tc:%s tuapi:%s dfield:%s' %
(tabn, kfv, tuapi, dfield))
conn = tuh.getMysqlConn()
cursor = conn.cursor()
rdf = None
maxdate = tuh.getmaxdate(TABMAXVAL[tabn], kfv,
tabc.get('sdate', tuh.TUSDATE))
if tabc['loop']:
def fetchData(ind):
logger = tul.TuLog('fetch_idx_' + ind + '_x', '/log').getlog()
conn = tuh.getMysqlConn()
cursor = conn.cursor(pymysql.cursors.DictCursor)
if __fav == 0 or __fav == 1:
cursor.execute(
"select t.ts_code from idx_basic t where t.del<>1 and t.fav=" +
str(__fav) + " order by t.ts_code;")
else:
cursor.execute(
"select t.ts_code from idx_basic t where t.del<>1 order by t.ts_code;"
)
idxcs = cursor.fetchall()
cursor.close()
cursor = conn.cursor(pymysql.cursors.DictCursor)
cursor.execute(
"select t.ts_code as idc,max(t.trade_date) as itd from idx_" + ind +
" t group by t.ts_code;")
idxsdd = tuh.listToDict(cursor.fetchall(), 'idc', 'itd')
cursor.close()
tuapi = tuh.tuApi
isql = ''
cursor = conn.cursor()
for idxd in idxcs:
idxc = idxd['ts_code']
sdate = cd.ymd2date(getIdxSdate(idxsdd, idxc, __force))
edate = cd.preday()
while edate >= sdate:
try:
df = fetchTuData(tuapi, ind, sdate.strftime('%Y%m%d'),
edate.strftime('%Y%m%d'), idxc)
if len(df) > 0:
cols = df.columns
isql = tuh.genInsSql(cols, 'idx_' + ind, isql)
rowvs = []
for index, row in df.iterrows():
rowv = []
for col in cols:
rowv.append(row[col])
rowvs.append(rowv)
emsgs = tuh.insertDatas({'sql': isql, 'vals': rowvs})
for m in emsgs:
logger.error(m)
print(idxc, '===>sd:', sdate, ' ed:', edate, ' ldf:',
len(df), ' mtd:', df['trade_date'].min(), ' emc:',
len(emsgs))
else:
print(idxc, '===>sd:', sdate, ' ed:', edate, ' ldf:',
len(df))
except BaseException as e:
print(e)
time.sleep(60)
continue
if pd.isnull(df['trade_date'].min()):
edate = cd.preday(sdate)
else:
edate = cd.preday(cd.ymd2date(df['trade_date'].min()))
cursor.close()
conn.close()
__author__ = 'Erik YU'
import time
import pandas as pd
import pymysql
import util.caldate as cd
import util.tuhelper as tuh
import util.tulog as tul
__force = False
__fav = 2
__batch = True
__logger = tul.TuLog('fetch_stk_adj_factor_x', '/log', True).getlog()
def fetchdataone():
logger = __logger
conn = tuh.getMysqlConn()
cursor = conn.cursor()
if __force:
cursor.execute("delete from stk_adj_factor")
cursor.close()
cursor = conn.cursor(pymysql.cursors.DictCursor)
if __fav == 0 or __fav == 1:
cursor.execute(
"select t.ts_code from stk_basic t where t.del<>1 and t.fav=" +
str(__fav) + " order by t.ts_code;")