def pull(self, dt=None, to_mysql=True, to_csv=False):
succ = True
df = None
try:
df = self._run(dt)
if (df.empty):
logger.info('get empty df for ' + self.__class__.__name__)
else:
if (to_mysql):
self._save_to_mysql(pm, df)
if (to_csv):
self._save_to_csv(df)
except Exception as e:
logger.exception('tbase puller error')
succ = False
digest_log.info('t-puller|%s|%s|%s|%s' %
(self.__class__.__name__, dt, str(to_mysql), succ))
return succ
def __parse_baostock_data(self, code, start, end, freq):
'''
虽然价格都进行了复权，但是成交量和换手率都没有前复权(tushare和baostock都没做)，同花顺也是， 东方财富做了。
前复权的价格和同方财富以及同花顺有出入。
'''
'''
能获取2011-01-01至当前时间的数据；
date 交易所行情日期 格式：YYYY-MM-DD
code 证券代码 格式：sh.600000。sh：上海，sz：深圳
open 今开盘价格 精度：小数点后4位；单位：人民币元
high 最高价 精度：小数点后4位；单位：人民币元
low 最低价 精度：小数点后4位；单位：人民币元
close 今收盘价 精度：小数点后4位；单位：人民币元
preclose 昨日收盘价 精度：小数点后4位；单位：人民币元
volume 成交数量 单位：股
amount 成交金额 精度：小数点后4位；单位：人民币元
adjustflag 复权状态 不复权、前复权、后复权
turn 换手率 精度：小数点后6位；单位：%
tradestatus 交易状态 1：正常交易 0：停牌
pctChg 涨跌幅 精度：小数点后6位
peTTM 动态市盈率 精度：小数点后4位
psTTM 市销率 精度：小数点后4位
pcfNcfTTM 市现率 精度：小数点后4位
pbMRQ 市净率 精度：小数点后4位
isST 是否ST 1是，0否'''
try:
rs = bs.query_history_k_data(
code,
"date,code,open,high,low,close,volume,turn",
start_date=start,
end_date=end,
frequency=freq,
adjustflag="2")
except BaseException as e:
logger.exception('query_history_k_data error')
raise e
# print(rs.data);
data_list = []
while (rs.error_code == '0') & rs.next():
# 获取一条记录，将记录合并在一起
data_list.append(rs.get_row_data())
result = pd.DataFrame(data_list, columns=rs.fields)
return result
def __init__(self,host=ConfigDict['ip'],user=ConfigDict['user'],password=ConfigDict['password'],db_name=ConfigDict['db_name']):
try:
logger.info('start to build pool');
self.pool = mysql.connector.pooling.MySQLConnectionPool(pool_name="mypool", pool_size=10, host=host,
port=3306, database=db_name,
user=user, password=password,
pool_reset_session=True)
# self.pool = pool = ConnectionPool(**config);
# PooledDB(MySQLdb, mincached=5,maxcached=5, maxconnections=20, host='localhost', user='******', passwd='pwd', db='myDB', port=3306)
# # 打开数据库连接
#self.__conn = pymysql.connect(host, user, password, db_name,charset='utf8');
#self.__cursor=self.__conn .cursor();
#self.__engine = create_engine('mysql+pymysql://'+user+':'+password+'@'+host+'/'+db_name+'?charset=utf8')
logger.info('build pool OK');
except BaseException as be:
logger.exception('build error');
raise be;
def update(self, table_name, df, primaryKeys):
if (df.empty):
logger.info('no date to update')
return
if (primaryKeys == '' or primaryKeys == None):
raise Exception('primaryKeys is None');
conn = self.get_conn();
cur = conn.cursor();
try:
sql_prefix = 'update ' + table_name + ' set ';
sql='';
#for i in np.arange(df.shape[0]):
for i in df.index:
chip = '';
for vi, val in enumerate(df.columns):
if(val not in primaryKeys):
chip =chip+ self.__parse_field__(val) + '=' + self.__parse_value__(df.loc[i, val]);
if (vi < len(df.columns) - 1):
chip = chip + ','
if(chip==''):
raise Exception('updated columns is None');
chip += ' where ';
for vi, val in enumerate(primaryKeys):
chip += self.__parse_field__(val) + '=' + self.__parse_value__(df.loc[i, val]);
if (vi < len(primaryKeys) - 1):
chip = chip + ' and ';
sql = sql_prefix + chip;
cur.execute(sql);
logger.debug('update sql count:'+str(df.shape[0]));
conn.commit();
except BaseException as be:
conn.rollback();
logger.exception('updat db error')
raise be;
finally:
cur.close();
conn.close();
def run(self, dt, to_mysql=False, to_csv=False, type=1):
succ = True
try:
df = self.__query(dt, type)
df = self._process(df)
if (to_mysql):
df = self._get_update_df(df)
pm.update(DbCreator.share_data_finance,
df,
primaryKeys=[Config.id])
if (to_csv):
df.to_csv(dt + '-fin.csv')
except Exception as e:
logger.exception('fin base error')
succ = False
digest_log.info('TFinBase|%s|%s|%s|%s' %
(self.__class__.__name__, dt, to_mysql, succ))
return succ
def pull(self,
code: str,
start: str,
end: str,
to_mysql=True,
to_csv=False):
succ = True
df = None
try:
df = self._run(code, start, end)
if (to_mysql):
self._save_to_mysql(pm, df)
if (to_csv):
self._save_to_csv(code, df)
except Exception as e:
logger.exception('puller error')
succ = False
digest_log.info(
'puller|%s|%s|%s|%s|%s|%s' %
(self.__class__.__name__, code, start, end, str(to_mysql), succ))
return succ
def run(self, code, df, start=0, to_mysql=False, to_csv=False):
succ = True
try:
if (df.empty or df.shape[0] == 0):
logger.info('no date to count kpi')
else:
df = self._process(df, start)
if (to_mysql):
df = df.loc[start:, :]
tb_name = self._get_update_table_name()
pm.update(tb_name, df, primaryKeys=[Config.id])
if (to_csv):
df.to_csv(code + '-base.csv')
except Exception as e:
logger.exception('base kpi error')
print(e.__traceback__)
succ = False
digest_log.info('Alg-Base|%s|%s|%d|%s|%s' %
(self.__class__.__name__, code, start, to_mysql, succ))
return succ
def process(self, df, start_data):
try:
return (True, self.__process(df, start_data))
except Exception as e:
logger.exception('pre process error')
return (False, -1)
