def pull(self, dt=None, to_mysql=True, to_csv=False): succ = True df = None try: df = self._run(dt) if (df.empty): logger.info('get empty df for ' + self.__class__.__name__) else: if (to_mysql): self._save_to_mysql(pm, df) if (to_csv): self._save_to_csv(df) except Exception as e: logger.exception('tbase puller error') succ = False digest_log.info('t-puller|%s|%s|%s|%s' % (self.__class__.__name__, dt, str(to_mysql), succ)) return succ
def __parse_baostock_data(self, code, start, end, freq): ''' 虽然价格都进行了复权,但是成交量和换手率都没有前复权(tushare和baostock都没做),同花顺也是, 东方财富做了。 前复权的价格和同方财富以及同花顺有出入。 ''' ''' 能获取2011-01-01至当前时间的数据; date 交易所行情日期 格式:YYYY-MM-DD code 证券代码 格式:sh.600000。sh:上海,sz:深圳 open 今开盘价格 精度:小数点后4位;单位:人民币元 high 最高价 精度:小数点后4位;单位:人民币元 low 最低价 精度:小数点后4位;单位:人民币元 close 今收盘价 精度:小数点后4位;单位:人民币元 preclose 昨日收盘价 精度:小数点后4位;单位:人民币元 volume 成交数量 单位:股 amount 成交金额 精度:小数点后4位;单位:人民币元 adjustflag 复权状态 不复权、前复权、后复权 turn 换手率 精度:小数点后6位;单位:% tradestatus 交易状态 1:正常交易 0:停牌 pctChg 涨跌幅 精度:小数点后6位 peTTM 动态市盈率 精度:小数点后4位 psTTM 市销率 精度:小数点后4位 pcfNcfTTM 市现率 精度:小数点后4位 pbMRQ 市净率 精度:小数点后4位 isST 是否ST 1是,0否''' try: rs = bs.query_history_k_data( code, "date,code,open,high,low,close,volume,turn", start_date=start, end_date=end, frequency=freq, adjustflag="2") except BaseException as e: logger.exception('query_history_k_data error') raise e # print(rs.data); data_list = [] while (rs.error_code == '0') & rs.next(): # 获取一条记录,将记录合并在一起 data_list.append(rs.get_row_data()) result = pd.DataFrame(data_list, columns=rs.fields) return result
def __init__(self,host=ConfigDict['ip'],user=ConfigDict['user'],password=ConfigDict['password'],db_name=ConfigDict['db_name']): try: logger.info('start to build pool'); self.pool = mysql.connector.pooling.MySQLConnectionPool(pool_name="mypool", pool_size=10, host=host, port=3306, database=db_name, user=user, password=password, pool_reset_session=True) # self.pool = pool = ConnectionPool(**config); # PooledDB(MySQLdb, mincached=5,maxcached=5, maxconnections=20, host='localhost', user='******', passwd='pwd', db='myDB', port=3306) # # 打开数据库连接 #self.__conn = pymysql.connect(host, user, password, db_name,charset='utf8'); #self.__cursor=self.__conn .cursor(); #self.__engine = create_engine('mysql+pymysql://'+user+':'+password+'@'+host+'/'+db_name+'?charset=utf8') logger.info('build pool OK'); except BaseException as be: logger.exception('build error'); raise be;
def update(self, table_name, df, primaryKeys): if (df.empty): logger.info('no date to update') return if (primaryKeys == '' or primaryKeys == None): raise Exception('primaryKeys is None'); conn = self.get_conn(); cur = conn.cursor(); try: sql_prefix = 'update ' + table_name + ' set '; sql=''; #for i in np.arange(df.shape[0]): for i in df.index: chip = ''; for vi, val in enumerate(df.columns): if(val not in primaryKeys): chip =chip+ self.__parse_field__(val) + '=' + self.__parse_value__(df.loc[i, val]); if (vi < len(df.columns) - 1): chip = chip + ',' if(chip==''): raise Exception('updated columns is None'); chip += ' where '; for vi, val in enumerate(primaryKeys): chip += self.__parse_field__(val) + '=' + self.__parse_value__(df.loc[i, val]); if (vi < len(primaryKeys) - 1): chip = chip + ' and '; sql = sql_prefix + chip; cur.execute(sql); logger.debug('update sql count:'+str(df.shape[0])); conn.commit(); except BaseException as be: conn.rollback(); logger.exception('updat db error') raise be; finally: cur.close(); conn.close();
def run(self, dt, to_mysql=False, to_csv=False, type=1): succ = True try: df = self.__query(dt, type) df = self._process(df) if (to_mysql): df = self._get_update_df(df) pm.update(DbCreator.share_data_finance, df, primaryKeys=[Config.id]) if (to_csv): df.to_csv(dt + '-fin.csv') except Exception as e: logger.exception('fin base error') succ = False digest_log.info('TFinBase|%s|%s|%s|%s' % (self.__class__.__name__, dt, to_mysql, succ)) return succ
def pull(self, code: str, start: str, end: str, to_mysql=True, to_csv=False): succ = True df = None try: df = self._run(code, start, end) if (to_mysql): self._save_to_mysql(pm, df) if (to_csv): self._save_to_csv(code, df) except Exception as e: logger.exception('puller error') succ = False digest_log.info( 'puller|%s|%s|%s|%s|%s|%s' % (self.__class__.__name__, code, start, end, str(to_mysql), succ)) return succ
def run(self, code, df, start=0, to_mysql=False, to_csv=False): succ = True try: if (df.empty or df.shape[0] == 0): logger.info('no date to count kpi') else: df = self._process(df, start) if (to_mysql): df = df.loc[start:, :] tb_name = self._get_update_table_name() pm.update(tb_name, df, primaryKeys=[Config.id]) if (to_csv): df.to_csv(code + '-base.csv') except Exception as e: logger.exception('base kpi error') print(e.__traceback__) succ = False digest_log.info('Alg-Base|%s|%s|%d|%s|%s' % (self.__class__.__name__, code, start, to_mysql, succ)) return succ
def process(self, df, start_data): try: return (True, self.__process(df, start_data)) except Exception as e: logger.exception('pre process error') return (False, -1)