def backTest(trainEndDate, code, testDate, predictDate): conn = db.get_history_data_db('D') df = None # train more date # model = pickle.load(open('%s/%s.pkl' % (config.model_dir, code), 'r')) rng = np.random.RandomState(1) model = AdaBoostRegressor(DecisionTreeRegressor(max_depth=4), n_estimators=1000, random_state=rng, loss='square') df = pd.read_sql_query( "select * from history_data where date([date])<='%s' and code='%s' order by code, date([date]) asc" % (trainEndDate, code), conn) shift_1 = df['close'].shift(-2) df['target'] = shift_1 data = df[df['target'] > -1000] X_train = data.ix[:, 'code':'turnover'] y_train = data.ix[:, 'target'] if len(X_train) < 500: return print len(X_train) # print data # for i in range(0, 10): # model.fit(X_train, y_train) model.fit(X_train, y_train) # predict tomorrow try: df = pd.read_sql_query( config.sql_history_data_by_code_date % (code, testDate), conn) # print df except Exception, e: print e
def backTest(trainEndDate, code, testDate, predictDate): conn = db.get_history_data_db('D') df = None # train more date # model = pickle.load(open('%s/%s.pkl' % (config.model_dir, code), 'r')) rng = np.random.RandomState(1) model = AdaBoostRegressor(DecisionTreeRegressor( max_depth=4), n_estimators=1000, random_state=rng, loss='square') df = pd.read_sql_query( "select * from history_data where date([date])<='%s' and code='%s' order by code, date([date]) asc" % ( trainEndDate, code), conn) shift_1 = df['close'].shift(-2) df['target'] = shift_1 data = df[df['target'] > -1000] X_train = data.ix[:, 'code':'turnover'] y_train = data.ix[:, 'target'] if len(X_train) < 500: return print len(X_train) # print data # for i in range(0, 10): # model.fit(X_train, y_train) model.fit(X_train, y_train) # predict tomorrow try: df = pd.read_sql_query(config.sql_history_data_by_code_date % (code, testDate), conn) # print df except Exception, e: print e
def get_all_stock_code(): ''' 返回全部股票代码 ''' conn = db.get_history_data_db() conn.text_factory = str try: df = pd.read_sql_query( 'select * from stock_basics where [timeToMarket] !=0', conn) return df['code'] except Exception as e: log.error(e)
def get_all_stock_code(): ''' 返回全部股票代码 ''' conn = db.get_history_data_db() conn.text_factory = str try: df = pd.read_sql_query( 'select * from stock_basics where [timeToMarket] !=0', conn) return df['code'] except Exception as e: log.error(e)
def download_history_data_fq(autype='qfq', startTime=None): ''' 获取前复权的历史k线数据 ''' conn = db.get_history_data_db('D') start = startTime if startTime is None: start = utils.today_last_year(6) for code in get_all_stock_code(): df = ts.get_h_data(code, start=start, drop_factor=False) if df is not None: try: df.insert(0, 'code', code) sql_df = df.loc[:, :] sql.to_sql(sql_df, name='history_data_%s' % autype, con=conn, index=True, if_exists='append') log.info('%s,%s history qfq data download ok.' % (code, start)) except Exception as e: log.error('error:code:%s,start:%s' % (code, start))
def updata_all_stock_basics(): ''' 更新所有股票的基本数据 return DataFrame code,代码 name,名称 industry,细分行业 area,地区 pe,市盈率 outstanding,流通股本 totals,总股本(万) totalAssets,总资产(万) liquidAssets,流动资产 fixedAssets,固定资产 reserved,公积金 reservedPerShare,每股公积金 eps,每股收益 bvps,每股净资 pb,市净率 timeToMarket,上市日期 ''' conn = db.get_history_data_db() conn.text_factory = str retry = 3 for i in range(retry): try: df = ts.get_stock_basics() if not df.empty: sql_df = df.loc[:, :] sql.to_sql(sql_df, name='stock_basics', con=conn, index=True, if_exists='replace') log.info('all stock basics updated, total size=%d' % len(df)) break except Exception as e: log.error(e) conn.close()
def updata_all_stock_basics(): ''' 更新所有股票的基本数据 return DataFrame code,代码 name,名称 industry,细分行业 area,地区 pe,市盈率 outstanding,流通股本 totals,总股本(万) totalAssets,总资产(万) liquidAssets,流动资产 fixedAssets,固定资产 reserved,公积金 reservedPerShare,每股公积金 eps,每股收益 bvps,每股净资 pb,市净率 timeToMarket,上市日期 ''' conn = db.get_history_data_db() conn.text_factory = str retry = 3 for i in range(retry): try: df = ts.get_stock_basics() if not df.empty: sql_df = df.loc[:, :] sql.to_sql(sql_df, name='stock_basics', con=conn, index=True, if_exists='replace') log.info('all stock basics updated, total size=%d' % len(df)) break except Exception as e: log.error(e) conn.close()
def download_history_data_fq(autype='qfq', startTime=None): ''' 获取前复权的历史k线数据 ''' conn = db.get_history_data_db('D') start = startTime if startTime is None: start = utils.today_last_year(6) for code in get_all_stock_code(): df = ts.get_h_data(code, start=start, drop_factor=False) if df is not None: try: df.insert(0, 'code', code) sql_df = df.loc[:, :] sql.to_sql(sql_df, name='history_data_%s' % autype, con=conn, index=True, if_exists='append') log.info('%s,%s history qfq data download ok.' % (code, start)) except Exception as e: log.error('error:code:%s,start:%s' % (code, start))
def __init__(self): self.log = utils.mylog self.history_data_db_conn = db.get_history_data_db() self.history_data_db_conn.text_factory = str
def download_index_history_data(start=None, end=None, init_run=False): ''' start:开始时间 yyyyMMdd,第一次调用空则取20100101,之后以数据表中最近时间为准 end:结束时间 yyyyMMdd,空则取当前日期 ''' cur_time = datetime.now() conn = db.get_history_data_db() if init_run: start = default_start_time if start is None: try: onerow = conn.execute(config.sql_last_date_index_all).fetchone() if onerow is not None: start = onerow[0] dt = datetime.strptime(start, '%Y-%m-%d') + timedelta(days=1) start = datetime.strftime(dt, '%Y%m%d') else: start = default_start_time except Exception as e: start = default_start_time if end is None: end = datetime.today().date().strftime('%Y%m%d') print(start, end) if int(end) <= int(start): return None for code in indices.keys(): for i in range(retry): try: url = sohu_history_api % (code, start, end) text = request.urlopen(url, timeout=10).read() text = text.decode('GBK') log.info('url=%s,size=%d, try=%d' % (url, len(text), i)) if len(text) < 20: continue j = demjson.decode(text, 'utf-8') head = [ 'date', 'open', 'close', 'chg', 'chg_r', 'low', 'high', 'vibration', 'volume', 'amount' ] # 日期 开盘 收盘 涨跌额 涨跌幅 最低 最高 成交量(手) 成交金额(万) # 日期 开盘 收盘 涨跌额 涨跌幅 最低 最高 成交量(手) 成交金额(万) 换手率 data = [] for x in j[0].get('hq'): date, open, close, change, _, low, high, valume, amount, _ = x chg_r = '%.4f' % ( (float(close) - float(open)) / float(open)) vibration = '%.4f' % (float( (float(high) - float(low)) / float(open))) # print date, vibration, str(float(vibration)) data.append([ date, float(open), float(close), float(change), float(chg_r), float(low), float(high), float(vibration), float(valume), float(amount) ]) # sql_str = 'insert OR IGNORE into all_index values(?,?,?,?,?,?,?,?,?,?,?)' # print len(data[0]) # conn.executemany(sql_str, data) df = pd.DataFrame(data, columns=head) if not df.empty: df.insert(1, 'code', code) sql_df = df.loc[:, :] sql.to_sql(sql_df, name='all_index', con=conn, index=False, if_exists='append') log.info('%s,%s index history download ok.' % (code, start)) break except Exception as e: log.error(e) conn.close() log.info('index history data download complete. cost %d s' % (datetime.now() - cur_time).seconds)
def download_history_data(ktype='D', start=None, end=None, init_run=False): ''' 获取近不复权的历史k线数据 ''' if init_run: start = default_start_time if end == None: end = datetime.today().date().strftime('%Y%m%d') conn = db.get_history_data_db(ktype) cost = 0 cur_time = datetime.now() for code in get_all_stock_code(): cost = datetime.now() if start is None: _start = get_laste_update_dt(code) if _start is not None: dt = datetime.strptime(_start, '%Y-%m-%d') + timedelta(days=1) start = datetime.strftime(dt, '%Y%m%d') else: row = conn.execute(config.sql_last_date_history_data_by_code % code).fetchone() if row is not None: start = row[0] dt = datetime.strptime(start, '%Y-%m-%d') + timedelta(days=1) start = datetime.strftime(dt, '%Y%m%d') else: start = default_start_time for i in range(retry): try: url = sohu_history_api % ('cn_' + code, start, end) text = request.urlopen(url, timeout=10).read() text = text.decode('GBK') log.info('url=%s,size=%d, try=%d' % (url, len(text), i)) if len(text) < 20: continue j = demjson.decode(text, 'utf-8') head = [ 'date', 'open', 'close', 'chg', 'chg_r', 'low', 'high', 'vibration', 'volume', 'amount', 'turnover' ] # 日期 开盘 收盘 涨跌额 涨跌幅 最低 最高 成交量(手) 成交金额(万) # 日期 开盘 收盘 涨跌额 涨跌幅 最低 最高 成交量(手) 成交金额(万) 换手率 data = [] for x in j[0].get('hq'): date, open, close, change, _, low, high, valume, amount, turnover = x if '-' == turnover: turnover = '0.0%' chg_r = '%.4f' % ( (float(close) - float(open)) / float(open)) vibration = '%.4f' % float( (float(high) - float(low)) / float(open)) chg_r = float(chg_r) vibration = float(vibration) data.append([ date, float(open), float(close), float(change), float(chg_r), float(low), float(high), float(vibration), float(valume), float(amount), float(turnover[:-1]) ]) df = pd.DataFrame(data, columns=head) if not df.empty: df.insert(1, 'code', code) sql_df = df.loc[:, :] sql.to_sql(sql_df, name='history_data', con=conn, index=False, if_exists='append') log.info('%s,%s,%d history data download ok.' % (code, str(start), len(sql_df))) slog.info('%s,%s' % (code, data[0][0])) break except Exception as e: log.error('error:code=%s,start=%s,msg=%s' % (code, start, e)) if str(e).find('UNIQUE constraint') > -1: break log.debug('%s,costs:%d s' % (code, (datetime.now() - cost).seconds)) conn.close() log.info('history data download complete. cost %d s' % (datetime.now() - cur_time).seconds)
import talib import numpy as np from talib.abstract import * close = np.random.random(100) output = talib.SMA(close, timeperiod=5) # print output from org.tradesafe.db import sqlite_db as db import pandas as pd import numpy as np # note that all ndarrays must be the same length! inputs = { 'high': np.random.random(100), 'open': np.random.random(100), 'low': np.random.random(100), # 'close': np.random.random(100), 'volume': np.random.random(100) } output = SMA(inputs, timeperiod=25, price='open') # print CCI(inputs, price='high') # print output conn = db.get_history_data_db('D') df = pd.read_sql_query( "select * from history_data where code='%s' order by date([date]) asc" % '600022', conn) df['sma15'] = talib.SMA(df['high'].values, timeperiod=15) print df.head(30) if __name__ == '__main__': pass
def __init__(self): self.log = utils.mylog self.history_data_db_conn = db.get_history_data_db() self.history_data_db_conn.text_factory = str
def download_index_history_data(start=None, end=None, init_run=False): ''' start:开始时间 yyyyMMdd,第一次调用空则取20100101,之后以数据表中最近时间为准 end:结束时间 yyyyMMdd,空则取当前日期 ''' cur_time = datetime.now() conn = db.get_history_data_db() if init_run: start = default_start_time if start is None: try: onerow = conn.execute(config.sql_last_date_index_all).fetchone() if onerow is not None: start = onerow[0] dt = datetime.strptime(start, '%Y-%m-%d') + timedelta(days=1) start = datetime.strftime(dt, '%Y%m%d') else: start = default_start_time except Exception as e: start = default_start_time if end is None: end = datetime.today().date().strftime('%Y%m%d') print(start, end) if int(end) <= int(start): return None for code in indices.keys(): for i in range(retry): try: url = sohu_history_api % (code, start, end) text = request.urlopen(url, timeout=10).read() text = text.decode('GBK') log.info('url=%s,size=%d, try=%d' % (url, len(text), i)) if len(text) < 20: continue j = demjson.decode(text, 'utf-8') head = ['date', 'open', 'close', 'chg', 'chg_r', 'low', 'high', 'vibration', 'volume', 'amount'] # 日期 开盘 收盘 涨跌额 涨跌幅 最低 最高 成交量(手) 成交金额(万) # 日期 开盘 收盘 涨跌额 涨跌幅 最低 最高 成交量(手) 成交金额(万) 换手率 data = [] for x in j[0].get('hq'): date, open, close, change, _, low, high, valume, amount, _ = x chg_r = '%.4f' % ((float(close) - float(open)) / float(open)) vibration = '%.4f' % (float((float(high) - float(low)) / float(open))) # print date, vibration, str(float(vibration)) data.append([date, float(open), float(close), float(change), float(chg_r), float(low), float(high), float(vibration), float(valume), float(amount)]) # sql_str = 'insert OR IGNORE into all_index values(?,?,?,?,?,?,?,?,?,?,?)' # print len(data[0]) # conn.executemany(sql_str, data) df = pd.DataFrame(data, columns=head) if not df.empty: df.insert(1, 'code', code) sql_df = df.loc[:, :] sql.to_sql(sql_df, name='all_index', con=conn, index=False, if_exists='append') log.info('%s,%s index history download ok.' % (code, start)) break except Exception as e: log.error(e) conn.close() log.info('index history data download complete. cost %d s' % (datetime.now() - cur_time).seconds)
def download_history_data(ktype='D', start=None, end=None, init_run=False): ''' 获取近不复权的历史k线数据 ''' if init_run: start = default_start_time if end == None: end = datetime.today().date().strftime('%Y%m%d') conn = db.get_history_data_db(ktype) cost = 0 cur_time = datetime.now() for code in get_all_stock_code(): cost = datetime.now() if start is None: _start = get_laste_update_dt(code) if _start is not None: dt = datetime.strptime(_start, '%Y-%m-%d') + timedelta(days=1) start = datetime.strftime(dt, '%Y%m%d') else: row = conn.execute(config.sql_last_date_history_data_by_code % code).fetchone() if row is not None: start = row[0] dt = datetime.strptime(start, '%Y-%m-%d') + timedelta(days=1) start = datetime.strftime(dt, '%Y%m%d') else: start = default_start_time for i in range(retry): try: url = sohu_history_api % ('cn_' + code, start, end) text = request.urlopen(url, timeout=10).read() text = text.decode('GBK') log.info('url=%s,size=%d, try=%d' % (url, len(text), i)) if len(text) < 20: continue j = demjson.decode(text, 'utf-8') head = ['date', 'open', 'close', 'chg', 'chg_r', 'low', 'high', 'vibration', 'volume', 'amount', 'turnover'] # 日期 开盘 收盘 涨跌额 涨跌幅 最低 最高 成交量(手) 成交金额(万) # 日期 开盘 收盘 涨跌额 涨跌幅 最低 最高 成交量(手) 成交金额(万) 换手率 data = [] for x in j[0].get('hq'): date, open, close, change, _, low, high, valume, amount, turnover = x if '-' == turnover: turnover = '0.0%' chg_r = '%.4f' % ((float(close) - float(open)) / float(open)) vibration = '%.4f' % float((float(high) - float(low)) / float(open)) chg_r = float(chg_r) vibration = float(vibration) data.append( [date, float(open), float(close), float(change), float(chg_r), float(low), float(high), float(vibration), float(valume), float(amount), float(turnover[:-1])]) df = pd.DataFrame(data, columns=head) if not df.empty: df.insert(1, 'code', code) sql_df = df.loc[:, :] sql.to_sql(sql_df, name='history_data', con=conn, index=False, if_exists='append') log.info('%s,%s,%d history data download ok.' % (code, str(start), len(sql_df))) slog.info('%s,%s' % (code, data[0][0])) break except Exception as e: log.error('error:code=%s,start=%s,msg=%s' % (code, start, e)) if str(e).find('UNIQUE constraint') > -1: break log.debug('%s,costs:%d s' % (code, (datetime.now() - cost).seconds)) conn.close() log.info('history data download complete. cost %d s' % (datetime.now() - cur_time).seconds)