def check(cls, code, start_date, end_date): table = 'kline_min5' table2 = 'tmp_kline_min5' path = 'f:/zjf/data/check6/' data = KlineData.read_data(code=code, start_date=start_date, end_date=end_date, kline=table, timemerge=True) if len(data) <= 0: return data = data.sort_values(by=['date'], ascending=True) data = data[data.time <= 955] data2 = KlineData.read_data(code=code, start_date=start_date, end_date=end_date, kline=table2, timemerge=True) if len(data2) <= 0: return data2 = data2.sort_values(by=['date'], ascending=True) data2 = data2[data2.time <= 955] print('start draw.......') # fig, ax = plt.subplots() # fig.subplots_adjust(bottom=0.2) ax = plt.subplot(211) # plt.title('本地') ax.set_xticks([]) candlestick2_ohlc(ax, data.open, data.high, data.low, data.close, width=1, colorup='red', colordown='green') plt.title(table + '--' + code) ax = plt.subplot(212) # plt.title('远程') plt.title(table2 + '--' + code) candlestick2_ohlc(ax, data2.open, data2.high, data2.low, data2.close, width=1, colorup='red', colordown='green') File.check_file(path=path) ax.set_xticks([]) plt.savefig(path + code + '.png', transparent=True) plt.close()
def unreplenishment(cls, data, method='local'): """ 将XDXR_day复权后的价格更新至现价 :param method:获取现价的方法:net表示从网络接口获取最新价作为现价 local表示从本地数据库获取。 目前反复权操作指针对中国A股XDXR_day 表,将其close-->现价 :param data: :return: """ try: if method == 'net': from Calf.data import RealData as rd real_data = rd.get_stocks_data(list(data.stock_code)) data.drop('close', axis=1, inplace=True) data = pd.merge(data, real_data.loc[:, ['stock_code', 'price']], on='stock_code') data.rename(columns={'price': 'open_price'}, inplace=True) if method == 'local': # 注意使用的是open_date from Calf.data import KlineData as kd data['open_price'] = data.apply(lambda r: kd.read_one( r['stock_code'], r['open_date'], 'kline_day')['close'], axis=1) return data except Exception as e: ExceptionInfo(e) return pd.DataFrame()
def get_data(self): data = KlineData.read_data(code='000001', kline='kline_day', start_date=dt.datetime(2015, 1, 1), end_date=dt.datetime(2018, 9, 3)) # print(data.loc[:,['date']]) return data
def get_data(self, stockno, start_date, end_date, kline): data = KlineData.read_data(code=stockno, start_date=start_date, end_date=end_date, kline=kline, timemerge=True) data = data[::-1] return data
def deal_data(code): data = KlineData.read_data(code=code, start_date=dt.datetime(2016, 1, 1), end_date=dt.datetime(2018, 9, 18), kline='index_min5',timemerge=True) if len(data): data = n_KDJ(data, 9, 3) # t1 = time.clock() # print(t1-t) # t = time.clock() data = n_MA(data) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_MACD(data, 12, 26, 9) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_RSI(data, 6, 12, 24) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_DMI(data, 14, 6) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_BRAR(data, 26) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_CR(data, 26, 10, 20, 40) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_VR(data, 26) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_WR(data, 10, 6) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_CCI(data, 14) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_BOLL(data, 20) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_PSY(data, 12) data.dropna(inplace=True) data=data[data.time==955] BaseModel('features_index_min5').insert_batch(data.to_dict(orient='records'))
def get(code, table='kline_tick'): print(code) start_date = dt.datetime(2018, 6, 12) end_date = dt.datetime(2018, 6, 12) data = KlineData.read_data(code=code, start_date=start_date, end_date=end_date, kline=table, timemerge=True) if len(data) <= 0: return {} data = data.sort_values(by=['_id']) data['pre_price'] = data.price.shift(-1) data['change'] = data.pre_price - data.price data['amount'] = data.price * data.volume data0 = data[data.price == data.price.max()] max_price_amount = data0.amount.sum() data1 = data[data.price == data.price.min()] min_price_amount = data1.amount.sum() up = (data[data.change > 0]) down = data[data.change < 0] ping = data[data.change == 0] up_volume = up['amount'].sum() down_volume = down['amount'].sum() ping_volume = ping['amount'].sum() up_max_amount = up['amount'].max() down_max_amount = down['amount'].max() return dict(code=code, date=start_date, up_amount=up_volume, down_amount=down_volume, ping_amount=ping_volume, up_max_amount=up_max_amount, down_max_amount=down_max_amount, max_price_amount=max_price_amount, min_price_amount=min_price_amount)
def get_result(sc, kline, start, end, table_name): data = KlineData.read_data(code=sc, start_date=start, end_date=end, kline=kline, timemerge=True) data = cal_ma(data) data = data.dropna() while len(data) >= 64: last_one = data.iloc[-1] if last_one.time != 1500: break elif abs(last_one.profit_self) >= 0.097: continue else: date = last_one.date data = data[data.date <= dt.datetime(date.year, date.month, date.day, 9, 55)] if len(data) >= 64: profit = data.iloc[-1].profit date = data.iloc[-1].date idata = data.tail(64) data = data[ data.date < dt.datetime(date.year, date.month, date.day)] idata = idata.loc[:, [ 'close', 'open', 'high', 'low', 'ma5', 'ma10', 'ma20', 'ma60', 'ma120', 'ma200' ]] idata = np.array(idata) amin, amax = idata.min(), idata.max() # 求最大最小值 idata = (idata - amin) / (amax - amin) BaseModel(table_name).insert_batch({ 'stock_code': sc, 'profit': profit, 'date': date, 'value': idata.tolist() })
def get_result(sc, kline, start, end, table_name): data = KlineData.read_data(code=sc, start_date=start, end_date=end, kline=kline, timemerge=True) data = cal_macd(data) data = data.dropna() while len(data) >= 64: last_one = data.iloc[-1] if last_one.time != 1500: break elif abs(last_one.profit_self) >= 0.097: continue else: date = last_one.date data = data[data.date <= dt.datetime(date.year, date.month, date.day, 9, 55)] if len(data) >= 64: profit = data.iloc[-1].profit date = data.iloc[-1].date idata = data.tail(64) idata = idata.loc[:, ['macd', 'dif', 'dea']] data = data[ data.date < dt.datetime(date.year, date.month, date.day)] idata = np.array(idata) amin, amax = idata.min(), idata.max() # 求最大最小值 max = -amax if abs(amax) > abs(amin) else amin idata = idata * (64 / 2 - 1) / max idata = idata + 64 / 2 BaseModel(table_name).insert_batch({ 'stock_code': sc, 'profit': profit, 'date': date, 'value': idata.tolist() })
import numpy as np def normal(employment): mean = employment.mean() # 计算平均数 deviation = employment.std() # 计算标准差 # 标准化数据的公式: (数据值 - 平均数) / 标准差 standardized_employment = (employment - mean) / deviation return standardized_employment if __name__ == '__main__': data = KlineData.read_data(code='880302', start_date=dt.datetime(2018, 9, 10), end_date=dt.datetime(2018, 9, 18), kline='index_min5', timemerge=True) name = data.columns.values.tolist() if len(data): data = data.drop(['_id', 'classtype', 'market'], axis=1) data = n_KDJ(data, 9, 3) # t1 = time.clock() # print(t1-t) # t = time.clock() data = n_MA(data) # t1 = time.clock() # print(t1 - t1) # t = time.clock()
def deal_data(code): data = KlineData.read_data(code=code, start_date=dt.datetime(2015, 1, 1), end_date=dt.datetime(2018, 9, 17), kline='kline_day', timemerge=True) name = data.columns.values.tolist() if len(data): data = data.drop(['_id', 'classtype', 'market'], axis=1) data = n_KDJ(data, 9, 3) # t1 = time.clock() # print(t1-t) # t = time.clock() data = n_MA(data) # t1 = time.clock() # print(t1 - t1) # t = time.clock() data = n_MACD(data, 12, 26, 9) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_RSI(data, 6, 12, 24) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_DMI(data, 14, 6) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_BRAR(data, 26) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_CR(data, 26, 10, 20, 40) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_VR(data, 26) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_WR(data, 10, 6) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_CCI(data, 14) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_BOLL(data, 20) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_PSY(data, 12) columns = data.columns.values.tolist() data = data.replace(to_replace=np.Infinity, value=np.NaN) fund_data = pd.DataFrame( list(BaseModel('jq_fund_data').query({'code': code}))) jq_name = [ 'capitalization', 'circulating_cap', 'circulating_market_cap', 'market_cap', 'pb_ratio', 'pcf_ratio', 'pe_ratio', 'pe_ratio_lyr', 'ps_ratio', 'turnover_ratio' ] for i in jq_name: data[i] = 0 # data['circulating_market_cap'] = 0 for i in columns: if i not in name and 'change_r_next' not in i: v = data[i].iloc[0] if type(v).__name__ != 'bool_': # print(i, v, type(v)) if data[i].min() == 0: # pass # print(i, data[i].loc[0]) data.loc[data[i] != 0, i] = (data.loc[data[i] != 0, i] - data.loc[data[i] != 0, i].min()) / ( data.loc[data[i] != 0, i].max() - data.loc[data[i] != 0, i].min()) # print(i,data[i].loc[0]) else: # print(i, v, type(v)) data[i] = (data[i] - data[i].min()) / (data[i].max() - data[i].min()) # data['up'] = data['up'] / (data['up'] + data['down']) # data['down'] = data['down'] / (data['up'] + data['down']) data.loc[ (data.date >= dt.datetime(2015, 12, 31)) & (data.date < dt.datetime(2016, 3, 31)), jq_name] = \ fund_data[fund_data.day == '2015-12-31'].loc[:,jq_name].iloc[0].tolist() data.loc[ (data.date >= dt.datetime(2016, 3, 31)) & (data.date <= dt.datetime(2016, 6, 30)), jq_name] = \ fund_data[fund_data.day == '2016-03-31'].loc[:,jq_name].iloc[0].tolist() data.loc[ (data.date < dt.datetime(2016, 9, 30)) & (data.date >= dt.datetime(2016, 6, 30)), jq_name] = \ fund_data[fund_data.day == '2016-06-30'].loc[:,jq_name].iloc[0].tolist() data.loc[ (data.date >= dt.datetime(2016, 9, 30)) & (data.date < dt.datetime(2016, 12, 30)), jq_name] = \ fund_data[fund_data.day == '2016-09-30'].loc[:,jq_name].iloc[0].tolist() data.loc[ (data.date < dt.datetime(2017, 3, 31)) & (data.date >= dt.datetime(2016, 12, 30)), jq_name] = \ fund_data[fund_data.day == '2016-12-30'].loc[:,jq_name].iloc[0].tolist() data.loc[ (data.date >= dt.datetime(2017, 3, 31)) & (data.date < dt.datetime(2017, 6, 30)), jq_name] = \ fund_data[fund_data.day == '2017-03-31'].loc[:,jq_name].iloc[0].tolist() data.loc[ (data.date >= dt.datetime(2017, 6, 30)) & (data.date < dt.datetime(2017, 9, 29)), jq_name] = \ fund_data[fund_data.day == '2017-06-30'].loc[:,jq_name].iloc[0].tolist() data.loc[ (data.date >= dt.datetime(2017, 9, 29)) & (data.date < dt.datetime(2017, 12, 29)), jq_name] = \ fund_data[fund_data.day == '2017-09-29'].loc[:,jq_name].iloc[0].tolist() data.loc[ (data.date >= dt.datetime(2017, 12, 29)) & (data.date < dt.datetime(2018, 3, 30)), jq_name] = \ fund_data[fund_data.day == '2017-12-29'].loc[:,jq_name].iloc[0].tolist() data.loc[ (data.date >= dt.datetime(2018, 3, 30)) & (data.date < dt.datetime(2018, 6, 29)), jq_name] = \ fund_data[fund_data.day == '2018-03-30'].loc[:,jq_name].iloc[0].tolist() data.loc[ (data.date >= dt.datetime(2018, 6, 29)), jq_name] = \ fund_data[fund_data.day == '2018-06-29'].loc[:,jq_name].iloc[0].tolist() name = ['open', 'close', 'high', 'low', 'amount', 'volume'] data['turnover_ratio'] = data.volume / (data.circulating_cap * 10000) for i in name: data[i] = (data[i] - data[i].min()) / (data[i].max() - data[i].min()) data = data[::-1] data = data.reset_index(drop=True) # data['count'] = data.index.tolist() curror = BaseModel('LaunchDate').query(sql={'stock_code': code}) data['count'] = 0 if curror.count(): launchdate = list(curror)[0]['date'] data['count'] = data.date.map(lambda x: (x - launchdate).days) data = data[data.date >= dt.datetime(2016, 1, 1)] BaseModel('features_kline_day').insert_batch( data.to_dict(orient='records'))
from Calf.data import KlineData import datetime as dt from sklearn.cross_validation import train_test_split import matplotlib.pyplot as plt from Calf.models.base_model import BaseModel import pandas as pd import numpy as np data = KlineData.read_data(code='000001', start_date=dt.datetime(2016, 1, 1), end_date=dt.datetime(2018, 9, 2), kline='index_min5', timemerge=True) data = data.loc[:, ['date', 'close', 'open', 'high', 'low', 'volume']] data['close2'] = data.close.shift(-48) data['profit'] = data.close / data.close2 - 1 data = data.drop(['close2'], axis=1) data = data.dropna() y = data.loc[:, ['profit']] X = data.loc[:, ['close', 'open', 'low', 'high', 'volume']] X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1) print(X_train.shape) print(y_test.shape) from sklearn.linear_model import LinearRegression linreg = LinearRegression() linreg.fit(X_train, y_train) a = (linreg.intercept_) print(a) print((linreg.coef_)) # 模型拟合测试集 y_pred = linreg.predict(X_test) from sklearn import metrics
from Calf.data import KlineData import datetime as dt import matplotlib.pyplot as plt from Visual.DMI.caculate import Caculate data=KlineData().read_data(code='001979',start_date=dt.datetime(2018,5,1),end_date=dt.datetime(2018,9,1),kline='kline_day') data=data.sort_values(by=['date'],ascending=True) data=Caculate.n_DMI(data) data=data[::-1] print(data.date.tolist()) fig=plt.figure(figsize=(9,6)) plt.plot(data.date,data['PDI']) plt.plot(data.date,data['MDI']) plt.plot(data.date,data['ADX']) plt.show()
from Calf.data import KlineData import datetime as dt def deal(data): columns = data.columns.values.tolist() for i in columns: if i not in ['_id', 'classtype']: v = data[i].iloc[0] if type(v).__name__ not in ['bool_', 'str']: data[i] = (data[i] - data[i].min()) / (data[i].max() - data[i].min()) return data data = KlineData.read_data(code='000001', start_date=dt.datetime(2018, 9, 1), end_date=dt.datetime(2018, 10, 10), kline='kline_day') data = deal(data) print(data.head(10))
def deal_data(code): data = KlineData.read_data(code=code, start_date=dt.datetime(2015, 1, 1), end_date=dt.datetime(2018, 9, 17), kline='index_day', timemerge=True) name = data.columns.values.tolist() if len(data): data = data.drop(['_id', 'classtype', 'market'], axis=1) data = n_KDJ(data, 9, 3) # t1 = time.clock() # print(t1-t) # t = time.clock() data = n_MA(data) # t1 = time.clock() # print(t1 - t1) # t = time.clock() data = n_MACD(data, 12, 26, 9) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_RSI(data, 6, 12, 24) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_DMI(data, 14, 6) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_BRAR(data, 26) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_CR(data, 26, 10, 20, 40) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_VR(data, 26) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_WR(data, 10, 6) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_CCI(data, 14) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_BOLL(data, 20) # t1 = time.clock() # print(t1 - t) # t = time.clock() data = n_PSY(data, 12) columns = data.columns.values.tolist() data = data.replace(to_replace=np.Infinity, value=np.NaN) for i in columns: if i not in name and 'change_r_next' not in i: v = data[i].iloc[0] if type(v).__name__ != 'bool_': # print(i, v, type(v)) if data[i].min() == 0: # pass # print(i, data[i].loc[0]) data.loc[data[i] != 0, i] = (data.loc[data[i] != 0, i] - data.loc[data[i] != 0, i].min()) / ( data.loc[data[i] != 0, i].max() - data.loc[data[i] != 0, i].min()) # print(i,data[i].loc[0]) else: # print(i, v, type(v)) data[i] = (data[i] - data[i].min()) / (data[i].max() - data[i].min()) data['up'] = data['up'] / (data['up'] + data['down']) data['down'] = data['down'] / (data['up'] + data['down']) name = ['open', 'close', 'high', 'low', 'amount', 'volume'] for i in name: data[i] = (data[i] - data[i].min()) / (data[i].max() - data[i].min()) data = data[::-1] data = data.reset_index(drop=True) data['count'] = data.index.tolist() data = data[data.date >= dt.datetime(2016, 1, 1)] BaseModel('features_index_day').insert_batch(data.to_dict(orient='records'))
def KScheduler(cls, action, tz=None, deep_sleep=list()): """ 适用于A股的信号采集实时任务,适用于跨时区任务 :param tz: :param deep_sleep: :param action: 模型的运行管理对象 :return: """ if not isinstance(action, type(ModelAction)): raise TypeError('Object action must be a subclass of ModelAction') if tz is not None: if tz not in pytz.all_timezones: raise ValueError('this tz: %s not in pytz time zones' % tz) else: tz = pytz.timezone(tz) profit_probe_period = dt.timedelta(minutes=1) profit_probe_next = dt.datetime.now( ) if tz is None else dt.datetime.now(tz=tz).replace(tzinfo=None) _f = False # 记录当天是否收盘 klines = action.klines # 需要跟踪的bar周期 _id = cls.open_kline_update_log() # 记录bar更新 from Calf.data import KlineData as kd while 1: try: crt = dt.datetime.now() if tz is None else dt.datetime.now( tz=tz).replace(tzinfo=None) if not action.is_trade_day( dt.datetime(crt.year, crt.month, crt.day)): print('{0} today is not in business'.format(crt)) time.sleep(60 * 60 * 2) # sleep two hours continue if action.trade_day_end(crt) and _f: print(fontcolor.F_GREEN + '-' * 80) print('# %s Today Overview #' % action.name) print('Signal:') action.signals_summary() print('Order:') action.orders_summary() print(fontcolor.F_GREEN + '-' * 80 + fontcolor.END) sound_notice('close.wav').start() _f = False # open_am <= crt <= close_am or open_pm <= crt < close_pm if action.trade_date(crt): # 交易日盘中 s_d = crt - dt.timedelta(minutes=3) e_d = crt + dt.timedelta(minutes=3) log = kd.read_log(start_date=s_d, end_date=e_d, status=200) if len(log): log['_id'] = log['_id'].astype('str') for i, r in log.iterrows(): if r.kline in klines and r['_id'] != _id[r.kline]: print('this kline %s find update' % r.kline) action.real(r.kline) print(r.kline + ' id update:' + _id[r.kline] + '-->' + r['_id']) _id[r.kline] = r['_id'] cls.set_kline_update_log(_id) if profit_probe_next <= crt: print('profit probing date:{0}'.format(crt)) action.probing() profit_probe_next += profit_probe_period _f = True time.sleep(5) else: print('{0} this datetime is not in business'.format(crt)) profit_probe_next = crt _f = False sleep = 60 * 5 if crt.hour in deep_sleep else 60 * 30 time.sleep(sleep) except Exception as e: ExceptionInfo(e)
def rerun(cls, action, deep_sleep, offset=None): """ 复盘演示 :param action: :param deep_sleep: :param offset: :return: """ from Calf.data import KlineData as kd profit_probe_period = dt.timedelta(minutes=1) profit_probe_next = CalfDateTime.now(offset=offset) _f = False # 记录当天是否收盘 klines = action.klines # 需要跟踪的bar周期 _id = cls.open_kline_update_log() # 记录bar更新 while 1: try: crt = CalfDateTime.now(offset=offset) if not trading.is_trade_day( dt.datetime(crt.year, crt.month, crt.day)): print('{0} today is not in business'.format(crt)) time.sleep(60 * 60 * 2) # sleep two hours continue if action.trade_day_end(crt) and _f: print(fontcolor.F_GREEN + '-' * 80) print('# %s Today Overview #' % action.name) print('Signal:') action.signals_summary() print('Order:') action.orders_summary() print(fontcolor.F_GREEN + '-' * 80 + fontcolor.END) sound_notice('close.wav').start() _f = False # open_am <= crt <= close_am or open_pm <= crt < close_pm if action.trade_date(crt): # 交易日盘中 s_d = crt - dt.timedelta(minutes=3) e_d = crt + dt.timedelta(minutes=3) log = kd.read_log(start_date=s_d, end_date=e_d, status=200) if len(log): log['_id'] = log['_id'].astype('str') for i, r in log.iterrows(): if r.kline in klines and r['_id'] != _id[r.kline]: print('this kline %s find update' % r.kline) action.real(r.kline, start_time=crt) print(r.kline + ' id update:' + _id[r.kline] + '-->' + r['_id']) _id[r.kline] = r['_id'] cls.set_kline_update_log(_id) if profit_probe_next <= crt: print('profit probing date:{0}'.format(crt)) # action.probing() profit_probe_next += profit_probe_period _f = True time.sleep(5) else: print('{0} this datetime is not in business'.format(crt)) profit_probe_next = crt _f = False # sleep = 60 * 30 # if crt.hour == 9 or crt.hour == 12: # sleep = 60 * 5 sleep = 60 * 5 if crt.hour in deep_sleep else 60 * 30 time.sleep(sleep) except Exception as e: ExceptionInfo(e)