def get_volumes(strategy): obj = model_list['kline_' + 'min5'] t = 0 if (strategy[-2:] == '30'): t = 1425 c = 6 if (strategy[-2:] == '60'): t = 1355 c = 12 print(strategy) dt = datetime.datetime.now() str1 = str(dt.year * 10000 + dt.month * 100 + dt.day) # data testing please update print(str1) sql = analyzer("time >= {} and time < {} and date = {}".format( t, 1455, int(str1))) print(sql) a = obj.query(sql) a = list(a) df = pd.DataFrame(a) print(list) if (len(df) == 0): print('df is None') df = df.sort_values(['stock_code', 'time'], ascending=True) df.reset_index(inplace=True) grouped = df['amount'].groupby(df['stock_code']) data = grouped.sum() # for i,r in data.iterrows(): # dic[r.stock_code]=r.volume return (dict(data))
def get_data(kline): t = 1425 if kline == 'min30' else 1355 today = dt.datetime.strftime(dt.date.today(), '%Y%m%d') tt = {'$gte': t, '$lte': 1450} sql = analyzer("date = {} and time <= {} and time >= {}".format( today, 1450, t)) print(sql) a = model_list['kline_min5'].query(sql) a = list(a) data = pd.DataFrame(a) # data = md.get_stock_data(kline='min5', start_date=today, end_date=today, axis=0, timemerge=False, time=t) kline_data = data[data.time == 1450] spl_data = data.amount.groupby(data.stock_code).sum() spl_data = spl_data.to_frame() spl_data['stock_code'] = spl_data.index data = pd.merge(kline_data, spl_data, on='stock_code') data['amount'] = data.amount_x + data.amount_y data['time'] = 1500 data.drop(['_id', 'classtype', 'amount_x', 'amount_y'], axis=1, inplace=True) dicts = [] for i, r in data.iterrows(): dicts.append(dict(r)) x = model_list['kline_' + kline].insert_batch(dicts) print(x) return data
def check_kline(cls, strategy='kline_min5'): dt = Calendar.today() # pre_date = Calendar.calc(dt, -10) dt = Calendar.calc(dt, -1)['date'] # sql = analyzer('date = {}'.format(str(dt.year * 10000 + dt.month * 100 + dt.day))) # model_list['check_data'].remove(sql) # print(pre_date) sql = analyzer("date >= {} and date <= {}".format( str(20180111), str(dt.year * 10000 + dt.month * 100 + dt.day))) print("____________________" + strategy) print(sql) print('稍等几秒') cursor = model_list[strategy].query(sql) print(cursor) all_count = cursor.count() print(all_count) day_count = cls.number_of_branches(strategy) print(day_count) dic = { 'date': dt, 'strategy': strategy, 'all_count': all_count, 'day_count': day_count, 'result': all_count % day_count == 0 } print('insert') model_list['check_data'].insert(dic) if all_count % day_count == 0: return strategy + ' is true' print("____________________") else: return strategy + "有问题"
def get_his(stratrgy): sql = get_sql(stratrgy) obj = model_list['kline_' + stratrgy] print(stratrgy) print('^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^') print(sql) dt = datetime.datetime.now() str1 = str(dt.year * 10000 + dt.month * 100 + dt.day) sql = dict(sql, **analyzer("date = {}".format(str1))) a = obj.query(sql) a = list(a) df = pd.DataFrame(a) df.reset_index(inplace=True) return df
def tushare_to_db(start_date=None, end_date=None): sd, ed = date_preprocess(start_date, end_date) obj = model_list['tushare'] dr = list() if start_date: season = (sd.month - 1) // 3 s1 = '{}{:02d}01'.format(sd.year, season * 3 if season > 0 else 1) cond = analyzer('date >= {}'.format(s1)) obj.remove(cond) dr = trans_tushare_data(sd.year, season + 1) else: for y in range(sd.year, ed.year + 1): for s in range(1, 5): dr.extend(trans_tushare_data(y, s)) obj.insert_batch(dr)
def date_range(start_date=None, end_date=None): if start_date: ssql = 'date >= {}'.format(Calendar.to(str, start_date)) else: ssql = 'date >= 20120130' if end_date: esql = 'date <= {}'.format(Calendar.to(str, end_date)) else: esql = 'date <= {}'.format(Calendar.to(str, Calendar.recent())) sql = ' and '.join([ssql, esql]) cond = analyzer(sql) calendar = list(Calendar().query(cond)) return calendar
def self_check(cls, date=Calendar.today()): date = Calendar.calc(date=date, offset=-1)['date'] sql = analyzer('date = {}'.format(date.year * 10000 + date.month * 100 + date.day)) print(sql) result = list(model_list['check_data'].query(sql)) data = pd.DataFrame(result) all_count = list(data['all_count']) for i in range(0, len(all_count) - 1): if i != 4: if all_count[i] % all_count[i + 1] != 0: print(all_count[i], all_count[i + 1]) return False print(data) return True
def get_data(): dt = datetime.now() # if dt.hour > 15: # dt = datetime(2018, 6, 12, 14, 10) now_time = dt.hour * 100 + dt.minute if (now_time >=1030 and now_time<1100)or (now_time>=1130 and now_time<1300) or (now_time>=1400 and now_time<1430) : start_time=0 if dt.hour >=11 and dt.hour<13: now_time=1300 for i in range(0,len(min60_time_list())): if now_time>=min60_time_list()[i]: now_time=min60_time_list()[i] start_time=min60_time_list()[i+1] print(now_time) print(start_time) break sql = analyzer('date = {} and time <= {} and time >= {}'.format(dt.year * 10000 + dt.month * 100 + dt.day, now_time, start_time)) print(sql) data = model_list['kline_min30'].query(sql) data = pd.DataFrame(list(data)) data.sort_values(by=['time']) kline_data = data[data.time == now_time] kline_data_his = data[data.time == start_time] grouped = data.groupby(['stock_code']).agg({'amount': 'sum','volume':'sum','high':'max','low':'min'}) grouped['stock_code'] = grouped.index data = pd.merge(kline_data, grouped, on='stock_code') data['time'] = now_time data['amount'] = data.amount_y data['volume'] = data.volume_y data['high'] = data.high_y data['low'] = data.low_y # data.open=kline_data_his.open data.drop(['_id', 'classtype', 'amount_x', 'amount_y','volume_x','volume_y','high_y','high_x','low_x','low_y'], axis=1, inplace=True) dicts = [] for i,r in data.iterrows(): dicts.append(dict(r)) x=model_list['kline_min60'].insert_batch(dicts) print(dicts[0:10]) return data else: exit()
def data_is_update(strategy, data_type): today = time.strftime('%Y%m%d', time.localtime(time.time())) cql = analyzer( "date = {s}".format(s=today)) strategy=data_type+"_"+strategy cursor=model_list[strategy].query(cql) stock_data = list(cursor) if len(stock_data) > 0: data = pd.DataFrame(stock_data) a=(data.time.max()) print (a) bf=a/100 af=a-bf*100 now_hour=datetime.now().hour now_minute=datetime.now().minute database_data_time=bf*60+af now_time=now_hour*60+now_minute if ((now_time - database_data_time) < change_status(strategy)['time']): return {'time': a, 'isupdate': True} return {'time': a, 'isupdate': False}
def get_history_tick(market_name, stock_code, start_date=None, end_date=None, his=False): sd, ed = date_preprocess(start_date, end_date) days = list(Calendar().query( analyzer('date >= {} and date <= {}'.format( Calendar.to(str, Calendar.recent(sd)), Calendar.to(str, Calendar.recent(ed)))))) tick_data_list = list() market = markets[market_name] for day in days: d = day['date'] loop = True start = 0 while loop: if his: err, count, result = TradeX.TdxHq_Connect( '61.152.249.56', 7709).GetHistoryTransactionData(market['id'], stock_code, start, 2000, Calendar.to(int, d)) else: err, count, result = TradeX.TdxHq_Connect( '61.152.249.56', 7709).GetTransactionData(market['id'], stock_code, start, 2000) # print(result.decode('gbk')) row = result.split('\n') tick_data_list.extend( Tick.trans_data(stock_code, market_name, d, row)) start += 2000 loop = count == 2000 # obj.insert_batch(tick_data_list) print('{} done: {}'.format(stock_code, len(tick_data_list))) return tick_data_list
def day_feature(stock_code, start_date, end_date, his=False): iobj = model_list['feature_day'] offset = 0 if end_date is None: end_date = Calendar.recent() if Calendar.in_business(datetime.now(), hard=False): end_date = Calendar.calc(end_date, -1) else: end_date = Calendar.recent(end_date) # 如果 start_date 为空,则定到 end_date 之前的60天 # 这样刚好可以算出 end_date 那一天的 ma60 # 实现了 daily update 的功能 if his: start_date = Calendar.recent(start_date) else: if start_date is None: d = end_date else: d = start_date offset = Calendar.calc(end_date['date'], Calendar.to('datetime', d)) start_date = Calendar.calc(d, -(offset + 60))['date'] # num = 60 sd, ed = date_preprocess(start_date, end_date) sql = 'stock_code = {} and date <= {}'.format(stock_code, Calendar.to(str, ed)) cond = analyzer(sql) # print(1, stock_code) cobj = model_list['capital'] cc = list(cobj.asc(cobj.query(cond), ['date'])) # print(2, stock_code) tobj = model_list['tushare'] tt = list(tobj.asc(tobj.query(cond), ['date'])) # print(3, stock_code) dd = calc_feature('XDXR_day', stock_code, start_date, end_date) if dd: dd = dd[-offset:] dl = len(dd) cl = len(cc) tl = len(tt) ic = 0 it = 0 capital_all = 0 capital = 0 dr = deepcopy(dd) sub_t = dict() for j in range(dl): # print(4, stock_code) d = dr[j] for ti in range(it, tl): t = tt[ti] if t['date'] <= d['date']: for k in t.keys(): if k == 'date': continue sub_t[k] = t[k] if pd.notnull(t[k]) else 0.0 it = ti + 1 else: break # print(5, stock_code) dict_combine(d, sub_t) # print(6, stock_code) for ci in range(ic, cl): c = cc[ci] if c['date'] <= d['date']: capital_all = c['capital_all'] capital = c['capital'] ic = ci + 1 else: break d['capital'] = capital d['capital_all'] = capital_all iobj.insert_batch(dr) print('insertttt', stock_code)
def calc_feature(tn, stock_code, start_date, end_date): obj = model_list[tn] sd, ed = date_preprocess(start_date, end_date) sql = 'stock_code = {} and date >= {} and date <= {}'.format( stock_code, Calendar.to(str, sd), Calendar.to(str, ed)) cond = analyzer(sql) raw_dl = list(obj.desc(obj.query(cond), ['date'])) if len(raw_dl) < 2: return result = list() dl = MA_s(pd.DataFrame(raw_dl)).to_dict() l = len(dl['date']) sn = 0 day1 = None for i in range(l): price_inc = 0 dt = Calendar.to(str, raw_dl[i]['date']) # 确定该股是不是次新股 if sn == 0: nc = analyzer('stock_code = {} and date <= {}'.format( raw_dl[i]['stock_code'], dt)) ds = obj.query(nc) sn = ds.count() day1 = obj.asc(ds, ['date'])[0] if 0 < sn < 100: sub_new = True if 3 < sn < 100: price_inc = raw_dl[i]['close'] / day1['open'] else: price_inc = 1 else: sub_new = False sn -= 1 d = dict(stock_code=raw_dl[i]['stock_code'], date=dl['date'][i], last_5=float(dl['last_5'][i]), last_10=float(dl['last_10'][i]), last_20=float(dl['last_20'][i]), last_30=float(dl['last_30'][i]), last_60=float(dl['last_60'][i]), ma5=float(dl['ma5'][i]), ma10=float(dl['ma10'][i]), ma20=float(dl['ma20'][i]), ma30=float(dl['ma30'][i]), ma60=float(dl['ma60'][i]), v_ma5=float(dl['v_ma5'][i]), v_ma10=float(dl['v_ma10'][i]), v_ma20=float(dl['v_ma20'][i]), v_ma30=float(dl['v_ma30'][i]), v_ma60=float(dl['v_ma60'][i]), close_z=float(dl['close_z'][i]), change_r=float(dl['change_r'][i]), volume_r=float(dl['volume_r'][i]), volume_z=float(dl['volume_z'][i]), v_dt_ma60=float(dl['v_dt_ma60'][i]), capital=0, price_inc=price_inc, sub_new=sub_new) d.update(raw_dl[i]) result.append(d) # print(dt, raw_dl[i]['stock_code'], sn) if result is None: print(stock_code, ' calc_feature is None') return sr = sorted(result, key=lambda x: (x['stock_code'], x['date'])) if sr is None: print(stock_code, ' calc_feature sort None') return return sr
obj = model_list['XDXR_' + _['category_name']] obj.insert_batch(calc_xdxr_specified(**_)) if __name__ == '__main__': # 参数含义见上例 market_name = ['sz', 'sh'] category_name = 'day' start_date = 20180706 end_date = None s = list() num = 10 for m in market_name: stock_code_list = get_stock_code_list(market=markets[m]) aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num)) )(stock_code_list) for s_c_l in aa: s.append( dict(category_name=category_name, start_date=start_date, end_date=end_date, stock_code_list=s_c_l)) model_list['XDXR_day'].remove( analyzer('date >= {} and date <= {}'.format(start_date, end_date))) cpus = 30 freeze_support() pool = Pool(cpus) pool.map(gt, s) pool.close() pool.join()
from app.models.base_model import BaseModel from app.query_str_analyzer import analyzer import pandas as pd sql = analyzer('date = {}'.format(20180620)) # sql['time']={'$lte':955} # data=BaseModel('tmp_kline_min5').remove(sql) data = BaseModel('kline_min5').query(sql) data = pd.DataFrame(list(data)) d = data.to_dict(orient='records') BaseModel('tmp_kline_min5').insert_batch(d)
mm = ['sz', 'sh'] s = list() sd = None ed = None for m in mm: sl = get_stock_code_list(markets[m]) num = 10 aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num)) )(sl) for a in aa: s.append( dict(market=m, stock_code_list=a, start_date=sd, end_date=ed, category='day')) model_list['XDXR_day'].remove( analyzer('date >= {} and date <= {}'.format(sd, ed))) freeze_support() pool = Pool(cpus) pool.map(gt, s) pool.close() pool.join() Mark.update_mark_success(strategy) exit() except Exception as e: x = '合并30异常:' + str(e.message) + "追踪:" + str(traceback.format_exc()) Mark.update_mark_fail(strategy=strategy, messages=x) exit(1)
# stock_codes = {'sh': model_list['kline_day'].query(market=markets['sh']['id']).distinct('stock_code'), # 'sz': model_list['kline_day'].query(market=markets['sz']['id']).distinct('stock_code')} # sd = Calendar.today() sd = 20180206 ed = None # 不传的话就是计算今天(15点后)/昨天(15点前) # sd = 20171215 # ed = 20171218 # tobj = model_list['tushare'] mm = ['sz', 'sh'] for m in mm: stock_codes = get_stock_code_list(markets[m]) sl = len(stock_codes) t = int(ceil(sl * 1.0 / cpus)) for i in range(cpus): s.append(dict(stock_code_list=stock_codes[i * t: (i + 1) * t], market_name=m, start_date=sd, end_date=ed, his=False)) print('done') freeze_support() pool = Pool(cpus) model_list['capital'].remove(analyzer('date >= {} and date <= {}'.format(sd, ed))) pool.map(ft, s) pool.close() pool.join() exit()
st = 20180604 et = None if Calendar().query(date=t): mm = ['sh', 'sz'] cc = ['day', 'min60', 'min30', 'min15', 'min5'] cc = ['min5'] s = list() for m in mm: sl = index_dict[m] num = 10 aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num)) )(sl) for k in cc: for a in aa: s.append( dict(market=m, category=k, stock_code_list=a, start_date=st, end_date=et)) sql = analyzer('date <= {} and date >= {}'.format(et, st)) for c in cc: model_list['index_{}'.format(c)].remove(sql) freeze_support() pool = Pool(cpus) pool.map(gt, s) pool.close() pool.join() # for ss in s: # gt(ss)
else: exit() # sd = 20170101 if his else None sd = 20180608 ed = None s = list() done = True dbsc = [] flag = False while done: try: for m in ['sh', 'sz']: sl = get_stock_code_list(markets[m]) dbsc = [ a for a in list(model_list['kline_tick'].query( analyzer('date >= {} and date <= {}'.format( sd, ed))).distinct('stock_code')) if (m == 'sz' and int(a) < 600000) or ( m == 'sh' and int(a) >= 600000) ] sl = list(set(sl).difference(set(dbsc))) num = 3 aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num)))(sl) print('aa ok') for a in aa: s.append( dict(market=m, stock_code_list=a, start_date=sd, end_date=ed, his=his))
# start_date=_['start_date'], end_date=_['end_date']) # obj.insert_batch(k_realtime_a(market_name=_['market'], category_name=_['category'], stock_code_list=_['stock_code_list'])) print(_['stock_code_list']) if __name__ == '__main__': print('in') t = Calendar.today() if Calendar().query(date=t): mm = ['usa'] cc = ['day', 'min60', 'min30', 'min15', 'min5'] # cc = ['min5'] # cc = ['day'] s = list() # sd = Calendar.today() sd = analyzer('date = {}'.format(20170101))['date'] ed = Calendar.today() for m in mm: pf = r'{}/{}'.format(project_dir, '{}.txt'.format(m)) with open(pf) as f: sl = f.read().split('\n') num = 30 aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num)) )(sl) for a in aa: for k in cc: s.append( dict(market=m, category=k, stock_code_list=a, start_date=sd,
func = gt for i in p: func(i) p = list() cs = list(Calendar().query(date={'$gte': sd, '$lte': ed})) # ts = model_list['feature_min5'].distinct('time') for cc in cs: for k, v in { 'sz': 'stock_code < 600000', 'cyb': 'stock_code >= 300000 and stock_code < 600000', 'sh': 'stock_code >= 600000', 'zxb': 'stock_code >= 002000 and stock_code < 300000' }.items(): # for t in ts: # p.append([analyzer('date = {} and time = {} and {}'.format(Calendar.to('str', cc['date']), t, v)), k]) p.append([ analyzer('date = {} and time = 1500 and {}'.format( Calendar.to('str', cc['date']), v)), k ]) func = ft freeze_support() pool = Pool(cpus) pool.map(func, p) pool.close() pool.join() # exit() e = datetime.now() print(s, e)
data = pd.DataFrame(result) all_count = list(data['all_count']) for i in range(0, len(all_count) - 1): if i != 4: if all_count[i] % all_count[i + 1] != 0: print(all_count[i], all_count[i + 1]) return False print(data) return True dt = Calendar.today() # pre_date = Calendar.calc(dt, -10) dt = Calendar.calc(dt, -1)['date'] sql = analyzer('date = {}'.format( str(dt.year * 10000 + dt.month * 100 + dt.day))) model_list['check_data'].remove(sql) print(Check_Data.check_kline('kline_min5')) print(Check_Data.check_kline('kline_min15')) print(Check_Data.check_kline('kline_min30')) print(Check_Data.check_kline('kline_min60')) print(Check_Data.check_kline('kline_day')) print(Check_Data.check_kline('index_min5')) print(Check_Data.check_kline('index_min15')) print(Check_Data.check_kline('index_min30')) print(Check_Data.check_kline('index_min60')) print(Check_Data.check_kline('index_day')) # # a = Check_Data.self_check() # print(a)
from app import cpus from app.actions import day_feature_index from app.models import Calendar, model_list from app.query_str_analyzer import analyzer def gti(*args): _ = args[0] day_feature_index(_[0], _[1]) if __name__ == '__main__': sd = 20180625 ed = None dates = list(Calendar().query( analyzer('date >= {} and date <= {}'.format(sd, ed)))) p = list() for d in dates: for k, v in { 'sz': 'stock_code < 600000', 'cyb': 'stock_code >= 300000 and stock_code < 600000', 'sh': 'stock_code >= 600000', 'zxb': 'stock_code >= 002000 and stock_code < 300000' }.items(): p.append([ analyzer('date = {} and {}'.format( Calendar.to('str', d['date']), v)), k ]) sql = analyzer('date >= {} and date <= {}'.format(sd, ed)) iobj = model_list['feature_index_day'].remove(sql) print('done')
from app.query_str_analyzer import analyzer def gti(*args): _ = args[0] day_feature_index(_[0], _[1]) # # daily if __name__ == '__main__': if not Calendar.in_business(dt=Calendar.today(), day=True): exit() p = list() d = None c = list(Calendar().asc(Calendar().query(analyzer('date = {}'.format(d))), ['num'])) for cc in c: for k, v in { 'sz': 'stock_code < 600000', 'cyb': 'stock_code >= 300000 and stock_code < 600000', 'sh': 'stock_code >= 600000', 'zxb': 'stock_code >= 002000 and stock_code < 300000' }.items(): p.append([ analyzer('date = {} and {}'.format( Calendar.to('str', cc['date']), v)), k ]) print('done') model_list['feature_index_day'].remove(date=Calendar.today()) cpus = 10
for s_c in _['stock_code_list']: day_feature(s_c, sd, ed, his) if __name__ == '__main__': s = list() market_name = ["sz", 'sh'] sd = None ed = None mm = ['sz', 'sh'] for m in mm: stock_codes = get_stock_code_list(markets[m]) sl = len(stock_codes) t = int(ceil(sl * 1.0 / cpus)) for i in range(cpus): s.append(dict(stock_code_list=stock_codes[i * t: (i + 1) * t], market_name=m, start_date=sd, end_date=ed, his=True)) print('done') model_list['feature_day'].remove(analyzer('date >= {} and date <= {}'.format(sd, ed))) freeze_support() pool = Pool(cpus) pool.map(gt, s) pool.close() pool.join() print('ok') # for i in s: # gt(i)
if __name__ == '__main__': his = True if len(sys.argv) > 1 and sys.argv[1] == 'his' else False print('mod: {}'.format(his)) if not Calendar.in_business(Calendar.today(), day=True): if his: pass else: exit() s = list() sd = 20170101 if his else None ed = None for m in ['sh', 'sz']: sl = get_stock_code_list(markets[m]) dbsc = [ a for a in list(model_list['kline_min1'].query( analyzer('date >= {} and date <= {}'.format(sd, ed))).distinct( 'stock_code')) if (m == 'sz' and int(a) < 600000) or ( m == 'sh' and int(a) >= 600000) ] sl = sorted(list(set(sl).difference(set(dbsc)))) num = 10 aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num)))(sl) for a in aa: s.append(dict(stock_code_list=a, start_date=sd, end_date=ed)) print(s) dql = analyzer('date >= {} and date <= {}'.format(sd, ed)) model_list['kline_min1'].remove(dql) print(datetime.now()) cpus = 30 freeze_support()