def get_volumes(strategy):
obj = model_list['kline_' + 'min5']
t = 0
if (strategy[-2:] == '30'):
t = 1425
c = 6
if (strategy[-2:] == '60'):
t = 1355
c = 12
print(strategy)
dt = datetime.datetime.now()
str1 = str(dt.year * 10000 + dt.month * 100 +
dt.day) # data testing please update
print(str1)
sql = analyzer("time >= {} and time < {} and date = {}".format(
t, 1455, int(str1)))
print(sql)
a = obj.query(sql)
a = list(a)
df = pd.DataFrame(a)
print(list)
if (len(df) == 0):
print('df is None')
df = df.sort_values(['stock_code', 'time'], ascending=True)
df.reset_index(inplace=True)
grouped = df['amount'].groupby(df['stock_code'])
data = grouped.sum()
# for i,r in data.iterrows():
# dic[r.stock_code]=r.volume
return (dict(data))
def get_data(kline):
t = 1425 if kline == 'min30' else 1355
today = dt.datetime.strftime(dt.date.today(), '%Y%m%d')
tt = {'$gte': t, '$lte': 1450}
sql = analyzer("date = {} and time <= {} and time >= {}".format(
today, 1450, t))
print(sql)
a = model_list['kline_min5'].query(sql)
a = list(a)
data = pd.DataFrame(a)
# data = md.get_stock_data(kline='min5', start_date=today, end_date=today, axis=0, timemerge=False, time=t)
kline_data = data[data.time == 1450]
spl_data = data.amount.groupby(data.stock_code).sum()
spl_data = spl_data.to_frame()
spl_data['stock_code'] = spl_data.index
data = pd.merge(kline_data, spl_data, on='stock_code')
data['amount'] = data.amount_x + data.amount_y
data['time'] = 1500
data.drop(['_id', 'classtype', 'amount_x', 'amount_y'],
axis=1,
inplace=True)
dicts = []
for i, r in data.iterrows():
dicts.append(dict(r))
x = model_list['kline_' + kline].insert_batch(dicts)
print(x)
return data
def check_kline(cls, strategy='kline_min5'):
dt = Calendar.today()
# pre_date = Calendar.calc(dt, -10)
dt = Calendar.calc(dt, -1)['date']
# sql = analyzer('date = {}'.format(str(dt.year * 10000 + dt.month * 100 + dt.day)))
# model_list['check_data'].remove(sql)
# print(pre_date)
sql = analyzer("date >= {} and date <= {}".format(
str(20180111), str(dt.year * 10000 + dt.month * 100 + dt.day)))
print("____________________" + strategy)
print(sql)
print('稍等几秒')
cursor = model_list[strategy].query(sql)
print(cursor)
all_count = cursor.count()
print(all_count)
day_count = cls.number_of_branches(strategy)
print(day_count)
dic = {
'date': dt,
'strategy': strategy,
'all_count': all_count,
'day_count': day_count,
'result': all_count % day_count == 0
}
print('insert')
model_list['check_data'].insert(dic)
if all_count % day_count == 0:
return strategy + ' is true'
print("____________________")
else:
return strategy + "有问题"
def get_his(stratrgy):
sql = get_sql(stratrgy)
obj = model_list['kline_' + stratrgy]
print(stratrgy)
print('^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^')
print(sql)
dt = datetime.datetime.now()
str1 = str(dt.year * 10000 + dt.month * 100 + dt.day)
sql = dict(sql, **analyzer("date = {}".format(str1)))
a = obj.query(sql)
a = list(a)
df = pd.DataFrame(a)
df.reset_index(inplace=True)
return df
def tushare_to_db(start_date=None, end_date=None):
sd, ed = date_preprocess(start_date, end_date)
obj = model_list['tushare']
dr = list()
if start_date:
season = (sd.month - 1) // 3
s1 = '{}{:02d}01'.format(sd.year, season * 3 if season > 0 else 1)
cond = analyzer('date >= {}'.format(s1))
obj.remove(cond)
dr = trans_tushare_data(sd.year, season + 1)
else:
for y in range(sd.year, ed.year + 1):
for s in range(1, 5):
dr.extend(trans_tushare_data(y, s))
obj.insert_batch(dr)
def date_range(start_date=None, end_date=None):
if start_date:
ssql = 'date >= {}'.format(Calendar.to(str, start_date))
else:
ssql = 'date >= 20120130'
if end_date:
esql = 'date <= {}'.format(Calendar.to(str, end_date))
else:
esql = 'date <= {}'.format(Calendar.to(str, Calendar.recent()))
sql = ' and '.join([ssql, esql])
cond = analyzer(sql)
calendar = list(Calendar().query(cond))
return calendar
def self_check(cls, date=Calendar.today()):
date = Calendar.calc(date=date, offset=-1)['date']
sql = analyzer('date = {}'.format(date.year * 10000 +
date.month * 100 + date.day))
print(sql)
result = list(model_list['check_data'].query(sql))
data = pd.DataFrame(result)
all_count = list(data['all_count'])
for i in range(0, len(all_count) - 1):
if i != 4:
if all_count[i] % all_count[i + 1] != 0:
print(all_count[i], all_count[i + 1])
return False
print(data)
return True
def get_data():
dt = datetime.now()
# if dt.hour > 15:
# dt = datetime(2018, 6, 12, 14, 10)
now_time = dt.hour * 100 + dt.minute
if (now_time >=1030 and now_time<1100)or (now_time>=1130 and now_time<1300) or (now_time>=1400 and now_time<1430) :
start_time=0
if dt.hour >=11 and dt.hour<13:
now_time=1300
for i in range(0,len(min60_time_list())):
if now_time>=min60_time_list()[i]:
now_time=min60_time_list()[i]
start_time=min60_time_list()[i+1]
print(now_time)
print(start_time)
break
sql = analyzer('date = {} and time <= {} and time >= {}'.format(dt.year * 10000 + dt.month * 100 + dt.day, now_time,
start_time))
print(sql)
data = model_list['kline_min30'].query(sql)
data = pd.DataFrame(list(data))
data.sort_values(by=['time'])
kline_data = data[data.time == now_time]
kline_data_his = data[data.time == start_time]
grouped = data.groupby(['stock_code']).agg({'amount': 'sum','volume':'sum','high':'max','low':'min'})
grouped['stock_code'] = grouped.index
data = pd.merge(kline_data, grouped, on='stock_code')
data['time'] = now_time
data['amount'] = data.amount_y
data['volume'] = data.volume_y
data['high'] = data.high_y
data['low'] = data.low_y
# data.open=kline_data_his.open
data.drop(['_id', 'classtype', 'amount_x', 'amount_y','volume_x','volume_y','high_y','high_x','low_x','low_y'], axis=1, inplace=True)
dicts = []
for i,r in data.iterrows():
dicts.append(dict(r))
x=model_list['kline_min60'].insert_batch(dicts)
print(dicts[0:10])
return data
else:
exit()
def data_is_update(strategy, data_type):
today = time.strftime('%Y%m%d', time.localtime(time.time()))
cql = analyzer(
"date = {s}".format(s=today))
strategy=data_type+"_"+strategy
cursor=model_list[strategy].query(cql)
stock_data = list(cursor)
if len(stock_data) > 0:
data = pd.DataFrame(stock_data)
a=(data.time.max())
print (a)
bf=a/100
af=a-bf*100
now_hour=datetime.now().hour
now_minute=datetime.now().minute
database_data_time=bf*60+af
now_time=now_hour*60+now_minute
if ((now_time - database_data_time) < change_status(strategy)['time']):
return {'time': a, 'isupdate': True}
return {'time': a, 'isupdate': False}
def get_history_tick(market_name,
stock_code,
start_date=None,
end_date=None,
his=False):
sd, ed = date_preprocess(start_date, end_date)
days = list(Calendar().query(
analyzer('date >= {} and date <= {}'.format(
Calendar.to(str, Calendar.recent(sd)),
Calendar.to(str, Calendar.recent(ed))))))
tick_data_list = list()
market = markets[market_name]
for day in days:
d = day['date']
loop = True
start = 0
while loop:
if his:
err, count, result = TradeX.TdxHq_Connect(
'61.152.249.56',
7709).GetHistoryTransactionData(market['id'], stock_code,
start, 2000,
Calendar.to(int, d))
else:
err, count, result = TradeX.TdxHq_Connect(
'61.152.249.56',
7709).GetTransactionData(market['id'], stock_code, start,
2000)
# print(result.decode('gbk'))
row = result.split('\n')
tick_data_list.extend(
Tick.trans_data(stock_code, market_name, d, row))
start += 2000
loop = count == 2000
# obj.insert_batch(tick_data_list)
print('{} done: {}'.format(stock_code, len(tick_data_list)))
return tick_data_list
def day_feature(stock_code, start_date, end_date, his=False):
iobj = model_list['feature_day']
offset = 0
if end_date is None:
end_date = Calendar.recent()
if Calendar.in_business(datetime.now(), hard=False):
end_date = Calendar.calc(end_date, -1)
else:
end_date = Calendar.recent(end_date)
# 如果 start_date 为空，则定到 end_date 之前的60天
# 这样刚好可以算出 end_date 那一天的 ma60
# 实现了 daily update 的功能
if his:
start_date = Calendar.recent(start_date)
else:
if start_date is None:
d = end_date
else:
d = start_date
offset = Calendar.calc(end_date['date'], Calendar.to('datetime', d))
start_date = Calendar.calc(d, -(offset + 60))['date']
# num = 60
sd, ed = date_preprocess(start_date, end_date)
sql = 'stock_code = {} and date <= {}'.format(stock_code,
Calendar.to(str, ed))
cond = analyzer(sql)
# print(1, stock_code)
cobj = model_list['capital']
cc = list(cobj.asc(cobj.query(cond), ['date']))
# print(2, stock_code)
tobj = model_list['tushare']
tt = list(tobj.asc(tobj.query(cond), ['date']))
# print(3, stock_code)
dd = calc_feature('XDXR_day', stock_code, start_date, end_date)
if dd:
dd = dd[-offset:]
dl = len(dd)
cl = len(cc)
tl = len(tt)
ic = 0
it = 0
capital_all = 0
capital = 0
dr = deepcopy(dd)
sub_t = dict()
for j in range(dl):
# print(4, stock_code)
d = dr[j]
for ti in range(it, tl):
t = tt[ti]
if t['date'] <= d['date']:
for k in t.keys():
if k == 'date':
continue
sub_t[k] = t[k] if pd.notnull(t[k]) else 0.0
it = ti + 1
else:
break
# print(5, stock_code)
dict_combine(d, sub_t)
# print(6, stock_code)
for ci in range(ic, cl):
c = cc[ci]
if c['date'] <= d['date']:
capital_all = c['capital_all']
capital = c['capital']
ic = ci + 1
else:
break
d['capital'] = capital
d['capital_all'] = capital_all
iobj.insert_batch(dr)
print('insertttt', stock_code)
def calc_feature(tn, stock_code, start_date, end_date):
obj = model_list[tn]
sd, ed = date_preprocess(start_date, end_date)
sql = 'stock_code = {} and date >= {} and date <= {}'.format(
stock_code, Calendar.to(str, sd), Calendar.to(str, ed))
cond = analyzer(sql)
raw_dl = list(obj.desc(obj.query(cond), ['date']))
if len(raw_dl) < 2:
return
result = list()
dl = MA_s(pd.DataFrame(raw_dl)).to_dict()
l = len(dl['date'])
sn = 0
day1 = None
for i in range(l):
price_inc = 0
dt = Calendar.to(str, raw_dl[i]['date'])
# 确定该股是不是次新股
if sn == 0:
nc = analyzer('stock_code = {} and date <= {}'.format(
raw_dl[i]['stock_code'], dt))
ds = obj.query(nc)
sn = ds.count()
day1 = obj.asc(ds, ['date'])[0]
if 0 < sn < 100:
sub_new = True
if 3 < sn < 100:
price_inc = raw_dl[i]['close'] / day1['open']
else:
price_inc = 1
else:
sub_new = False
sn -= 1
d = dict(stock_code=raw_dl[i]['stock_code'],
date=dl['date'][i],
last_5=float(dl['last_5'][i]),
last_10=float(dl['last_10'][i]),
last_20=float(dl['last_20'][i]),
last_30=float(dl['last_30'][i]),
last_60=float(dl['last_60'][i]),
ma5=float(dl['ma5'][i]),
ma10=float(dl['ma10'][i]),
ma20=float(dl['ma20'][i]),
ma30=float(dl['ma30'][i]),
ma60=float(dl['ma60'][i]),
v_ma5=float(dl['v_ma5'][i]),
v_ma10=float(dl['v_ma10'][i]),
v_ma20=float(dl['v_ma20'][i]),
v_ma30=float(dl['v_ma30'][i]),
v_ma60=float(dl['v_ma60'][i]),
close_z=float(dl['close_z'][i]),
change_r=float(dl['change_r'][i]),
volume_r=float(dl['volume_r'][i]),
volume_z=float(dl['volume_z'][i]),
v_dt_ma60=float(dl['v_dt_ma60'][i]),
capital=0,
price_inc=price_inc,
sub_new=sub_new)
d.update(raw_dl[i])
result.append(d)
# print(dt, raw_dl[i]['stock_code'], sn)
if result is None:
print(stock_code, ' calc_feature is None')
return
sr = sorted(result, key=lambda x: (x['stock_code'], x['date']))
if sr is None:
print(stock_code, ' calc_feature sort None')
return
return sr
obj = model_list['XDXR_' + _['category_name']]
obj.insert_batch(calc_xdxr_specified(**_))
if __name__ == '__main__':
# 参数含义见上例
market_name = ['sz', 'sh']
category_name = 'day'
start_date = 20180706
end_date = None
s = list()
num = 10
for m in market_name:
stock_code_list = get_stock_code_list(market=markets[m])
aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num))
)(stock_code_list)
for s_c_l in aa:
s.append(
dict(category_name=category_name,
start_date=start_date,
end_date=end_date,
stock_code_list=s_c_l))
model_list['XDXR_day'].remove(
analyzer('date >= {} and date <= {}'.format(start_date, end_date)))
cpus = 30
freeze_support()
pool = Pool(cpus)
pool.map(gt, s)
pool.close()
pool.join()
from app.models.base_model import BaseModel
from app.query_str_analyzer import analyzer
import pandas as pd
sql = analyzer('date = {}'.format(20180620))
# sql['time']={'$lte':955}
# data=BaseModel('tmp_kline_min5').remove(sql)
data = BaseModel('kline_min5').query(sql)
data = pd.DataFrame(list(data))
d = data.to_dict(orient='records')
BaseModel('tmp_kline_min5').insert_batch(d)
mm = ['sz', 'sh']
s = list()
sd = None
ed = None
for m in mm:
sl = get_stock_code_list(markets[m])
num = 10
aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num))
)(sl)
for a in aa:
s.append(
dict(market=m,
stock_code_list=a,
start_date=sd,
end_date=ed,
category='day'))
model_list['XDXR_day'].remove(
analyzer('date >= {} and date <= {}'.format(sd, ed)))
freeze_support()
pool = Pool(cpus)
pool.map(gt, s)
pool.close()
pool.join()
Mark.update_mark_success(strategy)
exit()
except Exception as e:
x = '合并30异常:' + str(e.message) + "追踪：" + str(traceback.format_exc())
Mark.update_mark_fail(strategy=strategy, messages=x)
exit(1)
# stock_codes = {'sh': model_list['kline_day'].query(market=markets['sh']['id']).distinct('stock_code'),
# 'sz': model_list['kline_day'].query(market=markets['sz']['id']).distinct('stock_code')}
# sd = Calendar.today()
sd = 20180206
ed = None
# 不传的话就是计算今天（15点后）/昨天（15点前）
# sd = 20171215
# ed = 20171218
# tobj = model_list['tushare']
mm = ['sz', 'sh']
for m in mm:
stock_codes = get_stock_code_list(markets[m])
sl = len(stock_codes)
t = int(ceil(sl * 1.0 / cpus))
for i in range(cpus):
s.append(dict(stock_code_list=stock_codes[i * t: (i + 1) * t], market_name=m,
start_date=sd,
end_date=ed,
his=False))
print('done')
freeze_support()
pool = Pool(cpus)
model_list['capital'].remove(analyzer('date >= {} and date <= {}'.format(sd, ed)))
pool.map(ft, s)
pool.close()
pool.join()
exit()
st = 20180604
et = None
if Calendar().query(date=t):
mm = ['sh', 'sz']
cc = ['day', 'min60', 'min30', 'min15', 'min5']
cc = ['min5']
s = list()
for m in mm:
sl = index_dict[m]
num = 10
aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num))
)(sl)
for k in cc:
for a in aa:
s.append(
dict(market=m,
category=k,
stock_code_list=a,
start_date=st,
end_date=et))
sql = analyzer('date <= {} and date >= {}'.format(et, st))
for c in cc:
model_list['index_{}'.format(c)].remove(sql)
freeze_support()
pool = Pool(cpus)
pool.map(gt, s)
pool.close()
pool.join()
# for ss in s:
# gt(ss)
else:
exit()
# sd = 20170101 if his else None
sd = 20180608
ed = None
s = list()
done = True
dbsc = []
flag = False
while done:
try:
for m in ['sh', 'sz']:
sl = get_stock_code_list(markets[m])
dbsc = [
a for a in list(model_list['kline_tick'].query(
analyzer('date >= {} and date <= {}'.format(
sd, ed))).distinct('stock_code'))
if (m == 'sz' and int(a) < 600000) or (
m == 'sh' and int(a) >= 600000)
]
sl = list(set(sl).difference(set(dbsc)))
num = 3
aa = (lambda a: map(lambda b: a[b:b + num],
range(0, len(a), num)))(sl)
print('aa ok')
for a in aa:
s.append(
dict(market=m,
stock_code_list=a,
start_date=sd,
end_date=ed,
his=his))
# start_date=_['start_date'], end_date=_['end_date'])
# obj.insert_batch(k_realtime_a(market_name=_['market'], category_name=_['category'], stock_code_list=_['stock_code_list']))
print(_['stock_code_list'])
if __name__ == '__main__':
print('in')
t = Calendar.today()
if Calendar().query(date=t):
mm = ['usa']
cc = ['day', 'min60', 'min30', 'min15', 'min5']
# cc = ['min5']
# cc = ['day']
s = list()
# sd = Calendar.today()
sd = analyzer('date = {}'.format(20170101))['date']
ed = Calendar.today()
for m in mm:
pf = r'{}/{}'.format(project_dir, '{}.txt'.format(m))
with open(pf) as f:
sl = f.read().split('\n')
num = 30
aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num))
)(sl)
for a in aa:
for k in cc:
s.append(
dict(market=m,
category=k,
stock_code_list=a,
start_date=sd,
func = gt
for i in p:
func(i)
p = list()
cs = list(Calendar().query(date={'$gte': sd, '$lte': ed}))
# ts = model_list['feature_min5'].distinct('time')
for cc in cs:
for k, v in {
'sz': 'stock_code < 600000',
'cyb': 'stock_code >= 300000 and stock_code < 600000',
'sh': 'stock_code >= 600000',
'zxb': 'stock_code >= 002000 and stock_code < 300000'
}.items():
# for t in ts:
# p.append([analyzer('date = {} and time = {} and {}'.format(Calendar.to('str', cc['date']), t, v)), k])
p.append([
analyzer('date = {} and time = 1500 and {}'.format(
Calendar.to('str', cc['date']), v)), k
])
func = ft
freeze_support()
pool = Pool(cpus)
pool.map(func, p)
pool.close()
pool.join()
# exit()
e = datetime.now()
print(s, e)
data = pd.DataFrame(result)
all_count = list(data['all_count'])
for i in range(0, len(all_count) - 1):
if i != 4:
if all_count[i] % all_count[i + 1] != 0:
print(all_count[i], all_count[i + 1])
return False
print(data)
return True
dt = Calendar.today()
# pre_date = Calendar.calc(dt, -10)
dt = Calendar.calc(dt, -1)['date']
sql = analyzer('date = {}'.format(
str(dt.year * 10000 + dt.month * 100 + dt.day)))
model_list['check_data'].remove(sql)
print(Check_Data.check_kline('kline_min5'))
print(Check_Data.check_kline('kline_min15'))
print(Check_Data.check_kline('kline_min30'))
print(Check_Data.check_kline('kline_min60'))
print(Check_Data.check_kline('kline_day'))
print(Check_Data.check_kline('index_min5'))
print(Check_Data.check_kline('index_min15'))
print(Check_Data.check_kline('index_min30'))
print(Check_Data.check_kline('index_min60'))
print(Check_Data.check_kline('index_day'))
#
# a = Check_Data.self_check()
# print(a)
from app import cpus
from app.actions import day_feature_index
from app.models import Calendar, model_list
from app.query_str_analyzer import analyzer
def gti(*args):
_ = args[0]
day_feature_index(_[0], _[1])
if __name__ == '__main__':
sd = 20180625
ed = None
dates = list(Calendar().query(
analyzer('date >= {} and date <= {}'.format(sd, ed))))
p = list()
for d in dates:
for k, v in {
'sz': 'stock_code < 600000',
'cyb': 'stock_code >= 300000 and stock_code < 600000',
'sh': 'stock_code >= 600000',
'zxb': 'stock_code >= 002000 and stock_code < 300000'
}.items():
p.append([
analyzer('date = {} and {}'.format(
Calendar.to('str', d['date']), v)), k
])
sql = analyzer('date >= {} and date <= {}'.format(sd, ed))
iobj = model_list['feature_index_day'].remove(sql)
print('done')
from app.query_str_analyzer import analyzer
def gti(*args):
_ = args[0]
day_feature_index(_[0], _[1])
# # daily
if __name__ == '__main__':
if not Calendar.in_business(dt=Calendar.today(), day=True):
exit()
p = list()
d = None
c = list(Calendar().asc(Calendar().query(analyzer('date = {}'.format(d))),
['num']))
for cc in c:
for k, v in {
'sz': 'stock_code < 600000',
'cyb': 'stock_code >= 300000 and stock_code < 600000',
'sh': 'stock_code >= 600000',
'zxb': 'stock_code >= 002000 and stock_code < 300000'
}.items():
p.append([
analyzer('date = {} and {}'.format(
Calendar.to('str', cc['date']), v)), k
])
print('done')
model_list['feature_index_day'].remove(date=Calendar.today())
cpus = 10
for s_c in _['stock_code_list']:
day_feature(s_c, sd, ed, his)
if __name__ == '__main__':
s = list()
market_name = ["sz", 'sh']
sd = None
ed = None
mm = ['sz', 'sh']
for m in mm:
stock_codes = get_stock_code_list(markets[m])
sl = len(stock_codes)
t = int(ceil(sl * 1.0 / cpus))
for i in range(cpus):
s.append(dict(stock_code_list=stock_codes[i * t: (i + 1) * t], market_name=m,
start_date=sd,
end_date=ed,
his=True))
print('done')
model_list['feature_day'].remove(analyzer('date >= {} and date <= {}'.format(sd, ed)))
freeze_support()
pool = Pool(cpus)
pool.map(gt, s)
pool.close()
pool.join()
print('ok')
# for i in s:
# gt(i)
if __name__ == '__main__':
his = True if len(sys.argv) > 1 and sys.argv[1] == 'his' else False
print('mod: {}'.format(his))
if not Calendar.in_business(Calendar.today(), day=True):
if his:
pass
else:
exit()
s = list()
sd = 20170101 if his else None
ed = None
for m in ['sh', 'sz']:
sl = get_stock_code_list(markets[m])
dbsc = [
a for a in list(model_list['kline_min1'].query(
analyzer('date >= {} and date <= {}'.format(sd, ed))).distinct(
'stock_code')) if (m == 'sz' and int(a) < 600000) or (
m == 'sh' and int(a) >= 600000)
]
sl = sorted(list(set(sl).difference(set(dbsc))))
num = 10
aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num)))(sl)
for a in aa:
s.append(dict(stock_code_list=a, start_date=sd, end_date=ed))
print(s)
dql = analyzer('date >= {} and date <= {}'.format(sd, ed))
model_list['kline_min1'].remove(dql)
print(datetime.now())
cpus = 30
freeze_support()
