def get_kline_data(strategy):
"""
this function acts on kline
first get all stock code
then call the function named get_kline by multiprocessing
:param strategy:
:return:
"""
dd = Calendar.today()
if not Calendar.in_business(dt=Calendar.today(), day=True):
exit()
print('kline_in')
# t = Calendar.today()
st = None
et = None
t = Calendar.today()
if Calendar().query(date=t):
mm = ['sh', 'sz']
#cc = ['day', 'min60', 'min30', 'min15', 'min5']
cc = [strategy]
strategy = 'kline_' + strategy
s = list()
for m in mm:
sl = get_stock_code_list(markets[m])
num = 50
aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num))
)(sl)
for a in aa:
for k in cc:
s.append(
dict(market=m,
category=k,
stock_code_list=a,
start_date=st,
end_date=et))
# if st == None and et == None:
# model_list[strategy].remove(date=t)
# else:
# sql = analyzer('date >= {} and date <= {}'.format(st, et))
# model_list[strategy].remove(sql)
pool = multiprocessing.Pool(processes=10)
for i in s:
pool.apply_async(gt_kline, (i, ))
pool.close()
pool.join()
# for i in s:
# gt_kline(i)
# model_list[strategy].remove(volume=0)
print('-----------------' + str())
print('kline_done')
def gt_kline(*args):
# print('gt')
_ = args[0]
obj = model_list['kline_' + _['category']]
t = Calendar.today()
data = None
# print(os.getpid())
try:
data = k_specified_a(market_name=_['market'],
category_name=_['category'],
stock_code_list=_['stock_code_list'],
start_date=_['start_date'],
end_date=_['end_date'])
except Exception as e:
print(e)
# if _['category'] == 'min5':
# if datetime.now() <= n and datetime.now() >= n2:
# obj2 = model_list['tmp_kline_min5']
# obj2.insert_batch(data)
# print('ojb2 ok')
# Mark.update_mark_success('tmp_kline_min5')
# for sc in _['stock_code_list']:
# obj.remove(date=t, stock_code=sc)
obj.insert_batch(data)
# gc.collect()
# print('____')
print(os.getpid(), _['stock_code_list'])
return 0
def get_index_data(strategy):
print('index_in')
t = Calendar.today()
if Calendar().query(date=t):
mm = ['sh', 'sz']
cc = [strategy]
strategy = 'index_' + strategy
s = list()
for m in mm:
sl = index_dict[m]
num = 10
aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num))
)(sl)
for a in aa:
for k in cc:
s.append(
dict(market=m,
category=k,
stock_code_list=a,
start_date=None,
end_date=None))
print('item')
model_list[strategy].remove(date=t)
print(s)
pool = Pool(7)
pool.map(gt_index, s)
# p=multiprocessing.Process(target=gt_index,args=s)
# p.start()
# p.join(timeout=60)
print('index_done')
def check_kline(cls, strategy='kline_min5'):
dt = Calendar.today()
# pre_date = Calendar.calc(dt, -10)
dt = Calendar.calc(dt, -1)['date']
# sql = analyzer('date = {}'.format(str(dt.year * 10000 + dt.month * 100 + dt.day)))
# model_list['check_data'].remove(sql)
# print(pre_date)
sql = analyzer("date >= {} and date <= {}".format(
str(20180111), str(dt.year * 10000 + dt.month * 100 + dt.day)))
print("____________________" + strategy)
print(sql)
print('稍等几秒')
cursor = model_list[strategy].query(sql)
print(cursor)
all_count = cursor.count()
print(all_count)
day_count = cls.number_of_branches(strategy)
print(day_count)
dic = {
'date': dt,
'strategy': strategy,
'all_count': all_count,
'day_count': day_count,
'result': all_count % day_count == 0
}
print('insert')
model_list['check_data'].insert(dic)
if all_count % day_count == 0:
return strategy + ' is true'
print("____________________")
else:
return strategy + "有问题"
def get_kline_data(strategy):
# print('kline_in')
# file_log(strategy,'time:'+str(datetime.now())+'\n'+'op:start')
# t = Calendar.today()
print('step into')
t = Calendar.today()
# 111=555
if Calendar().query(date=t):
mm = ['sh', 'sz']
# cc = ['day', 'min60', 'min30', 'min15', 'min5']
cc = [strategy]
strategy = 'kline_' + strategy
s = list()
# ss = datetime.now()
# print(ss)
for m in mm:
sl = get_stock_code_list(markets[m])
num = 10
aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num))
)(sl)
print(aa)
for a in aa:
for k in cc:
s.append(
dict(market=m,
category=k,
stock_code_list=a,
start_date=None,
end_date=None))
# e = datetime.now()
# print(e)
# file_log(strategy, 'time:' + str(datetime.now()) + '\n' + 'op:get stock_code')
print('step getstocks')
model_list[strategy].remove(date=t)
print('remove over')
# file_log(strategy, 'time:' + str(datetime.now()) + '\n' + 'op:remove today')
# f = datetime.now()
# print(f)
pool = Pool(10)
pool.map(gt_kline, s)
# for i in s:
# gt_kline(i)
# pool.close()
# pool.join()
# print('s=',s)
# p = multiprocessing.Process(target=gt_kline, args=s)
# p.start()
# p.join(timeout=300)
# for i in s:
# gt_kline(i)
# model_list[strategy].remove(volume=0)
# file_log(strategy, 'time:' + str(datetime.now()) + '\n' + 'op:complate')
print('kline_done')
return 0
def get_data_allocate(strategy, data_type):
dd = Calendar.today()
a = Calendar.in_business(dd, day=True)
if (a == False):
print('非工作日！')
exit(0)
t1 = time.clock()
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
command = "python {}\daily_{}_{}.py".format(project_dir, data_type,
strategy)
list = []
if (strategy == 'min5'):
list = get_min5_time()
if (strategy == 'min15'):
list = get_min15_time()
if (strategy == 'min30'):
list = get_min30_time()
if (strategy == 'min60'):
list = get_min60_time()
if (strategy == 'day'):
list = get_day_time()
print(strategy)
while True:
# if in_business(datetime.now(), strategy, True):
x = datetime.now().hour * 60 + datetime.now().minute
# print ( datetime.now().hour,datetime.now().minute)
# print(x)
# if(x>list[-1]):
if (x > list[-1]):
print(list[-1])
print('收盘！')
exit(0)
if (x == 11 * 60 + 45):
print('上午结束')
exit(0)
if (x in list):
print('finded')
t1 = time.clock()
subprocess.call(command, shell=True)
t3 = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
print(t3)
dit = {'date': t3, 'status': 1, 'kline': strategy, 'other': ''}
model_list['kline_data_update_mark'].insert(dit)
t2 = time.clock()
a = (change_status(strategy)['time']) - ((t2 - t1) / 60)
print(a)
if (a > 0):
time.sleep(a * 60)
print('睡眠结束')
else:
print('finding:')
time.sleep(10)
def self_check(cls, date=Calendar.today()):
date = Calendar.calc(date=date, offset=-1)['date']
sql = analyzer('date = {}'.format(date.year * 10000 +
date.month * 100 + date.day))
print(sql)
result = list(model_list['check_data'].query(sql))
data = pd.DataFrame(result)
all_count = list(data['all_count'])
for i in range(0, len(all_count) - 1):
if i != 4:
if all_count[i] % all_count[i + 1] != 0:
print(all_count[i], all_count[i + 1])
return False
print(data)
return True
def get_kline_data(strategy):
print('kline_in')
t = Calendar.today()
if Calendar().query(date=t):
mm = ['sh', 'sz']
cc = [strategy]
strategy = 'kline_' + strategy
s = list()
for m in mm:
sl = get_stock_code_list(markets[m])
num = 50
aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num))
)(sl)
for a in aa:
for k in cc:
s.append(
dict(market=m,
category=k,
stock_code_list=a,
start_date=None,
end_date=None))
# model_list[strategy].remove(date=t)
# model_list['kline_min15'].remove(date=t)
# model_list['kline_min5'].remove(date=t)
# model_list['kline_min30'].remove(date=t)
# model_list['kline_min60'].remove(date=t)
# model_list['kline_day'].remove(date=t)
print("((((((((((((")
print(len(s))
freeze_support()
pool = Pool(5)
r = pool.map(gt_kline, s)
df = pd.concat(r, join='outer', ignore_index=False)
df.reset_index(inplace=True)
# print(df.head(10))
pool.close()
pool.join()
return df
# -*- coding: utf-8 -*-
from os.path import abspath, pardir, join
from sys import path
from app import markets
from app.models import Calendar
from app.actions import get_stock_code_list, get_xdxr
path.append(abspath(join(abspath(join(abspath(__file__), pardir)), pardir)))
if __name__ == '__main__':
if not Calendar.in_business(dt=Calendar.today(), day=True):
exit()
start_date = None
end_date = None
stock_code_list = []
for m in ['sh', 'sz']:
stock_code_list = get_stock_code_list(market=markets[m])
get_xdxr(market_name=m,
stock_code_list=stock_code_list,
start_date=start_date,
end_date=end_date)
k_specified_ex(market_name=_['market'],
category_name=_['category'],
stock_code_list=_['stock_code_list'],
start_date=_['start_date'],
end_date=_['end_date']))
# k_specified_a(market_name=_['market'], category_name=_['category'], stock_code_list=_['stock_code_list'],
# start_date=_['start_date'], end_date=_['end_date'])
# k_history_a(market_name=_['market'], category_name=_['category'], stock_code_list=_['stock_code_list'],
# start_date=_['start_date'], end_date=_['end_date'])
# obj.insert_batch(k_realtime_a(market_name=_['market'], category_name=_['category'], stock_code_list=_['stock_code_list']))
print(_['stock_code_list'])
if __name__ == '__main__':
print('in')
t = Calendar.today()
if Calendar().query(date=t):
mm = ['usa']
cc = ['day', 'min60', 'min30', 'min15', 'min5']
# cc = ['min5']
# cc = ['day']
s = list()
# sd = Calendar.today()
sd = analyzer('date = {}'.format(20170101))['date']
ed = Calendar.today()
for m in mm:
pf = r'{}/{}'.format(project_dir, '{}.txt'.format(m))
with open(pf) as f:
sl = f.read().split('\n')
num = 30
aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num))
from multiprocessing import freeze_support, Pool
from app import cpus
from app.actions import day_feature_index
from app.models import Calendar, model_list
from app.query_str_analyzer import analyzer
def gti(*args):
_ = args[0]
day_feature_index(_[0], _[1])
# # daily
if __name__ == '__main__':
if not Calendar.in_business(dt=Calendar.today(), day=True):
exit()
p = list()
d = None
c = list(Calendar().asc(Calendar().query(analyzer('date = {}'.format(d))),
['num']))
for cc in c:
for k, v in {
'sz': 'stock_code < 600000',
'cyb': 'stock_code >= 300000 and stock_code < 600000',
'sh': 'stock_code >= 600000',
'zxb': 'stock_code >= 002000 and stock_code < 300000'
}.items():
p.append([
analyzer('date = {} and {}'.format(
print(sql)
result = list(model_list['check_data'].query(sql))
data = pd.DataFrame(result)
all_count = list(data['all_count'])
for i in range(0, len(all_count) - 1):
if i != 4:
if all_count[i] % all_count[i + 1] != 0:
print(all_count[i], all_count[i + 1])
return False
print(data)
return True
dt = Calendar.today()
# pre_date = Calendar.calc(dt, -10)
dt = Calendar.calc(dt, -1)['date']
sql = analyzer('date = {}'.format(
str(dt.year * 10000 + dt.month * 100 + dt.day)))
model_list['check_data'].remove(sql)
print(Check_Data.check_kline('kline_min5'))
print(Check_Data.check_kline('kline_min15'))
print(Check_Data.check_kline('kline_min30'))
print(Check_Data.check_kline('kline_min60'))
print(Check_Data.check_kline('kline_day'))
print(Check_Data.check_kline('index_min5'))
print(Check_Data.check_kline('index_min15'))
print(Check_Data.check_kline('index_min30'))
print(Check_Data.check_kline('index_min60'))
print(Check_Data.check_kline('index_day'))