def to_min(self, stock_code, day):
ticks = list(
self.query(date=Calendar.to('datetime', day),
stock_code=stock_code))
if len(ticks):
ticks = sorted(ticks, key=lambda x: (x['time']))
pd_ticks = pd.DataFrame(ticks)
pd_ticks['open'] = pd_ticks.price
pd_ticks['close'] = pd_ticks.price
pd_ticks['high'] = pd_ticks.price
pd_ticks['low'] = pd_ticks.price
pd_ticks['volume'] *= 100
pd_ticks['amount'] = pd_ticks['price'] * pd_ticks['volume']
x1 = pd_ticks.groupby(by='time').agg({
'volume': 'sum',
'close': 'last',
'open': 'first',
'high': 'max',
'low': 'min',
'amount': 'sum'
})
tmp_data = x1.to_dict()
min_data = list()
open = tmp_data['open']
close = tmp_data['close']
high = tmp_data['high']
low = tmp_data['low']
volume = tmp_data['volume']
amount = tmp_data['amount']
times = sorted(open.keys())
for t in times:
min_data.append({
'stock_code':
stock_code,
'date':
day,
'time':
int(t),
'open':
round(open[t], 2),
'close':
round(close[t], 2),
'high':
round(high[t], 2),
'low':
round(float(low[t]), 2),
'amount':
int(amount[t]),
'volume':
int(volume[t]),
'market':
1 if stock_code.startswith('60') else 0
})
return min_data
else:
return []
def get_zybk_ttm(start_date=None, end_date=None):
obj = model_list['fund_zybk_ttm']
start_date, end_date = date_preprocess(start_date, end_date)
calendar = date_range(start_date, end_date)
print('calendar', len(calendar))
i_l = 0
result = list()
cl = len(calendar)
for ci in range(cl):
c = calendar[ci]
t = randint(1, 2)
params = {'type': 'zy2', 'date': Calendar.to(str, c, _format='%Y-%m-%d')}
try:
e = crawl(url, params)
html = etree.HTML(e.text)
html_tbody_tr = html.xpath('//tbody//tr/td')
tr_list = [_.text for _ in html_tbody_tr]
for i in range(6):
sub_tr = tr_list[8 * i:8 * (1 + i)]
sub_tr.append(c['date'])
data = obj.trans_data(sub_tr)
result.append(data)
except Exception as e:
print(e)
finally:
params['t'] = t
print(params)
i_l += 1
if i_l > 500 or ci == cl - 1:
print('insert')
obj.insert_batch(result)
i_l = 0
result = list()
sleep(t)
pf = r'{}/{}'.format(project_dir, '{}.txt'.format(m))
with open(pf) as f:
sl = f.read().split('\n')
num = 30
aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num))
)(sl)
for a in aa:
for k in cc:
s.append(
dict(market=m,
category=k,
stock_code_list=a,
start_date=sd,
end_date=ed))
dql = analyzer('date >= {} and date <= {}'.format(
Calendar.to(str,
Calendar.calc(sd, -1)['date']),
Calendar.to(str,
Calendar.calc(ed, -1)['date'])))
for m in mm:
for c in cc:
model_list['{}_kline_{}'.format(m, c)].remove(dql)
# exit()
print(s)
st = datetime.now()
freeze_support()
pool = Pool(7)
pool.map(gt_ex, s)
pool.close()
pool.join()
# for i in s:
def gti(*args):
_ = args[0]
day_feature_index(_[0], _[1])
if __name__ == '__main__':
sd = 20180625
ed = None
dates = list(Calendar().query(
analyzer('date >= {} and date <= {}'.format(sd, ed))))
p = list()
for d in dates:
for k, v in {
'sz': 'stock_code < 600000',
'cyb': 'stock_code >= 300000 and stock_code < 600000',
'sh': 'stock_code >= 600000',
'zxb': 'stock_code >= 002000 and stock_code < 300000'
}.items():
p.append([
analyzer('date = {} and {}'.format(
Calendar.to('str', d['date']), v)), k
])
sql = analyzer('date >= {} and date <= {}'.format(sd, ed))
iobj = model_list['feature_index_day'].remove(sql)
print('done')
freeze_support()
pool = Pool(cpus)
pool.map(gti, p)
pool.close()
pool.join()
def get_ttm(p_type='zz', start_date=None, end_date=None):
start_date, end_date = date_preprocess(start_date, end_date)
calendar = date_range(start_date, end_date)
print('calendar', len(calendar))
for c in calendar:
t = randint(1, 2)
params = {'type': p_type + '2', 'date': Calendar.to(str, c, _format='%Y-%m-%d')}
obj = model_list['fund_{}_ttm'.format(p_type)]
try:
e = crawl(url, params)
html = etree.HTML(e.text)
name_data = html.xpath('//table[@class="list-div-table"]//span')
list_name = [_.text for _ in name_data]
sp_data = html.xpath('//table[@class="list-div-table"]//tbody//tr//td//div//a')
list_sp = [_.text for _ in sp_data]
normal_data = html.xpath('//table[@class="list-div-table"]//tbody//tr//td//div')
list_normal = [_.text for _ in normal_data]
lnl = len(list_normal) / 9
name_loop = 0
sp_loop = 0
for i in range(lnl):
sub_normal = list_normal[9 * i: 9 * (i + 1)]
name = list_name[name_loop]
name_loop += 1
code = sub_normal[0]
cl = len(code)
def sll(sp_loop):
lv = cl / 2
s_n = list_sp[sp_loop]
if s_n is None:
sp_loop += 1
s_n = list_sp[sp_loop]
sp_loop += 1
l_n = list_sp[sp_loop]
sp_loop += 1
return s_n, l_n, lv, sp_loop
# if cl == 2:
# lv = 1
# stock_num = list_sp[sp_loop]
# if stock_num is None:
# sp_loop += 1
# stock_num = list_sp[sp_loop]
# sp_loop += 1
# loss_num = list_sp[sp_loop]
# sp_loop += 1
# elif cl == 4:
# lv = 2
# stock_num = list_sp[sp_loop]
# if stock_num is None:
# sp_loop += 1
# stock_num = list_sp[sp_loop]
# sp_loop += 1
# loss_num = list_sp[sp_loop]
# sp_loop += 1
# elif cl == 6:
# lv = 3
# stock_num = list_sp[sp_loop]
# if stock_num is None:
# sp_loop += 1
# stock_num = list_sp[sp_loop]
# sp_loop += 1
# loss_num = list_sp[sp_loop]
# sp_loop += 1
# elif cl == 8:
# lv = 4
# stock_num = list_sp[sp_loop]
# if stock_num is None:
# sp_loop += 1
# stock_num = list_sp[sp_loop]
# sp_loop += 1
# loss_num = list_sp[sp_loop]
# sp_loop += 1
# else:
# continue
stock_num, loss_num, lv, sp_loop = sll(sp_loop)
data = obj.trans_data([
name,
sub_normal[2],
stock_num,
loss_num,
sub_normal[5],
sub_normal[6],
sub_normal[7],
sub_normal[8],
code,
lv,
c['date']
])
obj.insert(data)
except Exception as e:
print(e)
finally:
params['t'] = t
print(params)
sleep(t)
fmobj = model_list['feature_min5']
fmobj.insert_batch(result)
def ft(_):
fmobj = model_list['feature_min5']
data = list(fmobj.query(_[0]))
model_list['feature_index_min5'].insert_batch(
minute_feature_index(data, _[1]))
print('{} done'.format(_))
if __name__ == '__main__':
mm = ['sz', 'sh']
sd = Calendar.to('datetime', 20180101)
ed = Calendar.today()
# sd = Calendar.today()
s = datetime.now()
p = list()
for m in mm:
sl = get_stock_code_list(markets[m])
num = 30
aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num)))(sl)
for a in aa:
p.append({'sd': sd, 'sl': a})
func = gt
for i in p:
func(i)
p = list()
