def to_min(self, stock_code, day): ticks = list( self.query(date=Calendar.to('datetime', day), stock_code=stock_code)) if len(ticks): ticks = sorted(ticks, key=lambda x: (x['time'])) pd_ticks = pd.DataFrame(ticks) pd_ticks['open'] = pd_ticks.price pd_ticks['close'] = pd_ticks.price pd_ticks['high'] = pd_ticks.price pd_ticks['low'] = pd_ticks.price pd_ticks['volume'] *= 100 pd_ticks['amount'] = pd_ticks['price'] * pd_ticks['volume'] x1 = pd_ticks.groupby(by='time').agg({ 'volume': 'sum', 'close': 'last', 'open': 'first', 'high': 'max', 'low': 'min', 'amount': 'sum' }) tmp_data = x1.to_dict() min_data = list() open = tmp_data['open'] close = tmp_data['close'] high = tmp_data['high'] low = tmp_data['low'] volume = tmp_data['volume'] amount = tmp_data['amount'] times = sorted(open.keys()) for t in times: min_data.append({ 'stock_code': stock_code, 'date': day, 'time': int(t), 'open': round(open[t], 2), 'close': round(close[t], 2), 'high': round(high[t], 2), 'low': round(float(low[t]), 2), 'amount': int(amount[t]), 'volume': int(volume[t]), 'market': 1 if stock_code.startswith('60') else 0 }) return min_data else: return []
def get_zybk_ttm(start_date=None, end_date=None): obj = model_list['fund_zybk_ttm'] start_date, end_date = date_preprocess(start_date, end_date) calendar = date_range(start_date, end_date) print('calendar', len(calendar)) i_l = 0 result = list() cl = len(calendar) for ci in range(cl): c = calendar[ci] t = randint(1, 2) params = {'type': 'zy2', 'date': Calendar.to(str, c, _format='%Y-%m-%d')} try: e = crawl(url, params) html = etree.HTML(e.text) html_tbody_tr = html.xpath('//tbody//tr/td') tr_list = [_.text for _ in html_tbody_tr] for i in range(6): sub_tr = tr_list[8 * i:8 * (1 + i)] sub_tr.append(c['date']) data = obj.trans_data(sub_tr) result.append(data) except Exception as e: print(e) finally: params['t'] = t print(params) i_l += 1 if i_l > 500 or ci == cl - 1: print('insert') obj.insert_batch(result) i_l = 0 result = list() sleep(t)
pf = r'{}/{}'.format(project_dir, '{}.txt'.format(m)) with open(pf) as f: sl = f.read().split('\n') num = 30 aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num)) )(sl) for a in aa: for k in cc: s.append( dict(market=m, category=k, stock_code_list=a, start_date=sd, end_date=ed)) dql = analyzer('date >= {} and date <= {}'.format( Calendar.to(str, Calendar.calc(sd, -1)['date']), Calendar.to(str, Calendar.calc(ed, -1)['date']))) for m in mm: for c in cc: model_list['{}_kline_{}'.format(m, c)].remove(dql) # exit() print(s) st = datetime.now() freeze_support() pool = Pool(7) pool.map(gt_ex, s) pool.close() pool.join() # for i in s:
def gti(*args): _ = args[0] day_feature_index(_[0], _[1]) if __name__ == '__main__': sd = 20180625 ed = None dates = list(Calendar().query( analyzer('date >= {} and date <= {}'.format(sd, ed)))) p = list() for d in dates: for k, v in { 'sz': 'stock_code < 600000', 'cyb': 'stock_code >= 300000 and stock_code < 600000', 'sh': 'stock_code >= 600000', 'zxb': 'stock_code >= 002000 and stock_code < 300000' }.items(): p.append([ analyzer('date = {} and {}'.format( Calendar.to('str', d['date']), v)), k ]) sql = analyzer('date >= {} and date <= {}'.format(sd, ed)) iobj = model_list['feature_index_day'].remove(sql) print('done') freeze_support() pool = Pool(cpus) pool.map(gti, p) pool.close() pool.join()
def get_ttm(p_type='zz', start_date=None, end_date=None): start_date, end_date = date_preprocess(start_date, end_date) calendar = date_range(start_date, end_date) print('calendar', len(calendar)) for c in calendar: t = randint(1, 2) params = {'type': p_type + '2', 'date': Calendar.to(str, c, _format='%Y-%m-%d')} obj = model_list['fund_{}_ttm'.format(p_type)] try: e = crawl(url, params) html = etree.HTML(e.text) name_data = html.xpath('//table[@class="list-div-table"]//span') list_name = [_.text for _ in name_data] sp_data = html.xpath('//table[@class="list-div-table"]//tbody//tr//td//div//a') list_sp = [_.text for _ in sp_data] normal_data = html.xpath('//table[@class="list-div-table"]//tbody//tr//td//div') list_normal = [_.text for _ in normal_data] lnl = len(list_normal) / 9 name_loop = 0 sp_loop = 0 for i in range(lnl): sub_normal = list_normal[9 * i: 9 * (i + 1)] name = list_name[name_loop] name_loop += 1 code = sub_normal[0] cl = len(code) def sll(sp_loop): lv = cl / 2 s_n = list_sp[sp_loop] if s_n is None: sp_loop += 1 s_n = list_sp[sp_loop] sp_loop += 1 l_n = list_sp[sp_loop] sp_loop += 1 return s_n, l_n, lv, sp_loop # if cl == 2: # lv = 1 # stock_num = list_sp[sp_loop] # if stock_num is None: # sp_loop += 1 # stock_num = list_sp[sp_loop] # sp_loop += 1 # loss_num = list_sp[sp_loop] # sp_loop += 1 # elif cl == 4: # lv = 2 # stock_num = list_sp[sp_loop] # if stock_num is None: # sp_loop += 1 # stock_num = list_sp[sp_loop] # sp_loop += 1 # loss_num = list_sp[sp_loop] # sp_loop += 1 # elif cl == 6: # lv = 3 # stock_num = list_sp[sp_loop] # if stock_num is None: # sp_loop += 1 # stock_num = list_sp[sp_loop] # sp_loop += 1 # loss_num = list_sp[sp_loop] # sp_loop += 1 # elif cl == 8: # lv = 4 # stock_num = list_sp[sp_loop] # if stock_num is None: # sp_loop += 1 # stock_num = list_sp[sp_loop] # sp_loop += 1 # loss_num = list_sp[sp_loop] # sp_loop += 1 # else: # continue stock_num, loss_num, lv, sp_loop = sll(sp_loop) data = obj.trans_data([ name, sub_normal[2], stock_num, loss_num, sub_normal[5], sub_normal[6], sub_normal[7], sub_normal[8], code, lv, c['date'] ]) obj.insert(data) except Exception as e: print(e) finally: params['t'] = t print(params) sleep(t)
fmobj = model_list['feature_min5'] fmobj.insert_batch(result) def ft(_): fmobj = model_list['feature_min5'] data = list(fmobj.query(_[0])) model_list['feature_index_min5'].insert_batch( minute_feature_index(data, _[1])) print('{} done'.format(_)) if __name__ == '__main__': mm = ['sz', 'sh'] sd = Calendar.to('datetime', 20180101) ed = Calendar.today() # sd = Calendar.today() s = datetime.now() p = list() for m in mm: sl = get_stock_code_list(markets[m]) num = 30 aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num)))(sl) for a in aa: p.append({'sd': sd, 'sl': a}) func = gt for i in p: func(i) p = list()