def get_index_data(strategy): print('index_in') t = Calendar.today() if Calendar().query(date=t): mm = ['sh', 'sz'] cc = [strategy] strategy = 'index_' + strategy s = list() for m in mm: sl = index_dict[m] num = 10 aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num)) )(sl) for a in aa: for k in cc: s.append( dict(market=m, category=k, stock_code_list=a, start_date=None, end_date=None)) print('item') model_list[strategy].remove(date=t) print(s) pool = Pool(7) pool.map(gt_index, s) # p=multiprocessing.Process(target=gt_index,args=s) # p.start() # p.join(timeout=60) print('index_done')
def check_kline(cls, strategy='kline_min5'): dt = Calendar.today() # pre_date = Calendar.calc(dt, -10) dt = Calendar.calc(dt, -1)['date'] # sql = analyzer('date = {}'.format(str(dt.year * 10000 + dt.month * 100 + dt.day))) # model_list['check_data'].remove(sql) # print(pre_date) sql = analyzer("date >= {} and date <= {}".format( str(20180111), str(dt.year * 10000 + dt.month * 100 + dt.day))) print("____________________" + strategy) print(sql) print('稍等几秒') cursor = model_list[strategy].query(sql) print(cursor) all_count = cursor.count() print(all_count) day_count = cls.number_of_branches(strategy) print(day_count) dic = { 'date': dt, 'strategy': strategy, 'all_count': all_count, 'day_count': day_count, 'result': all_count % day_count == 0 } print('insert') model_list['check_data'].insert(dic) if all_count % day_count == 0: return strategy + ' is true' print("____________________") else: return strategy + "有问题"
def get_kline_data(strategy): # print('kline_in') # file_log(strategy,'time:'+str(datetime.now())+'\n'+'op:start') # t = Calendar.today() print('step into') t = Calendar.today() # 111=555 if Calendar().query(date=t): mm = ['sh', 'sz'] # cc = ['day', 'min60', 'min30', 'min15', 'min5'] cc = [strategy] strategy = 'kline_' + strategy s = list() # ss = datetime.now() # print(ss) for m in mm: sl = get_stock_code_list(markets[m]) num = 10 aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num)) )(sl) print(aa) for a in aa: for k in cc: s.append( dict(market=m, category=k, stock_code_list=a, start_date=None, end_date=None)) # e = datetime.now() # print(e) # file_log(strategy, 'time:' + str(datetime.now()) + '\n' + 'op:get stock_code') print('step getstocks') model_list[strategy].remove(date=t) print('remove over') # file_log(strategy, 'time:' + str(datetime.now()) + '\n' + 'op:remove today') # f = datetime.now() # print(f) pool = Pool(10) pool.map(gt_kline, s) # for i in s: # gt_kline(i) # pool.close() # pool.join() # print('s=',s) # p = multiprocessing.Process(target=gt_kline, args=s) # p.start() # p.join(timeout=300) # for i in s: # gt_kline(i) # model_list[strategy].remove(volume=0) # file_log(strategy, 'time:' + str(datetime.now()) + '\n' + 'op:complate') print('kline_done') return 0
def get_data_allocate(strategy, data_type): dd = Calendar.today() a = Calendar.in_business(dd, day=True) if (a == False): print('非工作日!') exit(0) t1 = time.clock() print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))) command = "python {}\daily_{}_{}.py".format(project_dir, data_type, strategy) list = [] if (strategy == 'min5'): list = get_min5_time() if (strategy == 'min15'): list = get_min15_time() if (strategy == 'min30'): list = get_min30_time() if (strategy == 'min60'): list = get_min60_time() if (strategy == 'day'): list = get_day_time() print(strategy) while True: # if in_business(datetime.now(), strategy, True): x = datetime.now().hour * 60 + datetime.now().minute # print ( datetime.now().hour,datetime.now().minute) # print(x) # if(x>list[-1]): if (x > list[-1]): print(list[-1]) print('收盘!') exit(0) if (x == 11 * 60 + 45): print('上午结束') exit(0) if (x in list): print('finded') t1 = time.clock() subprocess.call(command, shell=True) t3 = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) print(t3) dit = {'date': t3, 'status': 1, 'kline': strategy, 'other': ''} model_list['kline_data_update_mark'].insert(dit) t2 = time.clock() a = (change_status(strategy)['time']) - ((t2 - t1) / 60) print(a) if (a > 0): time.sleep(a * 60) print('睡眠结束') else: print('finding:') time.sleep(10)
def add_plate_event(): data = request.get_json() calendar = Calendar( title=data['title'], user_id=data['user_id'], date=data['date'], plate_id=data['plate_id'], url=data['url'], ) db.session.add(calendar) db.session.commit() return calendar.to_dict()
def get_kline_data(strategy): """ this function acts on kline first get all stock code then call the function named get_kline by multiprocessing :param strategy: :return: """ dd = Calendar.today() if not Calendar.in_business(dt=Calendar.today(), day=True): exit() print('kline_in') # t = Calendar.today() st = None et = None t = Calendar.today() if Calendar().query(date=t): mm = ['sh', 'sz'] #cc = ['day', 'min60', 'min30', 'min15', 'min5'] cc = [strategy] strategy = 'kline_' + strategy s = list() for m in mm: sl = get_stock_code_list(markets[m]) num = 50 aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num)) )(sl) for a in aa: for k in cc: s.append( dict(market=m, category=k, stock_code_list=a, start_date=st, end_date=et)) # if st == None and et == None: # model_list[strategy].remove(date=t) # else: # sql = analyzer('date >= {} and date <= {}'.format(st, et)) # model_list[strategy].remove(sql) pool = multiprocessing.Pool(processes=10) for i in s: pool.apply_async(gt_kline, (i, )) pool.close() pool.join() # for i in s: # gt_kline(i) # model_list[strategy].remove(volume=0) print('-----------------' + str()) print('kline_done')
def gt_kline(*args): # print('gt') _ = args[0] obj = model_list['kline_' + _['category']] t = Calendar.today() data = None # print(os.getpid()) try: data = k_specified_a(market_name=_['market'], category_name=_['category'], stock_code_list=_['stock_code_list'], start_date=_['start_date'], end_date=_['end_date']) except Exception as e: print(e) # if _['category'] == 'min5': # if datetime.now() <= n and datetime.now() >= n2: # obj2 = model_list['tmp_kline_min5'] # obj2.insert_batch(data) # print('ojb2 ok') # Mark.update_mark_success('tmp_kline_min5') # for sc in _['stock_code_list']: # obj.remove(date=t, stock_code=sc) obj.insert_batch(data) # gc.collect() # print('____') print(os.getpid(), _['stock_code_list']) return 0
def self_check(cls, date=Calendar.today()): date = Calendar.calc(date=date, offset=-1)['date'] sql = analyzer('date = {}'.format(date.year * 10000 + date.month * 100 + date.day)) print(sql) result = list(model_list['check_data'].query(sql)) data = pd.DataFrame(result) all_count = list(data['all_count']) for i in range(0, len(all_count) - 1): if i != 4: if all_count[i] % all_count[i + 1] != 0: print(all_count[i], all_count[i + 1]) return False print(data) return True
def addCalendar(): form = AddCalendar() if form.validate_on_submit(): race = Calendar(country=form.country.data, track_name=form.track_name.data, start_date=form.start_date.data, end_date=form.end_date.data) db.session.add(race) db.session.commit() flash('You have successfully added a new race') return render_template('addCalendar.html', title='Add Calendar', form=form)
def to_min(self, stock_code, day): ticks = list( self.query(date=Calendar.to('datetime', day), stock_code=stock_code)) if len(ticks): ticks = sorted(ticks, key=lambda x: (x['time'])) pd_ticks = pd.DataFrame(ticks) pd_ticks['open'] = pd_ticks.price pd_ticks['close'] = pd_ticks.price pd_ticks['high'] = pd_ticks.price pd_ticks['low'] = pd_ticks.price pd_ticks['volume'] *= 100 pd_ticks['amount'] = pd_ticks['price'] * pd_ticks['volume'] x1 = pd_ticks.groupby(by='time').agg({ 'volume': 'sum', 'close': 'last', 'open': 'first', 'high': 'max', 'low': 'min', 'amount': 'sum' }) tmp_data = x1.to_dict() min_data = list() open = tmp_data['open'] close = tmp_data['close'] high = tmp_data['high'] low = tmp_data['low'] volume = tmp_data['volume'] amount = tmp_data['amount'] times = sorted(open.keys()) for t in times: min_data.append({ 'stock_code': stock_code, 'date': day, 'time': int(t), 'open': round(open[t], 2), 'close': round(close[t], 2), 'high': round(high[t], 2), 'low': round(float(low[t]), 2), 'amount': int(amount[t]), 'volume': int(volume[t]), 'market': 1 if stock_code.startswith('60') else 0 }) return min_data else: return []
def initModel(self, obj): if obj == 'calendar': return Calendar() elif obj == 'file': return File() elif obj == 'position': return Position() elif obj == 'client' or obj == 'user': return Client() else: return 0
def get_kline_data(strategy): print('kline_in') t = Calendar.today() if Calendar().query(date=t): mm = ['sh', 'sz'] cc = [strategy] strategy = 'kline_' + strategy s = list() for m in mm: sl = get_stock_code_list(markets[m]) num = 50 aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num)) )(sl) for a in aa: for k in cc: s.append( dict(market=m, category=k, stock_code_list=a, start_date=None, end_date=None)) # model_list[strategy].remove(date=t) # model_list['kline_min15'].remove(date=t) # model_list['kline_min5'].remove(date=t) # model_list['kline_min30'].remove(date=t) # model_list['kline_min60'].remove(date=t) # model_list['kline_day'].remove(date=t) print("((((((((((((") print(len(s)) freeze_support() pool = Pool(5) r = pool.map(gt_kline, s) df = pd.concat(r, join='outer', ignore_index=False) df.reset_index(inplace=True) # print(df.head(10)) pool.close() pool.join() return df
def appendSummaryPeriodDetail(staff,task_or_bug): """ appendTheSummary """ # staff=Staff.objects.get(id=staffID) # task=Task.objects.get(id=taskID) mySummary=task_or_bug.name.strip() start=task_or_bug.begin_date end=task_or_bug.end_date calendars=Calendar.objects.filter(date__range=[start,end],creator=staff).order_by('date').all() for c in calendars: if(c.summary): c.summary+="; "+mySummary else: c.summary=mySummary if calendars and len(calendars)>0: bulk_update(calendars,update_fields=['summary'],batch_size=400) transaction.commit_unless_managed() index=0 length=len(calendars) new_calendar=[] for n in range(int ((end - start).days)+1): date=start + datetime.timedelta(n) lYear = int(date.year) lMonth = int(date.month) lDay=int(date.day) if(index<length and date==calendars[index].date): index=index+1 else: tmp_cal=Calendar() tmp_cal.date=datetime.datetime(lYear, lMonth, lDay, 0) tmp_cal.summary=mySummary tmp_cal.creator=staff new_calendar.append(tmp_cal) if new_calendar and len(new_calendar)>0: Calendar.objects.bulk_create(new_calendar) del new_calendar
def new_user(): name = request.args.get('name') instance = db.session.query(User).filter_by(name=name).first() if instance: return json.dumps(instance.as_dict(), ensure_ascii=False) util = Utilities() id = util.new_uuid() calendar_id = util.new_uuid() calendar = Calendar(id=calendar_id) user = User(id=id, name=name, avatar="") user.calendar = calendar db.session.add(user) db.session.commit() return json.dumps(user.as_dict(), ensure_ascii=False)
def add_cal_to_user(google_id, cal_url, cal_id, filters): """ Adds a user to the database :param google_id: The users Google ID :param cal_url: The url to ical file the user uses :param cal_id: The Google Calendar ID the user uses :param filters: The users filters :param cred: The users credentials :return: """ user = User.query.filter_by(google_id=google_id).first() calendar = Calendar(cal_id=cal_id, cal_url=cal_url, owner=user) db.session.add(calendar) for cal_filter in filters: filter_object = Filter(course_code=cal_filter.course_code, description=cal_filter.description, group_name=cal_filter.group_name, owner=calendar) db.session.add(filter_object) db.session.commit()
def get_zybk_ttm(start_date=None, end_date=None): obj = model_list['fund_zybk_ttm'] start_date, end_date = date_preprocess(start_date, end_date) calendar = date_range(start_date, end_date) print('calendar', len(calendar)) i_l = 0 result = list() cl = len(calendar) for ci in range(cl): c = calendar[ci] t = randint(1, 2) params = {'type': 'zy2', 'date': Calendar.to(str, c, _format='%Y-%m-%d')} try: e = crawl(url, params) html = etree.HTML(e.text) html_tbody_tr = html.xpath('//tbody//tr/td') tr_list = [_.text for _ in html_tbody_tr] for i in range(6): sub_tr = tr_list[8 * i:8 * (1 + i)] sub_tr.append(c['date']) data = obj.trans_data(sub_tr) result.append(data) except Exception as e: print(e) finally: params['t'] = t print(params) i_l += 1 if i_l > 500 or ci == cl - 1: print('insert') obj.insert_batch(result) i_l = 0 result = list() sleep(t)
def get_ttm(p_type='zz', start_date=None, end_date=None): start_date, end_date = date_preprocess(start_date, end_date) calendar = date_range(start_date, end_date) print('calendar', len(calendar)) for c in calendar: t = randint(1, 2) params = {'type': p_type + '2', 'date': Calendar.to(str, c, _format='%Y-%m-%d')} obj = model_list['fund_{}_ttm'.format(p_type)] try: e = crawl(url, params) html = etree.HTML(e.text) name_data = html.xpath('//table[@class="list-div-table"]//span') list_name = [_.text for _ in name_data] sp_data = html.xpath('//table[@class="list-div-table"]//tbody//tr//td//div//a') list_sp = [_.text for _ in sp_data] normal_data = html.xpath('//table[@class="list-div-table"]//tbody//tr//td//div') list_normal = [_.text for _ in normal_data] lnl = len(list_normal) / 9 name_loop = 0 sp_loop = 0 for i in range(lnl): sub_normal = list_normal[9 * i: 9 * (i + 1)] name = list_name[name_loop] name_loop += 1 code = sub_normal[0] cl = len(code) def sll(sp_loop): lv = cl / 2 s_n = list_sp[sp_loop] if s_n is None: sp_loop += 1 s_n = list_sp[sp_loop] sp_loop += 1 l_n = list_sp[sp_loop] sp_loop += 1 return s_n, l_n, lv, sp_loop # if cl == 2: # lv = 1 # stock_num = list_sp[sp_loop] # if stock_num is None: # sp_loop += 1 # stock_num = list_sp[sp_loop] # sp_loop += 1 # loss_num = list_sp[sp_loop] # sp_loop += 1 # elif cl == 4: # lv = 2 # stock_num = list_sp[sp_loop] # if stock_num is None: # sp_loop += 1 # stock_num = list_sp[sp_loop] # sp_loop += 1 # loss_num = list_sp[sp_loop] # sp_loop += 1 # elif cl == 6: # lv = 3 # stock_num = list_sp[sp_loop] # if stock_num is None: # sp_loop += 1 # stock_num = list_sp[sp_loop] # sp_loop += 1 # loss_num = list_sp[sp_loop] # sp_loop += 1 # elif cl == 8: # lv = 4 # stock_num = list_sp[sp_loop] # if stock_num is None: # sp_loop += 1 # stock_num = list_sp[sp_loop] # sp_loop += 1 # loss_num = list_sp[sp_loop] # sp_loop += 1 # else: # continue stock_num, loss_num, lv, sp_loop = sll(sp_loop) data = obj.trans_data([ name, sub_normal[2], stock_num, loss_num, sub_normal[5], sub_normal[6], sub_normal[7], sub_normal[8], code, lv, c['date'] ]) obj.insert(data) except Exception as e: print(e) finally: params['t'] = t print(params) sleep(t)
# -*- coding: utf-8 -*- from os.path import abspath, pardir, join from sys import path from app import markets from app.models import Calendar from app.actions import get_stock_code_list, get_xdxr path.append(abspath(join(abspath(join(abspath(__file__), pardir)), pardir))) if __name__ == '__main__': if not Calendar.in_business(dt=Calendar.today(), day=True): exit() start_date = None end_date = None stock_code_list = [] for m in ['sh', 'sz']: stock_code_list = get_stock_code_list(market=markets[m]) get_xdxr(market_name=m, stock_code_list=stock_code_list, start_date=start_date, end_date=end_date)
fmobj = model_list['feature_min5'] fmobj.insert_batch(result) def ft(_): fmobj = model_list['feature_min5'] data = list(fmobj.query(_[0])) model_list['feature_index_min5'].insert_batch( minute_feature_index(data, _[1])) print('{} done'.format(_)) if __name__ == '__main__': mm = ['sz', 'sh'] sd = Calendar.to('datetime', 20180101) ed = Calendar.today() # sd = Calendar.today() s = datetime.now() p = list() for m in mm: sl = get_stock_code_list(markets[m]) num = 30 aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num)))(sl) for a in aa: p.append({'sd': sd, 'sl': a}) func = gt for i in p: func(i) p = list()
k_specified_ex(market_name=_['market'], category_name=_['category'], stock_code_list=_['stock_code_list'], start_date=_['start_date'], end_date=_['end_date'])) # k_specified_a(market_name=_['market'], category_name=_['category'], stock_code_list=_['stock_code_list'], # start_date=_['start_date'], end_date=_['end_date']) # k_history_a(market_name=_['market'], category_name=_['category'], stock_code_list=_['stock_code_list'], # start_date=_['start_date'], end_date=_['end_date']) # obj.insert_batch(k_realtime_a(market_name=_['market'], category_name=_['category'], stock_code_list=_['stock_code_list'])) print(_['stock_code_list']) if __name__ == '__main__': print('in') t = Calendar.today() if Calendar().query(date=t): mm = ['usa'] cc = ['day', 'min60', 'min30', 'min15', 'min5'] # cc = ['min5'] # cc = ['day'] s = list() # sd = Calendar.today() sd = analyzer('date = {}'.format(20170101))['date'] ed = Calendar.today() for m in mm: pf = r'{}/{}'.format(project_dir, '{}.txt'.format(m)) with open(pf) as f: sl = f.read().split('\n') num = 30 aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num))
from app import cpus from app.actions import day_feature_index from app.models import Calendar, model_list from app.query_str_analyzer import analyzer def gti(*args): _ = args[0] day_feature_index(_[0], _[1]) if __name__ == '__main__': sd = 20180625 ed = None dates = list(Calendar().query( analyzer('date >= {} and date <= {}'.format(sd, ed)))) p = list() for d in dates: for k, v in { 'sz': 'stock_code < 600000', 'cyb': 'stock_code >= 300000 and stock_code < 600000', 'sh': 'stock_code >= 600000', 'zxb': 'stock_code >= 002000 and stock_code < 300000' }.items(): p.append([ analyzer('date = {} and {}'.format( Calendar.to('str', d['date']), v)), k ]) sql = analyzer('date >= {} and date <= {}'.format(sd, ed)) iobj = model_list['feature_index_day'].remove(sql) print('done')
def gt(*args): _ = args[0] obj = model_list['XDXR_' + _['category']] obj.insert_batch( calc_xdxr_specified(stock_code_list=_['stock_code_list'], category_name=_['category'], start_date=_['start_date'], end_date=_['end_date'])) # calc_xdxr_specified(stock_code_list=[600015], category_name=_['category'], start_date=_['start_date'], # end_date=_['end_date']) if __name__ == '__main__': d = Calendar.today() if not Calendar.in_business(dt=d, day=True): print('not_ok') exit() print('0000') strategy = 'XDXR_day' # day_XDXR ���ܸIJ��ܸIJ��ܸ�Ӵ try: print('in') mm = ['sz', 'sh'] s = list() sd = None ed = None for m in mm: sl = get_stock_code_list(markets[m]) num = 10 aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num))
from app.models import Calendar, model_list def gt(*args): _ = args[0] i_history_a(market_name=_['market'], category_name=_['category'], start_date=_['start_date'], end_date=_['end_date'], stock_code_list=_['stock_code_list']) print(args, 'ok') if __name__ == '__main__': print('in') t = Calendar.today() st = 20180604 et = None if Calendar().query(date=t): mm = ['sh', 'sz'] cc = ['day', 'min60', 'min30', 'min15', 'min5'] cc = ['min5'] s = list() for m in mm: sl = index_dict[m] num = 10 aa = (lambda a: map(lambda b: a[b:b + num], range(0, len(a), num)) )(sl) for k in cc: for a in aa: s.append(
from multiprocessing import freeze_support, Pool from app import cpus from app.actions import day_feature_index from app.models import Calendar, model_list from app.query_str_analyzer import analyzer def gti(*args): _ = args[0] day_feature_index(_[0], _[1]) # # daily if __name__ == '__main__': if not Calendar.in_business(dt=Calendar.today(), day=True): exit() p = list() d = None c = list(Calendar().asc(Calendar().query(analyzer('date = {}'.format(d))), ['num'])) for cc in c: for k, v in { 'sz': 'stock_code < 600000', 'cyb': 'stock_code >= 300000 and stock_code < 600000', 'sh': 'stock_code >= 600000', 'zxb': 'stock_code >= 002000 and stock_code < 300000' }.items(): p.append([ analyzer('date = {} and {}'.format(
# -*- coding: utf-8 -*- from app.actions import f_zjh_ttm_a, f_zybk_ttm_a, f_zz_ttm_a from app.models import model_list, Calendar from app.query_str_analyzer import analyzer def change_date(t): print(t.year * 10000 + t.month * 100 + t.day) return t.year * 10000 + t.month * 100 + t.day if __name__ == '__main__': t = Calendar.today() print('t', t) t2 = Calendar.calc(t, -1)['date'] print('t2', t2) sd = change_date(t2) ed = change_date(t) dql = analyzer('date >= {} and date <= {}'.format(sd, ed)) for i in ['fund_zz_ttm', 'fund_zjh_ttm', 'fund_zybk_ttm']: model_list[i].remove(dql) f_zz_ttm_a(start_date=sd, end_date=ed) f_zjh_ttm_a(p_type='zjh', start_date=sd, end_date=ed) f_zybk_ttm_a(start_date=sd, end_date=ed)
def add_calendar(): data = request.get_json() calendar = Calendar() db.session.add(calendar) db.session.commit() return jsonify(status='OK')
print(sql) result = list(model_list['check_data'].query(sql)) data = pd.DataFrame(result) all_count = list(data['all_count']) for i in range(0, len(all_count) - 1): if i != 4: if all_count[i] % all_count[i + 1] != 0: print(all_count[i], all_count[i + 1]) return False print(data) return True dt = Calendar.today() # pre_date = Calendar.calc(dt, -10) dt = Calendar.calc(dt, -1)['date'] sql = analyzer('date = {}'.format( str(dt.year * 10000 + dt.month * 100 + dt.day))) model_list['check_data'].remove(sql) print(Check_Data.check_kline('kline_min5')) print(Check_Data.check_kline('kline_min15')) print(Check_Data.check_kline('kline_min30')) print(Check_Data.check_kline('kline_min60')) print(Check_Data.check_kline('kline_day')) print(Check_Data.check_kline('index_min5')) print(Check_Data.check_kline('index_min15')) print(Check_Data.check_kline('index_min30')) print(Check_Data.check_kline('index_min60')) print(Check_Data.check_kline('index_day'))
# -*- coding: utf-8 -*- from app.models import model_list, Calendar if __name__ == '__main__': cbj = model_list['tmp_kline_min5'] cbj.remove_all() t = Calendar.today() kbj = model_list['kline_min5'] # kbj.copyto(cbj, condition={'date': Calendar.calc(t, -1)['date']}) # 补数据 kbj.copyto(cbj, condition={'date': t}) print('done')