def load(self, start, end, freq, ascending, limit=0): helper = KlinedbHelper.load_db_helper(freq) if start is not None: start = trade_calendar.create_v2(start, freq) if end is not None: end = trade_calendar.create_v2(end, freq) df = helper.load(self.symbol, start, end, limit=limit) if isinstance(df, pd.DataFrame) and df.empty == False: df = df.sort_values(by='datetime', ascending=ascending) return df.reset_index(drop=True) else: return None
def request_kline(symbol, freq , **kwargs): query = kline.KlineQuery(symbol) log.debug('begin to request [%s]' % symbol) if query.isEmpty(freq): ## TODO request all info = composite.get_basic().info(symbol) if info is None: raise ValueError('%s not exist'% symbol) born = info['timeToMarket'] return request_kline_core(symbol, start_date=trade_calendar.create_v2(born, freq), end_date = trade_calendar.last_trade_date(freq), freq = freq, **kwargs) else: db_start, db_end = query.duration(freq) if db_start is None and db_end is None: log.error('duration in db is error') raise RuntimeError('[%s] isn\'t exist, but duration ???' % symbol) last_date = trade_calendar.last_trade_date(freq) if trade_calendar.compare_v2(db_end, last_date, freq) == 0: ## TODO check suspend stock log.info('%s needn\'t update' % symbol) return 1 elif trade_calendar.compare_v2(db_end, last_date, freq) < 0: db_end.shift(count = 1) return request_kline_core(symbol, db_end, last_date, freq, **kwargs) else: raise RuntimeError("????")
def get_count_bars(code, last, limit, freq, ascending = False): query = kline.KlineQuery(code) if last is None: end = trade_calendar.last_trade_date(freq) else: end = trade_calendar.create_v2(last, freq) return query.load(None, end, freq, ascending = ascending, limit = limit)
def test_shift(): v1 = trade_calendar.create_v2("2018-02-10", freq=util.FREQ_DAY) v1.shift(1) assert v1.datetime().year == 2018 and v1.datetime( ).month == 2 and v1.datetime().day == 12 v1.shift(3) assert v1.datetime().year == 2018 and v1.datetime( ).month == 2 and v1.datetime().day == 22
def __append_or_replace(self, src, des): opt = self.NONE START = 0 END = 1 src = list(map(lambda x: trade_calendar.create_v2(x, self.freq), src)) des = list(map(lambda x: trade_calendar.create_v2(x, self.freq), des)) if src[START] is None or src[END] is None or (des[START] < src[START] and des[END] > des[END]): opt = self.REPLACE elif des[START] is None or des[END] is None or ( des[START] > src[START] and des[END] < src[END]): opt = self.NONE elif des[START] < src[START] and des[END] > src[START]: ## TODO have no solution to do opt = self.NONE elif des[START] > src[END]: opt = self.APPEND return opt
def day_sync_time(name): now = arrow.now() r = trade_calendar.create_v2(now, freq = util.FREQ_DAY) v = r.datetime() if v.date() == now.datetime.date() and now.hour <= 16: log.error('request %s need after 16:00:00' % name) return False else: return True
def __append(self, symbol, start, end, df): log.info('append %s from %s to %s' % (symbol, start, end)) klines = util.to_json(df) for k in klines: k['datetime'] = trade_calendar.create_v2(k['datetime'], self.freq).datetime() cursor = self.table.update({'_id': symbol}, { '$set':{'info.end_date': end.datetime()}, \ ## TODO waiting for $position '$push': {'kline': {'$each':klines}}}, upsert=True)
def __sync_day_kline(symbols, start, end): freq = util.FREQ_DAY s = trade_calendar.create_v2(start, freq) e = trade_calendar.create_v2(end, freq) s.shift(1) success = True log.info("merge from %s -> %s" % (s, e)) ## check we can use daily csv to merge daily_merge = True _iter = trade_calendar.create_v2(s, freq) while _iter <= e: if not TodayAllDailyKline.has_cache(_iter): daily_merge = False break else: _iter.shift(1) ## use daily merge if daily_merge: while trade_calendar.compare_v2(e, s, freq = freq) >= 0: if TodayAllDailyKline.has_cache(s): today = TodayAllDailyKline(s) for symbol in symbols: k = today.trade(symbol) if k is not None: kline.append_kline(symbol, s, s, freq = util.FREQ_DAY, df = k) else: log.info('{} with no trade in {}'.format(symbol, s)) else: raise RuntimeError('have no daily data %s' % s.format()) s.shift(1) kline.set_sync_result(True, freq, e) else: request_symbols_kline(symbols, freq) log.info('no error')
def __replace(self, symbol, start, end, df): log.info('replace %s from %s to %s' % (symbol, start, end)) info = {'symbol': symbol, 'start_date':start.datetime(),\ 'end_date':end.datetime()} klines = util.to_json(df) for k in klines: k['datetime'] = trade_calendar.create_v2(k['datetime'], self.freq).datetime() self.table.update({'_id': symbol}, { '$set':{'info': info}, \ ## TODO sort operation '$setOnInsert':{'kline': klines} \ }, upsert=True) log.debug('replace end')
def test_trade_calendarv2(): v1 = trade_calendar.create_v2('2018-01-01', util.FREQ_DAY) assert (v1.shift(1) == trade_calendar.create_v2('2018-01-02', util.FREQ_DAY)) assert (v1.shift(-4) == trade_calendar.create_v2('2017-12-26', util.FREQ_DAY)) v2 = trade_calendar.create_v2('2018-01-01 10:30:00', util.FREQ_30M) assert (v2.shift(1) == trade_calendar.create_v2('2018-01-02 09:30:00', util.FREQ_30M)) assert (v2.shift(-4) == trade_calendar.create_v2('2017-12-29 13:30:00', util.FREQ_30M)) assert trade_calendar.compare_v2('2018-01-01', '2017-12-31', util.FREQ_DAY) == 0
def test_tradecalendar_create(): v1 = trade_calendar.create_v2("2018-01-01", freq=util.FREQ_DAY) assert v1.datetime().year == 2017 and v1.datetime( ).month == 12 and v1.datetime().day == 29
def test_load(): helper = kline.KlinedbHelper.load_db_helper('DAY') start = trade_calendar.create_v2('2018-01-10', util.FREQ_DAY) end = trade_calendar.create_v2('2018-03-10', util.FREQ_DAY) k = helper.load('000002', start, end) assert k.empty is False
def duration(self, freq): helper = KlinedbHelper.load_db_helper(freq) start, end = helper.duration(self.symbol) return trade_calendar.create_v2(start, freq), trade_calendar.create_v2( end, freq)
def append_kline(symbol, start_date, end_date, freq, df): helper = KlinedbHelper.load_db_helper(freq) ## TODO add more check helper.save(symbol, trade_calendar.create_v2(start_date, freq), trade_calendar.create_v2(end_date, freq), df)
def date(self): self.__load() if self.__date is None: return None return trade_calendar.create_v2(self.__date, util.FREQ_DAY)