def request_kline(symbol, freq , **kwargs):
query = kline.KlineQuery(symbol)
log.debug('begin to request [%s]' % symbol)
if query.isEmpty(freq):
## TODO request all
info = composite.get_basic().info(symbol)
if info is None:
raise ValueError('%s not exist'% symbol)
born = info['timeToMarket']
return request_kline_core(symbol, start_date=trade_calendar.create_v2(born, freq), end_date = trade_calendar.last_trade_date(freq), freq = freq, **kwargs)
else:
db_start, db_end = query.duration(freq)
if db_start is None and db_end is None:
log.error('duration in db is error')
raise RuntimeError('[%s] isn\'t exist, but duration ???' % symbol)
last_date = trade_calendar.last_trade_date(freq)
if trade_calendar.compare_v2(db_end, last_date, freq) == 0:
## TODO check suspend stock
log.info('%s needn\'t update' % symbol)
return 1
elif trade_calendar.compare_v2(db_end, last_date, freq) < 0:
db_end.shift(count = 1)
return request_kline_core(symbol, db_end, last_date, freq, **kwargs)
else:
raise RuntimeError("????")
def newest(self):
dbdate = self.date()
if dbdate is None or self.__df is None:
return False
else:
return trade_calendar.compare_v2(dbdate,
trade_calendar.last_trade_date(
util.FREQ_DAY),
freq=util.FREQ_DAY) == 0
def request_kline_core(symbol, start_date, end_date, freq = util.FREQ_DAY , **kwargs):
## check start < end
if trade_calendar.compare_v2(start_date, end_date, freq) > 0:
log.warn('invalid {} -> {}'.format(start_date, end_date))
return -1
else:
df = request.request_stock_bar(symbol, start = start_date, end = end_date, freq = freq)
if not df.empty:
log.info('request %s [%s -> %s] success'% (symbol, start_date, end_date))
kline.append_kline(symbol, start_date, end_date, freq, df)
else:
log.warn('request %s empty'% (symbol))
return 0
def test_trade_calendarv2():
v1 = trade_calendar.create_v2('2018-01-01', util.FREQ_DAY)
assert (v1.shift(1) == trade_calendar.create_v2('2018-01-02',
util.FREQ_DAY))
assert (v1.shift(-4) == trade_calendar.create_v2('2017-12-26',
util.FREQ_DAY))
v2 = trade_calendar.create_v2('2018-01-01 10:30:00', util.FREQ_30M)
assert (v2.shift(1) == trade_calendar.create_v2('2018-01-02 09:30:00',
util.FREQ_30M))
assert (v2.shift(-4) == trade_calendar.create_v2('2017-12-29 13:30:00',
util.FREQ_30M))
assert trade_calendar.compare_v2('2018-01-01', '2017-12-31',
util.FREQ_DAY) == 0
def sync_history_bar(symbols, freq):
ret = kline.get_sync_result(freq)
end = trade_calendar.last_trade_date(freq)
if ret is not None and ret.get('result', False) == True and \
ret.get('date') is not None and trade_calendar.compare_v2(ret.get('date'), end, freq) >= 0:
log.info('need\'t update')
else:
if freq == util.FREQ_DAY:
if ret is None or ret.get('date') is None:
## no date in db , request all data
log.info("request symbol list")
request_symbols_kline(symbols, freq)
else:
request_all_today_kline()
__sync_day_kline(symbols, ret.get('date'), end)
elif freq == util.FREQ_WEEK:
now = datetime.datetime.now()
if now.weekday() >= 5: ## rest day
request_symbols_kline(symbols, freq)
else:
log.info('request weekly data in rest day ignore freq week')
else:
## TODO
request_symbols_kline(symbols, freq)
def __sync_day_kline(symbols, start, end):
freq = util.FREQ_DAY
s = trade_calendar.create_v2(start, freq)
e = trade_calendar.create_v2(end, freq)
s.shift(1)
success = True
log.info("merge from %s -> %s" % (s, e))
## check we can use daily csv to merge
daily_merge = True
_iter = trade_calendar.create_v2(s, freq)
while _iter <= e:
if not TodayAllDailyKline.has_cache(_iter):
daily_merge = False
break
else:
_iter.shift(1)
## use daily merge
if daily_merge:
while trade_calendar.compare_v2(e, s, freq = freq) >= 0:
if TodayAllDailyKline.has_cache(s):
today = TodayAllDailyKline(s)
for symbol in symbols:
k = today.trade(symbol)
if k is not None:
kline.append_kline(symbol, s, s, freq = util.FREQ_DAY, df = k)
else:
log.info('{} with no trade in {}'.format(symbol, s))
else:
raise RuntimeError('have no daily data %s' % s.format())
s.shift(1)
kline.set_sync_result(True, freq, e)
else:
request_symbols_kline(symbols, freq)
log.info('no error')