def load(self, start, end, freq, ascending, limit=0):
helper = KlinedbHelper.load_db_helper(freq)
if start is not None:
start = trade_calendar.create_v2(start, freq)
if end is not None:
end = trade_calendar.create_v2(end, freq)
df = helper.load(self.symbol, start, end, limit=limit)
if isinstance(df, pd.DataFrame) and df.empty == False:
df = df.sort_values(by='datetime', ascending=ascending)
return df.reset_index(drop=True)
else:
return None
def request_kline(symbol, freq , **kwargs):
query = kline.KlineQuery(symbol)
log.debug('begin to request [%s]' % symbol)
if query.isEmpty(freq):
## TODO request all
info = composite.get_basic().info(symbol)
if info is None:
raise ValueError('%s not exist'% symbol)
born = info['timeToMarket']
return request_kline_core(symbol, start_date=trade_calendar.create_v2(born, freq), end_date = trade_calendar.last_trade_date(freq), freq = freq, **kwargs)
else:
db_start, db_end = query.duration(freq)
if db_start is None and db_end is None:
log.error('duration in db is error')
raise RuntimeError('[%s] isn\'t exist, but duration ???' % symbol)
last_date = trade_calendar.last_trade_date(freq)
if trade_calendar.compare_v2(db_end, last_date, freq) == 0:
## TODO check suspend stock
log.info('%s needn\'t update' % symbol)
return 1
elif trade_calendar.compare_v2(db_end, last_date, freq) < 0:
db_end.shift(count = 1)
return request_kline_core(symbol, db_end, last_date, freq, **kwargs)
else:
raise RuntimeError("????")
def get_count_bars(code, last, limit, freq, ascending = False):
query = kline.KlineQuery(code)
if last is None:
end = trade_calendar.last_trade_date(freq)
else:
end = trade_calendar.create_v2(last, freq)
return query.load(None, end, freq, ascending = ascending, limit = limit)
def test_shift():
v1 = trade_calendar.create_v2("2018-02-10", freq=util.FREQ_DAY)
v1.shift(1)
assert v1.datetime().year == 2018 and v1.datetime(
).month == 2 and v1.datetime().day == 12
v1.shift(3)
assert v1.datetime().year == 2018 and v1.datetime(
).month == 2 and v1.datetime().day == 22
def __append_or_replace(self, src, des):
opt = self.NONE
START = 0
END = 1
src = list(map(lambda x: trade_calendar.create_v2(x, self.freq), src))
des = list(map(lambda x: trade_calendar.create_v2(x, self.freq), des))
if src[START] is None or src[END] is None or (des[START] < src[START]
and des[END] > des[END]):
opt = self.REPLACE
elif des[START] is None or des[END] is None or (
des[START] > src[START] and des[END] < src[END]):
opt = self.NONE
elif des[START] < src[START] and des[END] > src[START]:
## TODO have no solution to do
opt = self.NONE
elif des[START] > src[END]:
opt = self.APPEND
return opt
def day_sync_time(name):
now = arrow.now()
r = trade_calendar.create_v2(now, freq = util.FREQ_DAY)
v = r.datetime()
if v.date() == now.datetime.date() and now.hour <= 16:
log.error('request %s need after 16:00:00' % name)
return False
else:
return True
def __append(self, symbol, start, end, df):
log.info('append %s from %s to %s' % (symbol, start, end))
klines = util.to_json(df)
for k in klines:
k['datetime'] = trade_calendar.create_v2(k['datetime'],
self.freq).datetime()
cursor = self.table.update({'_id': symbol}, {
'$set':{'info.end_date': end.datetime()}, \
## TODO waiting for $position
'$push': {'kline': {'$each':klines}}}, upsert=True)
def __sync_day_kline(symbols, start, end):
freq = util.FREQ_DAY
s = trade_calendar.create_v2(start, freq)
e = trade_calendar.create_v2(end, freq)
s.shift(1)
success = True
log.info("merge from %s -> %s" % (s, e))
## check we can use daily csv to merge
daily_merge = True
_iter = trade_calendar.create_v2(s, freq)
while _iter <= e:
if not TodayAllDailyKline.has_cache(_iter):
daily_merge = False
break
else:
_iter.shift(1)
## use daily merge
if daily_merge:
while trade_calendar.compare_v2(e, s, freq = freq) >= 0:
if TodayAllDailyKline.has_cache(s):
today = TodayAllDailyKline(s)
for symbol in symbols:
k = today.trade(symbol)
if k is not None:
kline.append_kline(symbol, s, s, freq = util.FREQ_DAY, df = k)
else:
log.info('{} with no trade in {}'.format(symbol, s))
else:
raise RuntimeError('have no daily data %s' % s.format())
s.shift(1)
kline.set_sync_result(True, freq, e)
else:
request_symbols_kline(symbols, freq)
log.info('no error')
def __replace(self, symbol, start, end, df):
log.info('replace %s from %s to %s' % (symbol, start, end))
info = {'symbol': symbol, 'start_date':start.datetime(),\
'end_date':end.datetime()}
klines = util.to_json(df)
for k in klines:
k['datetime'] = trade_calendar.create_v2(k['datetime'],
self.freq).datetime()
self.table.update({'_id': symbol}, {
'$set':{'info': info}, \
## TODO sort operation
'$setOnInsert':{'kline': klines} \
}, upsert=True)
log.debug('replace end')
def test_trade_calendarv2():
v1 = trade_calendar.create_v2('2018-01-01', util.FREQ_DAY)
assert (v1.shift(1) == trade_calendar.create_v2('2018-01-02',
util.FREQ_DAY))
assert (v1.shift(-4) == trade_calendar.create_v2('2017-12-26',
util.FREQ_DAY))
v2 = trade_calendar.create_v2('2018-01-01 10:30:00', util.FREQ_30M)
assert (v2.shift(1) == trade_calendar.create_v2('2018-01-02 09:30:00',
util.FREQ_30M))
assert (v2.shift(-4) == trade_calendar.create_v2('2017-12-29 13:30:00',
util.FREQ_30M))
assert trade_calendar.compare_v2('2018-01-01', '2017-12-31',
util.FREQ_DAY) == 0
def test_tradecalendar_create():
v1 = trade_calendar.create_v2("2018-01-01", freq=util.FREQ_DAY)
assert v1.datetime().year == 2017 and v1.datetime(
).month == 12 and v1.datetime().day == 29
def test_load():
helper = kline.KlinedbHelper.load_db_helper('DAY')
start = trade_calendar.create_v2('2018-01-10', util.FREQ_DAY)
end = trade_calendar.create_v2('2018-03-10', util.FREQ_DAY)
k = helper.load('000002', start, end)
assert k.empty is False
def duration(self, freq):
helper = KlinedbHelper.load_db_helper(freq)
start, end = helper.duration(self.symbol)
return trade_calendar.create_v2(start, freq), trade_calendar.create_v2(
end, freq)
def append_kline(symbol, start_date, end_date, freq, df):
helper = KlinedbHelper.load_db_helper(freq)
## TODO add more check
helper.save(symbol, trade_calendar.create_v2(start_date, freq),
trade_calendar.create_v2(end_date, freq), df)
def date(self):
self.__load()
if self.__date is None:
return None
return trade_calendar.create_v2(self.__date, util.FREQ_DAY)
