def request_kline(symbol, freq , **kwargs):
query = kline.KlineQuery(symbol)
log.debug('begin to request [%s]' % symbol)
if query.isEmpty(freq):
## TODO request all
info = composite.get_basic().info(symbol)
if info is None:
raise ValueError('%s not exist'% symbol)
born = info['timeToMarket']
return request_kline_core(symbol, start_date=trade_calendar.create_v2(born, freq), end_date = trade_calendar.last_trade_date(freq), freq = freq, **kwargs)
else:
db_start, db_end = query.duration(freq)
if db_start is None and db_end is None:
log.error('duration in db is error')
raise RuntimeError('[%s] isn\'t exist, but duration ???' % symbol)
last_date = trade_calendar.last_trade_date(freq)
if trade_calendar.compare_v2(db_end, last_date, freq) == 0:
## TODO check suspend stock
log.info('%s needn\'t update' % symbol)
return 1
elif trade_calendar.compare_v2(db_end, last_date, freq) < 0:
db_end.shift(count = 1)
return request_kline_core(symbol, db_end, last_date, freq, **kwargs)
else:
raise RuntimeError("????")
def save(self, df):
df['code'] = df['code'].astype(str).str.zfill(6)
data = util.to_json(df)
last = trade_calendar.last_trade_date(util.FREQ_DAY)
db_format = {'_id': 'basic', 'date': last.datetime(), 'stocks': data}
self.collection.replace_one({'_id': 'basic'}, db_format, upsert=True)
self.__load(force=True)
def get_count_bars(code, last, limit, freq, ascending = False):
query = kline.KlineQuery(code)
if last is None:
end = trade_calendar.last_trade_date(freq)
else:
end = trade_calendar.create_v2(last, freq)
return query.load(None, end, freq, ascending = ascending, limit = limit)
def newest(self):
dbdate = self.date()
if dbdate is None or self.__df is None:
return False
else:
return trade_calendar.compare_v2(dbdate,
trade_calendar.last_trade_date(
util.FREQ_DAY),
freq=util.FREQ_DAY) == 0
def request_all_today_kline():
df = None
last_date = trade_calendar.last_trade_date(freq = util.FREQ_DAY)
if not TodayAllDailyKline.has_cache(last_date):
if day_sync_time('today_all'):
log.info('request today_all')
df = request.get_today_all()
daily = TodayAllDailyKline(last_date)
daily.catch(df)
else:
daily = TodayAllDailyKline(last_date)
df = daily.load()
return df
def test_stock_basic():
basic = composite.get_basic()
basic.clear()
date1 = basic.date()
assert date1 is None
df1 = pd.read_csv('./debug/get_stock_basic_action.csv')
date2 = trade_calendar.last_trade_date(util.FREQ_DAY)
df1 = df1[df1['timeToMarket'] != 0]
df1['timeToMarket'] = pd.to_datetime(df1['timeToMarket'].astype(str),
format='%Y%m%d').astype(str)
basic.save(df1)
print(date2)
print(basic.date())
assert date2 == basic.date()
assert basic.count() == 3499
assert basic.info('000002')['name'] == '万 科Ａ'
assert basic.info('000002')['timeToMarket'] == '1991-01-29'
def request_symbols_kline(symbols, freq):
def child_fork_init():
trade_calendar.TradeCalendarV2.initialze_reader(True)
db.MongoDB.open_or_create_client(True)
## TODO if symbols more than 50 try use multi process
if len(symbols) > 50:
db.MongoDB.release()
pool = multiprocessing.Pool(initializer=child_fork_init)
inner_worker = functools.partial(worker, freq)
pool.map(inner_worker, symbols)
pool.close()
pool.join()
else:
for symbol in symbols:
request_kline(symbol, freq = freq)
# run here mean all sucess
date = trade_calendar.last_trade_date(freq)
kline.set_sync_result(True, freq, date)
def sync_history_bar(symbols, freq):
ret = kline.get_sync_result(freq)
end = trade_calendar.last_trade_date(freq)
if ret is not None and ret.get('result', False) == True and \
ret.get('date') is not None and trade_calendar.compare_v2(ret.get('date'), end, freq) >= 0:
log.info('need\'t update')
else:
if freq == util.FREQ_DAY:
if ret is None or ret.get('date') is None:
## no date in db , request all data
log.info("request symbol list")
request_symbols_kline(symbols, freq)
else:
request_all_today_kline()
__sync_day_kline(symbols, ret.get('date'), end)
elif freq == util.FREQ_WEEK:
now = datetime.datetime.now()
if now.weekday() >= 5: ## rest day
request_symbols_kline(symbols, freq)
else:
log.info('request weekly data in rest day ignore freq week')
else:
## TODO
request_symbols_kline(symbols, freq)