def __init__(self, ticker, **kwargs): """ Args: ticker: ticker """ self.ticker = ticker self.exch = const.exch_info(ticker=ticker, **kwargs)
def time_range(dt, ticker, session='allday', tz='UTC', **kwargs) -> intervals.Session: """ Time range in UTC (for intraday bar) or other timezone Args: dt: date ticker: ticker session: market session defined in xbbg/markets/exch.yml tz: timezone Returns: intervals.Session """ ss = intervals.get_interval(ticker=ticker, session=session, **kwargs) ex_info = const.exch_info(ticker=ticker, **kwargs) cur_dt = pd.Timestamp(dt).strftime('%Y-%m-%d') time_fmt = '%Y-%m-%dT%H:%M:%S' time_idx = pd.DatetimeIndex([ f'{cur_dt} {ss.start_time}', f'{cur_dt} {ss.end_time}' ]).tz_localize(ex_info.tz).tz_convert(DEFAULT_TZ).tz_convert(tz) if time_idx[0] > time_idx[1]: time_idx -= pd.TimedeltaIndex(['1D', '0D']) return intervals.Session(time_idx[0].strftime(time_fmt), time_idx[1].strftime(time_fmt))
def get_tz(tz) -> str: """ Convert tz from ticker / shorthands to timezone Args: tz: ticker or timezone shorthands Returns: str: Python timzone Examples: >>> get_tz('NY') 'America/New_York' >>> get_tz(TimeZone.NY) 'America/New_York' >>> get_tz('BHP AU Equity') 'Australia/Sydney' """ from xbbg.const import exch_info if tz is None: return DEFAULT_TZ to_tz = tz if isinstance(tz, str): if hasattr(TimeZone, tz): to_tz = getattr(TimeZone, tz) else: exch = exch_info(ticker=tz) if 'tz' in exch.index: to_tz = exch.tz return to_tz
def intraday(ticker, dt, session='', **kwargs) -> pd.DataFrame: """ Bloomberg intraday bar data within market session Args: ticker: ticker dt: date session: examples include day_open_30, am_normal_30_30, day_close_30, allday_exact_0930_1000 **kwargs: ref: reference ticker or exchange for timezone keep_tz: if keep tz if reference ticker / exchange is given start_time: start time end_time: end time typ: [TRADE, BID, ASK, BID_BEST, ASK_BEST, BEST_BID, BEST_ASK] Returns: pd.DataFrame """ from xbbg.core import intervals cur_data = bdib(ticker=ticker, dt=dt, typ=kwargs.get('typ', 'TRADE')) if cur_data.empty: return pd.DataFrame() fmt = '%H:%M:%S' ss = intervals.SessNA ref = kwargs.get('ref', None) exch = pd.Series() if ref is None else const.exch_info(ticker=ref) if session: ss = intervals.get_interval(ticker=kwargs.get('ref', ticker), session=session) start_time = kwargs.get('start_time', None) end_time = kwargs.get('end_time', None) if ss != intervals.SessNA: start_time = pd.Timestamp(ss.start_time).strftime(fmt) end_time = pd.Timestamp(ss.end_time).strftime(fmt) if start_time and end_time: kw = dict(start_time=start_time, end_time=end_time) if not exch.empty: cur_tz = cur_data.index.tz res = cur_data.tz_convert(exch.tz).between_time(**kw) if kwargs.get('keep_tz', False): res = res.tz_convert(cur_tz) return pd.DataFrame(res) return pd.DataFrame(cur_data.between_time(**kw)) return cur_data
def save_intraday(data: pd.DataFrame, ticker: str, dt, typ='TRADE', **kwargs): """ Check whether data is done for the day and save Args: data: data ticker: ticker dt: date typ: [TRADE, BID, ASK, BID_BEST, ASK_BEST, BEST_BID, BEST_ASK] Examples: >>> os.environ['BBG_ROOT'] = 'xbbg/tests/data' >>> sample = pd.read_parquet('xbbg/tests/data/aapl.parq') >>> save_intraday(sample, 'AAPL US Equity', '2018-11-02') >>> # Invalid exchange >>> save_intraday(sample, 'AAPL XX Equity', '2018-11-02') >>> # Invalid empty data >>> save_intraday(pd.DataFrame(), 'AAPL US Equity', '2018-11-02') >>> # Invalid date - too close >>> cur_dt = utils.cur_time() >>> save_intraday(sample, 'AAPL US Equity', cur_dt) """ cur_dt = pd.Timestamp(dt).strftime('%Y-%m-%d') logger = logs.get_logger(save_intraday, level='debug') info = f'{ticker} / {cur_dt} / {typ}' data_file = bar_file(ticker=ticker, dt=dt, typ=typ) if not data_file: return if data.empty: logger.warning(f'data is empty for {info} ...') return exch = const.exch_info(ticker=ticker, **kwargs) if exch.empty: return end_time = pd.Timestamp( const.market_timing(ticker=ticker, dt=dt, timing='FINISHED', **kwargs)).tz_localize(exch.tz) now = pd.Timestamp('now', tz=exch.tz) - pd.Timedelta('1H') if end_time > now: logger.debug( f'skip saving cause market close ({end_time}) < now - 1H ({now}) ...' ) return logger.info(f'saving data to {data_file} ...') files.create_folder(data_file, is_file=True) data.to_parquet(data_file)
def bdtick(ticker, dt, session='allday', time_range=None, types=None, **kwargs) -> pd.DataFrame: """ Bloomberg tick data Args: ticker: ticker name dt: date to download session: [allday, day, am, pm, pre, post] time_range: tuple of start and end time (must be converted into UTC) if this is given, `dt` and `session` will be ignored types: str or list, one or combinations of [ TRADE, AT_TRADE, BID, ASK, MID_PRICE, BID_BEST, ASK_BEST, BEST_BID, BEST_ASK, ] Returns: pd.DataFrame """ logger = logs.get_logger(bdtick, **kwargs) if types is None: types = ['TRADE'] exch = const.exch_info(ticker=ticker, **kwargs) if exch.empty: raise LookupError(f'Cannot find exchange info for {ticker}') if isinstance(time_range, (tuple, list)) and (len(time_range) == 2): cur_dt = pd.Timestamp(dt).strftime('%Y-%m-%d') time_rng = (pd.DatetimeIndex([ f'{cur_dt} {time_range[0]}', f'{cur_dt} {time_range[1]}', ]).tz_localize(exch.tz).tz_convert( process.DEFAULT_TZ).tz_convert('UTC')) else: time_rng = process.time_range(dt=dt, ticker=ticker, session=session, **kwargs) while conn.bbg_session(**kwargs).tryNextEvent(): pass request = process.create_request( service='//blp/refdata', request='IntradayTickRequest', settings=[ ('security', ticker), ('startDateTime', time_rng[0]), ('endDateTime', time_rng[1]), ('includeConditionCodes', True), ('includeExchangeCodes', True), ('includeNonPlottableEvents', True), ('includeBrokerCodes', True), ('includeRpsCodes', True), ('includeTradeTime', True), ('includeActionCodes', True), ('includeIndicatorCodes', True), ], append={'eventTypes': types}, **kwargs, ) logger.debug(f'Sending request to Bloomberg ...\n{request}') conn.send_request(request=request) res = pd.DataFrame( process.rec_events(func=process.process_bar, typ='t', **kwargs)) if kwargs.get('raw', False): return res if res.empty or ('time' not in res): return pd.DataFrame() return (res.set_index('time').rename_axis( index=None).tz_localize('UTC').tz_convert(exch.tz).pipe( pipeline.add_ticker, ticker=ticker).rename( columns={ 'size': 'volume', 'type': 'typ', 'conditionCodes': 'cond', 'exchangeCode': 'exch', 'tradeTime': 'trd_time', }))
def bdib(ticker: str, dt, session='allday', typ='TRADE', **kwargs) -> pd.DataFrame: """ Bloomberg intraday bar data Args: ticker: ticker name dt: date to download session: [allday, day, am, pm, pre, post] typ: [TRADE, BID, ASK, BID_BEST, ASK_BEST, BEST_BID, BEST_ASK] **kwargs: ref: reference ticker or exchange used as supplement if exchange info is not defined for `ticker` batch: whether is batch process to download data log: level of logs Returns: pd.DataFrame """ from xbbg.core import trials logger = logs.get_logger(bdib, **kwargs) ex_info = const.exch_info(ticker=ticker, **kwargs) if ex_info.empty: raise KeyError(f'Cannot find exchange info for {ticker}') ss_rng = process.time_range(dt=dt, ticker=ticker, session=session, tz=ex_info.tz, **kwargs) data_file = storage.bar_file(ticker=ticker, dt=dt, typ=typ) if files.exists(data_file) and kwargs.get( 'cache', True) and (not kwargs.get('reload', False)): res = (pd.read_parquet(data_file).pipe( pipeline.add_ticker, ticker=ticker).loc[ss_rng[0]:ss_rng[1]]) if not res.empty: logger.debug(f'Loading Bloomberg intraday data from: {data_file}') return res if not process.check_current(dt=dt, logger=logger, **kwargs): return pd.DataFrame() cur_dt = pd.Timestamp(dt).strftime('%Y-%m-%d') q_tckr = ticker if ex_info.get('is_fut', False): is_sprd = ex_info.get( 'has_sprd', False) and (len(ticker[:-1]) != ex_info['tickers'][0]) if not is_sprd: q_tckr = fut_ticker(gen_ticker=ticker, dt=dt, freq=ex_info['freq']) if q_tckr == '': logger.error(f'cannot find futures ticker for {ticker} ...') return pd.DataFrame() info_log = f'{q_tckr} / {cur_dt} / {typ}' trial_kw = dict(ticker=ticker, dt=dt, typ=typ, func='bdib') num_trials = trials.num_trials(**trial_kw) if num_trials >= 2: if kwargs.get('batch', False): return pd.DataFrame() logger.info(f'{num_trials} trials with no data {info_log}') return pd.DataFrame() while conn.bbg_session(**kwargs).tryNextEvent(): pass time_rng = process.time_range(dt=dt, ticker=ticker, session='allday', **kwargs) request = process.create_request( service='//blp/refdata', request='IntradayBarRequest', settings=[ ('security', ticker), ('eventType', typ), ('interval', kwargs.get('interval', 1)), ('startDateTime', time_rng[0]), ('endDateTime', time_rng[1]), ], **kwargs, ) logger.debug(f'Sending request to Bloomberg ...\n{request}') conn.send_request(request=request, **kwargs) res = pd.DataFrame(process.rec_events(func=process.process_bar, **kwargs)) if res.empty or ('time' not in res): logger.warning(f'No data for {info_log} ...') trials.update_trials(cnt=num_trials + 1, **trial_kw) return pd.DataFrame() data = (res.set_index('time').rename_axis(index=None).rename( columns={ 'numEvents': 'num_trds' }).tz_localize('UTC').tz_convert(ex_info.tz).pipe(pipeline.add_ticker, ticker=ticker)) if kwargs.get('cache', True): storage.save_intraday(data=data[ticker], ticker=ticker, dt=dt, typ=typ, **kwargs) return data.loc[ss_rng[0]:ss_rng[1]]
def bdib(ticker, dt, typ='TRADE', batch=False, log=logs.LOG_LEVEL) -> pd.DataFrame: """ Download intraday data and save to cache Args: ticker: ticker name dt: date to download typ: [TRADE, BID, ASK, BID_BEST, ASK_BEST, BEST_BID, BEST_ASK] batch: whether is batch process to download data log: level of logs Returns: pd.DataFrame """ from xbbg.core import missing logger = logs.get_logger(bdib, level=log) t_1 = pd.Timestamp('today').date() - pd.Timedelta('1D') whole_day = pd.Timestamp(dt).date() < t_1 if (not whole_day) and batch: logger.warning(f'querying date {t_1} is too close, ignoring download ...') return pd.DataFrame() cur_dt = pd.Timestamp(dt).strftime('%Y-%m-%d') asset = ticker.split()[-1] info_log = f'{ticker} / {cur_dt} / {typ}' if asset in ['Equity', 'Curncy', 'Index', 'Comdty']: exch = const.exch_info(ticker=ticker) if exch.empty: return pd.DataFrame() else: logger.error(f'unknown asset type: {asset}') return pd.DataFrame() time_fmt = '%Y-%m-%dT%H:%M:%S' time_idx = pd.DatetimeIndex([ f'{cur_dt} {exch.allday[0]}', f'{cur_dt} {exch.allday[-1]}'] ).tz_localize(exch.tz).tz_convert(DEFAULT_TZ).tz_convert('UTC') if time_idx[0] > time_idx[1]: time_idx -= pd.TimedeltaIndex(['1D', '0D']) q_tckr = ticker if exch.get('is_fut', False): if 'freq' not in exch: logger.error(f'[freq] missing in info for {info_log} ...') is_sprd = exch.get('has_sprd', False) and (len(ticker[:-1]) != exch['tickers'][0]) if not is_sprd: q_tckr = fut_ticker(gen_ticker=ticker, dt=dt, freq=exch['freq']) if q_tckr == '': logger.error(f'cannot find futures ticker for {ticker} ...') return pd.DataFrame() info_log = f'{q_tckr} / {cur_dt} / {typ}' miss_kw = dict(ticker=ticker, dt=dt, typ=typ, func='bdib') cur_miss = missing.current_missing(**miss_kw) if cur_miss >= 2: if batch: return pd.DataFrame() logger.info(f'{cur_miss} trials with no data {info_log}') return pd.DataFrame() logger.info(f'loading data from Bloomberg: {info_log} ...') con, _ = create_connection() data = con.bdib( ticker=q_tckr, event_type=typ, interval=1, start_datetime=time_idx[0].strftime(time_fmt), end_datetime=time_idx[1].strftime(time_fmt), ) if not isinstance(data, pd.DataFrame): raise ValueError(f'unknown output format: {type(data)}') if data.empty: logger.warning(f'no data for {info_log} ...') missing.update_missing(**miss_kw) return pd.DataFrame() data = data.tz_localize('UTC').tz_convert(exch.tz) storage.save_intraday(data=data, ticker=ticker, dt=dt, typ=typ) return pd.DataFrame() if batch else assist.format_intraday(data=data, ticker=ticker)
def bdtick(ticker, dt, session='allday', types=None, **kwargs) -> pd.DataFrame: """ Bloomberg tick data Args: ticker: ticker name dt: date to download session: [allday, day, am, pm, pre, post] types: str or list, one or combinations of [ TRADE, AT_TRADE, BID, ASK, MID_PRICE, BID_BEST, ASK_BEST, BEST_BID, BEST_ASK, ] Returns: pd.DataFrame """ logger = logs.get_logger(bdtick, **kwargs) exch = const.exch_info(ticker=ticker, **kwargs) time_rng = process.time_range(dt=dt, ticker=ticker, session=session, tz=exch.tz, **kwargs) service = conn.bbg_service(service='//blp/refdata', **kwargs) request = service.createRequest('IntradayTickRequest') while conn.bbg_session(**kwargs).tryNextEvent(): pass if types is None: types = ['TRADE'] if isinstance(types, str): types = [types] request.set('security', ticker) for typ in types: request.append('eventTypes', typ) request.set('startDateTime', time_rng[0]) request.set('endDateTime', time_rng[1]) request.set('includeConditionCodes', True) request.set('includeExchangeCodes', True) request.set('includeNonPlottableEvents', True) request.set('includeBrokerCodes', True) request.set('includeRpsCodes', True) request.set('includeTradeTime', True) request.set('includeActionCodes', True) request.set('includeIndicatorCodes', True) logger.debug(f'Sending request to Bloomberg ...\n{request}') conn.send_request(request=request) res = pd.DataFrame( process.rec_events(func=process.process_bar, typ='t', **kwargs)) if kwargs.get('raw', False): return res if res.empty or ('time' not in res): return pd.DataFrame() return (res.set_index('time').rename_axis( index=None).tz_localize('UTC').tz_convert(exch.tz).pipe( pipeline.add_ticker, ticker=ticker).rename( columns={ 'size': 'volume', 'type': 'typ', 'conditionCodes': 'cond', 'exchangeCode': 'exch', 'tradeTime': 'trd_time', }))