def __init__(self, ticker, **kwargs):
"""
Args:
ticker: ticker
"""
self.ticker = ticker
self.exch = const.exch_info(ticker=ticker, **kwargs)
def time_range(dt,
ticker,
session='allday',
tz='UTC',
**kwargs) -> intervals.Session:
"""
Time range in UTC (for intraday bar) or other timezone
Args:
dt: date
ticker: ticker
session: market session defined in xbbg/markets/exch.yml
tz: timezone
Returns:
intervals.Session
"""
ss = intervals.get_interval(ticker=ticker, session=session, **kwargs)
ex_info = const.exch_info(ticker=ticker, **kwargs)
cur_dt = pd.Timestamp(dt).strftime('%Y-%m-%d')
time_fmt = '%Y-%m-%dT%H:%M:%S'
time_idx = pd.DatetimeIndex([
f'{cur_dt} {ss.start_time}', f'{cur_dt} {ss.end_time}'
]).tz_localize(ex_info.tz).tz_convert(DEFAULT_TZ).tz_convert(tz)
if time_idx[0] > time_idx[1]: time_idx -= pd.TimedeltaIndex(['1D', '0D'])
return intervals.Session(time_idx[0].strftime(time_fmt),
time_idx[1].strftime(time_fmt))
def get_tz(tz) -> str:
"""
Convert tz from ticker / shorthands to timezone
Args:
tz: ticker or timezone shorthands
Returns:
str: Python timzone
Examples:
>>> get_tz('NY')
'America/New_York'
>>> get_tz(TimeZone.NY)
'America/New_York'
>>> get_tz('BHP AU Equity')
'Australia/Sydney'
"""
from xbbg.const import exch_info
if tz is None: return DEFAULT_TZ
to_tz = tz
if isinstance(tz, str):
if hasattr(TimeZone, tz):
to_tz = getattr(TimeZone, tz)
else:
exch = exch_info(ticker=tz)
if 'tz' in exch.index:
to_tz = exch.tz
return to_tz
def intraday(ticker, dt, session='', **kwargs) -> pd.DataFrame:
"""
Bloomberg intraday bar data within market session
Args:
ticker: ticker
dt: date
session: examples include
day_open_30, am_normal_30_30, day_close_30, allday_exact_0930_1000
**kwargs:
ref: reference ticker or exchange for timezone
keep_tz: if keep tz if reference ticker / exchange is given
start_time: start time
end_time: end time
typ: [TRADE, BID, ASK, BID_BEST, ASK_BEST, BEST_BID, BEST_ASK]
Returns:
pd.DataFrame
"""
from xbbg.core import intervals
cur_data = bdib(ticker=ticker, dt=dt, typ=kwargs.get('typ', 'TRADE'))
if cur_data.empty: return pd.DataFrame()
fmt = '%H:%M:%S'
ss = intervals.SessNA
ref = kwargs.get('ref', None)
exch = pd.Series() if ref is None else const.exch_info(ticker=ref)
if session:
ss = intervals.get_interval(ticker=kwargs.get('ref', ticker),
session=session)
start_time = kwargs.get('start_time', None)
end_time = kwargs.get('end_time', None)
if ss != intervals.SessNA:
start_time = pd.Timestamp(ss.start_time).strftime(fmt)
end_time = pd.Timestamp(ss.end_time).strftime(fmt)
if start_time and end_time:
kw = dict(start_time=start_time, end_time=end_time)
if not exch.empty:
cur_tz = cur_data.index.tz
res = cur_data.tz_convert(exch.tz).between_time(**kw)
if kwargs.get('keep_tz', False):
res = res.tz_convert(cur_tz)
return pd.DataFrame(res)
return pd.DataFrame(cur_data.between_time(**kw))
return cur_data
def save_intraday(data: pd.DataFrame, ticker: str, dt, typ='TRADE', **kwargs):
"""
Check whether data is done for the day and save
Args:
data: data
ticker: ticker
dt: date
typ: [TRADE, BID, ASK, BID_BEST, ASK_BEST, BEST_BID, BEST_ASK]
Examples:
>>> os.environ['BBG_ROOT'] = 'xbbg/tests/data'
>>> sample = pd.read_parquet('xbbg/tests/data/aapl.parq')
>>> save_intraday(sample, 'AAPL US Equity', '2018-11-02')
>>> # Invalid exchange
>>> save_intraday(sample, 'AAPL XX Equity', '2018-11-02')
>>> # Invalid empty data
>>> save_intraday(pd.DataFrame(), 'AAPL US Equity', '2018-11-02')
>>> # Invalid date - too close
>>> cur_dt = utils.cur_time()
>>> save_intraday(sample, 'AAPL US Equity', cur_dt)
"""
cur_dt = pd.Timestamp(dt).strftime('%Y-%m-%d')
logger = logs.get_logger(save_intraday, level='debug')
info = f'{ticker} / {cur_dt} / {typ}'
data_file = bar_file(ticker=ticker, dt=dt, typ=typ)
if not data_file: return
if data.empty:
logger.warning(f'data is empty for {info} ...')
return
exch = const.exch_info(ticker=ticker, **kwargs)
if exch.empty: return
end_time = pd.Timestamp(
const.market_timing(ticker=ticker, dt=dt, timing='FINISHED',
**kwargs)).tz_localize(exch.tz)
now = pd.Timestamp('now', tz=exch.tz) - pd.Timedelta('1H')
if end_time > now:
logger.debug(
f'skip saving cause market close ({end_time}) < now - 1H ({now}) ...'
)
return
logger.info(f'saving data to {data_file} ...')
files.create_folder(data_file, is_file=True)
data.to_parquet(data_file)
def bdtick(ticker,
dt,
session='allday',
time_range=None,
types=None,
**kwargs) -> pd.DataFrame:
"""
Bloomberg tick data
Args:
ticker: ticker name
dt: date to download
session: [allday, day, am, pm, pre, post]
time_range: tuple of start and end time (must be converted into UTC)
if this is given, `dt` and `session` will be ignored
types: str or list, one or combinations of [
TRADE, AT_TRADE, BID, ASK, MID_PRICE,
BID_BEST, ASK_BEST, BEST_BID, BEST_ASK,
]
Returns:
pd.DataFrame
"""
logger = logs.get_logger(bdtick, **kwargs)
if types is None: types = ['TRADE']
exch = const.exch_info(ticker=ticker, **kwargs)
if exch.empty: raise LookupError(f'Cannot find exchange info for {ticker}')
if isinstance(time_range, (tuple, list)) and (len(time_range) == 2):
cur_dt = pd.Timestamp(dt).strftime('%Y-%m-%d')
time_rng = (pd.DatetimeIndex([
f'{cur_dt} {time_range[0]}',
f'{cur_dt} {time_range[1]}',
]).tz_localize(exch.tz).tz_convert(
process.DEFAULT_TZ).tz_convert('UTC'))
else:
time_rng = process.time_range(dt=dt,
ticker=ticker,
session=session,
**kwargs)
while conn.bbg_session(**kwargs).tryNextEvent():
pass
request = process.create_request(
service='//blp/refdata',
request='IntradayTickRequest',
settings=[
('security', ticker),
('startDateTime', time_rng[0]),
('endDateTime', time_rng[1]),
('includeConditionCodes', True),
('includeExchangeCodes', True),
('includeNonPlottableEvents', True),
('includeBrokerCodes', True),
('includeRpsCodes', True),
('includeTradeTime', True),
('includeActionCodes', True),
('includeIndicatorCodes', True),
],
append={'eventTypes': types},
**kwargs,
)
logger.debug(f'Sending request to Bloomberg ...\n{request}')
conn.send_request(request=request)
res = pd.DataFrame(
process.rec_events(func=process.process_bar, typ='t', **kwargs))
if kwargs.get('raw', False): return res
if res.empty or ('time' not in res): return pd.DataFrame()
return (res.set_index('time').rename_axis(
index=None).tz_localize('UTC').tz_convert(exch.tz).pipe(
pipeline.add_ticker, ticker=ticker).rename(
columns={
'size': 'volume',
'type': 'typ',
'conditionCodes': 'cond',
'exchangeCode': 'exch',
'tradeTime': 'trd_time',
}))
def bdib(ticker: str,
dt,
session='allday',
typ='TRADE',
**kwargs) -> pd.DataFrame:
"""
Bloomberg intraday bar data
Args:
ticker: ticker name
dt: date to download
session: [allday, day, am, pm, pre, post]
typ: [TRADE, BID, ASK, BID_BEST, ASK_BEST, BEST_BID, BEST_ASK]
**kwargs:
ref: reference ticker or exchange
used as supplement if exchange info is not defined for `ticker`
batch: whether is batch process to download data
log: level of logs
Returns:
pd.DataFrame
"""
from xbbg.core import trials
logger = logs.get_logger(bdib, **kwargs)
ex_info = const.exch_info(ticker=ticker, **kwargs)
if ex_info.empty: raise KeyError(f'Cannot find exchange info for {ticker}')
ss_rng = process.time_range(dt=dt,
ticker=ticker,
session=session,
tz=ex_info.tz,
**kwargs)
data_file = storage.bar_file(ticker=ticker, dt=dt, typ=typ)
if files.exists(data_file) and kwargs.get(
'cache', True) and (not kwargs.get('reload', False)):
res = (pd.read_parquet(data_file).pipe(
pipeline.add_ticker, ticker=ticker).loc[ss_rng[0]:ss_rng[1]])
if not res.empty:
logger.debug(f'Loading Bloomberg intraday data from: {data_file}')
return res
if not process.check_current(dt=dt, logger=logger, **kwargs):
return pd.DataFrame()
cur_dt = pd.Timestamp(dt).strftime('%Y-%m-%d')
q_tckr = ticker
if ex_info.get('is_fut', False):
is_sprd = ex_info.get(
'has_sprd', False) and (len(ticker[:-1]) != ex_info['tickers'][0])
if not is_sprd:
q_tckr = fut_ticker(gen_ticker=ticker, dt=dt, freq=ex_info['freq'])
if q_tckr == '':
logger.error(f'cannot find futures ticker for {ticker} ...')
return pd.DataFrame()
info_log = f'{q_tckr} / {cur_dt} / {typ}'
trial_kw = dict(ticker=ticker, dt=dt, typ=typ, func='bdib')
num_trials = trials.num_trials(**trial_kw)
if num_trials >= 2:
if kwargs.get('batch', False): return pd.DataFrame()
logger.info(f'{num_trials} trials with no data {info_log}')
return pd.DataFrame()
while conn.bbg_session(**kwargs).tryNextEvent():
pass
time_rng = process.time_range(dt=dt,
ticker=ticker,
session='allday',
**kwargs)
request = process.create_request(
service='//blp/refdata',
request='IntradayBarRequest',
settings=[
('security', ticker),
('eventType', typ),
('interval', kwargs.get('interval', 1)),
('startDateTime', time_rng[0]),
('endDateTime', time_rng[1]),
],
**kwargs,
)
logger.debug(f'Sending request to Bloomberg ...\n{request}')
conn.send_request(request=request, **kwargs)
res = pd.DataFrame(process.rec_events(func=process.process_bar, **kwargs))
if res.empty or ('time' not in res):
logger.warning(f'No data for {info_log} ...')
trials.update_trials(cnt=num_trials + 1, **trial_kw)
return pd.DataFrame()
data = (res.set_index('time').rename_axis(index=None).rename(
columns={
'numEvents': 'num_trds'
}).tz_localize('UTC').tz_convert(ex_info.tz).pipe(pipeline.add_ticker,
ticker=ticker))
if kwargs.get('cache', True):
storage.save_intraday(data=data[ticker],
ticker=ticker,
dt=dt,
typ=typ,
**kwargs)
return data.loc[ss_rng[0]:ss_rng[1]]
def bdib(ticker, dt, typ='TRADE', batch=False, log=logs.LOG_LEVEL) -> pd.DataFrame:
"""
Download intraday data and save to cache
Args:
ticker: ticker name
dt: date to download
typ: [TRADE, BID, ASK, BID_BEST, ASK_BEST, BEST_BID, BEST_ASK]
batch: whether is batch process to download data
log: level of logs
Returns:
pd.DataFrame
"""
from xbbg.core import missing
logger = logs.get_logger(bdib, level=log)
t_1 = pd.Timestamp('today').date() - pd.Timedelta('1D')
whole_day = pd.Timestamp(dt).date() < t_1
if (not whole_day) and batch:
logger.warning(f'querying date {t_1} is too close, ignoring download ...')
return pd.DataFrame()
cur_dt = pd.Timestamp(dt).strftime('%Y-%m-%d')
asset = ticker.split()[-1]
info_log = f'{ticker} / {cur_dt} / {typ}'
if asset in ['Equity', 'Curncy', 'Index', 'Comdty']:
exch = const.exch_info(ticker=ticker)
if exch.empty: return pd.DataFrame()
else:
logger.error(f'unknown asset type: {asset}')
return pd.DataFrame()
time_fmt = '%Y-%m-%dT%H:%M:%S'
time_idx = pd.DatetimeIndex([
f'{cur_dt} {exch.allday[0]}', f'{cur_dt} {exch.allday[-1]}']
).tz_localize(exch.tz).tz_convert(DEFAULT_TZ).tz_convert('UTC')
if time_idx[0] > time_idx[1]: time_idx -= pd.TimedeltaIndex(['1D', '0D'])
q_tckr = ticker
if exch.get('is_fut', False):
if 'freq' not in exch:
logger.error(f'[freq] missing in info for {info_log} ...')
is_sprd = exch.get('has_sprd', False) and (len(ticker[:-1]) != exch['tickers'][0])
if not is_sprd:
q_tckr = fut_ticker(gen_ticker=ticker, dt=dt, freq=exch['freq'])
if q_tckr == '':
logger.error(f'cannot find futures ticker for {ticker} ...')
return pd.DataFrame()
info_log = f'{q_tckr} / {cur_dt} / {typ}'
miss_kw = dict(ticker=ticker, dt=dt, typ=typ, func='bdib')
cur_miss = missing.current_missing(**miss_kw)
if cur_miss >= 2:
if batch: return pd.DataFrame()
logger.info(f'{cur_miss} trials with no data {info_log}')
return pd.DataFrame()
logger.info(f'loading data from Bloomberg: {info_log} ...')
con, _ = create_connection()
data = con.bdib(
ticker=q_tckr, event_type=typ, interval=1,
start_datetime=time_idx[0].strftime(time_fmt),
end_datetime=time_idx[1].strftime(time_fmt),
)
if not isinstance(data, pd.DataFrame):
raise ValueError(f'unknown output format: {type(data)}')
if data.empty:
logger.warning(f'no data for {info_log} ...')
missing.update_missing(**miss_kw)
return pd.DataFrame()
data = data.tz_localize('UTC').tz_convert(exch.tz)
storage.save_intraday(data=data, ticker=ticker, dt=dt, typ=typ)
return pd.DataFrame() if batch else assist.format_intraday(data=data, ticker=ticker)
def bdtick(ticker, dt, session='allday', types=None, **kwargs) -> pd.DataFrame:
"""
Bloomberg tick data
Args:
ticker: ticker name
dt: date to download
session: [allday, day, am, pm, pre, post]
types: str or list, one or combinations of [
TRADE, AT_TRADE, BID, ASK, MID_PRICE,
BID_BEST, ASK_BEST, BEST_BID, BEST_ASK,
]
Returns:
pd.DataFrame
"""
logger = logs.get_logger(bdtick, **kwargs)
exch = const.exch_info(ticker=ticker, **kwargs)
time_rng = process.time_range(dt=dt,
ticker=ticker,
session=session,
tz=exch.tz,
**kwargs)
service = conn.bbg_service(service='//blp/refdata', **kwargs)
request = service.createRequest('IntradayTickRequest')
while conn.bbg_session(**kwargs).tryNextEvent():
pass
if types is None: types = ['TRADE']
if isinstance(types, str): types = [types]
request.set('security', ticker)
for typ in types:
request.append('eventTypes', typ)
request.set('startDateTime', time_rng[0])
request.set('endDateTime', time_rng[1])
request.set('includeConditionCodes', True)
request.set('includeExchangeCodes', True)
request.set('includeNonPlottableEvents', True)
request.set('includeBrokerCodes', True)
request.set('includeRpsCodes', True)
request.set('includeTradeTime', True)
request.set('includeActionCodes', True)
request.set('includeIndicatorCodes', True)
logger.debug(f'Sending request to Bloomberg ...\n{request}')
conn.send_request(request=request)
res = pd.DataFrame(
process.rec_events(func=process.process_bar, typ='t', **kwargs))
if kwargs.get('raw', False): return res
if res.empty or ('time' not in res): return pd.DataFrame()
return (res.set_index('time').rename_axis(
index=None).tz_localize('UTC').tz_convert(exch.tz).pipe(
pipeline.add_ticker, ticker=ticker).rename(
columns={
'size': 'volume',
'type': 'typ',
'conditionCodes': 'cond',
'exchangeCode': 'exch',
'tradeTime': 'trd_time',
}))
