def process_response_event(self, event):
data_frame_list = []
logger = LoggerManager().getLogger(__name__)
for msg in event:
# Generates a lot of output - so don't use unless for
# debugging purposes
# logger.info(msg)
if msg.hasElement(self.RESPONSE_ERROR):
logger.error("REQUEST FAILED: " +
str(msg.getElement(self.RESPONSE_ERROR)))
continue
data_frame_slice = self.process_message(msg)
if (data_frame_slice is not None):
data_frame_list.append(data_frame_slice)
if data_frame_list == []:
logger.warn("No elements for ticker.")
return None
else:
return pd.concat(data_frame_list)
def start_bloomberg_session(self):
constants = DataConstants()
tries = 0
session = None
logger = LoggerManager().getLogger(__name__)
# Try up to 5 times to start a session
while (tries < 5):
try:
# fill SessionOptions
sessionOptions = blpapi.SessionOptions()
sessionOptions.setServerHost(constants.bbg_server)
sessionOptions.setServerPort(constants.bbg_server_port)
logger.info("Starting Bloomberg session...")
# create a Session
session = blpapi.Session(sessionOptions)
# start a Session
if not session.start():
logger.error("Failed to start session.")
return
logger.info("Returning session...")
tries = 5
except:
tries = tries + 1
# BBGLowLevelTemplate._session = session
if session is None:
logger.error("Failed to start session.")
return
return session
def process_message(self, msg):
logger = LoggerManager().getLogger(__name__)
data = collections.defaultdict(dict)
# process received events
securityDataArray = msg.getElement('securityData')
index = 0
single = False
for securityData in list(securityDataArray.values()):
ticker = securityData.getElementAsString("security")
fieldData = securityData.getElement("fieldData")
for field in fieldData.elements():
if not field.isValid():
field_name = "%s" % field.name()
logger.error(field_name + " is NULL")
elif field.isArray():
# iterate over complex data returns.
field_name = "%s" % field.name()
for i, row in enumerate(field.values()):
try:
field_val = re.findall(r'"(.*?)"', "%s" % row)[0]
except:
e = row.getElement(0)
# k = str(e.name())
field_val = e.getValue()
data[(field_name, ticker)][index] = field_val
index = index + 1
else:
field_name = "%s" % field.name()
data[(field_name, ticker)][0] = field.getValueAsString()
index = index + 1
single = True # no need to create multi-index late,
# because just row!! CAREFUL!! needed for futures expiries
fieldExceptionArray = securityData.getElement("fieldExceptions")
for fieldException in list(fieldExceptionArray.values()):
errorInfo = fieldException.getElement("errorInfo")
print(errorInfo.getElementAsString("category"), ":", \
fieldException.getElementAsString("fieldId"))
print("stop")
# Explicitly state from_dict (buggy if create pd.DataFrame(data)
data_frame = pd.DataFrame.from_dict(data)
# If obsolete ticker could return no values
if data_frame.empty:
return None
else:
logger.info("Reading: " + ticker + ' ' + str(data_frame.index[0]) +
' - ' + str(data_frame.index[-1]))
return data_frame
def load_time_series(self, md_request):
# if(BBGLowLevelTemplate._session is None):
logger = LoggerManager().getLogger(__name__)
session = self.start_bloomberg_session()
# else:
# session = BBGLowLevelTemplate._session
def download_data_frame(sess, eventQ, opt, ci):
if opt.security is not None:
self.send_bar_request(sess, eventQ, opt, ci)
logger.info("Waiting for data to be returned...")
return self.event_loop(sess)
else:
logger.warn("No ticker or field specified!")
return None
try:
# if can't open the session, kill existing one
# then try reopen (up to 5 times...)
i = 0
while i < 5:
if session is not None:
if not session.openService("//blp/refdata"):
logger.info("Try reopening Bloomberg session... try " +
str(i))
self.kill_session(
session) # need to forcibly kill_session since
# can't always reopen
session = self.start_bloomberg_session()
if session is not None:
if session.openService("//blp/refdata"): i = 6
else:
logger.info("Try opening Bloomberg session... try " +
str(i))
session = self.start_bloomberg_session()
i = i + 1
# Give error if still doesn't work after several tries..
if not session.openService("//blp/refdata"):
logger.error("Failed to open //blp/refdata")
return
logger.info("Creating request...")
eventQueue = blpapi.EventQueue()
# eventQueue = None
# Create a request
from blpapi import CorrelationId
options = self.fill_options(md_request)
# In some instances we might split the options if need to have
# different overrides
if isinstance(options, list):
data_frame_list = []
for op in options:
cid = CorrelationId()
data_frame_list.append(
download_data_frame(session, eventQueue, op, cid))
data_frame = Calculations().join(data_frame_list)
else:
cid = CorrelationId()
data_frame = download_data_frame(session, eventQueue, options,
cid)
finally:
# stop the session (will fail if NoneType)
try:
session.stop()
except:
pass
return data_frame
class BBGLowLevelRef(BBGLowLevelTemplate):
def __init__(self):
super(BBGLowLevelRef, self).__init__()
self.logger = LoggerManager().getLogger(__name__)
self._options = []
# populate options for Bloomberg request for asset intraday request
def fill_options(self, market_data_request):
self._options = OptionsBBG()
self._options.security = market_data_request.tickers
self._options.startDateTime = market_data_request.start_date
self._options.endDateTime = market_data_request.finish_date
self._options.fields = market_data_request.fields
return self._options
def process_message(self, msg):
data = collections.defaultdict(dict)
# process received events
securityDataArray = msg.getElement('securityData')
index = 0
for securityData in list(securityDataArray.values()):
ticker = securityData.getElementAsString("security")
fieldData = securityData.getElement("fieldData")
for field in fieldData.elements():
if not field.isValid():
field_name = "%s" % field.name()
self.logger.error(field_name + " is NULL")
elif field.isArray():
# iterate over complex data returns.
field_name = "%s" % field.name()
for i, row in enumerate(field.values()):
data[(field_name, ticker)][index] = re.findall(r'"(.*?)"', "%s" % row)[0]
index = index + 1
# else:
# vals.append(re.findall(r'"(.*?)"', "%s" % row)[0])
# print("%s = %s" % (field.name(), field.getValueAsString()))
fieldExceptionArray = securityData.getElement("fieldExceptions")
for fieldException in list(fieldExceptionArray.values()):
errorInfo = fieldException.getElement("errorInfo")
print(errorInfo.getElementAsString("category"), ":", \
fieldException.getElementAsString("fieldId"))
data_frame = pandas.DataFrame(data)
# if obsolete ticker could return no values
if (not(data_frame.empty)):
data_frame.columns = pandas.MultiIndex.from_tuples(data, names=['field', 'ticker'])
self.logger.info("Reading: " + ticker + ' ' + str(data_frame.index[0]) + ' - ' + str(data_frame.index[-1]))
else:
return None
return data_frame
def combine_slices(self, data_frame, data_frame_slice):
if (data_frame_slice.columns.get_level_values(1).values[0]
not in data_frame.columns.get_level_values(1).values):
return data_frame.join(data_frame_slice, how="outer")
return data_frame
# create request for data
def send_bar_request(self, session, eventQueue):
refDataService = session.getService("//blp/refdata")
request = refDataService.createRequest('ReferenceDataRequest')
self.add_override(request, 'TIME_ZONE_OVERRIDE', 23) # force GMT time
self.add_override(request, 'START_DT', self._options.startDateTime.strftime('%Y%m%d'))
self.add_override(request, 'END_DT', self._options.endDateTime.strftime('%Y%m%d'))
# only one security/eventType per request
for field in self._options.fields:
request.getElement("fields").appendValue(field)
for security in self._options.security:
request.getElement("securities").appendValue(security)
self.logger.info("Sending Bloomberg Ref Request:" + str(request))
session.sendRequest(request)
def load_time_series(self, market_data_request):
# if(BBGLowLevelTemplate._session is None):
logger = LoggerManager().getLogger(__name__)
session = self.start_bloomberg_session()
# else:
# session = BBGLowLevelTemplate._session
try:
# if can't open the session, kill existing one
# then try reopen (up to 5 times...)
i = 0
while i < 5:
if session is not None:
if not session.openService("//blp/refdata"):
logger.info("Try reopening Bloomberg session... try " +
str(i))
self.kill_session(
session
) # need to forcibly kill_session since can't always reopen
session = self.start_bloomberg_session()
if session is not None:
if session.openService("//blp/refdata"): i = 6
else:
logger.info("Try opening Bloomberg session... try " +
str(i))
session = self.start_bloomberg_session()
i = i + 1
# give error if still doesn't work after several tries..
if not session.openService("//blp/refdata"):
logger.error("Failed to open //blp/refdata")
return
logger.info("Creating request...")
eventQueue = blpapi.EventQueue()
# eventQueue = None
# create a request
from blpapi import CorrelationId
cid = CorrelationId()
options = self.fill_options(market_data_request)
if options.security is not None:
self.send_bar_request(session, eventQueue, options, cid)
logger.info("Waiting for data to be returned...")
data_frame = self.event_loop(session)
else:
logger.warn("No ticker or field specified!")
data_frame = None
finally:
# stop the session (will fail if NoneType)
try:
session.stop()
except:
pass
return data_frame
class BBGLowLevelRef(BBGLowLevelTemplate):
def __init__(self):
super(BBGLowLevelRef, self).__init__()
self.logger = LoggerManager().getLogger(__name__)
# populate options for Bloomberg request for asset intraday request
def fill_options(self, market_data_request):
options = OptionsBBG()
options.security = market_data_request.tickers
options.startDateTime = market_data_request.start_date
options.endDateTime = market_data_request.finish_date
options.fields = market_data_request.fields
options.overrides = market_data_request.overrides
return options
def process_message(self, msg):
data = collections.defaultdict(dict)
# process received events
securityDataArray = msg.getElement('securityData')
index = 0
single = False
for securityData in list(securityDataArray.values()):
ticker = securityData.getElementAsString("security")
fieldData = securityData.getElement("fieldData")
for field in fieldData.elements():
if not field.isValid():
field_name = "%s" % field.name()
self.logger.error(field_name + " is NULL")
elif field.isArray():
# iterate over complex data returns.
field_name = "%s" % field.name()
for i, row in enumerate(field.values()):
try:
field_val = re.findall(r'"(.*?)"', "%s" % row)[0]
except:
e = row.getElement(0)
# k = str(e.name())
field_val = e.getValue()
data[(field_name, ticker)][index] = field_val
index = index + 1
else:
field_name = "%s" % field.name()
data[(field_name, ticker)][0] = field.getValueAsString()
index = index + 1
single = True # no need to create multi-index late, because just row!! CAREFUL!! needed for futures expiries
fieldExceptionArray = securityData.getElement("fieldExceptions")
for fieldException in list(fieldExceptionArray.values()):
errorInfo = fieldException.getElement("errorInfo")
print(errorInfo.getElementAsString("category"), ":", \
fieldException.getElementAsString("fieldId"))
print("stop")
# explicitly state from_dict (buggy if create pandas.DataFrame(data)
data_frame = pandas.DataFrame.from_dict(data)
# if obsolete ticker could return no values
if (not (data_frame.empty)):
# if not(single):
# pass
# data_frame.columns = pandas.MultiIndex.from_tuples(data, names=['field', 'ticker'])
self.logger.info("Reading: " + ticker + ' ' +
str(data_frame.index[0]) + ' - ' +
str(data_frame.index[-1]))
else:
return None
return data_frame
def combine_slices(self, data_frame_cols, data_frame_slice):
if (data_frame_slice.columns.get_level_values(1).values[0]
not in data_frame_cols):
# return data_frame.join(data_frame_slice, how="outer")
return data_frame_slice
return None
# create request for data
def send_bar_request(self, session, eventQueue, options, cid):
refDataService = session.getService("//blp/refdata")
request = refDataService.createRequest('ReferenceDataRequest')
self.add_override(request, 'TIME_ZONE_OVERRIDE', 23) # force GMT time
self.add_override(request, 'INCLUDE_EXPIRED_CONTRACTS',
"Y") # include expired contracts
self.add_override(request, 'START_DT',
options.startDateTime.strftime('%Y%m%d'))
self.add_override(request, 'END_DT',
options.endDateTime.strftime('%Y%m%d'))
# only one security/eventType per request
for field in options.fields:
request.getElement("fields").appendValue(field)
for security in options.security:
request.getElement("securities").appendValue(security)
if options.overrides != {}:
for k in options.overrides.keys():
new_k = k
# is there a pretty name for this?
if k in super().convert_override_fields:
new_k = super().convert_override_fields[k]
self.add_override(request, new_k, options.overrides[k])
self.logger.info("Sending Bloomberg Ref Request:" + str(request))
session.sendRequest(request=request, correlationId=cid)
