def download_instrument(self, inst, **kwargs):
"""
:param kwargs: 'clean=True' will force delete and re-download all data,
otherwise db will be updated with the new data
"""
clean = kwargs.get('clean', False)
if not clean: # if clean is False, but no existing data found - force it to be True
try:
tmp_data = Store(self.library, '', inst.bitmex_symbol).get()
except KeyError:
clean = True
if clean:
self.drop_symbol(inst.bitmex_symbol)
last_timestamp = None
else: # get the last timestamp and contract from the sorted index
last_index = tmp_data.index[-1]
last_timestamp = last_index[1].to_pydatetime()
last_contract = last_index[0]
c = str(inst.first_contract)
logger.info('Downloading contracts for instrument: %s' % inst.name)
while c is not None:
nrows = self.download_contract(inst, c, start_time=last_timestamp)
logger.debug('Got %d rows for contract %s' % (nrows, c))
# keep downloading contracts until nothing is returned by the server for the next one.
# if not doing a clean re-download, most dataframes for historical contracts will be
# empty, so add another condition: c < last_contract
try_next = (nrows > 0) or ((not clean) and
(int(c) < last_contract))
c = inst.next_contract(c) if try_next else None
return True
def download_table(self, symbol, start_time=None, price_func=None, store_symbol=None):
# Get bravado client on first download call
if self.client is None:
if self.auth:
self.client = get_request_client()
else:
self.client = SwaggerClient.from_url(SPEC_URI, config=api_config)
# In general we may need separate symbol strings for storage and to pass to API
# (as for futures), but they also may be equal (as for currencies)
if store_symbol is None:
store_symbol = symbol
if price_func is None:
price_func = self._augment_trades
store = Store(self.library, '', store_symbol)
df = pd.DataFrame()
i = 0
n = 0
while True:
try:
res = pd.DataFrame(self.client.Trade.Trade_getBucketed(count=500, start=i*500,
symbol=symbol, binSize='1d', startTime=start_time).result())
except bravado.exception.HTTPTooManyRequests:
logger.info('Requests limit exceeded, sleeping for 5 mins...')
sleep(300)
logger.info('Resuming download')
continue
if len(res) == 0:
break
# keep track of total records count for calling functions to know if we got any data
n += len(res)
df = df.append(res)
# send data to db each 150K rows and empty local df so it won't get too big
if len(df) > 150000: # make this number a class property?
store.write(price_func(df.fillna(value=np.nan)))
df = pd.DataFrame()
i += 1
if len(df) > 0:
store.write(price_func(df.fillna(value=np.nan)))
return n
def _get_data(library, q_type, database, symbol, **kwargs):
"""
General method to get quotes data from storage
:param library: storage library name (usually corresponds to a data provider name)
:param q_type: one of 'futures' | 'currency' | 'others'
:param database: local storage database name
:param symbol: local storage symbol name
:return: pd.DataFrame or None in case of error
"""
try:
return Store(library, q_type, database + '_' + symbol).get()
except Exception as e:
logger.warning("Something went wrong on symbol %s_%s request from storage: %s" %
(database, symbol, e))
return None
def download_table(self,
q_type,
database,
symbol,
db_symbol=None,
**kwargs):
# symbol name for the DB storage may be different from what we send to quandl API (e.g. for futures)
if db_symbol is None:
db_symbol = symbol
# specify the format function for the table (depends on quotes type)
formnat_fn = self.quotes_formats[q_type]
# for some spot prices the data column is specified explicitly in instruments.py
# in such cases we pass this column to a format function and save it to database as 'close'
if 'col' in kwargs.keys():
formnat_fn = partial(formnat_fn, column=kwargs.get('col'))
# pass currency object to scale the rate values where needed
if 'currency' in kwargs.keys():
formnat_fn = partial(formnat_fn, currency=kwargs.get('currency'))
# pass spot object to apply multiplier on data format
if 'spot' in kwargs.keys():
formnat_fn = partial(formnat_fn, spot=kwargs.get('spot'))
# pass instrument and contract to format futures data
if 'instrument' in kwargs.keys():
formnat_fn = partial(formnat_fn,
instrument=kwargs.get('instrument'),
contract=kwargs.get('contract'))
try:
data = quandl.get(database + '/' + symbol)
Store(self.library, q_type,
database + '_' + db_symbol).update(formnat_fn(data=data))
logger.debug('Wrote data for %s/%s' % (database, symbol))
sleep(self.api_delay)
except JSONDecodeError:
logger.warning("JSONDecodeError")
return False
except quandl.errors.quandl_error.NotFoundError:
logger.debug('Symbol %s not found on database %s' %
(symbol, database))
return False
except quandl.errors.quandl_error.LimitExceededError:
logger.warning('Quandl API limit exceeded!')
return False
except Exception as e:
logger.warning('Unexpected error occured: %s' % e)
return False
return True
def download_currency(self, currency, **kwargs):
"""
:param currency: core.trade.Currency object
:param kwargs: 'clean=True' will force delete and re-download all data,
otherwise db will be updated with the new data
"""
logger.info('Downloading currency data for %s' %
currency.bitmex_symbol)
clean = kwargs.get('clean', False)
if clean:
self.drop_symbol(currency.bitmex_symbol)
last_timestamp = None
else: # get the last timestamp and contract from the sorted index
last_timestamp = Store(
self.library, '',
currency.bitmex_symbol).get().index[-1].to_pydatetime()
n = self.download_table(currency.bitmex_symbol, last_timestamp,
self._augment_trades)
return n > 0
def download_table(self,
symbol,
start_time=None,
price_func=None,
store_symbol=None):
# Get bravado client on first download call
if self.client is None:
if self.auth:
self.client = get_request_client()
else:
self.client = SwaggerClient.from_url(SPEC_URI,
config=api_config)
# In general we may need separate symbol strings for storage and to pass to API
# (as for futures), but they also may be equal (as for currencies)
if store_symbol is None:
store_symbol = symbol
if price_func is None:
price_func = self._augment_trades
store = Store(self.library, '', store_symbol)
df = pd.DataFrame()
i = 0
n = 0
while True:
try:
res = pd.DataFrame(
self.client.Trade.Trade_getBucketed(
count=500,
start=i * 500,
symbol=symbol,
binSize='1d',
startTime=start_time).result())
except bravado.exception.HTTPTooManyRequests:
logger.info('Requests limit exceeded, sleeping for 5 mins...')
sleep(300)
logger.info('Resuming download')
continue
if len(res) == 0:
break
# keep track of total records count for calling functions to know if we got any data
n += len(res)
df = df.append(res)
# send data to db each 150K rows and empty local df so it won't get too big
if len(df) > 150000: # make this number a class property?
store.write(price_func(df.fillna(value=np.nan)))
df = pd.DataFrame()
i += 1
if len(df) > 0:
store.write(price_func(df.fillna(value=np.nan)))
return n
def _historical_data_handler(self, msg):
if int(msg.reqId) not in self.historical_data.keys():
self.historical_data[int(msg.reqId)] = []
msg_dict = dict(zip(msg.keys(), msg.values()))
if msg.close > 0:
# We've got an incoming history line, save it to a buffer
self.historical_data[int(msg.reqId)].append(msg_dict)
else:
# If msg.close = 0 then we've reached the end of the history. IB sends a zero line to signify the end.
data = pd.DataFrame(self.historical_data[int(msg.reqId)])
data['date'] = pd.to_datetime(data['date'], format="%Y%m%d")
contract = self.historical_data_req_contract[int(msg.reqId)]
dbg_message = 'Wrote data for symbol %s' % contract.m_symbol
if contract.m_secType == 'FUT': # futures
data = self._format_future(data, contract.m_expiry)
dbg_message += ', contract %s' % contract.m_expiry
store_symbol = '_'.join([contract.m_exchange, contract.m_symbol])
q_type = QuotesType.futures
elif contract.m_secType == 'CASH': # currency
data = self._format_currency(data, contract.currency_object)
dbg_message += contract.m_currency
store_symbol = '_'.join([contract.m_exchange, contract.m_symbol + contract.m_currency])
q_type = QuotesType.currency
elif contract.m_secType == 'IND': # indices
data = self._format_other(data, contract.spot_object)
dbg_message += ' (index)'
q_type = QuotesType.others
store_symbol = '_'.join([contract.m_exchange, contract.m_symbol])
else:
raise Exception('Attempt to download data of unsupported secType')
Store(self.library, q_type, store_symbol).update(data)
logger.debug(dbg_message)
self.historical_data_result[int(msg.reqId)] = True
self.historical_data_event.set()
def drop_symbol(self, q_type, database, symbol, **kwargs):
Store(self.library, q_type, database + '_' + symbol).delete()
def drop_symbol(self, symbol, database=''):
store = Store(self.library, database, symbol)
store.delete()
def drop_symbol(self, q_type, exchange, symbol, **kwargs):
Store(self.library, q_type, exchange + '_' + symbol).delete()
def drop_symbol(self, symbol, database=''):
store = Store(self.library, database, symbol)
store.delete()