def get_market_data(self, market_request, return_cache_handles=False):
# Handles returns a pointer
volatile_cache = Mediator.get_volatile_cache(volatile_cache_engine=self._volatile_cache_engine)
cache = True
# Don't attempt to cache DataFrames
if hasattr(market_request, 'market_data_store'):
if (isinstance(market_request.market_data_store, pd.DataFrame)):
cache = False
elif isinstance(market_request.data_store, pd.DataFrame):
cache = False
# If we have allowed the caching of monthly/periodic market data
if market_request.multithreading_params['cache_period_market_data'] and cache:
old_start_date = market_request.start_date;
old_finish_date = market_request.finish_date
# so we can also take TCARequest objects
if hasattr(market_request, 'market_data_store'):
data_store = market_request.market_data_store
data_offset_ms = market_request.market_data_offset_ms
else:
data_store = market_request.data_store
data_offset_ms = market_request.data_offset_ms
# See if we can fetch from the cache (typically Redis)
start_date, finish_date, market_key, market_df = \
volatile_cache.get_data_request_cache(market_request, data_store, 'market_df',
data_offset_ms)
# If data is already cached, just return the existing CacheHandle (which is like a pointer to the reference
# in Redis)
if market_df is not None and start_date == old_start_date and finish_date == old_finish_date and return_cache_handles:
return CacheHandle(market_key, add_time_expiry=False)
if market_df is None:
market_request_copy = MarketRequest(market_request=market_request)
market_request_copy.start_date = start_date
market_request_copy.finish_date = finish_date
market_df = super(TCATickerLoaderImpl, self).get_market_data(market_request_copy)
volatile_cache.put_data_request_cache(market_request_copy, market_key, market_df)
market_df = self._strip_start_finish_dataframe(market_df, old_start_date, old_finish_date, market_request)
else:
market_df = super(TCATickerLoaderImpl, self).get_market_data(market_request)
# Return as a cache handle (which can be easily passed across Celery for example)
# Only if use_multithreading
if return_cache_handles and market_request.use_multithreading:
return volatile_cache.put_dataframe_handle(market_df,
use_cache_handles=market_request.multithreading_params['cache_period_market_data'])
return market_df
def get_trade_order_data(self, tca_request, trade_order_type, start_date=None, finish_date=None, return_cache_handles=True):
# return_cache_handles returns a pointer
logger = LoggerManager().getLogger(__name__)
volatile_cache = Mediator.get_volatile_cache(volatile_cache_engine=self._volatile_cache_engine)
# by default, assume we want trade data (rather than order data)
if trade_order_type is None:
trade_order_type = 'trade_df'
trade_order_contents = tca_request.trade_order_mapping[trade_order_type]
cache = True
# Don't attempt to catch DataFrames (or CSVs of trades)
if isinstance(trade_order_contents, pd.DataFrame):
cache = False
elif isinstance(trade_order_contents, str):
if 'csv' in trade_order_contents:
cache = False
# If we have allowed the caching of monthly/weekly trade data
if tca_request.multithreading_params['cache_period_trade_data'] and cache:
old_start_date = tca_request.start_date; old_finish_date = tca_request.finish_date
# See if we can fetch from the cache (usually Redis)
start_date, finish_date, trade_key, trade_df = \
volatile_cache.get_data_request_cache(
tca_request, tca_request.trade_data_store, trade_order_type, tca_request.trade_data_offset_ms)
# If data is already cached, just return the existing CacheHandle
if trade_df is not None and start_date == old_start_date and finish_date == old_finish_date:
return CacheHandle(trade_key, add_time_expiry=False)
# If it wasn't in the cache then fetch it and push into the cache
if trade_df is None:
logger.debug('Key not found for ' + trade_key + ".. now need to load")
# Call the superclass (get back DataFrames not return_cache_handles)
trade_df = super(TCATickerLoaderImpl, self).get_trade_order_data(tca_request, trade_order_type,
start_date=start_date,
finish_date=finish_date)
# Cache this periodic monthly/weekly data
volatile_cache.put_data_request_cache(tca_request, trade_key, trade_df)
# Strip off the start/finish dates (because when we load from cache, we get full months)
trade_df = self._strip_start_finish_dataframe(trade_df, start_date, finish_date, tca_request)
else:
if start_date is None or finish_date is None:
start_date = tca_request.start_date
finish_date = tca_request.finish_date
# Call the superclass (get back DataFrames not return_cache_handles)
trade_df = super(TCATickerLoaderImpl, self).get_trade_order_data(tca_request, trade_order_type,
start_date=start_date,
finish_date=finish_date)
if return_cache_handles and tca_request.use_multithreading:
# Return as a cache handle (which can be easily passed across Celery for example)
return volatile_cache.put_dataframe_handle(trade_df,
use_cache_handles=tca_request.multithreading_params['cache_period_trade_data'])
return trade_df