def construct_vendor_md_request(self, md_request):
"""Creates a MarketDataRequest with the vendor tickers
Parameters
----------
md_request : MarketDataRequest
contains all the various parameters detailing time series start and
finish, tickers etc
Returns
-------
MarketDataRequest
"""
symbols_vendor = self.translate_to_vendor_ticker(md_request)
fields_vendor = self.translate_to_vendor_field(md_request)
md_request_vendor = MarketDataRequest(
md_request=md_request)
md_request_vendor.tickers = symbols_vendor
md_request_vendor.fields = fields_vendor
md_request_vendor.old_tickers = md_request.tickers
return md_request_vendor
def get_fx_cross_tick(self,
start,
end,
cross,
cut="NYC",
data_source="dukascopy",
cache_algo='internet_load_return',
type='spot',
environment='backtest',
fields=['bid', 'ask']):
if isinstance(cross, str):
cross = [cross]
market_data_request = MarketDataRequest(
gran_freq="tick",
freq_mult=1,
freq='tick',
cut=cut,
fields=['bid', 'ask', 'bidv', 'askv'],
cache_algo=cache_algo,
environment=environment,
start_date=start,
finish_date=end,
data_source=data_source,
category='fx')
market_data_generator = self.market_data_generator
data_frame_agg = None
for cr in cross:
if (type == 'spot'):
market_data_request.tickers = cr
cross_vals = market_data_generator.fetch_market_data(
market_data_request)
# if user only wants 'close' calculate that from the bid/ask fields
if fields == ['close']:
cross_vals = cross_vals[[cr + '.bid',
cr + '.ask']].mean(axis=1)
cross_vals.columns = [cr + '.close']
else:
filter = Filter()
filter_columns = [cr + '.' + f for f in fields]
cross_vals = filter.filter_time_series_by_columns(
filter_columns, cross_vals)
if data_frame_agg is None:
data_frame_agg = cross_vals
else:
data_frame_agg = data_frame_agg.join(cross_vals, how='outer')
# strip the nan elements
data_frame_agg = data_frame_agg.dropna()
return data_frame_agg
def fetch_single_time_series(self, market_data_request):
market_data_request = MarketDataRequest(md_request=market_data_request)
# only includes those tickers have not expired yet!
start_date = pandas.Timestamp(market_data_request.start_date).date()
import datetime
current_date = datetime.datetime.utcnow().date()
from datetime import timedelta
tickers = market_data_request.tickers
vendor_tickers = market_data_request.vendor_tickers
expiry_date = market_data_request.expiry_date
config = ConfigManager().get_instance()
# in many cases no expiry is defined so skip them
for i in range(0, len(tickers)):
try:
expiry_date = config.get_expiry_for_ticker(
market_data_request.data_source, tickers[i])
except:
pass
if expiry_date is not None:
expiry_date = pandas.Timestamp(expiry_date).date()
# use pandas Timestamp, a bit more robust with weird dates (can fail if comparing date vs datetime)
# if the expiry is before the start date of our download don't bother downloading this ticker
if expiry_date < start_date:
tickers[i] = None
# special case for futures-contracts which are intraday
# avoid downloading if the expiry date is very far in the past
# (we need this before there might be odd situations where we run on an expiry date, but still want to get
# data right till expiry time)
if market_data_request.category == 'futures-contracts' and market_data_request.freq == 'intraday' \
and self.days_expired_intraday_contract_download > 0:
if expiry_date + timedelta(
days=self.days_expired_intraday_contract_download
) < current_date:
tickers[i] = None
if vendor_tickers is not None and tickers[i] is None:
vendor_tickers[i] = None
market_data_request.tickers = [e for e in tickers if e != None]
if vendor_tickers is not None:
market_data_request.vendor_tickers = [
e for e in vendor_tickers if e != None
]
data_frame_single = None
if len(market_data_request.tickers) > 0:
data_frame_single = self.get_data_vendor(
market_data_request.data_source).load_ticker(
market_data_request)
#print(data_frame_single.head(n=10))
if data_frame_single is not None:
if data_frame_single.empty == False:
data_frame_single.index.name = 'Date'
# will fail for dataframes which includes dates/strings (eg. futures contract names)
try:
data_frame_single = data_frame_single.astype('float32')
except:
self.logger.warning('Could not convert to float')
if market_data_request.freq == "second":
data_frame_single = data_frame_single.resample("1s")
return data_frame_single
def fetch_single_time_series(self, md_request):
md_request = MarketDataRequest(md_request=md_request)
# Only includes those tickers have not expired yet!
start_date = pd.Timestamp(md_request.start_date).date()
current_date = pd.Timestamp(datetime.datetime.utcnow().date())
tickers = md_request.tickers
vendor_tickers = md_request.vendor_tickers
expiry_date = pd.Timestamp(md_request.expiry_date)
config = ConfigManager().get_instance()
# In many cases no expiry is defined so skip them
for i in range(0, len(tickers)):
try:
expiry_date = config.get_expiry_for_ticker(
md_request.data_source, tickers[i])
except:
pass
if expiry_date is not None:
expiry_date = pd.Timestamp(expiry_date)
if not (pd.isna(expiry_date)):
# Use pandas Timestamp, a bit more robust with weird dates
# (can fail if comparing date vs datetime)
# if the expiry is before the start date of our download
# don"t bother downloading this ticker
if expiry_date < start_date:
tickers[i] = None
# Special case for futures-contracts which are intraday
# avoid downloading if the expiry date is very far in the
# past
# (we need this before there might be odd situations where
# we run on an expiry date, but still want to get
# data right till expiry time)
if md_request.category == "futures-contracts" \
and md_request.freq == "intraday" \
and self._days_expired_intraday_contract_download \
> 0:
if expiry_date + pd.Timedelta(
days=
self._days_expired_intraday_contract_download) \
< current_date:
tickers[i] = None
if vendor_tickers is not None and tickers[i] is None:
vendor_tickers[i] = None
md_request.tickers = [e for e in tickers if e != None]
if vendor_tickers is not None:
md_request.vendor_tickers = [e for e in vendor_tickers if
e != None]
df_single = None
if len(md_request.tickers) > 0:
df_single = self.get_data_vendor(
md_request).load_ticker(md_request)
if df_single is not None:
if df_single.empty == False:
df_single.index.name = "Date"
# Will fail for DataFrames which includes dates/strings
# eg. futures contract names
df_single = Calculations().convert_to_numeric_dataframe(
df_single)
if md_request.freq == "second":
df_single = df_single.resample("1s")
return df_single
