def construct_vendor_md_request(self, md_request): """Creates a MarketDataRequest with the vendor tickers Parameters ---------- md_request : MarketDataRequest contains all the various parameters detailing time series start and finish, tickers etc Returns ------- MarketDataRequest """ symbols_vendor = self.translate_to_vendor_ticker(md_request) fields_vendor = self.translate_to_vendor_field(md_request) md_request_vendor = MarketDataRequest( md_request=md_request) md_request_vendor.tickers = symbols_vendor md_request_vendor.fields = fields_vendor md_request_vendor.old_tickers = md_request.tickers return md_request_vendor
def get_fx_cross_tick(self, start, end, cross, cut="NYC", data_source="dukascopy", cache_algo='internet_load_return', type='spot', environment='backtest', fields=['bid', 'ask']): if isinstance(cross, str): cross = [cross] market_data_request = MarketDataRequest( gran_freq="tick", freq_mult=1, freq='tick', cut=cut, fields=['bid', 'ask', 'bidv', 'askv'], cache_algo=cache_algo, environment=environment, start_date=start, finish_date=end, data_source=data_source, category='fx') market_data_generator = self.market_data_generator data_frame_agg = None for cr in cross: if (type == 'spot'): market_data_request.tickers = cr cross_vals = market_data_generator.fetch_market_data( market_data_request) # if user only wants 'close' calculate that from the bid/ask fields if fields == ['close']: cross_vals = cross_vals[[cr + '.bid', cr + '.ask']].mean(axis=1) cross_vals.columns = [cr + '.close'] else: filter = Filter() filter_columns = [cr + '.' + f for f in fields] cross_vals = filter.filter_time_series_by_columns( filter_columns, cross_vals) if data_frame_agg is None: data_frame_agg = cross_vals else: data_frame_agg = data_frame_agg.join(cross_vals, how='outer') # strip the nan elements data_frame_agg = data_frame_agg.dropna() return data_frame_agg
def fetch_single_time_series(self, market_data_request): market_data_request = MarketDataRequest(md_request=market_data_request) # only includes those tickers have not expired yet! start_date = pandas.Timestamp(market_data_request.start_date).date() import datetime current_date = datetime.datetime.utcnow().date() from datetime import timedelta tickers = market_data_request.tickers vendor_tickers = market_data_request.vendor_tickers expiry_date = market_data_request.expiry_date config = ConfigManager().get_instance() # in many cases no expiry is defined so skip them for i in range(0, len(tickers)): try: expiry_date = config.get_expiry_for_ticker( market_data_request.data_source, tickers[i]) except: pass if expiry_date is not None: expiry_date = pandas.Timestamp(expiry_date).date() # use pandas Timestamp, a bit more robust with weird dates (can fail if comparing date vs datetime) # if the expiry is before the start date of our download don't bother downloading this ticker if expiry_date < start_date: tickers[i] = None # special case for futures-contracts which are intraday # avoid downloading if the expiry date is very far in the past # (we need this before there might be odd situations where we run on an expiry date, but still want to get # data right till expiry time) if market_data_request.category == 'futures-contracts' and market_data_request.freq == 'intraday' \ and self.days_expired_intraday_contract_download > 0: if expiry_date + timedelta( days=self.days_expired_intraday_contract_download ) < current_date: tickers[i] = None if vendor_tickers is not None and tickers[i] is None: vendor_tickers[i] = None market_data_request.tickers = [e for e in tickers if e != None] if vendor_tickers is not None: market_data_request.vendor_tickers = [ e for e in vendor_tickers if e != None ] data_frame_single = None if len(market_data_request.tickers) > 0: data_frame_single = self.get_data_vendor( market_data_request.data_source).load_ticker( market_data_request) #print(data_frame_single.head(n=10)) if data_frame_single is not None: if data_frame_single.empty == False: data_frame_single.index.name = 'Date' # will fail for dataframes which includes dates/strings (eg. futures contract names) try: data_frame_single = data_frame_single.astype('float32') except: self.logger.warning('Could not convert to float') if market_data_request.freq == "second": data_frame_single = data_frame_single.resample("1s") return data_frame_single
def fetch_single_time_series(self, md_request): md_request = MarketDataRequest(md_request=md_request) # Only includes those tickers have not expired yet! start_date = pd.Timestamp(md_request.start_date).date() current_date = pd.Timestamp(datetime.datetime.utcnow().date()) tickers = md_request.tickers vendor_tickers = md_request.vendor_tickers expiry_date = pd.Timestamp(md_request.expiry_date) config = ConfigManager().get_instance() # In many cases no expiry is defined so skip them for i in range(0, len(tickers)): try: expiry_date = config.get_expiry_for_ticker( md_request.data_source, tickers[i]) except: pass if expiry_date is not None: expiry_date = pd.Timestamp(expiry_date) if not (pd.isna(expiry_date)): # Use pandas Timestamp, a bit more robust with weird dates # (can fail if comparing date vs datetime) # if the expiry is before the start date of our download # don"t bother downloading this ticker if expiry_date < start_date: tickers[i] = None # Special case for futures-contracts which are intraday # avoid downloading if the expiry date is very far in the # past # (we need this before there might be odd situations where # we run on an expiry date, but still want to get # data right till expiry time) if md_request.category == "futures-contracts" \ and md_request.freq == "intraday" \ and self._days_expired_intraday_contract_download \ > 0: if expiry_date + pd.Timedelta( days= self._days_expired_intraday_contract_download) \ < current_date: tickers[i] = None if vendor_tickers is not None and tickers[i] is None: vendor_tickers[i] = None md_request.tickers = [e for e in tickers if e != None] if vendor_tickers is not None: md_request.vendor_tickers = [e for e in vendor_tickers if e != None] df_single = None if len(md_request.tickers) > 0: df_single = self.get_data_vendor( md_request).load_ticker(md_request) if df_single is not None: if df_single.empty == False: df_single.index.name = "Date" # Will fail for DataFrames which includes dates/strings # eg. futures contract names df_single = Calculations().convert_to_numeric_dataframe( df_single) if md_request.freq == "second": df_single = df_single.resample("1s") return df_single