def fetch_symbol_map(self, is_local=False): index = 1 if is_local else 0 if self._symbol_maps[index] is not None: return self._symbol_maps[index] else: symbol_map = get_exchange_symbols(self.name, is_local) self._symbol_maps[index] = symbol_map return symbol_map
def universe(context, lookback_date, current_date): # get all the pairs for the given exchange json_symbols = get_exchange_symbols(context.exchange) # convert into a DataFrame for easier processing df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str) df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1], axis=1) df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0], axis=1) # Filter all the pairs to get only the ones for a given base_currency df = df[df['base_currency'] == context.base_currency] # Filter all pairs to ensure that pair existed in the current date range df = df[df.start_date < lookback_date] df = df[df.end_daily >= current_date] context.coins = symbols(*df.symbol) # convert all the pairs to symbols return df.symbol.tolist()
def update_symbols_file(self, assets): if self.exchange is None: # Avoid circular dependencies from catalyst.exchange.utils.factory import get_exchange self.exchange = get_exchange(self.exchange_name) # check if the symbols.json file was updated today try: root = get_exchange_folder(self.exchange_name) timestamp = os.path.getmtime(os.path.join(root, 'symbols.json')) file_dt = pd.to_datetime(timestamp, unit='s', utc=True) except FileNotFoundError: file_dt = None log.info("updating symbols.json") try: existing_symbols_defs = get_exchange_symbols(self.exchange_name) except ExchangeSymbolsNotFound: existing_symbols_defs = {} self.exchange.api.load_markets() results = {} for asset in assets: if asset.symbol in INGEST_PAIRS_INCLUDED or self._matches_included_quote( asset.symbol): if asset.exchange_symbol in existing_symbols_defs: existing_def = existing_symbols_defs[asset.exchange_symbol] if self.exchange.api.markets[asset.asset_name.replace( ' ', '')]['active']: end_date = pd.Timestamp.utcnow().floor('1D') existing_def['end_minute'] = end_date existing_def['end_daily'] = end_date log.debug("updated {} symbol -> [still active]", asset.symbol) results[asset.exchange_symbol] = existing_def continue elif file_dt is not None and pd.Timestamp( existing_def['end_daily']) < file_dt.floor('1D'): log.debug("updated {} symbol -> [already delisted]", asset.symbol) results[asset.exchange_symbol] = existing_def continue # either the symbol is new or it has been delisted since the last update try: end_results = self.exchange.get_candles( freq='1H', assets=asset, start_dt=None, end_dt=None, bar_count=1, keep_empty_start=True) if len(end_results) == 0: raise Exception("no end cancles found for {}", asset.symbol) last_date = end_results[-1]['last_traded'].floor('1D') start_results = self.exchange.get_candles( freq='1D', assets=asset, start_dt=pd.Timestamp("2009-01-01", tz='utc'), end_dt=None, bar_count=1, keep_empty_start=True) if len(start_results) == 0: raise Exception("no start cancles found for {}", asset.symbol) first_date = start_results[-1]['last_traded'].floor('1D') symbol_dates = { 'end_minute': last_date, 'end_daily': last_date, 'start_date': first_date, 'symbol': asset.symbol } if last_date != pd.Timestamp.utcnow().floor('1D'): log.info("updated {} symbol [new delisted]", asset.symbol) else: log.info("updated {} symbol [new listed]", asset.symbol) results[asset.exchange_symbol] = symbol_dates except: log.exception("error building symbol dates for {}".format( asset.symbol)) pass save_exchange_symbols_dicts(self.exchange_name, results)