def test_markets(self): population = 3 results = dict() exchanges = select_random_exchanges(population) # Type: list[Exchange] for exchange in exchanges: assets = self._test_markets_exchange(exchange) if assets is not None: results[exchange.name] = len(assets) folder = get_exchange_folder(exchange.name) filename = os.path.join(folder, 'whitelist.json') symbols = [asset.symbol for asset in assets] with open(filename, 'wt') as handle: json.dump(symbols, handle, indent=4) series = pd.Series(results) print('the tested markets\n{}'.format(series)) if population is not None: assert (len(results) == population) pass
def test_markets(self): population = 3 results = dict() exchanges = select_random_exchanges(population) # Type: list[Exchange] for exchange in exchanges: assets = self._test_markets_exchange(exchange) if assets is not None: results[exchange.name] = len(assets) folder = get_exchange_folder(exchange.name) filename = os.path.join(folder, 'whitelist.json') symbols = [asset.symbol for asset in assets] with open(filename, 'wt') as handle: json.dump(symbols, handle, indent=4) series = pd.Series(results) print('the tested markets\n{}'.format(series)) if population is not None: assert (len(results) == population) pass
def get_reader(self, data_frequency, path=None): """ Get a data writer object, either a new object or from cache Returns ------- BcolzMinuteBarReader | BcolzDailyBarReader """ if path is None: root = get_exchange_folder(self.exchange_name) path = BUNDLE_NAME_TEMPLATE.format( root=root, frequency=data_frequency ) if path in self._readers and self._readers[path] is not None: return self._readers[path] try: self._readers[path] = BcolzExchangeBarReader( rootdir=path, data_frequency=data_frequency ) except IOError: self._readers[path] = None return self._readers[path]
def init(self): if self._is_init: return exchange_folder = get_exchange_folder(self.name) filename = os.path.join(exchange_folder, 'cctx_markets.json') if os.path.exists(filename): timestamp = os.path.getmtime(filename) dt = pd.to_datetime(timestamp, unit='s', utc=True) if dt >= pd.Timestamp.utcnow().floor('1D'): with open(filename) as f: self.markets = json.load(f) log.debug('loaded markets for {}'.format(self.name)) if self.markets is None: try: markets_symbols = self.api.load_markets() log.debug( 'fetching {} markets:\n{}'.format( self.name, markets_symbols ) ) self.markets = self.api.fetch_markets() with open(filename, 'w+') as f: json.dump(self.markets, f, indent=4) except ExchangeNotAvailable as e: raise ExchangeRequestError(error=e) self.load_assets() self._is_init = True
def get_exchange(exchange_name, quote_currency=None, must_authenticate=False, skip_init=False, auth_alias=None): key = (exchange_name, quote_currency) if key in exchange_cache: return exchange_cache[key] exchange_auth = get_exchange_auth(exchange_name, alias=auth_alias) has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '') if must_authenticate and not has_auth: raise ExchangeAuthEmpty(exchange=exchange_name.title(), filename=os.path.join( get_exchange_folder(exchange_name), 'auth.json')) exchange = CCXT( exchange_name=exchange_name, key=exchange_auth['key'], secret=exchange_auth['secret'], password=exchange_auth['password'] if 'password' in exchange_auth.keys() else '', quote_currency=quote_currency, ) exchange_cache[key] = exchange if not skip_init: exchange.init() return exchange
def get_exchange(exchange_name, base_currency=None, must_authenticate=False, skip_init=False): key = (exchange_name, base_currency) if key in exchange_cache: return exchange_cache[key] exchange_auth = get_exchange_auth(exchange_name) has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '') if must_authenticate and not has_auth: raise ExchangeAuthEmpty(exchange=exchange_name.title(), filename=os.path.join( get_exchange_folder(exchange_name), 'auth.json')) exchange = CCXT( exchange_name=exchange_name, key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, ) exchange_cache[key] = exchange if not skip_init: exchange.init() return exchange
def get_exchange(exchange_name, base_currency=None, must_authenticate=False, skip_init=False, auth_alias=None): key = (exchange_name, base_currency) if key in exchange_cache: return exchange_cache[key] exchange_auth = get_exchange_auth(exchange_name, alias=auth_alias) has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '') if must_authenticate and not has_auth: raise ExchangeAuthEmpty( exchange=exchange_name.title(), filename=os.path.join( get_exchange_folder(exchange_name), 'auth.json' ) ) exchange = CCXT( exchange_name=exchange_name, key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, ) exchange_cache[key] = exchange if not skip_init: exchange.init() return exchange
def get_writer(self, start_dt, end_dt, data_frequency): """ Get a data writer object, either a new object or from cache Returns ------- BcolzMinuteBarWriter | BcolzDailyBarWriter """ root = get_exchange_folder(self.exchange_name) path = BUNDLE_NAME_TEMPLATE.format( root=root, frequency=data_frequency ) if path in self._writers: return self._writers[path] ensure_directory(path) if len(os.listdir(path)) > 0: metadata = BcolzMinuteBarMetadata.read(path) write_metadata = False if start_dt < metadata.start_session: write_metadata = True start_session = start_dt else: start_session = metadata.start_session if end_dt > metadata.end_session: write_metadata = True end_session = end_dt else: end_session = metadata.end_session self._writers[path] = \ BcolzExchangeBarWriter( rootdir=path, start_session=start_session, end_session=end_session, write_metadata=write_metadata, data_frequency=data_frequency ) else: self._writers[path] = BcolzExchangeBarWriter( rootdir=path, start_session=start_dt, end_session=end_dt, write_metadata=True, data_frequency=data_frequency ) return self._writers[path]
def handle_exchange_error(exchange, e): try: message = '{}: {}'.format(e.__class__, e.message.decode('ascii', 'ignore')) except Exception: message = 'unexpected error' folder = get_exchange_folder(exchange.name) filename = os.path.join(folder, 'blacklist.txt') with open(filename, 'wt') as handle: handle.write(message)
def handle_exchange_error(exchange, e): try: message = '{}: {}'.format( e.__class__, e.message.decode('ascii', 'ignore') ) except Exception: message = 'unexpected error' folder = get_exchange_folder(exchange.name) filename = os.path.join(folder, 'blacklist.txt') with open(filename, 'wt') as handle: handle.write(message)
def clean(self, data_frequency): """ Removing the bundle data from the catalyst folder. Parameters ---------- data_frequency: str """ log.debug('cleaning exchange {}, frequency {}'.format( self.exchange_name, data_frequency )) root = get_exchange_folder(self.exchange_name) symbols = os.path.join(root, 'symbols.json') if os.path.isfile(symbols): os.remove(symbols) local_symbols = os.path.join(root, 'symbols_local.json') if os.path.isfile(local_symbols): os.remove(local_symbols) temp_bundles = os.path.join(root, 'temp_bundles') if os.path.isdir(temp_bundles): log.debug('removing folder and content: {}'.format(temp_bundles)) shutil.rmtree(temp_bundles) log.debug('{} removed'.format(temp_bundles)) frequencies = ['daily', 'minute'] if data_frequency is None \ else [data_frequency] for frequency in frequencies: label = '{}_bundle'.format(frequency) frequency_bundle = os.path.join(root, label) if os.path.isdir(frequency_bundle): log.debug( 'removing folder and content: {}'.format(frequency_bundle) ) shutil.rmtree(frequency_bundle) log.debug('{} removed'.format(frequency_bundle))
def init(self): if self._is_init: return exchange_folder = get_exchange_folder(self.name) filename = os.path.join(exchange_folder, 'cctx_markets.json') if os.path.exists(filename): timestamp = os.path.getmtime(filename) dt = pd.to_datetime(timestamp, unit='s', utc=True) if dt >= pd.Timestamp.utcnow().floor('1D'): with open(filename) as f: self.markets = json.load(f) log.debug('loaded markets for {}'.format(self.name)) if self.markets is None: try: markets_symbols = self.api.load_markets() log.debug( 'fetching {} markets:\n{}'.format( self.name, markets_symbols ) ) self.markets = self.api.fetch_markets() with open(filename, 'w+') as f: json.dump(self.markets, f, indent=4) except (ExchangeError, NetworkError) as e: log.warn( 'unable to fetch markets {}: {}'.format( self.name, e ) ) raise ExchangeRequestError(error=e) self.load_assets() self._is_init = True
def init(self): if self._is_init: return exchange_folder = get_exchange_folder(self.name) filename = os.path.join(exchange_folder, 'cctx_markets.json') if os.path.exists(filename): timestamp = os.path.getmtime(filename) dt = pd.to_datetime(timestamp, unit='s', utc=True) # offline Manual update # if dt >= pd.Timestamp.utcnow().floor('1D'): if True: log.warn( '==> loaded markets for {}, file update time {}'.format( self.name, dt)) with open(filename) as f: self.markets = json.load(f) log.debug('loaded markets for {}'.format(self.name)) if self.markets is None: try: markets_symbols = self.api.load_markets() log.debug('fetching {} markets:\n{}'.format( self.name, markets_symbols)) self.markets = self.api.fetch_markets() with open(filename, 'w+') as f: json.dump(self.markets, f, indent=4) except (ExchangeError, NetworkError) as e: log.warn('unable to fetch markets {}: {}'.format(self.name, e)) raise ExchangeRequestError(error=e) self.load_assets() self._is_init = True
def update_symbols_file(self, assets): if self.exchange is None: # Avoid circular dependencies from catalyst.exchange.utils.factory import get_exchange self.exchange = get_exchange(self.exchange_name) # check if the symbols.json file was updated today try: root = get_exchange_folder(self.exchange_name) timestamp = os.path.getmtime(os.path.join(root, 'symbols.json')) file_dt = pd.to_datetime(timestamp, unit='s', utc=True) except FileNotFoundError: file_dt = None log.info("updating symbols.json") try: existing_symbols_defs = get_exchange_symbols(self.exchange_name) except ExchangeSymbolsNotFound: existing_symbols_defs = {} self.exchange.api.load_markets() results = {} for asset in assets: if asset.symbol in INGEST_PAIRS_INCLUDED or self._matches_included_quote( asset.symbol): if asset.exchange_symbol in existing_symbols_defs: existing_def = existing_symbols_defs[asset.exchange_symbol] if self.exchange.api.markets[asset.asset_name.replace( ' ', '')]['active']: end_date = pd.Timestamp.utcnow().floor('1D') existing_def['end_minute'] = end_date existing_def['end_daily'] = end_date log.debug("updated {} symbol -> [still active]", asset.symbol) results[asset.exchange_symbol] = existing_def continue elif file_dt is not None and pd.Timestamp( existing_def['end_daily']) < file_dt.floor('1D'): log.debug("updated {} symbol -> [already delisted]", asset.symbol) results[asset.exchange_symbol] = existing_def continue # either the symbol is new or it has been delisted since the last update try: end_results = self.exchange.get_candles( freq='1H', assets=asset, start_dt=None, end_dt=None, bar_count=1, keep_empty_start=True) if len(end_results) == 0: raise Exception("no end cancles found for {}", asset.symbol) last_date = end_results[-1]['last_traded'].floor('1D') start_results = self.exchange.get_candles( freq='1D', assets=asset, start_dt=pd.Timestamp("2009-01-01", tz='utc'), end_dt=None, bar_count=1, keep_empty_start=True) if len(start_results) == 0: raise Exception("no start cancles found for {}", asset.symbol) first_date = start_results[-1]['last_traded'].floor('1D') symbol_dates = { 'end_minute': last_date, 'end_daily': last_date, 'start_date': first_date, 'symbol': asset.symbol } if last_date != pd.Timestamp.utcnow().floor('1D'): log.info("updated {} symbol [new delisted]", asset.symbol) else: log.info("updated {} symbol [new listed]", asset.symbol) results[asset.exchange_symbol] = symbol_dates except: log.exception("error building symbol dates for {}".format( asset.symbol)) pass save_exchange_symbols_dicts(self.exchange_name, results)
def test_daily_data_to_minute_table(self): exchange_name = 'poloniex' # Switch between daily and minute for testing data_frequency = 'daily' # data_frequency = 'minute' exchange = get_exchange(exchange_name) assets = [ exchange.get_asset('eth_btc'), exchange.get_asset('etc_btc'), ] start = pd.to_datetime('2017-9-1', utc=True) end = pd.to_datetime('2017-9-30', utc=True) # Preparing the bundle folder root = get_exchange_folder(exchange.name) path = BUNDLE_NAME_TEMPLATE.format(root=root, frequency=data_frequency) ensure_directory(path) exchange_bundle = ExchangeBundle(exchange) # We are using a BcolzMinuteBarWriter even though the data is daily # Each day has a maximum of one bar # I tried setting the minutes_per_day to 1 will not create # unnecessary bars writer = BcolzExchangeBarWriter(rootdir=path, data_frequency=data_frequency, start_session=start, end_session=end, write_metadata=True) # This will read the daily data in a bundle created by # the daily writer. It will write to the minute writer which # we are passing. # Ingesting a second asset to ensure that multiple chunks # don't override each other for asset in assets: exchange_bundle.ingest_ctable(asset=asset, data_frequency=data_frequency, period='2017', start_dt=start, end_dt=end, writer=writer, empty_rows_behavior='strip') reader = BcolzExchangeBarReader(rootdir=path, data_frequency=data_frequency) # Reading the two assets to ensure that no data was lost for asset in assets: sid = asset.sid daily_values = reader.load_raw_arrays( fields=['open', 'high', 'low', 'close', 'volume'], start_dt=start, end_dt=end, sids=[sid], ) print('found {} rows for last ingestion'.format( len(daily_values[0]))) pass