def test_markets(self):
population = 3
results = dict()
exchanges = select_random_exchanges(population) # Type: list[Exchange]
for exchange in exchanges:
assets = self._test_markets_exchange(exchange)
if assets is not None:
results[exchange.name] = len(assets)
folder = get_exchange_folder(exchange.name)
filename = os.path.join(folder, 'whitelist.json')
symbols = [asset.symbol for asset in assets]
with open(filename, 'wt') as handle:
json.dump(symbols, handle, indent=4)
series = pd.Series(results)
print('the tested markets\n{}'.format(series))
if population is not None:
assert (len(results) == population)
pass
def test_markets(self):
population = 3
results = dict()
exchanges = select_random_exchanges(population) # Type: list[Exchange]
for exchange in exchanges:
assets = self._test_markets_exchange(exchange)
if assets is not None:
results[exchange.name] = len(assets)
folder = get_exchange_folder(exchange.name)
filename = os.path.join(folder, 'whitelist.json')
symbols = [asset.symbol for asset in assets]
with open(filename, 'wt') as handle:
json.dump(symbols, handle, indent=4)
series = pd.Series(results)
print('the tested markets\n{}'.format(series))
if population is not None:
assert (len(results) == population)
pass
def get_reader(self, data_frequency, path=None):
"""
Get a data writer object, either a new object or from cache
Returns
-------
BcolzMinuteBarReader | BcolzDailyBarReader
"""
if path is None:
root = get_exchange_folder(self.exchange_name)
path = BUNDLE_NAME_TEMPLATE.format(
root=root,
frequency=data_frequency
)
if path in self._readers and self._readers[path] is not None:
return self._readers[path]
try:
self._readers[path] = BcolzExchangeBarReader(
rootdir=path,
data_frequency=data_frequency
)
except IOError:
self._readers[path] = None
return self._readers[path]
def init(self):
if self._is_init:
return
exchange_folder = get_exchange_folder(self.name)
filename = os.path.join(exchange_folder, 'cctx_markets.json')
if os.path.exists(filename):
timestamp = os.path.getmtime(filename)
dt = pd.to_datetime(timestamp, unit='s', utc=True)
if dt >= pd.Timestamp.utcnow().floor('1D'):
with open(filename) as f:
self.markets = json.load(f)
log.debug('loaded markets for {}'.format(self.name))
if self.markets is None:
try:
markets_symbols = self.api.load_markets()
log.debug(
'fetching {} markets:\n{}'.format(
self.name, markets_symbols
)
)
self.markets = self.api.fetch_markets()
with open(filename, 'w+') as f:
json.dump(self.markets, f, indent=4)
except ExchangeNotAvailable as e:
raise ExchangeRequestError(error=e)
self.load_assets()
self._is_init = True
def get_exchange(exchange_name,
quote_currency=None,
must_authenticate=False,
skip_init=False,
auth_alias=None):
key = (exchange_name, quote_currency)
if key in exchange_cache:
return exchange_cache[key]
exchange_auth = get_exchange_auth(exchange_name, alias=auth_alias)
has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
if must_authenticate and not has_auth:
raise ExchangeAuthEmpty(exchange=exchange_name.title(),
filename=os.path.join(
get_exchange_folder(exchange_name),
'auth.json'))
exchange = CCXT(
exchange_name=exchange_name,
key=exchange_auth['key'],
secret=exchange_auth['secret'],
password=exchange_auth['password']
if 'password' in exchange_auth.keys() else '',
quote_currency=quote_currency,
)
exchange_cache[key] = exchange
if not skip_init:
exchange.init()
return exchange
def get_exchange(exchange_name,
base_currency=None,
must_authenticate=False,
skip_init=False):
key = (exchange_name, base_currency)
if key in exchange_cache:
return exchange_cache[key]
exchange_auth = get_exchange_auth(exchange_name)
has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
if must_authenticate and not has_auth:
raise ExchangeAuthEmpty(exchange=exchange_name.title(),
filename=os.path.join(
get_exchange_folder(exchange_name),
'auth.json'))
exchange = CCXT(
exchange_name=exchange_name,
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
)
exchange_cache[key] = exchange
if not skip_init:
exchange.init()
return exchange
def get_exchange(exchange_name, base_currency=None, must_authenticate=False,
skip_init=False, auth_alias=None):
key = (exchange_name, base_currency)
if key in exchange_cache:
return exchange_cache[key]
exchange_auth = get_exchange_auth(exchange_name, alias=auth_alias)
has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
if must_authenticate and not has_auth:
raise ExchangeAuthEmpty(
exchange=exchange_name.title(),
filename=os.path.join(
get_exchange_folder(exchange_name), 'auth.json'
)
)
exchange = CCXT(
exchange_name=exchange_name,
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
)
exchange_cache[key] = exchange
if not skip_init:
exchange.init()
return exchange
def get_writer(self, start_dt, end_dt, data_frequency):
"""
Get a data writer object, either a new object or from cache
Returns
-------
BcolzMinuteBarWriter | BcolzDailyBarWriter
"""
root = get_exchange_folder(self.exchange_name)
path = BUNDLE_NAME_TEMPLATE.format(
root=root,
frequency=data_frequency
)
if path in self._writers:
return self._writers[path]
ensure_directory(path)
if len(os.listdir(path)) > 0:
metadata = BcolzMinuteBarMetadata.read(path)
write_metadata = False
if start_dt < metadata.start_session:
write_metadata = True
start_session = start_dt
else:
start_session = metadata.start_session
if end_dt > metadata.end_session:
write_metadata = True
end_session = end_dt
else:
end_session = metadata.end_session
self._writers[path] = \
BcolzExchangeBarWriter(
rootdir=path,
start_session=start_session,
end_session=end_session,
write_metadata=write_metadata,
data_frequency=data_frequency
)
else:
self._writers[path] = BcolzExchangeBarWriter(
rootdir=path,
start_session=start_dt,
end_session=end_dt,
write_metadata=True,
data_frequency=data_frequency
)
return self._writers[path]
def handle_exchange_error(exchange, e):
try:
message = '{}: {}'.format(e.__class__,
e.message.decode('ascii', 'ignore'))
except Exception:
message = 'unexpected error'
folder = get_exchange_folder(exchange.name)
filename = os.path.join(folder, 'blacklist.txt')
with open(filename, 'wt') as handle:
handle.write(message)
def handle_exchange_error(exchange, e):
try:
message = '{}: {}'.format(
e.__class__, e.message.decode('ascii', 'ignore')
)
except Exception:
message = 'unexpected error'
folder = get_exchange_folder(exchange.name)
filename = os.path.join(folder, 'blacklist.txt')
with open(filename, 'wt') as handle:
handle.write(message)
def clean(self, data_frequency):
"""
Removing the bundle data from the catalyst folder.
Parameters
----------
data_frequency: str
"""
log.debug('cleaning exchange {}, frequency {}'.format(
self.exchange_name, data_frequency
))
root = get_exchange_folder(self.exchange_name)
symbols = os.path.join(root, 'symbols.json')
if os.path.isfile(symbols):
os.remove(symbols)
local_symbols = os.path.join(root, 'symbols_local.json')
if os.path.isfile(local_symbols):
os.remove(local_symbols)
temp_bundles = os.path.join(root, 'temp_bundles')
if os.path.isdir(temp_bundles):
log.debug('removing folder and content: {}'.format(temp_bundles))
shutil.rmtree(temp_bundles)
log.debug('{} removed'.format(temp_bundles))
frequencies = ['daily', 'minute'] if data_frequency is None \
else [data_frequency]
for frequency in frequencies:
label = '{}_bundle'.format(frequency)
frequency_bundle = os.path.join(root, label)
if os.path.isdir(frequency_bundle):
log.debug(
'removing folder and content: {}'.format(frequency_bundle)
)
shutil.rmtree(frequency_bundle)
log.debug('{} removed'.format(frequency_bundle))
def init(self):
if self._is_init:
return
exchange_folder = get_exchange_folder(self.name)
filename = os.path.join(exchange_folder, 'cctx_markets.json')
if os.path.exists(filename):
timestamp = os.path.getmtime(filename)
dt = pd.to_datetime(timestamp, unit='s', utc=True)
if dt >= pd.Timestamp.utcnow().floor('1D'):
with open(filename) as f:
self.markets = json.load(f)
log.debug('loaded markets for {}'.format(self.name))
if self.markets is None:
try:
markets_symbols = self.api.load_markets()
log.debug(
'fetching {} markets:\n{}'.format(
self.name, markets_symbols
)
)
self.markets = self.api.fetch_markets()
with open(filename, 'w+') as f:
json.dump(self.markets, f, indent=4)
except (ExchangeError, NetworkError) as e:
log.warn(
'unable to fetch markets {}: {}'.format(
self.name, e
)
)
raise ExchangeRequestError(error=e)
self.load_assets()
self._is_init = True
def init(self):
if self._is_init:
return
exchange_folder = get_exchange_folder(self.name)
filename = os.path.join(exchange_folder, 'cctx_markets.json')
if os.path.exists(filename):
timestamp = os.path.getmtime(filename)
dt = pd.to_datetime(timestamp, unit='s', utc=True)
# offline Manual update
# if dt >= pd.Timestamp.utcnow().floor('1D'):
if True:
log.warn(
'==> loaded markets for {}, file update time {}'.format(
self.name, dt))
with open(filename) as f:
self.markets = json.load(f)
log.debug('loaded markets for {}'.format(self.name))
if self.markets is None:
try:
markets_symbols = self.api.load_markets()
log.debug('fetching {} markets:\n{}'.format(
self.name, markets_symbols))
self.markets = self.api.fetch_markets()
with open(filename, 'w+') as f:
json.dump(self.markets, f, indent=4)
except (ExchangeError, NetworkError) as e:
log.warn('unable to fetch markets {}: {}'.format(self.name, e))
raise ExchangeRequestError(error=e)
self.load_assets()
self._is_init = True
def update_symbols_file(self, assets):
if self.exchange is None:
# Avoid circular dependencies
from catalyst.exchange.utils.factory import get_exchange
self.exchange = get_exchange(self.exchange_name)
# check if the symbols.json file was updated today
try:
root = get_exchange_folder(self.exchange_name)
timestamp = os.path.getmtime(os.path.join(root, 'symbols.json'))
file_dt = pd.to_datetime(timestamp, unit='s', utc=True)
except FileNotFoundError:
file_dt = None
log.info("updating symbols.json")
try:
existing_symbols_defs = get_exchange_symbols(self.exchange_name)
except ExchangeSymbolsNotFound:
existing_symbols_defs = {}
self.exchange.api.load_markets()
results = {}
for asset in assets:
if asset.symbol in INGEST_PAIRS_INCLUDED or self._matches_included_quote(
asset.symbol):
if asset.exchange_symbol in existing_symbols_defs:
existing_def = existing_symbols_defs[asset.exchange_symbol]
if self.exchange.api.markets[asset.asset_name.replace(
' ', '')]['active']:
end_date = pd.Timestamp.utcnow().floor('1D')
existing_def['end_minute'] = end_date
existing_def['end_daily'] = end_date
log.debug("updated {} symbol -> [still active]",
asset.symbol)
results[asset.exchange_symbol] = existing_def
continue
elif file_dt is not None and pd.Timestamp(
existing_def['end_daily']) < file_dt.floor('1D'):
log.debug("updated {} symbol -> [already delisted]",
asset.symbol)
results[asset.exchange_symbol] = existing_def
continue
# either the symbol is new or it has been delisted since the last update
try:
end_results = self.exchange.get_candles(
freq='1H',
assets=asset,
start_dt=None,
end_dt=None,
bar_count=1,
keep_empty_start=True)
if len(end_results) == 0:
raise Exception("no end cancles found for {}",
asset.symbol)
last_date = end_results[-1]['last_traded'].floor('1D')
start_results = self.exchange.get_candles(
freq='1D',
assets=asset,
start_dt=pd.Timestamp("2009-01-01", tz='utc'),
end_dt=None,
bar_count=1,
keep_empty_start=True)
if len(start_results) == 0:
raise Exception("no start cancles found for {}",
asset.symbol)
first_date = start_results[-1]['last_traded'].floor('1D')
symbol_dates = {
'end_minute': last_date,
'end_daily': last_date,
'start_date': first_date,
'symbol': asset.symbol
}
if last_date != pd.Timestamp.utcnow().floor('1D'):
log.info("updated {} symbol [new delisted]",
asset.symbol)
else:
log.info("updated {} symbol [new listed]",
asset.symbol)
results[asset.exchange_symbol] = symbol_dates
except:
log.exception("error building symbol dates for {}".format(
asset.symbol))
pass
save_exchange_symbols_dicts(self.exchange_name, results)
def test_daily_data_to_minute_table(self):
exchange_name = 'poloniex'
# Switch between daily and minute for testing
data_frequency = 'daily'
# data_frequency = 'minute'
exchange = get_exchange(exchange_name)
assets = [
exchange.get_asset('eth_btc'),
exchange.get_asset('etc_btc'),
]
start = pd.to_datetime('2017-9-1', utc=True)
end = pd.to_datetime('2017-9-30', utc=True)
# Preparing the bundle folder
root = get_exchange_folder(exchange.name)
path = BUNDLE_NAME_TEMPLATE.format(root=root, frequency=data_frequency)
ensure_directory(path)
exchange_bundle = ExchangeBundle(exchange)
# We are using a BcolzMinuteBarWriter even though the data is daily
# Each day has a maximum of one bar
# I tried setting the minutes_per_day to 1 will not create
# unnecessary bars
writer = BcolzExchangeBarWriter(rootdir=path,
data_frequency=data_frequency,
start_session=start,
end_session=end,
write_metadata=True)
# This will read the daily data in a bundle created by
# the daily writer. It will write to the minute writer which
# we are passing.
# Ingesting a second asset to ensure that multiple chunks
# don't override each other
for asset in assets:
exchange_bundle.ingest_ctable(asset=asset,
data_frequency=data_frequency,
period='2017',
start_dt=start,
end_dt=end,
writer=writer,
empty_rows_behavior='strip')
reader = BcolzExchangeBarReader(rootdir=path,
data_frequency=data_frequency)
# Reading the two assets to ensure that no data was lost
for asset in assets:
sid = asset.sid
daily_values = reader.load_raw_arrays(
fields=['open', 'high', 'low', 'close', 'volume'],
start_dt=start,
end_dt=end,
sids=[sid],
)
print('found {} rows for last ingestion'.format(
len(daily_values[0])))
pass
