def main(only_exchange=None):
skip = [EXX]
files = glob.glob('*')
for f in files:
for e in EXCHANGE_MAP.keys():
if e + "." in f:
skip.append(e.split(".")[0])
print(f'Generating test data. This will take approximately {(len(EXCHANGE_MAP) - len(set(skip))) * 0.5} minutes.')
loop = asyncio.get_event_loop()
for exch_str, exchange in EXCHANGE_MAP.items() if only_exchange is None else [(only_exchange, EXCHANGE_MAP[only_exchange])]:
if exch_str in skip:
continue
print(f"Collecting data for {exch_str}")
fh = FeedHandler(raw_data_collection=AsyncFileCallback("./"), config={'uvloop': False, 'log': {'filename': 'feedhandler.log', 'level': 'WARNING'}, 'rest': {'log': {'filename': 'rest.log', 'level': 'WARNING'}}})
info = exchange.info()
channels = list(set.intersection(set(info['channels']['websocket']), set([L2_BOOK, TRADES, TICKER, CANDLES])))
sample_size = 10
if exch_str in (BINANCE_US, BINANCE):
# books of size 5000 count significantly against rate limits
sample_size = 4
while True:
try:
symbols = random.sample(info['symbols'], sample_size)
if exch_str == BINANCE_FUTURES:
symbols = [s for s in symbols if 'PINDEX' not in s]
elif exch_str == BITFINEX:
symbols = [s for s in symbols if '-' in s]
except ValueError:
sample_size -= 1
else:
break
fh.add_feed(exchange(symbols=symbols, channels=channels))
fh.run(start_loop=False)
loop.call_later(31, stop)
print("Starting feedhandler. Will run for 30 seconds...")
loop.run_forever()
fh.stop(loop=loop)
del fh
print("Checking raw message dumps for errors...")
for exch_str, _ in EXCHANGE_MAP.items():
for file in glob.glob(exch_str + "*"):
try:
print(f"Checking {file}")
check_dump(file)
except Exception as e:
print(f"File {file} failed")
print(e)
def start_cryptofeed():
async def liquidations_cb(data, receipt):
# Add raw data to CryptofeedDataTypeEnum.LIQUIDATION_DATA queue
CryptofeedService.data[CryptofeedDataTypeEnum.LIQUIDATIONS].put(
data)
async def open_interest_cb(data, receipt):
# Add raw data to CryptofeedDataTypeEnum.OPEN_INTEREST queue
CryptofeedService.data[CryptofeedDataTypeEnum.OPEN_INTEREST].put(
data)
# There is no current event loop in thread
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
f = FeedHandler()
configured = []
print("Querying exchange metadata")
for exchange_string, exchange_class in EXCHANGE_MAP.items():
if exchange_string not in EXCHANGES:
continue
if exchange_string in ['BITFLYER', 'EXX', 'OKEX'
]: # We have issues with these exchanges
continue
if all(channel in exchange_class.info()['channels']['websocket']
for channel in [LIQUIDATIONS, OPEN_INTEREST]):
configured.append(exchange_string)
print(f"Configuring {exchange_string}...", end='')
symbols = [
sym for sym in exchange_class.symbols()
if 'PINDEX' not in sym
]
try:
f.add_feed(
exchange_class(subscription={
LIQUIDATIONS: symbols,
OPEN_INTEREST: symbols
},
callbacks={
LIQUIDATIONS: liquidations_cb,
OPEN_INTEREST: open_interest_cb
}))
print(" Done")
except Exception as e:
print(e, exchange_string)
pass
print(configured)
print("Starting feedhandler for exchanges:", ', '.join(configured))
f.run(install_signal_handlers=False)
def main():
f = FeedHandler()
configured = []
print("Querying exchange metadata")
for exchange_string, exchange_class in EXCHANGE_MAP.items():
if LIQUIDATIONS in exchange_class.info()['channels']['websocket']:
configured.append(exchange_string)
symbols = [
sym for sym in exchange_class.symbols() if 'PINDEX' not in sym
]
f.add_feed(
exchange_class(subscription={LIQUIDATIONS: symbols},
callbacks={LIQUIDATIONS: liquidations}))
print("Starting feedhandler for exchanges:", ', '.join(configured))
f.run()
def load_all(self):
from cryptofeed.exchanges import EXCHANGE_MAP
for _, exchange in EXCHANGE_MAP.items():
exchange.symbols(refresh=True)