def main(only_exchange=None):
skip = [EXX]
files = glob.glob('*')
for f in files:
for e in EXCHANGE_MAP.keys():
if e + "." in f:
skip.append(e.split(".")[0])
print(f'Generating test data. This will take approximately {(len(EXCHANGE_MAP) - len(set(skip))) * 0.5} minutes.')
loop = asyncio.get_event_loop()
for exch_str, exchange in EXCHANGE_MAP.items() if only_exchange is None else [(only_exchange, EXCHANGE_MAP[only_exchange])]:
if exch_str in skip:
continue
print(f"Collecting data for {exch_str}")
fh = FeedHandler(raw_data_collection=AsyncFileCallback("./"), config={'uvloop': False, 'log': {'filename': 'feedhandler.log', 'level': 'WARNING'}, 'rest': {'log': {'filename': 'rest.log', 'level': 'WARNING'}}})
info = exchange.info()
channels = list(set.intersection(set(info['channels']['websocket']), set([L2_BOOK, TRADES, TICKER, CANDLES])))
sample_size = 10
if exch_str in (BINANCE_US, BINANCE):
# books of size 5000 count significantly against rate limits
sample_size = 4
while True:
try:
symbols = random.sample(info['symbols'], sample_size)
if exch_str == BINANCE_FUTURES:
symbols = [s for s in symbols if 'PINDEX' not in s]
elif exch_str == BITFINEX:
symbols = [s for s in symbols if '-' in s]
except ValueError:
sample_size -= 1
else:
break
fh.add_feed(exchange(symbols=symbols, channels=channels))
fh.run(start_loop=False)
loop.call_later(31, stop)
print("Starting feedhandler. Will run for 30 seconds...")
loop.run_forever()
fh.stop(loop=loop)
del fh
print("Checking raw message dumps for errors...")
for exch_str, _ in EXCHANGE_MAP.items():
for file in glob.glob(exch_str + "*"):
try:
print(f"Checking {file}")
check_dump(file)
except Exception as e:
print(f"File {file} failed")
print(e)
def start_cryptofeed():
async def liquidations_cb(data, receipt):
# Add raw data to CryptofeedDataTypeEnum.LIQUIDATION_DATA queue
CryptofeedService.data[CryptofeedDataTypeEnum.LIQUIDATIONS].put(
data)
async def open_interest_cb(data, receipt):
# Add raw data to CryptofeedDataTypeEnum.OPEN_INTEREST queue
CryptofeedService.data[CryptofeedDataTypeEnum.OPEN_INTEREST].put(
data)
# There is no current event loop in thread
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
f = FeedHandler()
configured = []
print("Querying exchange metadata")
for exchange_string, exchange_class in EXCHANGE_MAP.items():
if exchange_string not in EXCHANGES:
continue
if exchange_string in ['BITFLYER', 'EXX', 'OKEX'
]: # We have issues with these exchanges
continue
if all(channel in exchange_class.info()['channels']['websocket']
for channel in [LIQUIDATIONS, OPEN_INTEREST]):
configured.append(exchange_string)
print(f"Configuring {exchange_string}...", end='')
symbols = [
sym for sym in exchange_class.symbols()
if 'PINDEX' not in sym
]
try:
f.add_feed(
exchange_class(subscription={
LIQUIDATIONS: symbols,
OPEN_INTEREST: symbols
},
callbacks={
LIQUIDATIONS: liquidations_cb,
OPEN_INTEREST: open_interest_cb
}))
print(" Done")
except Exception as e:
print(e, exchange_string)
pass
print(configured)
print("Starting feedhandler for exchanges:", ', '.join(configured))
f.run(install_signal_handlers=False)
def main():
f = FeedHandler()
configured = []
print("Querying exchange metadata")
for exchange_string, exchange_class in EXCHANGE_MAP.items():
if LIQUIDATIONS in exchange_class.info()['channels']['websocket']:
configured.append(exchange_string)
symbols = [
sym for sym in exchange_class.symbols() if 'PINDEX' not in sym
]
f.add_feed(
exchange_class(subscription={LIQUIDATIONS: symbols},
callbacks={LIQUIDATIONS: liquidations}))
print("Starting feedhandler for exchanges:", ', '.join(configured))
f.run()
def test_exchanges_fh():
"""
Ensure all exchanges are in feedhandler's string to class mapping
"""
path = os.path.dirname(os.path.abspath(__file__))
files = os.listdir(f"{path}/../../cryptofeed/exchanges")
files = [
f.replace("cryptodotcom", "CRYPTO.COM") for f in files
if '__' not in f and 'mixins' not in f
]
files = [
f.replace("bitdotcom", "BIT.COM") for f in files
if '__' not in f and 'mixins' not in f
]
files = [f[:-3].upper() for f in files] # Drop extension .py and uppercase
assert sorted(files) == sorted(EXCHANGE_MAP.keys())
def get_message_count(filenames: str):
counter = 0
for filename in filenames:
if '.ws.' not in filename:
continue
with open(filename, 'r') as fp:
for line in fp.readlines():
if line == "\n":
continue
start = line[:3]
if start == 'wss':
continue
counter += 1
return counter
@pytest.mark.parametrize(
"exchange", [e for e in EXCHANGE_MAP.keys() if e not in [COINGECKO, EXX]])
def test_exchange_playback(exchange):
Symbols.clear()
dir = os.path.dirname(os.path.realpath(__file__))
pcap = glob.glob(f"{dir}/../../sample_data/{exchange}.*")
results = playback(exchange, pcap)
message_count = get_message_count(pcap)
assert results['messages_processed'] == message_count
assert lookup_table[exchange] == results['callbacks']
Symbols.clear()
def load_all(self):
from cryptofeed.exchanges import EXCHANGE_MAP
for _, exchange in EXCHANGE_MAP.items():
exchange.symbols(refresh=True)
for filename in filenames:
if '.ws.' not in filename:
continue
with open(filename, 'r') as fp:
for line in fp.readlines():
if line == "\n":
continue
start = line[:3]
if start == 'wss':
continue
counter += 1
return counter
@pytest.mark.parametrize("exchange",
[e for e in EXCHANGE_MAP.keys() if e not in [EXX]])
def test_exchange_playback(exchange):
Symbols.clear()
dir = os.path.dirname(os.path.realpath(__file__))
pcap = glob.glob(f"{dir}/../../sample_data/{exchange}.*")
results = playback(exchange, pcap, config="tests/config_test.yaml")
message_count = get_message_count(pcap)
assert results['messages_processed'] == message_count
if exchange == BEQUANT:
# for some unknown reason on the github build servers this test always
# fails even though it works fine on my local mac and linux machines
expected = dict(lookup_table[exchange])
expected[L2_BOOK] = 990
def get_message_count(filenames: str):
counter = 0
for filename in filenames:
if '.ws.' not in filename:
continue
with open(filename, 'r') as fp:
for line in fp.readlines():
if line == "\n":
continue
start = line[:3]
if start == 'wss':
continue
counter += 1
return counter
@pytest.mark.parametrize("exchange", [e for e in EXCHANGE_MAP.keys() if e not in [COINGECKO, EXX]])
def test_exchange_playback(exchange):
Symbols.clear()
dir = os.path.dirname(os.path.realpath(__file__))
pcap = glob.glob(f"{dir}/../../sample_data/{exchange}.*")
results = playback(exchange, pcap)
message_count = get_message_count(pcap)
assert results['messages_processed'] == message_count
if isinstance(lookup_table[exchange], list):
assert results['callbacks'] in lookup_table[exchange]
else:
assert lookup_table[exchange] == results['callbacks']
Symbols.clear()