def main(only_exchange=None):
skip = [EXX]
files = glob.glob('*')
for f in files:
for e in EXCHANGE_MAP.keys():
if e + "." in f:
skip.append(e.split(".")[0])
print(f'Generating test data. This will take approximately {(len(EXCHANGE_MAP) - len(set(skip))) * 0.5} minutes.')
loop = asyncio.get_event_loop()
for exch_str, exchange in EXCHANGE_MAP.items() if only_exchange is None else [(only_exchange, EXCHANGE_MAP[only_exchange])]:
if exch_str in skip:
continue
print(f"Collecting data for {exch_str}")
fh = FeedHandler(raw_data_collection=AsyncFileCallback("./"), config={'uvloop': False, 'log': {'filename': 'feedhandler.log', 'level': 'WARNING'}, 'rest': {'log': {'filename': 'rest.log', 'level': 'WARNING'}}})
info = exchange.info()
channels = list(set.intersection(set(info['channels']['websocket']), set([L2_BOOK, TRADES, TICKER, CANDLES])))
sample_size = 10
if exch_str in (BINANCE_US, BINANCE):
# books of size 5000 count significantly against rate limits
sample_size = 4
while True:
try:
symbols = random.sample(info['symbols'], sample_size)
if exch_str == BINANCE_FUTURES:
symbols = [s for s in symbols if 'PINDEX' not in s]
elif exch_str == BITFINEX:
symbols = [s for s in symbols if '-' in s]
except ValueError:
sample_size -= 1
else:
break
fh.add_feed(exchange(symbols=symbols, channels=channels))
fh.run(start_loop=False)
loop.call_later(31, stop)
print("Starting feedhandler. Will run for 30 seconds...")
loop.run_forever()
fh.stop(loop=loop)
del fh
print("Checking raw message dumps for errors...")
for exch_str, _ in EXCHANGE_MAP.items():
for file in glob.glob(exch_str + "*"):
try:
print(f"Checking {file}")
check_dump(file)
except Exception as e:
print(f"File {file} failed")
print(e)
def test_exchanges_fh():
"""
Ensure all exchanges are in feedhandler's string to class mapping
"""
path = os.path.dirname(os.path.abspath(__file__))
files = os.listdir(f"{path}/../../cryptofeed/exchanges")
files = [
f.replace("cryptodotcom", "CRYPTO.COM") for f in files
if '__' not in f and 'mixins' not in f
]
files = [
f.replace("bitdotcom", "BIT.COM") for f in files
if '__' not in f and 'mixins' not in f
]
files = [f[:-3].upper() for f in files] # Drop extension .py and uppercase
assert sorted(files) == sorted(EXCHANGE_MAP.keys())
def get_message_count(filenames: str):
counter = 0
for filename in filenames:
if '.ws.' not in filename:
continue
with open(filename, 'r') as fp:
for line in fp.readlines():
if line == "\n":
continue
start = line[:3]
if start == 'wss':
continue
counter += 1
return counter
@pytest.mark.parametrize(
"exchange", [e for e in EXCHANGE_MAP.keys() if e not in [COINGECKO, EXX]])
def test_exchange_playback(exchange):
Symbols.clear()
dir = os.path.dirname(os.path.realpath(__file__))
pcap = glob.glob(f"{dir}/../../sample_data/{exchange}.*")
results = playback(exchange, pcap)
message_count = get_message_count(pcap)
assert results['messages_processed'] == message_count
assert lookup_table[exchange] == results['callbacks']
Symbols.clear()
for filename in filenames:
if '.ws.' not in filename:
continue
with open(filename, 'r') as fp:
for line in fp.readlines():
if line == "\n":
continue
start = line[:3]
if start == 'wss':
continue
counter += 1
return counter
@pytest.mark.parametrize("exchange",
[e for e in EXCHANGE_MAP.keys() if e not in [EXX]])
def test_exchange_playback(exchange):
Symbols.clear()
dir = os.path.dirname(os.path.realpath(__file__))
pcap = glob.glob(f"{dir}/../../sample_data/{exchange}.*")
results = playback(exchange, pcap, config="tests/config_test.yaml")
message_count = get_message_count(pcap)
assert results['messages_processed'] == message_count
if exchange == BEQUANT:
# for some unknown reason on the github build servers this test always
# fails even though it works fine on my local mac and linux machines
expected = dict(lookup_table[exchange])
expected[L2_BOOK] = 990
def get_message_count(filenames: str):
counter = 0
for filename in filenames:
if '.ws.' not in filename:
continue
with open(filename, 'r') as fp:
for line in fp.readlines():
if line == "\n":
continue
start = line[:3]
if start == 'wss':
continue
counter += 1
return counter
@pytest.mark.parametrize("exchange", [e for e in EXCHANGE_MAP.keys() if e not in [COINGECKO, EXX]])
def test_exchange_playback(exchange):
Symbols.clear()
dir = os.path.dirname(os.path.realpath(__file__))
pcap = glob.glob(f"{dir}/../../sample_data/{exchange}.*")
results = playback(exchange, pcap)
message_count = get_message_count(pcap)
assert results['messages_processed'] == message_count
if isinstance(lookup_table[exchange], list):
assert results['callbacks'] in lookup_table[exchange]
else:
assert lookup_table[exchange] == results['callbacks']
Symbols.clear()
