def main(only_exchange=None): skip = [EXX] files = glob.glob('*') for f in files: for e in EXCHANGE_MAP.keys(): if e + "." in f: skip.append(e.split(".")[0]) print(f'Generating test data. This will take approximately {(len(EXCHANGE_MAP) - len(set(skip))) * 0.5} minutes.') loop = asyncio.get_event_loop() for exch_str, exchange in EXCHANGE_MAP.items() if only_exchange is None else [(only_exchange, EXCHANGE_MAP[only_exchange])]: if exch_str in skip: continue print(f"Collecting data for {exch_str}") fh = FeedHandler(raw_data_collection=AsyncFileCallback("./"), config={'uvloop': False, 'log': {'filename': 'feedhandler.log', 'level': 'WARNING'}, 'rest': {'log': {'filename': 'rest.log', 'level': 'WARNING'}}}) info = exchange.info() channels = list(set.intersection(set(info['channels']['websocket']), set([L2_BOOK, TRADES, TICKER, CANDLES]))) sample_size = 10 if exch_str in (BINANCE_US, BINANCE): # books of size 5000 count significantly against rate limits sample_size = 4 while True: try: symbols = random.sample(info['symbols'], sample_size) if exch_str == BINANCE_FUTURES: symbols = [s for s in symbols if 'PINDEX' not in s] elif exch_str == BITFINEX: symbols = [s for s in symbols if '-' in s] except ValueError: sample_size -= 1 else: break fh.add_feed(exchange(symbols=symbols, channels=channels)) fh.run(start_loop=False) loop.call_later(31, stop) print("Starting feedhandler. Will run for 30 seconds...") loop.run_forever() fh.stop(loop=loop) del fh print("Checking raw message dumps for errors...") for exch_str, _ in EXCHANGE_MAP.items(): for file in glob.glob(exch_str + "*"): try: print(f"Checking {file}") check_dump(file) except Exception as e: print(f"File {file} failed") print(e)
def start_cryptofeed(): async def liquidations_cb(data, receipt): # Add raw data to CryptofeedDataTypeEnum.LIQUIDATION_DATA queue CryptofeedService.data[CryptofeedDataTypeEnum.LIQUIDATIONS].put( data) async def open_interest_cb(data, receipt): # Add raw data to CryptofeedDataTypeEnum.OPEN_INTEREST queue CryptofeedService.data[CryptofeedDataTypeEnum.OPEN_INTEREST].put( data) # There is no current event loop in thread loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) f = FeedHandler() configured = [] print("Querying exchange metadata") for exchange_string, exchange_class in EXCHANGE_MAP.items(): if exchange_string not in EXCHANGES: continue if exchange_string in ['BITFLYER', 'EXX', 'OKEX' ]: # We have issues with these exchanges continue if all(channel in exchange_class.info()['channels']['websocket'] for channel in [LIQUIDATIONS, OPEN_INTEREST]): configured.append(exchange_string) print(f"Configuring {exchange_string}...", end='') symbols = [ sym for sym in exchange_class.symbols() if 'PINDEX' not in sym ] try: f.add_feed( exchange_class(subscription={ LIQUIDATIONS: symbols, OPEN_INTEREST: symbols }, callbacks={ LIQUIDATIONS: liquidations_cb, OPEN_INTEREST: open_interest_cb })) print(" Done") except Exception as e: print(e, exchange_string) pass print(configured) print("Starting feedhandler for exchanges:", ', '.join(configured)) f.run(install_signal_handlers=False)
def main(): f = FeedHandler() configured = [] print("Querying exchange metadata") for exchange_string, exchange_class in EXCHANGE_MAP.items(): if LIQUIDATIONS in exchange_class.info()['channels']['websocket']: configured.append(exchange_string) symbols = [ sym for sym in exchange_class.symbols() if 'PINDEX' not in sym ] f.add_feed( exchange_class(subscription={LIQUIDATIONS: symbols}, callbacks={LIQUIDATIONS: liquidations})) print("Starting feedhandler for exchanges:", ', '.join(configured)) f.run()
def test_exchanges_fh(): """ Ensure all exchanges are in feedhandler's string to class mapping """ path = os.path.dirname(os.path.abspath(__file__)) files = os.listdir(f"{path}/../../cryptofeed/exchanges") files = [ f.replace("cryptodotcom", "CRYPTO.COM") for f in files if '__' not in f and 'mixins' not in f ] files = [ f.replace("bitdotcom", "BIT.COM") for f in files if '__' not in f and 'mixins' not in f ] files = [f[:-3].upper() for f in files] # Drop extension .py and uppercase assert sorted(files) == sorted(EXCHANGE_MAP.keys())
def get_message_count(filenames: str): counter = 0 for filename in filenames: if '.ws.' not in filename: continue with open(filename, 'r') as fp: for line in fp.readlines(): if line == "\n": continue start = line[:3] if start == 'wss': continue counter += 1 return counter @pytest.mark.parametrize( "exchange", [e for e in EXCHANGE_MAP.keys() if e not in [COINGECKO, EXX]]) def test_exchange_playback(exchange): Symbols.clear() dir = os.path.dirname(os.path.realpath(__file__)) pcap = glob.glob(f"{dir}/../../sample_data/{exchange}.*") results = playback(exchange, pcap) message_count = get_message_count(pcap) assert results['messages_processed'] == message_count assert lookup_table[exchange] == results['callbacks'] Symbols.clear()
def load_all(self): from cryptofeed.exchanges import EXCHANGE_MAP for _, exchange in EXCHANGE_MAP.items(): exchange.symbols(refresh=True)
for filename in filenames: if '.ws.' not in filename: continue with open(filename, 'r') as fp: for line in fp.readlines(): if line == "\n": continue start = line[:3] if start == 'wss': continue counter += 1 return counter @pytest.mark.parametrize("exchange", [e for e in EXCHANGE_MAP.keys() if e not in [EXX]]) def test_exchange_playback(exchange): Symbols.clear() dir = os.path.dirname(os.path.realpath(__file__)) pcap = glob.glob(f"{dir}/../../sample_data/{exchange}.*") results = playback(exchange, pcap, config="tests/config_test.yaml") message_count = get_message_count(pcap) assert results['messages_processed'] == message_count if exchange == BEQUANT: # for some unknown reason on the github build servers this test always # fails even though it works fine on my local mac and linux machines expected = dict(lookup_table[exchange]) expected[L2_BOOK] = 990
def get_message_count(filenames: str): counter = 0 for filename in filenames: if '.ws.' not in filename: continue with open(filename, 'r') as fp: for line in fp.readlines(): if line == "\n": continue start = line[:3] if start == 'wss': continue counter += 1 return counter @pytest.mark.parametrize("exchange", [e for e in EXCHANGE_MAP.keys() if e not in [COINGECKO, EXX]]) def test_exchange_playback(exchange): Symbols.clear() dir = os.path.dirname(os.path.realpath(__file__)) pcap = glob.glob(f"{dir}/../../sample_data/{exchange}.*") results = playback(exchange, pcap) message_count = get_message_count(pcap) assert results['messages_processed'] == message_count if isinstance(lookup_table[exchange], list): assert results['callbacks'] in lookup_table[exchange] else: assert lookup_table[exchange] == results['callbacks'] Symbols.clear()