def run(self): LOG.info("Collector for %s running on PID %d", self.exchange, os.getpid()) fh = FeedHandler() cb = {} depth = None window = 1000 if 'book_delta_window' in self.config: window = self.config['book_delta_window'] if 'book_depth' in self.config: depth = self.config['book_depth'] cache = self.config['cache'] if cache == 'redis': from cryptofeed.backends.redis import TradeStream, BookStream, BookDeltaStream trade_cb = TradeStream book_cb = BookStream book_up = BookDeltaStream if not depth and self.config['book_delta'] else None kwargs = {'host': self.config['redis']['ip'], 'port': self.config['redis']['port'], 'numeric_type': float} elif cache == 'kafka': from cryptofeed.backends.kafka import TradeKafka, BookKafka, BookDeltaKafka trade_cb = TradeKafka book_cb = BookKafka book_up = BookDeltaKafka if not depth and self.config['book_delta'] else None kwargs = {'host': self.config['kafka']['ip'], 'port': self.config['kafka']['port']} if TRADES in self.exchange_config: cb[TRADES] = [trade_cb(**kwargs)] if L2_BOOK in self.exchange_config: cb[L2_BOOK] = [book_cb(key=L2_BOOK, depth=depth, **kwargs)] if book_up: cb[BOOK_DELTA] = [book_up(key=L2_BOOK, **kwargs)] if L3_BOOK in self.exchange_config: cb[L3_BOOK] = [book_cb(key=L3_BOOK, depth=depth, **kwargs)] if book_up: cb[BOOK_DELTA] = [book_up(key=L3_BOOK, **kwargs)] if 'pass_through' in self.config: if self.config['pass_through']['type'] == 'zmq': from cryptofeed.backends.zmq import TradeZMQ, BookDeltaZMQ, BookZMQ import zmq host = self.config['pass_through']['host'] port = self.config['pass_through']['port'] if TRADES in cb: cb[TRADES].append(TradeZMQ(host=host, port=port, zmq_type=zmq.PUB)) if BOOK_DELTA in cb: cb[BOOK_DELTA].append(BookDeltaZMQ(host=host, port=port, zmq_type=zmq.PUB)) if L2_BOOK in cb: cb[L2_BOOK].append(BookZMQ(host=host, port=port, zmq_type=zmq.PUB)) if L3_BOOK in cb: cb[L3_BOOK].append(BookZMQ(host=host, port=port, zmq_type=zmq.PUB)) fh.add_feed(self.exchange, book_interval=window, config=self.exchange_config, callbacks=cb) fh.run()
def run(self): LOG.info("Collector for %s running on PID %d", self.exchange, os.getpid()) cache = self.config['cache'] retries = self.exchange_config.pop('retries', 30) fh = FeedHandler(retries=retries) feed_config = {} callbacks = {} for callback_type, value in self.exchange_config.items(): cb = {} depth = None window = 1000 delta = False if 'book_interval' in value: window = value['book_interval'] if 'book_delta' in value and value['book_delta']: delta = True if 'max_depth' in value: depth = value['max_depth'] if callback_type in (L2_BOOK, L3_BOOK): self.exchange_config[callback_type] = self.exchange_config[ callback_type]['symbols'] if cache == 'redis': if 'ip' in self.config['redis'] and self.config['redis']['ip']: kwargs = { 'host': self.config['redis']['ip'], 'port': self.config['redis']['port'], 'numeric_type': float } else: kwargs = { 'socket': self.config['redis']['socket'], 'numeric_type': float } from cryptofeed.backends.redis import TradeStream, BookStream, BookDeltaStream, TickerStream, FundingStream trade_cb = TradeStream book_cb = BookStream book_up = BookDeltaStream if delta else None ticker_cb = TickerStream funding_cb = FundingStream elif cache == 'kafka': from cryptofeed.backends.kafka import TradeKafka, BookKafka, BookDeltaKafka, TickerKafka, FundingKafka trade_cb = TradeKafka book_cb = BookKafka book_up = BookDeltaKafka if delta else None ticker_cb = TickerKafka funding_cb = FundingKafka kwargs = { 'host': self.config['kafka']['ip'], 'port': self.config['kafka']['port'] } if callback_type == TRADES: cb[TRADES] = [trade_cb(**kwargs)] elif callback_type == FUNDING: cb[FUNDING] = [funding_cb(**kwargs)] elif callback_type == TICKER: cb[TICKER] = [ticker_cb(**kwargs)] elif callback_type == L2_BOOK: cb[L2_BOOK] = [book_cb(key=L2_BOOK, **kwargs)] if book_up: cb[BOOK_DELTA] = [book_up(key=L2_BOOK, **kwargs)] elif callback_type == L3_BOOK: cb[L3_BOOK] = [book_cb(key=L3_BOOK, **kwargs)] if book_up: cb[BOOK_DELTA] = [book_up(key=L3_BOOK, **kwargs)] if 'pass_through' in self.config: if self.config['pass_through']['type'] == 'zmq': from cryptofeed.backends.zmq import TradeZMQ, BookDeltaZMQ, BookZMQ, FundingZMQ import zmq host = self.config['pass_through']['host'] port = self.config['pass_through']['port'] if callback_type == TRADES: cb[TRADES].append( TradeZMQ(host=host, port=port, zmq_type=zmq.PUB)) elif callback_type == FUNDING: cb[FUNDING].append( FundingZMQ(host=host, port=port, zmq_type=zmq.PUB)) elif callback_type == L2_BOOK: cb[L2_BOOK].append( BookZMQ(host=host, port=port, zmq_type=zmq.PUB)) elif callback_type == L3_BOOK: cb[L3_BOOK].append( BookZMQ(host=host, port=port, zmq_type=zmq.PUB)) if BOOK_DELTA in cb: cb[BOOK_DELTA].append( BookDeltaZMQ(host=host, port=port, zmq_type=zmq.PUB)) feed_config[callback_type] = self.exchange_config[callback_type] callbacks.update(cb) fh.add_feed(self.exchange, max_depth=depth, book_interval=window, config=feed_config, callbacks=callbacks) fh.run()
def run(self): LOG.info("Collector for %s running on PID %d", self.exchange, os.getpid()) cache = self.config['cache'] retries = self.exchange_config.pop('retries', 30) timeouts = self.exchange_config.pop('channel_timeouts', {}) fh = FeedHandler(retries=retries) for callback_type, value in self.exchange_config.items(): cb = {} depth = None window = 1000 delta = False timeout = timeouts.get(callback_type, 120) snap_interval = False if 'book_interval' in value: window = value['book_interval'] if 'book_delta' in value and value['book_delta']: delta = True if 'max_depth' in value: depth = value['max_depth'] if 'snapshot_interval' in value: snap_interval = value['snapshot_interval'] if callback_type in (L2_BOOK, L3_BOOK): self.exchange_config[callback_type] = self.exchange_config[callback_type]['symbols'] if cache == 'redis': if 'ip' in self.config['redis'] and self.config['redis']['ip']: kwargs = {'host': self.config['redis']['ip'], 'port': self.config['redis']['port'], 'numeric_type': float} else: kwargs = {'socket': self.config['redis']['socket'], 'numeric_type': float} from cryptofeed.backends.redis import TradeStream, BookStream, BookDeltaStream, TickerStream, FundingStream, OpenInterestStream, LiquidationsStream trade_cb = TradeStream book_cb = BookStream book_up = BookDeltaStream if delta else None ticker_cb = TickerStream funding_cb = FundingStream oi_cb = OpenInterestStream liq_cb = LiquidationsStream elif cache == 'kafka': from cryptofeed.backends.kafka import TradeKafka, BookKafka, BookDeltaKafka, TickerKafka, FundingKafka, OpenInterestKafka, LiquidationsKafka trade_cb = TradeKafka book_cb = BookKafka book_up = BookDeltaKafka if delta else None ticker_cb = TickerKafka funding_cb = FundingKafka oi_cb = OpenInterestKafka liq_cb = LiquidationsKafka kwargs = {'host': self.config['kafka']['ip'], 'port': self.config['kafka']['port']} if callback_type == TRADES: cb[TRADES] = [trade_cb(**kwargs)] elif callback_type == LIQUIDATIONS: cb[LIQUIDATIONS] = [liq_cb(**kwargs)] elif callback_type == FUNDING: cb[FUNDING] = [funding_cb(**kwargs)] elif callback_type == TICKER: cb[TICKER] = [ticker_cb(**kwargs)] elif callback_type == L2_BOOK: cb[L2_BOOK] = [book_cb(key=L2_BOOK, **kwargs)] if book_up: cb[BOOK_DELTA] = [book_up(key=L2_BOOK, **kwargs)] elif callback_type == L3_BOOK: cb[L3_BOOK] = [book_cb(key=L3_BOOK, **kwargs)] if book_up: cb[BOOK_DELTA] = [book_up(key=L3_BOOK, **kwargs)] elif callback_type == OPEN_INTEREST: cb[OPEN_INTEREST] = [oi_cb(**kwargs)] if 'pass_through' in self.config: if self.config['pass_through']['type'] == 'zmq': from cryptofeed.backends.zmq import TradeZMQ, BookDeltaZMQ, BookZMQ, FundingZMQ, OpenInterestZMQ, TickerZMQ, LiquidationsZMQ import zmq host = self.config['pass_through']['host'] port = self.config['pass_through']['port'] if callback_type == TRADES: cb[TRADES].append(TradeZMQ(host=host, port=port)) elif callback_type == LIQUIDATIONS: cb[LIQUIDATIONS].append(LiquidationsZMQ(host=host, port=port)) elif callback_type == FUNDING: cb[FUNDING].append(FundingZMQ(host=host, port=port)) elif callback_type == L2_BOOK: cb[L2_BOOK].append(BookZMQ(host=host, port=port)) elif callback_type == L3_BOOK: cb[L3_BOOK].append(BookZMQ(host=host, port=port)) elif callback_type == OPEN_INTEREST: cb[OPEN_INTEREST].append(OpenInterestZMQ(host=host, port=port)) elif callback_type == TICKER: cb[TICKER].append(TickerZMQ(host=host, port=port)) if BOOK_DELTA in cb: cb[BOOK_DELTA].append(BookDeltaZMQ(host=host, port=port)) fh.add_feed(self.exchange, timeout=timeout, max_depth=depth, snapshot_interval=snap_interval, book_interval=window, config={callback_type: self.exchange_config[callback_type]}, callbacks=cb) fh.run()