def run(self):
LOG.info("Collector for %s running on PID %d", self.exchange, os.getpid())
fh = FeedHandler()
cb = {}
depth = None
window = 1000
if 'book_delta_window' in self.config:
window = self.config['book_delta_window']
if 'book_depth' in self.config:
depth = self.config['book_depth']
cache = self.config['cache']
if cache == 'redis':
from cryptofeed.backends.redis import TradeStream, BookStream, BookDeltaStream
trade_cb = TradeStream
book_cb = BookStream
book_up = BookDeltaStream if not depth and self.config['book_delta'] else None
kwargs = {'host': self.config['redis']['ip'], 'port': self.config['redis']['port'], 'numeric_type': float}
elif cache == 'kafka':
from cryptofeed.backends.kafka import TradeKafka, BookKafka, BookDeltaKafka
trade_cb = TradeKafka
book_cb = BookKafka
book_up = BookDeltaKafka if not depth and self.config['book_delta'] else None
kwargs = {'host': self.config['kafka']['ip'], 'port': self.config['kafka']['port']}
if TRADES in self.exchange_config:
cb[TRADES] = [trade_cb(**kwargs)]
if L2_BOOK in self.exchange_config:
cb[L2_BOOK] = [book_cb(key=L2_BOOK, depth=depth, **kwargs)]
if book_up:
cb[BOOK_DELTA] = [book_up(key=L2_BOOK, **kwargs)]
if L3_BOOK in self.exchange_config:
cb[L3_BOOK] = [book_cb(key=L3_BOOK, depth=depth, **kwargs)]
if book_up:
cb[BOOK_DELTA] = [book_up(key=L3_BOOK, **kwargs)]
if 'pass_through' in self.config:
if self.config['pass_through']['type'] == 'zmq':
from cryptofeed.backends.zmq import TradeZMQ, BookDeltaZMQ, BookZMQ
import zmq
host = self.config['pass_through']['host']
port = self.config['pass_through']['port']
if TRADES in cb:
cb[TRADES].append(TradeZMQ(host=host, port=port, zmq_type=zmq.PUB))
if BOOK_DELTA in cb:
cb[BOOK_DELTA].append(BookDeltaZMQ(host=host, port=port, zmq_type=zmq.PUB))
if L2_BOOK in cb:
cb[L2_BOOK].append(BookZMQ(host=host, port=port, zmq_type=zmq.PUB))
if L3_BOOK in cb:
cb[L3_BOOK].append(BookZMQ(host=host, port=port, zmq_type=zmq.PUB))
fh.add_feed(self.exchange, book_interval=window, config=self.exchange_config, callbacks=cb)
fh.run()
def main():
args = get_args()
print(f"\nq connection port : {args.port}" \
f"\nzmq (kdb) port : {args.zmq_port}" \
f"\ndepth : {args.depth}" \
f"\nconfig : {args.config}")
subscriptions = read_cfg(args.config)
print(f"IPC version: {q.protocol_version}. Is connected: {q.is_connected()}")
try:
p = Process(target=receiver, args=(args.zmq_port, args.depth,))
p.start()
f = FeedHandler()
if "binance_us" in subscriptions.keys():
f.add_feed(BinanceUS(
max_depth=args.depth,
channels=[L2_BOOK, TRADES],
pairs=subscriptions['binance_us'],
callbacks={
TRADES: [TradeZMQ(port=args.zmq_port)],
L2_BOOK: [BookZMQ(depth=args.depth, port=args.zmq_port)]}))
if "coinbase" in subscriptions.keys():
f.add_feed(Coinbase(
max_depth=args.depth,
channels=[L2_BOOK, TRADES],
pairs=subscriptions['coinbase'],
callbacks={
TRADES: [TradeZMQ(port=args.zmq_port)],
L2_BOOK: [BookZMQ(depth=args.depth, port=args.zmq_port)]}))
if "kraken" in subscriptions.keys():
f.add_feed(Kraken(
channels=[L2_BOOK, TRADES],
pairs=subscriptions['kraken'],
callbacks={
TRADES: [TradeZMQ(port=args.zmq_port)],
L2_BOOK: [BookZMQ(depth=args.depth, port=args.zmq_port)]}))
f.run()
except KeyboardInterrupt:
pass
finally:
p.terminate()
# save trades and quotes tables to disk
# data_path = os.getcwd()
# data_path = data_path.replace("\\", "/")
# trades_path = f"`:{data_path}/trades set trades"
# quotes_path = f"`:{data_path}/quotes set quotes"
# print(f"saving to disk quotes -> {quotes_path} trades -> {trades_path}")
q.close()
def main():
try:
p = Process(target=receiver, args=(5678, ))
p.start()
f = FeedHandler()
f.add_feed(
Kraken(max_depth=2,
channels=[L2_BOOK],
symbols=['ETH-USD', 'BTC-USD'],
callbacks={
L2_BOOK:
BookZMQ(snapshots_only=False,
snapshot_interval=2,
port=5678)
}))
f.add_feed(
Coinbase(channels=[TICKER],
symbols=['BTC-USD'],
callbacks={TICKER: TickerZMQ(port=5678)}))
f.run()
finally:
p.terminate()
def main():
try:
p = Process(target=receiver, args=(5678, ))
p.start()
f = FeedHandler()
f.add_feed(
Kraken(channels=[L2_BOOK],
pairs=['ETH-USD'],
callbacks={L2_BOOK: BookZMQ(depth=1, port=5678)}))
f.run()
finally:
p.terminate()
def main():
try:
p = Process(target=receiver, args=(5555, ))
p2 = Process(target=receiver, args=(5556, ))
p.start()
p2.start()
f = FeedHandler()
f.add_feed(
Coinbase(channels=[L3_BOOK, TRADES],
pairs=['BTC-USD'],
callbacks={
TRADES: TradeZMQ(),
L3_BOOK: BookZMQ(depth=1, port=5556)
}))
f.run()
finally:
p.terminate()
p2.terminate()
def main():
try:
p = Process(target=receiver, args=(5678, ))
p.start()
f = FeedHandler()
f.add_feed(
Kraken(max_depth=1,
channels=[L2_BOOK],
pairs=['ETH-USD'],
callbacks={L2_BOOK: BookZMQ(port=5678)}))
f.add_feed(
Coinbase(channels=[TICKER],
pairs=['BTC-USD'],
callbacks={TICKER: TickerZMQ(port=5678)}))
f.run()
finally:
p.terminate()
def run(self):
LOG.info("Collector for %s running on PID %d", self.exchange,
os.getpid())
cache = self.config['cache']
retries = self.exchange_config.pop('retries', 30)
fh = FeedHandler(retries=retries)
feed_config = {}
callbacks = {}
for callback_type, value in self.exchange_config.items():
cb = {}
depth = None
window = 1000
delta = False
if 'book_interval' in value:
window = value['book_interval']
if 'book_delta' in value and value['book_delta']:
delta = True
if 'max_depth' in value:
depth = value['max_depth']
if callback_type in (L2_BOOK, L3_BOOK):
self.exchange_config[callback_type] = self.exchange_config[
callback_type]['symbols']
if cache == 'redis':
if 'ip' in self.config['redis'] and self.config['redis']['ip']:
kwargs = {
'host': self.config['redis']['ip'],
'port': self.config['redis']['port'],
'numeric_type': float
}
else:
kwargs = {
'socket': self.config['redis']['socket'],
'numeric_type': float
}
from cryptofeed.backends.redis import TradeStream, BookStream, BookDeltaStream, TickerStream, FundingStream
trade_cb = TradeStream
book_cb = BookStream
book_up = BookDeltaStream if delta else None
ticker_cb = TickerStream
funding_cb = FundingStream
elif cache == 'kafka':
from cryptofeed.backends.kafka import TradeKafka, BookKafka, BookDeltaKafka, TickerKafka, FundingKafka
trade_cb = TradeKafka
book_cb = BookKafka
book_up = BookDeltaKafka if delta else None
ticker_cb = TickerKafka
funding_cb = FundingKafka
kwargs = {
'host': self.config['kafka']['ip'],
'port': self.config['kafka']['port']
}
if callback_type == TRADES:
cb[TRADES] = [trade_cb(**kwargs)]
elif callback_type == FUNDING:
cb[FUNDING] = [funding_cb(**kwargs)]
elif callback_type == TICKER:
cb[TICKER] = [ticker_cb(**kwargs)]
elif callback_type == L2_BOOK:
cb[L2_BOOK] = [book_cb(key=L2_BOOK, **kwargs)]
if book_up:
cb[BOOK_DELTA] = [book_up(key=L2_BOOK, **kwargs)]
elif callback_type == L3_BOOK:
cb[L3_BOOK] = [book_cb(key=L3_BOOK, **kwargs)]
if book_up:
cb[BOOK_DELTA] = [book_up(key=L3_BOOK, **kwargs)]
if 'pass_through' in self.config:
if self.config['pass_through']['type'] == 'zmq':
from cryptofeed.backends.zmq import TradeZMQ, BookDeltaZMQ, BookZMQ, FundingZMQ
import zmq
host = self.config['pass_through']['host']
port = self.config['pass_through']['port']
if callback_type == TRADES:
cb[TRADES].append(
TradeZMQ(host=host, port=port, zmq_type=zmq.PUB))
elif callback_type == FUNDING:
cb[FUNDING].append(
FundingZMQ(host=host, port=port, zmq_type=zmq.PUB))
elif callback_type == L2_BOOK:
cb[L2_BOOK].append(
BookZMQ(host=host, port=port, zmq_type=zmq.PUB))
elif callback_type == L3_BOOK:
cb[L3_BOOK].append(
BookZMQ(host=host, port=port, zmq_type=zmq.PUB))
if BOOK_DELTA in cb:
cb[BOOK_DELTA].append(
BookDeltaZMQ(host=host,
port=port,
zmq_type=zmq.PUB))
feed_config[callback_type] = self.exchange_config[callback_type]
callbacks.update(cb)
fh.add_feed(self.exchange,
max_depth=depth,
book_interval=window,
config=feed_config,
callbacks=callbacks)
fh.run()
def run(self):
LOG.info("Collector for %s running on PID %d", self.exchange, os.getpid())
cache = self.config['cache']
retries = self.exchange_config.pop('retries', 30)
timeouts = self.exchange_config.pop('channel_timeouts', {})
fh = FeedHandler(retries=retries)
for callback_type, value in self.exchange_config.items():
cb = {}
depth = None
window = 1000
delta = False
timeout = timeouts.get(callback_type, 120)
snap_interval = False
if 'book_interval' in value:
window = value['book_interval']
if 'book_delta' in value and value['book_delta']:
delta = True
if 'max_depth' in value:
depth = value['max_depth']
if 'snapshot_interval' in value:
snap_interval = value['snapshot_interval']
if callback_type in (L2_BOOK, L3_BOOK):
self.exchange_config[callback_type] = self.exchange_config[callback_type]['symbols']
if cache == 'redis':
if 'ip' in self.config['redis'] and self.config['redis']['ip']:
kwargs = {'host': self.config['redis']['ip'], 'port': self.config['redis']['port'], 'numeric_type': float}
else:
kwargs = {'socket': self.config['redis']['socket'], 'numeric_type': float}
from cryptofeed.backends.redis import TradeStream, BookStream, BookDeltaStream, TickerStream, FundingStream, OpenInterestStream, LiquidationsStream
trade_cb = TradeStream
book_cb = BookStream
book_up = BookDeltaStream if delta else None
ticker_cb = TickerStream
funding_cb = FundingStream
oi_cb = OpenInterestStream
liq_cb = LiquidationsStream
elif cache == 'kafka':
from cryptofeed.backends.kafka import TradeKafka, BookKafka, BookDeltaKafka, TickerKafka, FundingKafka, OpenInterestKafka, LiquidationsKafka
trade_cb = TradeKafka
book_cb = BookKafka
book_up = BookDeltaKafka if delta else None
ticker_cb = TickerKafka
funding_cb = FundingKafka
oi_cb = OpenInterestKafka
liq_cb = LiquidationsKafka
kwargs = {'host': self.config['kafka']['ip'], 'port': self.config['kafka']['port']}
if callback_type == TRADES:
cb[TRADES] = [trade_cb(**kwargs)]
elif callback_type == LIQUIDATIONS:
cb[LIQUIDATIONS] = [liq_cb(**kwargs)]
elif callback_type == FUNDING:
cb[FUNDING] = [funding_cb(**kwargs)]
elif callback_type == TICKER:
cb[TICKER] = [ticker_cb(**kwargs)]
elif callback_type == L2_BOOK:
cb[L2_BOOK] = [book_cb(key=L2_BOOK, **kwargs)]
if book_up:
cb[BOOK_DELTA] = [book_up(key=L2_BOOK, **kwargs)]
elif callback_type == L3_BOOK:
cb[L3_BOOK] = [book_cb(key=L3_BOOK, **kwargs)]
if book_up:
cb[BOOK_DELTA] = [book_up(key=L3_BOOK, **kwargs)]
elif callback_type == OPEN_INTEREST:
cb[OPEN_INTEREST] = [oi_cb(**kwargs)]
if 'pass_through' in self.config:
if self.config['pass_through']['type'] == 'zmq':
from cryptofeed.backends.zmq import TradeZMQ, BookDeltaZMQ, BookZMQ, FundingZMQ, OpenInterestZMQ, TickerZMQ, LiquidationsZMQ
import zmq
host = self.config['pass_through']['host']
port = self.config['pass_through']['port']
if callback_type == TRADES:
cb[TRADES].append(TradeZMQ(host=host, port=port))
elif callback_type == LIQUIDATIONS:
cb[LIQUIDATIONS].append(LiquidationsZMQ(host=host, port=port))
elif callback_type == FUNDING:
cb[FUNDING].append(FundingZMQ(host=host, port=port))
elif callback_type == L2_BOOK:
cb[L2_BOOK].append(BookZMQ(host=host, port=port))
elif callback_type == L3_BOOK:
cb[L3_BOOK].append(BookZMQ(host=host, port=port))
elif callback_type == OPEN_INTEREST:
cb[OPEN_INTEREST].append(OpenInterestZMQ(host=host, port=port))
elif callback_type == TICKER:
cb[TICKER].append(TickerZMQ(host=host, port=port))
if BOOK_DELTA in cb:
cb[BOOK_DELTA].append(BookDeltaZMQ(host=host, port=port))
fh.add_feed(self.exchange, timeout=timeout, max_depth=depth, snapshot_interval=snap_interval, book_interval=window, config={callback_type: self.exchange_config[callback_type]}, callbacks=cb)
fh.run()
def run(self):
LOG.info("Collector for %s running on PID %d", self.exchange,
os.getpid())
cache = self.config['cache']
retries = self.exchange_config.pop('retries', 30)
timeouts = self.exchange_config.pop('channel_timeouts', {})
http_proxy = self.exchange_config.pop('http_proxy', None)
fh = FeedHandler()
for callback_type, feed_config in self.exchange_config.items():
# config value can be a dict or list of symbols
feed_kwargs = {
'retries': retries,
'timeout': timeouts.get(callback_type, 120)
}
if http_proxy is not None:
feed_kwargs['http_proxy'] = http_proxy
if isinstance(feed_config, dict):
feed_kwargs.update(self._exchange_feed_options(feed_config))
cb = {}
if callback_type in (L2_BOOK, L3_BOOK):
self.exchange_config[callback_type] = self.exchange_config[
callback_type]['symbols']
if cache == 'redis':
if 'ip' in self.config['redis'] and self.config['redis']['ip']:
kwargs = {
'host': self.config['redis']['ip'],
'port': self.config['redis']['port'],
'numeric_type': float
}
else:
kwargs = {
'socket': self.config['redis']['socket'],
'numeric_type': float
}
from cryptofeed.backends.redis import TradeStream, BookStream, TickerStream, FundingStream, OpenInterestStream, LiquidationsStream, CandlesStream
trade_cb = TradeStream
book_cb = BookStream
ticker_cb = TickerStream
funding_cb = FundingStream
oi_cb = OpenInterestStream
liq_cb = LiquidationsStream
candles_cb = CandlesStream
elif cache == 'kafka':
from cryptofeed.backends.kafka import TradeKafka, BookKafka, TickerKafka, FundingKafka, OpenInterestKafka, LiquidationsKafka, CandlesKafka
trade_cb = TradeKafka
book_cb = BookKafka
ticker_cb = TickerKafka
funding_cb = FundingKafka
oi_cb = OpenInterestKafka
liq_cb = LiquidationsKafka
candles_cb = CandlesKafka
kwargs = {
'bootstrap': self.config['kafka']['ip'],
'port': self.config['kafka']['port']
}
if callback_type == TRADES:
cb[TRADES] = [trade_cb(**kwargs)]
elif callback_type == LIQUIDATIONS:
cb[LIQUIDATIONS] = [liq_cb(**kwargs)]
elif callback_type == FUNDING:
cb[FUNDING] = [funding_cb(**kwargs)]
elif callback_type == TICKER:
cb[TICKER] = [ticker_cb(**kwargs)]
elif callback_type == L2_BOOK:
cb[L2_BOOK] = [
book_cb(key=L2_BOOK,
snapshot_interval=feed_config.get(
'snapshot_interval', 1000),
**kwargs)
]
elif callback_type == L3_BOOK:
cb[L3_BOOK] = [
book_cb(key=L3_BOOK,
snapshot_interval=feed_config.get(
'snapshot_interval', 1000),
**kwargs)
]
elif callback_type == OPEN_INTEREST:
cb[OPEN_INTEREST] = [oi_cb(**kwargs)]
elif callback_type == CANDLES:
cb[CANDLES] = [candles_cb(**kwargs)]
if 'pass_through' in self.config:
if self.config['pass_through']['type'] == 'zmq':
from cryptofeed.backends.zmq import TradeZMQ, BookZMQ, FundingZMQ, OpenInterestZMQ, TickerZMQ, LiquidationsZMQ, CandlesZMQ
host = self.config['pass_through']['host']
port = self.config['pass_through']['port']
if callback_type == TRADES:
cb[TRADES].append(TradeZMQ(host=host, port=port))
elif callback_type == LIQUIDATIONS:
cb[LIQUIDATIONS].append(
LiquidationsZMQ(host=host, port=port))
elif callback_type == FUNDING:
cb[FUNDING].append(FundingZMQ(host=host, port=port))
elif callback_type == L2_BOOK:
cb[L2_BOOK].append(
BookZMQ(host=host,
port=port,
snapshot_interval=feed_config.get(
'snapshot_interval', 1000)))
elif callback_type == L3_BOOK:
cb[L3_BOOK].append(
BookZMQ(host=host,
port=port,
snapshot_interval=feed_config.get(
'snapshot_interval', 1000)))
elif callback_type == OPEN_INTEREST:
cb[OPEN_INTEREST].append(
OpenInterestZMQ(host=host, port=port))
elif callback_type == TICKER:
cb[TICKER].append(TickerZMQ(host=host, port=port))
elif callback_type == CANDLES:
cb[CANDLES].append(CandlesZMQ(host=host, port=port))
fh.add_feed(self.exchange,
subscription={
callback_type: self.exchange_config[callback_type]
},
callbacks=cb,
**feed_kwargs)
LOG.info(
f"Collector added feed handler - {self.exchange}({callback_type.upper()}, {feed_kwargs})"
)
# Signal handlers are set up inside FeedHandler objects
fh.run()
LOG.info("Collector for %s on PID %d stopped", self.exchange,
os.getpid())
