def __init__(self, config=None, raw_data_collection=None): """ config: str, dict or None if str, absolute path (including file name) of the config file. If not provided, config can also be a dictionary of values, or can be None, which will default options. See docs/config.md for more information. raw_data_collection: callback (see AsyncFileCallback) or None if set, enables collection of raw data from exchanges. ALL https/wss traffic from the exchanges will be collected. """ self.feeds = [] self.config = Config(config=config) self.raw_data_collection = None if raw_data_collection: Connection.raw_data_callback = raw_data_collection self.raw_data_collection = raw_data_collection get_logger('feedhandler', self.config.log.filename, self.config.log.level) if self.config.log_msg: LOG.info(self.config.log_msg) if self.config.uvloop: try: import uvloop asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) LOG.info('FH: uvloop initalized') except ImportError: LOG.info("FH: uvloop not initialized")
def __init__(self, retries=10, timeout_interval=10, log_messages_on_error=False, raw_message_capture=None, handler_enabled=True, config=None): """ retries: int number of times the connection will be retried (in the event of a disconnect or other failure) timeout_interval: int number of seconds between checks to see if a feed has timed out log_messages_on_error: boolean if true, log the message from the exchange on exceptions raw_message_capture: callback if defined, callback to save/process/handle raw message (primarily for debugging purposes) handler_enabled: boolean run message handlers (and any registered callbacks) when raw message capture is enabled config: str absolute path (including file name) of the config file. If not provided env var checked first, then local config.yaml """ self.feeds = [] self.retries = retries self.timeout = {} self.last_msg = defaultdict(lambda: None) self.timeout_interval = timeout_interval self.log_messages_on_error = log_messages_on_error self.raw_message_capture = raw_message_capture self.handler_enabled = handler_enabled self.config = Config(file_name=config) lfile = 'feedhandler.log' if not self.config or not self.config.log.filename else self.config.log.filename level = logging.WARNING if not self.config or not self.config.log.level else self.config.log.level get_logger('feedhandler', lfile, level)
def __init__(self, retries=10, timeout_interval=10, log_messages_on_error=False, raw_message_capture=None, handler_enabled=True, config=None): """ retries: int number of times the connection will be retried (in the event of a disconnect or other failure) timeout_interval: int number of seconds between checks to see if a feed has timed out log_messages_on_error: boolean if true, log the message from the exchange on exceptions raw_message_capture: callback if defined, callback to save/process/handle raw message (primarily for debugging purposes) handler_enabled: boolean run message handlers (and any registered callbacks) when raw message capture is enabled config: str, dict or None if str, absolute path (including file name) of the config file. If not provided, config can also be a dictionary of values, or can be None, which will default options. See docs/config.md for more information. """ self.feeds = [] self.retries = retries self.timeout = {} self.last_msg = defaultdict(lambda: None) self.timeout_interval = timeout_interval self.log_messages_on_error = log_messages_on_error self.raw_message_capture = raw_message_capture self.handler_enabled = handler_enabled self.config = Config(config=config) get_logger('feedhandler', self.config.log.filename, self.config.log.level)
def __init__(self, retries=10, timeout_interval=10, log_messages_on_error=False, raw_message_capture=None, config=None): """ retries: int number of times the connection will be retried (in the event of a disconnect or other failure) timeout_interval: int number of seconds between checks to see if a feed has timed out log_messages_on_error: boolean if true, log the message from the exchange on exceptions raw_message_capture: callback if defined, callback to save/process/handle raw message (primarily for debugging purposes) config: str, dict or None if str, absolute path (including file name) of the config file. If not provided, config can also be a dictionary of values, or can be None, which will default options. See docs/config.md for more information. """ self.feeds = [] self.retries = (retries + 1) if retries >= 0 else -1 self.timeout = {} self.last_msg = defaultdict(lambda: None) self.timeout_interval = timeout_interval self.log_messages_on_error = log_messages_on_error self.raw_message_capture = raw_message_capture self.config = Config(config=config) get_logger('feedhandler', self.config.log.filename, self.config.log.level) if self.config.log_msg: LOG.info(self.config.log_msg) if self.config.uvloop: try: import uvloop asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) LOG.info('FH: uvloop initalized') except ImportError: LOG.info("FH: uvloop not initialized")
def __init__(self, retries=10, timeout_interval=10, log_messages_on_error=False, raw_message_capture=None, config=None, exception_ignore: Optional[List[Exception]] = None): """ retries: int number of times the connection will be retried (in the event of a disconnect or other failure) timeout_interval: int number of seconds between checks to see if a feed has timed out log_messages_on_error: boolean if true, log the message from the exchange on exceptions raw_message_capture: callback if defined, callback to save/process/handle raw message (primarily for debugging purposes) config: str, dict or None if str, absolute path (including file name) of the config file. If not provided, config can also be a dictionary of values, or can be None, which will default options. See docs/config.md for more information. exception_ignore: list, or None an optional list of exceptions that cryptofeed should ignore (i.e. not handle). These will need to be handled by a user-defined exception handler (provided to run run method) or the exception will kill the task (but not the feedhandler). """ self.feeds = [] self.retries = (retries + 1) if retries >= 0 else -1 self.timeout = {} self.last_msg = defaultdict(lambda: None) self.timeout_interval = timeout_interval self.log_messages_on_error = log_messages_on_error self.raw_message_capture = raw_message_capture self.config = Config(config=config) if exception_ignore is not None and not isinstance( exception_ignore, list): raise ValueError( "exception_ignore must be a list of Exceptions or None") self.exceptions = exception_ignore get_logger('feedhandler', self.config.log.filename, self.config.log.level) if self.config.log_msg: LOG.info(self.config.log_msg) if self.config.uvloop: try: import uvloop asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) LOG.info('FH: uvloop initalized') except ImportError: LOG.info("FH: uvloop not initialized")
def __init__(self, config=None, sandbox=False): config = Config(config=config) get_logger('rest', config.rest.log.filename, config.rest.log.level) self.lookup = { 'bitmex': Bitmex(config.bitmex), 'bitfinex': Bitfinex(config.bitfinex), 'coinbase': Coinbase(config.coinbase, sandbox=sandbox), 'poloniex': Poloniex(config.poloniex), 'gemini': Gemini(config.gemini, sandbox=sandbox), 'kraken': Kraken(config.kraken), 'deribit': Deribit(config.deribit), 'binance_futures': BinanceFutures(config.binance_futures), 'binance_delivery': BinanceDelivery(config.binance_delivery), 'ftx': FTX(config.ftx) }
def __init__(self, config=None, sandbox=False): self.config = Config(file_name=config) lfile = 'rest.log' if not self.config or not self.config.restlog.filename else self.config.restlog.filename level = logging.WARNING if not self.config or not self.config.restlog.level else self.config.restlog.level get_logger('rest', lfile, level) self.lookup = { 'bitmex': Bitmex(config), 'bitfinex': Bitfinex(config), 'coinbase': Coinbase(config, sandbox=sandbox), 'poloniex': Poloniex(config), 'gemini': Gemini(config, sandbox=sandbox), 'kraken': Kraken(config), 'deribit': Deribit(config), 'binance_futures': BinanceFutures(config), 'binance_delivery': BinanceDelivery(config), 'ftx': FTX(config) }
def main(): config = Config( config={'log': { 'filename': 'ingester.log', 'level': 'DEBUG' }}) get_logger('ingester', config.log.filename, config.log.level) f = FeedHandler(config=config) callbacks = { TICKER: DeribitTickerPostgres(**postgres_cfg, cache_size=1000), TRADES: DeribitTradePostgres(**postgres_cfg), L2_BOOK: DeribitBookPostgres(**postgres_cfg, cache_size=1000) } deribit = Deribit(max_depth=2, subscription=get_new_subscription(), callbacks=callbacks) f.add_feed(deribit) f.run(start_loop=True, tasks=[ do_periodically_at( 8, 1, 1, functools.partial(subscribe_to_new_subscription, deribit)) ])
''' import asyncio from datetime import datetime as dt from datetime import timedelta from socket import error as socket_error import websockets from websockets import ConnectionClosed from cryptofeed.defines import L2_BOOK from cryptofeed.log import get_logger from cryptofeed.exchanges import GEMINI, HITBTC, BITFINEX, BITMEX, BITSTAMP, POLONIEX, COINBASE from cryptofeed import Gemini, HitBTC, Bitfinex, Bitmex, Bitstamp, Poloniex, Coinbase from cryptofeed.nbbo import NBBO LOG = get_logger('feedhandler', 'feedhandler.log') _EXCHANGES = { COINBASE: Coinbase, GEMINI: Gemini, HITBTC: HitBTC, POLONIEX: Poloniex, BITFINEX: Bitfinex, BITMEX: Bitmex, BITSTAMP: Bitstamp, } class FeedHandler: def __init__(self, retries=10, timeout_interval=5): """ retries: int
import time, json, hashlib, hmac, requests, base64 from time import sleep from datetime import datetime as dt import pandas as pd from cryptofeed.rest.api import API from cryptofeed.feeds import COINBASE from cryptofeed.log import get_logger from cryptofeed.standards import pair_std_to_exchange REQUEST_LIMIT = 10 LOG = get_logger('rest', 'rest.log') # API Docs https://docs.gdax.com/ class Coinbase(API): ID = COINBASE api = "https://api.pro.coinbase.com" sandbox_api = "https://api-public.sandbox.pro.coinbase.com" def _generate_signature(self, endpoint: str, method: str, body=''): timestamp = str(time.time()) message = ''.join([timestamp, method, endpoint, body]) hmac_key = base64.b64decode(self.key_secret) signature = hmac.new(hmac_key, message.encode('ascii'), hashlib.sha256) signature_b64 = base64.b64encode(signature.digest()).decode('utf-8') return { 'CB-ACCESS-KEY': self.key_id, # The api key as a string.
from cryptofeed.defines import (BINANCE, BINANCE_FUTURES, BINANCE_JERSEY, BINANCE_US, BITCOINCOM, BITFINEX, BITMAX, BITMEX, BITSTAMP, BITTREX, BLOCKCHAIN, BYBIT, COINBASE, COINBENE, DERIBIT) from cryptofeed.defines import EXX as EXX_str from cryptofeed.defines import FTX as FTX_str from cryptofeed.defines import (FTX_US, GATEIO, GEMINI, HITBTC, HUOBI, HUOBI_DM, HUOBI_SWAP, KRAKEN, KRAKEN_FUTURES, L2_BOOK, OKCOIN, OKEX, POLONIEX, UPBIT) from cryptofeed.exceptions import ExhaustedRetries from cryptofeed.exchange.blockchain import Blockchain from cryptofeed.exchanges import * from cryptofeed.feed import RestFeed from cryptofeed.log import get_logger from cryptofeed.nbbo import NBBO LOG = get_logger('feedhandler', os.environ.get('CRYPTOFEED_FEEDHANDLER_LOG_FILENAME', "feedhandler.log"), int(os.environ.get('CRYPTOFEED_FEEDHANDLER_LOG_LEVEL', logging.WARNING))) # Maps string name to class name for use with config _EXCHANGES = { BINANCE: Binance, BINANCE_US: BinanceUS, BINANCE_JERSEY: BinanceJersey, BINANCE_FUTURES: BinanceFutures, BITCOINCOM: BitcoinCom, BITFINEX: Bitfinex, BITMAX: Bitmax, BITMEX: Bitmex, BITSTAMP: Bitstamp, BITTREX: Bittrex, BLOCKCHAIN: Blockchain,
import os from cryptofeed.log import get_logger from cryptofeed.rest.bitfinex import Bitfinex from cryptofeed.rest.bitmex import Bitmex from cryptofeed.rest.coinbase import Coinbase from cryptofeed.rest.deribit import Deribit from cryptofeed.rest.ftx import FTX from cryptofeed.rest.gemini import Gemini from cryptofeed.rest.kraken import Kraken from cryptofeed.rest.poloniex import Poloniex from cryptofeed.standards import load_exchange_pair_mapping LOG = get_logger('rest', os.environ.get('CRYPTOFEED_REST_LOG_FILENAME', "rest.log"), int(os.environ.get('CRYPTOFEED_REST_LOG_LEVEL', logging.WARNING))) class Rest: """ The rest class is a common interface for accessing the individual exchanges r = Rest() r.bitmex.trades('XBTUSD', '2018-01-01', '2018-01-01') The Rest class optionally takes two exchange-related parameters, config, and sandbox. In the config file the api key and secrets can be specified. Sandbox enables sandbox mode, if supported by the exchange. """