def __init__(self, config=None, raw_data_collection=None):
"""
config: str, dict or None
if str, absolute path (including file name) of the config file. If not provided, config can also be a dictionary of values, or
can be None, which will default options. See docs/config.md for more information.
raw_data_collection: callback (see AsyncFileCallback) or None
if set, enables collection of raw data from exchanges. ALL https/wss traffic from the exchanges will be collected.
"""
self.feeds = []
self.config = Config(config=config)
self.raw_data_collection = None
if raw_data_collection:
Connection.raw_data_callback = raw_data_collection
self.raw_data_collection = raw_data_collection
get_logger('feedhandler', self.config.log.filename, self.config.log.level)
if self.config.log_msg:
LOG.info(self.config.log_msg)
if self.config.uvloop:
try:
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
LOG.info('FH: uvloop initalized')
except ImportError:
LOG.info("FH: uvloop not initialized")
def __init__(self,
retries=10,
timeout_interval=10,
log_messages_on_error=False,
raw_message_capture=None,
handler_enabled=True,
config=None):
"""
retries: int
number of times the connection will be retried (in the event of a disconnect or other failure)
timeout_interval: int
number of seconds between checks to see if a feed has timed out
log_messages_on_error: boolean
if true, log the message from the exchange on exceptions
raw_message_capture: callback
if defined, callback to save/process/handle raw message (primarily for debugging purposes)
handler_enabled: boolean
run message handlers (and any registered callbacks) when raw message capture is enabled
config: str
absolute path (including file name) of the config file. If not provided env var checked first, then local config.yaml
"""
self.feeds = []
self.retries = retries
self.timeout = {}
self.last_msg = defaultdict(lambda: None)
self.timeout_interval = timeout_interval
self.log_messages_on_error = log_messages_on_error
self.raw_message_capture = raw_message_capture
self.handler_enabled = handler_enabled
self.config = Config(file_name=config)
lfile = 'feedhandler.log' if not self.config or not self.config.log.filename else self.config.log.filename
level = logging.WARNING if not self.config or not self.config.log.level else self.config.log.level
get_logger('feedhandler', lfile, level)
def __init__(self,
retries=10,
timeout_interval=10,
log_messages_on_error=False,
raw_message_capture=None,
handler_enabled=True,
config=None):
"""
retries: int
number of times the connection will be retried (in the event of a disconnect or other failure)
timeout_interval: int
number of seconds between checks to see if a feed has timed out
log_messages_on_error: boolean
if true, log the message from the exchange on exceptions
raw_message_capture: callback
if defined, callback to save/process/handle raw message (primarily for debugging purposes)
handler_enabled: boolean
run message handlers (and any registered callbacks) when raw message capture is enabled
config: str, dict or None
if str, absolute path (including file name) of the config file. If not provided, config can also be a dictionary of values, or
can be None, which will default options. See docs/config.md for more information.
"""
self.feeds = []
self.retries = retries
self.timeout = {}
self.last_msg = defaultdict(lambda: None)
self.timeout_interval = timeout_interval
self.log_messages_on_error = log_messages_on_error
self.raw_message_capture = raw_message_capture
self.handler_enabled = handler_enabled
self.config = Config(config=config)
get_logger('feedhandler', self.config.log.filename,
self.config.log.level)
def __init__(self, retries=10, timeout_interval=10, log_messages_on_error=False, raw_message_capture=None, config=None):
"""
retries: int
number of times the connection will be retried (in the event of a disconnect or other failure)
timeout_interval: int
number of seconds between checks to see if a feed has timed out
log_messages_on_error: boolean
if true, log the message from the exchange on exceptions
raw_message_capture: callback
if defined, callback to save/process/handle raw message (primarily for debugging purposes)
config: str, dict or None
if str, absolute path (including file name) of the config file. If not provided, config can also be a dictionary of values, or
can be None, which will default options. See docs/config.md for more information.
"""
self.feeds = []
self.retries = (retries + 1) if retries >= 0 else -1
self.timeout = {}
self.last_msg = defaultdict(lambda: None)
self.timeout_interval = timeout_interval
self.log_messages_on_error = log_messages_on_error
self.raw_message_capture = raw_message_capture
self.config = Config(config=config)
get_logger('feedhandler', self.config.log.filename, self.config.log.level)
if self.config.log_msg:
LOG.info(self.config.log_msg)
if self.config.uvloop:
try:
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
LOG.info('FH: uvloop initalized')
except ImportError:
LOG.info("FH: uvloop not initialized")
def __init__(self,
retries=10,
timeout_interval=10,
log_messages_on_error=False,
raw_message_capture=None,
config=None,
exception_ignore: Optional[List[Exception]] = None):
"""
retries: int
number of times the connection will be retried (in the event of a disconnect or other failure)
timeout_interval: int
number of seconds between checks to see if a feed has timed out
log_messages_on_error: boolean
if true, log the message from the exchange on exceptions
raw_message_capture: callback
if defined, callback to save/process/handle raw message (primarily for debugging purposes)
config: str, dict or None
if str, absolute path (including file name) of the config file. If not provided, config can also be a dictionary of values, or
can be None, which will default options. See docs/config.md for more information.
exception_ignore: list, or None
an optional list of exceptions that cryptofeed should ignore (i.e. not handle). These will need to be handled
by a user-defined exception handler (provided to run run method) or the exception will kill the task (but not the feedhandler).
"""
self.feeds = []
self.retries = (retries + 1) if retries >= 0 else -1
self.timeout = {}
self.last_msg = defaultdict(lambda: None)
self.timeout_interval = timeout_interval
self.log_messages_on_error = log_messages_on_error
self.raw_message_capture = raw_message_capture
self.config = Config(config=config)
if exception_ignore is not None and not isinstance(
exception_ignore, list):
raise ValueError(
"exception_ignore must be a list of Exceptions or None")
self.exceptions = exception_ignore
get_logger('feedhandler', self.config.log.filename,
self.config.log.level)
if self.config.log_msg:
LOG.info(self.config.log_msg)
if self.config.uvloop:
try:
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
LOG.info('FH: uvloop initalized')
except ImportError:
LOG.info("FH: uvloop not initialized")
def __init__(self, config=None, sandbox=False):
config = Config(config=config)
get_logger('rest', config.rest.log.filename, config.rest.log.level)
self.lookup = {
'bitmex': Bitmex(config.bitmex),
'bitfinex': Bitfinex(config.bitfinex),
'coinbase': Coinbase(config.coinbase, sandbox=sandbox),
'poloniex': Poloniex(config.poloniex),
'gemini': Gemini(config.gemini, sandbox=sandbox),
'kraken': Kraken(config.kraken),
'deribit': Deribit(config.deribit),
'binance_futures': BinanceFutures(config.binance_futures),
'binance_delivery': BinanceDelivery(config.binance_delivery),
'ftx': FTX(config.ftx)
}
def __init__(self, config=None, sandbox=False):
self.config = Config(file_name=config)
lfile = 'rest.log' if not self.config or not self.config.restlog.filename else self.config.restlog.filename
level = logging.WARNING if not self.config or not self.config.restlog.level else self.config.restlog.level
get_logger('rest', lfile, level)
self.lookup = {
'bitmex': Bitmex(config),
'bitfinex': Bitfinex(config),
'coinbase': Coinbase(config, sandbox=sandbox),
'poloniex': Poloniex(config),
'gemini': Gemini(config, sandbox=sandbox),
'kraken': Kraken(config),
'deribit': Deribit(config),
'binance_futures': BinanceFutures(config),
'binance_delivery': BinanceDelivery(config),
'ftx': FTX(config)
}
def main():
config = Config(
config={'log': {
'filename': 'ingester.log',
'level': 'DEBUG'
}})
get_logger('ingester', config.log.filename, config.log.level)
f = FeedHandler(config=config)
callbacks = {
TICKER: DeribitTickerPostgres(**postgres_cfg, cache_size=1000),
TRADES: DeribitTradePostgres(**postgres_cfg),
L2_BOOK: DeribitBookPostgres(**postgres_cfg, cache_size=1000)
}
deribit = Deribit(max_depth=2,
subscription=get_new_subscription(),
callbacks=callbacks)
f.add_feed(deribit)
f.run(start_loop=True,
tasks=[
do_periodically_at(
8, 1, 1,
functools.partial(subscribe_to_new_subscription, deribit))
])
'''
import asyncio
from datetime import datetime as dt
from datetime import timedelta
from socket import error as socket_error
import websockets
from websockets import ConnectionClosed
from cryptofeed.defines import L2_BOOK
from cryptofeed.log import get_logger
from cryptofeed.exchanges import GEMINI, HITBTC, BITFINEX, BITMEX, BITSTAMP, POLONIEX, COINBASE
from cryptofeed import Gemini, HitBTC, Bitfinex, Bitmex, Bitstamp, Poloniex, Coinbase
from cryptofeed.nbbo import NBBO
LOG = get_logger('feedhandler', 'feedhandler.log')
_EXCHANGES = {
COINBASE: Coinbase,
GEMINI: Gemini,
HITBTC: HitBTC,
POLONIEX: Poloniex,
BITFINEX: Bitfinex,
BITMEX: Bitmex,
BITSTAMP: Bitstamp,
}
class FeedHandler:
def __init__(self, retries=10, timeout_interval=5):
"""
retries: int
import time, json, hashlib, hmac, requests, base64
from time import sleep
from datetime import datetime as dt
import pandas as pd
from cryptofeed.rest.api import API
from cryptofeed.feeds import COINBASE
from cryptofeed.log import get_logger
from cryptofeed.standards import pair_std_to_exchange
REQUEST_LIMIT = 10
LOG = get_logger('rest', 'rest.log')
# API Docs https://docs.gdax.com/
class Coinbase(API):
ID = COINBASE
api = "https://api.pro.coinbase.com"
sandbox_api = "https://api-public.sandbox.pro.coinbase.com"
def _generate_signature(self, endpoint: str, method: str, body=''):
timestamp = str(time.time())
message = ''.join([timestamp, method, endpoint, body])
hmac_key = base64.b64decode(self.key_secret)
signature = hmac.new(hmac_key, message.encode('ascii'), hashlib.sha256)
signature_b64 = base64.b64encode(signature.digest()).decode('utf-8')
return {
'CB-ACCESS-KEY': self.key_id, # The api key as a string.
from cryptofeed.defines import (BINANCE, BINANCE_FUTURES, BINANCE_JERSEY, BINANCE_US, BITCOINCOM, BITFINEX,
BITMAX, BITMEX, BITSTAMP, BITTREX, BLOCKCHAIN, BYBIT, COINBASE, COINBENE, DERIBIT)
from cryptofeed.defines import EXX as EXX_str
from cryptofeed.defines import FTX as FTX_str
from cryptofeed.defines import (FTX_US, GATEIO, GEMINI, HITBTC, HUOBI, HUOBI_DM, HUOBI_SWAP, KRAKEN,
KRAKEN_FUTURES, L2_BOOK, OKCOIN, OKEX, POLONIEX, UPBIT)
from cryptofeed.exceptions import ExhaustedRetries
from cryptofeed.exchange.blockchain import Blockchain
from cryptofeed.exchanges import *
from cryptofeed.feed import RestFeed
from cryptofeed.log import get_logger
from cryptofeed.nbbo import NBBO
LOG = get_logger('feedhandler',
os.environ.get('CRYPTOFEED_FEEDHANDLER_LOG_FILENAME', "feedhandler.log"),
int(os.environ.get('CRYPTOFEED_FEEDHANDLER_LOG_LEVEL', logging.WARNING)))
# Maps string name to class name for use with config
_EXCHANGES = {
BINANCE: Binance,
BINANCE_US: BinanceUS,
BINANCE_JERSEY: BinanceJersey,
BINANCE_FUTURES: BinanceFutures,
BITCOINCOM: BitcoinCom,
BITFINEX: Bitfinex,
BITMAX: Bitmax,
BITMEX: Bitmex,
BITSTAMP: Bitstamp,
BITTREX: Bittrex,
BLOCKCHAIN: Blockchain,
import os
from cryptofeed.log import get_logger
from cryptofeed.rest.bitfinex import Bitfinex
from cryptofeed.rest.bitmex import Bitmex
from cryptofeed.rest.coinbase import Coinbase
from cryptofeed.rest.deribit import Deribit
from cryptofeed.rest.ftx import FTX
from cryptofeed.rest.gemini import Gemini
from cryptofeed.rest.kraken import Kraken
from cryptofeed.rest.poloniex import Poloniex
from cryptofeed.standards import load_exchange_pair_mapping
LOG = get_logger('rest',
os.environ.get('CRYPTOFEED_REST_LOG_FILENAME', "rest.log"),
int(os.environ.get('CRYPTOFEED_REST_LOG_LEVEL', logging.WARNING)))
class Rest:
"""
The rest class is a common interface for accessing the individual exchanges
r = Rest()
r.bitmex.trades('XBTUSD', '2018-01-01', '2018-01-01')
The Rest class optionally takes two exchange-related parameters, config, and sandbox.
In the config file the api key and secrets can be specified. Sandbox enables sandbox
mode, if supported by the exchange.
"""
