def test_request(self):
"""Exchange list should be returned on request"""
cc = CryptoCompare()
result = cc.get_exchange_list()
self.assertIn('Bitstamp', result)
self.assertIn('Kraken', result)
self.assertIn('Poloniex', result)
def test_one_feed(self):
"""News for provided feeds only should be provided when specified"""
cc = CryptoCompare()
feed = 'cryptocompare'
result = cc.get_latest_news(feeds=feed)
for news in result:
self.assertEqual(news['source'], feed)
def test_specific_lang(self):
"""News should be in specific lang when provided"""
cc = CryptoCompare()
lang = 'PT'
result = cc.get_latest_news(lang=lang)
for news in result:
self.assertEqual(news['lang'], lang)
def test_request(self):
"""Coin list should be returned on request"""
cc = CryptoCompare()
result = cc.get_coin_list()
self.assertIn('BTC', result)
self.assertIn('LTC', result)
self.assertIn('ETH', result)
def test_multiple_tsyms(self):
"""Historical price should be returned with all quote symbols passed"""
cc = CryptoCompare()
result = cc.get_historical_for_timestamp('BTC', ('USD', 'EUR'),
int(time.time()))
self.assertIsInstance(result['BTC']['USD'], numbers.Real)
self.assertIsInstance(result['BTC']['EUR'], numbers.Real)
def test_bitcoin_multiple_tsyms(self):
"""Requesting multiple price for multiple quote symbols should work"""
cc = CryptoCompare()
tsyms = ['USD', 'EUR', 'JPY', 'ETH']
result = cc.get_price('BTC', tsyms)
for symbol in tsyms:
self.assertIn(symbol, result['BTC'].keys())
def test_before_date_from_datetime(self):
"""News should only be from provided datetime"""
cc = CryptoCompare()
dt = datetime.datetime(year=2017, month=6, day=1)
result = cc.get_latest_news(before=dt)
for news in result:
self.assertLessEqual(news['published_on'], dt.timestamp())
def test_some_feeds(self):
"""News for provided feeds only should be provided when specified"""
cc = CryptoCompare()
feeds = ['coindesk', 'cryptocompare']
result = cc.get_latest_news(feeds=feeds)
for news in result:
self.assertIn(news['source'], feeds)
def test_no_feed(self):
"""
Latest news for all feeds should be returned if no specific feed is
provided
"""
cc = CryptoCompare()
result = cc.get_latest_news()
self.assertGreater(len(result), 0)
def test_specific_exchange(self):
"""Requesting historical data from a specific exchange should work"""
cc = CryptoCompare()
result = cc.get_historical_for_timestamp('BTC',
'USD',
int(time.time()),
exchange='Kraken')
self.assertIsInstance(result['BTC']['USD'], numbers.Real)
def test_limit(self):
"""
When a limit is provided, historical data should only have that many
elements (at most)
"""
cc = CryptoCompare()
result = cc.get_historical('BTC', 'USD', limit=25)
# for some reason CryptoCompare returns limit + 1 points
self.assertEqual(len(result), 25 + 1)
def test_type_equal_orover_threshold(self):
"""
Response containing type field equal to or over CryptoCompare threshold
should not raise any error
"""
CryptoCompare._check_request_response_error(
{'Type': ERROR_TYPE_THRESHOLD})
CryptoCompare._check_request_response_error(
{'Type': ERROR_TYPE_THRESHOLD + 1})
def test_multiple_fsyms_and_tsyms(self):
"""Requesting multiple price for multiple base and quote symbols should work"""
cc = CryptoCompare()
fsyms = ['BTC', 'LTC', 'ETH']
tsyms = ['USD', 'EUR']
result = cc.get_price(fsyms, tsyms)
for fs in fsyms:
for ts in tsyms:
self.assertIn(ts, result[fs])
def __init__(self, config={}):
self.influxdb_measurement = config.get("influxdb_measurement",
"market_ohlc")
self.market = config.get("market", "USDT-BTC")
self.exchange = config.get("exchange", "bittrex")
self.currency = self.market.split("-")[1]
self.influx = InfluxDbWrapper.getInstance()
self.bittrex = Bittrex()
self.cryptocompare = CryptoCompare()
def test_error_has_message(self):
"""Raised error should provide message if in response"""
try:
CryptoCompare._check_request_response_error({
'Response':
'Error',
'Message':
'Lorem ipsum dolor sit amet'
})
except CryptoCompareApiError as e:
self.assertEqual(str(e), 'Lorem ipsum dolor sit amet')
else:
self.fail("Should have raised")
def test_to_ts(self):
"""
When timestamp limit is provided, historical data should only be before
that limit
"""
cc = CryptoCompare()
ts = int(time.time()) - 1000
result = cc.get_historical('BTC',
'USD',
period=Period.MINUTE,
to_ts=ts)
for e in result:
self.assertLessEqual(e['time'], ts)
def test_bitcoin_usd_specific_unknown_exchange(self):
"""Requesting price for a pair on an exchange that does not exist should fail"""
cc = CryptoCompare()
self.assertRaises(CryptoCompareApiError,
cc.get_symbols_full_data,
'BTC',
'USD',
exchange='foo')
def __init__(self,
database,
api_key,
host="localhost",
port=8086,
*args,
**kwargs):
self.logger = logging.getLogger(self.__class__.__name__)
self.crypto = CryptoCompare(api_key)
self.influx = InfluxDB(database=database, host=host, port=port)
self.loop = asyncio.get_event_loop()
import os, sys, json, time, threading
sys.path.append(os.getenv("CRYPTO_LIB", "/projects/apps/shared/crypto"))
import cryptolib
from influxdbwrapper import InfluxDbWrapper
from scanbot import ScanBot
from bittrex import Bittrex
from datetime import datetime
from mongowrapper import MongoWrapper
from cryptocompare import CryptoCompare
from concurrent.futures import ThreadPoolExecutor
bc = Bittrex()
cc = CryptoCompare()
def topscan(market):
print("{}".format(market))
try:
mongo = MongoWrapper.getInstance().getClient()
bot = ScanBot({
"market": market,
"candlesize": "1d"
},
name='topscanner')
res = bot.process({'scrape': True})
mongo.crypto.scanner.replace_one({
'market': market,
'candlesize': '1d'
class Scraper(object):
PERIODMAP = {
"oneMin": "1m",
"fiveMin": "5m",
"thirtyMin": "30m",
"hour": "1h",
"day": "1d"
}
def __init__(self, config={}):
self.influxdb_measurement = config.get("influxdb_measurement",
"market_ohlc")
self.market = config.get("market", "USDT-BTC")
self.exchange = config.get("exchange", "bittrex")
self.currency = self.market.split("-")[1]
self.influx = InfluxDbWrapper.getInstance()
self.bittrex = Bittrex()
self.cryptocompare = CryptoCompare()
def cc_lastprice(self):
resp = self.cryptocompare.lastprice(self.exchange, self.market)
return resp
def cc_scrapeCandle(self, period="1m"):
lasttime = self.getLastCandle(period)
# print("scraping market: {}, exchange: {}, period {}, last candle: {}".format(self.market,self.exchange,period,lasttime))
resp = self.cryptocompare.get_candles(self.exchange, self.market,
period)
data = resp.data
skip = True
for candle in data["Data"]:
t = datetime.datetime.utcfromtimestamp(int(
candle["time"])).strftime('%Y-%m-%dT%H:%M:%S')
if lasttime == "" or lasttime == t:
skip = False
if lasttime is None:
skip = False
else:
tt = time.mktime(
datetime.datetime.strptime(
t, "%Y-%m-%dT%H:%M:%S").timetuple())
lt = time.mktime(
datetime.datetime.strptime(
lasttime, "%Y-%m-%dT%H:%M:%S").timetuple())
if lt < tt and skip:
# print("missing a lot of data")
skip = False
# print("lasttime = {} , t = {}, skip = {}".format(lasttime,t,skip))
# if not skip and lasttime != t:
if not skip:
item = {
"measurement": self.influxdb_measurement,
"time": t,
"tags": {
"currency": self.currency,
"period": period,
"market": self.market,
"exchange": self.exchange,
"source": "cryptocompare",
},
"fields": {
"open": float(candle["open"]),
"high": float(candle["high"]),
"low": float(candle["low"]),
"close": float(candle["close"]),
"volume": float(candle["volumefrom"]),
"base_volume": float(candle["volumeto"])
}
}
#print(item)
self.influx.bulkAdd(item)
# print("saving exchange: {} market: {} time: {}".format(self.exchange,self.market,t))
self.influx.bulkSave()
return self.cc_normalize_candles(data["Data"])
def cc_normalize_candles(self, cdata):
psize = 0
cs = {
"open": [],
"closed": [],
"high": [],
"low": [],
"volume": [],
"basevolume": [],
"time": [],
"opening": [],
"closing": []
}
for point in cdata:
t = datetime.datetime.utcfromtimestamp(int(
point["time"])).strftime('%Y-%m-%dT%H:%M:%S')
psize += 1
cs["low"].extend([point["low"]])
cs["high"].extend([point["high"]])
cs["closed"].extend([point["close"]])
cs["open"].extend([point["open"]])
cs["volume"].extend([float(point["volumefrom"])])
cs["basevolume"].extend([float(point["volumeto"])])
cs["time"].extend([t])
return {
"low": numpy.array(cs["low"]),
"high": numpy.array(cs["high"]),
"closed": numpy.array(cs["closed"]),
"volume": numpy.array(cs["volume"]),
"basevolume": numpy.array(cs["basevolume"]),
"open": numpy.array(cs["open"]),
"time": cs["time"],
}
def scrapeCandle(self, period):
lasttime = self.getLastCandle(Scraper.PERIODMAP[period])
# print("scraping market: {}, period {}, last candle: {}".format(self.market,period,lasttime))
resp = self.bittrex.public_get_candles(market=self.market,
tickInterval=period)
data = resp.data
skip = True
for candle in data["result"]:
if lasttime == "" or lasttime == candle["T"]:
skip = False
if not skip and lasttime != candle["T"]:
item = {
"measurement": self.influxdb_measurement,
"time": candle["T"],
"tags": {
"currency": self.currency,
"period": Scraper.PERIODMAP[period],
"market": self.market,
"exchange": self.exchange,
},
"fields": {
"open": float(candle["O"]),
"high": float(candle["H"]),
"low": float(candle["L"]),
"close": float(candle["C"]),
"volume": float(candle["V"]),
"base_volume": float(candle["BV"])
}
}
self.influx.bulkAdd(item)
# print("saving exchange: {} market: {} time: {}".format(self.exchange,self.market,candle["T"]))
self.influx.bulkSave()
def getLastCandle(self, period):
query = """SELECT base_volume FROM "{}" WHERE market='{}' AND period='{}' AND exchange='{}' ORDER BY time DESC LIMIT 2 """.format(
self.influxdb_measurement, self.market, period, self.exchange)
res = list(self.influx.raw_query(query).get_points())
if len(res) > 1:
return res[1]["time"][:-1]
def test_btc_usd_lowercase(self):
"""Request lowercase written BTC/USD top exchanges should work"""
cc = CryptoCompare()
result = cc.get_top_exchanges('btc', 'usd')
self.assertGreater(len(result), 0)
import pandas as pd
from cryptocompare import CryptoCompare
import os
import sys
import time
import logging
DATA_DIR = 'data/'
cc = CryptoCompare()
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO,
format='%(asctime)s|%(levelname)s|%(message)s',
datefmt='%Y%m%d %H:%M:%S',
stream=sys.stdout)
# Get list of coins on binance
EXCHANGES = ['Binance', 'Bitfinex', 'Coinbase']
BASE_SYMS = ['BTC', 'USD', 'USDT', 'ETH']
SLEEP_TIME_S = 60
for exchange in EXCHANGES:
for base_sym in BASE_SYMS:
coin_pairs = cc.coins_by_exchange(exchange)[base_sym]
for fsym, sym in coin_pairs.items():
prices = cc.get_data(sym, exchange, bar_size='minute')
fpath = os.path.join(
DATA_DIR, '{}_{}.csv'.format(exchange.lower(),
sym.lower().replace('/', '')))
if prices is not None:
def test_unknown_pair(self):
"""Requesting unknown pair top exchanges should fail"""
cc = CryptoCompare()
self.assertRaises(CryptoCompareApiError, cc.get_top_exchanges, 'FBFBF',
'ZXXCD')
def test_bitcoin_usd(self):
"""Request BTC/USD price should return that pair rate"""
cc = CryptoCompare()
result = cc.get_price('BTC', 'USD')
self.assertIsInstance(result['BTC']['USD'], numbers.Real)
def test_bitcoin(self):
"""Request BTC top pairs should work"""
cc = CryptoCompare()
result = cc.get_top_pairs('BTC')
self.assertGreater(len(result), 0)
def test_bitcoin_lowercase(self):
"""Request lowercase written BTC top pairs should work"""
cc = CryptoCompare()
result = cc.get_top_pairs('btc')
self.assertGreater(len(result), 0)
def test_unknown_symbol(self):
"""Requesting unknown symbol top pairs should fail"""
cc = CryptoCompare()
self.assertRaises(CryptoCompareApiError, cc.get_top_pairs, 'XZSDFE')
def test_specific_limit(self):
"""Request top pairs with a specific limit should return that many pairs (at most)"""
cc = CryptoCompare()
result = cc.get_top_pairs('BTC', limit=10)
self.assertEqual(len(result), 10)
def test_unknown_lang(self):
"""Error should be raised when provided unknown lang"""
cc = CryptoCompare()
self.assertRaises(CryptoCompareApiError,
cc.get_latest_news,
lang='WXYZ')
def test_request(self):
"""News providers should be returned on request"""
cc = CryptoCompare()
result = cc.get_news_providers()
self.assertGreater(len(result), 0)