def default_market_data(*exchanges):
engine = create_engine(
DB_CONNECTION.format(database=config.database.name, **config_dict))
Session = sessionmaker(bind=engine)
session = Session()
# import ipdb; ipdb.set_trace()
for exchange in exchanges:
market_data = MarketData(exchange)
markets = market_data.exchange.fetch_markets()
for market in markets:
market_object = session.query(Market).filter_by(
ccxt_market_id=market.get('id'),
exchange_code=market_data.exchange.name,
symbol=market.get('symbol')).first()
if market_object:
print('Market: %s, %s exists' %
(exchange, market.get('symbol')))
else:
print('Adding market %s' % market)
market_object = Market(ccxt_market_id=market.get('id'),
exchange_code=market_data.exchange.name,
symbol=market.get('symbol'),
base_currency=market.get('base'),
quote_currency=market.get('quote'),
track_data='F',
trade_market='F',
last_updated=datetime.datetime.today())
session.add(market_object)
session.commit()
session.close()
def get_exchange_data(exchange):
market_data = MarketData(exchange)
while True:
_task = r.spop('%s-tasks' % exchange)
if _task is None:
break
logger.info('Performing task: %s' % _task)
_task = json.loads(_task.decode())
data_type = _task.get('task')
symbol = _task.get('symbol')
freq = _task.get('freq')
with market_data as m:
try:
if data_type == 'trade_data':
# TODO: Pull the retention policy out into config
m.get_trades(symbol).save(retention_policy="90day")
elif data_type == 'trend_data':
if m.exchange_code == 'gdax':
m = m.get_trend(symbol, freq=freq, latest=True)
else:
m = m.get_trend(symbol, freq=freq)
if not m.result.get('trend_data'):
r.sadd('%s-tasks' % exchange, json.dumps(_task))
m.save()
except Exception as e:
continue
def test_trades():
ipdb.set_trace()
bn = MarketData('binance')
gd = MarketData('gdax')
bn = bn.get_trades('ETH/BTC')
gd = gd.get_trades('BTC/USD')
bn = bn.save(retention_policy="test_policy") #
gd = gd.save(retention_policy="test_policy")
class TestTimeSeries(object):
def setup(self):
self.client = TimeSeriesClient()
self.exchange = MarketData(exchange_code=test_exchange)
@pytest.mark.usefixtures("mock_write_points")
def test_write_points(self):
test_data = self.exchange.get_trend('BTC/USD', '1d')
self.client.write_dataframe(
test_data,
'trend_data',
tags={
'exchange': test_exchange,
'symbol': 'BTC/USD',
'resolution': '1d'
},
field_columns=['open', 'high', 'low', 'close', 'volume'],
)
# This test will fail in the fixture... its was written to test the test.
assert True
@pytest.mark.usefixtures("mock_write_points")
def test_write_trades(self):
trades = self.exchange.get_trades(symbol, since=0)
# ipdb.set_trace()
last_timestamp = trades['timestamp'].max()
self.client.write_dataframe(
trades[['symbol', 'side', 'id', 'price', 'amount']],
'trade_data',
tags={'exchange': 'gdax'},
tag_columns=['symbol', 'side'])
time.sleep(1)
def test_get_last_trade(self):
#ipdb.set_trace()
last_saved_trade = self.client.get_last_trade(test_exchange, symbol)
# @pytest.mark.usefixtures("mock_get_last_query")
def test_get_last(self):
ipdb.set_trace()
last_val = self.client.get_last(test_exchange, symbol, 'trade')
def __init__(self, exchange, symbol, freq):
self.exchange = exchange
self.symbol = symbol
self.freq = freq
self.start_time = 0
self.end_time = None
self.run_count = 0
self.method = 'geometric_brownian_motion'
self.market_data = MarketData(exchange)
super().__init__()
def __init__(self, exchange, symbol, freq, **kwargs):
self.exchange = exchange
self.symbol = symbol
self.freq = freq
self.start_time = 0
self.end_time = None
self.run_count = 0
self.method = 'geometric_brownian_motion'
self.market_data = MarketData(exchange)
self.result = DataFrame()
self.configure(**kwargs)
super().__init__()
def __init__(self, exchange, symbol, freq, **kwargs):
self.exchange = exchange
self.symbol = symbol
self.freq = freq
self.start_time = 0
self.end_time = None
self.run_count = 0
self.center_of_mass = None
self.span = None
self.alpha = None
self.market_data = MarketData(exchange)
self.configure(**kwargs)
super().__init__()
def setup(self):
self.exchange = test_exchange
self.market_data = MarketData(exchange_code=self.exchange)
class TestMarketData(object):
def setup(self):
self.exchange = test_exchange
self.market_data = MarketData(exchange_code=self.exchange)
def test_set_exchange(self):
# import ipdb; ipdb.set_trace()
new_market_data = self.market_data.set_exchange('bitfinex')
assert new_market_data.exchange_code == 'bitfinex'
self.market_data = self.market_data.set_exchange(test_exchange)
def test_get_all_exchanges(self):
exchanges = self.market_data.get_all_exchanges()
time.sleep(1)
assert len(exchanges) > 0
def test_get_markets(self):
markets = self.market_data.get_markets()
time.sleep(1)
assert len(markets) > 0
def test_get_trend(self):
ohlcv = self.market_data.get_trend('BTC/USD', '1d')
time.sleep(1)
assert len(ohlcv) > 0
assert ohlcv.index.dtype == 'datetime64[ns]'
def test_get_trades(self):
trades = self.market_data.get_trades(symbol)
ts_client = TimeSeriesClient()
ts_client.write_dataframe(
trades[['symbol', 'side', 'id', 'price', 'amount', 'timestamp']],
'test_trade_data',
tags={'exchange': test_exchange},
tag_columns=['symbol', 'side'])
time.sleep(2)
last_saved_trade = self.market_data.latest_trades(
self.exchange, 'BTC/USD')
ipdb.set_trace()
# assert len(trades.index) > len(new_trades.index)
print(len(trades.index), len(new_trades.index))
# def test_config():
# print(config)
# def test_gdax_response():
# assert len(products(api_base)) > 0
# def test_gdax_request_string():
# auth = GDAXRequestAuth()
# def test_order():
# auth = GDAXRequestAuth()
# order_url = api_base + '/orders'
# order_data = {
# 'type': 'market',
# 'side': 'sell',
# 'product_id': 'BTC-USD',
# 'size': '0.1'
# }
# order_placed_response = requests.post(order_url, data=json.dumps(order_data), auth=auth)
# order_data = order_placed_response.json()
# order_id = order_data['id']
# order_url = api_base + '/orders/' + order_id
# order_status_response = requests.get(order_url, auth=auth)
# import ipdb;ipdb.set_trace()
# print(response.json())
def setup(self):
self.client = TimeSeriesClient()
self.exchange = MarketData(exchange_code=test_exchange)
class TestMarketData(object):
def setup(self):
self.exchange = test_exchange
self.market_data = MarketData(exchange_code=self.exchange)
def test_set_exchange(self):
new_market_data = self.market_data.set_exchange('bitfinex')
assert new_market_data.exchange_code == 'bitfinex'
self.market_data = self.market_data.set_exchange(test_exchange)
def test_get_all_exchanges(self):
exchanges = self.market_data.get_all_exchanges()
time.sleep(1)
assert len(exchanges) > 0
def test_get_markets(self):
markets = self.market_data.get_markets()
time.sleep(1)
assert len(markets) > 0
# @pytest.mark.usefixtures("mock_exchange")
def test_get_trend(self):
self.market_data = self.market_data.get_trend('BTC/USD',
freq='5m',
wait=False)
assert len(self.market_data.result['trend_data'].index) == 10
@pytest.mark.usefixtures("mock_exchange")
def test_get_trades(self):
self.market_data = self.market_data.get_trades('BTC/USD', wait=False)
assert len(self.market_data.result['trade_data'].index) == 3
def test_global_lock(self):
# TODO: this test depends on redis... mock?
md1 = MarketData('gdax')
md2 = MarketData('gdax')
# Check they have different id's
assert md1._id != md2._id
# Test manually locking
md1.lock()
with pytest.raises(Exception) as e_info:
md2.lock()
md1.release()
# Ensure I can release the lock
assert md1._check_lock() is None
# Test locking via context handler
with md1 as _md1:
with pytest.raises(Exception) as e_info:
md2.lock()
# Ensure context handler released cleanly
assert md1._check_lock() is None
@pytest.mark.usefixtures("mock_exchange")
def test_save(self):
self.market_data.result.clear()
self.market_data = self.market_data.get_trend('BTC/USD',
freq='1d',
wait=False)
self.market_data = self.market_data.get_trades('BTC/USD', wait=False)
self.market_data.save()
assert self.market_data.result == {}
def test_global_lock(self):
# TODO: this test depends on redis... mock?
md1 = MarketData('gdax')
md2 = MarketData('gdax')
# Check they have different id's
assert md1._id != md2._id
# Test manually locking
md1.lock()
with pytest.raises(Exception) as e_info:
md2.lock()
md1.release()
# Ensure I can release the lock
assert md1._check_lock() is None
# Test locking via context handler
with md1 as _md1:
with pytest.raises(Exception) as e_info:
md2.lock()
# Ensure context handler released cleanly
assert md1._check_lock() is None