def record(self, entity, start, end, size, timestamps):
if size < 20:
size = 20
ccxt_exchange = CCXTAccount.get_ccxt_exchange(entity.exchange)
if ccxt_exchange.has['fetchTrades']:
limit = CCXTAccount.get_tick_limit(entity.exchange)
limit = min(size, limit)
kdata_list = []
trades = ccxt_exchange.fetch_trades(entity.code, limit=limit)
for trade in trades:
kdata_json = {
'name': entity.name,
'provider': 'ccxt',
# 'id': trade['id'],
'level': 'tick',
'order': trade['order'],
'timestamp': to_pd_timestamp(trade['timestamp']),
'price': trade['price'],
'volume': trade['amount'],
'direction': trade['side'],
'order_type': trade['type'],
'turnover': trade['price'] * trade['amount']
}
kdata_list.append(kdata_json)
return kdata_list
else:
self.logger.warning("exchange:{} not support fetchOHLCV".format(entity.exchange))
def run(self):
for exchange_str in self.exchanges:
exchange = CCXTAccount.get_ccxt_exchange(exchange_str)
try:
markets = exchange.fetch_markets()
df = pd.DataFrame()
# markets有些为key=symbol的dict,有些为list
markets_type = type(markets)
if markets_type != dict and markets_type != list:
self.logger.exception(
"unknown return markets type {}".format(markets_type))
return
aa = []
for market in markets:
if markets_type == dict:
name = market
code = market
if markets_type == list:
code = market['symbol']
name = market['symbol']
if name not in COIN_PAIRS:
continue
aa.append(market)
security_item = {
'id':
'{}_{}_{}'.format(SecurityType.coin.value,
exchange_str, code),
'exchange':
exchange_str,
'type':
SecurityType.coin.value,
'code':
code,
'name':
name
}
df = df.append(security_item, ignore_index=True)
# 存储该交易所的数字货币列表
if not df.empty:
init_securities(df=df,
security_type=SecurityType.coin,
provider=self.provider)
self.logger.info(
"init_markets for {} success".format(exchange_str))
except Exception as e:
self.logger.exception(
"init_markets for {} failed".format(exchange_str), e)
def request(self, url=None, method='get', param=None, path_fields=None):
security_item = param['security_item']
start_timestamp = param['start_timestamp']
size = param['size']
ccxt_level = param['ccxt_level']
level = param['level']
ccxt_exchange = CCXTAccount.get_ccxt_exchange(security_item.exchange)
if ccxt_exchange.has['fetchOHLCV']:
limit = CCXTAccount.get_kdata_limit(security_item.exchange)
limit = min(size, limit)
kdata_list = []
try:
if CCXTAccount.exchange_conf[
security_item.exchange]['support_since']:
kdatas = ccxt_exchange.fetch_ohlcv(security_item.code,
timeframe=ccxt_level,
since=start_timestamp)
else:
kdatas = ccxt_exchange.fetch_ohlcv(security_item.code,
timeframe=ccxt_level,
limit=limit)
# always ignore the latest one,because it's not finished
for kdata in kdatas[0:-1]:
current_timestamp = kdata[0]
if level == TradingLevel.LEVEL_1DAY:
current_timestamp = to_time_str(current_timestamp)
kdata_json = {
'timestamp': to_pd_timestamp(current_timestamp),
'open': kdata[1],
'high': kdata[2],
'low': kdata[3],
'close': kdata[4],
'volume': kdata[5],
'name': security_item.name,
'provider': 'ccxt',
'level': level.value
}
kdata_list.append(kdata_json)
return kdata_list
except Exception as e:
logger.exception("record_kdata for security:{} failed".format(
security_item.id))
else:
logger.warning("exchange:{} not support fetchOHLCV".format(
security_item.exchange))
def record(self, entity, start, end, size, timestamps):
if self.start_timestamp:
start = max(self.start_timestamp, to_pd_timestamp(start))
start_timestamp = to_time_str(start)
ccxt_exchange = CCXTAccount.get_ccxt_exchange(entity.exchange)
if ccxt_exchange.has['fetchOHLCV']:
limit = CCXTAccount.get_kdata_limit(entity.exchange)
limit = min(size, limit)
kdata_list = []
if CCXTAccount.exchange_conf[entity.exchange]['support_since']:
kdatas = ccxt_exchange.fetch_ohlcv(
entity.code,
timeframe=self.ccxt_trading_level,
since=start_timestamp)
else:
kdatas = ccxt_exchange.fetch_ohlcv(
entity.code,
timeframe=self.ccxt_trading_level,
limit=limit)
# always ignore the latest one,because it's not finished
for kdata in kdatas[0:-1]:
current_timestamp = kdata[0]
if self.level == IntervalLevel.LEVEL_1DAY:
current_timestamp = to_time_str(current_timestamp)
kdata_json = {
'timestamp': to_pd_timestamp(current_timestamp),
'open': kdata[1],
'high': kdata[2],
'low': kdata[3],
'close': kdata[4],
'volume': kdata[5],
'name': entity.name,
'provider': 'ccxt',
'level': self.level.value
}
kdata_list.append(kdata_json)
return kdata_list
else:
self.logger.warning("exchange:{} not support fetchOHLCV".format(
entity.exchange))
def record(self, entity, start, end, size, timestamps):
if size < 20:
size = 20
entity = entity
ccxt_exchange = CCXTAccount.get_ccxt_exchange(entity.exchange)
if ccxt_exchange.has['fetchTrades']:
limit = CCXTAccount.get_tick_limit(entity.exchange)
limit = min(size, limit)
kdata_list = []
try:
trades = ccxt_exchange.fetch_trades(entity.code, limit=limit)
# always ignore the latest one,because it's not finished
for trade in trades[0:-1]:
kdata_json = {
'name': entity.name,
'provider': 'ccxt',
# 'id': trade['id'],
'level': 'tick',
'order': trade['order'],
'timestamp': to_pd_timestamp(trade['timestamp']),
'price': trade['price'],
'volume': trade['amount'],
'direction': trade['side'],
'orderType': trade['type'],
'turnover': trade['price'] * trade['amount']
}
kdata_list.append(kdata_json)
return kdata_list
except Exception as e:
logger.exception("record_kdata for security:{} failed".format(
entity.id))
else:
logger.warning("exchange:{} not support fetchOHLCV".format(
entity.exchange))
def request(self, url=None, method='get', param=None, path_fields=None):
security_item = param['security_item']
size = param['size']
ccxt_exchange = CCXTAccount.get_ccxt_exchange(security_item.exchange)
if ccxt_exchange.has['fetchTrades']:
limit = CCXTAccount.get_tick_limit(security_item.exchange)
limit = min(size, limit)
kdata_list = []
try:
trades = ccxt_exchange.fetch_trades(security_item.code,
limit=limit)
# always ignore the latest one,because it's not finished
for trade in trades[0:-1]:
kdata_json = {
'securityId': security_item.id,
'name': security_item.name,
'provider': 'ccxt',
# 'id': trade['id'],
'level': 'tick',
'order': trade['order'],
'timestamp': to_pd_timestamp(trade['timestamp']),
'price': trade['price'],
'volume': trade['amount'],
'direction': trade['side'],
'orderType': trade['type'],
'turnover': trade['price'] * trade['amount']
}
kdata_list.append(kdata_json)
return kdata_list
except Exception as e:
logger.exception("record_kdata for security:{} failed".format(
security_item.id))
else:
logger.warning("exchange:{} not support fetchOHLCV".format(
security_item.exchange))
def get_current_price(entity_ids=None, entity_type='coin'):
result = {}
if entity_type == 'coin':
if entity_ids:
for entity_id in entity_ids:
a, exchange, code = decode_entity_id(entity_id)
assert a == entity_type
ccxt_exchange = CCXTAccount.get_ccxt_exchange(exchange_str=exchange)
if not ccxt_exchange:
raise Exception('{} not support'.format(exchange))
orderbook = ccxt_exchange.fetch_order_book(code)
bid = orderbook['bids'][0][0] if len(orderbook['bids']) > 0 else None
ask = orderbook['asks'][0][0] if len(orderbook['asks']) > 0 else None
entity_id = f'coin_{exchange}_{code}'
result[entity_id] = (bid, ask)
return result
def get_current_price(security_list=None, security_type=SecurityType.coin):
result = {}
if security_type == SecurityType.coin:
if security_list:
for security_id in security_list:
a, exchange, code = decode_security_id(security_id)
assert SecurityType(a) == security_type
ccxt_exchange = CCXTAccount.get_ccxt_exchange(
exchange_str=exchange)
if not ccxt_exchange:
raise Exception('{} not support'.format(exchange))
orderbook = ccxt_exchange.fetch_order_book(code)
bid = orderbook['bids'][0][0] if len(
orderbook['bids']) > 0 else None
ask = orderbook['asks'][0][0] if len(
orderbook['asks']) > 0 else None
security_id = f'coin_{exchange}_{code}'
result[security_id] = (bid, ask)
return result
class CoinMetaRecorder(Recorder):
provider = Provider.CCXT
data_schema = Coin
def __init__(self,
batch_size=10,
force_update=False,
sleeping_time=10,
exchanges=COIN_EXCHANGES) -> None:
super().__init__(batch_size, force_update, sleeping_time)
self.ccxt_account = CCXTAccount(exchanges=exchanges)
self.exchanges = COIN_EXCHANGES
def run(self):
for exchange_str in self.exchanges:
exchange = self.ccxt_account.get_ccxt_exchange(exchange_str)
try:
markets = exchange.fetch_markets()
df = pd.DataFrame()
# markets有些为key=symbol的dict,有些为list
markets_type = type(markets)
if markets_type != dict and markets_type != list:
self.logger.exception(
"unknown return markets type {}".format(markets_type))
return
aa = []
for market in markets:
if markets_type == dict:
name = market
code = market
if markets_type == list:
code = market['symbol']
name = market['symbol']
if name not in COIN_PAIRS:
continue
aa.append(market)
security_item = {
'id':
'{}_{}_{}'.format(SecurityType.coin.value,
exchange_str, code),
'exchange':
exchange_str,
'type':
SecurityType.coin.value,
'code':
code,
'name':
name
}
df = df.append(security_item, ignore_index=True)
# 存储该交易所的数字货币列表
if not df.empty:
init_securities(df=df,
security_type=SecurityType.coin,
provider=self.provider)
self.logger.info(
"init_markets for {} success".format(exchange_str))
except Exception as e:
self.logger.exception(
"init_markets for {} failed".format(exchange_str), e)