def __init__(self):
self.r = ConnectRedis()
self.hb = huobipro()
# 获取交易界面上的交易对，
# 接口返回信息
self.response_symbols = None
# 交易所目前的市场 3/2
self.markets = None
# 交易所支持的市场名称 ETH/BTC
self.marketNames = None
# 市场id与市场name映射s
self.marketId_ccxtsymbol_mapping = None
self.getMarketInfos()
print(self.marketNames)
def __init__(self):
# 从coinbase获取法币的价格
self.r = ConnectRedis()
self.pool = multiprocessing.dummy.Pool(8)
self.bt = bytetrade()
# 获取交易界面上的交易对，
# 接口返回信息
self.response_symbols = None
# 交易所目前的市场 3/2
self.markets = None
# 交易所支持的市场名称 ETH/BTC
self.marketNames = None
# 市场id与市场name映射
self.marketId_ccxtsymbol_mapping = None
self.getMarketInfos()
self.dataReady = False
self.refExchanges = []
self.tarExchange = TARGETEXCHANGE
for v in PAIRS.values():
if v['mode'] == 'refDirect':
if v['exchange'] not in self.refExchanges:
self.refExchanges.append(v['exchange'])
if v['mode'] == 'refDouble':
if v['basePair']['exchange'] not in self.refExchanges:
self.refExchanges.append(v['basePair']['exchange'])
if v['quotePair']['exchange'] not in self.refExchanges:
self.refExchanges.append(v['quotePair']['exchange'])
super(Quote, self).__init__()
# 自动生成的价格
self.generatedPrice = {}
# 币对参考价
self.refPrice = {}
# 币对最后价格
self.lastPrice = {}
# 最后成交数据
self.lastDeal = {}
# 最后ticker数据
self.lastTicker = {}
# 最后深度数据
self.lastDepth = {}
self.marketData = {}
self.tickerSymbols = {}
self.tickLock = threading.Lock()
self.dealLock = threading.Lock()
self.depthLock = threading.Lock()
class Quote(object):
def __init__(self):
# 从coinbase获取法币的价格
self.r = ConnectRedis()
self.coinbase_api = COIN_BASE_URL + "v2/assets/summary"
self.pool = Pool(20)
# @retry(10, 3)
def sendRequest(self, base):
"""
构建请求并发送
:param base: CNY USD
:return:
"""
params = {
# "include_prices": "false",
"base": base,
"resolution": "hour"
}
response = requests.get(url=self.coinbase_api,
params=params).content.decode("utf-8")
return response
def updateQuote(self, base):
# 所有的法币名称
response = eval(self.sendRequest(base))
for symbol in response["data"]:
if symbol["base"] in COIN_CURRENCY:
k = symbol["base"] + "/" + symbol["currency"]
price = symbol["latest"]
# 将value写入到redis中
print(k, price)
self.r.hset("coinbase_currency_price1", k, price)
def start(self):
"""
维持quote字典最新
:return:
"""
self.r.set("Receive_the_data_coinbase1", time.time())
logger.info("更新法币价格")
for base in moneyLst:
# 所有的法币名称
self.pool.apply_async(self.updateQuote, (base, ))
def resolve_symbols(self, info, symbol_name=None, currency=None, **kwargs):
key_bytetrade = "price_server_bytetrade1"
key_huobipro = "price_server_huobipro1"
key_coin_base = "coinbase_currency_price1"
# key_path_price = "price_server_path_price1"
r = ConnectRedis()
bytetrade_price = r.hgetall(key_bytetrade)
huobi_price = r.hgetall(key_huobipro)
coinbase_price = r.hgetall(key_coin_base)
# path_price = r.hgetall(key_path_price)
calprice = CalPrice(bytetrade_price, huobi_price, coinbase_price)
if currency:
# 传来的法币是一个字符串 用,分割
temp = currency.split(",")
currency_list = [i.replace(" ", "") for i in temp]
else:
# 没传就返回所有的法币价格
currency_list = CURRENCY_LIST
if symbol_name:
# 如果传了市场的列表 分割
temp = symbol_name.split(",")
symbol_list = [i.replace(" ", "") for i in temp]
else:
symbol_list = SYMBOL_LIST
# symbol_list = MARKET_LIST
# all 就是我们交易所目前支持的所有的币对
# 返回所有
t1 = time.time()
res = [
create_symbol_obj(i, currency_list, calprice) for i in symbol_list
]
print(time.time() - t1)
return res
class Quote(object):
def __init__(self):
self.r = ConnectRedis()
self.hb = huobipro()
# 获取交易界面上的交易对，
# 接口返回信息
self.response_symbols = None
# 交易所目前的市场 3/2
self.markets = None
# 交易所支持的市场名称 ETH/BTC
self.marketNames = None
# 市场id与市场name映射s
self.marketId_ccxtsymbol_mapping = None
self.getMarketInfos()
print(self.marketNames)
def get_huobipro_symbols(self):
logger.info("获取火币交易对")
url = HUOBIPRO_API + "v1/common/symbols"
res = eval(requests.get(url).content.decode("utf-8"))
huobi_symbols = [
i["base-currency"].upper() + "/" + i["quote-currency"].upper()
for i in res["data"]
]
return huobi_symbols
def onDeal_huobipro(self, symbol, data):
print(symbol)
print(data)
# 将收到的symbol计算成 ccxtsymbol
self.r.set("Receive_the_data_huobi1", time.time())
if MARKET_NAME_NUM_MAPPING[symbol] in PAIRS.keys() and PAIRS[
MARKET_NAME_NUM_MAPPING[symbol]]["mode"] == "refSelf":
pass
else:
self.r.hset("next_price", MARKET_NAME_NUM_MAPPING[symbol],
data[0]["info"]["price"])
# self.r.publish("price_server_" + "huobipro_" + str(symbol), data[0]["info"]["price"])
self.r.hset("price_server_huobipro1", symbol, data[0]["info"]["price"])
def subscribeAllTicker(self):
"""
订阅所有的交易对的websocket ticker
:return:
"""
# 订阅我们有的火币所有的交易对
self.hb.start()
huobipro_symbols = set(self.get_huobipro_symbols())
# 我们有并且火币也有的交易对
bytetrade_symbols = set(
[i for i in self.marketNames if "BTT" not in i])
common_symbols = huobipro_symbols & bytetrade_symbols
common_symbols.update(["ETH/USDT", "BTC/USDT"])
for stock in common_symbols:
self.hb.subscribeDeals(stock, self.onDeal_huobipro)
logger.info("订阅火币各个交易对成交价格")
def getMarketInfos(self):
# 获取交易所正在进行的市场
logger.info("正在获取目前交易所支持的 Market，MarketName，marketId与ccxtSymbol映射等信息")
url = BYTETRADE_API + "?cmd=marketsPrice&channel=all"
res = eval(requests.get(url).content.decode("utf-8"))
markets = [
str(i["stockId"]) + "/" + str(i["moneyId"]) for i in res["result"]
] # "3/2"
marketNames = [i["name"] for i in res["result"]] # "CMT/KCASH"
res_symbols = res["result"]
coinId_ccxtsymbol_mapping = {
str(i["id"]): i["name"]
for i in res["result"]
}
# 接口返回信息
self.response_symbols = res_symbols
# 交易所目前的市场 3/2
self.markets = markets
# 交易所支持的市场名称
self.marketNames = marketNames
# 市场id与市场name映射
self.marketId_ccxtsymbol_mapping = coinId_ccxtsymbol_mapping
def get_price_by_rest(self):
huobipro_symbols = set(self.get_huobipro_symbols())
# 我们有并且火币也有的交易对
bytetrade_symbols = set(
[i for i in self.marketNames if "BTT" not in i])
common_symbols = huobipro_symbols & bytetrade_symbols
huobipro = ccxt.huobipro()
res = huobipro.fetch_tickers()
for h_symbol, v in res.items():
if h_symbol in common_symbols or h_symbol in [
"ETH/USDT", "BTC/USDT"
]:
self.r.hset("price_server_huobipro1", h_symbol, v["close"])
self.r.set("Receive_the_data_huobi1", time.time())
# 开始的时候将原来的键删掉，构建新的 一旦加了新的交易对，重启程序
def push_bear():
PUSH_BEAR_KEY = "11970-ba5f3d1644a4bd880a04ebdef3560f69"
import requests
url = "https://pushbear.ftqq.com/sub"
data = {
"sendkey": PUSH_BEAR_KEY,
"text": "PriceServer——GraphQL",
"desp": "huobipro数据获取重启"
}
headers = {
"Accept":
"application/json, text/javascript, */*; q=0.01",
"Accept-Encoding":
"gzip, deflate, br",
"User-Agent":
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0"
}
requests.post(url, data=data, headers=headers)
# push_bear()
r = ConnectRedis()
# r.delete("price_server_huobipro")
# 用来维护兑换法币的redis hash
q = Quote()
q.get_price_by_rest()
#
q.subscribeAllTicker() # 维护各个marketId的实时价格
def __init__(self):
# 从coinbase获取法币的价格
self.r = ConnectRedis()
self.coinbase_api = COIN_BASE_URL + "v2/assets/summary"
self.pool = Pool(20)
class Quote(object):
def __init__(self):
# 从coinbase获取法币的价格
self.r = ConnectRedis()
self.pool = multiprocessing.dummy.Pool(8)
self.bt = bytetrade()
# 获取交易界面上的交易对，
# 接口返回信息
self.response_symbols = None
# 交易所目前的市场 3/2
self.markets = None
# 交易所支持的市场名称 ETH/BTC
self.marketNames = None
# 市场id与市场name映射
self.marketId_ccxtsymbol_mapping = None
self.getMarketInfos()
self.dataReady = False
self.refExchanges = []
self.tarExchange = TARGETEXCHANGE
for v in PAIRS.values():
if v['mode'] == 'refDirect':
if v['exchange'] not in self.refExchanges:
self.refExchanges.append(v['exchange'])
if v['mode'] == 'refDouble':
if v['basePair']['exchange'] not in self.refExchanges:
self.refExchanges.append(v['basePair']['exchange'])
if v['quotePair']['exchange'] not in self.refExchanges:
self.refExchanges.append(v['quotePair']['exchange'])
super(Quote, self).__init__()
# 自动生成的价格
self.generatedPrice = {}
# 币对参考价
self.refPrice = {}
# 币对最后价格
self.lastPrice = {}
# 最后成交数据
self.lastDeal = {}
# 最后ticker数据
self.lastTicker = {}
# 最后深度数据
self.lastDepth = {}
self.marketData = {}
self.tickerSymbols = {}
self.tickLock = threading.Lock()
self.dealLock = threading.Lock()
self.depthLock = threading.Lock()
def start_price_manager(self):
timerThread = threading.Thread(target=self.runTimer)
timerThread.start()
def runTimer(self):
lastCheckRef = {}
while True:
try:
for pair in PAIRS:
interval = Activity[int(PAIRS[pair]['activity']) - 1]
if not lastCheckRef.get(pair) or time.time() - lastCheckRef[pair] >= interval:
lastCheckRef[pair] = time.time()
if PAIRS[pair]['mode'] == 'refSelf' and self.generatedPrice.get(pair):
self.refPrice[pair] = self.generatedPrice[pair].pop(0)
self.r.hset("next_price", pair, self.generatedPrice[pair].pop(0))
log.info('on time pop ref self price %s, left %s' % (
self.refPrice[pair], len(self.generatedPrice[pair])))
except Exception as e:
log.error(e)
time.sleep(1)
# 检查自报价
def __checkSelfRefPrice(self, symbol, lastPrice):
iniPrice = PAIRS[symbol]['iniPrice']
if not lastPrice:
lastPrice = iniPrice
priceRange = PAIRS[symbol]['priceRange']
redisSeed = self.r.get(f'seedPrice.{symbol}')
print(f"最后成交价格: {lastPrice}")
print(f"初始价格：{iniPrice}")
print(f"价格区间：{priceRange}")
print(f"redis价格：{redisSeed}")
print(f"参考价格：{self.refPrice}")
print(f"generatedPrice: {self.generatedPrice}")
seeds = {}
if redisSeed: seeds = eval(redisSeed)
if (self.refPrice.get(symbol) and lastPrice != iniPrice and
abs(lastPrice - self.refPrice[symbol]) / lastPrice > priceRange * 0.2
and abs(lastPrice - self.refPrice[symbol]) / lastPrice > 0.03 and self.generatedPrice.get(symbol)):
log.info('current price %s is so far from ref price %s, regenerate ref price to fit it.'
% (lastPrice, self.refPrice[symbol]))
self.generatedPrice[symbol] = None
seeds[symbol] = None
# 重新生成seedPrice，并存储
if not seeds.get(symbol):
seeds[symbol] = lastPrice
log.info('regenerate seedPrice: %s' % seeds[symbol])
self.r.set(f'seedPrice.{symbol}', f'{seeds}')
# generatePrice为空，生成
if not self.generatedPrice.get(symbol):
self.generatedPrice[symbol] = self.__generateRefPrice(lastPrice, seeds[symbol], priceRange)
log.info('generate ref price by seed %s, lastPrice %s, priceRange %s, data %s'
% (seeds[symbol], lastPrice, priceRange, len(self.generatedPrice[symbol])))
self.refPrice[symbol] = self.generatedPrice[symbol].pop(0)
# random_pop = numpy.random.uniform(0,1)
# random_fluc = numpy.random.uniform(-0.0005,0.0005) #config.settings['pairs'][symbol]['priceRange']*numpy.random.uniform(-0.1,0.1)
# if random_pop>numpy.random.uniform(0.5, 0.9) or not self.refPrice.get(symbol):
# self.refPrice[symbol] = self.generatedPrice[symbol].pop(0)
# log.info('pop last ref price %s'%self.refPrice[symbol])
# else:
# # self.refPrice[symbol] = self.refPrice[symbol]*(1+random_fluc)
# log.info('random last ref price %s'%self.refPrice[symbol])
def __generateRefPrice(self, curPrice, seedPrice, priceRange):
priceMin = min(curPrice, seedPrice / 1.05 * (1 + numpy.random.uniform(-priceRange * 0.1, priceRange * 0.4)))
priceMax = max(curPrice, seedPrice * 1.05 * (1 + numpy.random.uniform(-priceRange * 0.4, priceRange * 0.1)))
data_len = numpy.random.randint(10000, 30000)
# assert curPrice>=priceMin and curPrice<=priceMax,f"error: {curPrice}, {priceMin}, {priceMax}"
def smooth_data(data):
x = numpy.arange(0, len(data), 1)
x_new = numpy.arange(0, max(x), 0.01)
func = interpolate.interp1d(x, data, kind='quadratic')
smoothed = func(x_new)
return smoothed
while True:
dataset = random_walks(n_ts=1, sz=data_len * 2)
scaler = TimeSeriesScalerMinMax(min=float(priceMin), max=float(priceMax))
dataset_scaled = scaler.fit_transform(dataset)[0, :, 0]
for i in range(0, data_len):
if abs(dataset_scaled[i] - curPrice) / curPrice < 0.001:
# return list(smooth_data(dataset_scaled[i:i+data_len]))
with open('price.txt', 'w+') as f:
f.writelines([f'{p}\n' for p in dataset_scaled[i:i + data_len]])
return list(dataset_scaled[i:i + data_len])
def cal_market_id(self, symbol):
symbolPair = symbol.split('/')
return int(symbolPair[1]) * 2147483647 + int(symbolPair[0])
def cal_ccxt_symbol(self, market_id):
return self.marketId_ccxtsymbol_mapping[str(market_id)]
def onDeal_bytetrade(self, symbol, data):
"""
subscribe的回调函数 将data写入到redis中
:return:
"""
# with self.dealLock:
# self.saveDeals(exchange, symbol, data)
# for deal in data:
# self.lastPrice[exchange][symbol]=float(deal['price'])
# exchange = "bytetrade"
print(f"symbol: {symbol} onDeal{data}")
if symbol in PAIRS.keys():
print(symbol)
print("=" * 100)
# self.saveDeals(exchange, symbol, dealData)
self.__checkSelfRefPrice(symbol, float(data[0]["info"]["price"]))
self.r.set("Receive_the_data_bytetrade1", time.time())
s = self.cal_market_id(symbol)
ccxt_symbol = self.cal_ccxt_symbol(s)
# 将收到的symbol计算成 ccxtsymbol
# self.r.publish("price_server_" + "bytetrade_" + ccxt_symbol, data["last"])
self.r.hset("price_server_bytetrade1", ccxt_symbol, float(data[0]["info"]["price"]))
self.r.set("Receive_the_data_bytetrade1", time.time())
def onTicker_bytetrade(self, symbol, data):
"""
subscribe的回调函数 将data写入到redis中
:return:
"""
print(f"symbol: {symbol} onTicker{data}")
s = self.cal_market_id(symbol)
ccxt_symbol = self.cal_ccxt_symbol(s)
# 将收到的symbol计算成 ccxtsymbol
if symbol in ["48/2"]:
pass
else:
# self.r.publish("price_server_" + "bytetrade_" + ccxt_symbol, data["last"])
self.r.hset("price_server_bytetrade_today1", ccxt_symbol, str(data["info"]))
self.r.set("Receive_the_data_bytetrade1", time.time())
def subscribeAllDeal(self):
"""
订阅所有的交易对的websocket ticker
:return:
"""
self.bt.start()
self.bt.subscribeDeals(self.markets, self.onDeal_bytetrade)
log.info("订阅bytetrade各个交易对最近成交")
def subscribeAllTicker(self):
self.bt.start()
self.bt.subscribeTicker(self.markets, self.onTicker_bytetrade)
log.info("订阅bytetrade各个交易对today")
def getMarketInfos(self):
# 获取交易所正在进行的市场
log.info("正在获取目前交易所支持的 Market，MarketName，marketId与ccxtSymbol映射等信息")
url = BYTETRADE_API + "?cmd=marketsPrice&channel=all"
res = eval(requests.get(url).content.decode("utf-8"))
markets = [str(i["stockId"]) + "/" + str(i["moneyId"]) for i in res["result"] if i["moneyId"] != 1] # "3/2"
marketNames = [i["name"] for i in res["result"] if i["moneyId"] != 1] # "CMT/KCASH"
res_symbols = res["result"]
coinId_ccxtsymbol_mapping = {str(i["id"]): i["name"] for i in res["result"]}
# 接口返回信息
self.response_symbols = res_symbols
# 交易所目前的市场 3/2 除了／1 btt
self.markets = markets
# 交易所支持的市场名称
self.marketNames = marketNames
# 市场id与市场name映射
self.marketId_ccxtsymbol_mapping = coinId_ccxtsymbol_mapping
def get_price_by_rest(self):
# restful查一下最新的成交价格
for info in self.response_symbols:
ccxt_symbol = info["name"]
if info["stockId"] == 35:
pass
if info["moneyId"] == 1:
print("=")
else:
print(ccxt_symbol)
print(info)
try:
self.r.hset("price_server_bytetrade1", ccxt_symbol, info["today"]["last"])
except:
pass
def __init__(self, bytetrade_price, huobi_price, coinbase_price):
# 初始化图
self.r = ConnectRedis()
self.bytetrade_price = bytetrade_price
self.huobi_price = huobi_price
self.coinbase_price = coinbase_price
class CalPrice(object):
def __init__(self, bytetrade_price, huobi_price, coinbase_price):
# 初始化图
self.r = ConnectRedis()
self.bytetrade_price = bytetrade_price
self.huobi_price = huobi_price
self.coinbase_price = coinbase_price
def get_symbols(self):
"""
获取redis中保存的交易所的所有币对
:param exchange_name: 交易所名称，huobipro， bytetrade
:return: 交易所支持的币对的列表
"""
redis_key_huobi = "price_server_huobipro1"
redis_key_bytetrade = "price_server_bytetrade1"
symbols_huibo = list(self.r.hgetall(redis_key_huobi).keys())
symbols_bytetrade = list(self.r.hgetall(redis_key_bytetrade).keys())
res = list(self.r.hgetall("coinbase_currency_price1"))
symbols = list(set(symbols_huibo + symbols_bytetrade + res))
return symbols
def cal_price(self, path):
# 拼接成交易对
# print(path)
if path:
symbols = []
for i in range(len(path)):
if i + 1 == len(path):
pass
else:
symbol = path[i] + "/" + path[i + 1]
symbols.append(symbol)
dic = {}
def get_price(symbol):
# price_bytetrade = self.r.hget(key_bytetrade, symbol)
# price_huobipro = self.r.hget(key_huobipro, symbol)
# price_coinbase = self.r.hget(key_coin_base, symbol)
price_bytetrade = self.bytetrade_price.get(symbol, 0)
price_huobipro = self.huobi_price.get(symbol, 0)
price_coinbase = self.coinbase_price.get(symbol, 0)
if price_bytetrade and price_bytetrade != 0:
price = float(price_bytetrade)
else:
if price_huobipro and price_huobipro != 0:
price = float(price_huobipro)
else:
if price_coinbase and price_coinbase != 0:
price = float(price_coinbase)
else:
return 0
return price
for symbol in symbols:
# 币币价格
price = get_price(symbol)
if price:
dic[symbol] = price
else:
t1, t2 = symbol.split("/")
reverse_symbol = t2 + "/" + t1
price = get_price(reverse_symbol)
if price:
if float(price) == 0:
dic[symbol] = 0
else:
dic[symbol] = 1 / float(price)
else:
dic[symbol] = 0
res = 1
for i in dic.values():
res *= float(i)
else:
# logger.info("交易对没有路径，返回价格为0")
res = 0
return res
def search(self, symbols, start, end):
temp = []
node_list = []
for i in symbols:
stock = i.split("/")[0]
money = i.split("/")[1]
temp.append(stock)
temp.append(money)
t = (stock, money, 1)
node_list.append(t)
node = list(set(temp))
## node_map[i][j] 存储i到j的最短距离
node_map = [[INF_val for val in range(len(node))] for val in range(len(node))]
## path_map[i][j]=j 表示i到j的最短路径是经过顶点j
path_map = [[0 for val in range(len(node))] for val in range(len(node))]
## set node_map
set_node_map(node_map, node, node_list, path_map)
## select one node to obj node, e.g. A --> D(node[0] --> node[3])
from_node = node.index(start)
to_node = node.index(end)
Floydpath = Floyd_Path(node, node_map, path_map)
path = Floydpath(from_node, to_node)
return path
def calculate_price(self, start, end, mid=None):
# 从缓存中获取路径
try:
if mid:
key = start + "_" + mid + "_" + end
path = self.r.hget("price_server_path1", key)
else:
key = start + "_" + end
path = self.r.hget("price_server_path1", key)
if path:
path = eval(path)
# 从缓存中获取价格
# if self.path_price:
# try:
#
# price = float(self.path_price[str(path)])
# return price
# except:
#
# if mid:
# path = self.search(self.get_symbols(), mid, end)
# path = [start] + path
# else:
# path = self.search(self.get_symbols(), start, end)
price = self.cal_price(path)
# self.r.hset("price_server_path_price1", str(path), float(price))
# self.r.set("price_server_path_price_alive1", time.time())
return price
# else:
# price = self.cal_price(path)
# self.r.hset("price_server_path_price1", str(path), float(price))
# self.r.set("price_server_path_price_alive1", time.time())
# return price
else:
# 缓存中没有，就计算路径并加入缓存
if mid:
path = self.search(self.get_symbols(), mid, end)
path = [start] + path
else:
path = self.search(self.get_symbols(), start, end)
self.r.hset("price_server_path1", key, str(path))
price = self.cal_price(path)
# self.r.hset("price_server_path_price1", str(path), price)
# self.r.set("price_server_path_price_alive1", time.time())
return price
#
except:
logger.info(f"{start, end, mid}找不到这个路径")
return 0
# -*- coding: utf-8 -*-
# Author: zhangchao
# Date: 2019-07-18
# Desc: 监控获取价格是否正常，不正常，就重启
import datetime
import sys
sys.path.append("..")
from priceserver.common.db_connection import ConnectRedis
import os
import time
r = ConnectRedis()
while True:
now = time.time()
bytetrade_data = float(r.get("Receive_the_data_bytetrade1"))
huobi_data = float(r.get("Receive_the_data_huobi1"))
coinbase_data = float(r.get("Receive_the_data_coinbase1"))
bytetrade = now - bytetrade_data
huobi = now - huobi_data
coinbase = now - coinbase_data
if bytetrade > 60 * 5:
cmd_str = 'pm2 restart ' + "sub_bytetrade_price_test"
result = os.system(cmd_str)
print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
print("重启bytetrade")