def fix_exchange_symbols(self, exchange_id, ws_type):
"""
功能:
同步交易所symbol
"""
ex_symbols_key = settings.SYMBOL_STATUS_KEY_MAP[ws_type].format(exchange_id)
symbols_data = REDIS_CON.hgetall(ex_symbols_key)
symbols_map = {x.decode(): ujson.loads(symbols_data[x]) for x in symbols_data}
ex = getattr(ws_crawl, exchange_id)(loop=loop, http_proxy=proxy, ws_proxy=proxy)
delete_symbols = [x for x in symbols_map if x not in ex.symbols]
for s in delete_symbols:
print(f'{exchange_id} {ws_type} 删除{s}')
REDIS_CON.hdel(ex_symbols_key, s)
# 添加 交易所新增的交易对
add_symbols = [x for x in ex.symbols if x not in symbols_map]
add_symbols_map = {
x: ujson.dumps({
redis.TMS_KEY: int(time.time()),
redis.STATUS_KEY: SpiderStatus.pending.value,
redis.PID_KEY: ''})
for x in add_symbols
}
if add_symbols_map:
print(f'{exchange_id} {ws_type} 新增交易对: {add_symbols}')
REDIS_CON.hmset(ex_symbols_key, add_symbols_map)
def save_data_to_db(symbol, exchange_id, data):
"""
功能:
把消息存储到本地
"""
key = settings.EXCHANGE_SYMBOL_TRADE_KEY.format(exchange_id, symbol)
REDIS_CON.rpush(key, *eval(data))
def get_alarm_price_push_percent():
"""
功能:
获取 价格异动币种触发push 幅度
"""
key = settings.PAIR_ALARM_PUSH_PERCENT_KEY
redis_data = redis_conn.hgetall(key)
if redis_data:
return {x.decode(): float(redis_data[x]) for x in redis_data}
else:
redis_conn.hmset(key, settings.PLUNGE_ALARM_PERCENT_MAP)
print('设置触发幅度: ', ujson.dumps(settings.PLUNGE_ALARM_PERCENT_MAP))
return settings.PLUNGE_ALARM_PERCENT_MAP
def get_pass_price_coin():
"""
功能:
获取 整数关口币种缓存
"""
key = settings.PASS_PRICE_COIN_KEY
redis_data = redis_conn.get(key)
if redis_data:
return ujson.loads(redis_data)
else:
redis_conn.set(key, ujson.dumps(settings.PASS_PRICE_COIN_LIST))
print('设置整数关口币种: ', settings.PASS_PRICE_COIN_LIST)
return settings.PASS_PRICE_COIN_LIST
def get_alarm_price_coin():
"""
功能:
获取 价格异动的币种
"""
key = settings.PRICE_ALARM_COIN_KEY
redis_data = redis_conn.get(key)
if redis_data:
return ujson.loads(redis_data)
else:
redis_conn.set(key, ujson.dumps(settings.PRICE_ALARM_COIN_LIST))
print('设置覆盖币种: ', settings.PRICE_ALARM_COIN_LIST)
return settings.PRICE_ALARM_COIN_LIST
def parse_exchange_trade_csv(exchange_id_str=None, symbol_str=None, is_forever=False, ws_type=settings.BASE_WS_TYPE_TRADE):
"""
功能:
生成csv 文件
每个ex 一个task 采取 异步的方式
步骤:
1. 获取 所有 exchange
2. 获取 ex的所有symbol
"""
if not exchange_id_str:
exchange_ids = [x.decode() for x in REDIS_CON.hkeys(settings.EXCHANGE_STATUS_KEY_MAP[ws_type])]
elif ',' in exchange_id_str:
exchange_ids = exchange_id_str.split(',')
else:
exchange_ids = [exchange_id_str]
if not symbol_str:
symbols = None
elif ',' in symbol_str:
symbols = symbol_str.split(',')
else:
symbols = [symbol_str]
for exchange_id in exchange_ids:
loop.run_until_complete(save_exchange_trade_to_csv(exchange_id, symbols, ws_type=ws_type))
if is_forever:
tasks = [loop.create_task(get_pending_exchange_to_csv_forever(ws_type=ws_type)) for x in range(5)]
asyncio.gather(*tasks)
loop.run_forever()
def handle_ws_type_exchanges(self, ws_type):
if not ws_type:
print(f'{ws_type} Error !!!')
return
all_exchange_ids = (x.decode() for x in REDIS_CON.hkeys(S_EXCHANGE_STATUS_KEY_MAP[ws_type]))
for exchange_id in all_exchange_ids:
spider_key = S_SPIDER_STATUS_KEY_MAP[ws_type].format(exchange_id)
spider_data = REDIS_CON.hgetall(spider_key)
for pid in spider_data:
now = int(time.time())
pid_data = ujson.loads(spider_data[pid])
if now - pid_data[TMS_KEY] > AUTO_STOP_TMS:
print(f'{datetime.datetime.now()} 停止 {exchange_id} {pid}')
pid_data[STATUS_KEY] = SpiderStatus.stopped.value
pid_data[TMS_KEY] = now
REDIS_CON.hset(spider_key, pid, ujson.dumps(pid_data))
print(f'{datetime.datetime.now()} 完成一次 {ws_type} 监控!')
async def check_push_2_php(base_name, quote_name, exchange_id) -> bool:
"""
功能:
判断 是否 push 到 php 价格异动
"""
exchange_symbol = f'{base_name}{quote_name}:{exchange_id}'
r_data = REDIS_CON.hget(settings.COIN_PUSH_FROM_PAIR_KEY, base_name)
if r_data and r_data.decode() == exchange_symbol:
return True
return False
async def start(self, exchange_id, symbol, limit):
kline_list = self.get_ohlcv(exchange_id, symbol, limit)
print(len(kline_list))
base, quote = symbol.split('_')
redis_conn.hset(settings.COIN_PUSH_FROM_PAIR_KEY, base,
f'{symbol.replace("_", "")}:{exchange_id}')
routing_key = f"market.kline.{exchange_id}.{base}.{quote}.spot.0.0"
alarm = PairAlarm(loop=loop, is_test=True)
for kline in kline_list:
kline = await self.format_ohlcv(kline)
data = {
'c': 'kline', # channel
'e': exchange_id, # 交易所id
't': kline[0], # 时间戳
's': symbol.replace('_', ''), # 交易对
'd': kline
}
await alarm.pair_alarm_msg_process_func(routing_key=routing_key,
data=ujson.dumps(data))
def get_pair_price_source():
"""
功能:
设置 币种交易对源
"""
key = settings.PAIR_PRICE_SOURCE_CACHE_KEY
redis_data = redis_conn.hgetall(key)
if redis_data:
return {x.decode(): ujson.loads(redis_data[x]) for x in redis_data}
else:
raise BaseException('未设置 币种价格预警数据源!!!')
async def get_pair_ticker():
"""
功能:
获取一个交易对的ticker 信息, 包括: 当前价格, 今日涨跌幅
"""
ret_data = {}
for platform in PLATFORM_CACHE.values():
for exchange_id in platform.alarm_info:
redis_key = settings.EXCHANGE_LAST_OHLCV_KEY.format(exchange_id)
data_list = []
pg = OHLCVPg(database=exchange_id)
await pg.init
today_tms = int(
time.mktime(datetime.datetime.now().date().timetuple()))
noon_broadcast_pairs = platform.alarm_info[exchange_id][
'noon_broadcast_pairs']
for symbol in noon_broadcast_pairs:
tab_name = f't_{symbol}_1min'
now = await pg.now_tms
sql = f'select tms, open from {tab_name} where tms <= {now} and tms >= {today_tms} order by tms limit 1;'
try:
rows = await pg.pool.fetchrow(sql)
except Exception as e:
await async_logger.error(e)
continue
last_ohlcv = REDIS_CON.hget(redis_key, f'{symbol}_m1')
if last_ohlcv:
last_kline = ujson.loads(last_ohlcv)
now_price = last_kline[4]
today_open_price = float(rows[1])
pct = (now_price -
today_open_price) / today_open_price * 100
is_rise = True if pct > 0 else False
pct = round(abs(pct), 2)
now_price = now_price if len(
f'{now_price}'.split('.')[1]) <= 4 else round(
now_price, 4)
data_list.append({
'pair_name': noon_broadcast_pairs[symbol],
'is_rise': is_rise,
'pct': pct,
'now_price': now_price,
})
await pg.pool.close()
ret_data[exchange_id] = data_list
await platform.push_message_to_platform(
PushType.noon_broadcast.value, '', exchange_id, data_list)
return ret_data
def get_spider_host_map():
"""
功能:
获取 spider 服务器的映射, 从缓存读取, 如果异常 方便切换
return:
spider_host_map = {
'spider1': '47.240.24.174',
'spider2': '149.129.90.193',
'spider3': '47.240.28.97'
}
"""
redis_data = redis_conn.hgetall(settings.SPIDER_HOST_MAP_KEY)
if redis_data:
spider_host_map = {
x.decode(): redis_data[x].decode()
for x in redis_data
}
return spider_host_map
else:
return settings.SPIDER_HOST_MAP
def check_kline_is_update(self, symbol_exchange):
"""
功能:
通过缓存 查询数据是否在更新
"""
symbol, exchange_id = symbol_exchange.split(':')
try:
last_ohlcv = REDIS_CON.hget(
settings.EXCHANGE_LAST_OHLCV_KEY.format(exchange_id),
f'{symbol}_{TimeFrame("1min").name}')
if last_ohlcv:
data = ujson.loads(last_ohlcv)
last_tms = data[0]
if int(time.time()) - last_tms >= S_CHANGE_PAIR_PRICE_TMS:
return False
else:
return False
except:
...
return True
def main(self):
for base_name in S_PAIR_PRICE_SOURCE_CACHE:
print(f'Start {base_name} {datetime.datetime.now()}...')
sorted_pair_list = [
x[0] for x in sorted(S_PAIR_PRICE_SOURCE_CACHE[base_name],
key=lambda x: x[1],
reverse=True)
]
base_name = base_name.lower()
first_symbol_exchange = sorted_pair_list[0]
key = settings.COIN_PUSH_FROM_PAIR_KEY
symbol_exchange_data = REDIS_CON.hget(key, base_name)
if symbol_exchange_data:
symbol_exchange = symbol_exchange_data.decode()
now_source = symbol_exchange
if not self.check_kline_is_update(symbol_exchange):
print(
f'{datetime.datetime.now()} {base_name} now_source: {symbol_exchange} disconnect!'
)
for new_symbol_exchange in sorted_pair_list:
if self.check_kline_is_update(new_symbol_exchange):
print(
f'{datetime.datetime.now()} change_source -> {new_symbol_exchange}'
)
change_source = new_symbol_exchange
REDIS_CON.hset(key, base_name, new_symbol_exchange)
break
else:
print(
f'{datetime.datetime.now()} {base_name} all source is disconnect, change -> base source {first_symbol_exchange}!!!'
)
REDIS_CON.hset(key, base_name, first_symbol_exchange)
change_source = first_symbol_exchange
else:
print(
f'{datetime.datetime.now()} {base_name} now_source: {symbol_exchange} OK!'
)
change_source = symbol_exchange
if symbol_exchange != first_symbol_exchange:
if self.check_kline_is_update(first_symbol_exchange):
print(
f'{datetime.datetime.now()} {base_name} change_to_base: {symbol_exchange} >> {first_symbol_exchange}'
)
change_source = first_symbol_exchange
REDIS_CON.hset(key, base_name,
first_symbol_exchange)
else:
print(
f'{datetime.datetime.now()} {base_name} cache {base_name.upper()} has no source!!!'
)
print(
f'{datetime.datetime.now()} {base_name} init source: {first_symbol_exchange}'
)
REDIS_CON.hset(key, base_name, first_symbol_exchange)
now_source = change_source = first_symbol_exchange
if base_name in ['btc', 'eth']:
# btc eth 分析宝 通知PHP
if now_source != change_source:
symbol, exchange_id = change_source.split(':')
pair_data = funcs.get_ret_from_thread_loop(
get_exchange_pair_data(exchange_id,
base_name,
quote_name='usdt'))
pair_id = pair_data['pair_id']
pair_id_list = [
funcs.get_ret_from_thread_loop(
get_exchange_pair_data(
x.split(':')[1], base_name,
quote_name='usdt'))['pair_id'] for x in [
source[0]
for source in S_PAIR_PRICE_SOURCE_CACHE[
base_name.upper()]
]
]
data = {'pair_id': pair_id, 'pair_id_list': pair_id_list}
send_notice_to_php(settings.PHP_NOTICE_METHOD['fenxibao'],
data=data)
print(now_source, change_source, data)