def __init__(self): self.logger = get_logger(__name__) self.push_logger = get_logger(settings.APP_NAME) self.platform_name = '' self.http = HttpBase() self._redis = AsyncRedisBase() self._cur = None self.token = '' self.test_api = '' self.prod_api = '' if settings.APP_TYPE == 'WWW': self.push_url = f'{self.prod_api}/api/newsflash-push' else: self.push_url = f'{self.test_api}/api/newsflash-push' self.http.headers = { 'Content-Type': "application/x-www-form-urlencoded", } self.push_type = { PushType.rise_or_fall.value: 1, PushType.pass_price.value: 2, PushType.noon_broadcast.value: 3 } # 请求参数 self.push_params = { 'token': self.token, 'push_type': '', 'push_content': {}, } # 配置信息 self.alarm_info = { 'huobipro': { 'name': '火币', 'alarm_coin': ['btc', 'eth'], 'alarm_price_pass': {'btc': 500, 'eth': 100}, 'price_pass_cache': {}, # 同一价格 10分钟内只push 一次 'pass_cache_ex_tms': 10 * 60, # 整数关口的过期时间: 10分钟 'noon_broadcast_pairs': { 'btcusdt': 'BTC/USDT', 'ethusdt': 'ETH/USDT', 'xrpusdt': 'XRP/USDT', 'eosusdt': 'EOS/USDT', 'bchusdt': 'BCH/USDT', 'ltcusdt': 'LTC/USDT', 'bsvusdt': 'BSV/USDT', }, }, }
# -*- coding:utf-8 -*- import os import asyncio import logging from django.conf import settings from exspider.utils.redis_con import StoreKeeperAsyncRedis from exspider.utils.mysql_con import TradeMySQL from exspider.utils.logger_con import get_logger logger = get_logger(__name__) class ImportLocalTrades: def __init__(self, local_dir, loop, exchange_id, symbol, is_forever=False): self._local_dir = local_dir self._loop = loop self._exchange_id = exchange_id self._symbol = symbol self._trades_pg_dict = {} # exchange -> pg self._store_redis = StoreKeeperAsyncRedis(loop) self._exchange_symbol_trade_need_update = {} self.is_forever = is_forever # 持续插入 还是 只是插入一轮 self.one_time_read_count = 1000 # 每次读取的条数 入库 async def start(self): tasks = [] # task 按照交易所建立, 一个交易所一个task # /data1/trades/hk1/binance/btcusdt/csv/1557007828.csv local_dir = self._local_dir if not os.path.exists(local_dir):
#! /usr/bin/python # -*- coding:utf-8 -*- # @zhuchen : 2019-03-04 17:59 from django.core.management import BaseCommand from spider.tasks import start_spider, stop_spider from django.conf import settings from exspider.utils.logger_con import get_logger logger = get_logger(__name__, is_debug=settings.IS_DEBUG) class Command(BaseCommand): """ CMD: python manage.py set_spider huobipro btcusdt kline start dev 说明: 一次 设置多个交易所不能指定交易对, 所有交易对都会被设置 多个交易所, 交易对 用, 隔开 """ def add_arguments(self, parser): parser.add_argument('exchange_id', type=str) parser.add_argument('symbol', type=str) parser.add_argument('ws_type', type=str) parser.add_argument('spider_type', type=str) def handle(self, *args, **options): spider_map = {'start': start_spider, 'stop': stop_spider} exchange_id = options['exchange_id'] symbol = options['symbol'] ws_type = options['ws_type']
import asyncio import datetime import time import ujson from django.conf import settings from exspider.utils.socket_server import send_notice_to_php from exspider.utils.pg_con import OHLCVPg from exspider.utils.redis_con import REDIS_CON from exspider.utils.logger_con import get_logger from exspider.utils.enum_con import PushType from exspider.common import funcs from storekeeper import platform as platform_model async_logger = get_logger(__name__, is_debug=settings.IS_DEBUG) push_logger = get_logger(settings.APP_NAME) S_PASS_PRICE_COIN_LIST = funcs.get_pass_price_coin() S_PRICE_ALARM_COIN_LIST = funcs.get_alarm_price_coin() S_PRICE_PASS_BASE_INTEGER = settings.PRICE_PASS_BASE_INTEGER # 所有第三方 缓存 PLATFORM_CACHE = { x: getattr(platform_model, x)() for x in platform_model.platforms } # 整数关口缓存 PHP_PASS_PRICE_CACH_MAP = { # 'BTC': [13000]
#! /usr/bin/python # -*- coding:utf-8 -*- # @zhuchen : 2019-05-16 16:02 import asyncio import time import aiohttp import ujson from exspider.utils.logger_con import get_logger logger = get_logger(__name__, is_debug=True) class HttpBase: def __init__(self, loop=None, proxy=None, timeout=5): self.loop = loop if loop else asyncio.get_event_loop() self.headers = {} self.proxy = proxy self.timeout = timeout async def get_http_response(self, url, request_method='GET', is_json=True, **kwargs): """ 功能: 请求RESTFul 接口获取数据 """ data = None if 'headers' in kwargs:
import asyncio import time import datetime import hashlib import ujson from django.core.management import BaseCommand from django.conf import settings from spider import ws_crawl from spider.tasks import save_data_2_csv from exspider.utils.logger_con import get_logger from exspider.utils.redis_con import AsyncRedis logger = get_logger() loop = asyncio.get_event_loop() BURST_LAST_SAVE_KEY = 'hash:burst_last_save_cache' class Command(BaseCommand): """ CMD: python manage.py fetch_future_burst okex_future 功能: 循环抓取 爆仓记录 """ redis = AsyncRedis(loop=loop) def add_arguments(self, parser):