def do_crawl_exchange(config_file, log_file, exchange_adaptee,
kline_data_collection, market_depth_collection,
market_trade_collection, table_name_dict):
Configuration.get_configuration(config_file)
logger = Logger.get_logger(log_file)
logger.debug("[%s]Log path: %s" % (os.getpid(), Logger.get_log_file()))
try:
logger.debug("==== do_crawl ===== %s" %
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
configure_logging(install_root_handler=False)
huobi_adaptee = exchange_adaptee()
pixiu_adapter = PixiuAdapter(huobi_adaptee)
ExchangesSpider.pixiu_adapter = pixiu_adapter
process = CrawlerProcess(get_project_settings())
process.crawl(ExchangesSpider)
process.start()
for item in pixiu_adapter.kline_data_collection:
kline_data_collection.append(item)
for item in pixiu_adapter.market_depth_collection:
market_depth_collection.append(item)
for item in pixiu_adapter.market_trade_collection:
market_trade_collection.append(item)
table_name_dict[KLINE_TABLE_NAME] = pixiu_adapter.kline_table_name
table_name_dict[TRADE_TABLE_NAME] = pixiu_adapter.trade_table_name
except Exception as e:
print("Engine get exception: %s" % e)
def __init__(self):
self.data_collection = None
self.logger = Logger.get_logger()
self.config = Configuration.get_configuration()
self._database_name = self.config.mysql_database_name_legal_tender_huobi
self._huobi_table_name = self.config.mysql_table_name_legal_tender_huobi
self._total_page_statistics_table_name = self.config.mysql_table_name_legal_tender_total_page_statistics
def __init__(self, *args, **kw):
super(ExchangesSpider, self).__init__(*args, **kw)
self.config = Configuration.get_configuration()
if ExchangesSpider.pixiu_adapter is None:
error_msg = "please assign pixiu adapter function"
self.logger.error(error_msg)
raise Exception(error_msg)
def __init__(self):
self.config = Configuration.get_configuration()
self.logger = Logger.get_logger()
self._content = ""
self._trade_deficit = ""
self.__add_content_header()
self._add_arbitrage_part_header()
self.arbitrage_mail_subject = ""
def do_crawl(config_file, log_file):
Configuration.get_configuration(config_file)
logger = Logger.get_logger(log_file)
logger.debug("[%s]Log path: %s" % (os.getpid(), Logger.get_log_file()))
try:
logger.debug("==== do_crawl ===== %s" % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
configure_logging(install_root_handler=False)
huobi_adaptee = HuobiAdaptee()
pixiu_adapter = PixiuAdapter(huobi_adaptee)
ExchangesSpider.pixiu_adapter = pixiu_adapter
process = CrawlerProcess(get_project_settings())
process.crawl(ExchangesSpider)
process.start()
pixiu_adapter.save_trade_data_into_mysql()
except Exception as e:
print("Engine get exception: %s" % e)
def __init__(self):
super(BittrexAdaptee, self).__init__()
self.config = Configuration.get_configuration()
self.logger = Logger.get_logger()
self._data_base_name = self.config.mysql_database_name_virtual_currency
self._mysql_table_name_kline = self.config.mysql_table_name_virtual_currency_bittrex_kline
self._mysql_table_name_market_trade = self.config.mysql_table_name_virtual_currency_bittrex_market_trade
self._mysql_table_name_market_depth = self.config.mysql_table_name_virtual_currency_bittrex_market_depth
self._symbol = "USDT-BTC"
def __init__(self):
super(HuobiAdaptee, self).__init__()
self.config = Configuration.get_configuration()
self.logger = Logger.get_logger()
self._data_base_name = self.config.mysql_database_name_virtual_currency
self._mysql_table_name_kline = self.config.mysql_table_name_virtual_currency_huobi_kline
self._mysql_table_name_market_trade = self.config.mysql_table_name_virtual_currency_huobi_market_trade
self._mysql_table_name_market_depth = self.config.mysql_table_name_virtual_currency_huobi_market_depth
self._symbol = "btcusdt"
def __init__(self):
# API doc: https://developer.fcoin.com/zh.html
super(FcoinAdaptee, self).__init__()
self.config = Configuration.get_configuration()
self.logger = Logger.get_logger()
self._data_base_name = self.config.mysql_database_name_virtual_currency
self._mysql_table_name_kline = self.config.mysql_table_name_virtual_currency_fcoin_kline
self._mysql_table_name_market_trade = self.config.mysql_table_name_virtual_currency_fcoin_market_trade
self._mysql_table_name_market_depth = self.config.mysql_table_name_virtual_currency_fcoin_market_depth
self._symbol = "btcusdt"
def __init__(self):
# API doc: https://github.com/58COIN/58coin-api/blob/master/58coin-docs-ZH-cn.md
super(FiveEightCoinAdaptee, self).__init__()
self.config = Configuration.get_configuration()
self.logger = Logger.get_logger()
self._data_base_name = self.config.mysql_database_name_virtual_currency
self._mysql_table_name_kline = self.config.mysql_table_name_virtual_currency_fiveeightcoin_kline
self._mysql_table_name_market_trade = self.config.mysql_table_name_virtual_currency_fiveeightcoin_market_trade
self._mysql_table_name_market_depth = self.config.mysql_table_name_virtual_currency_fiveeightcoin_market_depth
self._symbol = "BTC_USDT"
def __init__(self, log_file=""):
print("[%s]log init only for once!!!!" % os.getpid())
self.config = Configuration.get_configuration()
self.logger = logging.getLogger()
formatter = logging.Formatter(
'[%(process)d] %(asctime)s [%(module)s] %(levelname)s: %(message)s'
)
# Set file log handler
if log_file == "":
log_path = os.path.join(self.config.log_root_dir,
datetime.now().strftime("%Y%m%d_%H%M%S"))
if not os.path.exists(log_path):
os.makedirs(log_path)
log_file = os.path.join(log_path, self.config.log_file)
self._log_file = log_file
rotating_file_handler = RotatingFileHandler(log_file,
maxBytes=10485760)
rotating_file_handler.setFormatter(formatter)
self.logger.addHandler(rotating_file_handler)
self.logger.setLevel(logging.DEBUG)
def get_kline_insert_procedure(self):
return Configuration.get_configuration(
).mysql_procedure_insert_fiveeightcoin_kline
def get_market_depth_insert_procedure(self):
return Configuration.get_configuration(
).mysql_procedure_insert_bittrex_market_depth
def get_kline_insert_procedure(self):
return Configuration.get_configuration(
).mysql_procedure_insert_bittrex_kline
def get_market_trade_insert_procedure(self):
return Configuration.get_configuration(
).mysql_procedure_insert_fcoin_market_trade
def __init__(self):
self.logger = Logger.get_logger()
self.config = Configuration.get_configuration()
required=True, help='configure file path')
parser.add_argument('-m', metavar='mode', dest='mode', type=int, action='store',
required=True, help="assign the crawler target\n"
"0: crawl legal tender info\n"
"1: crawl virtual currencies info\n")
args = parser.parse_args()
if not os.path.exists(args.configuration):
print("Configure file not exists!")
exit(1)
# Do not use the logger before ConfigurationSingleton initialization
print("args.configuration: %s" % args.configuration)
print("args.mode: %d" % args.mode)
print("args: %s" % args)
config = Configuration.get_configuration(args.configuration, args)
config.show(Logger.get_logger())
if args.mode == 0:
legal_tender_engine = LegalTenderEngine()
# legal_tender_engine.run()
legal_tender_engine.main_thread_run()
elif args.mode == 1:
virtual_currency_engine = VirtualCurrencyEngine(HuobiAdaptee)
# virtual_currency_engine.main_thread_run()
virtual_currency_engine.run()
elif args.mode == 2:
virtual_currency_engine = VirtualCurrencyEngine(BittrexAdaptee)
# virtual_currency_engine.main_thread_run()
virtual_currency_engine.run()
def get_market_depth_insert_procedure(self):
return Configuration.get_configuration(
).mysql_procedure_insert_fiveeightcoin_market_depth
def get_market_depth_insert_procedure(cls):
return Configuration.get_configuration(
).mysql_procedure_insert_huobi_market_depth
def get_kline_insert_procedure(cls):
return Configuration.get_configuration(
).mysql_procedure_insert_huobi_kline
def __init__(self, exchange_adaptee):
self.logger = Logger.get_logger()
self.config = Configuration.get_configuration()
self.mysql_connection = MysqlConnection.get_mysql_connection()
self._create_and_use_database_connection()
self._exchange_adaptee = exchange_adaptee