def do_crawl_exchange(config_file, log_file, exchange_adaptee,
kline_data_collection, market_depth_collection,
market_trade_collection, table_name_dict):
Configuration.get_configuration(config_file)
logger = Logger.get_logger(log_file)
logger.debug("[%s]Log path: %s" % (os.getpid(), Logger.get_log_file()))
try:
logger.debug("==== do_crawl ===== %s" %
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
configure_logging(install_root_handler=False)
huobi_adaptee = exchange_adaptee()
pixiu_adapter = PixiuAdapter(huobi_adaptee)
ExchangesSpider.pixiu_adapter = pixiu_adapter
process = CrawlerProcess(get_project_settings())
process.crawl(ExchangesSpider)
process.start()
for item in pixiu_adapter.kline_data_collection:
kline_data_collection.append(item)
for item in pixiu_adapter.market_depth_collection:
market_depth_collection.append(item)
for item in pixiu_adapter.market_trade_collection:
market_trade_collection.append(item)
table_name_dict[KLINE_TABLE_NAME] = pixiu_adapter.kline_table_name
table_name_dict[TRADE_TABLE_NAME] = pixiu_adapter.trade_table_name
except Exception as e:
print("Engine get exception: %s" % e)
def __init__(self):
self.data_collection = None
self.logger = Logger.get_logger()
self.config = Configuration.get_configuration()
self._database_name = self.config.mysql_database_name_legal_tender_huobi
self._huobi_table_name = self.config.mysql_table_name_legal_tender_huobi
self._total_page_statistics_table_name = self.config.mysql_table_name_legal_tender_total_page_statistics
def filter_market_trade_data(cla, market_trade_data_collection,
time_boundary):
logger = Logger.get_logger()
if len(market_trade_data_collection) == 0:
logger.debug("Market trade data collection is empty")
return list()
# sort by time
sorted_market_trade_data = \
sorted(market_trade_data_collection, key=lambda x: datetime.datetime.strptime(x[0], SQL_TIME_FORMAT_WITH_MILLISECOND))
target_item_index_list = [
i for i, item in enumerate(sorted_market_trade_data)
if datetime.datetime.strptime(
item[0], SQL_TIME_FORMAT_WITH_MILLISECOND) > time_boundary
]
if len(target_item_index_list) == 0:
logger.debug(
"After time filter[%s][%s], there is no market trade data to return",
time_boundary, sorted_market_trade_data[-1][0])
return list()
start_item_index = min(target_item_index_list)
return sorted_market_trade_data[start_item_index:]
def filter_kline_data(self, kline_data_collection, time_boundary):
logger = Logger.get_logger()
if len(kline_data_collection) == 0:
logger.debug("Kline data collection is empty")
return list()
# sort by time
sorted_kline_data = \
sorted(kline_data_collection, key=lambda x: datetime.datetime.strptime(x[0], SQL_TIME_FORMAT_WITHOUT_SECOND))
formatted_time_boundary = datetime.datetime.strftime(
time_boundary, SQL_TIME_FORMAT_WITHOUT_SECOND)
target_item_index_list = [
i for i, item in enumerate(sorted_kline_data)
if item[0] == formatted_time_boundary
]
# avoid target_item_index_list is empty, and then make start_item_index will be 0, and return entire kline data
target_item_index_list.append(-1)
start_item_index = max(target_item_index_list) + 1
if start_item_index >= len(sorted_kline_data):
logger.debug(
"After time filter[%s][%s], there is no kline data to return",
time_boundary, sorted_kline_data[-1][0])
return list()
return sorted_kline_data[start_item_index:]
def __init__(self):
self.config = Configuration.get_configuration()
self.logger = Logger.get_logger()
self._content = ""
self._trade_deficit = ""
self.__add_content_header()
self._add_arbitrage_part_header()
self.arbitrage_mail_subject = ""
def do_crawl(config_file, log_file):
Configuration.get_configuration(config_file)
logger = Logger.get_logger(log_file)
logger.debug("[%s]Log path: %s" % (os.getpid(), Logger.get_log_file()))
try:
logger.debug("==== do_crawl ===== %s" % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
configure_logging(install_root_handler=False)
huobi_adaptee = HuobiAdaptee()
pixiu_adapter = PixiuAdapter(huobi_adaptee)
ExchangesSpider.pixiu_adapter = pixiu_adapter
process = CrawlerProcess(get_project_settings())
process.crawl(ExchangesSpider)
process.start()
pixiu_adapter.save_trade_data_into_mysql()
except Exception as e:
print("Engine get exception: %s" % e)
def main_thread_run(self):
try:
mp.set_start_method('spawn')
do_crawl(self.config.config_file, Logger.get_log_file())
print("------> [main_thread_run]crawler down!!!")
except Exception as e:
self.logger.error("Engine get exception: %s", e)
print("Engine get exception %s" % e)
def __init__(self):
super(HuobiAdaptee, self).__init__()
self.config = Configuration.get_configuration()
self.logger = Logger.get_logger()
self._data_base_name = self.config.mysql_database_name_virtual_currency
self._mysql_table_name_kline = self.config.mysql_table_name_virtual_currency_huobi_kline
self._mysql_table_name_market_trade = self.config.mysql_table_name_virtual_currency_huobi_market_trade
self._mysql_table_name_market_depth = self.config.mysql_table_name_virtual_currency_huobi_market_depth
self._symbol = "btcusdt"
def __init__(self):
super(BittrexAdaptee, self).__init__()
self.config = Configuration.get_configuration()
self.logger = Logger.get_logger()
self._data_base_name = self.config.mysql_database_name_virtual_currency
self._mysql_table_name_kline = self.config.mysql_table_name_virtual_currency_bittrex_kline
self._mysql_table_name_market_trade = self.config.mysql_table_name_virtual_currency_bittrex_market_trade
self._mysql_table_name_market_depth = self.config.mysql_table_name_virtual_currency_bittrex_market_depth
self._symbol = "USDT-BTC"
def do_something(sc):
s.enter(self.config.crawl_period, 1, do_something, (sc, ))
mp_manager = mp.Manager()
kline_data_collection = mp_manager.list()
market_depth_collection = mp_manager.list()
market_trade_collection = mp_manager.list()
table_name_dict = mp_manager.dict()
# exchange_adaptee = BittrexAdaptee
exchange_adaptee = self._exchange_adaptee
process = mp.Process(target=do_crawl_exchange,
args=(
self.config.config_file,
Logger.get_log_file(),
exchange_adaptee,
kline_data_collection,
market_depth_collection,
market_trade_collection,
table_name_dict,
))
self.logger.info(
"[run] Start process: %s",
datetime.datetime.now().strftime(
SQL_TIME_FORMAT_WITH_MILLISECOND))
process.start()
process.join()
self.logger.info(
"[run] End process: %s",
datetime.datetime.now().strftime(
SQL_TIME_FORMAT_WITH_MILLISECOND))
kline_table_name = table_name_dict[KLINE_TABLE_NAME]
trade_table_name = table_name_dict[TRADE_TABLE_NAME]
max_time_kline = self._get_max_time_kline(kline_table_name)
filtered_kline_data_collection = KlineDataCollection.filter_kline_data(
kline_data_collection, max_time_kline)
kline_procedure_name = self._exchange_adaptee.get_kline_insert_procedure(
)
self._save_data_into_database_by_procedure(
kline_procedure_name, filtered_kline_data_collection)
max_time_market_trade = self._get_max_time_market_trade(
trade_table_name)
filtered_market_trade_data_collection = MarketTradeDataCollection.filter_market_trade_data(
market_trade_collection, max_time_market_trade)
market_trade_procedure_name = self._exchange_adaptee.get_market_trade_insert_procedure(
)
self._save_data_into_database_by_procedure(
market_trade_procedure_name,
filtered_market_trade_data_collection)
market_depth_procedure_name = self._exchange_adaptee.get_market_depth_insert_procedure(
)
self._save_data_into_database_by_procedure(
market_depth_procedure_name, market_depth_collection)
def do_something(sc):
s.enter(self.config.crawl_period, 1, do_something, (sc,))
process = mp.Process(target=do_crawl, args=(self.config.config_file, Logger.get_log_file()))
process.start()
time.sleep(self.config.crawl_period)
# Received SIGTERM, shutting down gracefully. Send again to force
process.terminate()
# INFO: Closing spider (shutdown)
process.terminate()
self.logger.error("Crawler is overtime, terminate is %d" % process.pid)
def __init__(self):
# API doc: https://github.com/58COIN/58coin-api/blob/master/58coin-docs-ZH-cn.md
super(FiveEightCoinAdaptee, self).__init__()
self.config = Configuration.get_configuration()
self.logger = Logger.get_logger()
self._data_base_name = self.config.mysql_database_name_virtual_currency
self._mysql_table_name_kline = self.config.mysql_table_name_virtual_currency_fiveeightcoin_kline
self._mysql_table_name_market_trade = self.config.mysql_table_name_virtual_currency_fiveeightcoin_market_trade
self._mysql_table_name_market_depth = self.config.mysql_table_name_virtual_currency_fiveeightcoin_market_depth
self._symbol = "BTC_USDT"
def __init__(self):
# API doc: https://developer.fcoin.com/zh.html
super(FcoinAdaptee, self).__init__()
self.config = Configuration.get_configuration()
self.logger = Logger.get_logger()
self._data_base_name = self.config.mysql_database_name_virtual_currency
self._mysql_table_name_kline = self.config.mysql_table_name_virtual_currency_fcoin_kline
self._mysql_table_name_market_trade = self.config.mysql_table_name_virtual_currency_fcoin_market_trade
self._mysql_table_name_market_depth = self.config.mysql_table_name_virtual_currency_fcoin_market_depth
self._symbol = "btcusdt"
def main_thread_run(self):
try:
mp.set_start_method('spawn')
queue = mp.Queue()
do_crawl_bittrex(queue, self.config.config_file,
Logger.get_log_file())
# pixiu_adapter = queue.get()
# do_crawl_huobi(self.config.config_file, Logger.get_log_file())
print("------> [main_thread_run]crawler down!!!")
except Exception as e:
self.logger.error("Engine get exception: %s", e)
print("Engine get exception %s" % e)
def __init__(self):
self._market_trade_data_collection = list()
self._logger = Logger.get_logger()
def __init__(self, exchange_adaptee):
self.logger = Logger.get_logger()
self.config = Configuration.get_configuration()
self.mysql_connection = MysqlConnection.get_mysql_connection()
self._create_and_use_database_connection()
self._exchange_adaptee = exchange_adaptee
def __init__(self):
self.logger = Logger.get_logger()
self.config = Configuration.get_configuration()
required=True, help="assign the crawler target\n"
"0: crawl legal tender info\n"
"1: crawl virtual currencies info\n")
args = parser.parse_args()
if not os.path.exists(args.configuration):
print("Configure file not exists!")
exit(1)
# Do not use the logger before ConfigurationSingleton initialization
print("args.configuration: %s" % args.configuration)
print("args.mode: %d" % args.mode)
print("args: %s" % args)
config = Configuration.get_configuration(args.configuration, args)
config.show(Logger.get_logger())
if args.mode == 0:
legal_tender_engine = LegalTenderEngine()
# legal_tender_engine.run()
legal_tender_engine.main_thread_run()
elif args.mode == 1:
virtual_currency_engine = VirtualCurrencyEngine(HuobiAdaptee)
# virtual_currency_engine.main_thread_run()
virtual_currency_engine.run()
elif args.mode == 2:
virtual_currency_engine = VirtualCurrencyEngine(BittrexAdaptee)
# virtual_currency_engine.main_thread_run()
virtual_currency_engine.run()
elif args.mode == 3:
virtual_currency_engine = VirtualCurrencyEngine(BianceAdaptee)
def __init__(self):
self._kline_data_collection = list()
self._logger = Logger.get_logger()