def filter_market_trade_data(cla, market_trade_data_collection,
time_boundary):
logger = Logger.get_logger()
if len(market_trade_data_collection) == 0:
logger.debug("Market trade data collection is empty")
return list()
# sort by time
sorted_market_trade_data = \
sorted(market_trade_data_collection, key=lambda x: datetime.datetime.strptime(x[0], SQL_TIME_FORMAT_WITH_MILLISECOND))
target_item_index_list = [
i for i, item in enumerate(sorted_market_trade_data)
if datetime.datetime.strptime(
item[0], SQL_TIME_FORMAT_WITH_MILLISECOND) > time_boundary
]
if len(target_item_index_list) == 0:
logger.debug(
"After time filter[%s][%s], there is no market trade data to return",
time_boundary, sorted_market_trade_data[-1][0])
return list()
start_item_index = min(target_item_index_list)
return sorted_market_trade_data[start_item_index:]
def filter_kline_data(self, kline_data_collection, time_boundary):
logger = Logger.get_logger()
if len(kline_data_collection) == 0:
logger.debug("Kline data collection is empty")
return list()
# sort by time
sorted_kline_data = \
sorted(kline_data_collection, key=lambda x: datetime.datetime.strptime(x[0], SQL_TIME_FORMAT_WITHOUT_SECOND))
formatted_time_boundary = datetime.datetime.strftime(
time_boundary, SQL_TIME_FORMAT_WITHOUT_SECOND)
target_item_index_list = [
i for i, item in enumerate(sorted_kline_data)
if item[0] == formatted_time_boundary
]
# avoid target_item_index_list is empty, and then make start_item_index will be 0, and return entire kline data
target_item_index_list.append(-1)
start_item_index = max(target_item_index_list) + 1
if start_item_index >= len(sorted_kline_data):
logger.debug(
"After time filter[%s][%s], there is no kline data to return",
time_boundary, sorted_kline_data[-1][0])
return list()
return sorted_kline_data[start_item_index:]
def do_crawl_exchange(config_file, log_file, exchange_adaptee,
kline_data_collection, market_depth_collection,
market_trade_collection, table_name_dict):
Configuration.get_configuration(config_file)
logger = Logger.get_logger(log_file)
logger.debug("[%s]Log path: %s" % (os.getpid(), Logger.get_log_file()))
try:
logger.debug("==== do_crawl ===== %s" %
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
configure_logging(install_root_handler=False)
huobi_adaptee = exchange_adaptee()
pixiu_adapter = PixiuAdapter(huobi_adaptee)
ExchangesSpider.pixiu_adapter = pixiu_adapter
process = CrawlerProcess(get_project_settings())
process.crawl(ExchangesSpider)
process.start()
for item in pixiu_adapter.kline_data_collection:
kline_data_collection.append(item)
for item in pixiu_adapter.market_depth_collection:
market_depth_collection.append(item)
for item in pixiu_adapter.market_trade_collection:
market_trade_collection.append(item)
table_name_dict[KLINE_TABLE_NAME] = pixiu_adapter.kline_table_name
table_name_dict[TRADE_TABLE_NAME] = pixiu_adapter.trade_table_name
except Exception as e:
print("Engine get exception: %s" % e)
def __init__(self):
self.data_collection = None
self.logger = Logger.get_logger()
self.config = Configuration.get_configuration()
self._database_name = self.config.mysql_database_name_legal_tender_huobi
self._huobi_table_name = self.config.mysql_table_name_legal_tender_huobi
self._total_page_statistics_table_name = self.config.mysql_table_name_legal_tender_total_page_statistics
def __init__(self):
self.config = Configuration.get_configuration()
self.logger = Logger.get_logger()
self._content = ""
self._trade_deficit = ""
self.__add_content_header()
self._add_arbitrage_part_header()
self.arbitrage_mail_subject = ""
def __init__(self):
super(BittrexAdaptee, self).__init__()
self.config = Configuration.get_configuration()
self.logger = Logger.get_logger()
self._data_base_name = self.config.mysql_database_name_virtual_currency
self._mysql_table_name_kline = self.config.mysql_table_name_virtual_currency_bittrex_kline
self._mysql_table_name_market_trade = self.config.mysql_table_name_virtual_currency_bittrex_market_trade
self._mysql_table_name_market_depth = self.config.mysql_table_name_virtual_currency_bittrex_market_depth
self._symbol = "USDT-BTC"
def __init__(self):
super(HuobiAdaptee, self).__init__()
self.config = Configuration.get_configuration()
self.logger = Logger.get_logger()
self._data_base_name = self.config.mysql_database_name_virtual_currency
self._mysql_table_name_kline = self.config.mysql_table_name_virtual_currency_huobi_kline
self._mysql_table_name_market_trade = self.config.mysql_table_name_virtual_currency_huobi_market_trade
self._mysql_table_name_market_depth = self.config.mysql_table_name_virtual_currency_huobi_market_depth
self._symbol = "btcusdt"
def __init__(self):
# API doc: https://developer.fcoin.com/zh.html
super(FcoinAdaptee, self).__init__()
self.config = Configuration.get_configuration()
self.logger = Logger.get_logger()
self._data_base_name = self.config.mysql_database_name_virtual_currency
self._mysql_table_name_kline = self.config.mysql_table_name_virtual_currency_fcoin_kline
self._mysql_table_name_market_trade = self.config.mysql_table_name_virtual_currency_fcoin_market_trade
self._mysql_table_name_market_depth = self.config.mysql_table_name_virtual_currency_fcoin_market_depth
self._symbol = "btcusdt"
def __init__(self):
# API doc: https://github.com/58COIN/58coin-api/blob/master/58coin-docs-ZH-cn.md
super(FiveEightCoinAdaptee, self).__init__()
self.config = Configuration.get_configuration()
self.logger = Logger.get_logger()
self._data_base_name = self.config.mysql_database_name_virtual_currency
self._mysql_table_name_kline = self.config.mysql_table_name_virtual_currency_fiveeightcoin_kline
self._mysql_table_name_market_trade = self.config.mysql_table_name_virtual_currency_fiveeightcoin_market_trade
self._mysql_table_name_market_depth = self.config.mysql_table_name_virtual_currency_fiveeightcoin_market_depth
self._symbol = "BTC_USDT"
def do_crawl(config_file, log_file):
Configuration.get_configuration(config_file)
logger = Logger.get_logger(log_file)
logger.debug("[%s]Log path: %s" % (os.getpid(), Logger.get_log_file()))
try:
logger.debug("==== do_crawl ===== %s" % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
configure_logging(install_root_handler=False)
huobi_adaptee = HuobiAdaptee()
pixiu_adapter = PixiuAdapter(huobi_adaptee)
ExchangesSpider.pixiu_adapter = pixiu_adapter
process = CrawlerProcess(get_project_settings())
process.crawl(ExchangesSpider)
process.start()
pixiu_adapter.save_trade_data_into_mysql()
except Exception as e:
print("Engine get exception: %s" % e)
def __init__(self):
self._market_trade_data_collection = list()
self._logger = Logger.get_logger()
def __init__(self):
self.logger = Logger.get_logger()
self.config = Configuration.get_configuration()
def __init__(self):
self._kline_data_collection = list()
self._logger = Logger.get_logger()
def __init__(self, exchange_adaptee):
self.logger = Logger.get_logger()
self.config = Configuration.get_configuration()
self.mysql_connection = MysqlConnection.get_mysql_connection()
self._create_and_use_database_connection()
self._exchange_adaptee = exchange_adaptee
required=True, help="assign the crawler target\n"
"0: crawl legal tender info\n"
"1: crawl virtual currencies info\n")
args = parser.parse_args()
if not os.path.exists(args.configuration):
print("Configure file not exists!")
exit(1)
# Do not use the logger before ConfigurationSingleton initialization
print("args.configuration: %s" % args.configuration)
print("args.mode: %d" % args.mode)
print("args: %s" % args)
config = Configuration.get_configuration(args.configuration, args)
config.show(Logger.get_logger())
if args.mode == 0:
legal_tender_engine = LegalTenderEngine()
# legal_tender_engine.run()
legal_tender_engine.main_thread_run()
elif args.mode == 1:
virtual_currency_engine = VirtualCurrencyEngine(HuobiAdaptee)
# virtual_currency_engine.main_thread_run()
virtual_currency_engine.run()
elif args.mode == 2:
virtual_currency_engine = VirtualCurrencyEngine(BittrexAdaptee)
# virtual_currency_engine.main_thread_run()
virtual_currency_engine.run()
elif args.mode == 3:
virtual_currency_engine = VirtualCurrencyEngine(BianceAdaptee)