def cryptocurrency_tick_to_kafka(exchange, pairs=None):
for tick in fetch_ticks(exchange, pairs=pairs):
producer.send(get_kafka_tick_topic(tick['securityId']),
bytes(json.dumps(tick), encoding='utf8'),
timestamp_ms=tick['timestamp'])
logger.debug("tick_to_kafka {}".format(tick))
def tick_to_kafka():
for _, security_item in get_security_list().iterrows():
for df in get_ticks(security_item):
for _, tick_item in df.iterrows():
the_json = tick_item.to_json(force_ascii=False)
producer.send(get_kafka_tick_topic(security_item['id']),
bytes(the_json, encoding='utf8'),
timestamp_ms=int(datetime.datetime.strptime(tick_item['timestamp'],
TIME_FORMAT_SEC).timestamp()))
logger.debug("tick_to_kafka {}".format(the_json))
def _tick_to_kafka(security_item):
security_item = to_security_item(security_item)
for df in get_ticks(security_item):
for _, tick_item in df.iterrows():
the_json = tick_item.to_json(force_ascii=False)
producer.send(get_kafka_tick_topic(security_item['id']),
bytes(the_json, encoding='utf8'),
timestamp_ms=int(datetime.datetime.strptime(tick_item['timestamp'],
TIME_FORMAT_SEC).timestamp()))
logger.debug("tick_to_kafka {}".format(the_json))
def run(self):
# 对相应标的的行情进行监听,可以多标的多级别同时进行
if self.universe:
consumer = KafkaConsumer(bootstrap_servers=[KAFKA_HOST])
current_topics = consumer.topics()
for security_id in self.universe:
if 'on_tick' in dir(self):
topic = get_kafka_tick_topic(security_id)
if topic in current_topics:
if self.level_step.get('on_tick') < self.step:
self.step = self.level_step.get('on_tick')
self.threads.append(
threading.Thread(target=self.__consume_topic_with_func, args=(topic, 'on_tick')))
self.trading_type = 'event'
else:
self.logger.error("topic:{} not in kafka".format(topic))
for level in ('week', 'day', '60', '30', '15', '5', '1'):
the_func = 'on_{}_bar'.format(level)
topic = get_kafka_kdata_topic(security_id, fuquan=self.stock_fuquan, level=level)
if the_func in dir(self):
if topic in current_topics:
if self.level_step.get(the_func) < self.step:
self.step = self.level_step.get(the_func)
self.threads.append(
threading.Thread(target=self.__consume_topic_with_func, args=(topic, the_func)))
self.trading_type = 'event'
else:
self.logger.error("topic:{} not in kafka".format(topic))
# 用于同步各级别行情消费
if len(self.threads) >= 1:
if self.only_event_mode:
self.barrier = threading.Barrier(len(self.threads))
else:
self.barrier = threading.Barrier(len(self.threads) + 1)
for the_thread in self.threads:
the_thread.start()
# 主线程,是时间漫步的方式,一般来说,step用日线就可以了,主要用在那种大级别的操作
# 账户的每日市值更新也是在这里计算的
if not self.only_event_mode:
while True:
self.on_time_elapsed()
if self.trading_type == 'time':
current_time = self.current_time
self.move_on(self.step)
time_delta = self.current_time.date() - current_time.date()
if time_delta.days >= 1:
self.account_service.save_account(current_time, trading_close=True)
else:
self.barrier.wait()
def __init__(self, security_id=None):
super().__init__()
self.security_id = security_id
self.start_timestamp = None
self.end_timestamp = None
# setup the user custom settings
self.on_init()
assert self.security_id is not None
self.security_item = to_security_item(self.security_id)
assert self.security_item is not None
self._threads = []
self.quote_topic = get_kafka_tick_topic(security_id=self.security_id)
self.logger.info("bot:{} listen to security:{} topic:{}".format(
self.bot_name, self.security_id, self.quote_topic))
def get_latest_timestamp_order(security_id):
topic = get_kafka_tick_topic(security_id)
return get_latest_timestamp_order_from_topic(topic)
def __init__(self, security_item=None, level=None):
self.logger = logging.getLogger(__name__)
self.on_init()
self.threads = []
if not hasattr(self, 'living_mode'):
self.living_mode = False
if not hasattr(self, 'start_date'):
self.topics = []
# 回测的开始日期
if not hasattr(self, 'start_date'):
self.start_date = pd.Timestamp('2013-01-01')
# 回测的结束日期,为None的话会一直运行
if not hasattr(self, 'end_date'):
self.end_date = pd.Timestamp.today()
# 交易机器人需要账户，只是做监听告警之类不需要
if not hasattr(self, 'need_account'):
self.need_account = True
if self.need_account:
if not hasattr(self, 'base_capital'):
self.base_capital = 1000000
if not hasattr(self, 'buy_cost'):
self.buy_cost = 0.001
if not hasattr(self, 'sell_cost'):
self.sell_cost = 0.001
if not hasattr(self, 'slippage'):
self.slippage = 0.001
if not hasattr(self, 'stock_fuquan'):
self.stock_fuquan = 'hfq'
self.bot_name = type(self).__name__.lower()
# 指定security_item就监听其某级别的行情，否则为只收到timer信息，需要自己主动去查询行情
if security_item is not None:
self.security_item = security_item
if level is not None:
self.level = level
if hasattr(self, 'security_item'):
if not self.security_item:
raise Exception("you must set one security item!")
self.security_item = to_security_item(self.security_item)
if self.security_item is None:
raise Exception("invalid security item:{}".format(
self.security_item))
# 默认日级别行情
if not hasattr(self, 'level') or not self.level:
self.level = 'day'
self.logger.info(
"bot:{} listen to security_item:{},level:{}".format(
self.bot_name, self.security_item, self.level))
if self.level == 'day':
self.quote_topic = get_kafka_kdata_topic(
security_id=self.security_item['id'], level=self.level)
elif self.level == 'tick':
self.quote_topic = get_kafka_tick_topic(
security_id=self.security_item['id'])
else:
self.logger.exception("wrong level:{}".format(self.level))
else:
# 默认日级别timer
if not hasattr(self, 'time_step'):
self.time_step = timedelta(days=1)
self.logger.info(
"bot:{} check the market by itself,time_step:{}".format(
self.bot_name, self.time_step))
self._after_init()
self.after_init()
def eos_ram_to_kafka():
ram_trade = db.ram_trade
logger.info("collection:{}".format(ram_trade))
earliest_record = ram_trade.find_one({
"$query": {},
"$orderby": {
"global_seq": 1
}
})
latest_record = ram_trade.find_one({
"$query": {},
"$orderby": {
"global_seq": -1
}
})
logger.info("earliest_record:{},latest_record:{}".format(
earliest_record, latest_record))
security_id = 'cryptocurrency_contract_RAM-EOS'
latest_timestamp, latest_order = get_latest_timestamp_order(security_id)
topic = get_kafka_tick_topic(security_id)
if not latest_timestamp:
latest_timestamp = earliest_record['block_time']
start_date, end_date = evaluate_time_range(latest_timestamp)
while True:
if latest_order and start_date and end_date:
condition = {
"block_time": {
"$gte": start_date,
"$lt": end_date
},
"global_seq": {
"$gt": latest_order
}
}
elif start_date and end_date:
condition = {"block_time": {"$gte": start_date, "$lt": end_date}}
elif latest_order:
condition = {"global_seq": {"$gt": latest_order}}
logger.info("start_date:{},end_date:{},order:{}".format(
start_date, end_date, latest_order))
latest_timestamp = end_date
for item in ram_trade.find(condition):
tick = to_tick(item)
record_meta = producer.send(
topic,
bytes(json.dumps(tick, ensure_ascii=False), encoding='utf8'),
key=bytes(security_id, encoding='utf8'),
timestamp_ms=int(item['block_time'].timestamp() * 1000))
record = record_meta.get(10)
latest_timestamp = to_timestamp(record.timestamp)
latest_order = tick['order']
logger.debug("tick_to_kafka {}".format(tick))
if datetime.now() - latest_timestamp < timedelta(minutes=5):
time.sleep(2)
logger.info("record latest_timestamp:{},now is:{}".format(
latest_timestamp, datetime.now()))
start_date = None
end_date = None
else:
start_date, end_date = evaluate_time_range(latest_timestamp)