def kdata_to_kafka(fuquan):
for _, security_item in get_security_list().iterrows():
for _, kdata_item in get_kdata(security_item, source='sina', fuquan=fuquan).iterrows():
the_json = kdata_item.to_json(force_ascii=False)
producer.send(get_kafka_kdata_topic(security_item['id'], fuquan),
bytes(the_json, encoding='utf8'),
timestamp_ms=int(datetime.datetime.strptime(kdata_item['timestamp'],
TIME_FORMAT_DAY).timestamp()))
logger.debug("kdata_to_kafka {}".format(the_json))
def _kdata_to_kafka(security_item, fuquan='hfq'):
security_item = to_security_item(security_item)
for _, kdata_item in get_kdata(security_item, fuquan=fuquan).iterrows():
the_json = kdata_item.to_json(force_ascii=False)
producer.send(get_kafka_kdata_topic(security_item['id'], fuquan),
bytes(the_json, encoding='utf8'),
timestamp_ms=int(datetime.datetime.strptime(kdata_item['timestamp'],
TIME_FORMAT_DAY).timestamp()))
logger.debug("kdata_to_kafka {}".format(the_json))
def run(self):
# 对相应标的的行情进行监听,可以多标的多级别同时进行
if self.universe:
consumer = KafkaConsumer(bootstrap_servers=[KAFKA_HOST])
current_topics = consumer.topics()
for security_id in self.universe:
if 'on_tick' in dir(self):
topic = get_kafka_tick_topic(security_id)
if topic in current_topics:
if self.level_step.get('on_tick') < self.step:
self.step = self.level_step.get('on_tick')
self.threads.append(
threading.Thread(target=self.__consume_topic_with_func, args=(topic, 'on_tick')))
self.trading_type = 'event'
else:
self.logger.error("topic:{} not in kafka".format(topic))
for level in ('week', 'day', '60', '30', '15', '5', '1'):
the_func = 'on_{}_bar'.format(level)
topic = get_kafka_kdata_topic(security_id, fuquan=self.stock_fuquan, level=level)
if the_func in dir(self):
if topic in current_topics:
if self.level_step.get(the_func) < self.step:
self.step = self.level_step.get(the_func)
self.threads.append(
threading.Thread(target=self.__consume_topic_with_func, args=(topic, the_func)))
self.trading_type = 'event'
else:
self.logger.error("topic:{} not in kafka".format(topic))
# 用于同步各级别行情消费
if len(self.threads) >= 1:
if self.only_event_mode:
self.barrier = threading.Barrier(len(self.threads))
else:
self.barrier = threading.Barrier(len(self.threads) + 1)
for the_thread in self.threads:
the_thread.start()
# 主线程,是时间漫步的方式,一般来说,step用日线就可以了,主要用在那种大级别的操作
# 账户的每日市值更新也是在这里计算的
if not self.only_event_mode:
while True:
self.on_time_elapsed()
if self.trading_type == 'time':
current_time = self.current_time
self.move_on(self.step)
time_delta = self.current_time.date() - current_time.date()
if time_delta.days >= 1:
self.account_service.save_account(current_time, trading_close=True)
else:
self.barrier.wait()
def __init__(self, security_item=None, level=None):
self.logger = logging.getLogger(__name__)
self.on_init()
self.threads = []
if not hasattr(self, 'living_mode'):
self.living_mode = False
if not hasattr(self, 'start_date'):
self.topics = []
# 回测的开始日期
if not hasattr(self, 'start_date'):
self.start_date = pd.Timestamp('2013-01-01')
# 回测的结束日期,为None的话会一直运行
if not hasattr(self, 'end_date'):
self.end_date = pd.Timestamp.today()
# 交易机器人需要账户，只是做监听告警之类不需要
if not hasattr(self, 'need_account'):
self.need_account = True
if self.need_account:
if not hasattr(self, 'base_capital'):
self.base_capital = 1000000
if not hasattr(self, 'buy_cost'):
self.buy_cost = 0.001
if not hasattr(self, 'sell_cost'):
self.sell_cost = 0.001
if not hasattr(self, 'slippage'):
self.slippage = 0.001
if not hasattr(self, 'stock_fuquan'):
self.stock_fuquan = 'hfq'
self.bot_name = type(self).__name__.lower()
# 指定security_item就监听其某级别的行情，否则为只收到timer信息，需要自己主动去查询行情
if security_item is not None:
self.security_item = security_item
if level is not None:
self.level = level
if hasattr(self, 'security_item'):
if not self.security_item:
raise Exception("you must set one security item!")
self.security_item = to_security_item(self.security_item)
if self.security_item is None:
raise Exception("invalid security item:{}".format(
self.security_item))
# 默认日级别行情
if not hasattr(self, 'level') or not self.level:
self.level = 'day'
self.logger.info(
"bot:{} listen to security_item:{},level:{}".format(
self.bot_name, self.security_item, self.level))
if self.level == 'day':
self.quote_topic = get_kafka_kdata_topic(
security_id=self.security_item['id'], level=self.level)
elif self.level == 'tick':
self.quote_topic = get_kafka_tick_topic(
security_id=self.security_item['id'])
else:
self.logger.exception("wrong level:{}".format(self.level))
else:
# 默认日级别timer
if not hasattr(self, 'time_step'):
self.time_step = timedelta(days=1)
self.logger.info(
"bot:{} check the market by itself,time_step:{}".format(
self.bot_name, self.time_step))
self._after_init()
self.after_init()