def __init__(self, locator, name="ledger_client"): self.commander_location = pyyjj.location(pyyjj.mode.LIVE, pyyjj.category.SYSTEM, 'service', 'ledger', locator) io_device = pyyjj.io_device(self.commander_location) self.cmd_sock = io_device.connect_socket(self.commander_location, pyyjj.protocol.REQUEST, 10000)
def __init__(self, ctx): self.ctx = ctx self.ctx.converter = self self.dest_id = 0 # dest_id 0 should be configurable TODO self.location = pyyjj.location(kfj.MODES['backtest'], kfj.CATEGORIES['md'], ctx.group, ctx.name, ctx.locator) self.io_device = pyyjj.io_device(self.location, False, True) self.writer = self.io_device.open_writer(self.dest_id) self.reader = self.io_device.open_reader_to_subscribe() self.reader.join(self.location, self.dest_id, 0) self.logger = create_logger("converter", ctx.log_level, self.location)
def trace_journal(ctx, session_id, io_type): trace_df = pd.DataFrame(columns=[ 'gen_time', 'trigger_time', 'source', 'dest', 'msg_type', 'frame_length', 'data_length' ]) session = find_session(ctx, session_id) uname = '{}/{}/{}/{}'.format(session['category'], session['group'], session['name'], session['mode']) uid = pyyjj.hash_str_32(uname) ctx.category = '*' ctx.group = '*' ctx.name = '*' ctx.mode = '*' locations = collect_journal_locations(ctx) location = locations[uid] home = make_location_from_dict(ctx, location) io_device = pyyjj.io_device(home) reader = io_device.open_reader_to_subscribe() if io_type == 'out' or io_type == 'all': for dest in location['readers']: dest_id = int(dest, 16) reader.join(home, dest_id, session['begin_time']) if (io_type == 'in' or io_type == 'all' ) and not (home.category == pyyjj.category.SYSTEM and home.group == 'master' and home.name == 'master'): master_cmd_uid = pyyjj.hash_str_32('system/master/{:08x}/live'.format( location['uid'])) master_cmd_location = make_location_from_dict( ctx, locations[master_cmd_uid]) reader.join(master_cmd_location, location['uid'], session['begin_time']) while reader.data_available( ) and reader.current_frame().gen_time <= session['end_time']: frame = reader.current_frame() trace_df.loc[len(trace_df)] = [ frame.gen_time, frame.trigger_time, locations[frame.source]['uname'], 'public' if frame.dest == 0 else locations[frame.dest]['uname'], frame.msg_type, frame.frame_length, frame.data_length ] if frame.dest == home.uid and (frame.msg_type == 10021 or frame.msg_type == 10022): request = pyyjj.get_RequestReadFrom(frame) source_location = make_location_from_dict( ctx, locations[request.source_id]) reader.join(source_location, location['uid'] if frame.msg_type == 10021 else 0, request.from_time) reader.next() return trace_df
def ledger(ctx): pass_ctx_from_parent(ctx) commander_location = pyyjj.location(pyyjj.mode.LIVE, pyyjj.category.SYSTEM, 'service', 'ledger', ctx.locator) ctx.logger = create_logger("ledger_test", ctx.log_level, commander_location) io_device = pyyjj.io_device(commander_location) cmd_sock = io_device.connect_socket(commander_location, pyyjj.protocol.REQUEST, 10000) calendar.calendar_request(cmd_sock, commander_location) # order.new_order_single(cmd_sock, commander_location, test_account) # ctx.logger.info('done') order.cancel_all_order_for_account(cmd_sock, commander_location, test_account) ctx.logger.info('done')
def find_sessions(ctx): io_device = pyyjj.io_device(ctx.journal_util_location) ctx.session_count = 1 sessions_df = pd.DataFrame(columns=[ 'id', 'mode', 'category', 'group', 'name', 'begin_time', 'end_time', 'closed', 'duration', 'frame_count' ]) locations = collect_journal_locations(ctx) dest_pub = '{:08x}'.format(0) for key in locations: record = locations[key] location = pyyjj.location(MODES[record['mode']], CATEGORIES[record['category']], record['group'], record['name'], ctx.locator) if dest_pub in record['readers']: reader = io_device.open_reader_to_subscribe() for dest_id in record['readers']: reader.join(location, int(dest_id, 16), 0) find_sessions_from_reader(ctx, sessions_df, reader, record['mode'], record['category'], record['group'], record['name']) return sessions_df
def __init__(self, ctx): self.ctx = ctx self.ctx.converter = self self.filepath = ctx.filepath self.from_time = int(ctx.time) self.data = None # dest_id 0 should be configurable TODO self.dest_id = 0 self.location = pyyjj.location(kfj.MODES['backtest'], kfj.CATEGORIES['md'], ctx.group, ctx.name, ctx.locator) self.io_device = pyyjj.io_device(self.location, False, True) self.writer = self.io_device.open_writer(self.dest_id) self.reader = self.io_device.open_reader_to_subscribe() self.reader.join(self.location, self.dest_id, self.from_time) #self.reader.join(pyyjj.location(kfj.MODES['live'], kfj.CATEGORIES['md'], 'xtp', 'xtp', ctx.locator) , 0, self.from_time) self.logger = create_logger("converter", ctx.log_level, self.location) self.logger.info("converter creation success!")
def ledger(ctx, protocol, msg_type): pass_ctx_from_parent(ctx) commander_location = pyyjj.location(pyyjj.mode.LIVE, pyyjj.category.SYSTEM, 'service', 'ledger', ctx.locator) ctx.logger = create_logger("ledger_test", ctx.log_level, commander_location) io_device = pyyjj.io_device(commander_location) cmd_sock = io_device.connect_socket( commander_location, pyyjj.protocol.SUBSCRIBE if protocol == 'sub' else pyyjj.protocol.REQUEST, 10000) if protocol == "sub": cmd_sock.setsockopt(int(pyyjj.protocol.SUBSCRIBE), 1, "") while True: cmd_sock.recv() print(cmd_sock.last_message()) calendar.calendar_request(cmd_sock, commander_location) broker.broker_state_request(cmd_sock, commander_location) order.cancel_all_order_for_account(cmd_sock, commander_location, test_account) asset.asset_request(cmd_sock, commander_location) asset.all_asset_info_request(cmd_sock, commander_location) ctx.logger.info('done')
def find_sessions(ctx): io_device = pyyjj.io_device(ctx.journal_util_location) ctx.session_count = 1 sessions_df = pd.DataFrame(columns=[ 'id', 'mode', 'category', 'group', 'name', 'begin_time', 'end_time', 'closed', 'duration' ]) locations = collect_journal_locations(ctx) dest_pub = '{:08x}'.format(0) for key in locations: record = locations[key] if record["mode"] == "live" and record["group"] != "master": master_command_location = pyyjj.location(pyyjj.mode.LIVE, pyyjj.category.SYSTEM, "master", '{:08x}'.format(key), ctx.locator) reader = io_device.open_reader_to_subscribe() reader.join(master_command_location, key, 0) find_sessions_from_reader(ctx, sessions_df, reader, record['mode'], record['category'], record['group'], record['name']) else: location = pyyjj.location(MODES[record['mode']], CATEGORIES[record['category']], record['group'], record['name'], ctx.locator) if dest_pub in record['readers']: reader = io_device.open_reader_to_subscribe() reader.join(location, int(dest_pub, 16), 0) find_sessions_from_reader(ctx, sessions_df, reader, record['mode'], record['category'], record['group'], record['name']) return sessions_df
def dev(ctx): pass_ctx_from_root(ctx) ctx.dev_location = pyyjj.location(pyyjj.mode.LIVE, pyyjj.category.SYSTEM, 'dev', 'dev', ctx.locator) ctx.io_device = pyyjj.io_device(ctx.dev_location)
def test(ctx): pass_ctx_from_root(ctx) ctx.test_location = pyyjj.location(pyyjj.mode.LIVE, pyyjj.category.SYSTEM, 'test', 'test', ctx.locator) ctx.io_device = pyyjj.io_device(ctx.test_location)
def reader(ctx, session_id, io_type, from_beginning, max_messages, msg, continuous, output): pass_ctx_from_parent(ctx) session = kfj.find_session(ctx, session_id) uname = '{}/{}/{}/{}'.format(session['category'], session['group'], session['name'], session['mode']) uid = pyyjj.hash_str_32(uname) ctx.category = '*' ctx.group = '*' ctx.name = '*' ctx.mode = '*' locations = kfj.collect_journal_locations(ctx) location = locations[uid] home = kfj.make_location_from_dict(ctx, location) io_device = pyyjj.io_device(home) reader = io_device.open_reader_to_subscribe() if io_type == 'out' or io_type == 'all': for dest in location['readers']: dest_id = int(dest, 16) reader.join(home, dest_id, session['begin_time']) if (io_type == 'in' or io_type == 'all' ) and not (home.category == pyyjj.category.SYSTEM and home.group == 'master' and home.name == 'master'): master_home_uid = pyyjj.hash_str_32('system/master/master/live') master_home_location = kfj.make_location_from_dict( ctx, locations[master_home_uid]) reader.join(master_home_location, 0, session['begin_time']) master_cmd_uid = pyyjj.hash_str_32('system/master/{:08x}/live'.format( location['uid'])) master_cmd_location = kfj.make_location_from_dict( ctx, locations[master_cmd_uid]) reader.join(master_cmd_location, location['uid'], session['begin_time']) start_time = pyyjj.now_in_nano( ) if not from_beginning else session["begin_time"] msg_count = 0 if output: if msg == "all": raise ValueError( "invalid msg {}, please choose from ('quote', 'order', 'trade')" .format(msg)) msg_type = wc_utils.get_msg_type(msg) fieldnames = wc_utils.get_csv_header(msg_type) csv_writer = csv.DictWriter( open(output, "w"), fieldnames=wc_utils.get_csv_header(msg_type)) csv_writer.writeheader() pp = pprint.PrettyPrinter(indent=4) while True: if reader.data_available() and msg_count < max_messages: frame = reader.current_frame() if frame.dest == home.uid and ( frame.msg_type == yjj_msg.RequestReadFrom or frame.msg_type == yjj_msg.RequestReadFromPublic): request = pyyjj.get_RequestReadFrom(frame) source_location = kfj.make_location_from_dict( ctx, locations[request.source_id]) reader.join( source_location, location['uid'] if frame.msg_type == yjj_msg.RequestReadFrom else 0, request.from_time) if frame.dest == home.uid and frame.msg_type == yjj_msg.Deregister: loc = json.loads(frame.data_as_string()) reader.disjoin(loc['uid']) if frame.msg_type == yjj_msg.SessionEnd: ctx.logger.info("session reach end at %s", kft.strftime(frame.gen_time)) break elif frame.gen_time >= start_time and ( msg == "all" or wc_utils.get_msg_type(msg) == frame.msg_type): dict_row = wc_utils.flatten_json( wc_utils.object_as_dict(frame.data)) if output: csv_writer.writerow(dict_row) else: pp.pprint(dict_row) msg_count += 1 reader.next() elif msg_count >= max_messages: ctx.logger.info("reach max messages {}".format(max_messages)) break elif not reader.data_available(): if not continuous: ctx.logger.info("no data is available") break else: time.sleep(0.1)
def reader(ctx, session_id, io_type, from_beginning, max_messages, msg, continuous, output, script): pass_ctx_from_parent(ctx) session = kfj.find_session(ctx, session_id) uname = '{}/{}/{}/{}'.format(session['category'], session['group'], session['name'], session['mode']) uid = pyyjj.hash_str_32(uname) ctx.category = '*' ctx.group = '*' ctx.name = '*' ctx.mode = '*' locations = kfj.collect_journal_locations(ctx) location = locations[uid] home = kfj.make_location_from_dict(ctx, location) io_device = pyyjj.io_device(home) reader = io_device.open_reader_to_subscribe() if io_type == 'out' or io_type == 'all': for dest in location['readers']: dest_id = int(dest, 16) reader.join(home, dest_id, session['begin_time']) if (io_type == 'in' or io_type == 'all' ) and not (home.category == pyyjj.category.SYSTEM and home.group == 'master' and home.name == 'master'): master_home_uid = pyyjj.hash_str_32('system/master/master/live') master_home_location = kfj.make_location_from_dict( ctx, locations[master_home_uid]) reader.join(master_home_location, 0, session['begin_time']) master_cmd_uid = pyyjj.hash_str_32('system/master/{:08x}/live'.format( location['uid'])) master_cmd_location = kfj.make_location_from_dict( ctx, locations[master_cmd_uid]) reader.join(master_cmd_location, location['uid'], session['begin_time']) start_time = pyyjj.now_in_nano( ) if not from_beginning else session["begin_time"] msg_count = 0 msg_type_to_read = None if msg == "all" else kungfu.msg.Registry.meta_from_name( msg)["id"] if output: if msg not in kungfu.msg.Registry.type_names(): raise ValueError("invalid msg {}, please choose from {}".format( kungfu.msg.Registry.type_names())) csv_writer = None def handle(frame): data_as_dict = frame["data"] dict_row = kungfu.msg.utils.flatten_json(data_as_dict) nonlocal csv_writer if not csv_writer: csv_writer = csv.DictWriter(open(output, "w"), fieldnames=dict_row.keys()) csv_writer.writeheader() csv_writer.writerow(dict_row) frame_handler = handle elif script: dir = os.path.dirname(script) name_no_ext = os.path.split(os.path.basename(script)) sys.path.append(os.path.relpath(dir)) impl = importlib.import_module(os.path.splitext(name_no_ext[1])[0]) frame_handler = getattr(impl, 'on_frame', lambda frame: None) else: pp = pprint.PrettyPrinter(indent=4) frame_handler = pp.pprint while True: if reader.data_available() and msg_count < max_messages: frame = reader.current_frame() if frame.dest == home.uid and ( frame.msg_type == yjj_msg.RequestReadFrom or frame.msg_type == yjj_msg.RequestReadFromPublic): request = pyyjj.get_RequestReadFrom(frame) source_location = kfj.make_location_from_dict( ctx, locations[request.source_id]) reader.join( source_location, location['uid'] if frame.msg_type == yjj_msg.RequestReadFrom else 0, request.from_time) if frame.dest == home.uid and frame.msg_type == yjj_msg.Deregister: loc = json.loads(frame.data_as_string()) reader.disjoin(loc['uid']) if frame.msg_type == yjj_msg.SessionEnd: ctx.logger.info("session reach end at %s", kft.strftime(frame.gen_time)) break elif frame.gen_time >= start_time and ( msg == "all" or msg_type_to_read == frame.msg_type): try: frame_handler(frame.as_dict()) except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() ctx.logger.error( 'error [%s] %s', exc_type, traceback.format_exception(exc_type, exc_obj, exc_tb)) msg_count += 1 reader.next() elif msg_count >= max_messages: ctx.logger.info("reach max messages {}".format(max_messages)) break elif not reader.data_available(): if not continuous: ctx.logger.info("no data is available") break else: time.sleep(0.1)