def __init__(self, locator, name="ledger_client"):
self.commander_location = pyyjj.location(pyyjj.mode.LIVE,
pyyjj.category.SYSTEM,
'service', 'ledger', locator)
io_device = pyyjj.io_device(self.commander_location)
self.cmd_sock = io_device.connect_socket(self.commander_location,
pyyjj.protocol.REQUEST, 10000)
def __init__(self, ctx):
self.ctx = ctx
self.ctx.converter = self
self.dest_id = 0 # dest_id 0 should be configurable TODO
self.location = pyyjj.location(kfj.MODES['backtest'],
kfj.CATEGORIES['md'], ctx.group,
ctx.name, ctx.locator)
self.io_device = pyyjj.io_device(self.location, False, True)
self.writer = self.io_device.open_writer(self.dest_id)
self.reader = self.io_device.open_reader_to_subscribe()
self.reader.join(self.location, self.dest_id, 0)
self.logger = create_logger("converter", ctx.log_level, self.location)
def trace_journal(ctx, session_id, io_type):
trace_df = pd.DataFrame(columns=[
'gen_time', 'trigger_time', 'source', 'dest', 'msg_type',
'frame_length', 'data_length'
])
session = find_session(ctx, session_id)
uname = '{}/{}/{}/{}'.format(session['category'], session['group'],
session['name'], session['mode'])
uid = pyyjj.hash_str_32(uname)
ctx.category = '*'
ctx.group = '*'
ctx.name = '*'
ctx.mode = '*'
locations = collect_journal_locations(ctx)
location = locations[uid]
home = make_location_from_dict(ctx, location)
io_device = pyyjj.io_device(home)
reader = io_device.open_reader_to_subscribe()
if io_type == 'out' or io_type == 'all':
for dest in location['readers']:
dest_id = int(dest, 16)
reader.join(home, dest_id, session['begin_time'])
if (io_type == 'in' or io_type == 'all'
) and not (home.category == pyyjj.category.SYSTEM
and home.group == 'master' and home.name == 'master'):
master_cmd_uid = pyyjj.hash_str_32('system/master/{:08x}/live'.format(
location['uid']))
master_cmd_location = make_location_from_dict(
ctx, locations[master_cmd_uid])
reader.join(master_cmd_location, location['uid'],
session['begin_time'])
while reader.data_available(
) and reader.current_frame().gen_time <= session['end_time']:
frame = reader.current_frame()
trace_df.loc[len(trace_df)] = [
frame.gen_time, frame.trigger_time,
locations[frame.source]['uname'],
'public' if frame.dest == 0 else locations[frame.dest]['uname'],
frame.msg_type, frame.frame_length, frame.data_length
]
if frame.dest == home.uid and (frame.msg_type == 10021
or frame.msg_type == 10022):
request = pyyjj.get_RequestReadFrom(frame)
source_location = make_location_from_dict(
ctx, locations[request.source_id])
reader.join(source_location,
location['uid'] if frame.msg_type == 10021 else 0,
request.from_time)
reader.next()
return trace_df
def ledger(ctx):
pass_ctx_from_parent(ctx)
commander_location = pyyjj.location(pyyjj.mode.LIVE, pyyjj.category.SYSTEM, 'service', 'ledger', ctx.locator)
ctx.logger = create_logger("ledger_test", ctx.log_level, commander_location)
io_device = pyyjj.io_device(commander_location)
cmd_sock = io_device.connect_socket(commander_location, pyyjj.protocol.REQUEST, 10000)
calendar.calendar_request(cmd_sock, commander_location)
# order.new_order_single(cmd_sock, commander_location, test_account)
# ctx.logger.info('done')
order.cancel_all_order_for_account(cmd_sock, commander_location, test_account)
ctx.logger.info('done')
def find_sessions(ctx):
io_device = pyyjj.io_device(ctx.journal_util_location)
ctx.session_count = 1
sessions_df = pd.DataFrame(columns=[
'id', 'mode', 'category', 'group', 'name', 'begin_time', 'end_time', 'closed', 'duration', 'frame_count'
])
locations = collect_journal_locations(ctx)
dest_pub = '{:08x}'.format(0)
for key in locations:
record = locations[key]
location = pyyjj.location(MODES[record['mode']], CATEGORIES[record['category']], record['group'], record['name'], ctx.locator)
if dest_pub in record['readers']:
reader = io_device.open_reader_to_subscribe()
for dest_id in record['readers']:
reader.join(location, int(dest_id, 16), 0)
find_sessions_from_reader(ctx, sessions_df, reader, record['mode'], record['category'], record['group'], record['name'])
return sessions_df
def __init__(self, ctx):
self.ctx = ctx
self.ctx.converter = self
self.filepath = ctx.filepath
self.from_time = int(ctx.time)
self.data = None
# dest_id 0 should be configurable TODO
self.dest_id = 0
self.location = pyyjj.location(kfj.MODES['backtest'],
kfj.CATEGORIES['md'], ctx.group,
ctx.name, ctx.locator)
self.io_device = pyyjj.io_device(self.location, False, True)
self.writer = self.io_device.open_writer(self.dest_id)
self.reader = self.io_device.open_reader_to_subscribe()
self.reader.join(self.location, self.dest_id, self.from_time)
#self.reader.join(pyyjj.location(kfj.MODES['live'], kfj.CATEGORIES['md'], 'xtp', 'xtp', ctx.locator) , 0, self.from_time)
self.logger = create_logger("converter", ctx.log_level, self.location)
self.logger.info("converter creation success!")
def ledger(ctx, protocol, msg_type):
pass_ctx_from_parent(ctx)
commander_location = pyyjj.location(pyyjj.mode.LIVE, pyyjj.category.SYSTEM,
'service', 'ledger', ctx.locator)
ctx.logger = create_logger("ledger_test", ctx.log_level,
commander_location)
io_device = pyyjj.io_device(commander_location)
cmd_sock = io_device.connect_socket(
commander_location, pyyjj.protocol.SUBSCRIBE
if protocol == 'sub' else pyyjj.protocol.REQUEST, 10000)
if protocol == "sub":
cmd_sock.setsockopt(int(pyyjj.protocol.SUBSCRIBE), 1, "")
while True:
cmd_sock.recv()
print(cmd_sock.last_message())
calendar.calendar_request(cmd_sock, commander_location)
broker.broker_state_request(cmd_sock, commander_location)
order.cancel_all_order_for_account(cmd_sock, commander_location,
test_account)
asset.asset_request(cmd_sock, commander_location)
asset.all_asset_info_request(cmd_sock, commander_location)
ctx.logger.info('done')
def find_sessions(ctx):
io_device = pyyjj.io_device(ctx.journal_util_location)
ctx.session_count = 1
sessions_df = pd.DataFrame(columns=[
'id', 'mode', 'category', 'group', 'name', 'begin_time', 'end_time',
'closed', 'duration'
])
locations = collect_journal_locations(ctx)
dest_pub = '{:08x}'.format(0)
for key in locations:
record = locations[key]
if record["mode"] == "live" and record["group"] != "master":
master_command_location = pyyjj.location(pyyjj.mode.LIVE,
pyyjj.category.SYSTEM,
"master",
'{:08x}'.format(key),
ctx.locator)
reader = io_device.open_reader_to_subscribe()
reader.join(master_command_location, key, 0)
find_sessions_from_reader(ctx, sessions_df, reader, record['mode'],
record['category'], record['group'],
record['name'])
else:
location = pyyjj.location(MODES[record['mode']],
CATEGORIES[record['category']],
record['group'], record['name'],
ctx.locator)
if dest_pub in record['readers']:
reader = io_device.open_reader_to_subscribe()
reader.join(location, int(dest_pub, 16), 0)
find_sessions_from_reader(ctx, sessions_df, reader,
record['mode'], record['category'],
record['group'], record['name'])
return sessions_df
def dev(ctx):
pass_ctx_from_root(ctx)
ctx.dev_location = pyyjj.location(pyyjj.mode.LIVE, pyyjj.category.SYSTEM, 'dev', 'dev', ctx.locator)
ctx.io_device = pyyjj.io_device(ctx.dev_location)
def test(ctx):
pass_ctx_from_root(ctx)
ctx.test_location = pyyjj.location(pyyjj.mode.LIVE, pyyjj.category.SYSTEM,
'test', 'test', ctx.locator)
ctx.io_device = pyyjj.io_device(ctx.test_location)
def reader(ctx, session_id, io_type, from_beginning, max_messages, msg,
continuous, output):
pass_ctx_from_parent(ctx)
session = kfj.find_session(ctx, session_id)
uname = '{}/{}/{}/{}'.format(session['category'], session['group'],
session['name'], session['mode'])
uid = pyyjj.hash_str_32(uname)
ctx.category = '*'
ctx.group = '*'
ctx.name = '*'
ctx.mode = '*'
locations = kfj.collect_journal_locations(ctx)
location = locations[uid]
home = kfj.make_location_from_dict(ctx, location)
io_device = pyyjj.io_device(home)
reader = io_device.open_reader_to_subscribe()
if io_type == 'out' or io_type == 'all':
for dest in location['readers']:
dest_id = int(dest, 16)
reader.join(home, dest_id, session['begin_time'])
if (io_type == 'in' or io_type == 'all'
) and not (home.category == pyyjj.category.SYSTEM
and home.group == 'master' and home.name == 'master'):
master_home_uid = pyyjj.hash_str_32('system/master/master/live')
master_home_location = kfj.make_location_from_dict(
ctx, locations[master_home_uid])
reader.join(master_home_location, 0, session['begin_time'])
master_cmd_uid = pyyjj.hash_str_32('system/master/{:08x}/live'.format(
location['uid']))
master_cmd_location = kfj.make_location_from_dict(
ctx, locations[master_cmd_uid])
reader.join(master_cmd_location, location['uid'],
session['begin_time'])
start_time = pyyjj.now_in_nano(
) if not from_beginning else session["begin_time"]
msg_count = 0
if output:
if msg == "all":
raise ValueError(
"invalid msg {}, please choose from ('quote', 'order', 'trade')"
.format(msg))
msg_type = wc_utils.get_msg_type(msg)
fieldnames = wc_utils.get_csv_header(msg_type)
csv_writer = csv.DictWriter(
open(output, "w"), fieldnames=wc_utils.get_csv_header(msg_type))
csv_writer.writeheader()
pp = pprint.PrettyPrinter(indent=4)
while True:
if reader.data_available() and msg_count < max_messages:
frame = reader.current_frame()
if frame.dest == home.uid and (
frame.msg_type == yjj_msg.RequestReadFrom
or frame.msg_type == yjj_msg.RequestReadFromPublic):
request = pyyjj.get_RequestReadFrom(frame)
source_location = kfj.make_location_from_dict(
ctx, locations[request.source_id])
reader.join(
source_location, location['uid'] if frame.msg_type
== yjj_msg.RequestReadFrom else 0, request.from_time)
if frame.dest == home.uid and frame.msg_type == yjj_msg.Deregister:
loc = json.loads(frame.data_as_string())
reader.disjoin(loc['uid'])
if frame.msg_type == yjj_msg.SessionEnd:
ctx.logger.info("session reach end at %s",
kft.strftime(frame.gen_time))
break
elif frame.gen_time >= start_time and (
msg == "all"
or wc_utils.get_msg_type(msg) == frame.msg_type):
dict_row = wc_utils.flatten_json(
wc_utils.object_as_dict(frame.data))
if output:
csv_writer.writerow(dict_row)
else:
pp.pprint(dict_row)
msg_count += 1
reader.next()
elif msg_count >= max_messages:
ctx.logger.info("reach max messages {}".format(max_messages))
break
elif not reader.data_available():
if not continuous:
ctx.logger.info("no data is available")
break
else:
time.sleep(0.1)
def reader(ctx, session_id, io_type, from_beginning, max_messages, msg,
continuous, output, script):
pass_ctx_from_parent(ctx)
session = kfj.find_session(ctx, session_id)
uname = '{}/{}/{}/{}'.format(session['category'], session['group'],
session['name'], session['mode'])
uid = pyyjj.hash_str_32(uname)
ctx.category = '*'
ctx.group = '*'
ctx.name = '*'
ctx.mode = '*'
locations = kfj.collect_journal_locations(ctx)
location = locations[uid]
home = kfj.make_location_from_dict(ctx, location)
io_device = pyyjj.io_device(home)
reader = io_device.open_reader_to_subscribe()
if io_type == 'out' or io_type == 'all':
for dest in location['readers']:
dest_id = int(dest, 16)
reader.join(home, dest_id, session['begin_time'])
if (io_type == 'in' or io_type == 'all'
) and not (home.category == pyyjj.category.SYSTEM
and home.group == 'master' and home.name == 'master'):
master_home_uid = pyyjj.hash_str_32('system/master/master/live')
master_home_location = kfj.make_location_from_dict(
ctx, locations[master_home_uid])
reader.join(master_home_location, 0, session['begin_time'])
master_cmd_uid = pyyjj.hash_str_32('system/master/{:08x}/live'.format(
location['uid']))
master_cmd_location = kfj.make_location_from_dict(
ctx, locations[master_cmd_uid])
reader.join(master_cmd_location, location['uid'],
session['begin_time'])
start_time = pyyjj.now_in_nano(
) if not from_beginning else session["begin_time"]
msg_count = 0
msg_type_to_read = None if msg == "all" else kungfu.msg.Registry.meta_from_name(
msg)["id"]
if output:
if msg not in kungfu.msg.Registry.type_names():
raise ValueError("invalid msg {}, please choose from {}".format(
kungfu.msg.Registry.type_names()))
csv_writer = None
def handle(frame):
data_as_dict = frame["data"]
dict_row = kungfu.msg.utils.flatten_json(data_as_dict)
nonlocal csv_writer
if not csv_writer:
csv_writer = csv.DictWriter(open(output, "w"),
fieldnames=dict_row.keys())
csv_writer.writeheader()
csv_writer.writerow(dict_row)
frame_handler = handle
elif script:
dir = os.path.dirname(script)
name_no_ext = os.path.split(os.path.basename(script))
sys.path.append(os.path.relpath(dir))
impl = importlib.import_module(os.path.splitext(name_no_ext[1])[0])
frame_handler = getattr(impl, 'on_frame', lambda frame: None)
else:
pp = pprint.PrettyPrinter(indent=4)
frame_handler = pp.pprint
while True:
if reader.data_available() and msg_count < max_messages:
frame = reader.current_frame()
if frame.dest == home.uid and (
frame.msg_type == yjj_msg.RequestReadFrom
or frame.msg_type == yjj_msg.RequestReadFromPublic):
request = pyyjj.get_RequestReadFrom(frame)
source_location = kfj.make_location_from_dict(
ctx, locations[request.source_id])
reader.join(
source_location, location['uid'] if frame.msg_type
== yjj_msg.RequestReadFrom else 0, request.from_time)
if frame.dest == home.uid and frame.msg_type == yjj_msg.Deregister:
loc = json.loads(frame.data_as_string())
reader.disjoin(loc['uid'])
if frame.msg_type == yjj_msg.SessionEnd:
ctx.logger.info("session reach end at %s",
kft.strftime(frame.gen_time))
break
elif frame.gen_time >= start_time and (
msg == "all" or msg_type_to_read == frame.msg_type):
try:
frame_handler(frame.as_dict())
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
ctx.logger.error(
'error [%s] %s', exc_type,
traceback.format_exception(exc_type, exc_obj, exc_tb))
msg_count += 1
reader.next()
elif msg_count >= max_messages:
ctx.logger.info("reach max messages {}".format(max_messages))
break
elif not reader.data_available():
if not continuous:
ctx.logger.info("no data is available")
break
else:
time.sleep(0.1)
