def _proto_table_storage_impl(self, request):
self._logger.info("Getting request of proto table storage read.")
read_params = dict(request.params)
if 'proto_module' in read_params:
read_params['message_type'] = ProtoUtil.infer_message_type_from_str(
message_type_str=read_params['message_type'],
modules=read_params['proto_module']
)
else:
read_params['message_type'] = ProtoUtil.infer_message_type_from_str(
message_type_str=read_params['message_type']
)
lru_key = (request.type, request.file_name)
storage = self._lru_cache_tool.get(key=lru_key)
if not storage:
self.sys_log("Did not find the storage in cache. Making a new one...")
storage = ProtoTableStorage()
storage.initialize_from_file(file_name=request.file_name)
self._lru_cache_tool.set(
key=lru_key,
value=storage
)
else:
self.sys_log("Found key in LRU cache.")
self._logger.info('Current cache size ' + str(self._lru_cache_tool.get_cur_capacity()))
read_params.pop('proto_module', None)
return storage.read(params=read_params)
def view_proto_table():
value_types = EnvUtil.get_all_schemas(
pslx_frontend_ui_app.config['schemas'])
if request.method == 'POST':
try:
proto_table_path = request.form['proto_table_path'].strip()
selected_value_type = request.form['value_type'].strip()
modules = selected_value_type.split('.')
module, value_type = '.'.join(modules[:-1]), modules[-1]
pslx_frontend_logger.info("Proto table viewer input path [" +
proto_table_path +
'] with value type [' + value_type +
'] in module name [' + module + '].')
result = FileUtil.read_proto_from_file(proto_type=ProtoTable,
file_name=proto_table_path)
value_type = ProtoUtil.infer_message_type_from_str(
message_type_str=value_type, modules=module)
proto_contents = []
result_content = dict(result.data)
for key in sorted(result_content.keys()):
proto_val = ProtoUtil.any_to_message(
message_type=value_type, any_message=result_content[key])
try:
proto_contents.append({
'key':
key,
'val':
ProtoUtil.message_to_text(proto_message=proto_val),
})
except Exception as err:
pslx_frontend_logger.error(
"Proto table viewer Parsing proto with error " +
str(err) + '.')
proto_contents.append({
'key': key,
'val': str(proto_val),
})
value_types.remove(selected_value_type)
return render_template('proto_table_viewer.html',
proto_contents=proto_contents,
value_types=value_types,
selected_value_type=selected_value_type)
except Exception as err:
pslx_frontend_logger.error(
"Got error rendering proto_table_viewer.html: " + str(err) +
'.')
return render_template('proto_table_viewer.html',
proto_contents=[],
value_types=value_types,
selected_value_type='')
else:
return render_template('proto_table_viewer.html',
proto_contents=[],
value_types=value_types,
selected_value_type='')
def get_response_and_status_impl(self, request):
proto_file = FileUtil.die_if_file_not_exist(
file_name=request.proto_file_path)
message_type = ProtoUtil.infer_message_type_from_str(
message_type_str=request.message_type,
modules=request.proto_module if request.proto_module else None)
response = ProtoViewerRPCResponse()
proto_message = FileUtil.read_proto_from_file(proto_type=message_type,
file_name=proto_file)
response.proto_content = ProtoUtil.message_to_text(
proto_message=proto_message)
file_info = FileInfo()
file_info.file_path = request.proto_file_path
file_info.file_size = FileUtil.get_file_size(file_name=proto_file)
file_info.modified_time = str(
FileUtil.get_file_modified_time(file_name=proto_file))
response.file_info.CopyFrom(file_info)
return response, Status.SUCCEEDED
def view_proto():
value_types = EnvUtil.get_all_schemas(pslx_frontend_ui_app.config['schemas'])
if request.method == 'POST':
try:
proto_file_path = request.form['proto_file_path'].strip()
selected_value_type = request.form['value_type'].strip()
modules = selected_value_type.split('.')
module, message_type_str = '.'.join(modules[:-1]), modules[-1]
pslx_frontend_logger.info("Proto viewer input path [" + proto_file_path + '] with message type [' +
message_type_str + '] in module name [' + module + '].')
message_type = ProtoUtil.infer_message_type_from_str(
message_type_str=message_type_str,
modules=module
)
proto_message = FileUtil.read_proto_from_file(
proto_type=message_type, file_name=proto_file_path)
result_ui = ProtoUtil.message_to_text(proto_message=proto_message)
return render_template(
'proto_viewer.html',
proto_content=result_ui,
value_types=value_types,
selected_value_type=selected_value_type
)
except Exception as err:
pslx_frontend_logger.error("Got error rendering proto_viewer.html: " + str(err) + '.')
return render_template(
'proto_viewer.html',
proto_content="Got error rendering proto_viewer.html: " + str(err) + '.',
value_types=value_types,
selected_value_type=''
)
else:
return render_template(
'proto_viewer.html',
proto_content="",
value_types=value_types,
selected_value_type=''
)
def test_infer_message_type_from_str(self):
message_type_str = 'NodeSnapshot'
self.assertEqual(
ProtoUtil.infer_message_type_from_str(
message_type_str=message_type_str), NodeSnapshot)
def _partitioner_storage_impl(self, request):
self._logger.info("Getting request of partitioner storage read.")
read_params = dict(request.params)
is_proto_table = True if read_params['is_proto_table'] == '1' else False
if 'base_name' in read_params:
base_name = read_params['base_name']
else:
base_name = 'data.pb' if is_proto_table else 'data'
lru_key = (read_params['PartitionerStorageType'], request.dir_name)
self._logger.info("Partitioner type is " + read_params['PartitionerStorageType'])
storage = self._lru_cache_tool.get(key=lru_key)
if not storage:
self.sys_log("Did not find the storage in cache. Making a new one...")
partitioner_type = ProtoUtil.get_value_by_name(
enum_type=PartitionerStorageType,
name=read_params['PartitionerStorageType']
)
storage = self.PARTITIONER_TYPE_TO_IMPL[partitioner_type]()
storage.initialize_from_dir(dir_name=request.dir_name)
self._lru_cache_tool.set(
key=lru_key,
value=storage
)
else:
self.sys_log("Found key in LRU cache.")
self._logger.info('Current cache size ' + str(self._lru_cache_tool.get_cur_capacity()))
read_params.pop('PartitionerStorageType', None)
read_params.pop('is_proto_table', None)
if is_proto_table:
proto_table_storage = ProtoTableStorage()
storage.set_underlying_storage(storage=proto_table_storage)
else:
read_params['num_line'] = -1
response = RPCIOResponse()
if 'start_time' not in read_params:
# calling read function
if is_proto_table:
# if underlying storage is proto table.
if 'message_type' in read_params:
assert 'proto_module' in read_params
read_params['message_type'] = ProtoUtil.infer_message_type_from_str(
message_type_str=read_params['message_type'],
modules=read_params['proto_module']
)
proto_storage = ProtoTableStorage()
if 'read_oldest' in read_params:
proto_storage.initialize_from_file(
file_name=FileUtil.join_paths_to_file(
root_dir=storage.get_oldest_dir_in_root_directory(),
base_name=base_name
)
)
else:
proto_storage.initialize_from_file(
file_name=FileUtil.join_paths_to_file(
root_dir=storage.get_latest_dir(),
base_name=base_name
)
)
data = proto_storage.read_all()
for key, val in data.items():
rpc_list_data = RPCIOResponse.RPCListData()
rpc_data = rpc_list_data.data.add()
rpc_data.proto_data.CopyFrom(val)
response.dict_data[key].CopyFrom(rpc_list_data)
else:
# if underlying storage is not proto table.
default_storage = DefaultStorage()
if 'read_oldest' in read_params:
default_storage.initialize_from_file(
file_name=FileUtil.join_paths_to_file(
root_dir=storage.get_oldest_dir_in_root_directory(),
base_name=base_name
)
)
else:
default_storage.initialize_from_file(
file_name=FileUtil.join_paths_to_file(
root_dir=storage.get_latest_dir(),
base_name=base_name
)
)
data = default_storage.read(params={
'num_line': -1,
})
rpc_list_data = RPCIOResponse.RPCListData()
for item in data:
rpc_data = rpc_list_data.data.add()
rpc_data.string_data = item
response.list_data.CopyFrom(rpc_list_data)
else:
# calling read_range function
if 'start_time' in read_params:
read_params['start_time'] = TimezoneUtil.cur_time_from_str(
time_str=read_params['start_time']
)
if 'end_time' in read_params:
read_params['end_time'] = TimezoneUtil.cur_time_from_str(
time_str=read_params['end_time']
)
data = storage.read_range(params=read_params)
if data:
for key, val in data.items():
rpc_list_data = RPCIOResponse.RPCListData()
if is_proto_table:
for proto_key, any_message in val.items():
rpc_data = rpc_list_data.data.add()
rpc_data.string_data = proto_key
rpc_data = rpc_list_data.data.add()
rpc_data.proto_data.CopyFrom(any_message)
else:
for entry in val:
rpc_data = rpc_list_data.data.add()
rpc_data.string_data = entry
response.dict_data[key].CopyFrom(rpc_list_data)
return response
def view_proto_table():
all_urls = sorted(server_urls)
if request.method == 'POST':
try:
server_url = request.form['server_url'].strip()
proto_table_path = request.form['proto_table_path'].strip()
value_type = request.form['value_type'].strip()
module = request.form['module'].strip()
pslx_frontend_logger.info("Proto table viewer selecting url [" + server_url + '] and input path [' +
proto_table_path + '] with value type [' + value_type +
'] in module name [' + module + '].')
result = client_map[server_url]['client'].view_proto(
proto_file_path=proto_table_path,
message_type="ProtoTable",
module="",
root_certificate=client_map[server_url]['root_certificate']
)
value_type = ProtoUtil.infer_message_type_from_str(
message_type_str=value_type,
modules=module
)
result_content = ProtoUtil.text_to_message(message_type=ProtoTable, text_str=result['proto_content'])
proto_contents = []
result_content = dict(result_content.data)
for key in sorted(result_content.keys()):
proto_val = ProtoUtil.any_to_message(
message_type=value_type,
any_message=result_content[key]
)
try:
proto_contents.append(
{
'key': key,
'val': ProtoUtil.message_to_text(proto_message=proto_val),
}
)
except Exception as err:
pslx_frontend_logger.error("Proto table viewer Parsing proto with error " + str(err) + '.')
proto_contents.append(
{
'key': key,
'val': str(proto_val),
}
)
all_urls.remove(server_url)
all_urls = [server_url] + all_urls
return render_template(
'proto_table_viewer.html',
proto_contents=proto_contents,
server_urls=all_urls
)
except Exception as err:
pslx_frontend_logger.error("Got error rendering proto_table_viewer.html: " + str(err) + '.')
return render_template(
'proto_table_viewer.html',
proto_contents=[],
server_urls=all_urls
)
else:
return render_template(
'proto_table_viewer.html',
proto_contents=[],
selected_server_url='',
server_urls=server_urls
)