def _proto_table_storage_impl(self, request): self._logger.info("Getting request of proto table storage read.") read_params = dict(request.params) if 'proto_module' in read_params: read_params['message_type'] = ProtoUtil.infer_message_type_from_str( message_type_str=read_params['message_type'], modules=read_params['proto_module'] ) else: read_params['message_type'] = ProtoUtil.infer_message_type_from_str( message_type_str=read_params['message_type'] ) lru_key = (request.type, request.file_name) storage = self._lru_cache_tool.get(key=lru_key) if not storage: self.sys_log("Did not find the storage in cache. Making a new one...") storage = ProtoTableStorage() storage.initialize_from_file(file_name=request.file_name) self._lru_cache_tool.set( key=lru_key, value=storage ) else: self.sys_log("Found key in LRU cache.") self._logger.info('Current cache size ' + str(self._lru_cache_tool.get_cur_capacity())) read_params.pop('proto_module', None) return storage.read(params=read_params)
def view_proto_table(): value_types = EnvUtil.get_all_schemas( pslx_frontend_ui_app.config['schemas']) if request.method == 'POST': try: proto_table_path = request.form['proto_table_path'].strip() selected_value_type = request.form['value_type'].strip() modules = selected_value_type.split('.') module, value_type = '.'.join(modules[:-1]), modules[-1] pslx_frontend_logger.info("Proto table viewer input path [" + proto_table_path + '] with value type [' + value_type + '] in module name [' + module + '].') result = FileUtil.read_proto_from_file(proto_type=ProtoTable, file_name=proto_table_path) value_type = ProtoUtil.infer_message_type_from_str( message_type_str=value_type, modules=module) proto_contents = [] result_content = dict(result.data) for key in sorted(result_content.keys()): proto_val = ProtoUtil.any_to_message( message_type=value_type, any_message=result_content[key]) try: proto_contents.append({ 'key': key, 'val': ProtoUtil.message_to_text(proto_message=proto_val), }) except Exception as err: pslx_frontend_logger.error( "Proto table viewer Parsing proto with error " + str(err) + '.') proto_contents.append({ 'key': key, 'val': str(proto_val), }) value_types.remove(selected_value_type) return render_template('proto_table_viewer.html', proto_contents=proto_contents, value_types=value_types, selected_value_type=selected_value_type) except Exception as err: pslx_frontend_logger.error( "Got error rendering proto_table_viewer.html: " + str(err) + '.') return render_template('proto_table_viewer.html', proto_contents=[], value_types=value_types, selected_value_type='') else: return render_template('proto_table_viewer.html', proto_contents=[], value_types=value_types, selected_value_type='')
def get_response_and_status_impl(self, request): proto_file = FileUtil.die_if_file_not_exist( file_name=request.proto_file_path) message_type = ProtoUtil.infer_message_type_from_str( message_type_str=request.message_type, modules=request.proto_module if request.proto_module else None) response = ProtoViewerRPCResponse() proto_message = FileUtil.read_proto_from_file(proto_type=message_type, file_name=proto_file) response.proto_content = ProtoUtil.message_to_text( proto_message=proto_message) file_info = FileInfo() file_info.file_path = request.proto_file_path file_info.file_size = FileUtil.get_file_size(file_name=proto_file) file_info.modified_time = str( FileUtil.get_file_modified_time(file_name=proto_file)) response.file_info.CopyFrom(file_info) return response, Status.SUCCEEDED
def view_proto(): value_types = EnvUtil.get_all_schemas(pslx_frontend_ui_app.config['schemas']) if request.method == 'POST': try: proto_file_path = request.form['proto_file_path'].strip() selected_value_type = request.form['value_type'].strip() modules = selected_value_type.split('.') module, message_type_str = '.'.join(modules[:-1]), modules[-1] pslx_frontend_logger.info("Proto viewer input path [" + proto_file_path + '] with message type [' + message_type_str + '] in module name [' + module + '].') message_type = ProtoUtil.infer_message_type_from_str( message_type_str=message_type_str, modules=module ) proto_message = FileUtil.read_proto_from_file( proto_type=message_type, file_name=proto_file_path) result_ui = ProtoUtil.message_to_text(proto_message=proto_message) return render_template( 'proto_viewer.html', proto_content=result_ui, value_types=value_types, selected_value_type=selected_value_type ) except Exception as err: pslx_frontend_logger.error("Got error rendering proto_viewer.html: " + str(err) + '.') return render_template( 'proto_viewer.html', proto_content="Got error rendering proto_viewer.html: " + str(err) + '.', value_types=value_types, selected_value_type='' ) else: return render_template( 'proto_viewer.html', proto_content="", value_types=value_types, selected_value_type='' )
def test_infer_message_type_from_str(self): message_type_str = 'NodeSnapshot' self.assertEqual( ProtoUtil.infer_message_type_from_str( message_type_str=message_type_str), NodeSnapshot)
def _partitioner_storage_impl(self, request): self._logger.info("Getting request of partitioner storage read.") read_params = dict(request.params) is_proto_table = True if read_params['is_proto_table'] == '1' else False if 'base_name' in read_params: base_name = read_params['base_name'] else: base_name = 'data.pb' if is_proto_table else 'data' lru_key = (read_params['PartitionerStorageType'], request.dir_name) self._logger.info("Partitioner type is " + read_params['PartitionerStorageType']) storage = self._lru_cache_tool.get(key=lru_key) if not storage: self.sys_log("Did not find the storage in cache. Making a new one...") partitioner_type = ProtoUtil.get_value_by_name( enum_type=PartitionerStorageType, name=read_params['PartitionerStorageType'] ) storage = self.PARTITIONER_TYPE_TO_IMPL[partitioner_type]() storage.initialize_from_dir(dir_name=request.dir_name) self._lru_cache_tool.set( key=lru_key, value=storage ) else: self.sys_log("Found key in LRU cache.") self._logger.info('Current cache size ' + str(self._lru_cache_tool.get_cur_capacity())) read_params.pop('PartitionerStorageType', None) read_params.pop('is_proto_table', None) if is_proto_table: proto_table_storage = ProtoTableStorage() storage.set_underlying_storage(storage=proto_table_storage) else: read_params['num_line'] = -1 response = RPCIOResponse() if 'start_time' not in read_params: # calling read function if is_proto_table: # if underlying storage is proto table. if 'message_type' in read_params: assert 'proto_module' in read_params read_params['message_type'] = ProtoUtil.infer_message_type_from_str( message_type_str=read_params['message_type'], modules=read_params['proto_module'] ) proto_storage = ProtoTableStorage() if 'read_oldest' in read_params: proto_storage.initialize_from_file( file_name=FileUtil.join_paths_to_file( root_dir=storage.get_oldest_dir_in_root_directory(), base_name=base_name ) ) else: proto_storage.initialize_from_file( file_name=FileUtil.join_paths_to_file( root_dir=storage.get_latest_dir(), base_name=base_name ) ) data = proto_storage.read_all() for key, val in data.items(): rpc_list_data = RPCIOResponse.RPCListData() rpc_data = rpc_list_data.data.add() rpc_data.proto_data.CopyFrom(val) response.dict_data[key].CopyFrom(rpc_list_data) else: # if underlying storage is not proto table. default_storage = DefaultStorage() if 'read_oldest' in read_params: default_storage.initialize_from_file( file_name=FileUtil.join_paths_to_file( root_dir=storage.get_oldest_dir_in_root_directory(), base_name=base_name ) ) else: default_storage.initialize_from_file( file_name=FileUtil.join_paths_to_file( root_dir=storage.get_latest_dir(), base_name=base_name ) ) data = default_storage.read(params={ 'num_line': -1, }) rpc_list_data = RPCIOResponse.RPCListData() for item in data: rpc_data = rpc_list_data.data.add() rpc_data.string_data = item response.list_data.CopyFrom(rpc_list_data) else: # calling read_range function if 'start_time' in read_params: read_params['start_time'] = TimezoneUtil.cur_time_from_str( time_str=read_params['start_time'] ) if 'end_time' in read_params: read_params['end_time'] = TimezoneUtil.cur_time_from_str( time_str=read_params['end_time'] ) data = storage.read_range(params=read_params) if data: for key, val in data.items(): rpc_list_data = RPCIOResponse.RPCListData() if is_proto_table: for proto_key, any_message in val.items(): rpc_data = rpc_list_data.data.add() rpc_data.string_data = proto_key rpc_data = rpc_list_data.data.add() rpc_data.proto_data.CopyFrom(any_message) else: for entry in val: rpc_data = rpc_list_data.data.add() rpc_data.string_data = entry response.dict_data[key].CopyFrom(rpc_list_data) return response
def view_proto_table(): all_urls = sorted(server_urls) if request.method == 'POST': try: server_url = request.form['server_url'].strip() proto_table_path = request.form['proto_table_path'].strip() value_type = request.form['value_type'].strip() module = request.form['module'].strip() pslx_frontend_logger.info("Proto table viewer selecting url [" + server_url + '] and input path [' + proto_table_path + '] with value type [' + value_type + '] in module name [' + module + '].') result = client_map[server_url]['client'].view_proto( proto_file_path=proto_table_path, message_type="ProtoTable", module="", root_certificate=client_map[server_url]['root_certificate'] ) value_type = ProtoUtil.infer_message_type_from_str( message_type_str=value_type, modules=module ) result_content = ProtoUtil.text_to_message(message_type=ProtoTable, text_str=result['proto_content']) proto_contents = [] result_content = dict(result_content.data) for key in sorted(result_content.keys()): proto_val = ProtoUtil.any_to_message( message_type=value_type, any_message=result_content[key] ) try: proto_contents.append( { 'key': key, 'val': ProtoUtil.message_to_text(proto_message=proto_val), } ) except Exception as err: pslx_frontend_logger.error("Proto table viewer Parsing proto with error " + str(err) + '.') proto_contents.append( { 'key': key, 'val': str(proto_val), } ) all_urls.remove(server_url) all_urls = [server_url] + all_urls return render_template( 'proto_table_viewer.html', proto_contents=proto_contents, server_urls=all_urls ) except Exception as err: pslx_frontend_logger.error("Got error rendering proto_table_viewer.html: " + str(err) + '.') return render_template( 'proto_table_viewer.html', proto_contents=[], server_urls=all_urls ) else: return render_template( 'proto_table_viewer.html', proto_contents=[], selected_server_url='', server_urls=server_urls )