def get_content_from_snapshot(cls, snapshot_file, message_type=OperatorContentPlain): try: snapshot = FileUtil.read_proto_from_file( proto_type=OperatorSnapshot, file_name=FileUtil.die_if_file_not_exist(file_name=snapshot_file) ) return cls.content_deserializer(content=snapshot.content) except FileNotExistException as _: return message_type()
def get_status_from_snapshot(cls, snapshot_file): try: snapshot = FileUtil.read_proto_from_file( proto_type=OperatorSnapshot, file_name=FileUtil.die_if_file_not_exist(file_name=snapshot_file) ) return snapshot.status except FileNotExistException as _: return Status.IDLE
def get_content_from_snapshot(cls, snapshot_file, message_type): try: with FileLockTool(snapshot_file, read_mode=True): snapshot = FileUtil.read_proto_from_file( proto_type=OperatorSnapshot, file_name=FileUtil.die_if_file_not_exist(file_name=snapshot_file) ) return ProtoUtil.any_to_message(message_type=message_type, any_message=snapshot.content) except FileNotExistException as _: return message_type()
def view_proto_table(): value_types = EnvUtil.get_all_schemas( pslx_frontend_ui_app.config['schemas']) if request.method == 'POST': try: proto_table_path = request.form['proto_table_path'].strip() selected_value_type = request.form['value_type'].strip() modules = selected_value_type.split('.') module, value_type = '.'.join(modules[:-1]), modules[-1] pslx_frontend_logger.info("Proto table viewer input path [" + proto_table_path + '] with value type [' + value_type + '] in module name [' + module + '].') result = FileUtil.read_proto_from_file(proto_type=ProtoTable, file_name=proto_table_path) value_type = ProtoUtil.infer_message_type_from_str( message_type_str=value_type, modules=module) proto_contents = [] result_content = dict(result.data) for key in sorted(result_content.keys()): proto_val = ProtoUtil.any_to_message( message_type=value_type, any_message=result_content[key]) try: proto_contents.append({ 'key': key, 'val': ProtoUtil.message_to_text(proto_message=proto_val), }) except Exception as err: pslx_frontend_logger.error( "Proto table viewer Parsing proto with error " + str(err) + '.') proto_contents.append({ 'key': key, 'val': str(proto_val), }) value_types.remove(selected_value_type) return render_template('proto_table_viewer.html', proto_contents=proto_contents, value_types=value_types, selected_value_type=selected_value_type) except Exception as err: pslx_frontend_logger.error( "Got error rendering proto_table_viewer.html: " + str(err) + '.') return render_template('proto_table_viewer.html', proto_contents=[], value_types=value_types, selected_value_type='') else: return render_template('proto_table_viewer.html', proto_contents=[], value_types=value_types, selected_value_type='')
def get_response_and_status_impl(self, request): proto_file = FileUtil.die_if_file_not_exist( file_name=request.proto_file_path) message_type = ProtoUtil.infer_message_type_from_str( message_type_str=request.message_type, modules=request.proto_module if request.proto_module else None) response = ProtoViewerRPCResponse() proto_message = FileUtil.read_proto_from_file(proto_type=message_type, file_name=proto_file) response.proto_content = ProtoUtil.message_to_text( proto_message=proto_message) file_info = FileInfo() file_info.file_path = request.proto_file_path file_info.file_size = FileUtil.get_file_size(file_name=proto_file) file_info.modified_time = str( FileUtil.get_file_modified_time(file_name=proto_file)) response.file_info.CopyFrom(file_info) return response, Status.SUCCEEDED
def initialize_from_file(self, file_name): if '.pb' not in file_name: self.sys_log("Please use .pb extension for proto files.") self._file_name = FileUtil.create_file_if_not_exist(file_name=file_name) if FileUtil.is_file_empty(file_name=self._file_name): self._table_message = ProtoTable() else: with FileLockTool(self._file_name, read_mode=True): self._table_message = FileUtil.read_proto_from_file( proto_type=ProtoTable, file_name=self._file_name ) if not self._table_message.table_path: self._table_message.table_path = self._file_name if not self._table_message.table_name: self._table_message.table_name = FileUtil.base_name(file_name=file_name) if not self._table_message.created_time: self._table_message.created_time = str(TimezoneUtil.cur_time_in_pst())
def initialize_from_file(self, file_name): if '.pb' not in file_name: self._SYS_LOGGER.warning( "Please use .pb extension for proto files.") self._file_name = FileUtil.normalize_file_name(file_name=file_name) self._table_message = FileUtil.read_proto_from_file( proto_type=ProtoTable, file_name=self._file_name) self.increment_rpc_count_by(n=1) if self._table_message is None: self._table_message = ProtoTable() if not self._table_message.table_path: self._table_message.table_path = self._file_name if not self._table_message.table_name: self._table_message.table_name = FileUtil.base_name( file_name=file_name) if not self._table_message.created_time: self._table_message.created_time = str( TimezoneUtil.cur_time_in_pst())
def view_proto(): value_types = EnvUtil.get_all_schemas(pslx_frontend_ui_app.config['schemas']) if request.method == 'POST': try: proto_file_path = request.form['proto_file_path'].strip() selected_value_type = request.form['value_type'].strip() modules = selected_value_type.split('.') module, message_type_str = '.'.join(modules[:-1]), modules[-1] pslx_frontend_logger.info("Proto viewer input path [" + proto_file_path + '] with message type [' + message_type_str + '] in module name [' + module + '].') message_type = ProtoUtil.infer_message_type_from_str( message_type_str=message_type_str, modules=module ) proto_message = FileUtil.read_proto_from_file( proto_type=message_type, file_name=proto_file_path) result_ui = ProtoUtil.message_to_text(proto_message=proto_message) return render_template( 'proto_viewer.html', proto_content=result_ui, value_types=value_types, selected_value_type=selected_value_type ) except Exception as err: pslx_frontend_logger.error("Got error rendering proto_viewer.html: " + str(err) + '.') return render_template( 'proto_viewer.html', proto_content="Got error rendering proto_viewer.html: " + str(err) + '.', value_types=value_types, selected_value_type='' ) else: return render_template( 'proto_viewer.html', proto_content="", value_types=value_types, selected_value_type='' )
def initialize_from_dir(self, dir_name): self._dir_name = dir_name self._index_map_file = FileUtil.join_paths_to_file( root_dir=dir_name, base_name='index_map.pb' ) self._index_map_file = FileUtil.normalize_file_name(file_name=self._index_map_file) self.increment_rpc_count_by(n=1) self._index_map = FileUtil.read_proto_from_file( proto_type=ProtoTableIndexMap, file_name=self._index_map_file ) if self._index_map is None: self._index_map = ProtoTableIndexMap() self._index_map.cur_shard = 0 assert self._size_per_shard > 0 self._index_map.size_per_shard = self._size_per_shard else: if self._size_per_shard and self._index_map.size_per_shard != self._size_per_shard: self._logger.error("Please use the correct size per shard of [" + str(self._size_per_shard) + '].') self._size_per_shard = self._index_map.size_per_shard self._logger.info("Using size per shard of [" + str(self._size_per_shard) + '].')
from pslx.util.env_util import EnvUtil from pslx.util.file_util import FileUtil CLIENT_NAME = 'PSLX_FRONTEND_UI' pslx_frontend_ui_app = Flask(__name__) pslx_frontend_ui_app.config.update(SECRET_KEY=CLIENT_NAME) pslx_frontend_logger = LoggingTool( name=CLIENT_NAME, ttl=EnvUtil.get_pslx_env_variable(var='PSLX_INTERNAL_TTL')) frontend_config_file = EnvUtil.get_pslx_env_variable( 'PSLX_FRONTEND_CONFIG_PROTO_PATH') assert frontend_config_file != '' pslx_frontend_ui_app.config['frontend_config'] = FileUtil.read_proto_from_file( proto_type=FrontendConfig, file_name=frontend_config_file) pslx_frontend_ui_app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False pslx_frontend_ui_app.config['SQLALCHEMY_DATABASE_URI'] =\ 'sqlite:///' + pslx_frontend_ui_app.config['frontend_config'].sqlalchemy_database_path pslx_frontend_logger.info( "sqlalchemy database uri " + str(pslx_frontend_ui_app.config['SQLALCHEMY_DATABASE_URI']) + '.') pslx_frontend_db = SQLAlchemy(pslx_frontend_ui_app) pslx_partitioner_lru_cache = LRUCacheTool( max_capacity=EnvUtil.get_pslx_env_variable('PSLX_INTERNAL_CACHE')) pslx_proto_table_lru_cache = LRUCacheTool( max_capacity=EnvUtil.get_pslx_env_variable('PSLX_INTERNAL_CACHE'))