def get_content_from_snapshot(cls, snapshot_file, message_type=OperatorContentPlain):
try:
snapshot = FileUtil.read_proto_from_file(
proto_type=OperatorSnapshot,
file_name=FileUtil.die_if_file_not_exist(file_name=snapshot_file)
)
return cls.content_deserializer(content=snapshot.content)
except FileNotExistException as _:
return message_type()
def get_status_from_snapshot(cls, snapshot_file):
try:
snapshot = FileUtil.read_proto_from_file(
proto_type=OperatorSnapshot,
file_name=FileUtil.die_if_file_not_exist(file_name=snapshot_file)
)
return snapshot.status
except FileNotExistException as _:
return Status.IDLE
def get_content_from_snapshot(cls, snapshot_file, message_type):
try:
with FileLockTool(snapshot_file, read_mode=True):
snapshot = FileUtil.read_proto_from_file(
proto_type=OperatorSnapshot,
file_name=FileUtil.die_if_file_not_exist(file_name=snapshot_file)
)
return ProtoUtil.any_to_message(message_type=message_type, any_message=snapshot.content)
except FileNotExistException as _:
return message_type()
def view_proto_table():
value_types = EnvUtil.get_all_schemas(
pslx_frontend_ui_app.config['schemas'])
if request.method == 'POST':
try:
proto_table_path = request.form['proto_table_path'].strip()
selected_value_type = request.form['value_type'].strip()
modules = selected_value_type.split('.')
module, value_type = '.'.join(modules[:-1]), modules[-1]
pslx_frontend_logger.info("Proto table viewer input path [" +
proto_table_path +
'] with value type [' + value_type +
'] in module name [' + module + '].')
result = FileUtil.read_proto_from_file(proto_type=ProtoTable,
file_name=proto_table_path)
value_type = ProtoUtil.infer_message_type_from_str(
message_type_str=value_type, modules=module)
proto_contents = []
result_content = dict(result.data)
for key in sorted(result_content.keys()):
proto_val = ProtoUtil.any_to_message(
message_type=value_type, any_message=result_content[key])
try:
proto_contents.append({
'key':
key,
'val':
ProtoUtil.message_to_text(proto_message=proto_val),
})
except Exception as err:
pslx_frontend_logger.error(
"Proto table viewer Parsing proto with error " +
str(err) + '.')
proto_contents.append({
'key': key,
'val': str(proto_val),
})
value_types.remove(selected_value_type)
return render_template('proto_table_viewer.html',
proto_contents=proto_contents,
value_types=value_types,
selected_value_type=selected_value_type)
except Exception as err:
pslx_frontend_logger.error(
"Got error rendering proto_table_viewer.html: " + str(err) +
'.')
return render_template('proto_table_viewer.html',
proto_contents=[],
value_types=value_types,
selected_value_type='')
else:
return render_template('proto_table_viewer.html',
proto_contents=[],
value_types=value_types,
selected_value_type='')
def get_response_and_status_impl(self, request):
proto_file = FileUtil.die_if_file_not_exist(
file_name=request.proto_file_path)
message_type = ProtoUtil.infer_message_type_from_str(
message_type_str=request.message_type,
modules=request.proto_module if request.proto_module else None)
response = ProtoViewerRPCResponse()
proto_message = FileUtil.read_proto_from_file(proto_type=message_type,
file_name=proto_file)
response.proto_content = ProtoUtil.message_to_text(
proto_message=proto_message)
file_info = FileInfo()
file_info.file_path = request.proto_file_path
file_info.file_size = FileUtil.get_file_size(file_name=proto_file)
file_info.modified_time = str(
FileUtil.get_file_modified_time(file_name=proto_file))
response.file_info.CopyFrom(file_info)
return response, Status.SUCCEEDED
def initialize_from_file(self, file_name):
if '.pb' not in file_name:
self.sys_log("Please use .pb extension for proto files.")
self._file_name = FileUtil.create_file_if_not_exist(file_name=file_name)
if FileUtil.is_file_empty(file_name=self._file_name):
self._table_message = ProtoTable()
else:
with FileLockTool(self._file_name, read_mode=True):
self._table_message = FileUtil.read_proto_from_file(
proto_type=ProtoTable,
file_name=self._file_name
)
if not self._table_message.table_path:
self._table_message.table_path = self._file_name
if not self._table_message.table_name:
self._table_message.table_name = FileUtil.base_name(file_name=file_name)
if not self._table_message.created_time:
self._table_message.created_time = str(TimezoneUtil.cur_time_in_pst())
def initialize_from_file(self, file_name):
if '.pb' not in file_name:
self._SYS_LOGGER.warning(
"Please use .pb extension for proto files.")
self._file_name = FileUtil.normalize_file_name(file_name=file_name)
self._table_message = FileUtil.read_proto_from_file(
proto_type=ProtoTable, file_name=self._file_name)
self.increment_rpc_count_by(n=1)
if self._table_message is None:
self._table_message = ProtoTable()
if not self._table_message.table_path:
self._table_message.table_path = self._file_name
if not self._table_message.table_name:
self._table_message.table_name = FileUtil.base_name(
file_name=file_name)
if not self._table_message.created_time:
self._table_message.created_time = str(
TimezoneUtil.cur_time_in_pst())
def view_proto():
value_types = EnvUtil.get_all_schemas(pslx_frontend_ui_app.config['schemas'])
if request.method == 'POST':
try:
proto_file_path = request.form['proto_file_path'].strip()
selected_value_type = request.form['value_type'].strip()
modules = selected_value_type.split('.')
module, message_type_str = '.'.join(modules[:-1]), modules[-1]
pslx_frontend_logger.info("Proto viewer input path [" + proto_file_path + '] with message type [' +
message_type_str + '] in module name [' + module + '].')
message_type = ProtoUtil.infer_message_type_from_str(
message_type_str=message_type_str,
modules=module
)
proto_message = FileUtil.read_proto_from_file(
proto_type=message_type, file_name=proto_file_path)
result_ui = ProtoUtil.message_to_text(proto_message=proto_message)
return render_template(
'proto_viewer.html',
proto_content=result_ui,
value_types=value_types,
selected_value_type=selected_value_type
)
except Exception as err:
pslx_frontend_logger.error("Got error rendering proto_viewer.html: " + str(err) + '.')
return render_template(
'proto_viewer.html',
proto_content="Got error rendering proto_viewer.html: " + str(err) + '.',
value_types=value_types,
selected_value_type=''
)
else:
return render_template(
'proto_viewer.html',
proto_content="",
value_types=value_types,
selected_value_type=''
)
def initialize_from_dir(self, dir_name):
self._dir_name = dir_name
self._index_map_file = FileUtil.join_paths_to_file(
root_dir=dir_name,
base_name='index_map.pb'
)
self._index_map_file = FileUtil.normalize_file_name(file_name=self._index_map_file)
self.increment_rpc_count_by(n=1)
self._index_map = FileUtil.read_proto_from_file(
proto_type=ProtoTableIndexMap,
file_name=self._index_map_file
)
if self._index_map is None:
self._index_map = ProtoTableIndexMap()
self._index_map.cur_shard = 0
assert self._size_per_shard > 0
self._index_map.size_per_shard = self._size_per_shard
else:
if self._size_per_shard and self._index_map.size_per_shard != self._size_per_shard:
self._logger.error("Please use the correct size per shard of [" + str(self._size_per_shard) + '].')
self._size_per_shard = self._index_map.size_per_shard
self._logger.info("Using size per shard of [" + str(self._size_per_shard) + '].')
from pslx.util.env_util import EnvUtil
from pslx.util.file_util import FileUtil
CLIENT_NAME = 'PSLX_FRONTEND_UI'
pslx_frontend_ui_app = Flask(__name__)
pslx_frontend_ui_app.config.update(SECRET_KEY=CLIENT_NAME)
pslx_frontend_logger = LoggingTool(
name=CLIENT_NAME,
ttl=EnvUtil.get_pslx_env_variable(var='PSLX_INTERNAL_TTL'))
frontend_config_file = EnvUtil.get_pslx_env_variable(
'PSLX_FRONTEND_CONFIG_PROTO_PATH')
assert frontend_config_file != ''
pslx_frontend_ui_app.config['frontend_config'] = FileUtil.read_proto_from_file(
proto_type=FrontendConfig, file_name=frontend_config_file)
pslx_frontend_ui_app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
pslx_frontend_ui_app.config['SQLALCHEMY_DATABASE_URI'] =\
'sqlite:///' + pslx_frontend_ui_app.config['frontend_config'].sqlalchemy_database_path
pslx_frontend_logger.info(
"sqlalchemy database uri " +
str(pslx_frontend_ui_app.config['SQLALCHEMY_DATABASE_URI']) + '.')
pslx_frontend_db = SQLAlchemy(pslx_frontend_ui_app)
pslx_partitioner_lru_cache = LRUCacheTool(
max_capacity=EnvUtil.get_pslx_env_variable('PSLX_INTERNAL_CACHE'))
pslx_proto_table_lru_cache = LRUCacheTool(
max_capacity=EnvUtil.get_pslx_env_variable('PSLX_INTERNAL_CACHE'))
