def __init__(self, rpc_storage):
super().__init__(service_name=self.get_class_name(),
rpc_storage=rpc_storage)
self._logger = LoggingTool(
name='PSLX_CONTAINER_BACKEND_RPC',
ttl=EnvUtil.get_pslx_env_variable(var='PSLX_INTERNAL_TTL'))
self._lru_cache_tool = LRUCacheTool(
max_capacity=EnvUtil.get_pslx_env_variable(
var='PSLX_INTERNAL_CACHE'))
self._backend_folder = FileUtil.join_paths_to_dir(
root_dir=EnvUtil.get_pslx_env_variable('PSLX_DATABASE'),
base_name='PSLX_CONTAINER_BACKEND_TABLE')
def __init__(self, rpc_storage):
super().__init__(service_name=self.get_class_name(), rpc_storage=rpc_storage)
self._lru_cache_tool = LRUCacheTool(
max_capacity=EnvUtil.get_pslx_env_variable(var='PSLX_INTERNAL_CACHE')
)
self._storage_type_to_impl_func = {
StorageType.DEFAULT_STORAGE: self._default_storage_impl,
StorageType.FIXED_SIZE_STORAGE: self._fixed_size_storage_impl,
StorageType.PROTO_TABLE_STORAGE: self._proto_table_storage_impl,
StorageType.PARTITIONER_STORAGE: self._partitioner_storage_impl,
}
self._logger = LoggingTool(
name='PSLX_RPC_IO_RPC',
ttl=EnvUtil.get_pslx_env_variable(var='PSLX_INTERNAL_TTL')
)
def __init__(self, rpc_storage):
super().__init__(service_name=self.get_class_name(),
rpc_storage=rpc_storage)
self._logger = glogging.get_logger(
log_name='PSLX_CONTAINER_BACKEND_RPC',
log_dir=EnvUtil.get_pslx_env_variable(var='PSLX_DEFAULT_LOG_DIR') +
'PSLX_INTERNAL/container_backend_rpc')
self._lru_cache_tool = LRUCacheTool(max_capacity=int(
EnvUtil.get_pslx_env_variable(var='PSLX_INTERNAL_CACHE')))
self._backend_folder = FileUtil.join_paths_to_dir(
root_dir=EnvUtil.get_pslx_env_variable(
'PSLX_INTERNAL_METADATA_DIR'),
base_name='PSLX_CONTAINER_BACKEND_TABLE')
ttl=EnvUtil.get_pslx_env_variable(var='PSLX_INTERNAL_TTL'))
frontend_config_file = EnvUtil.get_pslx_env_variable(
'PSLX_FRONTEND_CONFIG_PROTO_PATH')
assert frontend_config_file != ''
pslx_frontend_ui_app.config['frontend_config'] = FileUtil.read_proto_from_file(
proto_type=FrontendConfig, file_name=frontend_config_file)
pslx_frontend_ui_app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
pslx_frontend_ui_app.config['SQLALCHEMY_DATABASE_URI'] =\
'sqlite:///' + pslx_frontend_ui_app.config['frontend_config'].sqlalchemy_database_path
pslx_frontend_logger.info(
"sqlalchemy database uri " +
str(pslx_frontend_ui_app.config['SQLALCHEMY_DATABASE_URI']) + '.')
pslx_frontend_db = SQLAlchemy(pslx_frontend_ui_app)
pslx_partitioner_lru_cache = LRUCacheTool(
max_capacity=EnvUtil.get_pslx_env_variable('PSLX_INTERNAL_CACHE'))
pslx_proto_table_lru_cache = LRUCacheTool(
max_capacity=EnvUtil.get_pslx_env_variable('PSLX_INTERNAL_CACHE'))
pslx_dedicated_logging_storage_path = FileUtil.join_paths_to_dir_with_mode(
root_dir=EnvUtil.get_pslx_env_variable('PSLX_DATABASE') +
'/PSLX_DEDICATED_LOGGING',
base_name='dedicated_logging.pb')
from pslx.micro_service.frontend.renderer import index_renderer, file_viewer_renderer, proto_viewer_renderer, \
container_backend_renderer, proto_table_viewer_renderer, logging_renderer
def test_add(self):
lru_cache_tool = LRUCacheTool(max_capacity=2)
lru_cache_tool.set(key='key_1', value='value_1')
self.assertEqual(lru_cache_tool.get_cur_capacity(), 1)
lru_cache_tool.set(key='key_2', value='value_2')
self.assertEqual(lru_cache_tool.get_cur_capacity(), 2)
lru_cache_tool.set(key='key_3', value='value_3')
self.assertEqual(lru_cache_tool.get_cur_capacity(), 2)
def test_get(self):
lru_cache_tool = LRUCacheTool(max_capacity=2)
lru_cache_tool.set(key='key_1', value='value_1')
self.assertEqual(lru_cache_tool.get(key='key_1'), 'value_1')
lru_cache_tool.set(key='key_2', value='value_2')
lru_cache_tool.set(key='key_3', value='value_3')
self.assertEqual(lru_cache_tool.get(key='key_3'), 'value_3')
self.assertEqual(lru_cache_tool.get(key='key_1'), None)
lru_cache_tool.get(key='key_2')
lru_cache_tool.set(key='key_4', value='value_4')
self.assertEqual(lru_cache_tool.get(key='key_3'), None)
class ContainerBackendRPC(RPCBase):
REQUEST_MESSAGE_TYPE = ContainerSnapshot
def __init__(self, rpc_storage):
super().__init__(service_name=self.get_class_name(),
rpc_storage=rpc_storage)
self._logger = LoggingTool(
name='PSLX_CONTAINER_BACKEND_RPC',
ttl=EnvUtil.get_pslx_env_variable(var='PSLX_INTERNAL_TTL'))
self._lru_cache_tool = LRUCacheTool(
max_capacity=EnvUtil.get_pslx_env_variable(
var='PSLX_INTERNAL_CACHE'))
self._backend_folder = FileUtil.join_paths_to_dir(
root_dir=EnvUtil.get_pslx_env_variable('PSLX_DATABASE'),
base_name='PSLX_CONTAINER_BACKEND_TABLE')
def get_response_and_status_impl(self, request):
storage_value = ContainerBackendValue()
storage_value.container_name = request.container_name
storage_value.container_status = request.status
for operator_name, operator_snapshot in dict(
request.operator_snapshot_map).items():
operator_info = ContainerBackendValue.OperatorInfo()
operator_info.status = operator_snapshot.status
for parent in operator_snapshot.node_snapshot.parents_names:
operator_info.parents.append(parent)
operator_info.start_time = operator_snapshot.start_time
operator_info.end_time = operator_snapshot.end_time
storage_value.operator_info_map[operator_name].CopyFrom(
operator_info)
storage_value.mode = request.mode
storage_value.data_model = request.data_model
storage_value.updated_time = str(TimezoneUtil.cur_time_in_pst())
storage_value.start_time = request.start_time
storage_value.end_time = request.end_time
storage_value.log_dir = request.log_dir
for key in request.counters:
storage_value.counters[key] = request.counters[key]
partitioner_dir = FileUtil.join_paths_to_dir_with_mode(
root_dir=FileUtil.join_paths_to_dir(
root_dir=self._backend_folder,
base_name=ProtoUtil.get_name_by_value(
enum_type=DataModelType, value=storage_value.data_model)),
base_name=storage_value.container_name,
ttl=EnvUtil.get_pslx_env_variable('PSLX_INTERNAL_TTL'))
if storage_value.mode == ModeType.TEST:
partitioner_dir = partitioner_dir.replace('PROD', 'TEST')
storage = self._lru_cache_tool.get(key=partitioner_dir)
if not storage:
self.sys_log(
"Did not find the storage in cache. Making a new one...")
storage = DailyPartitionerStorage()
proto_table = ProtoTableStorage()
storage.set_underlying_storage(storage=proto_table)
storage.initialize_from_dir(dir_name=partitioner_dir)
self._lru_cache_tool.set(key=partitioner_dir, value=storage)
else:
self.sys_log("Found key in LRU cache.")
storage.write(data={storage_value.container_name: storage_value},
params={
'overwrite': True,
'make_partition': True,
})
return None, Status.SUCCEEDED
class RPCIO(RPCBase):
REQUEST_MESSAGE_TYPE = RPCIORequest
PARTITIONER_TYPE_TO_IMPL = {
PartitionerStorageType.YEARLY: partitioner.YearlyPartitionerStorage,
PartitionerStorageType.MONTHLY: partitioner.MonthlyPartitionerStorage,
PartitionerStorageType.DAILY: partitioner.DailyPartitionerStorage,
PartitionerStorageType.HOURLY: partitioner.HourlyPartitionerStorage,
PartitionerStorageType.MINUTELY: partitioner.MinutelyPartitionerStorage,
}
def __init__(self, rpc_storage):
super().__init__(service_name=self.get_class_name(), rpc_storage=rpc_storage)
self._lru_cache_tool = LRUCacheTool(
max_capacity=EnvUtil.get_pslx_env_variable(var='PSLX_INTERNAL_CACHE')
)
self._storage_type_to_impl_func = {
StorageType.DEFAULT_STORAGE: self._default_storage_impl,
StorageType.FIXED_SIZE_STORAGE: self._fixed_size_storage_impl,
StorageType.PROTO_TABLE_STORAGE: self._proto_table_storage_impl,
StorageType.PARTITIONER_STORAGE: self._partitioner_storage_impl,
}
self._logger = LoggingTool(
name='PSLX_RPC_IO_RPC',
ttl=EnvUtil.get_pslx_env_variable(var='PSLX_INTERNAL_TTL')
)
def _default_storage_impl(self, request):
self._logger.info("Getting request of default storage read.")
read_params = dict(request.params)
if 'num_line' in read_params:
read_params['num_line'] = int(read_params['num_line'])
lru_key = (request.type, request.file_name)
storage = self._lru_cache_tool.get(key=lru_key)
if not storage:
self.sys_log("Did not find the storage in cache. Making a new one...")
storage = DefaultStorage()
storage.initialize_from_file(file_name=request.file_name)
self._lru_cache_tool.set(
key=lru_key,
value=storage
)
else:
self.sys_log("Found key in LRU cache.")
self._logger.info('Current cache size ' + str(self._lru_cache_tool.get_cur_capacity()))
response = RPCIOResponse()
data = storage.read(params=read_params)
rpc_list_data = RPCIOResponse.RPCListData()
for item in data:
rpc_data = rpc_list_data.data.add()
rpc_data.string_data = item
response.list_data.CopyFrom(rpc_list_data)
return response
def _fixed_size_storage_impl(self, request):
self._logger.info("Getting request of fixed size storage read.")
read_params = dict(request.params)
if 'force_load' in read_params:
read_params['force_load'] = ast.literal_eval(read_params['force_load'])
if 'num_line' in read_params:
read_params['num_line'] = int(read_params['num_line'])
lru_key = (request.type, request.file_name)
if 'fixed_size' in read_params:
lru_key += (read_params['fixed_size'],)
storage = self._lru_cache_tool.get(key=lru_key)
if not storage:
self.sys_log("Did not find the storage in cache. Making a new one...")
if 'fixed_size' in read_params:
storage = FixedSizeStorage(fixed_size=int(read_params['fixed_size']))
else:
storage = FixedSizeStorage()
storage.initialize_from_file(file_name=request.file_name)
self._lru_cache_tool.set(
key=lru_key,
value=storage
)
else:
self.sys_log("Found key in LRU cache.")
self._logger.info('Current cache size ' + str(self._lru_cache_tool.get_cur_capacity()))
read_params.pop('fixed_size', None)
response = RPCIOResponse()
data = storage.read(params=read_params)
rpc_list_data = RPCIOResponse.RPCListData()
for item in data:
rpc_data = rpc_list_data.data.add()
rpc_data.string_data = item
response.list_data.CopyFrom(rpc_list_data)
return response
def _proto_table_storage_impl(self, request):
self._logger.info("Getting request of proto table storage read.")
read_params = dict(request.params)
if 'proto_module' in read_params:
read_params['message_type'] = ProtoUtil.infer_message_type_from_str(
message_type_str=read_params['message_type'],
modules=read_params['proto_module']
)
else:
read_params['message_type'] = ProtoUtil.infer_message_type_from_str(
message_type_str=read_params['message_type']
)
lru_key = (request.type, request.file_name)
storage = self._lru_cache_tool.get(key=lru_key)
if not storage:
self.sys_log("Did not find the storage in cache. Making a new one...")
storage = ProtoTableStorage()
storage.initialize_from_file(file_name=request.file_name)
self._lru_cache_tool.set(
key=lru_key,
value=storage
)
else:
self.sys_log("Found key in LRU cache.")
self._logger.info('Current cache size ' + str(self._lru_cache_tool.get_cur_capacity()))
read_params.pop('proto_module', None)
return storage.read(params=read_params)
def _partitioner_storage_impl(self, request):
self._logger.info("Getting request of partitioner storage read.")
read_params = dict(request.params)
is_proto_table = True if read_params['is_proto_table'] == '1' else False
if 'base_name' in read_params:
base_name = read_params['base_name']
else:
base_name = 'data.pb' if is_proto_table else 'data'
lru_key = (read_params['PartitionerStorageType'], request.dir_name)
self._logger.info("Partitioner type is " + read_params['PartitionerStorageType'])
storage = self._lru_cache_tool.get(key=lru_key)
if not storage:
self.sys_log("Did not find the storage in cache. Making a new one...")
partitioner_type = ProtoUtil.get_value_by_name(
enum_type=PartitionerStorageType,
name=read_params['PartitionerStorageType']
)
storage = self.PARTITIONER_TYPE_TO_IMPL[partitioner_type]()
storage.initialize_from_dir(dir_name=request.dir_name)
self._lru_cache_tool.set(
key=lru_key,
value=storage
)
else:
self.sys_log("Found key in LRU cache.")
self._logger.info('Current cache size ' + str(self._lru_cache_tool.get_cur_capacity()))
read_params.pop('PartitionerStorageType', None)
read_params.pop('is_proto_table', None)
if is_proto_table:
proto_table_storage = ProtoTableStorage()
storage.set_underlying_storage(storage=proto_table_storage)
else:
read_params['num_line'] = -1
response = RPCIOResponse()
if 'start_time' not in read_params:
# calling read function
if is_proto_table:
# if underlying storage is proto table.
if 'message_type' in read_params:
assert 'proto_module' in read_params
read_params['message_type'] = ProtoUtil.infer_message_type_from_str(
message_type_str=read_params['message_type'],
modules=read_params['proto_module']
)
proto_storage = ProtoTableStorage()
if 'read_oldest' in read_params:
proto_storage.initialize_from_file(
file_name=FileUtil.join_paths_to_file(
root_dir=storage.get_oldest_dir_in_root_directory(),
base_name=base_name
)
)
else:
proto_storage.initialize_from_file(
file_name=FileUtil.join_paths_to_file(
root_dir=storage.get_latest_dir(),
base_name=base_name
)
)
data = proto_storage.read_all()
for key, val in data.items():
rpc_list_data = RPCIOResponse.RPCListData()
rpc_data = rpc_list_data.data.add()
rpc_data.proto_data.CopyFrom(val)
response.dict_data[key].CopyFrom(rpc_list_data)
else:
# if underlying storage is not proto table.
default_storage = DefaultStorage()
if 'read_oldest' in read_params:
default_storage.initialize_from_file(
file_name=FileUtil.join_paths_to_file(
root_dir=storage.get_oldest_dir_in_root_directory(),
base_name=base_name
)
)
else:
default_storage.initialize_from_file(
file_name=FileUtil.join_paths_to_file(
root_dir=storage.get_latest_dir(),
base_name=base_name
)
)
data = default_storage.read(params={
'num_line': -1,
})
rpc_list_data = RPCIOResponse.RPCListData()
for item in data:
rpc_data = rpc_list_data.data.add()
rpc_data.string_data = item
response.list_data.CopyFrom(rpc_list_data)
else:
# calling read_range function
if 'start_time' in read_params:
read_params['start_time'] = TimezoneUtil.cur_time_from_str(
time_str=read_params['start_time']
)
if 'end_time' in read_params:
read_params['end_time'] = TimezoneUtil.cur_time_from_str(
time_str=read_params['end_time']
)
data = storage.read_range(params=read_params)
if data:
for key, val in data.items():
rpc_list_data = RPCIOResponse.RPCListData()
if is_proto_table:
for proto_key, any_message in val.items():
rpc_data = rpc_list_data.data.add()
rpc_data.string_data = proto_key
rpc_data = rpc_list_data.data.add()
rpc_data.proto_data.CopyFrom(any_message)
else:
for entry in val:
rpc_data = rpc_list_data.data.add()
rpc_data.string_data = entry
response.dict_data[key].CopyFrom(rpc_list_data)
return response
def get_response_and_status_impl(self, request):
if request.is_test:
return self.REQUEST_MESSAGE_TYPE(), Status.SUCCEEDED
response = self._storage_type_to_impl_func[request.type](request=request)
return response, Status.SUCCEEDED