def _proto_table_storage_impl(self, request):
self._logger.info("Getting request of proto table storage read.")
read_params = dict(request.params)
if 'proto_module' in read_params:
read_params['message_type'] = ProtoUtil.infer_message_type_from_str(
message_type_str=read_params['message_type'],
modules=read_params['proto_module']
)
else:
read_params['message_type'] = ProtoUtil.infer_message_type_from_str(
message_type_str=read_params['message_type']
)
lru_key = (request.type, request.file_name)
storage = self._lru_cache_tool.get(key=lru_key)
if not storage:
self.sys_log("Did not find the storage in cache. Making a new one...")
storage = ProtoTableStorage()
storage.initialize_from_file(file_name=request.file_name)
self._lru_cache_tool.set(
key=lru_key,
value=storage
)
else:
self.sys_log("Found key in LRU cache.")
self._logger.info('Current cache size ' + str(self._lru_cache_tool.get_cur_capacity()))
read_params.pop('proto_module', None)
return storage.read(params=read_params)
def wait_for_upstream_status(self):
unfinished_op = []
if self.DATA_MODEL != DataModelType.DEFAULT:
for parent in self.get_parents_nodes():
if parent.get_status() in [Status.IDLE, Status.WAITING, Status.RUNNING]:
self._SYS_LOGGER.info("Upstream operator [" + parent.get_node_name() + "] is still in status [" +
ProtoUtil.get_name_by_value(enum_type=Status, value=parent.get_status()) +
'].')
self._logger.info("Upstream operator [" + parent.get_node_name() + "] is still in status [" +
ProtoUtil.get_name_by_value(enum_type=Status, value=parent.get_status()) + '].')
unfinished_op.append(parent.get_node_name())
elif parent.get_status() == Status.FAILED:
self._SYS_LOGGER.info("Upstream operator [" + parent.get_node_name() + "] failed.")
# streaming mode allows failure from its dependencies.
if not self._config['allow_failure']:
self._SYS_LOGGER.error('This results in failure of all the following descendant operators.')
self._logger.error('This results in failure of all the following descendant operators.')
self.set_status(status=Status.FAILED)
unfinished_op = []
break
else:
self._SYS_LOGGER.warning("Failure is allowed in Streaming mode. The rest of operators will" +
" continue.")
self._logger.warning('This results in failure of all the following descendant operators.')
return unfinished_op
def test_write_2(self):
proto_table_storage = ProtoTableStorage()
proto_table_storage.initialize_from_file(file_name=self.TEST_DATA_4)
proto_table_storage.write(data={'test': self.EXAMPLE_PROTO_1})
proto_table_storage.write(data={'test_1': self.EXAMPLE_PROTO_2})
result_proto = proto_table_storage.read(
params={
'key': 'test_1',
'message_type': OperatorSnapshot
})
self.assertEqual(result_proto, self.EXAMPLE_PROTO_2)
proto_table_storage.write(data={'test_1': self.EXAMPLE_PROTO_3})
result_proto = proto_table_storage.read(
params={
'key': 'test_1',
'message_type': OperatorSnapshot
})
self.assertEqual(result_proto, self.EXAMPLE_PROTO_3)
result = proto_table_storage.read_all()
self.assertDictEqual(
result, {
'test': ProtoUtil.message_to_any(self.EXAMPLE_PROTO_1),
'test_1': ProtoUtil.message_to_any(self.EXAMPLE_PROTO_3),
})
self.assertEqual(proto_table_storage.get_num_entries(), 2)
gclient_ext.cp_file(self.TEST_DATA_2, self.TEST_DATA_4)
def set_status(self, status):
self._SYS_LOGGER.info(
'Container [' + self.get_container_name() + "] switching to [" +
ProtoUtil.get_name_by_value(enum_type=Status, value=status) +
"] status from [" +
ProtoUtil.get_name_by_value(enum_type=Status, value=self._status) +
'].')
self._status = status
def send_request(self, request, root_certificate=None):
generic_request = ProtoUtil.compose_generic_request(request=request)
if self.RESPONSE_MESSAGE_TYPE:
generic_request.message_type = ProtoUtil.infer_str_from_message_type(
message_type=self.RESPONSE_MESSAGE_TYPE)
self._logger.info("Client getting request of uuid [" +
generic_request.uuid + '] in client [' +
self.get_client_name() + '].')
self.sys_log("Client getting request of uuid [" +
generic_request.uuid + '] in client [' +
self.get_client_name() + '].')
try:
options = [
('grpc.max_receive_message_length',
int(
EnvUtil.get_pslx_env_variable(
var='PSLX_GRPC_MAX_MESSAGE_LENGTH'))),
('grpc.max_send_message_length',
int(
EnvUtil.get_pslx_env_variable(
var='PSLX_GRPC_MAX_MESSAGE_LENGTH'))),
]
timeout = int(
EnvUtil.get_pslx_env_variable(var='PSLX_GRPC_TIMEOUT'))
if not root_certificate:
self._logger.info("Start with insecure channel in client [" +
self.get_client_name() + '].')
with grpc.insecure_channel(self._server_url,
options=options) as channel:
stub = GenericRPCServiceStub(channel=channel)
response = stub.SendRequest(request=generic_request,
timeout=timeout)
else:
self._logger.info("Start with secure channel in client [" +
self.get_client_name() + '].')
channel_credential = grpc.ssl_channel_credentials(
root_certificate)
with grpc.secure_channel(self._server_url,
channel_credential,
options=options) as channel:
stub = GenericRPCServiceStub(channel=channel)
response = stub.SendRequest(request=generic_request,
timeout=timeout)
if not self.RESPONSE_MESSAGE_TYPE:
self.sys_log(
"Response message type unset, return None instead.")
return None
else:
return ProtoUtil.any_to_message(
message_type=self.RESPONSE_MESSAGE_TYPE,
any_message=response.response_data)
except Exception as err:
self._logger.error("send request with error " + str(err) +
' in client [' + self.get_client_name() + '].')
self.sys_log("send request with error " + str(err) +
' in client [' + self.get_client_name() + '].')
return None
def initialize_from_dir(self, dir_name):
self._SYS_LOGGER.fatal(
"Initialize_from_dir function is not implemented for storage type "
+ ProtoUtil.get_name_by_value(enum_type=StorageType,
value=self.STORAGE_TYPE) + '.')
self._logger.fatal(
"Initialize_from_dir function is not implemented for storage type "
+ ProtoUtil.get_name_by_value(enum_type=StorageType,
value=self.STORAGE_TYPE) + '.')
return
def view_proto_table():
value_types = EnvUtil.get_all_schemas(
pslx_frontend_ui_app.config['schemas'])
if request.method == 'POST':
try:
proto_table_path = request.form['proto_table_path'].strip()
selected_value_type = request.form['value_type'].strip()
modules = selected_value_type.split('.')
module, value_type = '.'.join(modules[:-1]), modules[-1]
pslx_frontend_logger.info("Proto table viewer input path [" +
proto_table_path +
'] with value type [' + value_type +
'] in module name [' + module + '].')
result = FileUtil.read_proto_from_file(proto_type=ProtoTable,
file_name=proto_table_path)
value_type = ProtoUtil.infer_message_type_from_str(
message_type_str=value_type, modules=module)
proto_contents = []
result_content = dict(result.data)
for key in sorted(result_content.keys()):
proto_val = ProtoUtil.any_to_message(
message_type=value_type, any_message=result_content[key])
try:
proto_contents.append({
'key':
key,
'val':
ProtoUtil.message_to_text(proto_message=proto_val),
})
except Exception as err:
pslx_frontend_logger.error(
"Proto table viewer Parsing proto with error " +
str(err) + '.')
proto_contents.append({
'key': key,
'val': str(proto_val),
})
value_types.remove(selected_value_type)
return render_template('proto_table_viewer.html',
proto_contents=proto_contents,
value_types=value_types,
selected_value_type=selected_value_type)
except Exception as err:
pslx_frontend_logger.error(
"Got error rendering proto_table_viewer.html: " + str(err) +
'.')
return render_template('proto_table_viewer.html',
proto_contents=[],
value_types=value_types,
selected_value_type='')
else:
return render_template('proto_table_viewer.html',
proto_contents=[],
value_types=value_types,
selected_value_type='')
def get_containers_info():
containers_info = []
existing_containers = {}
all_proto_files = set()
if not FileUtil.is_local_path(backend_folder):
all_cells = ['']
else:
all_cells = gclient.list_cells()
for cell_name in all_cells:
folder = FileUtil.convert_local_to_cell_path(
path=backend_folder, cell=cell_name)
proto_files = FileUtil.list_files_in_dir(folder)
all_proto_files = all_proto_files.union(set(proto_files))
for proto_file in all_proto_files:
storage = ProtoTableStorage()
storage.initialize_from_file(
file_name=proto_file
)
raw_data = storage.read_all()
if not raw_data:
continue
key = sorted(raw_data.keys())[-1]
val = raw_data[key]
result_proto = ProtoUtil.any_to_message(
message_type=ContainerBackendValue,
any_message=val
)
ttl = result_proto.ttl
if ttl > 0 and result_proto.updated_time and TimezoneUtil.cur_time_in_pst() - TimezoneUtil.cur_time_from_str(
result_proto.updated_time) >= datetime.timedelta(days=ttl):
FileUtil.remove_file(storage.get_file_name())
else:
container_info = {
'container_name': result_proto.container_name,
'status': ProtoUtil.get_name_by_value(
enum_type=Status, value=result_proto.container_status),
'updated_time': result_proto.updated_time,
'mode': ProtoUtil.get_name_by_value(enum_type=ModeType, value=result_proto.mode),
'data_model': ProtoUtil.get_name_by_value(
enum_type=DataModelType, value=result_proto.data_model),
'run_cell': result_proto.run_cell,
'snapshot_cell': result_proto.snapshot_cell,
}
if container_info['container_name'] not in existing_containers:
existing_containers[container_info['container_name']] = container_info['updated_time']
containers_info.append(container_info)
else:
if container_info['updated_time'] >= existing_containers[container_info['container_name']]:
containers_info.append(container_info)
return containers_info
def test_message_to_any(self):
node_snapshot = NodeSnapshot()
node_snapshot.node_name = 'test'
node_snapshot.children_names.append('children1')
node_snapshot.children_names.append('children2')
node_snapshot.parents_names.append('parent1')
node_snapshot.parents_names.append('parent2')
any_message = ProtoUtil.message_to_any(message=node_snapshot)
self.assertEqual(
ProtoUtil.any_to_message(message_type=NodeSnapshot,
any_message=any_message), node_snapshot)
def set_config(self, config):
super().set_config(config=config)
if 'override_to_prod' in self._config and self._file_name:
self._file_name = self._file_name.replace(
ProtoUtil.get_name_by_value(enum_type=ModeType,
value=ModeType.TEST),
ProtoUtil.get_name_by_value(enum_type=ModeType,
value=ModeType.PROD))
if 'override_to_test' in self._config and self._file_name:
self._file_name = self._file_name.replace(
ProtoUtil.get_name_by_value(enum_type=ModeType,
value=ModeType.PROD),
ProtoUtil.get_name_by_value(enum_type=ModeType,
value=ModeType.TEST))
def read(self,
file_or_dir_path,
params=None,
is_test=False,
root_certificate=None):
assert 'PartitionerStorageType' in params and 'start_time' not in params and 'end_time' not in params
request = RPCIORequest()
request.is_test = is_test
request.type = self.STORAGE_TYPE
request.dir_name = file_or_dir_path
if 'is_proto_table' in params and params['is_proto_table']:
params['is_proto_table'] = '1'
else:
params['is_proto_table'] = '0'
if 'message_type' in params:
request.params[
'message_type'] = ProtoUtil.infer_str_from_message_type(
message_type=params['message_type'])
if 'read_oldest' in params and params['read_oldest']:
request.params['read_oldest'] = '1'
for key, val in params.items():
if isinstance(val, str) or key in self.WHITELISTED_KEY:
request.params[key] = str(val)
request.params['PartitionerStorageType'] = ProtoUtil.get_name_by_value(
enum_type=PartitionerStorageType,
value=params['PartitionerStorageType'])
response = self.send_request(request=request,
root_certificate=root_certificate)
if response:
if params['is_proto_table'] == '1':
result = {}
for key, val in dict(response.dict_data).items():
result[key] = val.data[0].proto_data
return result
else:
return [
rpc_data.string_data
for rpc_data in response.list_data.data
]
else:
return None if params['is_proto_table'] == '1' else []
def content_serializer(self, content):
op_content = OperatorContentPlain()
try:
op_content.plan_content.bytes_val = str(content)
except Exception as err:
self._logger.warning("Seralize content with error: " + str(err) + '.')
return ProtoUtil.message_to_any(message=op_content)
def fetch_oldest(self):
try:
oldest_dir = self._partitioner.get_oldest_dir_in_root_directory()
if not oldest_dir:
self._logger.warning('[' + self._partitioner.get_dir_name() +
'] is empty.')
return None
proto_table = ProtoTableStorage()
proto_table.initialize_from_file(
file_name=FileUtil.join_paths_to_file(root_dir=oldest_dir,
base_name='data.pb'))
all_data = proto_table.read_all()
if all_data:
self._logger.info(
"Successfully get the oldest data in partition dir [" +
self._partitioner.get_dir_name() + '].')
min_key = min(all_data.keys())
return ProtoUtil.any_to_message(message_type=self.MESSAGE_TYPE,
any_message=all_data[min_key])
else:
return None
except Exception as err:
self._logger.error("Fetch oldest partition [" +
self._partitioner.get_dir_name() +
"] with error " + str(err) + '.')
return None
def __init__(self, container_name):
super().__init__(container_name)
self._logger = glogging.get_logger(
log_name=(ProtoUtil.get_name_by_value(enum_type=DataModelType,
value=self.DATA_MODEL) +
'__' + self.get_class_name() + '__' + container_name),
log_dir=EnvUtil.get_pslx_env_variable('PSLX_DEFAULT_LOG_DIR'))
def read(self,
file_or_dir_path,
params=None,
is_test=False,
root_certificate=None):
if 'message_type' in params:
assert 'proto_module' in params
request = RPCIORequest()
request.is_test = is_test
request.type = self.STORAGE_TYPE
request.file_name = file_or_dir_path
if 'message_type' in params:
self.RESPONSE_MESSAGE_TYPE = params['message_type']
request.params[
'message_type'] = ProtoUtil.infer_str_from_message_type(
message_type=params['message_type'])
else:
self.RESPONSE_MESSAGE_TYPE = Any
for key, val in params.items():
if isinstance(val, str) or key in self.WHITELISTED_KEY:
request.params[key] = str(val)
response = self.send_request(request=request,
root_certificate=root_certificate)
return response
def fetch_range(self, start_time, end_time):
try:
data = self._partitioner.read_range(params={
'start_time': start_time,
'end_time': end_time,
})
start_time, end_time = str(start_time.replace(tzinfo=None)), str(
end_time.replace(tzinfo=None))
data_content = {}
for val in data.values():
data_content.update(val)
result = {}
self._logger.info(
"Successfully get the range data in partition dir [" +
self._partitioner.get_dir_name() + '].')
for key in sorted(data_content.keys()):
if key < start_time or key > end_time:
continue
else:
item = ProtoUtil.any_to_message(
message_type=self.MESSAGE_TYPE,
any_message=data_content[key])
result[key] = item
return result
except Exception as err:
self._logger.error("Fetch range for partition [" +
self._partitioner.get_dir_name() +
"] with error " + str(err) + '.')
return {}
def __init__(self, container_name, ttl=-1):
super().__init__(container_name, ttl=ttl)
self._logger = LoggingTool(
name=(ProtoUtil.get_name_by_value(enum_type=DataModelType,
value=self.DATA_MODEL) + '__' +
self.get_class_name() + '__' + container_name),
ttl=ttl)
def read(self, params=None):
assert 'key' in params
while self._writer_status != Status.IDLE:
self.sys_log("Waiting for writer to finish.")
time.sleep(TimeSleepObj.ONE_SECOND)
while self._deleter_status != Status.IDLE:
self.sys_log("Waiting for deleter to finish.")
time.sleep(TimeSleepObj.ONE_SECOND)
self._reader_status = Status.RUNNING
if params['key'] in self._table_message.data:
try:
if 'message_type' in params:
result = ProtoUtil.any_to_message(
message_type=params['message_type'],
any_message=self._table_message.data[params['key']]
)
else:
result = self._table_message.data[params['key']]
self._reader_status = Status.IDLE
return result
except Exception as err:
self.sys_log("Read file [" + self.get_file_name() + "] got exception: " + str(err) + '.')
self._logger.error("Read file [" + self.get_file_name() + "] got exception: " + str(err) + '.')
raise StorageReadException("Read file [" + self.get_file_name() + "] got exception: " + str(err) + '.')
else:
return None
def write(self, data, params=None):
if not params:
params = {}
if 'overwrite' not in params:
params['overwrite'] = True
assert isinstance(data, dict)
try:
for key, val in data.items():
if not params['overwrite'] and key in self._table_message.data:
continue
any_message = ProtoUtil.message_to_any(message=val)
self._table_message.data[key].CopyFrom(any_message)
if len(self._table_message.data) > 1000:
self._SYS_LOGGER.warning(
"Warning: the table content is too large, considering using Partitioner "
"combined with proto table.")
self._table_message.updated_time = str(
TimezoneUtil.cur_time_in_pst())
self.increment_rpc_count_by(n=1)
FileUtil.write_proto_to_file(proto=self._table_message,
file_name=self._file_name)
except Exception as err:
self._SYS_LOGGER.error("Write to file [" + self.get_file_name() +
"] got exception: " + str(err) + '.')
self._logger.error("Write to file [" + self.get_file_name() +
"] got exception: " + str(err) + '.')
raise StorageWriteException("Write to file [" +
self.get_file_name() +
"] got exception: " + str(err) + '.')
def SendRequest(self, request, context):
self.sys_log("rpc getting request with uuid [" + request.uuid +
'] in service [' + self.get_rpc_service_name() + '].')
self._logger.info("rpc getting request with uuid [" + request.uuid +
'] in service [' + self.get_rpc_service_name() +
'].')
decomposed_request = self.request_decomposer(request=request)
response, status = self.get_response_and_status_impl(
request=decomposed_request)
generic_response = ProtoUtil.compose_generic_response(
response=response)
generic_response.request_uuid = request.uuid
generic_response.status = status
request.status = status
generic_response.message_type = request.message_type
request.message_type = ProtoUtil.infer_str_from_message_type(
message_type=self.REQUEST_MESSAGE_TYPE)
if self._rpc_storage:
request_response_pair = GenericRPCRequestResponsePair()
request_response_pair.uuid = request.uuid
request_response_pair.generic_rpc_request.CopyFrom(request)
request_response_pair.generic_rpc_response.CopyFrom(
generic_response)
self._request_response_pair[request.uuid] = request_response_pair
if len(self._request_response_pair) >= int(
EnvUtil.get_pslx_env_variable('PSLX_RPC_FLUSH_RATE')):
self._rpc_storage.write(data=self._request_response_pair,
params={
'overwrite': True,
'make_partition': True,
})
self.sys_log("Request response pairs flushed to [" +
self._rpc_storage.get_latest_dir() +
'] in service [' + self.get_rpc_service_name() +
'].')
self._logger.info("Request response pairs flushed to [" +
self._rpc_storage.get_latest_dir() +
'] in service [' +
self.get_rpc_service_name() + '].')
self._request_response_pair.clear()
self._add_request_timestamp()
return generic_response
def send_request(self, request):
self._response = None
generic_request = GenericRPCRequest()
generic_request.request_data.CopyFrom(
ProtoUtil.message_to_any(message=request))
generic_request.timestamp = str(TimezoneUtil.cur_time_in_pst())
self._corr_id = generic_request.uuid = str(uuid.uuid4())
if self.RESPONSE_MESSAGE_TYPE:
generic_request.message_type = ProtoUtil.infer_str_from_message_type(
message_type=self.RESPONSE_MESSAGE_TYPE)
self._SYS_LOGGER.info("Getting request of uuid [" +
generic_request.uuid + '] in queue [' +
self.get_queue_name() + '].')
self._logger.info("Getting request of uuid [" + generic_request.uuid +
'] in queue [' + self.get_queue_name() + '].')
try:
generic_request_str = ProtoUtil.message_to_string(
proto_message=generic_request)
self._channel.basic_publish(
exchange='',
routing_key=self._queue_name,
properties=pika.BasicProperties(reply_to=self._callback_queue,
correlation_id=self._corr_id),
body=base64.b64encode(generic_request_str))
wait_start_time = TimezoneUtil.cur_time_in_pst()
while not self._response:
self._connection.process_data_events(time_limit=int(
EnvUtil.get_pslx_env_variable('PSLX_QUEUE_TIMEOUT')))
if TimezoneUtil.cur_time_in_pst() - wait_start_time > \
datetime.timedelta(seconds=int(EnvUtil.get_pslx_env_variable('PSLX_QUEUE_TIMEOUT'))):
break
if not self.RESPONSE_MESSAGE_TYPE or self._response is None:
return None
else:
return ProtoUtil.any_to_message(
message_type=self.RESPONSE_MESSAGE_TYPE,
any_message=self._response.response_data)
except Exception as err:
self._logger.error('Queue [' + self.get_queue_name() +
"] send request with error " + str(err) + '.',
publish=True)
self._SYS_LOGGER.error('Queue [' + self.get_queue_name() +
"] send request with error " + str(err) +
'.')
def get_response_and_status_impl(self, request):
proto_file = FileUtil.die_if_file_not_exist(
file_name=request.proto_file_path)
message_type = ProtoUtil.infer_message_type_from_str(
message_type_str=request.message_type,
modules=request.proto_module if request.proto_module else None)
response = ProtoViewerRPCResponse()
proto_message = FileUtil.read_proto_from_file(proto_type=message_type,
file_name=proto_file)
response.proto_content = ProtoUtil.message_to_text(
proto_message=proto_message)
file_info = FileInfo()
file_info.file_path = request.proto_file_path
file_info.file_size = FileUtil.get_file_size(file_name=proto_file)
file_info.modified_time = str(
FileUtil.get_file_modified_time(file_name=proto_file))
response.file_info.CopyFrom(file_info)
return response, Status.SUCCEEDED
def _process_message(self, ch, method, props, body):
try:
generic_request = ProtoUtil.string_to_message(
message_type=GenericRPCRequest,
string=base64.b64decode(body)
)
self._logger.info("Getting request with uuid [" + generic_request.uuid + '] in consumer ['
+ self.get_consumer_name() + '].')
response = self._queue.send_request(request=generic_request)
response_str = ProtoUtil.message_to_string(proto_message=response)
ch.basic_publish(exchange='',
routing_key=props.reply_to,
properties=pika.BasicProperties(correlation_id=props.correlation_id),
body=base64.b64encode(response_str))
ch.basic_ack(delivery_tag=method.delivery_tag)
except Exception as err:
self._logger.error("Consumer [" + self.get_consumer_name() + "] processing message with error: " +
str(err) + '.')
def get_content_from_snapshot(cls, snapshot_file, message_type):
try:
with FileLockTool(snapshot_file, read_mode=True):
snapshot = FileUtil.read_proto_from_file(
proto_type=OperatorSnapshot,
file_name=FileUtil.die_if_file_not_exist(file_name=snapshot_file)
)
return ProtoUtil.any_to_message(message_type=message_type, any_message=snapshot.content)
except FileNotExistException as _:
return message_type()
def on_response(self, ch, method, properties, body):
try:
message_type = self._topic_names_to_types[method.exchange + ':' +
method.routing_key]
message = ProtoUtil.string_to_message(
message_type=message_type, string=base64.b64decode(body))
return self.parse_message(exchange_name=method.exchange,
topic_name=method.routing_key,
message=message)
except Exception as err:
self._logger.error("Getting response with error " + str(err) + '.')
def view_proto():
value_types = EnvUtil.get_all_schemas(pslx_frontend_ui_app.config['schemas'])
if request.method == 'POST':
try:
proto_file_path = request.form['proto_file_path'].strip()
selected_value_type = request.form['value_type'].strip()
modules = selected_value_type.split('.')
module, message_type_str = '.'.join(modules[:-1]), modules[-1]
pslx_frontend_logger.info("Proto viewer input path [" + proto_file_path + '] with message type [' +
message_type_str + '] in module name [' + module + '].')
message_type = ProtoUtil.infer_message_type_from_str(
message_type_str=message_type_str,
modules=module
)
proto_message = FileUtil.read_proto_from_file(
proto_type=message_type, file_name=proto_file_path)
result_ui = ProtoUtil.message_to_text(proto_message=proto_message)
return render_template(
'proto_viewer.html',
proto_content=result_ui,
value_types=value_types,
selected_value_type=selected_value_type
)
except Exception as err:
pslx_frontend_logger.error("Got error rendering proto_viewer.html: " + str(err) + '.')
return render_template(
'proto_viewer.html',
proto_content="Got error rendering proto_viewer.html: " + str(err) + '.',
value_types=value_types,
selected_value_type=''
)
else:
return render_template(
'proto_viewer.html',
proto_content="",
value_types=value_types,
selected_value_type=''
)
def streaming_data_generator():
last_checked_key = None
while True:
pslx_dedicated_logging_storage = ProtoTableStorage(logger=pslx_frontend_logger)
pslx_dedicated_logging_storage.initialize_from_file(
file_name=pslx_dedicated_logging_storage_path
)
if pslx_dedicated_logging_storage.get_num_entries() == 0:
time.sleep(TimeSleepObj.ONE_TENTH_SECOND)
continue
all_data = pslx_dedicated_logging_storage.read_all()
all_sorted_keys = sorted(list(all_data.keys()))
if all_sorted_keys[-1] == last_checked_key:
time.sleep(TimeSleepObj.ONE_TENTH_SECOND)
continue
last_checked_key = all_sorted_keys[-1]
pslx_dedicated_logging_list = []
for key in all_sorted_keys:
val = ProtoUtil.any_to_message(
message_type=LoggingMessageRequest,
any_message=all_data[key]
)
if ProtoUtil.get_name_by_value(enum_type=DiskLoggerLevel, value=val.level) in log_levels:
message = val.message
for string_to_replace, string_after_replacing in strings_to_replace.items():
message = message.replace(string_to_replace, string_after_replacing)
contain_key_word = False if key_words else True
for key_word in key_words:
if key_word in message:
contain_key_word = True
break
if contain_key_word:
pslx_dedicated_logging_list.append(message)
yield '\\n'.join(pslx_dedicated_logging_list)
time.sleep(TimeSleepObj.ONE_TENTH_SECOND)
def get_response_and_status_impl(self, request):
storage_value = ContainerBackendValue()
storage_value.container_name = request.container_name
storage_value.container_status = request.status
for operator_name, operator_snapshot in dict(
request.operator_snapshot_map).items():
operator_info = ContainerBackendValue.OperatorInfo()
operator_info.status = operator_snapshot.status
for parent in operator_snapshot.node_snapshot.parents_names:
operator_info.parents.append(parent)
operator_info.start_time = operator_snapshot.start_time
operator_info.end_time = operator_snapshot.end_time
storage_value.operator_info_map[operator_name].CopyFrom(
operator_info)
storage_value.mode = request.mode
storage_value.data_model = request.data_model
storage_value.updated_time = str(TimezoneUtil.cur_time_in_pst())
storage_value.start_time = request.start_time
storage_value.end_time = request.end_time
storage_value.log_dir = request.log_dir
for key in request.counters:
storage_value.counters[key] = request.counters[key]
partitioner_dir = FileUtil.join_paths_to_dir_with_mode(
root_dir=FileUtil.join_paths_to_dir(
root_dir=self._backend_folder,
base_name=ProtoUtil.get_name_by_value(
enum_type=DataModelType, value=storage_value.data_model)),
base_name=storage_value.container_name,
ttl=EnvUtil.get_pslx_env_variable('PSLX_INTERNAL_TTL'))
if storage_value.mode == ModeType.TEST:
partitioner_dir = partitioner_dir.replace('PROD', 'TEST')
storage = self._lru_cache_tool.get(key=partitioner_dir)
if not storage:
self.sys_log(
"Did not find the storage in cache. Making a new one...")
storage = DailyPartitionerStorage()
proto_table = ProtoTableStorage()
storage.set_underlying_storage(storage=proto_table)
storage.initialize_from_dir(dir_name=partitioner_dir)
self._lru_cache_tool.set(key=partitioner_dir, value=storage)
else:
self.sys_log("Found key in LRU cache.")
storage.write(data={storage_value.container_name: storage_value},
params={
'overwrite': True,
'make_partition': True,
})
return None, Status.SUCCEEDED
def read(self, params):
assert 'key' in params
try:
result = self.read_multiple(params={'keys': [params['key']]})
value = list(result.values())[0]
if 'message_type' in params:
value = ProtoUtil.any_to_message(
any_message=value,
message_type=params['message_type']
)
return value
except Exception as err:
self._SYS_LOGGER.error("Read dir [" + self.get_dir_name() + "] got exception: " + str(err) + '.')
self._logger.error("Read dir [" + self.get_dir_name() + "] got exception: " + str(err) + '.')
return None
def _get_latest_status_of_operators(self):
operator_status = {}
snapshot_files = FileUtil.get_file_names_in_dir(
dir_name=FileUtil.join_paths_to_dir(FileUtil.dir_name(self._snapshot_file_folder), 'operators'))
for snapshot_file in snapshot_files[::-1]:
operator_name = snapshot_file.split('_')[1]
if operator_name not in operator_status:
self._logger.info("Getting status for operator [" + operator_name + '].')
self.sys_log("Getting status for operator [" + operator_name + '].')
operator_status[operator_name] = self._node_name_to_node_dict[operator_name].get_status_from_snapshot(
snapshot_file=snapshot_file
)
self.sys_log("Status for operator [" + operator_name + '] is [' + ProtoUtil.get_name_by_value(
enum_type=Status, value=operator_status[operator_name]) + '].')
if len(operator_status) == len(self._node_name_to_node_dict):
break
return operator_status