def read(self, params=None):
assert 'key' in params
while self._writer_status != Status.IDLE:
self.sys_log("Waiting for writer to finish.")
time.sleep(TimeSleepObj.ONE_SECOND)
while self._deleter_status != Status.IDLE:
self.sys_log("Waiting for deleter to finish.")
time.sleep(TimeSleepObj.ONE_SECOND)
self._reader_status = Status.RUNNING
if params['key'] in self._table_message.data:
try:
if 'message_type' in params:
result = ProtoUtil.any_to_message(
message_type=params['message_type'],
any_message=self._table_message.data[params['key']]
)
else:
result = self._table_message.data[params['key']]
self._reader_status = Status.IDLE
return result
except Exception as err:
self.sys_log("Read file [" + self.get_file_name() + "] got exception: " + str(err) + '.')
self._logger.error("Read file [" + self.get_file_name() + "] got exception: " + str(err) + '.')
raise StorageReadException("Read file [" + self.get_file_name() + "] got exception: " + str(err) + '.')
else:
return None
def fetch_range(self, start_time, end_time):
try:
data = self._partitioner.read_range(params={
'start_time': start_time,
'end_time': end_time,
})
start_time, end_time = str(start_time.replace(tzinfo=None)), str(
end_time.replace(tzinfo=None))
data_content = {}
for val in data.values():
data_content.update(val)
result = {}
self._logger.info(
"Successfully get the range data in partition dir [" +
self._partitioner.get_dir_name() + '].')
for key in sorted(data_content.keys()):
if key < start_time or key > end_time:
continue
else:
item = ProtoUtil.any_to_message(
message_type=self.MESSAGE_TYPE,
any_message=data_content[key])
result[key] = item
return result
except Exception as err:
self._logger.error("Fetch range for partition [" +
self._partitioner.get_dir_name() +
"] with error " + str(err) + '.')
return {}
def fetch_oldest(self):
try:
oldest_dir = self._partitioner.get_oldest_dir_in_root_directory()
if not oldest_dir:
self._logger.warning('[' + self._partitioner.get_dir_name() +
'] is empty.')
return None
proto_table = ProtoTableStorage()
proto_table.initialize_from_file(
file_name=FileUtil.join_paths_to_file(root_dir=oldest_dir,
base_name='data.pb'))
all_data = proto_table.read_all()
if all_data:
self._logger.info(
"Successfully get the oldest data in partition dir [" +
self._partitioner.get_dir_name() + '].')
min_key = min(all_data.keys())
return ProtoUtil.any_to_message(message_type=self.MESSAGE_TYPE,
any_message=all_data[min_key])
else:
return None
except Exception as err:
self._logger.error("Fetch oldest partition [" +
self._partitioner.get_dir_name() +
"] with error " + str(err) + '.')
return None
def send_request(self, request, root_certificate=None):
generic_request = ProtoUtil.compose_generic_request(request=request)
if self.RESPONSE_MESSAGE_TYPE:
generic_request.message_type = ProtoUtil.infer_str_from_message_type(
message_type=self.RESPONSE_MESSAGE_TYPE)
self._logger.info("Client getting request of uuid [" +
generic_request.uuid + '] in client [' +
self.get_client_name() + '].')
self.sys_log("Client getting request of uuid [" +
generic_request.uuid + '] in client [' +
self.get_client_name() + '].')
try:
options = [
('grpc.max_receive_message_length',
int(
EnvUtil.get_pslx_env_variable(
var='PSLX_GRPC_MAX_MESSAGE_LENGTH'))),
('grpc.max_send_message_length',
int(
EnvUtil.get_pslx_env_variable(
var='PSLX_GRPC_MAX_MESSAGE_LENGTH'))),
]
timeout = int(
EnvUtil.get_pslx_env_variable(var='PSLX_GRPC_TIMEOUT'))
if not root_certificate:
self._logger.info("Start with insecure channel in client [" +
self.get_client_name() + '].')
with grpc.insecure_channel(self._server_url,
options=options) as channel:
stub = GenericRPCServiceStub(channel=channel)
response = stub.SendRequest(request=generic_request,
timeout=timeout)
else:
self._logger.info("Start with secure channel in client [" +
self.get_client_name() + '].')
channel_credential = grpc.ssl_channel_credentials(
root_certificate)
with grpc.secure_channel(self._server_url,
channel_credential,
options=options) as channel:
stub = GenericRPCServiceStub(channel=channel)
response = stub.SendRequest(request=generic_request,
timeout=timeout)
if not self.RESPONSE_MESSAGE_TYPE:
self.sys_log(
"Response message type unset, return None instead.")
return None
else:
return ProtoUtil.any_to_message(
message_type=self.RESPONSE_MESSAGE_TYPE,
any_message=response.response_data)
except Exception as err:
self._logger.error("send request with error " + str(err) +
' in client [' + self.get_client_name() + '].')
self.sys_log("send request with error " + str(err) +
' in client [' + self.get_client_name() + '].')
return None
def get_content_from_snapshot(cls, snapshot_file, message_type):
try:
with FileLockTool(snapshot_file, read_mode=True):
snapshot = FileUtil.read_proto_from_file(
proto_type=OperatorSnapshot,
file_name=FileUtil.die_if_file_not_exist(file_name=snapshot_file)
)
return ProtoUtil.any_to_message(message_type=message_type, any_message=snapshot.content)
except FileNotExistException as _:
return message_type()
def view_proto_table():
value_types = EnvUtil.get_all_schemas(
pslx_frontend_ui_app.config['schemas'])
if request.method == 'POST':
try:
proto_table_path = request.form['proto_table_path'].strip()
selected_value_type = request.form['value_type'].strip()
modules = selected_value_type.split('.')
module, value_type = '.'.join(modules[:-1]), modules[-1]
pslx_frontend_logger.info("Proto table viewer input path [" +
proto_table_path +
'] with value type [' + value_type +
'] in module name [' + module + '].')
result = FileUtil.read_proto_from_file(proto_type=ProtoTable,
file_name=proto_table_path)
value_type = ProtoUtil.infer_message_type_from_str(
message_type_str=value_type, modules=module)
proto_contents = []
result_content = dict(result.data)
for key in sorted(result_content.keys()):
proto_val = ProtoUtil.any_to_message(
message_type=value_type, any_message=result_content[key])
try:
proto_contents.append({
'key':
key,
'val':
ProtoUtil.message_to_text(proto_message=proto_val),
})
except Exception as err:
pslx_frontend_logger.error(
"Proto table viewer Parsing proto with error " +
str(err) + '.')
proto_contents.append({
'key': key,
'val': str(proto_val),
})
value_types.remove(selected_value_type)
return render_template('proto_table_viewer.html',
proto_contents=proto_contents,
value_types=value_types,
selected_value_type=selected_value_type)
except Exception as err:
pslx_frontend_logger.error(
"Got error rendering proto_table_viewer.html: " + str(err) +
'.')
return render_template('proto_table_viewer.html',
proto_contents=[],
value_types=value_types,
selected_value_type='')
else:
return render_template('proto_table_viewer.html',
proto_contents=[],
value_types=value_types,
selected_value_type='')
def test_message_to_any(self):
node_snapshot = NodeSnapshot()
node_snapshot.node_name = 'test'
node_snapshot.children_names.append('children1')
node_snapshot.children_names.append('children2')
node_snapshot.parents_names.append('parent1')
node_snapshot.parents_names.append('parent2')
any_message = ProtoUtil.message_to_any(message=node_snapshot)
self.assertEqual(
ProtoUtil.any_to_message(message_type=NodeSnapshot,
any_message=any_message), node_snapshot)
def get_containers_info():
containers_info = []
existing_containers = {}
all_proto_files = set()
if not FileUtil.is_local_path(backend_folder):
all_cells = ['']
else:
all_cells = gclient.list_cells()
for cell_name in all_cells:
folder = FileUtil.convert_local_to_cell_path(
path=backend_folder, cell=cell_name)
proto_files = FileUtil.list_files_in_dir(folder)
all_proto_files = all_proto_files.union(set(proto_files))
for proto_file in all_proto_files:
storage = ProtoTableStorage()
storage.initialize_from_file(
file_name=proto_file
)
raw_data = storage.read_all()
if not raw_data:
continue
key = sorted(raw_data.keys())[-1]
val = raw_data[key]
result_proto = ProtoUtil.any_to_message(
message_type=ContainerBackendValue,
any_message=val
)
ttl = result_proto.ttl
if ttl > 0 and result_proto.updated_time and TimezoneUtil.cur_time_in_pst() - TimezoneUtil.cur_time_from_str(
result_proto.updated_time) >= datetime.timedelta(days=ttl):
FileUtil.remove_file(storage.get_file_name())
else:
container_info = {
'container_name': result_proto.container_name,
'status': ProtoUtil.get_name_by_value(
enum_type=Status, value=result_proto.container_status),
'updated_time': result_proto.updated_time,
'mode': ProtoUtil.get_name_by_value(enum_type=ModeType, value=result_proto.mode),
'data_model': ProtoUtil.get_name_by_value(
enum_type=DataModelType, value=result_proto.data_model),
'run_cell': result_proto.run_cell,
'snapshot_cell': result_proto.snapshot_cell,
}
if container_info['container_name'] not in existing_containers:
existing_containers[container_info['container_name']] = container_info['updated_time']
containers_info.append(container_info)
else:
if container_info['updated_time'] >= existing_containers[container_info['container_name']]:
containers_info.append(container_info)
return containers_info
def read(self, params):
assert 'key' in params
try:
result = self.read_multiple(params={'keys': [params['key']]})
value = list(result.values())[0]
if 'message_type' in params:
value = ProtoUtil.any_to_message(
any_message=value,
message_type=params['message_type']
)
return value
except Exception as err:
self._SYS_LOGGER.error("Read dir [" + self.get_dir_name() + "] got exception: " + str(err) + '.')
self._logger.error("Read dir [" + self.get_dir_name() + "] got exception: " + str(err) + '.')
return None
def send_request(self, request):
self._response = None
generic_request = GenericRPCRequest()
generic_request.request_data.CopyFrom(
ProtoUtil.message_to_any(message=request))
generic_request.timestamp = str(TimezoneUtil.cur_time_in_pst())
self._corr_id = generic_request.uuid = str(uuid.uuid4())
if self.RESPONSE_MESSAGE_TYPE:
generic_request.message_type = ProtoUtil.infer_str_from_message_type(
message_type=self.RESPONSE_MESSAGE_TYPE)
self._SYS_LOGGER.info("Getting request of uuid [" +
generic_request.uuid + '] in queue [' +
self.get_queue_name() + '].')
self._logger.info("Getting request of uuid [" + generic_request.uuid +
'] in queue [' + self.get_queue_name() + '].')
try:
generic_request_str = ProtoUtil.message_to_string(
proto_message=generic_request)
self._channel.basic_publish(
exchange='',
routing_key=self._queue_name,
properties=pika.BasicProperties(reply_to=self._callback_queue,
correlation_id=self._corr_id),
body=base64.b64encode(generic_request_str))
wait_start_time = TimezoneUtil.cur_time_in_pst()
while not self._response:
self._connection.process_data_events(time_limit=int(
EnvUtil.get_pslx_env_variable('PSLX_QUEUE_TIMEOUT')))
if TimezoneUtil.cur_time_in_pst() - wait_start_time > \
datetime.timedelta(seconds=int(EnvUtil.get_pslx_env_variable('PSLX_QUEUE_TIMEOUT'))):
break
if not self.RESPONSE_MESSAGE_TYPE or self._response is None:
return None
else:
return ProtoUtil.any_to_message(
message_type=self.RESPONSE_MESSAGE_TYPE,
any_message=self._response.response_data)
except Exception as err:
self._logger.error('Queue [' + self.get_queue_name() +
"] send request with error " + str(err) + '.',
publish=True)
self._SYS_LOGGER.error('Queue [' + self.get_queue_name() +
"] send request with error " + str(err) +
'.')
def streaming_data_generator():
last_checked_key = None
while True:
pslx_dedicated_logging_storage = ProtoTableStorage(logger=pslx_frontend_logger)
pslx_dedicated_logging_storage.initialize_from_file(
file_name=pslx_dedicated_logging_storage_path
)
if pslx_dedicated_logging_storage.get_num_entries() == 0:
time.sleep(TimeSleepObj.ONE_TENTH_SECOND)
continue
all_data = pslx_dedicated_logging_storage.read_all()
all_sorted_keys = sorted(list(all_data.keys()))
if all_sorted_keys[-1] == last_checked_key:
time.sleep(TimeSleepObj.ONE_TENTH_SECOND)
continue
last_checked_key = all_sorted_keys[-1]
pslx_dedicated_logging_list = []
for key in all_sorted_keys:
val = ProtoUtil.any_to_message(
message_type=LoggingMessageRequest,
any_message=all_data[key]
)
if ProtoUtil.get_name_by_value(enum_type=DiskLoggerLevel, value=val.level) in log_levels:
message = val.message
for string_to_replace, string_after_replacing in strings_to_replace.items():
message = message.replace(string_to_replace, string_after_replacing)
contain_key_word = False if key_words else True
for key_word in key_words:
if key_word in message:
contain_key_word = True
break
if contain_key_word:
pslx_dedicated_logging_list.append(message)
yield '\\n'.join(pslx_dedicated_logging_list)
time.sleep(TimeSleepObj.ONE_TENTH_SECOND)
def request_decomposer(cls, request):
assert cls.REQUEST_MESSAGE_TYPE is not None
return ProtoUtil.any_to_message(
message_type=cls.REQUEST_MESSAGE_TYPE,
any_message=request.request_data
)
def content_deserializer(self, content):
return ProtoUtil.any_to_message(message_type=OperatorContentPlain, any_message=content)
def get_container_info(container_name, cell_name, start_time):
container_info = {
'log_file': '',
'start_time': '',
'end_time': '',
'counter_info': [],
}
operators_info = []
folder = FileUtil.convert_local_to_cell_path(
path=backend_folder, cell=cell_name)
pslx_frontend_logger.info(
"Container backend checking folder [" + folder + '].')
storage = ProtoTableStorage()
storage.initialize_from_file(
FileUtil.join_paths_to_file(
root_dir=folder,
base_name=container_name + '.pb'
)
)
raw_data = storage.read_all()
all_past_run = []
for key in sorted(list(raw_data.keys()), reverse=True):
val = ProtoUtil.any_to_message(
message_type=ContainerBackendValue,
any_message=raw_data[key]
)
all_past_run.append(
{
'start_time': val.start_time,
'updated_time': val.updated_time,
'end_time': val.end_time,
'status': ProtoUtil.get_name_by_value(
enum_type=Status, value=val.container_status),
'run_cell': val.run_cell,
'snapshot_cell': val.snapshot_cell,
}
)
if len(all_past_run) > 10:
break
key = start_time if start_time else sorted(raw_data.keys())[-1]
val = raw_data[key]
result_proto = ProtoUtil.any_to_message(
message_type=ContainerBackendValue,
any_message=val
)
container_info['log_file'] = galaxy_viewer_url + result_proto.log_file
container_info['start_time'] = result_proto.start_time
container_info['end_time'] = result_proto.end_time
for key in sorted(dict(result_proto.counters).keys()):
container_info['counter_info'].append(
{
'name': key,
'count': result_proto.counters[key],
}
)
for key, val in dict(result_proto.operator_info_map).items():
operators_info.append({
'operator_name': key,
'status': ProtoUtil.get_name_by_value(
enum_type=Status, value=val.status),
'start_time': val.start_time,
'end_time': val.end_time,
'dependencies': ', '.join(val.parents),
'log_file': galaxy_viewer_url + val.log_file,
})
return (container_info, sorted(operators_info, key=lambda x: (x['dependencies'], x['operator_name'])),
all_past_run)
def view_proto_table():
all_urls = sorted(server_urls)
if request.method == 'POST':
try:
server_url = request.form['server_url'].strip()
proto_table_path = request.form['proto_table_path'].strip()
value_type = request.form['value_type'].strip()
module = request.form['module'].strip()
pslx_frontend_logger.info("Proto table viewer selecting url [" + server_url + '] and input path [' +
proto_table_path + '] with value type [' + value_type +
'] in module name [' + module + '].')
result = client_map[server_url]['client'].view_proto(
proto_file_path=proto_table_path,
message_type="ProtoTable",
module="",
root_certificate=client_map[server_url]['root_certificate']
)
value_type = ProtoUtil.infer_message_type_from_str(
message_type_str=value_type,
modules=module
)
result_content = ProtoUtil.text_to_message(message_type=ProtoTable, text_str=result['proto_content'])
proto_contents = []
result_content = dict(result_content.data)
for key in sorted(result_content.keys()):
proto_val = ProtoUtil.any_to_message(
message_type=value_type,
any_message=result_content[key]
)
try:
proto_contents.append(
{
'key': key,
'val': ProtoUtil.message_to_text(proto_message=proto_val),
}
)
except Exception as err:
pslx_frontend_logger.error("Proto table viewer Parsing proto with error " + str(err) + '.')
proto_contents.append(
{
'key': key,
'val': str(proto_val),
}
)
all_urls.remove(server_url)
all_urls = [server_url] + all_urls
return render_template(
'proto_table_viewer.html',
proto_contents=proto_contents,
server_urls=all_urls
)
except Exception as err:
pslx_frontend_logger.error("Got error rendering proto_table_viewer.html: " + str(err) + '.')
return render_template(
'proto_table_viewer.html',
proto_contents=[],
server_urls=all_urls
)
else:
return render_template(
'proto_table_viewer.html',
proto_contents=[],
selected_server_url='',
server_urls=server_urls
)
