def read(self, params=None): assert 'key' in params while self._writer_status != Status.IDLE: self.sys_log("Waiting for writer to finish.") time.sleep(TimeSleepObj.ONE_SECOND) while self._deleter_status != Status.IDLE: self.sys_log("Waiting for deleter to finish.") time.sleep(TimeSleepObj.ONE_SECOND) self._reader_status = Status.RUNNING if params['key'] in self._table_message.data: try: if 'message_type' in params: result = ProtoUtil.any_to_message( message_type=params['message_type'], any_message=self._table_message.data[params['key']] ) else: result = self._table_message.data[params['key']] self._reader_status = Status.IDLE return result except Exception as err: self.sys_log("Read file [" + self.get_file_name() + "] got exception: " + str(err) + '.') self._logger.error("Read file [" + self.get_file_name() + "] got exception: " + str(err) + '.') raise StorageReadException("Read file [" + self.get_file_name() + "] got exception: " + str(err) + '.') else: return None
def fetch_range(self, start_time, end_time): try: data = self._partitioner.read_range(params={ 'start_time': start_time, 'end_time': end_time, }) start_time, end_time = str(start_time.replace(tzinfo=None)), str( end_time.replace(tzinfo=None)) data_content = {} for val in data.values(): data_content.update(val) result = {} self._logger.info( "Successfully get the range data in partition dir [" + self._partitioner.get_dir_name() + '].') for key in sorted(data_content.keys()): if key < start_time or key > end_time: continue else: item = ProtoUtil.any_to_message( message_type=self.MESSAGE_TYPE, any_message=data_content[key]) result[key] = item return result except Exception as err: self._logger.error("Fetch range for partition [" + self._partitioner.get_dir_name() + "] with error " + str(err) + '.') return {}
def fetch_oldest(self): try: oldest_dir = self._partitioner.get_oldest_dir_in_root_directory() if not oldest_dir: self._logger.warning('[' + self._partitioner.get_dir_name() + '] is empty.') return None proto_table = ProtoTableStorage() proto_table.initialize_from_file( file_name=FileUtil.join_paths_to_file(root_dir=oldest_dir, base_name='data.pb')) all_data = proto_table.read_all() if all_data: self._logger.info( "Successfully get the oldest data in partition dir [" + self._partitioner.get_dir_name() + '].') min_key = min(all_data.keys()) return ProtoUtil.any_to_message(message_type=self.MESSAGE_TYPE, any_message=all_data[min_key]) else: return None except Exception as err: self._logger.error("Fetch oldest partition [" + self._partitioner.get_dir_name() + "] with error " + str(err) + '.') return None
def send_request(self, request, root_certificate=None): generic_request = ProtoUtil.compose_generic_request(request=request) if self.RESPONSE_MESSAGE_TYPE: generic_request.message_type = ProtoUtil.infer_str_from_message_type( message_type=self.RESPONSE_MESSAGE_TYPE) self._logger.info("Client getting request of uuid [" + generic_request.uuid + '] in client [' + self.get_client_name() + '].') self.sys_log("Client getting request of uuid [" + generic_request.uuid + '] in client [' + self.get_client_name() + '].') try: options = [ ('grpc.max_receive_message_length', int( EnvUtil.get_pslx_env_variable( var='PSLX_GRPC_MAX_MESSAGE_LENGTH'))), ('grpc.max_send_message_length', int( EnvUtil.get_pslx_env_variable( var='PSLX_GRPC_MAX_MESSAGE_LENGTH'))), ] timeout = int( EnvUtil.get_pslx_env_variable(var='PSLX_GRPC_TIMEOUT')) if not root_certificate: self._logger.info("Start with insecure channel in client [" + self.get_client_name() + '].') with grpc.insecure_channel(self._server_url, options=options) as channel: stub = GenericRPCServiceStub(channel=channel) response = stub.SendRequest(request=generic_request, timeout=timeout) else: self._logger.info("Start with secure channel in client [" + self.get_client_name() + '].') channel_credential = grpc.ssl_channel_credentials( root_certificate) with grpc.secure_channel(self._server_url, channel_credential, options=options) as channel: stub = GenericRPCServiceStub(channel=channel) response = stub.SendRequest(request=generic_request, timeout=timeout) if not self.RESPONSE_MESSAGE_TYPE: self.sys_log( "Response message type unset, return None instead.") return None else: return ProtoUtil.any_to_message( message_type=self.RESPONSE_MESSAGE_TYPE, any_message=response.response_data) except Exception as err: self._logger.error("send request with error " + str(err) + ' in client [' + self.get_client_name() + '].') self.sys_log("send request with error " + str(err) + ' in client [' + self.get_client_name() + '].') return None
def get_content_from_snapshot(cls, snapshot_file, message_type): try: with FileLockTool(snapshot_file, read_mode=True): snapshot = FileUtil.read_proto_from_file( proto_type=OperatorSnapshot, file_name=FileUtil.die_if_file_not_exist(file_name=snapshot_file) ) return ProtoUtil.any_to_message(message_type=message_type, any_message=snapshot.content) except FileNotExistException as _: return message_type()
def view_proto_table(): value_types = EnvUtil.get_all_schemas( pslx_frontend_ui_app.config['schemas']) if request.method == 'POST': try: proto_table_path = request.form['proto_table_path'].strip() selected_value_type = request.form['value_type'].strip() modules = selected_value_type.split('.') module, value_type = '.'.join(modules[:-1]), modules[-1] pslx_frontend_logger.info("Proto table viewer input path [" + proto_table_path + '] with value type [' + value_type + '] in module name [' + module + '].') result = FileUtil.read_proto_from_file(proto_type=ProtoTable, file_name=proto_table_path) value_type = ProtoUtil.infer_message_type_from_str( message_type_str=value_type, modules=module) proto_contents = [] result_content = dict(result.data) for key in sorted(result_content.keys()): proto_val = ProtoUtil.any_to_message( message_type=value_type, any_message=result_content[key]) try: proto_contents.append({ 'key': key, 'val': ProtoUtil.message_to_text(proto_message=proto_val), }) except Exception as err: pslx_frontend_logger.error( "Proto table viewer Parsing proto with error " + str(err) + '.') proto_contents.append({ 'key': key, 'val': str(proto_val), }) value_types.remove(selected_value_type) return render_template('proto_table_viewer.html', proto_contents=proto_contents, value_types=value_types, selected_value_type=selected_value_type) except Exception as err: pslx_frontend_logger.error( "Got error rendering proto_table_viewer.html: " + str(err) + '.') return render_template('proto_table_viewer.html', proto_contents=[], value_types=value_types, selected_value_type='') else: return render_template('proto_table_viewer.html', proto_contents=[], value_types=value_types, selected_value_type='')
def test_message_to_any(self): node_snapshot = NodeSnapshot() node_snapshot.node_name = 'test' node_snapshot.children_names.append('children1') node_snapshot.children_names.append('children2') node_snapshot.parents_names.append('parent1') node_snapshot.parents_names.append('parent2') any_message = ProtoUtil.message_to_any(message=node_snapshot) self.assertEqual( ProtoUtil.any_to_message(message_type=NodeSnapshot, any_message=any_message), node_snapshot)
def get_containers_info(): containers_info = [] existing_containers = {} all_proto_files = set() if not FileUtil.is_local_path(backend_folder): all_cells = [''] else: all_cells = gclient.list_cells() for cell_name in all_cells: folder = FileUtil.convert_local_to_cell_path( path=backend_folder, cell=cell_name) proto_files = FileUtil.list_files_in_dir(folder) all_proto_files = all_proto_files.union(set(proto_files)) for proto_file in all_proto_files: storage = ProtoTableStorage() storage.initialize_from_file( file_name=proto_file ) raw_data = storage.read_all() if not raw_data: continue key = sorted(raw_data.keys())[-1] val = raw_data[key] result_proto = ProtoUtil.any_to_message( message_type=ContainerBackendValue, any_message=val ) ttl = result_proto.ttl if ttl > 0 and result_proto.updated_time and TimezoneUtil.cur_time_in_pst() - TimezoneUtil.cur_time_from_str( result_proto.updated_time) >= datetime.timedelta(days=ttl): FileUtil.remove_file(storage.get_file_name()) else: container_info = { 'container_name': result_proto.container_name, 'status': ProtoUtil.get_name_by_value( enum_type=Status, value=result_proto.container_status), 'updated_time': result_proto.updated_time, 'mode': ProtoUtil.get_name_by_value(enum_type=ModeType, value=result_proto.mode), 'data_model': ProtoUtil.get_name_by_value( enum_type=DataModelType, value=result_proto.data_model), 'run_cell': result_proto.run_cell, 'snapshot_cell': result_proto.snapshot_cell, } if container_info['container_name'] not in existing_containers: existing_containers[container_info['container_name']] = container_info['updated_time'] containers_info.append(container_info) else: if container_info['updated_time'] >= existing_containers[container_info['container_name']]: containers_info.append(container_info) return containers_info
def read(self, params): assert 'key' in params try: result = self.read_multiple(params={'keys': [params['key']]}) value = list(result.values())[0] if 'message_type' in params: value = ProtoUtil.any_to_message( any_message=value, message_type=params['message_type'] ) return value except Exception as err: self._SYS_LOGGER.error("Read dir [" + self.get_dir_name() + "] got exception: " + str(err) + '.') self._logger.error("Read dir [" + self.get_dir_name() + "] got exception: " + str(err) + '.') return None
def send_request(self, request): self._response = None generic_request = GenericRPCRequest() generic_request.request_data.CopyFrom( ProtoUtil.message_to_any(message=request)) generic_request.timestamp = str(TimezoneUtil.cur_time_in_pst()) self._corr_id = generic_request.uuid = str(uuid.uuid4()) if self.RESPONSE_MESSAGE_TYPE: generic_request.message_type = ProtoUtil.infer_str_from_message_type( message_type=self.RESPONSE_MESSAGE_TYPE) self._SYS_LOGGER.info("Getting request of uuid [" + generic_request.uuid + '] in queue [' + self.get_queue_name() + '].') self._logger.info("Getting request of uuid [" + generic_request.uuid + '] in queue [' + self.get_queue_name() + '].') try: generic_request_str = ProtoUtil.message_to_string( proto_message=generic_request) self._channel.basic_publish( exchange='', routing_key=self._queue_name, properties=pika.BasicProperties(reply_to=self._callback_queue, correlation_id=self._corr_id), body=base64.b64encode(generic_request_str)) wait_start_time = TimezoneUtil.cur_time_in_pst() while not self._response: self._connection.process_data_events(time_limit=int( EnvUtil.get_pslx_env_variable('PSLX_QUEUE_TIMEOUT'))) if TimezoneUtil.cur_time_in_pst() - wait_start_time > \ datetime.timedelta(seconds=int(EnvUtil.get_pslx_env_variable('PSLX_QUEUE_TIMEOUT'))): break if not self.RESPONSE_MESSAGE_TYPE or self._response is None: return None else: return ProtoUtil.any_to_message( message_type=self.RESPONSE_MESSAGE_TYPE, any_message=self._response.response_data) except Exception as err: self._logger.error('Queue [' + self.get_queue_name() + "] send request with error " + str(err) + '.', publish=True) self._SYS_LOGGER.error('Queue [' + self.get_queue_name() + "] send request with error " + str(err) + '.')
def streaming_data_generator(): last_checked_key = None while True: pslx_dedicated_logging_storage = ProtoTableStorage(logger=pslx_frontend_logger) pslx_dedicated_logging_storage.initialize_from_file( file_name=pslx_dedicated_logging_storage_path ) if pslx_dedicated_logging_storage.get_num_entries() == 0: time.sleep(TimeSleepObj.ONE_TENTH_SECOND) continue all_data = pslx_dedicated_logging_storage.read_all() all_sorted_keys = sorted(list(all_data.keys())) if all_sorted_keys[-1] == last_checked_key: time.sleep(TimeSleepObj.ONE_TENTH_SECOND) continue last_checked_key = all_sorted_keys[-1] pslx_dedicated_logging_list = [] for key in all_sorted_keys: val = ProtoUtil.any_to_message( message_type=LoggingMessageRequest, any_message=all_data[key] ) if ProtoUtil.get_name_by_value(enum_type=DiskLoggerLevel, value=val.level) in log_levels: message = val.message for string_to_replace, string_after_replacing in strings_to_replace.items(): message = message.replace(string_to_replace, string_after_replacing) contain_key_word = False if key_words else True for key_word in key_words: if key_word in message: contain_key_word = True break if contain_key_word: pslx_dedicated_logging_list.append(message) yield '\\n'.join(pslx_dedicated_logging_list) time.sleep(TimeSleepObj.ONE_TENTH_SECOND)
def request_decomposer(cls, request): assert cls.REQUEST_MESSAGE_TYPE is not None return ProtoUtil.any_to_message( message_type=cls.REQUEST_MESSAGE_TYPE, any_message=request.request_data )
def content_deserializer(self, content): return ProtoUtil.any_to_message(message_type=OperatorContentPlain, any_message=content)
def get_container_info(container_name, cell_name, start_time): container_info = { 'log_file': '', 'start_time': '', 'end_time': '', 'counter_info': [], } operators_info = [] folder = FileUtil.convert_local_to_cell_path( path=backend_folder, cell=cell_name) pslx_frontend_logger.info( "Container backend checking folder [" + folder + '].') storage = ProtoTableStorage() storage.initialize_from_file( FileUtil.join_paths_to_file( root_dir=folder, base_name=container_name + '.pb' ) ) raw_data = storage.read_all() all_past_run = [] for key in sorted(list(raw_data.keys()), reverse=True): val = ProtoUtil.any_to_message( message_type=ContainerBackendValue, any_message=raw_data[key] ) all_past_run.append( { 'start_time': val.start_time, 'updated_time': val.updated_time, 'end_time': val.end_time, 'status': ProtoUtil.get_name_by_value( enum_type=Status, value=val.container_status), 'run_cell': val.run_cell, 'snapshot_cell': val.snapshot_cell, } ) if len(all_past_run) > 10: break key = start_time if start_time else sorted(raw_data.keys())[-1] val = raw_data[key] result_proto = ProtoUtil.any_to_message( message_type=ContainerBackendValue, any_message=val ) container_info['log_file'] = galaxy_viewer_url + result_proto.log_file container_info['start_time'] = result_proto.start_time container_info['end_time'] = result_proto.end_time for key in sorted(dict(result_proto.counters).keys()): container_info['counter_info'].append( { 'name': key, 'count': result_proto.counters[key], } ) for key, val in dict(result_proto.operator_info_map).items(): operators_info.append({ 'operator_name': key, 'status': ProtoUtil.get_name_by_value( enum_type=Status, value=val.status), 'start_time': val.start_time, 'end_time': val.end_time, 'dependencies': ', '.join(val.parents), 'log_file': galaxy_viewer_url + val.log_file, }) return (container_info, sorted(operators_info, key=lambda x: (x['dependencies'], x['operator_name'])), all_past_run)
def view_proto_table(): all_urls = sorted(server_urls) if request.method == 'POST': try: server_url = request.form['server_url'].strip() proto_table_path = request.form['proto_table_path'].strip() value_type = request.form['value_type'].strip() module = request.form['module'].strip() pslx_frontend_logger.info("Proto table viewer selecting url [" + server_url + '] and input path [' + proto_table_path + '] with value type [' + value_type + '] in module name [' + module + '].') result = client_map[server_url]['client'].view_proto( proto_file_path=proto_table_path, message_type="ProtoTable", module="", root_certificate=client_map[server_url]['root_certificate'] ) value_type = ProtoUtil.infer_message_type_from_str( message_type_str=value_type, modules=module ) result_content = ProtoUtil.text_to_message(message_type=ProtoTable, text_str=result['proto_content']) proto_contents = [] result_content = dict(result_content.data) for key in sorted(result_content.keys()): proto_val = ProtoUtil.any_to_message( message_type=value_type, any_message=result_content[key] ) try: proto_contents.append( { 'key': key, 'val': ProtoUtil.message_to_text(proto_message=proto_val), } ) except Exception as err: pslx_frontend_logger.error("Proto table viewer Parsing proto with error " + str(err) + '.') proto_contents.append( { 'key': key, 'val': str(proto_val), } ) all_urls.remove(server_url) all_urls = [server_url] + all_urls return render_template( 'proto_table_viewer.html', proto_contents=proto_contents, server_urls=all_urls ) except Exception as err: pslx_frontend_logger.error("Got error rendering proto_table_viewer.html: " + str(err) + '.') return render_template( 'proto_table_viewer.html', proto_contents=[], server_urls=all_urls ) else: return render_template( 'proto_table_viewer.html', proto_contents=[], selected_server_url='', server_urls=server_urls )