def get_response_and_status_impl(self, request):
storage_value = ContainerBackendValue()
storage_value.container_name = request.container_name
storage_value.container_status = request.status
for operator_name, operator_snapshot in dict(
request.operator_snapshot_map).items():
operator_info = ContainerBackendValue.OperatorInfo()
operator_info.status = operator_snapshot.status
for parent in operator_snapshot.node_snapshot.parents_names:
operator_info.parents.append(parent)
operator_info.start_time = operator_snapshot.start_time
operator_info.end_time = operator_snapshot.end_time
storage_value.operator_info_map[operator_name].CopyFrom(
operator_info)
storage_value.mode = request.mode
storage_value.data_model = request.data_model
storage_value.updated_time = str(TimezoneUtil.cur_time_in_pst())
storage_value.start_time = request.start_time
storage_value.end_time = request.end_time
storage_value.log_dir = request.log_dir
for key in request.counters:
storage_value.counters[key] = request.counters[key]
partitioner_dir = FileUtil.join_paths_to_dir_with_mode(
root_dir=FileUtil.join_paths_to_dir(
root_dir=self._backend_folder,
base_name=ProtoUtil.get_name_by_value(
enum_type=DataModelType, value=storage_value.data_model)),
base_name=storage_value.container_name,
ttl=EnvUtil.get_pslx_env_variable('PSLX_INTERNAL_TTL'))
if storage_value.mode == ModeType.TEST:
partitioner_dir = partitioner_dir.replace('PROD', 'TEST')
storage = self._lru_cache_tool.get(key=partitioner_dir)
if not storage:
self.sys_log(
"Did not find the storage in cache. Making a new one...")
storage = DailyPartitionerStorage()
proto_table = ProtoTableStorage()
storage.set_underlying_storage(storage=proto_table)
storage.initialize_from_dir(dir_name=partitioner_dir)
self._lru_cache_tool.set(key=partitioner_dir, value=storage)
else:
self.sys_log("Found key in LRU cache.")
storage.write(data={storage_value.container_name: storage_value},
params={
'overwrite': True,
'make_partition': True,
})
return None, Status.SUCCEEDED
def __init__(self, container_name, logger=DummyUtil.dummy_logging(), ttl=-1):
super().__init__()
self._container_name = container_name
self._is_initialized = False
self._snapshot_file_folder = FileUtil.join_paths_to_dir_with_mode(
root_dir=FileUtil.join_paths_to_dir(
root_dir=EnvUtil.get_pslx_env_variable(var='PSLX_DATABASE'),
base_name='snapshots'
),
base_name=self.get_class_name() + '__' + container_name,
ttl=ttl
)
self._start_time = None
self._end_time = None
self._logger = logger
self._upstream_ops = []
self._backend = None
self._status = Status.IDLE
self._counter = defaultdict(int)
ttl=EnvUtil.get_pslx_env_variable(var='PSLX_INTERNAL_TTL'))
frontend_config_file = EnvUtil.get_pslx_env_variable(
'PSLX_FRONTEND_CONFIG_PROTO_PATH')
assert frontend_config_file != ''
pslx_frontend_ui_app.config['frontend_config'] = FileUtil.read_proto_from_file(
proto_type=FrontendConfig, file_name=frontend_config_file)
pslx_frontend_ui_app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
pslx_frontend_ui_app.config['SQLALCHEMY_DATABASE_URI'] =\
'sqlite:///' + pslx_frontend_ui_app.config['frontend_config'].sqlalchemy_database_path
pslx_frontend_logger.info(
"sqlalchemy database uri " +
str(pslx_frontend_ui_app.config['SQLALCHEMY_DATABASE_URI']) + '.')
pslx_frontend_db = SQLAlchemy(pslx_frontend_ui_app)
pslx_partitioner_lru_cache = LRUCacheTool(
max_capacity=EnvUtil.get_pslx_env_variable('PSLX_INTERNAL_CACHE'))
pslx_proto_table_lru_cache = LRUCacheTool(
max_capacity=EnvUtil.get_pslx_env_variable('PSLX_INTERNAL_CACHE'))
pslx_dedicated_logging_storage_path = FileUtil.join_paths_to_dir_with_mode(
root_dir=EnvUtil.get_pslx_env_variable('PSLX_DATABASE') +
'/PSLX_DEDICATED_LOGGING',
base_name='dedicated_logging.pb')
from pslx.micro_service.frontend.renderer import index_renderer, file_viewer_renderer, proto_viewer_renderer, \
container_backend_renderer, proto_table_viewer_renderer, logging_renderer
slack_payload = "payload={'text':'" + request.message + "\nCurrent time is "\
+ str(TimezoneUtil.cur_time_in_pst()) + "'}"
status = Status.SUCCEEDED
try:
requests.post(request.webhook_url,
data=slack_payload,
headers=header)
except Exception as err:
self._logger.error("Slack failed to send message with err " +
str(err))
status = Status.FAILED
return None, status
if __name__ == "__main__":
consumer = GenericConsumer(
connection_str='amqp://*****:*****@localhost:5672')
partitioner_dir = FileUtil.join_paths_to_dir_with_mode(
root_dir=FileUtil.join_paths_to_dir(
root_dir=EnvUtil.get_pslx_env_variable(var='PSLX_DATABASE'),
base_name='msg_queue'),
base_name='msg_queue_example',
ttl='1h')
storage = DailyPartitionerStorage()
storage.initialize_from_dir(dir_name=partitioner_dir)
slack_queue = SlackQueue(queue_name='slack_queue', queue_storage=storage)
consumer.bind_queue(exchange='slack_exchange', queue=slack_queue)
consumer.start_consumer()
from pslx.micro_service.proto_viewer.rpc import ProtoViewerRPC
from pslx.micro_service.rpc.generic_server import GenericServer
from pslx.storage.partitioner_storage import MinutelyPartitionerStorage
from pslx.util.file_util import FileUtil
if __name__ == "__main__":
server_url = "localhost:11444"
partitioner_dir = FileUtil.join_paths_to_dir_with_mode(
root_dir='database/proto_viewer/',
base_name='proto_viewer_example',
ttl='1h')
storage = MinutelyPartitionerStorage()
storage.initialize_from_dir(dir_name=partitioner_dir)
example_rpc = ProtoViewerRPC(rpc_storage=storage)
example_server = GenericServer(server_name='example')
example_server.create_server(max_worker=1, server_url=server_url)
example_server.bind_rpc(rpc=example_rpc)
example_server.start_server()
from pslx.micro_service.container_backend.rpc import ContainerBackendRPC
from pslx.micro_service.rpc.generic_server import GenericServer
from pslx.storage.partitioner_storage import MinutelyPartitionerStorage
from pslx.util.file_util import FileUtil
if __name__ == "__main__":
server_url = "localhost:11443"
partitioner_dir = FileUtil.join_paths_to_dir_with_mode(
root_dir='database/container_backend/',
base_name='container_backend_example',
ttl=1)
storage = MinutelyPartitionerStorage()
storage.initialize_from_dir(dir_name=partitioner_dir)
example_rpc = ContainerBackendRPC(rpc_storage=storage)
example_server = GenericServer(server_name='example_backend')
example_server.create_server(max_worker=1, server_url=server_url)
example_server.bind_rpc(rpc=example_rpc)
example_server.start_server()
