def _send_by_slack(self, is_test, webhook_url, message):
header = {
'Content-Type': "application/x-www-form-urlencoded",
'Cache-Control': "no-cache",
}
status = Status.SUCCEEDED
if not is_test and webhook_url:
slack_payload = "payload={'text':'" + message + "\nCurrent time is "\
+ str(TimezoneUtil.cur_time_in_pst()) + "'}"
try:
requests.post(
webhook_url,
data=slack_payload,
headers=header
)
except Exception as err:
self._logger.error("Slack failed to send message with err " + str(err))
status = Status.FAILED
error_payload = "payload={'text':'" + "Error: " + str(err) + "\nCurrent time is "\
+ str(TimezoneUtil.cur_time_in_pst()) + "'}"
requests.post(
webhook_url,
data=error_payload,
headers=header
)
return None, status
def watch_key(self, key):
start_time = TimezoneUtil.cur_time_in_pst()
delay = self._delay
while (self._timeout < 0 or TimezoneUtil.cur_time_in_pst() - start_time
< datetime.timedelta(seconds=self._timeout)):
try:
value = self._partitioner.read(
params={
'key': key,
'message_type': self.MESSAGE_TYPE,
'reinitialize_underlying_storage': True,
})
if value:
self._logger.info("Successfully got data for key [" + key +
"] in partitioner dir [" +
self._partitioner.get_dir_name() + '].')
return value
else:
self._logger.warning("Trying to fetch data for key [" +
key + "] again in partitioner dir [" +
self._partitioner.get_dir_name() +
'].')
delay = self._delay if delay < TimeSleepObj.ONE_HUNDREDTH_SECOND else delay / 2.0
time.sleep(delay)
except Exception as err:
self._logger.error("Read partition [" +
self._partitioner.get_dir_name() +
"] with error " + str(err) + '.')
return None
def _send_by_teams(self, is_test, webhook_url, message):
headers = {
'Content-Type': "application/json"
}
status = Status.SUCCEEDED
if not is_test and webhook_url:
teams_json_data = {
"text": message + ". Current time is " + str(TimezoneUtil.cur_time_in_pst()),
"@content": "http://schema.org/extensions",
"@type": "MessageCard",
}
try:
requests.post(
webhook_url,
json=teams_json_data,
headers=headers
)
except Exception as err:
self._logger.error("Teams failed to send message with err " + str(err))
status = Status.FAILED
error_json_data = {
"text": "Error: " + str(err) + ". Current time is " + str(TimezoneUtil.cur_time_in_pst()),
"@content": "http://schema.org/extensions",
"@type": "MessageCard",
}
requests.post(
webhook_url,
json=error_json_data,
headers=headers
)
return None, status
def get_container_snapshot(self, send_backend=True):
if not self._is_initialized:
self._logger.error("Warning: taking snapshot when the container [" + self.get_container_name() +
"] is not initialized.")
self.sys_log("Warning: taking snapshot when the container [" + self.get_container_name() +
"] is not initialized.")
snapshot = ContainerSnapshot()
snapshot.container_name = self._container_name
snapshot.is_initialized = self._is_initialized
snapshot.status = self._status
snapshot.class_name = self.get_full_class_name()
snapshot.mode = self._mode
snapshot.data_model = self.DATA_MODEL
snapshot.log_dir = self._logger.get_log_dir()
for key, val in self._counter.items():
snapshot.counters[key] = val
if self._start_time:
snapshot.start_time = str(self._start_time)
if self._end_time:
snapshot.end_time = str(self._end_time)
for op_name, op in self._node_name_to_node_dict.items():
if 'Dummy' in op.get_class_name():
continue
op_output_file = FileUtil.join_paths_to_file(
root_dir=FileUtil.join_paths_to_dir(FileUtil.dir_name(self._snapshot_file_folder), 'operators'),
base_name='SNAPSHOT_' + str(TimezoneUtil.cur_time_in_pst()) + '_' + op_name + '.pb'
)
snapshot.operator_snapshot_map[op_name].CopyFrom(op.get_operator_snapshot(output_file=op_output_file))
self.sys_log("Snapshot saved to folder [" + self._snapshot_file_folder + '].')
self._logger.info("Snapshot saved to folder [" + self._snapshot_file_folder + '].')
output_file_name = FileUtil.join_paths_to_file(
root_dir=FileUtil.dir_name(self._snapshot_file_folder),
base_name='SNAPSHOT_' + str(TimezoneUtil.cur_time_in_pst()) + '_' + self._container_name + '.pb'
)
with FileLockTool(output_file_name, read_mode=False):
FileUtil.write_proto_to_file(
proto=snapshot,
file_name=output_file_name
)
if self._backend and send_backend:
try:
self._backend.send_to_backend(snapshot=snapshot)
except Exception as err:
self._logger.error("Sending backend failed with error " + str(err) + '.')
return snapshot
def compose_generic_request(cls, request):
generic_request = GenericRPCRequest()
generic_request.request_data.CopyFrom(
cls.message_to_any(message=request))
generic_request.timestamp = str(TimezoneUtil.cur_time_in_pst())
generic_request.uuid = str(uuid.uuid4())
return generic_request
def write(self, data, params=None):
if not params:
params = {}
if 'overwrite' not in params:
params['overwrite'] = True
assert isinstance(data, dict)
try:
for key, val in data.items():
if not params['overwrite'] and key in self._table_message.data:
continue
any_message = ProtoUtil.message_to_any(message=val)
self._table_message.data[key].CopyFrom(any_message)
if len(self._table_message.data) > 1000:
self._SYS_LOGGER.warning(
"Warning: the table content is too large, considering using Partitioner "
"combined with proto table.")
self._table_message.updated_time = str(
TimezoneUtil.cur_time_in_pst())
self.increment_rpc_count_by(n=1)
FileUtil.write_proto_to_file(proto=self._table_message,
file_name=self._file_name)
except Exception as err:
self._SYS_LOGGER.error("Write to file [" + self.get_file_name() +
"] got exception: " + str(err) + '.')
self._logger.error("Write to file [" + self.get_file_name() +
"] got exception: " + str(err) + '.')
raise StorageWriteException("Write to file [" +
self.get_file_name() +
"] got exception: " + str(err) + '.')
def delete_all(self):
while self._reader_status != Status.IDLE:
self.sys_log("Waiting for reader to finish.")
time.sleep(TimeSleepObj.ONE_SECOND)
while self._writer_status != Status.IDLE:
self.sys_log("Waiting for writer to finish.")
time.sleep(TimeSleepObj.ONE_SECOND)
self._deleter_status = Status.RUNNING
all_keys = list(dict(self._table_message.data).keys())
for key in all_keys:
del self._table_message.data[key]
try:
self._table_message.updated_time = str(TimezoneUtil.cur_time_in_pst())
with FileLockTool(self._file_name, read_mode=False):
FileUtil.write_proto_to_file(
proto=self._table_message,
file_name=self._file_name
)
self._deleter_status = Status.IDLE
except Exception as err:
self.sys_log("Delete all of file [" + self.get_file_name() + "] got exception: " + str(err) + '.')
self._logger.error("Delete all of file [" + self.get_file_name() + "] got exception: " + str(err) + '.')
raise StorageDeleteException("Delete all of file [" + self.get_file_name() +
"] got exception: " + str(err) + '.')
def sys_log(cls, string):
if EnvUtil.get_pslx_env_variable(var='PSLX_LOG'):
try:
caller = getframeinfo(stack()[1][0])
print('[SYS-LOG] ' + ColorsUtil.make_foreground_green(
'[file: %s]' % FileUtil.base_name(caller.filename)) + ' ' +
ColorsUtil.make_foreground_yellow('[line: %d]' %
caller.lineno) + ' ' +
ColorsUtil.make_foreground_red(
'[%s]' % str(TimezoneUtil.cur_time_in_pst())) +
': ' + string)
except Exception as _:
print('[SYS-LOG] ' + ColorsUtil.make_foreground_red(
'[%s]' % str(TimezoneUtil.cur_time_in_pst())) + ': ' +
string)
def _add_request_timestamp(self):
cur_time = TimezoneUtil.cur_time_in_pst()
while self._request_timestamp and cur_time - self._request_timestamp[
0] >= datetime.timedelta(minutes=30):
self._request_timestamp.popleft()
self._request_timestamp.append(cur_time)
def compose_generic_response(cls, response):
generic_response = GenericRPCResponse()
if response:
generic_response.response_data.CopyFrom(
cls.message_to_any(response))
generic_response.timestamp = str(TimezoneUtil.cur_time_in_pst())
return generic_response
def send_request(self, request):
self._response = None
generic_request = GenericRPCRequest()
generic_request.request_data.CopyFrom(
ProtoUtil.message_to_any(message=request))
generic_request.timestamp = str(TimezoneUtil.cur_time_in_pst())
self._corr_id = generic_request.uuid = str(uuid.uuid4())
if self.RESPONSE_MESSAGE_TYPE:
generic_request.message_type = ProtoUtil.infer_str_from_message_type(
message_type=self.RESPONSE_MESSAGE_TYPE)
self._SYS_LOGGER.info("Getting request of uuid [" +
generic_request.uuid + '] in queue [' +
self.get_queue_name() + '].')
self._logger.info("Getting request of uuid [" + generic_request.uuid +
'] in queue [' + self.get_queue_name() + '].')
try:
generic_request_str = ProtoUtil.message_to_string(
proto_message=generic_request)
self._channel.basic_publish(
exchange='',
routing_key=self._queue_name,
properties=pika.BasicProperties(reply_to=self._callback_queue,
correlation_id=self._corr_id),
body=base64.b64encode(generic_request_str))
wait_start_time = TimezoneUtil.cur_time_in_pst()
while not self._response:
self._connection.process_data_events(time_limit=int(
EnvUtil.get_pslx_env_variable('PSLX_QUEUE_TIMEOUT')))
if TimezoneUtil.cur_time_in_pst() - wait_start_time > \
datetime.timedelta(seconds=int(EnvUtil.get_pslx_env_variable('PSLX_QUEUE_TIMEOUT'))):
break
if not self.RESPONSE_MESSAGE_TYPE or self._response is None:
return None
else:
return ProtoUtil.any_to_message(
message_type=self.RESPONSE_MESSAGE_TYPE,
any_message=self._response.response_data)
except Exception as err:
self._logger.error('Queue [' + self.get_queue_name() +
"] send request with error " + str(err) + '.',
publish=True)
self._SYS_LOGGER.error('Queue [' + self.get_queue_name() +
"] send request with error " + str(err) +
'.')
def _send_by_rocketchat(self, is_test, webhook_url, message):
status = Status.SUCCEEDED
if not is_test and webhook_url:
data = {
"text": message + "\nCurrent time is " + str(TimezoneUtil.cur_time_in_pst()) +
'\n-----------------------------------------------'
}
try:
requests.post(webhook_url, json.dumps(data))
except Exception as err:
self._logger.error("Rocketchat failed to send message with err " + str(err))
status = Status.FAILED
error_data = {
"text": "Error: " + str(err) + "\nCurrent time is " + str(TimezoneUtil.cur_time_in_pst()) +
'\n-----------------------------------------------'
}
requests.post(webhook_url, json.dumps(error_data))
return None, status
def write(self, data, params=None):
to_make_partition = True
if params and 'make_partition' in params:
to_make_partition = params['make_partition']
params.pop('make_partition', None)
if self._underlying_storage.get_storage_type(
) == StorageType.PROTO_TABLE_STORAGE:
file_base_name = 'data.pb'
else:
file_base_name = 'data'
if params and 'base_name' in params:
file_base_name = params['base_name']
params.pop('base_name', None)
while self._reader_status != Status.IDLE:
self.sys_log("Waiting for reader to finish.")
time.sleep(TimeSleepObj.ONE_SECOND)
self._writer_status = Status.RUNNING
if to_make_partition:
if not params or 'timezone' not in params or params[
'timezone'] == 'PST':
self.make_new_partition(
timestamp=TimezoneUtil.cur_time_in_pst())
elif params['timezone'] == 'UTC':
self.make_new_partition(
timestamp=TimezoneUtil.cur_time_in_utc())
elif params['timezone'] == 'EST':
self.make_new_partition(
timestamp=TimezoneUtil.cur_time_in_est())
self.initialize_from_dir(dir_name=self._file_tree.get_root_name())
file_name = FileUtil.join_paths_to_file(
root_dir=self._file_tree.get_leftmost_leaf(),
base_name=file_base_name)
if file_name != self._underlying_storage.get_file_name():
self.sys_log("Sync to the latest file to " + file_name)
self._underlying_storage.initialize_from_file(file_name=file_name)
try:
self._underlying_storage.write(data=data, params=params)
self._writer_status = Status.IDLE
except Exception as err:
self.sys_log("Write to dir [" + self.get_dir_name() +
"] got exception: " + str(err) + '.')
self._logger.error("Write to dir [" + self.get_dir_name() +
"] got exception: " + str(err) + '.')
raise StorageWriteException("Write to dir [" +
self.get_dir_name() +
"] got exception: " + str(err) + '.')
def get_containers_info():
containers_info = []
existing_containers = {}
all_proto_files = set()
if not FileUtil.is_local_path(backend_folder):
all_cells = ['']
else:
all_cells = gclient.list_cells()
for cell_name in all_cells:
folder = FileUtil.convert_local_to_cell_path(
path=backend_folder, cell=cell_name)
proto_files = FileUtil.list_files_in_dir(folder)
all_proto_files = all_proto_files.union(set(proto_files))
for proto_file in all_proto_files:
storage = ProtoTableStorage()
storage.initialize_from_file(
file_name=proto_file
)
raw_data = storage.read_all()
if not raw_data:
continue
key = sorted(raw_data.keys())[-1]
val = raw_data[key]
result_proto = ProtoUtil.any_to_message(
message_type=ContainerBackendValue,
any_message=val
)
ttl = result_proto.ttl
if ttl > 0 and result_proto.updated_time and TimezoneUtil.cur_time_in_pst() - TimezoneUtil.cur_time_from_str(
result_proto.updated_time) >= datetime.timedelta(days=ttl):
FileUtil.remove_file(storage.get_file_name())
else:
container_info = {
'container_name': result_proto.container_name,
'status': ProtoUtil.get_name_by_value(
enum_type=Status, value=result_proto.container_status),
'updated_time': result_proto.updated_time,
'mode': ProtoUtil.get_name_by_value(enum_type=ModeType, value=result_proto.mode),
'data_model': ProtoUtil.get_name_by_value(
enum_type=DataModelType, value=result_proto.data_model),
'run_cell': result_proto.run_cell,
'snapshot_cell': result_proto.snapshot_cell,
}
if container_info['container_name'] not in existing_containers:
existing_containers[container_info['container_name']] = container_info['updated_time']
containers_info.append(container_info)
else:
if container_info['updated_time'] >= existing_containers[container_info['container_name']]:
containers_info.append(container_info)
return containers_info
def execute(self):
self.update_log_date()
assert self.is_data_model_consistent() and self.is_status_consistent()
if self.get_status() == Status.SUCCEEDED:
self._SYS_LOGGER.info("Operator [" + self.get_node_name() +
"] already succeeded. It might have been finished by another process.")
self._logger.info("Operator [" + self.get_node_name() +
"] already succeeded. It might have been finished by another process.")
return
unfinished_parent_ops = self.wait_for_upstream_status()
while unfinished_parent_ops:
time.sleep(TimeSleepObj.ONE_SECOND)
self._SYS_LOGGER.info("Waiting for parent process to finish: " + ','.join(unfinished_parent_ops) + '.')
self._logger.info("Waiting for parent process to finish: " + ','.join(unfinished_parent_ops) + '.')
unfinished_parent_ops = self.wait_for_upstream_status()
if self.get_status() == Status.FAILED:
self._SYS_LOGGER.info("Operator [" + self.get_node_name() +
"] already failed because of upstream jobs. Existing...")
self._logger.error("Operator [" + self.get_node_name() +
"] already failed because of upstream jobs. Existing...")
return
self.set_status(status=Status.RUNNING)
self._start_time = TimezoneUtil.cur_time_in_pst()
for child_node in self.get_children_nodes():
if child_node.get_status() != Status.WAITING:
child_node.set_status(Status.WAITING)
if self._config['allow_container_snapshot']:
self._SYS_LOGGER.info("Taking snapshot when status is updated.")
self._container.get_container_snapshot()
try:
if self._execute_impl() == Signal.STOP:
self._end_time = TimezoneUtil.cur_time_in_pst()
self.set_status(status=Status.SUCCEEDED)
except OperatorFailureException as err:
self._SYS_LOGGER.error("Execute operator [" + self.get_node_name() + "] with error " + str(err) + '.')
self._logger.error("Execute operator [" + self.get_node_name() + "] with error " + str(err) + '.')
self.set_status(status=Status.FAILED)
def get_response_and_status_impl(self, request):
storage_value = ContainerBackendValue()
storage_value.container_name = request.container_name
storage_value.container_status = request.status
for operator_name, operator_snapshot in dict(
request.operator_snapshot_map).items():
operator_info = ContainerBackendValue.OperatorInfo()
operator_info.status = operator_snapshot.status
for parent in operator_snapshot.node_snapshot.parents_names:
operator_info.parents.append(parent)
operator_info.start_time = operator_snapshot.start_time
operator_info.end_time = operator_snapshot.end_time
storage_value.operator_info_map[operator_name].CopyFrom(
operator_info)
storage_value.mode = request.mode
storage_value.data_model = request.data_model
storage_value.updated_time = str(TimezoneUtil.cur_time_in_pst())
storage_value.start_time = request.start_time
storage_value.end_time = request.end_time
storage_value.log_dir = request.log_dir
for key in request.counters:
storage_value.counters[key] = request.counters[key]
partitioner_dir = FileUtil.join_paths_to_dir_with_mode(
root_dir=FileUtil.join_paths_to_dir(
root_dir=self._backend_folder,
base_name=ProtoUtil.get_name_by_value(
enum_type=DataModelType, value=storage_value.data_model)),
base_name=storage_value.container_name,
ttl=EnvUtil.get_pslx_env_variable('PSLX_INTERNAL_TTL'))
if storage_value.mode == ModeType.TEST:
partitioner_dir = partitioner_dir.replace('PROD', 'TEST')
storage = self._lru_cache_tool.get(key=partitioner_dir)
if not storage:
self.sys_log(
"Did not find the storage in cache. Making a new one...")
storage = DailyPartitionerStorage()
proto_table = ProtoTableStorage()
storage.set_underlying_storage(storage=proto_table)
storage.initialize_from_dir(dir_name=partitioner_dir)
self._lru_cache_tool.set(key=partitioner_dir, value=storage)
else:
self.sys_log("Found key in LRU cache.")
storage.write(data={storage_value.container_name: storage_value},
params={
'overwrite': True,
'make_partition': True,
})
return None, Status.SUCCEEDED
def get_response_impl(backend_folder, request, lru_cache=None):
storage_value = ContainerBackendValue()
storage_value.container_name = request.container_name
storage_value.container_status = request.status
for operator_name, operator_snapshot in dict(
request.operator_snapshot_map).items():
operator_info = ContainerBackendValue.OperatorInfo()
operator_info.status = operator_snapshot.status
for parent in operator_snapshot.node_snapshot.parents_names:
operator_info.parents.append(parent)
operator_info.start_time = operator_snapshot.start_time
operator_info.end_time = operator_snapshot.end_time
operator_info.log_file = operator_snapshot.log_file
storage_value.operator_info_map[operator_name].CopyFrom(
operator_info)
storage_value.mode = request.mode
storage_value.data_model = request.data_model
storage_value.updated_time = str(TimezoneUtil.cur_time_in_pst())
storage_value.start_time = request.start_time
storage_value.end_time = request.end_time
storage_value.log_file = request.log_file
storage_value.run_cell = request.run_cell
storage_value.snapshot_cell = request.snapshot_cell
for key in request.counters:
storage_value.counters[key] = request.counters[key]
storage_value.ttl = int(
EnvUtil.get_pslx_env_variable('PSLX_BACKEND_CONTAINER_TTL'))
storage = lru_cache.get(
key=storage_value.container_name) if lru_cache else None
if not storage:
storage = ProtoTableStorage()
storage.initialize_from_file(file_name=FileUtil.join_paths_to_file(
root_dir=backend_folder,
base_name=storage_value.container_name + '.pb'))
if lru_cache:
lru_cache.set(key=backend_folder, value=storage)
all_data = storage.read_all()
if len(all_data) >= int(
EnvUtil.get_pslx_env_variable('PSLX_INTERNAL_CACHE')) > 0:
key_to_delete = sorted(all_data.keys())[0]
storage.delete(key=key_to_delete)
storage.write(data={storage_value.start_time: storage_value})
def get_response_and_status_impl(self, request):
header = {
'Content-Type': "application/x-www-form-urlencoded",
'Cache-Control': "no-cache",
}
slack_payload = "payload={'text':'" + request.message + "\nCurrent time is "\
+ str(TimezoneUtil.cur_time_in_pst()) + "'}"
status = Status.SUCCEEDED
try:
requests.post(request.webhook_url,
data=slack_payload,
headers=header)
except Exception as err:
self._logger.error("Slack failed to send message with err " +
str(err))
status = Status.FAILED
return None, status
def CheckHealth(self, request, context):
self.sys_log("Checking health for url [" + request.server_url + '].')
self._logger.info("Checking health for url [" + request.server_url +
'].')
response = HealthCheckerResponse()
response.server_url = request.server_url
response.server_status = Status.RUNNING
self._add_request_timestamp()
if len(self._request_timestamp) < 1:
response.server_qps = 0
else:
duration = (TimezoneUtil.cur_time_in_pst() -
self._request_timestamp[0]) / datetime.timedelta(
seconds=1)
response.server_qps = len(self._request_timestamp) / max(
duration, 1)
return response
def write(self, data, params=None):
to_make_partition = True
if params and 'make_partition' in params:
to_make_partition = params['make_partition']
params.pop('make_partition', None)
if self._underlying_storage.get_storage_type(
) == StorageType.PROTO_TABLE_STORAGE:
file_base_name = 'data.pb'
else:
file_base_name = 'data'
if params and 'base_name' in params:
file_base_name = params['base_name']
if to_make_partition:
if not params or 'timezone' not in params or params[
'timezone'] == 'PST':
self.make_new_partition(
timestamp=TimezoneUtil.cur_time_in_pst())
elif params['timezone'] == 'UTC':
self.make_new_partition(
timestamp=TimezoneUtil.cur_time_in_utc())
elif params['timezone'] == 'EST':
self.make_new_partition(
timestamp=TimezoneUtil.cur_time_in_est())
file_name = FileUtil.join_paths_to_file(
root_dir=self._file_tree.get_leftmost_leaf(),
base_name=file_base_name)
if file_name != self._underlying_storage.get_file_name():
self._SYS_LOGGER.info("Sync to the latest file to " + file_name)
self._underlying_storage.initialize_from_file(file_name=file_name)
try:
self._underlying_storage.write(data=data, params=params)
self.increment_rpc_count_by(
n=self._underlying_storage.get_rpc_call_count_and_reset())
except Exception as err:
self._SYS_LOGGER.error("Write to dir [" + self.get_dir_name() +
"] got exception: " + str(err) + '.')
self._logger.error("Write to dir [" + self.get_dir_name() +
"] got exception: " + str(err) + '.')
raise StorageWriteException("Write to dir [" +
self.get_dir_name() +
"] got exception: " + str(err) + '.')
def CheckHealth(self, request, context):
if not self._auth(context=context):
raise RPCNotAuthenticatedException("Checking health not authenticated for service [" +
self.get_rpc_service_name() + '].')
self._SYS_LOGGER.info("Checking health for url [" + request.server_url + '].')
self._logger.info("Checking health for url [" + request.server_url + '].')
response = HealthCheckerResponse()
response.server_url = request.server_url
response.server_status = Status.RUNNING
self._add_request_timestamp()
if len(self._request_timestamp) < 1:
response.server_qps = 0
else:
duration = (TimezoneUtil.cur_time_in_pst() - self._request_timestamp[0]) / datetime.timedelta(seconds=1)
response.server_qps = len(self._request_timestamp) / max(duration, 1)
return response
def pubsub_msg_parser(self, exchange_name, topic_name, message):
self._cached_logging[str(TimezoneUtil.cur_time_in_pst())] = message
if len(self._cached_logging) >= int(EnvUtil.get_pslx_env_variable('PSLX_RPC_FLUSH_RATE')):
self._pslx_dedicated_logging_storage.write(
data=self._cached_logging
)
self._cached_logging.clear()
total_entries = self._pslx_dedicated_logging_storage.get_num_entries()
if total_entries > self._logging_storage_capacity:
try:
all_data = self._pslx_dedicated_logging_storage.read_all()
all_sorted_keys = sorted(list(all_data.keys()))
for i in range(total_entries - self._logging_storage_capacity):
key_to_delete = all_sorted_keys[i]
self._pslx_dedicated_logging_storage.delete(key=key_to_delete)
except Exception as _:
pass
def initialize_from_file(self, file_name):
if '.pb' not in file_name:
self.sys_log("Please use .pb extension for proto files.")
self._file_name = FileUtil.create_file_if_not_exist(file_name=file_name)
if FileUtil.is_file_empty(file_name=self._file_name):
self._table_message = ProtoTable()
else:
with FileLockTool(self._file_name, read_mode=True):
self._table_message = FileUtil.read_proto_from_file(
proto_type=ProtoTable,
file_name=self._file_name
)
if not self._table_message.table_path:
self._table_message.table_path = self._file_name
if not self._table_message.table_name:
self._table_message.table_name = FileUtil.base_name(file_name=file_name)
if not self._table_message.created_time:
self._table_message.created_time = str(TimezoneUtil.cur_time_in_pst())
def initialize_from_file(self, file_name):
if '.pb' not in file_name:
self._SYS_LOGGER.warning(
"Please use .pb extension for proto files.")
self._file_name = FileUtil.normalize_file_name(file_name=file_name)
self._table_message = FileUtil.read_proto_from_file(
proto_type=ProtoTable, file_name=self._file_name)
self.increment_rpc_count_by(n=1)
if self._table_message is None:
self._table_message = ProtoTable()
if not self._table_message.table_path:
self._table_message.table_path = self._file_name
if not self._table_message.table_name:
self._table_message.table_name = FileUtil.base_name(
file_name=file_name)
if not self._table_message.created_time:
self._table_message.created_time = str(
TimezoneUtil.cur_time_in_pst())
def delete_multiple(self, keys):
for key in keys:
if key in self._table_message.data:
del self._table_message.data[key]
try:
self._table_message.updated_time = str(
TimezoneUtil.cur_time_in_pst())
self.increment_rpc_count_by(n=1)
FileUtil.write_proto_to_file(proto=self._table_message,
file_name=self._file_name)
except Exception as err:
self._SYS_LOGGER.error("Delete file [" + self.get_file_name() +
"] got exception: " + str(err))
self._logger.error("Delete file [" + self.get_file_name() +
"] got exception: " + str(err))
raise StorageDeleteException("Delete file [" +
self.get_file_name() +
"] got exception: " + str(err))
def delete_all(self):
all_keys = list(dict(self._table_message.data).keys())
for key in all_keys:
del self._table_message.data[key]
try:
self._table_message.updated_time = str(
TimezoneUtil.cur_time_in_pst())
self.increment_rpc_count_by(n=1)
FileUtil.write_proto_to_file(proto=self._table_message,
file_name=self._file_name)
except Exception as err:
self._SYS_LOGGER.error("Delete all of file [" +
self.get_file_name() + "] got exception: " +
str(err) + '.')
self._logger.error("Delete all of file [" + self.get_file_name() +
"] got exception: " + str(err) + '.')
raise StorageDeleteException("Delete all of file [" +
self.get_file_name() +
"] got exception: " + str(err) + '.')
def write(self, data, params=None):
if not params:
params = {}
if 'overwrite' not in params:
params['overwrite'] = True
while self._reader_status != Status.IDLE:
self.sys_log("Waiting for reader to finish.")
time.sleep(TimeSleepObj.ONE_SECOND)
while self._deleter_status != Status.IDLE:
self.sys_log("Waiting for deleter to finish.")
time.sleep(TimeSleepObj.ONE_SECOND)
self._writer_status = Status.RUNNING
assert isinstance(data, dict)
try:
for key, val in data.items():
if not params['overwrite'] and key in self._table_message.data:
continue
any_message = ProtoUtil.message_to_any(message=val)
self._table_message.data[key].CopyFrom(any_message)
if len(self._table_message.data) > 1000:
self.sys_log("Warning: the table content is too large, considering using Partitioner "
"combined with proto table.")
self._table_message.updated_time = str(TimezoneUtil.cur_time_in_pst())
with FileLockTool(self._file_name, read_mode=False):
FileUtil.write_proto_to_file(
proto=self._table_message,
file_name=self._file_name
)
self._writer_status = Status.IDLE
except Exception as err:
self.sys_log("Write to file [" + self.get_file_name() + "] got exception: " + str(err) + '.')
self._logger.error("Write to file [" + self.get_file_name() + "] got exception: " + str(err) + '.')
raise StorageWriteException("Write to file [" + self.get_file_name() + "] got exception: " + str(err) + '.')
def get_container_snapshot(self, send_backend=True):
if not self._is_initialized:
self._logger.error(
"Warning: taking snapshot when the container [" +
self.get_container_name() + "] is not initialized.")
self._SYS_LOGGER.error(
"Warning: taking snapshot when the container [" +
self.get_container_name() + "] is not initialized.")
snapshot = ContainerSnapshot()
snapshot.container_name = self._container_name
snapshot.is_initialized = self._is_initialized
snapshot.status = self._status
snapshot.class_name = self.get_full_class_name()
snapshot.mode = self._mode
snapshot.data_model = self.DATA_MODEL
snapshot.log_file = FileUtil.convert_local_to_cell_path(
glogging.get_logger_file(self._logger))
snapshot.run_cell = EnvUtil.get_other_env_variable(
var='GALAXY_fs_cell', fallback_value='')
snapshot.snapshot_cell = FileUtil.get_cell_from_path(
FileUtil.convert_local_to_cell_path(self._snapshot_file_folder))
for key, val in self._counter.items():
snapshot.counters[key] = val
if self._start_time:
snapshot.start_time = str(self._start_time)
if self._end_time:
snapshot.end_time = str(self._end_time)
for op_name, op in self._node_name_to_node_dict.items():
if 'Dummy' in op.get_class_name():
continue
op_output_file = FileUtil.join_paths_to_file(
root_dir=FileUtil.join_paths_to_dir(
FileUtil.dir_name(self._snapshot_file_folder),
'operators'),
base_name=op_name + '_SNAPSHOT_' +
str(TimezoneUtil.cur_time_in_pst()) + '.pb')
snapshot.operator_snapshot_map[op_name].CopyFrom(
op.get_operator_snapshot(output_file=op_output_file))
self._SYS_LOGGER.info(
"Snapshot saved to folder [" +
FileUtil.convert_local_to_cell_path(self._snapshot_file_folder) +
'].')
self._logger.info(
"Snapshot saved to folder [" +
FileUtil.convert_local_to_cell_path(self._snapshot_file_folder) +
'].')
output_file_name = FileUtil.join_paths_to_file(
root_dir=FileUtil.join_paths_to_dir(
FileUtil.dir_name(self._snapshot_file_folder), 'containers'),
base_name=self._container_name + '_SNAPSHOT_' +
str(TimezoneUtil.cur_time_in_pst()) + '.pb')
FileUtil.write_proto_to_file(proto=snapshot,
file_name=output_file_name)
if self._backend and send_backend:
try:
self._backend.send_to_backend(snapshot=snapshot)
except Exception as err:
self._logger.error("Sending backend failed with error " +
str(err) + '.')
return snapshot
def execute(self, is_backfill=False, num_threads=1):
self.update_log_date()
if is_backfill:
if self.DATA_MODEL == DataModelType.STREAMING:
self._logger.warning('STREAMING container [' +
self.get_container_name() +
'] does not support backfill mode.')
return
self._SYS_LOGGER.debug("Running in backfill mode for container [" +
self.get_container_name() + '].')
self._logger.debug("Running in backfill mode for container [" +
self.get_container_name() + '].')
if not self._is_initialized:
self._SYS_LOGGER.error("Cannot execute if the container [" +
self.get_container_name() +
"] is not initialized.")
self._logger.error("Cannot execute if the container [" +
self.get_container_name() +
"] is not initialized.")
raise exception.ContainerUninitializedException(
"Cannot execute if the container [" +
self.get_container_name() + "] is not initialized.")
self._SYS_LOGGER.info('Upstream operators are: ' +
', '.join(self._upstream_ops))
unblocked_blocker = 0
while unblocked_blocker < len(self._upstream_ops):
for blocker in self._upstream_ops:
latest_snapshot_files = FileUtil.get_file_names_from_pattern(
pattern=blocker)
if not latest_snapshot_files:
continue
latest_snapshot_file = latest_snapshot_files[-1]
if OperatorBase.get_status_from_snapshot(
snapshot_file=latest_snapshot_file
) != Status.SUCCEEDED:
continue
else:
unblocked_blocker += 1
self._start_time = TimezoneUtil.cur_time_in_pst()
self._end_time = None
self.set_status(status=Status.RUNNING)
self._SYS_LOGGER.info("Taking snapshot of container [" +
self.get_container_name() + "] at start.")
self._logger.info("Taking snapshot of container [" +
self.get_container_name() + "] at start.")
self.get_container_snapshot()
operator_status = {}
if is_backfill:
self._SYS_LOGGER.info(
"Updating the status of operators from previous snapshots.")
operator_status = self._get_latest_status_of_operators()
task_queue, finished_queue, num_tasks = Queue(), Queue(), 0
node_levels = self.get_node_levels()
max_level = max(node_levels.keys())
for level in range(max_level + 1):
for operator_name in node_levels[level]:
if is_backfill and operator_status[
operator_name] == Status.SUCCEEDED:
self._node_name_to_node_dict[operator_name].set_status(
status=Status.SUCCEEDED)
continue
self._SYS_LOGGER.info("Adding operator [" + operator_name +
"] to task queue for container [" +
self.get_container_name() + '].')
self._logger.info("Adding operator [" + operator_name +
"] to task queue for container [" +
self.get_container_name() + '].')
task_queue.put(operator_name)
num_tasks += 1
for _ in range(num_threads):
task_queue.put(Signal.STOP)
if num_threads > 1:
thread_list = []
self._logger.info("Using [" + str(num_threads) +
"] threads. Enter multi-threading mode.")
for _ in range(num_threads):
thread = threading.Thread(target=self._execute,
args=(task_queue, finished_queue))
thread.daemon = True
thread.start()
thread_list.append(thread)
self._SYS_LOGGER.info("Joining all threads.")
for thread in thread_list:
thread.join()
self._SYS_LOGGER.info("Done Joining all threads.")
else:
# this is mainly for streaming case.
self._logger.info(
"Using only 1 thread. Execute the container directly.")
self._execute(task_queue=task_queue, finished_queue=finished_queue)
self._end_time = TimezoneUtil.cur_time_in_pst()
self.set_status(status=Status.SUCCEEDED)
for operator in self._node_name_to_node_dict.values():
if operator.get_status() == Status.FAILED:
self.set_status(status=Status.FAILED)
break
log_str = 'Tasks finishing order is: '
tasks_list = []
for _ in range(num_tasks):
tasks_list.append(finished_queue.get())
log_str += '-->'.join(tasks_list)
self._logger.info(log_str + '.')
self._SYS_LOGGER.info(log_str + '.')
self.get_container_snapshot()
self._SYS_LOGGER.info("Taking snapshot of container [" +
self.get_container_name() + "] in the end.")
self._logger.info("Taking snapshot of container [" +
self.get_container_name() + "] in the end.")