def get_container_snapshot(self, send_backend=True):
if not self._is_initialized:
self._logger.error("Warning: taking snapshot when the container [" + self.get_container_name() +
"] is not initialized.")
self.sys_log("Warning: taking snapshot when the container [" + self.get_container_name() +
"] is not initialized.")
snapshot = ContainerSnapshot()
snapshot.container_name = self._container_name
snapshot.is_initialized = self._is_initialized
snapshot.status = self._status
snapshot.class_name = self.get_full_class_name()
snapshot.mode = self._mode
snapshot.data_model = self.DATA_MODEL
snapshot.log_dir = self._logger.get_log_dir()
for key, val in self._counter.items():
snapshot.counters[key] = val
if self._start_time:
snapshot.start_time = str(self._start_time)
if self._end_time:
snapshot.end_time = str(self._end_time)
for op_name, op in self._node_name_to_node_dict.items():
if 'Dummy' in op.get_class_name():
continue
op_output_file = FileUtil.join_paths_to_file(
root_dir=FileUtil.join_paths_to_dir(FileUtil.dir_name(self._snapshot_file_folder), 'operators'),
base_name='SNAPSHOT_' + str(TimezoneUtil.cur_time_in_pst()) + '_' + op_name + '.pb'
)
snapshot.operator_snapshot_map[op_name].CopyFrom(op.get_operator_snapshot(output_file=op_output_file))
self.sys_log("Snapshot saved to folder [" + self._snapshot_file_folder + '].')
self._logger.info("Snapshot saved to folder [" + self._snapshot_file_folder + '].')
output_file_name = FileUtil.join_paths_to_file(
root_dir=FileUtil.dir_name(self._snapshot_file_folder),
base_name='SNAPSHOT_' + str(TimezoneUtil.cur_time_in_pst()) + '_' + self._container_name + '.pb'
)
with FileLockTool(output_file_name, read_mode=False):
FileUtil.write_proto_to_file(
proto=snapshot,
file_name=output_file_name
)
if self._backend and send_backend:
try:
self._backend.send_to_backend(snapshot=snapshot)
except Exception as err:
self._logger.error("Sending backend failed with error " + str(err) + '.')
return snapshot
def _get_latest_status_of_operators(self):
operator_status = {}
snapshot_files = FileUtil.get_file_names_in_dir(
dir_name=FileUtil.join_paths_to_dir(FileUtil.dir_name(self._snapshot_file_folder), 'operators'))
for snapshot_file in snapshot_files[::-1]:
operator_name = snapshot_file.split('_')[1]
if operator_name not in operator_status:
self._logger.info("Getting status for operator [" + operator_name + '].')
self.sys_log("Getting status for operator [" + operator_name + '].')
operator_status[operator_name] = self._node_name_to_node_dict[operator_name].get_status_from_snapshot(
snapshot_file=snapshot_file
)
self.sys_log("Status for operator [" + operator_name + '] is [' + ProtoUtil.get_name_by_value(
enum_type=Status, value=operator_status[operator_name]) + '].')
if len(operator_status) == len(self._node_name_to_node_dict):
break
return operator_status
def test_dir_name(self):
file_name = 'foo/bar/file.txt'
self.assertEqual(FileUtil.dir_name(file_name), 'foo/bar')
def get_container_snapshot(self, send_backend=True):
if not self._is_initialized:
self._logger.error(
"Warning: taking snapshot when the container [" +
self.get_container_name() + "] is not initialized.")
self._SYS_LOGGER.error(
"Warning: taking snapshot when the container [" +
self.get_container_name() + "] is not initialized.")
snapshot = ContainerSnapshot()
snapshot.container_name = self._container_name
snapshot.is_initialized = self._is_initialized
snapshot.status = self._status
snapshot.class_name = self.get_full_class_name()
snapshot.mode = self._mode
snapshot.data_model = self.DATA_MODEL
snapshot.log_file = FileUtil.convert_local_to_cell_path(
glogging.get_logger_file(self._logger))
snapshot.run_cell = EnvUtil.get_other_env_variable(
var='GALAXY_fs_cell', fallback_value='')
snapshot.snapshot_cell = FileUtil.get_cell_from_path(
FileUtil.convert_local_to_cell_path(self._snapshot_file_folder))
for key, val in self._counter.items():
snapshot.counters[key] = val
if self._start_time:
snapshot.start_time = str(self._start_time)
if self._end_time:
snapshot.end_time = str(self._end_time)
for op_name, op in self._node_name_to_node_dict.items():
if 'Dummy' in op.get_class_name():
continue
op_output_file = FileUtil.join_paths_to_file(
root_dir=FileUtil.join_paths_to_dir(
FileUtil.dir_name(self._snapshot_file_folder),
'operators'),
base_name=op_name + '_SNAPSHOT_' +
str(TimezoneUtil.cur_time_in_pst()) + '.pb')
snapshot.operator_snapshot_map[op_name].CopyFrom(
op.get_operator_snapshot(output_file=op_output_file))
self._SYS_LOGGER.info(
"Snapshot saved to folder [" +
FileUtil.convert_local_to_cell_path(self._snapshot_file_folder) +
'].')
self._logger.info(
"Snapshot saved to folder [" +
FileUtil.convert_local_to_cell_path(self._snapshot_file_folder) +
'].')
output_file_name = FileUtil.join_paths_to_file(
root_dir=FileUtil.join_paths_to_dir(
FileUtil.dir_name(self._snapshot_file_folder), 'containers'),
base_name=self._container_name + '_SNAPSHOT_' +
str(TimezoneUtil.cur_time_in_pst()) + '.pb')
FileUtil.write_proto_to_file(proto=snapshot,
file_name=output_file_name)
if self._backend and send_backend:
try:
self._backend.send_to_backend(snapshot=snapshot)
except Exception as err:
self._logger.error("Sending backend failed with error " +
str(err) + '.')
return snapshot
