def get_container_snapshot(self, send_backend=True): if not self._is_initialized: self._logger.error("Warning: taking snapshot when the container [" + self.get_container_name() + "] is not initialized.") self.sys_log("Warning: taking snapshot when the container [" + self.get_container_name() + "] is not initialized.") snapshot = ContainerSnapshot() snapshot.container_name = self._container_name snapshot.is_initialized = self._is_initialized snapshot.status = self._status snapshot.class_name = self.get_full_class_name() snapshot.mode = self._mode snapshot.data_model = self.DATA_MODEL snapshot.log_dir = self._logger.get_log_dir() for key, val in self._counter.items(): snapshot.counters[key] = val if self._start_time: snapshot.start_time = str(self._start_time) if self._end_time: snapshot.end_time = str(self._end_time) for op_name, op in self._node_name_to_node_dict.items(): if 'Dummy' in op.get_class_name(): continue op_output_file = FileUtil.join_paths_to_file( root_dir=FileUtil.join_paths_to_dir(FileUtil.dir_name(self._snapshot_file_folder), 'operators'), base_name='SNAPSHOT_' + str(TimezoneUtil.cur_time_in_pst()) + '_' + op_name + '.pb' ) snapshot.operator_snapshot_map[op_name].CopyFrom(op.get_operator_snapshot(output_file=op_output_file)) self.sys_log("Snapshot saved to folder [" + self._snapshot_file_folder + '].') self._logger.info("Snapshot saved to folder [" + self._snapshot_file_folder + '].') output_file_name = FileUtil.join_paths_to_file( root_dir=FileUtil.dir_name(self._snapshot_file_folder), base_name='SNAPSHOT_' + str(TimezoneUtil.cur_time_in_pst()) + '_' + self._container_name + '.pb' ) with FileLockTool(output_file_name, read_mode=False): FileUtil.write_proto_to_file( proto=snapshot, file_name=output_file_name ) if self._backend and send_backend: try: self._backend.send_to_backend(snapshot=snapshot) except Exception as err: self._logger.error("Sending backend failed with error " + str(err) + '.') return snapshot
def _get_latest_status_of_operators(self): operator_status = {} snapshot_files = FileUtil.get_file_names_in_dir( dir_name=FileUtil.join_paths_to_dir(FileUtil.dir_name(self._snapshot_file_folder), 'operators')) for snapshot_file in snapshot_files[::-1]: operator_name = snapshot_file.split('_')[1] if operator_name not in operator_status: self._logger.info("Getting status for operator [" + operator_name + '].') self.sys_log("Getting status for operator [" + operator_name + '].') operator_status[operator_name] = self._node_name_to_node_dict[operator_name].get_status_from_snapshot( snapshot_file=snapshot_file ) self.sys_log("Status for operator [" + operator_name + '] is [' + ProtoUtil.get_name_by_value( enum_type=Status, value=operator_status[operator_name]) + '].') if len(operator_status) == len(self._node_name_to_node_dict): break return operator_status
def test_dir_name(self): file_name = 'foo/bar/file.txt' self.assertEqual(FileUtil.dir_name(file_name), 'foo/bar')
def get_container_snapshot(self, send_backend=True): if not self._is_initialized: self._logger.error( "Warning: taking snapshot when the container [" + self.get_container_name() + "] is not initialized.") self._SYS_LOGGER.error( "Warning: taking snapshot when the container [" + self.get_container_name() + "] is not initialized.") snapshot = ContainerSnapshot() snapshot.container_name = self._container_name snapshot.is_initialized = self._is_initialized snapshot.status = self._status snapshot.class_name = self.get_full_class_name() snapshot.mode = self._mode snapshot.data_model = self.DATA_MODEL snapshot.log_file = FileUtil.convert_local_to_cell_path( glogging.get_logger_file(self._logger)) snapshot.run_cell = EnvUtil.get_other_env_variable( var='GALAXY_fs_cell', fallback_value='') snapshot.snapshot_cell = FileUtil.get_cell_from_path( FileUtil.convert_local_to_cell_path(self._snapshot_file_folder)) for key, val in self._counter.items(): snapshot.counters[key] = val if self._start_time: snapshot.start_time = str(self._start_time) if self._end_time: snapshot.end_time = str(self._end_time) for op_name, op in self._node_name_to_node_dict.items(): if 'Dummy' in op.get_class_name(): continue op_output_file = FileUtil.join_paths_to_file( root_dir=FileUtil.join_paths_to_dir( FileUtil.dir_name(self._snapshot_file_folder), 'operators'), base_name=op_name + '_SNAPSHOT_' + str(TimezoneUtil.cur_time_in_pst()) + '.pb') snapshot.operator_snapshot_map[op_name].CopyFrom( op.get_operator_snapshot(output_file=op_output_file)) self._SYS_LOGGER.info( "Snapshot saved to folder [" + FileUtil.convert_local_to_cell_path(self._snapshot_file_folder) + '].') self._logger.info( "Snapshot saved to folder [" + FileUtil.convert_local_to_cell_path(self._snapshot_file_folder) + '].') output_file_name = FileUtil.join_paths_to_file( root_dir=FileUtil.join_paths_to_dir( FileUtil.dir_name(self._snapshot_file_folder), 'containers'), base_name=self._container_name + '_SNAPSHOT_' + str(TimezoneUtil.cur_time_in_pst()) + '.pb') FileUtil.write_proto_to_file(proto=snapshot, file_name=output_file_name) if self._backend and send_backend: try: self._backend.send_to_backend(snapshot=snapshot) except Exception as err: self._logger.error("Sending backend failed with error " + str(err) + '.') return snapshot