def get_containers_info(): containers_info = [] existing_containers = {} all_proto_files = set() if not FileUtil.is_local_path(backend_folder): all_cells = [''] else: all_cells = gclient.list_cells() for cell_name in all_cells: folder = FileUtil.convert_local_to_cell_path( path=backend_folder, cell=cell_name) proto_files = FileUtil.list_files_in_dir(folder) all_proto_files = all_proto_files.union(set(proto_files)) for proto_file in all_proto_files: storage = ProtoTableStorage() storage.initialize_from_file( file_name=proto_file ) raw_data = storage.read_all() if not raw_data: continue key = sorted(raw_data.keys())[-1] val = raw_data[key] result_proto = ProtoUtil.any_to_message( message_type=ContainerBackendValue, any_message=val ) ttl = result_proto.ttl if ttl > 0 and result_proto.updated_time and TimezoneUtil.cur_time_in_pst() - TimezoneUtil.cur_time_from_str( result_proto.updated_time) >= datetime.timedelta(days=ttl): FileUtil.remove_file(storage.get_file_name()) else: container_info = { 'container_name': result_proto.container_name, 'status': ProtoUtil.get_name_by_value( enum_type=Status, value=result_proto.container_status), 'updated_time': result_proto.updated_time, 'mode': ProtoUtil.get_name_by_value(enum_type=ModeType, value=result_proto.mode), 'data_model': ProtoUtil.get_name_by_value( enum_type=DataModelType, value=result_proto.data_model), 'run_cell': result_proto.run_cell, 'snapshot_cell': result_proto.snapshot_cell, } if container_info['container_name'] not in existing_containers: existing_containers[container_info['container_name']] = container_info['updated_time'] containers_info.append(container_info) else: if container_info['updated_time'] >= existing_containers[container_info['container_name']]: containers_info.append(container_info) return containers_info
def get_operator_snapshot(self, output_file=None): snapshot = OperatorSnapshot() snapshot.operator_name = self.get_node_name() snapshot.data_model = self.get_data_model() snapshot.status = self.get_status() snapshot.node_snapshot.CopyFrom(self.get_node_snapshot()) snapshot.class_name = self.get_full_class_name() snapshot.log_file = FileUtil.convert_local_to_cell_path(glogging.get_logger_file(self._logger)) if self._start_time: snapshot.start_time = str(self._start_time) if self.get_status() == Status.SUCCEEDED and self._end_time: snapshot.end_time = str(self._end_time) if self._persistent: snapshot.content.CopyFrom(self.content_serializer(self._content)) if output_file and self._config['save_snapshot'] and 'Dummy' not in self.get_class_name(): self._SYS_LOGGER.info("Saved to file " + output_file + '.') FileUtil.write_proto_to_file( proto=snapshot, file_name=output_file ) return snapshot
def get_container_info(container_name, cell_name, start_time): container_info = { 'log_file': '', 'start_time': '', 'end_time': '', 'counter_info': [], } operators_info = [] folder = FileUtil.convert_local_to_cell_path( path=backend_folder, cell=cell_name) pslx_frontend_logger.info( "Container backend checking folder [" + folder + '].') storage = ProtoTableStorage() storage.initialize_from_file( FileUtil.join_paths_to_file( root_dir=folder, base_name=container_name + '.pb' ) ) raw_data = storage.read_all() all_past_run = [] for key in sorted(list(raw_data.keys()), reverse=True): val = ProtoUtil.any_to_message( message_type=ContainerBackendValue, any_message=raw_data[key] ) all_past_run.append( { 'start_time': val.start_time, 'updated_time': val.updated_time, 'end_time': val.end_time, 'status': ProtoUtil.get_name_by_value( enum_type=Status, value=val.container_status), 'run_cell': val.run_cell, 'snapshot_cell': val.snapshot_cell, } ) if len(all_past_run) > 10: break key = start_time if start_time else sorted(raw_data.keys())[-1] val = raw_data[key] result_proto = ProtoUtil.any_to_message( message_type=ContainerBackendValue, any_message=val ) container_info['log_file'] = galaxy_viewer_url + result_proto.log_file container_info['start_time'] = result_proto.start_time container_info['end_time'] = result_proto.end_time for key in sorted(dict(result_proto.counters).keys()): container_info['counter_info'].append( { 'name': key, 'count': result_proto.counters[key], } ) for key, val in dict(result_proto.operator_info_map).items(): operators_info.append({ 'operator_name': key, 'status': ProtoUtil.get_name_by_value( enum_type=Status, value=val.status), 'start_time': val.start_time, 'end_time': val.end_time, 'dependencies': ', '.join(val.parents), 'log_file': galaxy_viewer_url + val.log_file, }) return (container_info, sorted(operators_info, key=lambda x: (x['dependencies'], x['operator_name'])), all_past_run)
def get_container_snapshot(self, send_backend=True): if not self._is_initialized: self._logger.error( "Warning: taking snapshot when the container [" + self.get_container_name() + "] is not initialized.") self._SYS_LOGGER.error( "Warning: taking snapshot when the container [" + self.get_container_name() + "] is not initialized.") snapshot = ContainerSnapshot() snapshot.container_name = self._container_name snapshot.is_initialized = self._is_initialized snapshot.status = self._status snapshot.class_name = self.get_full_class_name() snapshot.mode = self._mode snapshot.data_model = self.DATA_MODEL snapshot.log_file = FileUtil.convert_local_to_cell_path( glogging.get_logger_file(self._logger)) snapshot.run_cell = EnvUtil.get_other_env_variable( var='GALAXY_fs_cell', fallback_value='') snapshot.snapshot_cell = FileUtil.get_cell_from_path( FileUtil.convert_local_to_cell_path(self._snapshot_file_folder)) for key, val in self._counter.items(): snapshot.counters[key] = val if self._start_time: snapshot.start_time = str(self._start_time) if self._end_time: snapshot.end_time = str(self._end_time) for op_name, op in self._node_name_to_node_dict.items(): if 'Dummy' in op.get_class_name(): continue op_output_file = FileUtil.join_paths_to_file( root_dir=FileUtil.join_paths_to_dir( FileUtil.dir_name(self._snapshot_file_folder), 'operators'), base_name=op_name + '_SNAPSHOT_' + str(TimezoneUtil.cur_time_in_pst()) + '.pb') snapshot.operator_snapshot_map[op_name].CopyFrom( op.get_operator_snapshot(output_file=op_output_file)) self._SYS_LOGGER.info( "Snapshot saved to folder [" + FileUtil.convert_local_to_cell_path(self._snapshot_file_folder) + '].') self._logger.info( "Snapshot saved to folder [" + FileUtil.convert_local_to_cell_path(self._snapshot_file_folder) + '].') output_file_name = FileUtil.join_paths_to_file( root_dir=FileUtil.join_paths_to_dir( FileUtil.dir_name(self._snapshot_file_folder), 'containers'), base_name=self._container_name + '_SNAPSHOT_' + str(TimezoneUtil.cur_time_in_pst()) + '.pb') FileUtil.write_proto_to_file(proto=snapshot, file_name=output_file_name) if self._backend and send_backend: try: self._backend.send_to_backend(snapshot=snapshot) except Exception as err: self._logger.error("Sending backend failed with error " + str(err) + '.') return snapshot