def get_containers_info():
containers_info = []
existing_containers = {}
all_proto_files = set()
if not FileUtil.is_local_path(backend_folder):
all_cells = ['']
else:
all_cells = gclient.list_cells()
for cell_name in all_cells:
folder = FileUtil.convert_local_to_cell_path(
path=backend_folder, cell=cell_name)
proto_files = FileUtil.list_files_in_dir(folder)
all_proto_files = all_proto_files.union(set(proto_files))
for proto_file in all_proto_files:
storage = ProtoTableStorage()
storage.initialize_from_file(
file_name=proto_file
)
raw_data = storage.read_all()
if not raw_data:
continue
key = sorted(raw_data.keys())[-1]
val = raw_data[key]
result_proto = ProtoUtil.any_to_message(
message_type=ContainerBackendValue,
any_message=val
)
ttl = result_proto.ttl
if ttl > 0 and result_proto.updated_time and TimezoneUtil.cur_time_in_pst() - TimezoneUtil.cur_time_from_str(
result_proto.updated_time) >= datetime.timedelta(days=ttl):
FileUtil.remove_file(storage.get_file_name())
else:
container_info = {
'container_name': result_proto.container_name,
'status': ProtoUtil.get_name_by_value(
enum_type=Status, value=result_proto.container_status),
'updated_time': result_proto.updated_time,
'mode': ProtoUtil.get_name_by_value(enum_type=ModeType, value=result_proto.mode),
'data_model': ProtoUtil.get_name_by_value(
enum_type=DataModelType, value=result_proto.data_model),
'run_cell': result_proto.run_cell,
'snapshot_cell': result_proto.snapshot_cell,
}
if container_info['container_name'] not in existing_containers:
existing_containers[container_info['container_name']] = container_info['updated_time']
containers_info.append(container_info)
else:
if container_info['updated_time'] >= existing_containers[container_info['container_name']]:
containers_info.append(container_info)
return containers_info
def get_operator_snapshot(self, output_file=None):
snapshot = OperatorSnapshot()
snapshot.operator_name = self.get_node_name()
snapshot.data_model = self.get_data_model()
snapshot.status = self.get_status()
snapshot.node_snapshot.CopyFrom(self.get_node_snapshot())
snapshot.class_name = self.get_full_class_name()
snapshot.log_file = FileUtil.convert_local_to_cell_path(glogging.get_logger_file(self._logger))
if self._start_time:
snapshot.start_time = str(self._start_time)
if self.get_status() == Status.SUCCEEDED and self._end_time:
snapshot.end_time = str(self._end_time)
if self._persistent:
snapshot.content.CopyFrom(self.content_serializer(self._content))
if output_file and self._config['save_snapshot'] and 'Dummy' not in self.get_class_name():
self._SYS_LOGGER.info("Saved to file " + output_file + '.')
FileUtil.write_proto_to_file(
proto=snapshot,
file_name=output_file
)
return snapshot
def get_container_info(container_name, cell_name, start_time):
container_info = {
'log_file': '',
'start_time': '',
'end_time': '',
'counter_info': [],
}
operators_info = []
folder = FileUtil.convert_local_to_cell_path(
path=backend_folder, cell=cell_name)
pslx_frontend_logger.info(
"Container backend checking folder [" + folder + '].')
storage = ProtoTableStorage()
storage.initialize_from_file(
FileUtil.join_paths_to_file(
root_dir=folder,
base_name=container_name + '.pb'
)
)
raw_data = storage.read_all()
all_past_run = []
for key in sorted(list(raw_data.keys()), reverse=True):
val = ProtoUtil.any_to_message(
message_type=ContainerBackendValue,
any_message=raw_data[key]
)
all_past_run.append(
{
'start_time': val.start_time,
'updated_time': val.updated_time,
'end_time': val.end_time,
'status': ProtoUtil.get_name_by_value(
enum_type=Status, value=val.container_status),
'run_cell': val.run_cell,
'snapshot_cell': val.snapshot_cell,
}
)
if len(all_past_run) > 10:
break
key = start_time if start_time else sorted(raw_data.keys())[-1]
val = raw_data[key]
result_proto = ProtoUtil.any_to_message(
message_type=ContainerBackendValue,
any_message=val
)
container_info['log_file'] = galaxy_viewer_url + result_proto.log_file
container_info['start_time'] = result_proto.start_time
container_info['end_time'] = result_proto.end_time
for key in sorted(dict(result_proto.counters).keys()):
container_info['counter_info'].append(
{
'name': key,
'count': result_proto.counters[key],
}
)
for key, val in dict(result_proto.operator_info_map).items():
operators_info.append({
'operator_name': key,
'status': ProtoUtil.get_name_by_value(
enum_type=Status, value=val.status),
'start_time': val.start_time,
'end_time': val.end_time,
'dependencies': ', '.join(val.parents),
'log_file': galaxy_viewer_url + val.log_file,
})
return (container_info, sorted(operators_info, key=lambda x: (x['dependencies'], x['operator_name'])),
all_past_run)
def get_container_snapshot(self, send_backend=True):
if not self._is_initialized:
self._logger.error(
"Warning: taking snapshot when the container [" +
self.get_container_name() + "] is not initialized.")
self._SYS_LOGGER.error(
"Warning: taking snapshot when the container [" +
self.get_container_name() + "] is not initialized.")
snapshot = ContainerSnapshot()
snapshot.container_name = self._container_name
snapshot.is_initialized = self._is_initialized
snapshot.status = self._status
snapshot.class_name = self.get_full_class_name()
snapshot.mode = self._mode
snapshot.data_model = self.DATA_MODEL
snapshot.log_file = FileUtil.convert_local_to_cell_path(
glogging.get_logger_file(self._logger))
snapshot.run_cell = EnvUtil.get_other_env_variable(
var='GALAXY_fs_cell', fallback_value='')
snapshot.snapshot_cell = FileUtil.get_cell_from_path(
FileUtil.convert_local_to_cell_path(self._snapshot_file_folder))
for key, val in self._counter.items():
snapshot.counters[key] = val
if self._start_time:
snapshot.start_time = str(self._start_time)
if self._end_time:
snapshot.end_time = str(self._end_time)
for op_name, op in self._node_name_to_node_dict.items():
if 'Dummy' in op.get_class_name():
continue
op_output_file = FileUtil.join_paths_to_file(
root_dir=FileUtil.join_paths_to_dir(
FileUtil.dir_name(self._snapshot_file_folder),
'operators'),
base_name=op_name + '_SNAPSHOT_' +
str(TimezoneUtil.cur_time_in_pst()) + '.pb')
snapshot.operator_snapshot_map[op_name].CopyFrom(
op.get_operator_snapshot(output_file=op_output_file))
self._SYS_LOGGER.info(
"Snapshot saved to folder [" +
FileUtil.convert_local_to_cell_path(self._snapshot_file_folder) +
'].')
self._logger.info(
"Snapshot saved to folder [" +
FileUtil.convert_local_to_cell_path(self._snapshot_file_folder) +
'].')
output_file_name = FileUtil.join_paths_to_file(
root_dir=FileUtil.join_paths_to_dir(
FileUtil.dir_name(self._snapshot_file_folder), 'containers'),
base_name=self._container_name + '_SNAPSHOT_' +
str(TimezoneUtil.cur_time_in_pst()) + '.pb')
FileUtil.write_proto_to_file(proto=snapshot,
file_name=output_file_name)
if self._backend and send_backend:
try:
self._backend.send_to_backend(snapshot=snapshot)
except Exception as err:
self._logger.error("Sending backend failed with error " +
str(err) + '.')
return snapshot
