def _delete_file(self, cur_time, path_name):
num_file_removed, num_file_failed = 0, 0
if FileUtil.is_file(path_name=path_name):
ttl = FileUtil.get_ttl_from_path(path_name=path_name)
if ttl and cur_time - FileUtil.get_file_modified_time(
file_name=path_name) > ttl:
self._logger.info("Removing file " + path_name + '...')
try:
with FileLockTool(protected_file_path=path_name,
read_mode=True,
timeout=TimeSleepObj.ONE_TENTH_SECOND):
FileUtil.remove_file(file_name=path_name)
num_file_removed += 1
self.counter_increment("num_file_removed")
except Exception as err:
num_file_failed += 1
self.counter_increment("num_file_failed_to_be_removed")
self._logger.error("Removing file " + path_name +
' failed with err ' + str(err) + '.')
else:
for file_name in FileUtil.list_files_in_dir(dir_name=path_name):
stats = self._delete_file(cur_time=cur_time,
path_name=file_name)
num_file_removed += stats[0]
num_file_failed += stats[1]
for dir_name in FileUtil.list_dirs_in_dir(dir_name=path_name):
stats = self._delete_file(cur_time=cur_time,
path_name=dir_name)
num_file_removed += stats[0]
num_file_failed += stats[1]
return num_file_removed, num_file_failed
def get_containers_info():
containers_info = []
existing_containers = {}
all_proto_files = set()
if not FileUtil.is_local_path(backend_folder):
all_cells = ['']
else:
all_cells = gclient.list_cells()
for cell_name in all_cells:
folder = FileUtil.convert_local_to_cell_path(
path=backend_folder, cell=cell_name)
proto_files = FileUtil.list_files_in_dir(folder)
all_proto_files = all_proto_files.union(set(proto_files))
for proto_file in all_proto_files:
storage = ProtoTableStorage()
storage.initialize_from_file(
file_name=proto_file
)
raw_data = storage.read_all()
if not raw_data:
continue
key = sorted(raw_data.keys())[-1]
val = raw_data[key]
result_proto = ProtoUtil.any_to_message(
message_type=ContainerBackendValue,
any_message=val
)
ttl = result_proto.ttl
if ttl > 0 and result_proto.updated_time and TimezoneUtil.cur_time_in_pst() - TimezoneUtil.cur_time_from_str(
result_proto.updated_time) >= datetime.timedelta(days=ttl):
FileUtil.remove_file(storage.get_file_name())
else:
container_info = {
'container_name': result_proto.container_name,
'status': ProtoUtil.get_name_by_value(
enum_type=Status, value=result_proto.container_status),
'updated_time': result_proto.updated_time,
'mode': ProtoUtil.get_name_by_value(enum_type=ModeType, value=result_proto.mode),
'data_model': ProtoUtil.get_name_by_value(
enum_type=DataModelType, value=result_proto.data_model),
'run_cell': result_proto.run_cell,
'snapshot_cell': result_proto.snapshot_cell,
}
if container_info['container_name'] not in existing_containers:
existing_containers[container_info['container_name']] = container_info['updated_time']
containers_info.append(container_info)
else:
if container_info['updated_time'] >= existing_containers[container_info['container_name']]:
containers_info.append(container_info)
return containers_info
