def _delete_file(self, cur_time, path_name): num_file_removed, num_file_failed = 0, 0 if FileUtil.is_file(path_name=path_name): ttl = FileUtil.get_ttl_from_path(path_name=path_name) if ttl and cur_time - FileUtil.get_file_modified_time( file_name=path_name) > ttl: self._logger.info("Removing file " + path_name + '...') try: with FileLockTool(protected_file_path=path_name, read_mode=True, timeout=TimeSleepObj.ONE_TENTH_SECOND): FileUtil.remove_file(file_name=path_name) num_file_removed += 1 self.counter_increment("num_file_removed") except Exception as err: num_file_failed += 1 self.counter_increment("num_file_failed_to_be_removed") self._logger.error("Removing file " + path_name + ' failed with err ' + str(err) + '.') else: for file_name in FileUtil.list_files_in_dir(dir_name=path_name): stats = self._delete_file(cur_time=cur_time, path_name=file_name) num_file_removed += stats[0] num_file_failed += stats[1] for dir_name in FileUtil.list_dirs_in_dir(dir_name=path_name): stats = self._delete_file(cur_time=cur_time, path_name=dir_name) num_file_removed += stats[0] num_file_failed += stats[1] return num_file_removed, num_file_failed
def get_containers_info(): containers_info = [] existing_containers = {} all_proto_files = set() if not FileUtil.is_local_path(backend_folder): all_cells = [''] else: all_cells = gclient.list_cells() for cell_name in all_cells: folder = FileUtil.convert_local_to_cell_path( path=backend_folder, cell=cell_name) proto_files = FileUtil.list_files_in_dir(folder) all_proto_files = all_proto_files.union(set(proto_files)) for proto_file in all_proto_files: storage = ProtoTableStorage() storage.initialize_from_file( file_name=proto_file ) raw_data = storage.read_all() if not raw_data: continue key = sorted(raw_data.keys())[-1] val = raw_data[key] result_proto = ProtoUtil.any_to_message( message_type=ContainerBackendValue, any_message=val ) ttl = result_proto.ttl if ttl > 0 and result_proto.updated_time and TimezoneUtil.cur_time_in_pst() - TimezoneUtil.cur_time_from_str( result_proto.updated_time) >= datetime.timedelta(days=ttl): FileUtil.remove_file(storage.get_file_name()) else: container_info = { 'container_name': result_proto.container_name, 'status': ProtoUtil.get_name_by_value( enum_type=Status, value=result_proto.container_status), 'updated_time': result_proto.updated_time, 'mode': ProtoUtil.get_name_by_value(enum_type=ModeType, value=result_proto.mode), 'data_model': ProtoUtil.get_name_by_value( enum_type=DataModelType, value=result_proto.data_model), 'run_cell': result_proto.run_cell, 'snapshot_cell': result_proto.snapshot_cell, } if container_info['container_name'] not in existing_containers: existing_containers[container_info['container_name']] = container_info['updated_time'] containers_info.append(container_info) else: if container_info['updated_time'] >= existing_containers[container_info['container_name']]: containers_info.append(container_info) return containers_info