def untrack(files: T.Iterable[T.Path]) -> None:
""" Untrack the given files """
for f in files:
if not file.is_regular(f):
# Skip non-regular files
log.warning(f"Cannot untrack {f}: Doesn't exist or is not regular")
log.info(
"Contact HGI if a file exists in the vault, but has been deleted outside"
)
continue
vault = _create_vault(f)
if (branch := vault.branch(f)) is not None:
try:
vault.remove(branch, f)
except core.vault.exception.VaultCorruption as e:
# Corruption detected
log.critical(f"Corruption detected: {e}")
log.info("Contact HGI to resolve this corruption")
except core.vault.exception.PermissionDenied as e:
# User doesn't have permission to remove files
log.error(f"Permission denied: {e}")
except core.idm.exception.NoSuchIdentity as e:
# IdM doesn't know about the vault's group
log.critical(f"Unknown vault group: {e}")
log.info("Contact HGI to resolve this inconsistency")
except core.vault.exception.PhysicalVaultFile:
# This wouldn't make sense, so we just skip it sans log
pass
def drain(persistence: core.persistence.base.Persistence, *,
force: bool = False) -> int:
""" Drain phase """
handler = _Handler(config.archive.handler)
criteria = Filter(state=State.Staged(notified=True), stakeholder=Anything)
try:
with persistence.files(criteria) as staging_queue:
# NOTE The returned files will be purged on exit of this
# context manager. An exception MUST be raised to avoid that
# (e.g., if we need to cancel the drain, or if the
# downstream handler fails, etc.)
if (count := len(staging_queue)) == 0:
raise _StagingQueueEmpty()
if count < config.archive.threshold and not force:
raise _StagingQueueUnderThreshold(
f"Only {count} files to archive; use --force-drain to ignore the threshold")
required_capacity = staging_queue.accumulator
log.info(
f"Checking downstream handler is ready for {human_size(required_capacity)}B...")
handler.preflight(required_capacity)
log.info("Handler is ready; beginning drain...")
handler.consume(f.key for f in staging_queue)
log.info(
f"Successfully drained {count} files into the downstream handler")
except _StagingQueueEmpty:
log.info("Staging queue is empty")
except _StagingQueueUnderThreshold as e:
log.info(f"Skipping: {e}")
except _HandlerBusy:
log.warning("The downstream handler is busy; try again later...")
except _DownstreamFull:
log.error(
"The downstream handler is reporting it is out of capacity and cannot proceed")
return 1
except _UnknownHandlerError:
log.critical(
"The downstream handler failed unexpectedly; please check its logs for details...")
return 1
return 0
def consume(self, files: T.Iterator[T.Path]) -> None:
"""
Drain the files, NULL-delimited, through the handler's stdin
@param files File queue
@raises _UnknownHandlerError Handler did not accept the queue
"""
log = self.log
handler = subprocess.Popen(self._handler, stdin=PIPE,
stdout=DEVNULL,
stderr=DEVNULL)
for file in files:
if not is_regular(file):
log.error(
"Skipping: {file} is not a regular file or does not exist")
continue
log.info(f"Draining: {file}")
handler.stdin.write(bytes(file))
handler.stdin.write(b"\0")
handler.stdin.close()
if handler.wait() != 0:
raise _UnknownHandlerError()
def add(branch: Branch, files: T.Iterable[T.Path]) -> None:
""" Add the given files to the appropriate branch """
for f in files:
if not file.is_regular(f):
# Skip non-regular files
log.warning(f"Cannot add {f}: Doesn't exist or is not regular")
continue
try:
vault = _create_vault(f)
vault.add(branch, f)
except core.vault.exception.VaultCorruption as e:
# Corruption detected
log.critical(f"Corruption detected: {e}")
log.info("Contact HGI to resolve this corruption")
except core.vault.exception.PermissionDenied as e:
# User does have permission to add files
log.error(f"Permission denied: {e}")
except core.vault.exception.PhysicalVaultFile as e:
# Trying to add a vault file to the vault
log.error(f"Cannot add: {e}")
def recover(files: T.Optional[T.Iterable[T.Path]] = None) -> None:
"""
Recover the given files from Limbo branch or Recover all files from
the Limbo branch
@ param list of file paths relative to the working directory
example: ["../file1" "file1"]
"""
cwd = file.cwd()
vault = _create_vault(cwd)
bpath = vault.location / Branch.Limbo
RECOVER_ALL = files is None
if not RECOVER_ALL:
vault_root = vault.root
files_to_recover = {
vault_root / derelativise(path, cwd, vault_root): path
for path in files
}
for dirname, _, files in os.walk(bpath):
for f in files:
limbo_relative_path = T.Path(dirname, f).relative_to(bpath)
try:
vfk = VaultFileKey.Reconstruct(limbo_relative_path)
except Exception as e:
raise core.vault.exception.VaultCorruption(
f"Failed to reconstruct VaultFileKey for {limbo_relative_path}"
)
original_file = vault.root / vfk.source
vfkpath = bpath / vfk.path
if RECOVER_ALL or original_file in files_to_recover:
try:
move_with_path_safety_checks(vfkpath, original_file)
except RecoverExc.NoSourceFound as e:
log.error(f"Recovery source {vfkpath} does not exist: {e}")
except RecoverExc.NoParentForDestination:
log.error(
f"Source path exists {vfkpath} but destination parent {original_file.parent} does not exist"
)
except RecoverExc.DestinationAlreadyExists:
log.error(
f"Destination {original_file} already has an existing file"
)