def untrack(files: T.Iterable[T.Path]) -> None:
""" Untrack the given files """
for f in files:
if not file.is_regular(f):
# Skip non-regular files
log.warning(f"Cannot untrack {f}: Doesn't exist or is not regular")
log.info(
"Contact HGI if a file exists in the vault, but has been deleted outside"
)
continue
vault = _create_vault(f)
if (branch := vault.branch(f)) is not None:
try:
vault.remove(branch, f)
except core.vault.exception.VaultCorruption as e:
# Corruption detected
log.critical(f"Corruption detected: {e}")
log.info("Contact HGI to resolve this corruption")
except core.vault.exception.PermissionDenied as e:
# User doesn't have permission to remove files
log.error(f"Permission denied: {e}")
except core.idm.exception.NoSuchIdentity as e:
# IdM doesn't know about the vault's group
log.critical(f"Unknown vault group: {e}")
log.info("Contact HGI to resolve this inconsistency")
except core.vault.exception.PhysicalVaultFile:
# This wouldn't make sense, so we just skip it sans log
pass
def _create_vault(relative_to: T.Path,
idm: IDMBase.IdentityManager = idm) -> Vault:
# Defensively create a vault with the common IdM
try:
return Vault(relative_to, idm=idm, min_owners=config.min_group_owners)
except (core.vault.exception.VaultConflict,
core.vault.exception.MinimumNumberOfOwnersNotMet) as e:
# Non-managed file(s) exists where the vault should be
log.critical(e)
sys.exit(1)
def drain(persistence: core.persistence.base.Persistence, *,
force: bool = False) -> int:
""" Drain phase """
handler = _Handler(config.archive.handler)
criteria = Filter(state=State.Staged(notified=True), stakeholder=Anything)
try:
with persistence.files(criteria) as staging_queue:
# NOTE The returned files will be purged on exit of this
# context manager. An exception MUST be raised to avoid that
# (e.g., if we need to cancel the drain, or if the
# downstream handler fails, etc.)
if (count := len(staging_queue)) == 0:
raise _StagingQueueEmpty()
if count < config.archive.threshold and not force:
raise _StagingQueueUnderThreshold(
f"Only {count} files to archive; use --force-drain to ignore the threshold")
required_capacity = staging_queue.accumulator
log.info(
f"Checking downstream handler is ready for {human_size(required_capacity)}B...")
handler.preflight(required_capacity)
log.info("Handler is ready; beginning drain...")
handler.consume(f.key for f in staging_queue)
log.info(
f"Successfully drained {count} files into the downstream handler")
except _StagingQueueEmpty:
log.info("Staging queue is empty")
except _StagingQueueUnderThreshold as e:
log.info(f"Skipping: {e}")
except _HandlerBusy:
log.warning("The downstream handler is busy; try again later...")
except _DownstreamFull:
log.error(
"The downstream handler is reporting it is out of capacity and cannot proceed")
return 1
except _UnknownHandlerError:
log.critical(
"The downstream handler failed unexpectedly; please check its logs for details...")
return 1
return 0
def main(argv: T.List[str] = sys.argv) -> None:
args = usage.parse_args(argv[1:])
log.info("Enter Sandman")
# Cheery thoughts
if args.weaponise:
log.warning("Weaponised: Now I am become Death, "
"the destroyer of worlds")
else:
log.info("Dry Run: The filesystem will not be affected "
"and the drain phase will not run")
persistence = Persistence(config.persistence, idm)
# Sweep Phase
log.info("Starting the sweep phase")
try:
if args.stats is not None:
log.info(f"Walking mpistat output from {args.stats}")
log.warning("mpistat data may not be up to date")
walker = mpistatWalker(args.stats, *args.vaults)
else:
log.info("Walking the filesystem directly")
log.warning("This is an expensive operation")
walker = FilesystemWalker(*args.vaults)
except InvalidVaultBases as e:
# Safety checks failed on input Vault paths
log.critical(e)
sys.exit(1)
Sweeper(walker, persistence, args.weaponise)
# Drain Phase
if args.weaponise:
log.info("Starting the drain phase")
if (exit_code := drain(persistence, force=args.force_drain)) != 0:
sys.exit(exit_code)
def add(branch: Branch, files: T.Iterable[T.Path]) -> None:
""" Add the given files to the appropriate branch """
for f in files:
if not file.is_regular(f):
# Skip non-regular files
log.warning(f"Cannot add {f}: Doesn't exist or is not regular")
continue
try:
vault = _create_vault(f)
vault.add(branch, f)
except core.vault.exception.VaultCorruption as e:
# Corruption detected
log.critical(f"Corruption detected: {e}")
log.info("Contact HGI to resolve this corruption")
except core.vault.exception.PermissionDenied as e:
# User does have permission to add files
log.error(f"Permission denied: {e}")
except core.vault.exception.PhysicalVaultFile as e:
# Trying to add a vault file to the vault
log.error(f"Cannot add: {e}")
from core import config, typing as T
from api.config import Config
from api.idm import IdentityManager
from api.logging import log
# Executable versioning
class version(T.SimpleNamespace):
vault = "0.0.9"
sandman = "0.0.7"
# Common configuration
try:
if 'unittest' in sys.modules:
os.environ["VAULTRC"] = "eg/.vaultrc"
_cfg_path = config.utils.path("VAULTRC", T.Path("~/.vaultrc"),
T.Path("/etc/vaultrc"))
config = Config(_cfg_path)
except (config.exception.ConfigurationNotFound,
config.exception.InvalidConfiguration) as e:
log.critical(e)
sys.exit(1)
# Common identity manager
# NOTE This is mutable global state...which is not cool, but "should be
# fine"™ provided we maintain serial execution
idm = IdentityManager(config.identity)