def run(cls, command, args, settings, options):
# read command line
cmdline = docopt(cls.USAGE, argv=[command] + args)
paths = cmdline['<path>']
archive = cmdline['--archive']
date = cmdline['--date']
# make sure source directories are given as absolute paths
for src_dir in settings.src_dirs:
if not src_dir.is_absolute():
raise Error('restore command cannot be used',
'with relative source directories',
culprit=src_dir)
# convert to absolute resolved paths
paths = [to_path(p).resolve() for p in paths]
# assure that paths correspond to src_dirs
src_dirs = settings.src_dirs
unknown_path = False
for path in paths:
if not any([str(path).startswith(str(sd)) for sd in src_dirs]):
unknown_path = True
warn('unknown path.', culprit=path)
if unknown_path:
codicil('Paths should start with:', conjoin(src_dirs,
conj=', or '))
# remove leading / from paths
paths = [str(p).lstrip('/') for p in paths]
# get the desired archive
if date and not archive:
archive = get_name_of_nearest_archive(settings, date)
if not archive:
archive = get_name_of_latest_archive(settings)
output('Archive:', archive)
# run borg
cd('/')
borg = settings.run_borg(
cmd='extract',
args=[settings.destination(archive)] + paths,
emborg_opts=options,
)
out = borg.stdout
if out:
output(out.rstrip())
def test_run_layer():
with cd(wd):
cmd = "./test_prog 3"
p = Run(cmd, "sOEW0,3")
assert p.stdout == "this is stdout.\n"
assert p.stderr == "this is stderr.\n"
assert p.status == 3
def test_ls_abominate():
"""recursive list of directory"""
# setup
mkdir("work")
with cd("work"):
d1 = to_path("d1")
mkdir(d1)
d1d1 = to_path("d1/d1")
mkdir(d1d1)
d1d2 = to_path("d1/d2")
mkdir(d1d2)
d1d1f1 = to_path("d1/d1/f1")
touch(d1d1f1)
d1d2f2 = to_path("d1/d2/f2")
touch(d1d2f2)
# run test
paths = ls(".", select="**/*")
# check
assert set(str(f) for f in paths) == set(
["d1", "d1/d1", "d1/d2", "d1/d1/f1", "d1/d2/f2"])
# cleanup
rm("work")
def test_run_downturn():
with cd(wd):
cmd = "./test_prog 0"
p = Run(cmd, "sOEW")
assert p.stdout == "this is stdout.\n"
assert p.stderr == "this is stderr.\n"
assert p.status == 0
def test_ls_abominate():
"""recursive list of directory"""
# setup
mkdir('work')
with cd('work'):
d1 = to_path('d1')
mkdir(d1)
d1d1 = to_path('d1/d1')
mkdir(d1d1)
d1d2 = to_path('d1/d2')
mkdir(d1d2)
d1d1f1 = to_path('d1/d1/f1')
touch(d1d1f1)
d1d2f2 = to_path('d1/d2/f2')
touch(d1d2f2)
# run test
paths = ls('.', select='**/*')
# check
assert set(str(f) for f in paths) == set(
['d1', 'd1/d1', 'd1/d2', 'd1/d1/f1', 'd1/d2/f2'])
# cleanup
rm('work')
def run_borg_raw(self, args):
# prepare the command
os.environ.update(self.publish_passcode())
os.environ["BORG_DISPLAY_PASSPHRASE"] = "no"
executable = self.value("borg_executable", BORG)
remote_path = self.value("remote_path")
remote_path = ["--remote-path", remote_path] if remote_path else []
repository = str(self.repository)
command = ([executable] + remote_path +
[a.replace('@repo', repository) for a in args])
# run the command
narrate("running:\n{}".format(
indent(render_command(command, borg_options_arg_count))))
with cd(self.working_dir):
narrate("running in:", cwd())
starts_at = arrow.now()
log("starts at: {!s}".format(starts_at))
borg = Run(command, modes="soeW", env=os.environ, log=False)
ends_at = arrow.now()
log("ends at: {!s}".format(ends_at))
log("elapsed = {!s}".format(ends_at - starts_at))
if borg.stdout:
narrate("Borg stdout:")
narrate(indent(borg.stdout))
if borg.stderr:
narrate("Borg stderr:")
narrate(indent(borg.stderr))
if borg.status:
narrate("Borg exit status:", borg.status)
return borg
def test_ls_abominate():
"""recursive list of directory"""
# setup
mkdir('work')
with cd('work'):
d1 = to_path('d1')
mkdir(d1)
d1d1 = to_path('d1/d1')
mkdir(d1d1)
d1d2 = to_path('d1/d2')
mkdir(d1d2)
d1d1f1 = to_path('d1/d1/f1')
touch(d1d1f1)
d1d2f2 = to_path('d1/d2/f2')
touch(d1d2f2)
# run test
paths = ls('.', select='**/*')
# check
assert set(str(f) for f in paths) == set(
['d1', 'd1/d1', 'd1/d2', 'd1/d1/f1', 'd1/d2/f2']
)
# cleanup
rm('work')
def test_run_entree():
with cd(wd):
cmd = "./test_prog 1"
p = Run(cmd, "sOEW1")
assert p.stdout == "this is stdout.\n"
assert p.stderr == "this is stderr.\n"
assert p.status == 1
def encrypt_archive(config, workspace, passcode):
narrate("Encrypting the archive")
with cd(workspace):
run('tar -cf archive.tgz archive', 'soEW')
with open('archive.tgz', 'rb') as f:
cleartext = f.read()
gpg = GPG()
encrypted = gpg.encrypt(
cleartext,
recipients=None,
symmetric=True,
passphrase=str(passcode),
)
if encrypted.ok:
with open('archive.tgz.gpg', 'w') as f:
f.write(str(encrypted))
else:
raise EncryptionFailed(encrypted)
rm('archive.tgz', 'archive')
script = workspace / 'decrypt.sh'
script.write_text('''\
#!/bin/sh
# Decrypts the archive.
gpg -d -o - archive.tgz.gpg | tar xvf -
''')
chmod(0o700, script, workspace / 'archive.tgz.gpg')
narrate(f"Local archive '{workspace.name}' created.")
def test_run_endorse():
with cd(wd):
cmd = "./test_prog 1"
try:
Run(cmd, "sOEW")
assert False, "expected exception"
except OSError as e:
assert str(e) == "[Errno None] this is stderr."
def test_run_parakeet():
with cd(wd):
cmd = "./test_prog 3"
try:
Run(cmd, "sOEW0,1,2,4")
assert False, "expected exception"
except OSError as e:
assert str(e) == "[Errno None] this is stderr."
def initialize_configs(initialize):
with cd(tests_dir):
cp("CONFIGS", ".config/emborg")
rm(".config/emborg/subdir")
for p in lsf(".config/emborg"):
contents = p.read_text()
contents = contents.replace('⟪EMBORG⟫', emborg_dir)
p.write_text(contents)
touch(".config/.nobackup")
def initialize():
with cd(tests_dir):
rm("configs .config .local repositories configs.symlink".split())
cp("CONFIGS", "configs")
mkdir(".config repositories .local".split())
ln("~/.local/bin", ".local/bin")
ln("~/.local/lib", ".local/lib")
ln("configs", "configs.symlink")
os.environ["HOME"] = str(cwd())
def test_cryptocurrency():
if sys.version_info < (3, 8):
return # cryptocurrency example uses walrus operator
with cd(tests_dir):
# the cryptocurrency example outputs the current prices, so just do some
# simple sanity checking on the output.
cc = Run('./cryptocurrency', modes='sOEW')
assert '5 BTC =' in cc.stdout
assert '50 ETH =' in cc.stdout
assert '50 kXLM =' in cc.stdout
def test_postmortem():
if sys.version_info < (3, 8):
return # postmortem example uses walrus operator
with cd(tests_dir):
expected = Path('postmortem.json').read_text()
user_home_dir = str(to_path('~'))
expected = expected.replace('~', user_home_dir)
pm = Run('./postmortem', modes='sOEW')
assert pm.stdout.strip() == expected.strip()
def __enter__(self):
# change to working directory
working_dir = self.value('working_dir')
if not working_dir:
working_dir = self.resolve(DEFAULT_WORKING_DIR)
self.working_dir = to_path(working_dir)
mkdir(self.working_dir)
narrate('changing to working_dir:', working_dir)
self.starting_dir = cd(self.working_dir).starting_dir
# resolve src and dest directories
src_dir = self.resolve(self.src_dir)
self.src_dir = to_path(src_dir)
dest_dir = self.resolve(self.dest_dir)
self.dest_dir = to_path(dest_dir)
# resolve other files and directories
config_dir = self.resolve(CONFIG_DIR)
self.config_dir = to_path(config_dir, config_dir)
logfile = self.resolve(EMBALM_LOG_FILE)
self.logfile = to_path(working_dir, logfile)
incr_date_file = self.resolve(INCR_DATE_FILE)
self.incr_date_file = to_path(working_dir, incr_date_file)
full_date_file = self.resolve(FULL_DATE_FILE)
self.full_date_file = to_path(working_dir, full_date_file)
restore_dir = self.resolve(RESTORE_DIR)
self.restore_dir = to_path(working_dir, restore_dir)
archive_dir = self.resolve(ARCHIVE_DIR)
self.archive_dir = to_path(working_dir, archive_dir)
# perform locking
if self.requires_exclusivity:
# check for existance of lockfile
lockfile = self.lockfile = to_path(working_dir, LOCK_FILE)
if lockfile.exists():
raise Error(f'currently running (see {lockfile} for details).')
# create lockfile
now = arrow.now()
pid = os.getpid()
lockfile.write_text(
dedent(f'''
started = {now!s}
pid = {pid}
''').lstrip())
# open logfile
get_informer().set_logfile(self.logfile)
return self
def test_cd_downturn():
"""change into directory"""
# setup
dot = cwd()
d1 = to_path('d1')
mkdir(d1)
d1p = to_path(d1).resolve()
d1f1 = to_path('d1/f1')
touch(d1f1)
f1 = to_path('f1')
rm(f1)
# run test
cd(d1)
assert to_path(f1).is_file()
assert str(d1p) == str(cwd())
cd('..')
assert str(dot) == str(cwd())
# clean up
rm(d1)
def test_cd_downturn():
"""change into directory"""
# setup
dot = cwd()
d1 = to_path('d1')
mkdir(d1)
d1p = to_path(d1).resolve()
d1f1 = to_path('d1/f1')
touch(d1f1)
f1 = to_path('f1')
rm(f1)
# run test
cd(d1)
assert to_path(f1).is_file()
assert str(d1p) == str(cwd())
cd('..')
assert str(dot) == str(cwd())
# clean up
rm(d1)
def test_emborg_with_configs(initialize_configs, name, args, expected,
expected_type, cmp_dirs, remove, dependencies):
if skip_test(dependencies):
return
with cd(tests_dir):
tester = EmborgTester(args, expected, expected_type, cmp_dirs, remove)
passes = tester.run()
if not passes:
result = tester.get_result()
expected = tester.get_expected()
assert result == expected, name
raise AssertionError('test code failure')
def test_cd_endorse():
"""change into directory"""
# setup
dot = cwd()
d1 = "d1"
mkdir(d1)
d1p = to_path(d1).resolve()
d1f1 = "d1/f1"
touch(d1f1)
f1 = "f1"
rm(f1)
# run test
cd(d1)
assert to_path(f1).is_file()
assert str(d1p) == str(cwd())
cd("..")
assert str(dot) == str(cwd())
# clean up
rm(d1)
def test_emborg_api(initialize):
with cd(tests_dir):
from emborg import Emborg
with Emborg('tests') as emborg:
configs = emborg.configs
assert configs == 'test0 test1 test2 test3'.split()
for config in configs:
# get the name of latest archive
with Emborg(config) as emborg:
borg = emborg.run_borg(cmd='list',
args=['--json',
emborg.destination()])
response = json.loads(borg.stdout)
print(response)
archive = response['archives'][-1]['archive']
# list files in latest archive
borg = emborg.run_borg(
cmd='list',
args=['--json-lines',
emborg.destination(archive)])
json_data = '[' + ','.join(borg.stdout.splitlines()) + ']'
response = json.loads(json_data)
paths = sorted([entry['path'] for entry in response])
for each in [
'⟪EMBORG⟫/tests/configs',
'⟪EMBORG⟫/tests/configs/README',
'⟪EMBORG⟫/tests/configs/overdue.conf',
'⟪EMBORG⟫/tests/configs/settings',
'⟪EMBORG⟫/tests/configs/subdir',
'⟪EMBORG⟫/tests/configs/subdir/file',
'⟪EMBORG⟫/tests/configs/test0',
'⟪EMBORG⟫/tests/configs/test1',
'⟪EMBORG⟫/tests/configs/test2',
'⟪EMBORG⟫/tests/configs/test2excludes',
'⟪EMBORG⟫/tests/configs/test2passphrase',
'⟪EMBORG⟫/tests/configs/test3',
'⟪EMBORG⟫/tests/configs/test4',
'⟪EMBORG⟫/tests/configs/test5',
'⟪EMBORG⟫/tests/configs/test6',
'⟪EMBORG⟫/tests/configs/test6patterns',
'⟪EMBORG⟫/tests/configs/test7',
'⟪EMBORG⟫/tests/configs/test7patterns',
'⟪EMBORG⟫/tests/configs/test8',
]:
each = each.replace("⟪EMBORG⟫", emborg_dir_wo_slash)
assert each in paths
def test_cwd_downturn():
"""change into directory"""
# setup
d1 = to_path('d1')
mkdir(d1)
# run test
bef = cwd()
with cd(d1):
aft = cwd()
delta = aft.relative_to(bef)
assert str(delta) == 'd1'
# cleanup
rm(d1)
def test_cwd_downturn():
"""change into directory"""
# setup
d1 = to_path("d1")
mkdir(d1)
# run test
bef = cwd()
with cd(d1):
aft = cwd()
delta = aft.relative_to(bef)
assert str(delta) == "d1"
# cleanup
rm(d1)
def test_run_ground():
with cd(wd):
Inform(prog_name=False, logfile=False)
set_prefs(use_inform=True)
cmd = "./test_prog 1"
try:
Run(cmd, "sOEW")
assert False, "expected exception"
except Error as e:
assert str(e) == "this is stderr."
assert e.cmd == "./test_prog 1"
assert e.stdout == "this is stdout."
assert e.stderr == "this is stderr."
assert e.status == 1
assert e.msg == "this is stderr."
set_prefs(use_inform=False)
def test_address():
if sys.version_info < (3, 8):
return # address example uses walrus operator
with cd(tests_dir):
cc = Run('./address fumiko', modes='sOEW')
assert cc.stdout.strip() == dedent("""
Fumiko Purvis
position: treasurer
address:
3636 Buffalo Ave
Topeka, Kansas 20692
phone: 1-268-555-0280
email: [email protected]
additional roles:
- accounting task force
""").strip()
def test_emborg_overdue(initialize, name, conf, args, expected, expected_type,
dependencies):
if skip_test(dependencies):
return
with cd(tests_dir):
if conf:
with open('.config/overdue.conf', 'w') as f:
f.write(conf)
try:
args = args.split() if is_str(args) else args
overdue = Run(emborg_overdue_exe + args, "sOEW")
if 'regex' in expected_type.split():
assert bool(re.fullmatch(expected, overdue.stdout)), name
else:
assert expected == overdue.stdout, name
except Error as e:
assert str(e) == expected, name
def test_cd_quoit():
"""change into directory"""
# setup
dot = cwd()
d1 = 'd1'
mkdir(d1)
d1p = to_path(d1).resolve()
d1f1 = 'd1/f1'
touch(d1f1)
f1 = 'f1'
rm(f1)
# run test
with cd(d1):
assert to_path(f1).is_file()
assert str(d1p) == str(cwd())
assert str(dot) == str(cwd())
# clean up
rm(d1)
def test_cd_thinner():
"""change into directory"""
# setup
dot = cwd()
d1 = to_path("d1")
mkdir(d1)
d1p = to_path(d1).resolve()
d1f1 = to_path("d1/f1")
touch(d1f1)
f1 = to_path("f1")
rm(f1)
# run test
with cd(d1):
assert to_path(f1).is_file()
assert str(d1p) == str(cwd())
assert str(dot) == str(cwd())
# clean up
rm(d1)
def test_cd_quoit():
"""change into directory"""
# setup
dot = cwd()
d1 = 'd1'
mkdir(d1)
d1p = to_path(d1).resolve()
d1f1 = 'd1/f1'
touch(d1f1)
f1 = 'f1'
rm(f1)
# run test
with cd(d1):
assert to_path(f1).is_file()
assert str(d1p) == str(cwd())
assert str(dot) == str(cwd())
# clean up
rm(d1)
def run_borg_raw(self, args):
# run the run_before_borg commands
self.run_user_commands('run_before_borg')
# prepare the command
self.publish_passcode()
os.environ["BORG_DISPLAY_PASSPHRASE"] = "no"
executable = self.value("borg_executable", BORG)
remote_path = self.value("remote_path")
remote_path = ["--remote-path", remote_path] if remote_path else []
repository = str(self.repository)
command = ([executable] + remote_path +
[a.replace('@repo', repository) for a in args])
# run the command
narrate("running:\n{}".format(
indent(render_command(command, borg_options_arg_count))))
with cd(self.working_dir):
narrate("running in:", cwd())
starts_at = arrow.now()
log("starts at: {!s}".format(starts_at))
try:
borg = Run(command, modes="soeW1", env=os.environ, log=False)
except Error as e:
self.report_borg_error(e, executable)
ends_at = arrow.now()
log("ends at: {!s}".format(ends_at))
log("elapsed = {!s}".format(ends_at - starts_at))
if borg.status == 1:
warn('Warning emitted by Borg, see logfile for details.')
if borg.stdout:
narrate("Borg stdout:")
narrate(indent(borg.stdout))
if borg.stderr:
narrate("Borg stderr:")
narrate(indent(borg.stderr))
if borg.status:
narrate("Borg exit status:", borg.status)
return borg
def main():
try:
# Read command line {{{1
cmdline = docopt(__doc__)
keys = cmdline["--keys"].split(",") if cmdline["--keys"] else []
update = cmdline["--update"].split(",") if cmdline["--update"] else []
skip = cmdline["--skip"].split(",") if cmdline["--skip"] else []
Inform(
narrate=cmdline["--narrate"] or cmdline["--verbose"],
verbose=cmdline["--verbose"],
logfile=".sshdeploy.log",
prog_name=False,
flush=True,
version=__version__,
)
if keys and not cmdline["--trial-run"]:
fatal(
"Using the --keys option results in incomplete authorized_keys files.",
"It may only be used for testing purposes.",
"As such, --trial-run must also be specified when using --keys.",
sep="\n",
)
# Generated detailed help {{{1
if cmdline["manual"]:
from pkg_resources import resource_string
try:
Run(cmd=["less"], modes="soeW0", stdin=resource_string("src", "manual.rst").decode("utf8"))
except OSError as err:
error(os_error(err))
terminate()
# Read config file {{{1
try:
config_file = cmdline.get("--config-file")
config_file = config_file if config_file else "sshdeploy.conf"
contents = to_path(config_file).read_text()
except OSError as err:
fatal(os_error(err))
code = compile(contents, config_file, "exec")
config = {}
try:
exec(code, config)
except Exception as err:
fatal(err)
# Move into keydir {{{1
keydir = cmdline["--keydir"]
keydir = to_path(keydir if keydir else "keys-" + date)
if cmdline["generate"]:
comment("creating key directory:", keydir)
rm(keydir)
mkdir(keydir)
cd(keydir)
elif cmdline["distribute"]:
cd(keydir)
# determine default values for key options
defaults = {}
for name, default in [
("keygen-options", DefaultKeygenOpts),
("abraxas-account", DefaultAbraxasAccount),
("remote-include-filename", DefaultRemoteIncludeFilename),
]:
defaults[name] = config.get(name, default)
# Generate keys {{{1
if cmdline["generate"]:
for keyname in sorted(config["keys"].keys()):
data = config["keys"][keyname]
if keys and keyname not in keys:
# user did not request this key
continue
# get default values for missing key options
for option in defaults:
data[option] = data.get(option, defaults[option])
# generate the key
key = Key(keyname, data, update, skip, cmdline["--trial-run"])
key.generate()
# Publish keys {{{1
elif cmdline["distribute"]:
for keyname in sorted(config["keys"].keys()):
data = config["keys"][keyname]
if keys and keyname not in keys:
continue # user did not request this key
# get default values for missing key options
for option in defaults:
data[option] = data.get(option, defaults[option])
# publish the key pair to clients
key = Key(keyname, data, update, skip, cmdline["--trial-run"])
key.publish_private_key()
key.gather_public_keys()
# publish authorized_keys files to servers {{{1
if cmdline["distribute"]:
for each in sorted(AuthKeys.known):
authkey = AuthKeys.known[each]
authkey.publish()
authkey.verify()
# Process hosts {{{1
elif cmdline["test"] or cmdline["clean"] or cmdline["hosts"]:
hosts = set()
for keyname, data in config["keys"].items():
if keys and keyname not in keys:
continue # user did not request this key
# add servers to list of hosts
for server, options in data["servers"].items():
if update and server not in update or server in skip:
continue
if "bypass" not in options:
hosts.add(server)
# add clients to list of hosts
for client in data["clients"].keys():
if update and client not in update or client in skip:
continue
hosts.add(client)
# process the hosts
if cmdline["test"]:
# test host
for host in sorted(hosts):
test_access(host)
elif cmdline["clean"]:
# clean host
for host in sorted(hosts):
clean(host)
else:
# list hosts
for host in sorted(hosts):
display(host)
except OSError as err:
error(os_error(err))
except KeyboardInterrupt:
display("Killed by user")
done()
def test_deploy_voluptuous():
with cd(tests_dir):
expected = Path('deploy_voluptuous.nt').read_text()
dv = Run('python3 deploy_voluptuous.py', modes='sOEW')
assert dv.stdout.strip() == expected.strip()
def test_deploy_pydantic():
with cd(tests_dir):
expected = Path('deploy_pydantic.nt').read_text()
dp = Run('python3 deploy_pydantic.py', modes='sOEW')
assert dp.stdout.strip() == expected.strip()
def test_csv_to_nestedtext():
with cd(tests_dir):
stimulus = Path('percent_bachelors_degrees_women_usa.csv').read_text()
expected = Path('percent_bachelors_degrees_women_usa.nt').read_text()
csv2nt = Run('./csv-to-nestedtext -n', stdin=stimulus, modes='sOEW')
assert csv2nt.stdout.strip() == expected.strip()
def test_toml_to_nestedtext():
with cd(tests_dir):
stimulus = Path('sparekeys.toml').read_text()
expected = Path('sparekeys.nt').read_text()
toml2nt = Run('./toml-to-nestedtext', stdin=stimulus, modes='sOEW')
assert toml2nt.stdout.strip() == expected.strip()