def read_manifests(self): # {{{2
if self.name_index:
return
cache_dir = get_setting('cache_dir')
manifests_path = cache_dir / MANIFESTS_FILENAME
try:
encrypted = manifests_path.read_bytes()
user_key = get_setting('user_key')
if not user_key:
raise Error('no user key.')
key = base64.urlsafe_b64encode(sha256(user_key.encode('ascii')).digest())
fernet = Fernet(key)
contents = fernet.decrypt(encrypted)
try:
cache = pickle.loads(contents, **PICKLE_ARGS)
self.name_manifests = cache['names']
self.url_manifests = cache['urls']
self.title_manifests = cache['titles']
# build the name_index by inverting the name_manifests
self.name_index = {
h:n for n,l in self.name_manifests.items() for h in l
}
except (ValueError, pickle.UnpicklingError) as e:
warn('garbled manifest.', culprit=manifests_path, codicil=str(e))
manifests_path.unlink()
assert isinstance(self.name_index, dict)
except OSErrors as e:
comment(os_error(e))
except Exception as e:
comment(e)
def verify(self):
if self.bypass:
narrate('skipping post update connection test for', self.server)
return
comment('%s: post update connection test.' % (self.server))
test_access(self.server)
def __new__(cls, server, include_file, bypass, trial_run):
if server in AuthKeys.known:
self = AuthKeys.known[server]
if include_file != self.include_file:
warn(
'inconsistent remote include file:',
fmt('{include_file} != {self.include_file} in {server}.')
)
return self
self = super(AuthKeys, cls).__new__(cls)
AuthKeys.known[server] = self
self.server = server
self.bypass = bypass
self.trial_run = trial_run
self.keys = {}
self.comment = {}
self.restrictions = {}
self.include_file = include_file
self.include = None
# get remote include file if it exists
if include_file and not bypass:
narrate(fmt(' retrieving remote include file from {server}.'))
try:
try:
run_sftp(self.server, [
fmt('get .ssh/{inc} {inc}.{server}', inc=include_file)
])
self.include = to_path(include_file + '.' + server).read_text()
except OSError as err:
comment(fmt(' sftp {server}: {include_file} not found.'))
except OSError as err:
error(os_error(err))
return self
def run_sftp(server, cmds):
cmd = ['sftp', '-q', '-b', '-', server]
comment(fmt(' sftp {server}:'), '; '.join(cmds))
try:
Run(cmd, stdin='\n'.join(cmds), modes='sOEW')
except KeyboardInterrupt:
display('Continuing')
def converter(cls, to, data):
try:
# SYMBOL is not unique
#return UnitConversion(to, (cls.SYMBOL, cls.UNITS), data[cls.UNITS][to[-1]])
return UnitConversion(to, cls.UNITS, data[cls.UNITS][to[-1]])
except KeyError as e:
comment(f'missing price in {e}.', culprit=cls.UNITS)
def read_config():
if Config.get('READ'):
return # already read
# First open the config file
from .gpg import PythonFile
path = get_setting('config_file')
assert path.suffix.lower() not in ['.gpg', '.asc']
config_file = PythonFile(path)
try:
contents = config_file.run()
for k, v in contents.items():
if k.startswith('_'):
continue
if k not in CONFIG_DEFAULTS:
warn('%s: unknown.' % k, culprit=config_file)
continue
if k.endswith('_executable'):
argv = v.split() if is_str(v) else list(v)
path = to_path(argv[0])
if not path.is_absolute():
warn(
'should use absolute path for executables.',
culprit=(config_file, k)
)
Config[k] = v
Config['READ'] = True
except Error as err:
comment('not found.', culprit=config_file)
# Now open the hashes file
hashes_file = PythonFile(get_setting('hashes_file'))
try:
contents = hashes_file.run()
Config.update({k.lower(): v for k,v in contents.items()})
except Error as err:
pass
# Now open the account list file
account_list_file = PythonFile(get_setting('account_list_file'))
try:
contents = account_list_file.run()
Config.update({k.lower(): v for k,v in contents.items()})
except Error as err:
pass
# initilize GPG
from .gpg import GnuPG
GnuPG.initialize()
# Now read the user key file
user_key_file = get_setting('user_key_file')
if user_key_file:
user_key_file = PythonFile(get_setting('user_key_file'))
try:
contents = user_key_file.run()
Config.update({k.lower(): v for k,v in contents.items()})
except Error as err:
pass
def clean(host):
try:
narrate(fmt('Cleaning {host}.'))
run_sftp(host, ['rm .ssh/*.provisional'])
except OSError as err:
if 'no such file or directory' in str(err).lower():
comment(os_error(err))
else:
error('cannot connect.', culprit=host)
def gather_public_keys(self):
comment(' gathering public keys')
keyname = self.keyname
data = self.data
clients = conjoin(self.data.get('clients', []))
default_purpose = fmt('This key allows access from {clients}.')
purpose = self.data.get('purpose', default_purpose)
servers = self.data.get('servers', [])
prov = '.provisional' if self.trial_run else ''
# read contents of public key
try:
pubkey = to_path(keyname + '.pub')
key = pubkey.read_text().strip()
except OSError as err:
narrate('%s, skipping.' % os_error(err))
return
# get fingerprint of public key
try:
keygen = Run(['ssh-keygen', '-l', '-f', pubkey], modes='wOeW')
fields = keygen.stdout.strip().split()
fingerprint = ' '.join([fields[0], fields[1], fields[-1]])
except OSError as err:
error(os_error(err))
return
# contribute commented and restricted public key to the authorized_key
# file for each server
for server in servers:
if self.update and server not in self.update:
continue
if server in self.skip:
continue
server_data = servers[server]
description = server_data.get('description', None)
restrictions = server_data.get('restrictions', [])
remarks = [
'# %s' % t
for t in cull([purpose, description, self.warning, fingerprint])
if t
]
include_file = server_data.get(
'remote-include-filename', data['remote-include-filename']
)
bypass = server_data.get('bypass')
authkeys = AuthKeys(server, include_file, bypass, self.trial_run)
authkeys.add_public_key(keyname, key, remarks, restrictions)
if not servers:
warn(
'no servers specified, you must update them manually.',
culprit=keyname
)
def publish_passcode(self):
for v in ['BORG_PASSPHRASE', 'BORG_PASSCOMMAND', 'BORG_PASSPHRASE_FD']:
if v in os.environ:
narrate(f"Using existing {v}.")
return
passcommand = self.value('passcommand')
passcode = self.passphrase
# process passcomand
if passcommand:
if passcode:
warn("passphrase unneeded.", culprit="passcommand")
narrate(f"Setting BORG_PASSCOMMAND.")
os.environ['BORG_PASSCOMMAND'] = passcommand
self.borg_passcode_env_var_set_by_emborg = 'BORG_PASSCOMMAND'
return
# get passphrase from avendesora
if not passcode and self.avendesora_account:
narrate("running avendesora to access passphrase.")
try:
from avendesora import PasswordGenerator
pw = PasswordGenerator()
account_spec = self.value("avendesora_account")
if ':' in account_spec:
passcode = str(pw.get_value(account_spec))
else:
account = pw.get_account(self.value("avendesora_account"))
field = self.value("avendesora_field", None)
passcode = str(account.get_value(field))
except ImportError:
raise Error(
"Avendesora is not available",
"you must specify passphrase in settings.",
sep=", ",
)
if passcode:
os.environ['BORG_PASSPHRASE'] = passcode
narrate(f"Setting BORG_PASSPHRASE.")
self.borg_passcode_env_var_set_by_emborg = 'BORG_PASSPHRASE'
return
if self.encryption is None:
self.encryption = "none"
if self.encryption == "none" or self.encryption.startswith(
'authenticated'):
comment("Encryption is disabled.")
return
raise Error("Cannot determine the encryption passphrase.")
def generate(self):
comment(' creating key')
keyname = self.keyname
data = self.data
servers = self.data.get('servers', [])
opts = data['keygen-options']
account_name = data['abraxas-account']
if account_name:
pw.get_account(account_name)
passcode = pw.generate_password()
description = fmt("{keyname} (created {date})")
else:
passcode = ''
self.warning = 'This key is not protected with a passcode!'
description = fmt("{keyname} (created {date} -- no passcode!)")
# warn user if they have a key with no passcode and no restrictions
for server in servers:
server_data = servers[server]
restrictions = server_data.get('restrictions')
if not restrictions:
warn(
'unprotected key being sent to', server,
'without restrictions.'
)
args = ['-C', description, '-f', keyname] + opts.split()
try:
comment(' running:', 'ssh-keygen', *args)
keygen = pexpect.spawn('ssh-keygen', args, timeout=None)
keygen.expect('Enter passphrase.*: ')
keygen.sendline(passcode)
keygen.expect('Enter same passphrase again: ')
keygen.sendline(passcode)
keygen.expect(pexpect.EOF)
keygen.close()
except pexpect.ExceptionPexpect as err:
fatal(err)
# remove group/other permissions from keyfiles
to_path(keyname).chmod(0o600)
to_path(keyname + '.pub').chmod(0o600)
def test_access(host):
try:
narrate(fmt('Testing connection to {host}.'))
payload = fmt('test payload for {host}')
ref = to_path('.ref')
test = to_path('.test')
ref.write_text(payload)
rm(test)
run_sftp(host, [
fmt('put {ref}'),
fmt('get {ref} {test}'),
fmt('rm {ref}')
])
if test.read_text() == payload:
comment('connection successful.', culprit=host)
else:
error('cannot connect.', culprit=host)
except OSError as err:
error('cannot connect.', culprit=host)
rm(ref, test)
def publish_passcode(self):
passcommand = self.value('passcommand')
passcode = self.passphrase
# process passcomand
if passcommand:
if passcode:
warn("passphrase unneeded.", culprit="passcommand")
return dict(BORG_PASSCOMMAND=passcommand)
# get passphrase from avendesora
if not passcode and self.avendesora_account:
narrate("running avendesora to access passphrase.")
try:
from avendesora import PasswordGenerator
pw = PasswordGenerator()
account = pw.get_account(self.value("avendesora_account"))
field = self.value("avendesora_field", None)
passcode = str(account.get_value(field))
except ImportError:
raise Error(
"Avendesora is not available",
"you must specify passphrase in settings.",
sep=", ",
)
if passcode:
return dict(BORG_PASSPHRASE=passcode)
if self.encryption is None:
self.encryption = "none"
if self.encryption == "none":
comment("passphrase is not available, encryption disabled.")
return {}
raise Error("Cannot determine the encryption passphrase.")
def read_config():
if Config.get('READ'):
return # already read
# First open the config file
from .gpg import PythonFile
path = get_setting('config_file')
assert path.suffix.lower() not in ['.gpg', '.asc']
config_file = PythonFile(path)
if not config_file.exists():
# have not yet initialized this account
return
try:
contents = config_file.run()
for k, v in contents.items():
if k.startswith('_'):
continue
if k not in CONFIG_DEFAULTS:
warn('%s: unknown.' % k, culprit=config_file)
continue
if k.endswith('_executable'):
argv = v.split() if is_str(v) else list(v)
path = Path(argv[0])
if not path.is_absolute():
warn('should use absolute path for executables.',
culprit=(config_file, k))
Config[k] = v
Config['READ'] = True
except PasswordError:
comment('not found.', culprit=config_file)
# Now open the hashes file
hashes_file = PythonFile(get_setting('hashes_file'))
try:
contents = hashes_file.run()
Config.update({k.lower(): v for k, v in contents.items()})
except PasswordError:
pass
# Now open the account list file
account_list_file = PythonFile(get_setting('account_list_file'))
try:
contents = account_list_file.run()
Config.update({k.lower(): v for k, v in contents.items()})
except PasswordError:
pass
# initilize GPG
from .gpg import GnuPG
GnuPG.initialize()
# Now read the user key file
user_key_file = get_setting('user_key_file')
if user_key_file:
user_key_file = PythonFile(get_setting('user_key_file'))
try:
contents = user_key_file.run()
Config.update({
k.lower(): v
for k, v in contents.items() if not k.startswith('__')
})
except PasswordError:
pass
# Set the user-selected colors
Config['_label_color'] = Color(color=get_setting('label_color'),
scheme=get_setting('color_scheme'),
enable=Color.isTTY())
Config['_highlight_color'] = Color(color=get_setting('highlight_color'),
scheme=get_setting('color_scheme'),
enable=Color.isTTY())
def run(cls, command, args, settings, options):
# read command line
cmdline = docopt(cls.USAGE, argv=[command] + args)
archive = cmdline["--archive"]
date = cmdline["--date"]
# get the desired archive
if date and not archive:
archive = get_name_of_nearest_archive(settings, date)
if not archive:
archive = get_name_of_latest_archive(settings)
output("Archive:", archive)
# run borg
borg = settings.run_borg(
cmd="list",
args=[settings.destination(archive)],
emborg_opts=options,
)
out = borg.stdout
# define available formats
formats = dict(
name="{path}",
date="{day} {date} {time} {path}",
size="{Size:<5.2r} {path}",
owner="{owner:<8} {path}",
group="{group:<8} {path}",
long="{Size:<5.2r} {date} {time} {path}",
full=
"{permissions:<10} {owner:<6} {group:<6} {size:>8} {Date:YYMMDD HH:mm} {path}",
)
user_formats = settings.manifest_formats
if user_formats:
unknown = user_formats.keys() - formats.keys()
if unknown:
warn("unknown formats:",
", ".join(unknown),
culprit="manifest_formats")
formats.update(user_formats)
# process sort options
if cmdline["--sort-by-name"]:
fmt = "name"
def get_key(columns):
return columns[7]
elif cmdline["--sort-by-date"]:
fmt = "date"
def get_key(columns):
date_time = " ".join(columns[5:7])
return arrow.get(date_time, "YYYY-MM-DD HH:mm:ss")
elif cmdline["--sort-by-size"]:
fmt = "size"
def get_key(columns):
return int(columns[3])
elif cmdline["--sort-by-owner"]:
fmt = "owner"
def get_key(columns):
return columns[1]
elif cmdline["--sort-by-group"]:
fmt = "group"
def get_key(columns):
return columns[2]
else:
fmt = None
get_key = None
# process format options
if cmdline["--name-only"]:
fmt = "name"
elif cmdline["--long"]:
fmt = "long"
elif cmdline["--full"]:
fmt = "full"
# sort the output
lines = [l.split(maxsplit=7) for l in out.rstrip().splitlines()]
if get_key:
lines = sorted(lines, key=get_key)
if cmdline["--reverse-sort"]:
lines.reverse()
# echo borg output if no formatting is necessary
if not fmt:
output(out)
return
# import QuantiPhy
try:
from quantiphy import Quantity
Quantity.set_prefs(spacer="")
except ImportError:
comment("Could not import QuantiPhy.")
def Quantity(value, units):
return value
# generate formatted output
for columns in lines:
output(
formats[fmt].format(
permissions=columns[0],
owner=columns[1],
group=columns[2],
size=columns[3],
Size=Quantity(columns[3], "B"),
Date=arrow.get(" ".join(columns[5:7]),
"YYYY-MM-DD HH:mm:ss"),
day=columns[4].rstrip(","),
date=columns[5],
time=columns[6],
path=columns[7],
),
sep="\n",
)
def main():
try:
# Read command line {{{1
cmdline = docopt(__doc__)
keys = cmdline["--keys"].split(",") if cmdline["--keys"] else []
update = cmdline["--update"].split(",") if cmdline["--update"] else []
skip = cmdline["--skip"].split(",") if cmdline["--skip"] else []
Inform(
narrate=cmdline["--narrate"] or cmdline["--verbose"],
verbose=cmdline["--verbose"],
logfile=".sshdeploy.log",
prog_name=False,
flush=True,
version=__version__,
)
if keys and not cmdline["--trial-run"]:
fatal(
"Using the --keys option results in incomplete authorized_keys files.",
"It may only be used for testing purposes.",
"As such, --trial-run must also be specified when using --keys.",
sep="\n",
)
# Generated detailed help {{{1
if cmdline["manual"]:
from pkg_resources import resource_string
try:
Run(cmd=["less"], modes="soeW0", stdin=resource_string("src", "manual.rst").decode("utf8"))
except OSError as err:
error(os_error(err))
terminate()
# Read config file {{{1
try:
config_file = cmdline.get("--config-file")
config_file = config_file if config_file else "sshdeploy.conf"
contents = to_path(config_file).read_text()
except OSError as err:
fatal(os_error(err))
code = compile(contents, config_file, "exec")
config = {}
try:
exec(code, config)
except Exception as err:
fatal(err)
# Move into keydir {{{1
keydir = cmdline["--keydir"]
keydir = to_path(keydir if keydir else "keys-" + date)
if cmdline["generate"]:
comment("creating key directory:", keydir)
rm(keydir)
mkdir(keydir)
cd(keydir)
elif cmdline["distribute"]:
cd(keydir)
# determine default values for key options
defaults = {}
for name, default in [
("keygen-options", DefaultKeygenOpts),
("abraxas-account", DefaultAbraxasAccount),
("remote-include-filename", DefaultRemoteIncludeFilename),
]:
defaults[name] = config.get(name, default)
# Generate keys {{{1
if cmdline["generate"]:
for keyname in sorted(config["keys"].keys()):
data = config["keys"][keyname]
if keys and keyname not in keys:
# user did not request this key
continue
# get default values for missing key options
for option in defaults:
data[option] = data.get(option, defaults[option])
# generate the key
key = Key(keyname, data, update, skip, cmdline["--trial-run"])
key.generate()
# Publish keys {{{1
elif cmdline["distribute"]:
for keyname in sorted(config["keys"].keys()):
data = config["keys"][keyname]
if keys and keyname not in keys:
continue # user did not request this key
# get default values for missing key options
for option in defaults:
data[option] = data.get(option, defaults[option])
# publish the key pair to clients
key = Key(keyname, data, update, skip, cmdline["--trial-run"])
key.publish_private_key()
key.gather_public_keys()
# publish authorized_keys files to servers {{{1
if cmdline["distribute"]:
for each in sorted(AuthKeys.known):
authkey = AuthKeys.known[each]
authkey.publish()
authkey.verify()
# Process hosts {{{1
elif cmdline["test"] or cmdline["clean"] or cmdline["hosts"]:
hosts = set()
for keyname, data in config["keys"].items():
if keys and keyname not in keys:
continue # user did not request this key
# add servers to list of hosts
for server, options in data["servers"].items():
if update and server not in update or server in skip:
continue
if "bypass" not in options:
hosts.add(server)
# add clients to list of hosts
for client in data["clients"].keys():
if update and client not in update or client in skip:
continue
hosts.add(client)
# process the hosts
if cmdline["test"]:
# test host
for host in sorted(hosts):
test_access(host)
elif cmdline["clean"]:
# clean host
for host in sorted(hosts):
clean(host)
else:
# list hosts
for host in sorted(hosts):
display(host)
except OSError as err:
error(os_error(err))
except KeyboardInterrupt:
display("Killed by user")
done()
def run(cmd, stdin=None, modes=None):
comment(' running:', *cmd)
Run(cmd, stdin=stdin, modes=modes)