def fail(self, *msg, comment=''): msg = full_stop(' '.join(str(m) for m in msg)) try: if self.notify: Run(['mail', f'-s "{PROGRAM_NAME}: {msg}"', self.notify], stdin=dedent(f''' {msg} {comment} config = {self.config_name} source = {hostname}:{self.src_dir} destination = {self.dest_server}:{self.dest_dir} ''').lstrip(), modes='soeW') except OSError as e: pass try: if self.notifier: Run(self.notifier.format( msg=msg, host_name=hostname, user_name=username, prog_name=PROGRAM_NAME, ), modes='soeW') except OSError as e: pass except KeyError as e: warn('unknown key.', culprit=(self.settings_file, 'notifier', e)) raise Error(msg)
def fail(self, *msg, comment=''): msg = full_stop(' '.join(str(m) for m in msg)) try: if self.notify: Run(['mail', '-s', f'{PROGRAM_NAME} on {hostname}: {msg}'] + self.notify.split(), stdin=dedent(f''' {msg} {comment} config = {self.config_name} source = {username}@{hostname}:{', '.join(str(d) for d in self.src_dirs)} destination = {self.repository} ''').lstrip(), modes='soeW') except Error: pass try: if self.notifier: Run(self.notifier.format( msg=msg, hostname=hostname, user_name=username, prog_name=PROGRAM_NAME, ), modes='soeW') except Error: pass except KeyError as e: warn('unknown key.', culprit=(self.settings_file, 'notifier', e)) raise Error(msg)
def test_run_downturn(): with cd(wd): cmd = "./test_prog 0" p = Run(cmd, "sOEW") assert p.stdout == "this is stdout.\n" assert p.stderr == "this is stderr.\n" assert p.status == 0
def test_run_endorse(): cmd = './test_prog 1' try: p = Run(cmd, 'sOEW') assert False, 'expected exception' except OSError as e: assert str(e) == '[Errno None] this is stderr.'
def test_run_parakeet(): cmd = './test_prog 3' try: p = Run(cmd, 'sOEW0,1,2,4') assert False, 'expected exception' except OSError as e: assert str(e) == '[Errno None] this is stderr.'
def run_borg_raw(self, args): # prepare the command os.environ.update(self.publish_passcode()) os.environ["BORG_DISPLAY_PASSPHRASE"] = "no" executable = self.value("borg_executable", BORG) remote_path = self.value("remote_path") remote_path = ["--remote-path", remote_path] if remote_path else [] repository = str(self.repository) command = ([executable] + remote_path + [a.replace('@repo', repository) for a in args]) # run the command narrate("running:\n{}".format( indent(render_command(command, borg_options_arg_count)))) with cd(self.working_dir): narrate("running in:", cwd()) starts_at = arrow.now() log("starts at: {!s}".format(starts_at)) borg = Run(command, modes="soeW", env=os.environ, log=False) ends_at = arrow.now() log("ends at: {!s}".format(ends_at)) log("elapsed = {!s}".format(ends_at - starts_at)) if borg.stdout: narrate("Borg stdout:") narrate(indent(borg.stdout)) if borg.stderr: narrate("Borg stderr:") narrate(indent(borg.stderr)) if borg.status: narrate("Borg exit status:", borg.status) return borg
def test_run_entree(): with cd(wd): cmd = "./test_prog 1" p = Run(cmd, "sOEW1") assert p.stdout == "this is stdout.\n" assert p.stderr == "this is stderr.\n" assert p.status == 1
def run_borg(self, cmd, args='', borg_opts=None, emborg_opts=()): # prepare the command os.environ.update(self.publish_passcode()) os.environ['BORG_DISPLAY_PASSPHRASE'] = 'no' if self.ssh_command: os.environ['BORG_RSH'] = self.ssh_command executable = self.value('borg_executable', BORG) if borg_opts is None: borg_opts = self.borg_options(cmd, emborg_opts) command = ([executable] + cmd.split() + borg_opts + (args.split() if is_str(args) else args)) environ = { k: v for k, v in os.environ.items() if k.startswith('BORG_') } if 'BORG_PASSPHRASE' in environ: environ['BORG_PASSPHRASE'] = '<redacted>' narrate('setting environment variables:', render(environ)) # check if ssh agent is present if self.needs_ssh_agent: for ssh_var in 'SSH_AGENT_PID SSH_AUTH_SOCK'.split(): if ssh_var not in os.environ: warn( 'environment variable not found, is ssh-agent running?', culprit=ssh_var) # run the command narrate('running:\n{}'.format( indent(render_command(command, borg_options_arg_count)))) narrating = 'verbose' in emborg_opts or 'narrate' in emborg_opts modes = 'soeW' if narrating else 'sOEW' return Run(command, modes=modes, stdin='', env=os.environ, log=False)
def test_run_layer(): with cd(wd): cmd = "./test_prog 3" p = Run(cmd, "sOEW0,3") assert p.stdout == "this is stdout.\n" assert p.stderr == "this is stderr.\n" assert p.status == 3
def test_run_parakeet(): with cd(wd): cmd = "./test_prog 3" try: Run(cmd, "sOEW0,1,2,4") assert False, "expected exception" except OSError as e: assert str(e) == "[Errno None] this is stderr."
def test_run_endorse(): with cd(wd): cmd = "./test_prog 1" try: Run(cmd, "sOEW") assert False, "expected exception" except OSError as e: assert str(e) == "[Errno None] this is stderr."
def fail(self, *msg, cmd='<unknown>'): msg = join(*msg) try: msg = msg.decode('ascii', errors='replace') except AttributeError: pass try: if self.notify and not Color.isTTY(): Run( [ "mail", "-s", f"{PROGRAM_NAME} failed on {username}@{hostname}" ] + self.notify.split(), stdin=dedent(f"""\ {PROGRAM_NAME} fails. command: {cmd} config: {self.config_name} source: {username}@{fullhostname}:{', '.join(str(d) for d in self.src_dirs)} destination: {self.repository!s} error message: """) + indent(msg) + "\n", modes="soeW", encoding="ascii", ) except Error: pass try: notifier = self.settings.get("notifier") # don't use self.value as we don't want arguments expanded yet if notifier and not Color.isTTY(): Run(self.notifier.format( cmd=cmd, msg=msg, hostname=hostname, user_name=username, prog_name=PROGRAM_NAME, ), modes="SoeW" # need to use the shell as user will generally quote msg ) except Error: pass except KeyError as e: warn("unknown key.", culprit=(self.settings_file, "notifier", e))
def run_duplicity(cmd, settings, narrating): os.environ.update(publish_passcode(settings)) for ssh_var in 'SSH_AGENT_PID SSH_AUTH_SOCK'.split(): if ssh_var not in os.environ: warn('environment variable not found, is ssh-agent running?', culprit=ssh_var) narrate('running:\n{}'.format(indent(render_command(cmd)))) modes = 'soeW' if narrating else 'sOEW' Run(cmd, modes=modes, env=os.environ)
def run(self): # remove requested files and directories if self.remove: rm(self.remove.split()) if '<PASS>' in self.args: return True # run command emborg = Run(self.cmd, "sOMW*") self.result = dedent(Color.strip_colors(emborg.stdout)).strip("\n") # check stdout matches = True if 'ignore' not in self.expected_type: expected, result = self.expected, self.result if 'sort-lines' in self.expected_type: expected = '\n'.join(sorted(expected.splitlines())) result = '\n'.join(sorted(result.splitlines())) if 'regex' in self.expected_type: matches = bool(re.fullmatch(expected, result)) else: matches = expected == result if matches and self.cmp_dirs: gen_dir, ref_dir = self.cmp_dirs #self.expected = '' try: diff = Run(["diff", "--recursive", gen_dir, ref_dir], "sOMW") self.result = diff.stdout except Error as e: self.result = e.stdout matches = False # check exit status self.exit_status = emborg.status self.expected_exit_status = 0 if 'diff' in self.expected_type: self.expected_exit_status = 1 if 'error' in self.expected_type: self.expected_exit_status = 2 if self.exit_status != self.expected_exit_status: matches = False return matches
def test_cryptocurrency(): if sys.version_info < (3, 8): return # cryptocurrency example uses walrus operator with cd(tests_dir): # the cryptocurrency example outputs the current prices, so just do some # simple sanity checking on the output. cc = Run('./cryptocurrency', modes='sOEW') assert '5 BTC =' in cc.stdout assert '50 ETH =' in cc.stdout assert '50 kXLM =' in cc.stdout
def test_postmortem(): if sys.version_info < (3, 8): return # postmortem example uses walrus operator with cd(tests_dir): expected = Path('postmortem.json').read_text() user_home_dir = str(to_path('~')) expected = expected.replace('~', user_home_dir) pm = Run('./postmortem', modes='sOEW') assert pm.stdout.strip() == expected.strip()
def run_user_commands(self, setting): for i, cmd in enumerate(self.values(setting)): narrate(f"staging {setting}[{i}] command.") try: Run(cmd, "SoEW") except Error as e: e.reraise(culprit=(setting, i, cmd.split()[0])) # the following two statements are only useful from run_before_borg self.settings[setting] = [] # erase the setting so it is not run again self.borg_ran = True # indicate that before has run so after should run
def report(msg): Run([ 'mail', '-s', f'{PROGRAM_NAME}: backup is overdue', email ], stdin=dedent(f''' {msg} config = {settings.config_name} source host = {hostname} source directories = {', '.join(str(d) for d in settings.src_dirs)} destination = {settings.repository} ''').lstrip(), modes='soeW')
def run_borg_raw(self, args): # prepare the command os.environ.update(self.publish_passcode()) os.environ['BORG_DISPLAY_PASSPHRASE'] = 'no' executable = self.value('borg_executable', BORG) repository = str(self.repository) command = ([executable] + [(repository if a == '@repo' else a) for a in args]) # run the command narrate('running:\n{}'.format( indent(render_command(command, borg_options_arg_count)))) return Run(command, modes='soeW', env=os.environ, log=False)
def run(cls, command, args, settings, options): # read command line cmdline = docopt(cls.USAGE, argv=[command] + args) kind = 'full' if command in 'full f'.split() else 'incr' # check the dependencies are available for each in settings.values('must_exist'): path = to_path(each) if not path.exists(): raise Error('does not exist, perform setup and restart.', culprit=each) # run prerequisites for each in settings.values('run_before_backup'): narrate('running:', each) Run(each, 'SoeW') rm('duplicity.log') # run duplicity cmd = (f'duplicity {kind}'.split() + duplicity_options(settings, options) + archive_dir_command(settings) + sftp_command(settings) + excludes(settings) + [render_path(settings.src_dir), destination(settings)]) run_duplicity(cmd, settings, 'narrate' in options) # update the date files now = arrow.now() if kind == 'full': settings.full_date_file.write_text(str(now)) settings.incr_date_file.write_text(str(now)) # run any scripts specified to be run after a backup for each in settings.values('run_after_backup'): narrate('running:', each) Run(each, 'SoeW')
def test_run_ground(): Inform(prog_name=False, logfile=False) set_prefs(use_inform=True) cmd = './test_prog 1' try: p = Run(cmd, 'sOEW') assert False, 'expected exception' except Error as e: assert str(e) == 'this is stderr.' assert e.cmd == './test_prog 1' assert e.stdout == 'this is stdout.' assert e.stderr == 'this is stderr.' assert e.status == 1 assert e.msg == 'this is stderr.' set_prefs(use_inform=False)
def initialize_network(self): network = self.network # run the init script if given try: if network.init_script: script = Run(network.init_script, "sOEW") if script.stdout: display(script.stdout.rstrip()) except AttributeError: pass except Error as e: warn("{} network init_script failed: {}".format( network.name(), network.init_script)) codicil(e.get_message())
def test_run_ground(): with cd(wd): Inform(prog_name=False, logfile=False) set_prefs(use_inform=True) cmd = "./test_prog 1" try: Run(cmd, "sOEW") assert False, "expected exception" except Error as e: assert str(e) == "this is stderr." assert e.cmd == "./test_prog 1" assert e.stdout == "this is stdout." assert e.stderr == "this is stderr." assert e.status == 1 assert e.msg == "this is stderr." set_prefs(use_inform=False)
def test_address(): if sys.version_info < (3, 8): return # address example uses walrus operator with cd(tests_dir): cc = Run('./address fumiko', modes='sOEW') assert cc.stdout.strip() == dedent(""" Fumiko Purvis position: treasurer address: 3636 Buffalo Ave Topeka, Kansas 20692 phone: 1-268-555-0280 email: [email protected] additional roles: - accounting task force """).strip()
def test_emborg_overdue(initialize, name, conf, args, expected, expected_type, dependencies): if skip_test(dependencies): return with cd(tests_dir): if conf: with open('.config/overdue.conf', 'w') as f: f.write(conf) try: args = args.split() if is_str(args) else args overdue = Run(emborg_overdue_exe + args, "sOEW") if 'regex' in expected_type.split(): assert bool(re.fullmatch(expected, overdue.stdout)), name else: assert expected == overdue.stdout, name except Error as e: assert str(e) == expected, name
def run_late(cls, command, args, settings, options): # read command line cmdline = docopt(cls.USAGE, argv=[command] + args) email = cmdline['--email'] if not cls.MESSAGES: return if cmdline['--oldest']: message = cls.MESSAGES[cls.OLDEST_CONFIG] else: message = '\n'.join(cls.MESSAGES.values()) if email: Run(['mail', '-s', f'{PROGRAM_NAME}: backup is overdue', email], stdin=message, modes='soeW') else: output(message)
def run_borg_raw(self, args): # run the run_before_borg commands self.run_user_commands('run_before_borg') # prepare the command self.publish_passcode() os.environ["BORG_DISPLAY_PASSPHRASE"] = "no" executable = self.value("borg_executable", BORG) remote_path = self.value("remote_path") remote_path = ["--remote-path", remote_path] if remote_path else [] repository = str(self.repository) command = ([executable] + remote_path + [a.replace('@repo', repository) for a in args]) # run the command narrate("running:\n{}".format( indent(render_command(command, borg_options_arg_count)))) with cd(self.working_dir): narrate("running in:", cwd()) starts_at = arrow.now() log("starts at: {!s}".format(starts_at)) try: borg = Run(command, modes="soeW1", env=os.environ, log=False) except Error as e: self.report_borg_error(e, executable) ends_at = arrow.now() log("ends at: {!s}".format(ends_at)) log("elapsed = {!s}".format(ends_at - starts_at)) if borg.status == 1: warn('Warning emitted by Borg, see logfile for details.') if borg.stdout: narrate("Borg stdout:") narrate(indent(borg.stdout)) if borg.stderr: narrate("Borg stderr:") narrate(indent(borg.stderr)) if borg.status: narrate("Borg exit status:", borg.status) return borg
def run_late(cls, command, args, settings, options): # read command line cmdline = docopt(cls.USAGE, argv=[command] + args) email = cmdline["--email"] if not cls.MESSAGES: return if cmdline["--oldest"]: message = cls.MESSAGES[cls.OLDEST_CONFIG] else: message = "\n".join(cls.MESSAGES.values()) if email: Run( ["mail", "-s", f"{PROGRAM_NAME}: backup is overdue", email], stdin=message, modes="soeW", ) else: output(message)
def run_borg_raw(self, args): # prepare the command os.environ.update(self.publish_passcode()) os.environ['BORG_DISPLAY_PASSPHRASE'] = 'no' executable = self.value('borg_executable', BORG) remote_path = self.value('remote_path') remote_path = ['--remote-path', remote_path] if remote_path else [] repository = str(self.repository) command = ([executable] + remote_path + [(repository if a == '@repo' else a) for a in args]) # run the command narrate('running:\n{}'.format( indent(render_command(command, borg_options_arg_count)))) starts_at = arrow.now() narrate('starts at: {!s}'.format(starts_at)) borg = Run(command, modes='soeW', env=os.environ, log=False) ends_at = arrow.now() narrate('ends at: {!s}'.format(ends_at)) narrate('elapsed = {!s}'.format(ends_at - starts_at)) return borg
def identify_network(self): # get MAC address of gateway try: arp = Run(ARP, "sOeW") arp_table = arp.stdout except Error as e: e.report() return gateway_macs = [] other_macs = [] for row in arp_table.split("\n"): try: name, ipaddr, at, mac, hwtype, on, interface = row.split() if name == "_gateway": gateway_macs.append(mac) else: other_macs.append(mac) except ValueError: continue def choose(preferred): # First offer the preferred networks, in order for name in preferred: network = NetworkEntry.find(name) if network: yield network # Offer the remaining networks in arbitrary order for network in NetworkEntry.all_networks(): yield network for network in choose(self.preferred_networks): for mac in gateway_macs + other_macs: if mac in network.routers: # We are on a known network return network