def run(cls, command, args, settings, options):
# read command line
cmdline = docopt(cls.USAGE, argv=[command] + args)
show_available = cmdline['--available']
unknown = Color('yellow')
known = Color('cyan')
if show_available:
output('Emborg settings:')
for name, desc in EMBORG_SETTINGS.items():
output(f'{known(name):>33s}: {desc}')
output()
output('Borg settings:')
for name, attrs in BORG_SETTINGS.items():
output(f"{known(name):>33s}: {attrs['desc']}")
return 0
if settings:
for k, v in settings:
is_known = k in EMBORG_SETTINGS or k in BORG_SETTINGS
key = known(k) if is_known else unknown(k)
if k == 'passphrase':
v = '<set>'
output(f'{key:>33}: {render(v, level=6)}')
def run(cls, command, args, settings, options):
# read command line
cmdline = docopt(cls.USAGE, argv=[command] + args)
show_available = cmdline["--available"]
unknown = Color("yellow")
known = Color("cyan")
if show_available:
output("Emborg settings:")
for name, desc in EMBORG_SETTINGS.items():
output(f"{known(name):>33s}: {desc}")
output()
output("Borg settings:")
for name, attrs in BORG_SETTINGS.items():
output(f"{known(name):>33s}: {attrs['desc']}")
return 0
if settings:
for k, v in settings:
is_known = k in EMBORG_SETTINGS or k in BORG_SETTINGS
key = known(k) if is_known else unknown(k)
if k == "passphrase":
v = "<set>"
output(f"{key:>33}: {render(v, level=6)}")
try:
if is_str(v) and "{" in v:
output(
f'{"":>24}{render(settings.resolve(v), level=6)}')
except Error:
pass
def fail(self, *msg, comment=''):
msg = full_stop(' '.join(str(m) for m in msg))
try:
if self.notify and not Color.isTTY():
Run(['mail', '-s', f'{PROGRAM_NAME} on {fullhostname}: {msg}']
+ self.notify.split(),
stdin=dedent(f'''
{msg}
{comment}
config = {self.config_name}
source = {username}@{hostname}:{', '.join(str(d) for d in self.src_dirs)}
destination = {self.repository}
''').lstrip(),
modes='soeW')
except Error:
pass
try:
if self.notifier and not Color.isTTY():
Run(self.notifier.format(
msg=msg,
hostname=hostname,
user_name=username,
prog_name=PROGRAM_NAME,
),
modes='soeW')
except Error:
pass
except KeyError as e:
warn('unknown key.', culprit=(self.settings_file, 'notifier', e))
def run(cls, command, args, settings, options):
# read command line
cmdline = docopt(cls.USAGE, argv=[command] + args)
highlight = Color('yellow')
normal = Color('cyan')
for k, v in settings:
key = f'{k:>22s}'
key = normal(key) if k in KNOWN_SETTINGS else highlight(key)
output(f'{key}: {render(v, level=6)}')
def fail(self, *msg, cmd='<unknown>'):
msg = join(*msg)
try:
msg = msg.decode('ascii', errors='replace')
except AttributeError:
pass
try:
if self.notify and not Color.isTTY():
Run(
[
"mail", "-s",
f"{PROGRAM_NAME} failed on {username}@{hostname}"
] + self.notify.split(),
stdin=dedent(f"""\
{PROGRAM_NAME} fails.
command: {cmd}
config: {self.config_name}
source: {username}@{fullhostname}:{', '.join(str(d) for d in self.src_dirs)}
destination: {self.repository!s}
error message:
""") + indent(msg) + "\n",
modes="soeW",
encoding="ascii",
)
except Error:
pass
try:
notifier = self.settings.get("notifier")
# don't use self.value as we don't want arguments expanded yet
if notifier and not Color.isTTY():
Run(self.notifier.format(
cmd=cmd,
msg=msg,
hostname=hostname,
user_name=username,
prog_name=PROGRAM_NAME,
),
modes="SoeW"
# need to use the shell as user will generally quote msg
)
except Error:
pass
except KeyError as e:
warn("unknown key.", culprit=(self.settings_file, "notifier", e))
def get_writer(tty=None, clipboard=False, stdout=False):
if clipboard:
return ClipboardWriter()
if stdout:
return StdoutWriter()
if tty is True:
return TTY_Writer()
if tty is False:
return KeyboardWriter()
if Color.isTTY():
return TTY_Writer()
else:
return KeyboardWriter()
def display_field(self, account, field):
# get string to display
value, is_secret, name, desc = tuple(account.get_value(field))
label = '%s (%s)' % (name, desc) if desc else name
value = dedent(str(value)).strip()
label_color = get_setting('_label_color')
# indent multiline outputs
sep = ' '
if '\n' in value:
if is_secret:
warn('secret contains newlines, will not be fully concealed.')
value = indent(value, get_setting('indent')).strip('\n')
sep = '\n'
if label:
if label[0] == '_':
# hidden field
label = '!' + label[1:]
text = label_color(label.replace('_', ' ') + ':') + sep + value
else:
text = value
label = field
log('Writing to TTY:', label)
if is_secret:
if Color.isTTY():
# Write only if output is a TTY. This is a security feature.
# The ideas is that when the TTY writer is called it is because
# the user is expecting the output to go to the tty. This
# eliminates the chance that the output can be intercepted and
# recorded by replacing Avendesora with an alias or shell
# script. If the user really want the output to go to something
# other than the TTY, the user should use the --stdout option.
try:
cursor.write(text)
cursor.conceal()
sleep(get_setting('display_time'))
except KeyboardInterrupt:
pass
cursor.reveal()
cursor.clear()
else:
error('output is not a TTY.')
codicil(
'Use --stdout option if you want to send secret',
'to a file or a pipe.'
)
else:
output(text)
def run(self):
# remove requested files and directories
if self.remove:
rm(self.remove.split())
if '<PASS>' in self.args:
return True
# run command
emborg = Run(self.cmd, "sOMW*")
self.result = dedent(Color.strip_colors(emborg.stdout)).strip("\n")
# check stdout
matches = True
if 'ignore' not in self.expected_type:
expected, result = self.expected, self.result
if 'sort-lines' in self.expected_type:
expected = '\n'.join(sorted(expected.splitlines()))
result = '\n'.join(sorted(result.splitlines()))
if 'regex' in self.expected_type:
matches = bool(re.fullmatch(expected, result))
else:
matches = expected == result
if matches and self.cmp_dirs:
gen_dir, ref_dir = self.cmp_dirs
#self.expected = ''
try:
diff = Run(["diff", "--recursive", gen_dir, ref_dir], "sOMW")
self.result = diff.stdout
except Error as e:
self.result = e.stdout
matches = False
# check exit status
self.exit_status = emborg.status
self.expected_exit_status = 0
if 'diff' in self.expected_type:
self.expected_exit_status = 1
if 'error' in self.expected_type:
self.expected_exit_status = 2
if self.exit_status != self.expected_exit_status:
matches = False
return matches
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see http://www.gnu.org/licenses/.
# Imports {{{1
from .config import get_setting
from .shlib import Run
from inform import Error, output, is_str, indent, Color
from textwrap import dedent
import os
import sys
# Globals {{{1
HighlightColor = Color(color=get_setting('highlight_color'),
scheme=get_setting('color_scheme'),
enable=Color.isTTY())
# OSErrors {{{1
# Define OSError to include IOError with Python2 for backward compatibility
if sys.version_info.major < 3:
OSErrors = (OSError, IOError)
else:
OSErrors = (OSError, )
# gethostname {{{1
# returns short version of the hostname (the hostname without any domain name)
def gethostname():
import socket
return socket.gethostname().split('.')[0]
error,
log,
notify,
output,
warn,
indent,
os_error,
)
from time import sleep
from textwrap import dedent
import string
import re
# Globals {{{1
LabelColor = Color(color=get_setting('label_color'),
scheme=get_setting('color_scheme'),
enable=Color.isTTY())
KEYSYMS = {
'!': 'exclam',
'"': 'quotedbl',
'#': 'numbersign',
'$': 'dollar',
'%': 'percent',
'&': 'ampersand',
"'": 'apostrophe',
'(': 'parenleft',
')': 'parenright',
'*': 'asterisk',
'+': 'plus',
',': 'comma',
'-': 'minus',
from .collection import Collection
from .config import get_setting
from .obscure import Obscure
from .preferences import TOOL_FIELDS
from .recognize import Recognizer
from .secrets import Secret
from inform import Color, conjoin, cull, Error, is_collection, is_str, log, output, warn
from textwrap import indent, dedent
from urllib.parse import urlparse
import re
import sys
# Globals {{{1
VECTOR_PATTERN = re.compile(r"\A(\w+)\[(\w+)\]\Z")
LabelColor = Color(color=get_setting("label_color"), scheme=get_setting("color_scheme"), enable=Color.isTTY())
# Account class {{{1
class Account:
__NO_MASTER = True
# prevents master password from being added to this base class
# all_accounts() {{{2
@classmethod
def all_accounts(cls):
for sub in cls.__subclasses__():
yield sub
for each in sub.all_accounts():
yield each
# fields() {{{2
def main():
version = f"{__version__} ({__released__})"
cmdline = docopt(__doc__, version=version)
quiet = cmdline["--quiet"]
problem = False
use_color = Color.isTTY() and not cmdline["--no-color"]
passes = Color("green", enable=use_color)
fails = Color("red", enable=use_color)
if cmdline["--verbose"]:
overdue_message = verbose_overdue_message
else:
overdue_message = terse_overdue_message
# prepare to create logfile
log = to_path(DATA_DIR, OVERDUE_LOG_FILE) if OVERDUE_LOG_FILE else False
if log:
data_dir = to_path(DATA_DIR)
if not data_dir.exists():
try:
# data dir does not exist, create it
data_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
except OSError as e:
warn(os_error(e))
log = False
with Inform(flush=True, quiet=quiet, logfile=log, version=version):
# read the settings file
try:
settings_file = PythonFile(CONFIG_DIR, OVERDUE_FILE)
settings = settings_file.run()
except Error as e:
e.terminate()
# gather needed settings
default_maintainer = settings.get("default_maintainer")
default_max_age = settings.get("default_max_age", 28)
dumper = settings.get("dumper", f"{username}@{hostname}")
repositories = settings.get("repositories")
root = settings.get("root")
# process repositories table
backups = []
if is_str(repositories):
for line in repositories.split("\n"):
line = line.split("#")[0].strip() # discard comments
if not line:
continue
backups.append([c.strip() for c in line.split("|")])
else:
for each in repositories:
backups.append([
each.get("host"),
each.get("path"),
each.get("maintainer"),
each.get("max_age"),
])
def send_mail(recipient, subject, message):
if cmdline["--mail"]:
if cmdline['--verbose']:
display(f"Reporting to {recipient}.\n")
mail_cmd = ["mailx", "-r", dumper, "-s", subject, recipient]
Run(mail_cmd, stdin=message, modes="soeW0")
# check age of repositories
for host, path, maintainer, max_age in backups:
maintainer = default_maintainer if not maintainer else maintainer
max_age = float(max_age) if max_age else default_max_age
try:
path = to_path(root, path)
if path.is_dir():
paths = list(path.glob("index.*"))
if not paths:
raise Error("no sentinel file found.", culprit=path)
if len(paths) > 1:
raise Error("too many sentinel files.",
*paths,
sep="\n ")
path = paths[0]
mtime = arrow.get(path.stat().st_mtime)
delta = now - mtime
age = 24 * delta.days + delta.seconds / 3600
report = age > max_age
overdue = ' -- overdue' if report else ''
color = fails if report else passes
if report or not cmdline["--no-passes"]:
display(color(fmt(overdue_message)))
if report:
problem = True
subject = f"backup of {host} is overdue"
msg = fmt(mail_overdue_message)
send_mail(maintainer, subject, msg)
except OSError as e:
problem = True
msg = os_error(e)
error(msg)
if maintainer:
send_mail(
maintainer,
f"{get_prog_name()} error",
error_message.format(msg),
)
except Error as e:
problem = True
e.report()
if maintainer:
send_mail(
maintainer,
f"{get_prog_name()} error",
error_message.format(str(e)),
)
terminate(problem)
def read_config():
if Config.get('READ'):
return # already read
# First open the config file
from .gpg import PythonFile
path = get_setting('config_file')
assert path.suffix.lower() not in ['.gpg', '.asc']
config_file = PythonFile(path)
if not config_file.exists():
# have not yet initialized this account
return
try:
contents = config_file.run()
for k, v in contents.items():
if k.startswith('_'):
continue
if k not in CONFIG_DEFAULTS:
warn('%s: unknown.' % k, culprit=config_file)
continue
if k.endswith('_executable'):
argv = v.split() if is_str(v) else list(v)
path = Path(argv[0])
if not path.is_absolute():
warn('should use absolute path for executables.',
culprit=(config_file, k))
Config[k] = v
Config['READ'] = True
except PasswordError:
comment('not found.', culprit=config_file)
# Now open the hashes file
hashes_file = PythonFile(get_setting('hashes_file'))
try:
contents = hashes_file.run()
Config.update({k.lower(): v for k, v in contents.items()})
except PasswordError:
pass
# Now open the account list file
account_list_file = PythonFile(get_setting('account_list_file'))
try:
contents = account_list_file.run()
Config.update({k.lower(): v for k, v in contents.items()})
except PasswordError:
pass
# initilize GPG
from .gpg import GnuPG
GnuPG.initialize()
# Now read the user key file
user_key_file = get_setting('user_key_file')
if user_key_file:
user_key_file = PythonFile(get_setting('user_key_file'))
try:
contents = user_key_file.run()
Config.update({
k.lower(): v
for k, v in contents.items() if not k.startswith('__')
})
except PasswordError:
pass
# Set the user-selected colors
Config['_label_color'] = Color(color=get_setting('label_color'),
scheme=get_setting('color_scheme'),
enable=Color.isTTY())
Config['_highlight_color'] = Color(color=get_setting('highlight_color'),
scheme=get_setting('color_scheme'),
enable=Color.isTTY())
def main():
version = f'{__version__} ({__released__})'
cmdline = docopt(__doc__, version=version)
quiet = cmdline['--quiet']
problem = False
use_color = Color.isTTY() and not cmdline['--no-color']
passes = Color('green', enable=use_color)
fails = Color('red', enable=use_color)
# prepare to create logfile
log = to_path(DATA_DIR, OVERDUE_LOG_FILE) if OVERDUE_LOG_FILE else False
if log:
data_dir = to_path(DATA_DIR)
if not data_dir.exists():
try:
# data dir does not exist, create it
data_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
except OSError as e:
warn(os_error(e))
log = False
with Inform(flush=True, quiet=quiet, logfile=log, version=version):
# read the settings file
try:
settings_file = PythonFile(CONFIG_DIR, OVERDUE_FILE)
settings = settings_file.run()
except Error as e:
e.terminate()
# gather needed settings
default_maintainer = settings.get('default_maintainer')
default_max_age = settings.get('default_max_age', 28)
dumper = settings.get('dumper', f'{username}@{hostname}')
repositories = settings.get('repositories')
root = settings.get('root')
# process repositories table
backups = []
if is_str(repositories):
for line in repositories.split('\n'):
line = line.split('#')[0].strip() # discard comments
if not line:
continue
backups.append([c.strip() for c in line.split('|')])
else:
for each in repositories:
backups.append([
each.get('host'),
each.get('path'),
each.get('maintainer'),
each.get('max_age')
])
def send_mail(recipient, subject, message):
if cmdline['--mail']:
display(f'Reporting to {recipient}.\n')
mail_cmd = ['mailx', '-r', dumper, '-s', subject, recipient]
Run(mail_cmd, stdin=message, modes='soeW0')
# check age of repositories
for host, path, maintainer, max_age in backups:
maintainer = default_maintainer if not maintainer else maintainer
max_age = float(max_age) if max_age else default_max_age
try:
path = to_path(root, path)
if path.is_dir():
paths = list(path.glob('index.*'))
if not paths:
raise Error('no sentinel file found.', culprit=path)
if len(paths) > 1:
raise Error('too many sentinel files.',
*paths,
sep='\n ')
path = paths[0]
mtime = arrow.get(path.stat().st_mtime)
delta = now - mtime
age = 24 * delta.days + delta.seconds / 3600
report = age > max_age
color = fails if report else passes
if report or not cmdline['--no-passes']:
display(
color(
dedent(f"""
HOST: {host}
sentinel file: {path!s}
last modified: {mtime}
since last change: {age:0.1f} hours
maximum age: {max_age} hours
overdue: {report}
""").lstrip()))
if report:
problem = True
subject = f"backup of {host} is overdue"
msg = overdue_message.format(host=host, path=path, age=age)
send_mail(maintainer, subject, msg)
except OSError as e:
problem = True
msg = os_error(e)
error(msg)
if maintainer:
send_mail(maintainer, f'{get_prog_name()} error',
error_message.format(msg))
except Error as e:
problem = True
e.report()
if maintainer:
send_mail(maintainer, f'{get_prog_name()} error',
error_message.format(str(e)))
terminate(problem)
# Imports {{{1
from . import cursor
from .config import get_setting
from .preferences import INITIAL_AUTOTYPE_DELAY
from shlib import Run
from inform import Color, Error, error, fatal, log, output, warn
from time import sleep
from textwrap import indent, dedent
import string
import re
# Globals {{{1
LabelColor = Color(
color=get_setting('label_color'),
scheme=get_setting('color_scheme'),
enable=Color.isTTY()
)
KEYSYMS = {
'!': 'exclam',
'"': 'quotedbl',
'#': 'numbersign',
'$': 'dollar',
'%': 'percent',
'&': 'ampersand',
"'": 'apostrophe',
'(': 'parenleft',
')': 'parenright',
'*': 'asterisk',
'+': 'plus',
',': 'comma',
'-': 'minus',
file for a python script should be named './.test.name.sum'. The summary file
for a directory should be named './dir/.test.sum'. The invocation of runTests
will create a file in the current working directory named './.test.sum'. These
summary files should contain a dictionary with keys: 'tests', 'failures'.
"""
# preliminaries {{{1
# imports {{{2
from __future__ import division, print_function
import os, sys
from json import load as loadSummary, dump as dumpSummary
from inform import Color
import argparse
# Globals {{{2
status = Color('blue', 'dark')
info = Color('magenta', 'dark')
succeed = Color('green', 'dark')
fail = Color('red', 'dark')
warning = Color('yellow', 'dark')
error = Color('red', 'dark')
exception = Color('red', 'light')
# default python
def pythonCmd(version=None):
version = version if version else "{0}.{1}".format(*sys.version_info)
return "python{}".format(version)
def coverageCmd(version=None, source=None):