class Installer:
BACKUP_SUFFIX = '.dotdropbak'
def __init__(self,
base='.',
create=True,
backup=True,
dry=False,
safe=False,
debug=False,
diff=True):
self.create = create
self.backup = backup
self.dry = dry
self.safe = safe
self.base = base
self.debug = debug
self.diff = diff
self.comparing = False
self.log = Logger(debug=self.debug)
def install(self, templater, profile, src, dst):
"""install the src to dst using a template"""
src = os.path.join(self.base, os.path.expanduser(src))
dst = os.path.join(self.base, os.path.expanduser(dst))
self.log.dbg('install {} to {}'.format(src, dst))
if os.path.isdir(src):
return self._handle_dir(templater, profile, src, dst)
return self._handle_file(templater, profile, src, dst)
def link(self, src, dst):
"""set src as the link target of dst"""
src = os.path.join(self.base, os.path.expanduser(src))
dst = os.path.join(self.base, os.path.expanduser(dst))
if os.path.exists(dst):
if os.path.realpath(dst) == os.path.realpath(src):
self.log.sub('ignoring "{}", link exists'.format(dst))
return []
if self.dry:
self.log.dry('would remove {} and link to {}'.format(dst, src))
return []
msg = 'Remove "{}" for link creation?'.format(dst)
if self.safe and not self.log.ask(msg):
msg = 'ignoring "{}", link was not created'
self.log.warn(msg.format(dst))
return []
try:
utils.remove(dst)
except OSError:
self.log.err('something went wrong with {}'.format(src))
return []
if self.dry:
self.log.dry('would link {} to {}'.format(dst, src))
return []
base = os.path.dirname(dst)
if not self._create_dirs(base):
self.log.err('creating directory for \"{}\"'.format(dst))
return []
os.symlink(src, dst)
self.log.sub('linked {} to {}'.format(dst, src))
return [(src, dst)]
def _handle_file(self, templater, profile, src, dst):
"""install src to dst when is a file"""
self.log.dbg('generate template for {}'.format(src))
content = templater.generate(src, profile)
if content is None:
self.log.err('generate from template \"{}\"'.format(src))
return []
if not os.path.exists(src):
self.log.err('source dotfile does not exist: \"{}\"'.format(src))
return []
st = os.stat(src)
ret = self._write(dst, content, st.st_mode)
if ret < 0:
self.log.err('installing \"{}\" to \"{}\"'.format(src, dst))
return []
if ret > 0:
self.log.dbg('ignoring \"{}\", same content'.format(dst))
return []
if ret == 0:
if not self.dry and not self.comparing:
self.log.sub('copied \"{}\" to \"{}\"'.format(src, dst))
return [(src, dst)]
return []
def _handle_dir(self, templater, profile, src, dst):
"""install src to dst when is a directory"""
ret = []
for entry in os.listdir(src):
f = os.path.join(src, entry)
if not os.path.isdir(f):
res = self._handle_file(templater, profile, f,
os.path.join(dst, entry))
ret.extend(res)
else:
res = self._handle_dir(templater, profile, f,
os.path.join(dst, entry))
ret.extend(res)
return ret
def _fake_diff(self, dst, content):
"""fake diff by comparing file content with content"""
cur = ''
with open(dst, 'br') as f:
cur = f.read()
return cur == content
def _write(self, dst, content, rights):
"""write content to file
return 0 for success,
1 when already exists
-1 when error"""
if self.dry:
self.log.dry('would install {}'.format(dst))
return 0
if os.path.exists(dst):
samerights = os.stat(dst).st_mode == rights
if self.diff and self._fake_diff(dst, content) and samerights:
self.log.dbg('{} is the same'.format(dst))
return 1
if self.safe and not self.log.ask('Overwrite \"{}\"'.format(dst)):
self.log.warn('ignoring \"{}\", already present'.format(dst))
return 1
if self.backup and os.path.exists(dst):
self._backup(dst)
base = os.path.dirname(dst)
if not self._create_dirs(base):
self.log.err('creating directory for \"{}\"'.format(dst))
return -1
self.log.dbg('write content to {}'.format(dst))
try:
with open(dst, 'wb') as f:
f.write(content)
except NotADirectoryError as e:
self.log.err('opening dest file: {}'.format(e))
return -1
os.chmod(dst, rights)
return 0
def _create_dirs(self, directory):
"""mkdir -p <directory>"""
if not self.create and not os.path.exists(directory):
return False
if os.path.exists(directory):
return True
self.log.dbg('mkdir -p {}'.format(directory))
os.makedirs(directory)
return os.path.exists(directory)
def _backup(self, path):
"""backup file pointed by path"""
if self.dry:
return
dst = path.rstrip(os.sep) + self.BACKUP_SUFFIX
self.log.log('backup {} to {}'.format(path, dst))
os.rename(path, dst)
def _install_to_temp(self, templater, profile, src, dst, tmpdir):
"""install a dotfile to a tempdir for comparing"""
sub = dst
if dst[0] == os.sep:
sub = dst[1:]
tmpdst = os.path.join(tmpdir, sub)
return self.install(templater, profile, src, tmpdst), tmpdst
def compare(self, templater, tmpdir, profile, src, dst, opts=''):
"""compare a temporary generated dotfile with the local one"""
self.comparing = True
retval = False, ''
drysaved = self.dry
self.dry = False
diffsaved = self.diff
self.diff = False
createsaved = self.create
self.create = True
src = os.path.expanduser(src)
dst = os.path.expanduser(dst)
self.log.dbg('comparing {} and {}'.format(src, dst))
if not os.path.exists(dst):
retval = False, '\"{}\" does not exist on local\n'.format(dst)
else:
ret, tmpdst = self._install_to_temp(templater, profile, src, dst,
tmpdir)
if ret:
self.log.dbg('diffing {} and {}'.format(tmpdst, dst))
diff = utils.diff(tmpdst, dst, raw=False, opts=opts)
if diff == '':
retval = True, ''
else:
retval = False, diff
self.dry = drysaved
self.diff = diffsaved
self.comparing = False
self.create = createsaved
return retval
class Comparator:
def __init__(self, diff_cmd='', debug=False):
"""constructor
@diff_cmd: diff command to use
@debug: enable debug
"""
self.diff_cmd = diff_cmd
self.debug = debug
self.log = Logger()
def compare(self, left, right, ignore=[]):
"""diff left (dotdrop dotfile) and right (deployed file)"""
left = os.path.expanduser(left)
right = os.path.expanduser(right)
if self.debug:
self.log.dbg('comparing {} and {}'.format(left, right))
self.log.dbg('ignore pattern(s): {}'.format(ignore))
# test type of file
if os.path.isdir(left) and not os.path.isdir(right):
return '\"{}\" is a dir while \"{}\" is a file\n'.format(
left, right)
if not os.path.isdir(left) and os.path.isdir(right):
return '\"{}\" is a file while \"{}\" is a dir\n'.format(
left, right)
# test content
if not os.path.isdir(left):
if self.debug:
self.log.dbg('is file')
return self._comp_file(left, right, ignore)
if self.debug:
self.log.dbg('is directory')
return self._comp_dir(left, right, ignore)
def _comp_file(self, left, right, ignore):
"""compare a file"""
if self.debug:
self.log.dbg('compare file {} with {}'.format(left, right))
if must_ignore([left, right], ignore, debug=self.debug):
if self.debug:
self.log.dbg('ignoring diff {} and {}'.format(left, right))
return ''
return self._diff(left, right)
def _comp_dir(self, left, right, ignore):
"""compare a directory"""
if self.debug:
self.log.dbg('compare directory {} with {}'.format(left, right))
if not os.path.exists(right):
return ''
if must_ignore([left, right], ignore, debug=self.debug):
if self.debug:
self.log.dbg('ignoring diff {} and {}'.format(left, right))
return ''
if not os.path.isdir(right):
return '\"{}\" is a file\n'.format(right)
if self.debug:
self.log.dbg('compare {} and {}'.format(left, right))
ret = []
comp = filecmp.dircmp(left, right)
# handle files only in deployed dir
for i in comp.left_only:
if must_ignore([os.path.join(left, i)], ignore, debug=self.debug):
continue
ret.append('=> \"{}\" does not exist on local\n'.format(i))
# handle files only in dotpath dir
for i in comp.right_only:
if must_ignore([os.path.join(right, i)], ignore, debug=self.debug):
continue
ret.append('=> \"{}\" does not exist in dotdrop\n'.format(i))
# same left and right but different type
funny = comp.common_funny
for i in funny:
lfile = os.path.join(left, i)
rfile = os.path.join(right, i)
if must_ignore([lfile, rfile], ignore, debug=self.debug):
continue
short = os.path.basename(lfile)
# file vs dir
ret.append('=> different type: \"{}\"\n'.format(short))
# content is different
funny = comp.diff_files
funny.extend(comp.funny_files)
funny = uniq_list(funny)
for i in funny:
lfile = os.path.join(left, i)
rfile = os.path.join(right, i)
if must_ignore([lfile, rfile], ignore, debug=self.debug):
continue
diff = self._diff(lfile, rfile, header=True)
ret.append(diff)
# recursively compare subdirs
for i in comp.common_dirs:
subleft = os.path.join(left, i)
subright = os.path.join(right, i)
ret.extend(self._comp_dir(subleft, subright, ignore))
return ''.join(ret)
def _diff(self, left, right, header=False):
"""diff two files"""
out = diff(modified=left,
original=right,
raw=False,
diff_cmd=self.diff_cmd,
debug=self.debug)
if header:
lshort = os.path.basename(left)
out = '=> diff \"{}\":\n{}'.format(lshort, out)
return out
class Comparator:
def __init__(self,
diff_cmd='',
debug=False,
ignore_missing_in_dotdrop=False):
"""constructor
@diff_cmd: diff command to use
@debug: enable debug
"""
self.diff_cmd = diff_cmd
self.debug = debug
self.log = Logger()
self.ignore_missing_in_dotdrop = ignore_missing_in_dotdrop
def compare(self, local_path, deployed_path, ignore=[]):
"""diff local_path (dotdrop dotfile) and
deployed_path (destination file)"""
local_path = os.path.expanduser(local_path)
deployed_path = os.path.expanduser(deployed_path)
if self.debug:
self.log.dbg('comparing {} and {}'.format(
local_path,
deployed_path,
))
self.log.dbg('ignore pattern(s): {}'.format(ignore))
# test type of file
if os.path.isdir(local_path) and not os.path.isdir(deployed_path):
return '\"{}\" is a dir while \"{}\" is a file\n'.format(
local_path,
deployed_path,
)
if not os.path.isdir(local_path) and os.path.isdir(deployed_path):
return '\"{}\" is a file while \"{}\" is a dir\n'.format(
local_path,
deployed_path,
)
# test content
if not os.path.isdir(local_path):
if self.debug:
self.log.dbg('{} is a file'.format(local_path))
if self.debug:
self.log.dbg('is file')
ret = self._comp_file(local_path, deployed_path, ignore)
if not ret:
ret = self._comp_mode(local_path, deployed_path)
return ret
if self.debug:
self.log.dbg('{} is a directory'.format(local_path))
ret = self._comp_dir(local_path, deployed_path, ignore)
if not ret:
ret = self._comp_mode(local_path, deployed_path)
return ret
def _comp_mode(self, local_path, deployed_path):
"""compare mode"""
local_mode = get_file_perm(local_path)
deployed_mode = get_file_perm(deployed_path)
if local_mode == deployed_mode:
return ''
if self.debug:
msg = 'mode differ {} ({:o}) and {} ({:o})'
self.log.dbg(
msg.format(local_path, local_mode, deployed_path,
deployed_mode))
ret = 'modes differ for {} ({:o}) vs {:o}\n'
return ret.format(deployed_path, deployed_mode, local_mode)
def _comp_file(self, local_path, deployed_path, ignore):
"""compare a file"""
if self.debug:
self.log.dbg('compare file {} with {}'.format(
local_path,
deployed_path,
))
if (self.ignore_missing_in_dotdrop and not
os.path.exists(local_path)) \
or must_ignore([local_path, deployed_path], ignore,
debug=self.debug):
if self.debug:
self.log.dbg('ignoring diff {} and {}'.format(
local_path,
deployed_path,
))
return ''
return self._diff(local_path, deployed_path)
def _comp_dir(self, local_path, deployed_path, ignore):
"""compare a directory"""
if self.debug:
self.log.dbg('compare directory {} with {}'.format(
local_path,
deployed_path,
))
if not os.path.exists(deployed_path):
return ''
if (self.ignore_missing_in_dotdrop and not
os.path.exists(local_path)) \
or must_ignore([local_path, deployed_path], ignore,
debug=self.debug):
if self.debug:
self.log.dbg('ignoring diff {} and {}'.format(
local_path,
deployed_path,
))
return ''
if not os.path.isdir(deployed_path):
return '\"{}\" is a file\n'.format(deployed_path)
if self.debug:
self.log.dbg('compare {} and {}'.format(local_path, deployed_path))
ret = []
comp = filecmp.dircmp(local_path, deployed_path)
# handle files only in deployed dir
for i in comp.left_only:
if self.ignore_missing_in_dotdrop:
continue
if must_ignore([os.path.join(local_path, i)],
ignore,
debug=self.debug):
continue
ret.append('=> \"{}\" does not exist on destination\n'.format(i))
# handle files only in dotpath dir
for i in comp.right_only:
if must_ignore([os.path.join(deployed_path, i)],
ignore,
debug=self.debug):
continue
if not self.ignore_missing_in_dotdrop:
ret.append('=> \"{}\" does not exist in dotdrop\n'.format(i))
# same local_path and deployed_path but different type
funny = comp.common_funny
for i in funny:
source_file = os.path.join(local_path, i)
deployed_file = os.path.join(deployed_path, i)
if self.ignore_missing_in_dotdrop and \
not os.path.exists(source_file):
continue
if must_ignore([source_file, deployed_file],
ignore,
debug=self.debug):
continue
short = os.path.basename(source_file)
# file vs dir
ret.append('=> different type: \"{}\"\n'.format(short))
# content is different
funny = comp.diff_files
funny.extend(comp.funny_files)
funny = uniq_list(funny)
for i in funny:
source_file = os.path.join(local_path, i)
deployed_file = os.path.join(deployed_path, i)
if self.ignore_missing_in_dotdrop and \
not os.path.exists(source_file):
continue
if must_ignore([source_file, deployed_file],
ignore,
debug=self.debug):
continue
ret.append(self._diff(source_file, deployed_file, header=True))
# recursively compare subdirs
for i in comp.common_dirs:
sublocal_path = os.path.join(local_path, i)
subdeployed_path = os.path.join(deployed_path, i)
ret.extend(self._comp_dir(sublocal_path, subdeployed_path, ignore))
return ''.join(ret)
def _diff(self, local_path, deployed_path, header=False):
"""diff two files"""
out = diff(modified=local_path,
original=deployed_path,
diff_cmd=self.diff_cmd,
debug=self.debug)
if header:
lshort = os.path.basename(local_path)
out = '=> diff \"{}\":\n{}'.format(lshort, out)
return out
class Installer:
def __init__(self, base='.', create=True, backup=True,
dry=False, safe=False, workdir='~/.config/dotdrop',
debug=False, diff=True, totemp=None, showdiff=False,
backup_suffix='.dotdropbak', diff_cmd=''):
"""constructor
@base: directory path where to search for templates
@create: create directory hierarchy if missing when installing
@backup: backup existing dotfile when installing
@dry: just simulate
@safe: ask for any overwrite
@workdir: where to install template before symlinking
@debug: enable debug
@diff: diff when installing if True
@totemp: deploy to this path instead of dotfile dst if not None
@showdiff: show the diff before overwriting (or asking for)
@backup_suffix: suffix for dotfile backup file
@diff_cmd: diff command to use
"""
self.create = create
self.backup = backup
self.dry = dry
self.safe = safe
self.workdir = os.path.expanduser(workdir)
self.base = base
self.debug = debug
self.diff = diff
self.totemp = totemp
self.showdiff = showdiff
self.backup_suffix = backup_suffix
self.diff_cmd = diff_cmd
self.comparing = False
self.action_executed = False
self.log = Logger()
def _log_install(self, boolean, err):
if not self.debug:
return boolean, err
if boolean:
self.log.dbg('install: SUCCESS')
else:
if err:
self.log.dbg('install: ERROR: {}'.format(err))
else:
self.log.dbg('install: IGNORED')
return boolean, err
def install(self, templater, src, dst,
actionexec=None, noempty=False,
ignore=[], template=True):
"""
install src to dst using a template
@templater: the templater object
@src: dotfile source path in dotpath
@dst: dotfile destination path in the FS
@actionexec: action executor callback
@noempty: render empty template flag
@ignore: pattern to ignore when installing
@template: template this dotfile
return
- True, None : success
- False, error_msg : error
- False, None : ignored
"""
if self.debug:
self.log.dbg('installing \"{}\" to \"{}\"'.format(src, dst))
if not dst or not src:
if self.debug:
self.log.dbg('empty dst for {}'.format(src))
return self._log_install(True, None)
self.action_executed = False
src = os.path.join(self.base, os.path.expanduser(src))
if not os.path.exists(src):
err = 'source dotfile does not exist: {}'.format(src)
return self._log_install(False, err)
dst = os.path.expanduser(dst)
if self.totemp:
dst = self._pivot_path(dst, self.totemp)
if utils.samefile(src, dst):
# symlink loop
err = 'dotfile points to itself: {}'.format(dst)
return self._log_install(False, err)
isdir = os.path.isdir(src)
if self.debug:
self.log.dbg('install {} to {}'.format(src, dst))
self.log.dbg('is a directory \"{}\": {}'.format(src, isdir))
if isdir:
b, e = self._install_dir(templater, src, dst,
actionexec=actionexec,
noempty=noempty, ignore=ignore,
template=template)
return self._log_install(b, e)
b, e = self._install_file(templater, src, dst,
actionexec=actionexec,
noempty=noempty, ignore=ignore,
template=template)
return self._log_install(b, e)
def link(self, templater, src, dst, actionexec=None, template=True):
"""
set src as the link target of dst
@templater: the templater
@src: dotfile source path in dotpath
@dst: dotfile destination path in the FS
@actionexec: action executor callback
@template: template this dotfile
return
- True, None : success
- False, error_msg : error
- False, None : ignored
"""
if self.debug:
self.log.dbg('link \"{}\" to \"{}\"'.format(src, dst))
if not dst or not src:
if self.debug:
self.log.dbg('empty dst for {}'.format(src))
return self._log_install(True, None)
self.action_executed = False
src = os.path.normpath(os.path.join(self.base,
os.path.expanduser(src)))
if not os.path.exists(src):
err = 'source dotfile does not exist: {}'.format(src)
return self._log_install(False, err)
dst = os.path.normpath(os.path.expanduser(dst))
if self.totemp:
# ignore actions
b, e = self.install(templater, src, dst, actionexec=None,
template=template)
return self._log_install(b, e)
if template and Templategen.is_template(src):
if self.debug:
self.log.dbg('dotfile is a template')
self.log.dbg('install to {} and symlink'.format(self.workdir))
tmp = self._pivot_path(dst, self.workdir, striphome=True)
i, err = self.install(templater, src, tmp, actionexec=actionexec,
template=template)
if not i and not os.path.exists(tmp):
return self._log_install(i, err)
src = tmp
b, e = self._link(src, dst, actionexec=actionexec)
return self._log_install(b, e)
def link_children(self, templater, src, dst, actionexec=None,
template=True):
"""
link all files under a given directory
@templater: the templater
@src: dotfile source path in dotpath
@dst: dotfile destination path in the FS
@actionexec: action executor callback
@template: template this dotfile
return
- True, None: success
- False, error_msg: error
- False, None, ignored
"""
if self.debug:
self.log.dbg('link_children \"{}\" to \"{}\"'.format(src, dst))
if not dst or not src:
if self.debug:
self.log.dbg('empty dst for {}'.format(src))
return self._log_install(True, None)
self.action_executed = False
parent = os.path.join(self.base, os.path.expanduser(src))
# Fail if source doesn't exist
if not os.path.exists(parent):
err = 'source dotfile does not exist: {}'.format(parent)
return self._log_install(False, err)
# Fail if source not a directory
if not os.path.isdir(parent):
if self.debug:
self.log.dbg('symlink children of {} to {}'.format(src, dst))
err = 'source dotfile is not a directory: {}'.format(parent)
return self._log_install(False, err)
dst = os.path.normpath(os.path.expanduser(dst))
if not os.path.lexists(dst):
self.log.sub('creating directory "{}"'.format(dst))
os.makedirs(dst)
if os.path.isfile(dst):
msg = ''.join([
'Remove regular file {} and ',
'replace with empty directory?',
]).format(dst)
if self.safe and not self.log.ask(msg):
err = 'ignoring "{}", nothing installed'.format(dst)
return self._log_install(False, err)
os.unlink(dst)
os.mkdir(dst)
children = os.listdir(parent)
srcs = [os.path.normpath(os.path.join(parent, child))
for child in children]
dsts = [os.path.normpath(os.path.join(dst, child))
for child in children]
installed = 0
for i in range(len(children)):
src = srcs[i]
dst = dsts[i]
if self.debug:
self.log.dbg('symlink child {} to {}'.format(src, dst))
if template and Templategen.is_template(src):
if self.debug:
self.log.dbg('dotfile is a template')
self.log.dbg('install to {} and symlink'
.format(self.workdir))
tmp = self._pivot_path(dst, self.workdir, striphome=True)
r, e = self.install(templater, src, tmp, actionexec=actionexec,
template=template)
if not r and e and not os.path.exists(tmp):
continue
src = tmp
ret, err = self._link(src, dst, actionexec=actionexec)
if ret:
installed += 1
# void actionexec if dotfile installed
# to prevent from running actions multiple times
actionexec = None
else:
if err:
return self._log_install(ret, err)
return self._log_install(installed > 0, None)
def _link(self, src, dst, actionexec=None):
"""
set src as a link target of dst
return
- True, None: success
- False, error_msg: error
- False, None, ignored
"""
overwrite = not self.safe
if os.path.lexists(dst):
if os.path.realpath(dst) == os.path.realpath(src):
msg = 'ignoring "{}", link already exists'.format(dst)
if self.debug:
self.log.dbg(msg)
return False, None
if self.dry:
self.log.dry('would remove {} and link to {}'.format(dst, src))
return True, None
if self.showdiff:
self._diff_before_write(src, dst, quiet=False)
msg = 'Remove "{}" for link creation?'.format(dst)
if self.safe and not self.log.ask(msg):
err = 'ignoring "{}", link was not created'.format(dst)
return False, err
overwrite = True
try:
utils.removepath(dst)
except OSError as e:
err = 'something went wrong with {}: {}'.format(src, e)
return False, err
if self.dry:
self.log.dry('would link {} to {}'.format(dst, src))
return True, None
base = os.path.dirname(dst)
if not self._create_dirs(base):
err = 'error creating directory for {}'.format(dst)
return False, err
r, e = self._exec_pre_actions(actionexec)
if not r:
return False, e
# re-check in case action created the file
if os.path.lexists(dst):
msg = 'Remove "{}" for link creation?'.format(dst)
if self.safe and not overwrite and not self.log.ask(msg):
err = 'ignoring "{}", link was not created'.format(dst)
return False, err
try:
utils.removepath(dst)
except OSError as e:
err = 'something went wrong with {}: {}'.format(src, e)
return False, err
os.symlink(src, dst)
self.log.sub('linked {} to {}'.format(dst, src))
return True, None
def _get_tmp_file_vars(self, src, dst):
tmp = {}
tmp['_dotfile_sub_abs_src'] = src
tmp['_dotfile_sub_abs_dst'] = dst
return tmp
def _install_file(self, templater, src, dst,
actionexec=None, noempty=False,
ignore=[], template=True):
"""install src to dst when is a file"""
if self.debug:
self.log.dbg('deploy file: {}'.format(src))
self.log.dbg('ignore empty: {}'.format(noempty))
self.log.dbg('ignore pattern: {}'.format(ignore))
self.log.dbg('template: {}'.format(template))
self.log.dbg('no empty: {}'.format(noempty))
if utils.must_ignore([src, dst], ignore, debug=self.debug):
if self.debug:
self.log.dbg('ignoring install of {} to {}'.format(src, dst))
return False, None
if utils.samefile(src, dst):
# symlink loop
err = 'dotfile points to itself: {}'.format(dst)
return False, err
if not os.path.exists(src):
err = 'source dotfile does not exist: {}'.format(src)
return False, err
# handle the file
content = None
if template:
# template the file
saved = templater.add_tmp_vars(self._get_tmp_file_vars(src, dst))
try:
content = templater.generate(src)
except UndefinedException as e:
return False, str(e)
finally:
templater.restore_vars(saved)
if noempty and utils.content_empty(content):
if self.debug:
self.log.dbg('ignoring empty template: {}'.format(src))
return False, None
if content is None:
err = 'empty template {}'.format(src)
return False, err
ret, err = self._write(src, dst,
content=content,
actionexec=actionexec,
template=template)
# build return values
if ret < 0:
# error
return False, err
if ret > 0:
# already exists
if self.debug:
self.log.dbg('ignoring {}'.format(dst))
return False, None
if ret == 0:
# success
if not self.dry and not self.comparing:
self.log.sub('copied {} to {}'.format(src, dst))
return True, None
# error
err = 'installing {} to {}'.format(src, dst)
return False, err
def _install_dir(self, templater, src, dst,
actionexec=None, noempty=False,
ignore=[], template=True):
"""install src to dst when is a directory"""
if self.debug:
self.log.dbg('install dir {}'.format(src))
self.log.dbg('ignore empty: {}'.format(noempty))
# default to nothing installed and no error
ret = False, None
if not self._create_dirs(dst):
err = 'creating directory for {}'.format(dst)
return False, err
# handle all files in dir
for entry in os.listdir(src):
f = os.path.join(src, entry)
if not os.path.isdir(f):
# is file
res, err = self._install_file(templater, f,
os.path.join(dst, entry),
actionexec=actionexec,
noempty=noempty,
ignore=ignore,
template=template)
if not res and err:
# error occured
ret = res, err
break
elif res:
# something got installed
ret = True, None
else:
# is directory
res, err = self._install_dir(templater, f,
os.path.join(dst, entry),
actionexec=actionexec,
noempty=noempty,
ignore=ignore,
template=template)
if not res and err:
# error occured
ret = res, err
break
elif res:
# something got installed
ret = True, None
return ret
def _fake_diff(self, dst, content):
"""
fake diff by comparing file content with content
returns True if same
"""
cur = ''
with open(dst, 'br') as f:
cur = f.read()
return cur == content
def _write(self, src, dst, content=None,
actionexec=None, template=True):
"""
copy dotfile / write content to file
return 0, None: for success,
1, None: when already exists
-1, err: when error
content is always empty if template is False
and is to be ignored
"""
overwrite = not self.safe
if self.dry:
self.log.dry('would install {}'.format(dst))
return 0, None
if os.path.lexists(dst):
rights = os.stat(src).st_mode
samerights = False
try:
samerights = os.stat(dst).st_mode == rights
except OSError as e:
if e.errno == errno.ENOENT:
# broken symlink
err = 'broken symlink {}'.format(dst)
return -1, err
diff = None
if self.diff:
diff = self._diff_before_write(src, dst,
content=content,
quiet=True)
if not diff and samerights:
if self.debug:
self.log.dbg('{} is the same'.format(dst))
return 1, None
if self.safe:
if self.debug:
self.log.dbg('change detected for {}'.format(dst))
if self.showdiff:
if diff is None:
# get diff
diff = self._diff_before_write(src, dst,
content=content,
quiet=True)
if diff:
self._print_diff(src, dst, diff)
if not self.log.ask('Overwrite \"{}\"'.format(dst)):
self.log.warn('ignoring {}'.format(dst))
return 1, None
overwrite = True
if self.backup and os.path.lexists(dst):
self._backup(dst)
base = os.path.dirname(dst)
if not self._create_dirs(base):
err = 'creating directory for {}'.format(dst)
return -1, err
r, e = self._exec_pre_actions(actionexec)
if not r:
return -1, e
if self.debug:
self.log.dbg('install dotfile to \"{}\"'.format(dst))
# re-check in case action created the file
if self.safe and not overwrite and os.path.lexists(dst):
if not self.log.ask('Overwrite \"{}\"'.format(dst)):
self.log.warn('ignoring {}'.format(dst))
return 1, None
if template:
# write content the file
try:
with open(dst, 'wb') as f:
f.write(content)
shutil.copymode(src, dst)
except NotADirectoryError as e:
err = 'opening dest file: {}'.format(e)
return -1, err
except Exception as e:
return -1, str(e)
else:
# copy file
try:
shutil.copyfile(src, dst)
shutil.copymode(src, dst)
except Exception as e:
return -1, str(e)
return 0, None
def _diff_before_write(self, src, dst, content=None, quiet=False):
"""
diff before writing
using a temp file if content is not None
returns diff string ('' if same)
"""
tmp = None
if content:
tmp = utils.write_to_tmpfile(content)
src = tmp
diff = utils.diff(modified=src, original=dst, raw=False,
diff_cmd=self.diff_cmd)
if tmp:
utils.removepath(tmp, logger=self.log)
if not quiet and diff:
self._print_diff(src, dst, diff)
return diff
def _print_diff(self, src, dst, diff):
"""show diff to user"""
self.log.log('diff \"{}\" VS \"{}\"'.format(dst, src))
self.log.emph(diff)
def _create_dirs(self, directory):
"""mkdir -p <directory>"""
if not self.create and not os.path.exists(directory):
if self.debug:
self.log.dbg('no mkdir as \"create\" set to false in config')
return False
if os.path.exists(directory):
return True
if self.dry:
self.log.dry('would mkdir -p {}'.format(directory))
return True
if self.debug:
self.log.dbg('mkdir -p {}'.format(directory))
os.makedirs(directory)
return os.path.exists(directory)
def _backup(self, path):
"""backup file pointed by path"""
if self.dry:
return
dst = path.rstrip(os.sep) + self.backup_suffix
self.log.log('backup {} to {}'.format(path, dst))
os.rename(path, dst)
def _pivot_path(self, path, newdir, striphome=False):
"""change path to be under newdir"""
if self.debug:
self.log.dbg('pivot new dir: \"{}\"'.format(newdir))
self.log.dbg('strip home: {}'.format(striphome))
if striphome:
path = utils.strip_home(path)
sub = path.lstrip(os.sep)
new = os.path.join(newdir, sub)
if self.debug:
self.log.dbg('pivot \"{}\" to \"{}\"'.format(path, new))
return new
def _exec_pre_actions(self, actionexec):
"""execute action executor"""
if self.action_executed:
return True, None
if not actionexec:
return True, None
ret, err = actionexec()
self.action_executed = True
return ret, err
def _install_to_temp(self, templater, src, dst, tmpdir, template=True):
"""install a dotfile to a tempdir"""
tmpdst = self._pivot_path(dst, tmpdir)
r = self.install(templater, src, tmpdst, template=template)
return r, tmpdst
def install_to_temp(self, templater, tmpdir, src, dst, template=True):
"""install a dotfile to a tempdir"""
ret = False
tmpdst = ''
# save some flags while comparing
self.comparing = True
drysaved = self.dry
self.dry = False
diffsaved = self.diff
self.diff = False
createsaved = self.create
self.create = True
# normalize src and dst
src = os.path.expanduser(src)
dst = os.path.expanduser(dst)
if self.debug:
self.log.dbg('tmp install {} (defined dst: {})'.format(src, dst))
# install the dotfile to a temp directory for comparing
r, tmpdst = self._install_to_temp(templater, src, dst, tmpdir,
template=template)
ret, err = r
if self.debug:
self.log.dbg('tmp installed in {}'.format(tmpdst))
# reset flags
self.dry = drysaved
self.diff = diffsaved
self.comparing = False
self.create = createsaved
return ret, err, tmpdst
class Updater:
def __init__(self,
conf,
dotpath,
profile,
variables,
dry,
safe,
iskey=False,
debug=False,
ignore=[],
showpatch=False):
"""constructor
@conf: configuration
@dotpath: path where dotfiles are stored
@profile: profile selected
@variables: dictionary of variables for the templates
@dry: simulate
@safe: ask for overwrite if True
@iskey: will the update be called on keys or path
@debug: enable debug
@ignore: pattern to ignore when updating
@showpatch: show patch if dotfile to update is a template
"""
self.conf = conf
self.dotpath = dotpath
self.profile = profile
self.variables = variables
self.dry = dry
self.safe = safe
self.iskey = iskey
self.debug = debug
self.ignore = ignore
self.showpatch = showpatch
self.log = Logger()
def update_path(self, path):
"""update the dotfile installed on path"""
if not os.path.lexists(path):
self.log.err('\"{}\" does not exist!'.format(path))
return False
path = self._normalize(path)
dotfile = self._get_dotfile_by_path(path)
if not dotfile:
return False
path = os.path.expanduser(path)
if self.debug:
self.log.dbg('updating {} from path \"{}\"'.format(dotfile, path))
return self._update(path, dotfile)
def update_key(self, key):
"""update the dotfile referenced by key"""
dotfile = self._get_dotfile_by_key(key)
if not dotfile:
return False
if self.debug:
self.log.dbg('updating {} from key \"{}\"'.format(dotfile, key))
path = self._normalize(dotfile.dst)
return self._update(path, dotfile)
def _update(self, path, dotfile):
"""update dotfile from file pointed by path"""
ret = False
new_path = None
self.ignores = list(set(self.ignore + dotfile.upignore))
if self.debug:
self.log.dbg('ignore pattern(s): {}'.format(self.ignores))
left = os.path.expanduser(path)
right = os.path.join(self.conf.abs_or_rel(self.dotpath), dotfile.src)
right = os.path.expanduser(right)
if self._ignore([left, right]):
return True
if dotfile.trans_w:
# apply write transformation if any
new_path = self._apply_trans_w(path, dotfile)
if not new_path:
return False
left = new_path
if os.path.isdir(left):
ret = self._handle_dir(left, right)
else:
ret = self._handle_file(left, right)
# clean temporary files
if new_path and os.path.exists(new_path):
utils.remove(new_path)
return ret
def _apply_trans_w(self, path, dotfile):
"""apply write transformation to dotfile"""
trans = dotfile.trans_w
if self.debug:
self.log.dbg('executing write transformation {}'.format(trans))
tmp = utils.get_unique_tmp_name()
if not trans.transform(path, tmp):
msg = 'transformation \"{}\" failed for {}'
self.log.err(msg.format(trans.key, dotfile.key))
if os.path.exists(tmp):
utils.remove(tmp)
return None
return tmp
def _normalize(self, path):
"""normalize the path to match dotfile"""
path = os.path.expanduser(path)
path = os.path.expandvars(path)
path = os.path.abspath(path)
home = os.path.expanduser(TILD) + os.sep
# normalize the path
if path.startswith(home):
path = path[len(home):]
path = os.path.join(TILD, path)
return path
def _get_dotfile_by_key(self, key):
"""get the dotfile matching this key"""
dotfiles = self.conf.get_dotfiles(self.profile)
subs = [d for d in dotfiles if d.key == key]
if not subs:
self.log.err('key \"{}\" not found!'.format(key))
return None
if len(subs) > 1:
found = ','.join([d.src for d in dotfiles])
self.log.err('multiple dotfiles found: {}'.format(found))
return None
return subs[0]
def _get_dotfile_by_path(self, path):
"""get the dotfile matching this path"""
dotfiles = self.conf.get_dotfiles(self.profile)
subs = [d for d in dotfiles if d.dst == path]
if not subs:
self.log.err('\"{}\" is not managed!'.format(path))
return None
if len(subs) > 1:
found = ','.join([d.src for d in dotfiles])
self.log.err('multiple dotfiles found: {}'.format(found))
return None
return subs[0]
def _is_template(self, path):
if not Templategen.is_template(path):
return False
self.log.warn('{} uses template, update manually'.format(path))
return True
def _show_patch(self, tpath, fpath):
"""provide a way to manually patch the template"""
content = self._resolve_template(tpath)
tmp = utils.write_to_tmpfile(content)
cmds = ['diff', '-u', tmp, fpath, '|', 'patch', tpath]
self.log.warn('try patching with: \"{}\"'.format(' '.join(cmds)))
return False
def _resolve_template(self, tpath):
"""resolve the template to a temporary file"""
t = Templategen(variables=self.variables,
base=self.dotpath,
debug=self.debug)
return t.generate(tpath)
def _handle_file(self, left, right, compare=True):
"""sync left (deployed file) and right (dotdrop dotfile)"""
if self._ignore([left, right]):
return True
if self.debug:
self.log.dbg('update for file {} and {}'.format(left, right))
if self._is_template(right):
# dotfile is a template
if self.debug:
self.log.dbg('{} is a template'.format(right))
if self.showpatch:
self._show_patch(right, left)
return False
if compare and filecmp.cmp(left, right, shallow=True):
# no difference
if self.debug:
self.log.dbg('identical files: {} and {}'.format(left, right))
return True
if not self._overwrite(left, right):
return False
try:
if self.dry:
self.log.dry('would cp {} {}'.format(left, right))
else:
if self.debug:
self.log.dbg('cp {} {}'.format(left, right))
shutil.copyfile(left, right)
except IOError as e:
self.log.warn('{} update failed, do manually: {}'.format(left, e))
return False
return True
def _handle_dir(self, left, right):
"""sync left (deployed dir) and right (dotdrop dir)"""
if self.debug:
self.log.dbg('handle update for dir {} to {}'.format(left, right))
# paths must be absolute (no tildes)
left = os.path.expanduser(left)
right = os.path.expanduser(right)
if self._ignore([left, right]):
return True
# find the differences
diff = filecmp.dircmp(left, right, ignore=None)
# handle directories diff
return self._merge_dirs(diff)
def _merge_dirs(self, diff):
"""Synchronize directories recursively."""
left, right = diff.left, diff.right
if self.debug:
self.log.dbg('sync dir {} to {}'.format(left, right))
if self._ignore([left, right]):
return True
# create dirs that don't exist in dotdrop
for toadd in diff.left_only:
exist = os.path.join(left, toadd)
if not os.path.isdir(exist):
# ignore files for now
continue
# match to dotdrop dotpath
new = os.path.join(right, toadd)
if self._ignore([exist, new]):
continue
if self.dry:
self.log.dry('would cp -r {} {}'.format(exist, new))
continue
if self.debug:
self.log.dbg('cp -r {} {}'.format(exist, new))
# Newly created directory should be copied as is (for efficiency).
shutil.copytree(exist, new)
# remove dirs that don't exist in deployed version
for toremove in diff.right_only:
old = os.path.join(right, toremove)
if not os.path.isdir(old):
# ignore files for now
continue
if self._ignore([old]):
continue
if self.dry:
self.log.dry('would rm -r {}'.format(old))
continue
if self.debug:
self.log.dbg('rm -r {}'.format(old))
if not self._confirm_rm_r(old):
continue
utils.remove(old)
# handle files diff
# sync files that exist in both but are different
fdiff = diff.diff_files
fdiff.extend(diff.funny_files)
fdiff.extend(diff.common_funny)
for f in fdiff:
fleft = os.path.join(left, f)
fright = os.path.join(right, f)
if self._ignore([fleft, fright]):
continue
if self.dry:
self.log.dry('would cp {} {}'.format(fleft, fright))
continue
if self.debug:
self.log.dbg('cp {} {}'.format(fleft, fright))
self._handle_file(fleft, fright, compare=False)
# copy files that don't exist in dotdrop
for toadd in diff.left_only:
exist = os.path.join(left, toadd)
if os.path.isdir(exist):
# ignore dirs, done above
continue
new = os.path.join(right, toadd)
if self._ignore([exist, new]):
continue
if self.dry:
self.log.dry('would cp {} {}'.format(exist, new))
continue
if self.debug:
self.log.dbg('cp {} {}'.format(exist, new))
shutil.copyfile(exist, new)
# remove files that don't exist in deployed version
for toremove in diff.right_only:
new = os.path.join(right, toremove)
if not os.path.exists(new):
continue
if os.path.isdir(new):
# ignore dirs, done above
continue
if self._ignore([new]):
continue
if self.dry:
self.log.dry('would rm {}'.format(new))
continue
if self.debug:
self.log.dbg('rm {}'.format(new))
utils.remove(new)
# Recursively decent into common subdirectories.
for subdir in diff.subdirs.values():
self._merge_dirs(subdir)
# Nothing more to do here.
return True
def _overwrite(self, src, dst):
"""ask for overwritting"""
msg = 'Overwrite \"{}\" with \"{}\"?'.format(dst, src)
if self.safe and not self.log.ask(msg):
return False
return True
def _confirm_rm_r(self, directory):
"""ask for rm -r directory"""
msg = 'Recursively remove \"{}\"?'.format(directory)
if self.safe and not self.log.ask(msg):
return False
return True
def _ignore(self, paths):
if utils.must_ignore(paths, self.ignores, debug=self.debug):
if self.debug:
self.log.dbg('ignoring update for {}'.format(paths))
return True
return False
class Updater:
BACKUP_SUFFIX = '.dotdropbak'
def __init__(self, conf, dotpath, dry, safe, debug):
self.home = os.path.expanduser(TILD)
self.conf = conf
self.dotpath = dotpath
self.dry = dry
self.safe = safe
self.debug = debug
self.log = Logger()
def _normalize(self, path):
"""normalize the path to match dotfile"""
path = os.path.expanduser(path)
path = os.path.expandvars(path)
# normalize the path
if path.startswith(self.home):
path = path.lstrip(self.home)
path = os.path.join(TILD, path)
return path
def _get_dotfile(self, path, profile):
"""get the dotfile matching this path"""
dotfiles = self.conf.get_dotfiles(profile)
subs = [d for d in dotfiles if d.dst == path]
if not subs:
self.log.err('\"{}\" is not managed!'.format(path))
return None
if len(subs) > 1:
found = ','.join([d.src for d in dotfiles])
self.log.err('multiple dotfiles found: {}'.format(found))
return None
return subs[0]
def update(self, path, profile):
"""update the dotfile installed on path"""
if not os.path.lexists(path):
self.log.err('\"{}\" does not exist!'.format(path))
return False
left = self._normalize(path)
dotfile = self._get_dotfile(path, profile)
if not dotfile:
return False
if self.debug:
self.log.dbg('updating {} from {}'.format(dotfile, path))
right = os.path.join(self.conf.abs_dotpath(self.dotpath), dotfile.src)
# go through all files and update
if os.path.isdir(path):
return self._handle_dir(left, right)
return self._handle_file(left, right)
def _is_template(self, path):
if Templategen.get_marker() not in open(path, 'r').read():
return False
self.log.warn('{} uses template, update manually'.format(right))
return True
def _handle_file(self, left, right, compare=True):
"""sync left (deployed file) and right (dotdrop dotfile)"""
if self.debug:
self.log.dbg('update for file {} and {}'.format(left, right))
if self._is_template(right):
return False
if compare and filecmp.cmp(left, right, shallow=True):
# no difference
if self.debug:
self.log.dbg('identical files: {} and {}'.format(left, right))
return True
if not self._overwrite(left, right):
return False
try:
if self.dry:
self.log.dry('would cp {} {}'.format(left, right))
else:
if self.debug:
self.log.dbg('cp {} {}'.format(left, right))
shutil.copyfile(left, right)
except IOError as e:
self.log.warn('{} update failed, do manually: {}'.format(left, e))
return False
return True
def _handle_dir(self, left, right):
"""sync left (deployed dir) and right (dotdrop dir)"""
if self.debug:
self.log.dbg('handle update for dir {} to {}'.format(left, right))
# find the difference
diff = filecmp.dircmp(left, right, ignore=None)
# handle directories diff
# create dirs that don't exist in dotdrop
if self.debug:
self.log.dbg('handle dirs that do not exist in dotdrop')
for toadd in diff.left_only:
exist = os.path.join(left, toadd)
if not os.path.isdir(exist):
# ignore files for now
continue
# match to dotdrop dotpath
new = os.path.join(right, toadd)
if self.dry:
self.log.dry('would mkdir -p {}'.format(new))
continue
if self.debug:
self.log.dbg('mkdir -p {}'.format(new))
self._create_dirs(new)
# remove dirs that don't exist in deployed version
if self.debug:
self.log.dbg('remove dirs that do not exist in deployed version')
for toremove in diff.right_only:
new = os.path.join(right, toremove)
if self.dry:
self.log.dry('would rm -r {}'.format(new))
continue
if self.debug:
self.log.dbg('rm -r {}'.format(new))
if not self._confirm_rm_r(new):
continue
utils.remove(new)
# handle files diff
# sync files that exist in both but are different
if self.debug:
self.log.dbg('sync files that exist in both but are different')
fdiff = diff.diff_files
fdiff.extend(diff.funny_files)
fdiff.extend(diff.common_funny)
for f in fdiff:
fleft = os.path.join(left, f)
fright = os.path.join(right, f)
if self.dry:
self.log.dry('would cp {} {}'.format(fleft, fright))
continue
if self.debug:
self.log.dbg('cp {} {}'.format(fleft, fright))
self._handle_file(fleft, fright, compare=False)
# copy files that don't exist in dotdrop
if self.debug:
self.log.dbg('copy files not existing in dotdrop')
for toadd in diff.left_only:
exist = os.path.join(left, toadd)
if os.path.isdir(exist):
# ignore dirs, done above
continue
new = os.path.join(right, toadd)
if self.dry:
self.log.dry('would cp {} {}'.format(exist, new))
continue
if self.debug:
self.log.dbg('cp {} {}'.format(exist, new))
shutil.copyfile(exist, new)
# remove files that don't exist in deployed version
if self.debug:
self.log.dbg('remove files that do not exist in deployed version')
for toremove in diff.right_only:
new = os.path.join(right, toremove)
if not os.path.exists(new):
continue
if os.path.isdir(new):
# ignore dirs, done above
continue
if self.dry:
self.log.dry('would rm {}'.format(new))
continue
if self.debug:
self.log.dbg('rm {}'.format(new))
utils.remove(new)
return True
def _create_dirs(self, directory):
"""mkdir -p <directory>"""
if os.path.exists(directory):
return True
if self.dry:
self.log.dry('would mkdir -p {}'.format(directory))
return True
if self.debug:
self.log.dbg('mkdir -p {}'.format(directory))
os.makedirs(directory)
return os.path.exists(directory)
def _overwrite(self, src, dst):
"""ask for overwritting"""
msg = 'Overwrite \"{}\" with \"{}\"?'.format(dst, src)
if self.safe and not self.log.ask(msg):
return False
return True
def _confirm_rm_r(self, directory):
"""ask for rm -r directory"""
msg = 'Recursively remove \"{}\"?'.format(directory)
if self.safe and not self.log.ask(msg):
return False
return True
class Options(AttrMonitor):
def __init__(self, args=None):
"""constructor
@args: argument dictionary (if None use sys)
"""
self.args = args
if not args:
self.args = docopt(USAGE, version=VERSION)
self.log = Logger()
self.debug = self.args['--verbose']
if not self.debug and ENV_DEBUG in os.environ:
self.debug = True
if ENV_NODEBUG in os.environ:
self.debug = False
self.profile = self.args['--profile']
self.confpath = os.path.expanduser(self.args['--cfg'])
if self.debug:
self.log.dbg('config file: {}'.format(self.confpath))
self._read_config(self.profile)
self._apply_args()
self._fill_attr()
if ENV_NOBANNER not in os.environ \
and self.banner \
and not self.args['--no-banner']:
self._header()
self._print_attr()
# start monitoring for bad attribute
self._set_attr_err = True
def _header(self):
"""print the header"""
self.log.log(BANNER)
self.log.log('')
def _read_config(self, profile=None):
"""read the config file"""
self.conf = Cfg(self.confpath, profile=profile, debug=self.debug)
# transform the configs in attribute
for k, v in self.conf.get_settings().items():
setattr(self, k, v)
def _apply_args(self):
"""apply cli args as attribute"""
# the commands
self.cmd_list = self.args['list']
self.cmd_listfiles = self.args['listfiles']
self.cmd_install = self.args['install']
self.cmd_compare = self.args['compare']
self.cmd_import = self.args['import']
self.cmd_update = self.args['update']
self.cmd_detail = self.args['detail']
# adapt attributes based on arguments
self.dry = self.args['--dry']
self.safe = not self.args['--force']
self.link = LinkTypes.NOLINK
if self.link_by_default:
self.link = LinkTypes.PARENTS
if self.args['--inv-link']:
# Only invert link type from NOLINK to PARENTS and vice-versa
if self.link == LinkTypes.NOLINK:
self.link = LinkTypes.PARENTS
elif self.link == LinkTypes.PARENTS:
self.link = LinkTypes.NOLINK
# "listfiles" specifics
self.listfiles_templateonly = self.args['--template']
# "install" specifics
self.install_temporary = self.args['--temp']
self.install_keys = self.args['<key>']
self.install_diff = not self.args['--nodiff']
self.install_showdiff = self.showdiff or self.args['--showdiff']
self.install_backup_suffix = BACKUP_SUFFIX
# "compare" specifics
self.compare_dopts = self.args['--dopts']
self.compare_focus = self.args['--file']
self.compare_ignore = self.args['--ignore']
self.compare_ignore.append('*{}'.format(self.install_backup_suffix))
# "import" specifics
self.import_path = self.args['<path>']
# "update" specifics
self.update_path = self.args['<path>']
self.update_iskey = self.args['--key']
self.update_ignore = self.args['--ignore']
self.update_ignore.append('*{}'.format(self.install_backup_suffix))
self.update_showpatch = self.args['--show-patch']
# "detail" specifics
self.detail_keys = self.args['<key>']
def _fill_attr(self):
"""create attributes from conf"""
# variables
self.variables = self.conf.get_variables(self.profile,
debug=self.debug).copy()
# the dotfiles
self.dotfiles = self.conf.eval_dotfiles(self.profile,
self.variables,
debug=self.debug).copy()
# the profiles
self.profiles = self.conf.get_profiles()
def _print_attr(self):
"""print all of this class attributes"""
if not self.debug:
return
self.log.dbg('options:')
for att in dir(self):
if att.startswith('_'):
continue
val = getattr(self, att)
if callable(val):
continue
self.log.dbg('- {}: \"{}\"'.format(att, val))
def _attr_set(self, attr):
"""error when some inexistent attr is set"""
raise Exception('bad option: {}'.format(attr))
class Templategen:
def __init__(self, base='.', variables={},
func_file=[], filter_file=[], debug=False):
"""constructor
@base: directory path where to search for templates
@variables: dictionary of variables for templates
@func_file: file path to load functions from
@filter_file: file path to load filters from
@debug: enable debug
"""
self.base = base.rstrip(os.sep)
self.debug = debug
self.log = Logger()
loader1 = FileSystemLoader(self.base)
loader2 = FunctionLoader(self._template_loader)
loader = ChoiceLoader([loader1, loader2])
self.env = Environment(loader=loader,
trim_blocks=True, lstrip_blocks=True,
keep_trailing_newline=True,
block_start_string=BLOCK_START,
block_end_string=BLOCK_END,
variable_start_string=VAR_START,
variable_end_string=VAR_END,
comment_start_string=COMMENT_START,
comment_end_string=COMMENT_END)
# adding variables
self.env.globals['env'] = os.environ
if variables:
self.env.globals.update(variables)
# adding header method
self.env.globals['header'] = self._header
# adding helper methods
if self.debug:
self.log.dbg('load global functions:')
self._load_funcs_to_dic(jhelpers, self.env.globals)
if func_file:
for f in func_file:
if self.debug:
self.log.dbg('load custom functions from {}'.format(f))
self._load_path_to_dic(f, self.env.globals)
if filter_file:
for f in filter_file:
if self.debug:
self.log.dbg('load custom filters from {}'.format(f))
self._load_path_to_dic(f, self.env.filters)
if self.debug:
self.log.dbg('template additional variables: {}'.format(variables))
def generate(self, src):
"""render template from path"""
if not os.path.exists(src):
return ''
return self._handle_file(src)
def generate_string(self, string):
"""render template from string"""
if not string:
return ''
return self.env.from_string(string).render()
def add_tmp_vars(self, newvars={}):
"""add vars to the globals, make sure to call restore_vars"""
saved_globals = self.env.globals.copy()
if not newvars:
return saved_globals
self.env.globals.update(newvars)
return saved_globals
def restore_vars(self, saved_globals):
"""restore globals from add_tmp_vars"""
self.env.globals = saved_globals.copy()
def update_variables(self, variables):
"""update variables"""
self.env.globals.update(variables)
def _load_path_to_dic(self, path, dic):
mod = utils.get_module_from_path(path)
if not mod:
self.log.warn('cannot load module \"{}\"'.format(path))
return
self._load_funcs_to_dic(mod, dic)
def _load_funcs_to_dic(self, mod, dic):
"""dynamically load functions from module to dic"""
if not mod or not dic:
return
funcs = utils.get_module_functions(mod)
for name, func in funcs:
if self.debug:
self.log.dbg('load function \"{}\"'.format(name))
dic[name] = func
def _header(self, prepend=''):
"""add a comment usually in the header of a dotfile"""
return '{}{}'.format(prepend, utils.header())
def _handle_file(self, src):
"""generate the file content from template"""
_, filetype = utils.run(['file', '-b', src],
raw=False, debug=self.debug)
filetype = filetype.strip()
if self.debug:
self.log.dbg('\"{}\" filetype: {}'.format(src, filetype))
istext = self._is_text(filetype)
if self.debug:
self.log.dbg('\"{}\" is text: {}'.format(src, istext))
if not istext:
return self._handle_bin_file(src)
return self._handle_text_file(src)
def _is_text(self, fileoutput):
"""return if `file -b` output is ascii text"""
out = fileoutput.lower()
if 'text' in out:
return True
if 'empty' in out:
return True
if 'json' in out:
return True
return False
def _template_loader(self, relpath):
"""manually load template when outside of base"""
path = os.path.join(self.base, relpath)
path = os.path.normpath(path)
if not os.path.exists(path):
raise TemplateNotFound(path)
with open(path, 'r') as f:
content = f.read()
return content
def _handle_text_file(self, src):
"""write text to file"""
template_rel_path = os.path.relpath(src, self.base)
try:
template = self.env.get_template(template_rel_path)
content = template.render()
except UnicodeDecodeError:
data = self._read_bad_encoded_text(src)
content = self.generate_string(data)
return content.encode('utf-8')
def _handle_bin_file(self, src):
"""write binary to file"""
# this is dirty
if not src.startswith(self.base):
src = os.path.join(self.base, src)
with open(src, 'rb') as f:
content = f.read()
return content
def _read_bad_encoded_text(self, path):
"""decode non utf-8 data"""
with open(path, 'rb') as f:
data = f.read()
return data.decode('utf-8', 'replace')
@staticmethod
def is_template(path):
"""recursively check if any file is a template within path"""
path = os.path.expanduser(path)
if not os.path.exists(path):
return False
if os.path.isfile(path):
# is file
return Templategen._is_template(path)
for entry in os.listdir(path):
fpath = os.path.join(path, entry)
if not os.path.isfile(fpath):
# recursively explore directory
if Templategen.is_template(fpath):
return True
else:
# check if file is a template
if Templategen._is_template(fpath):
return True
return False
@staticmethod
def var_is_template(string):
"""check if variable contains template(s)"""
return VAR_START in str(string)
@staticmethod
def _is_template(path):
"""test if file pointed by path is a template"""
if not os.path.isfile(path):
return False
try:
with open(path, 'r') as f:
data = f.read()
except UnicodeDecodeError:
# is binary so surely no template
return False
markers = [BLOCK_START, VAR_START, COMMENT_START]
for marker in markers:
if marker in data:
return True
return False
class Updater:
def __init__(self,
dotpath,
variables,
dotfile_key_getter,
dotfile_dst_getter,
dotfile_path_normalizer,
dry=False,
safe=True,
debug=False,
ignore=[],
showpatch=False):
"""constructor
@dotpath: path where dotfiles are stored
@variables: dictionary of variables for the templates
@dotfile_key_getter: func to get a dotfile by key
@dotfile_dst_getter: func to get a dotfile by dst
@dotfile_path_normalizer: func to normalize dotfile dst
@dry: simulate
@safe: ask for overwrite if True
@debug: enable debug
@ignore: pattern to ignore when updating
@showpatch: show patch if dotfile to update is a template
"""
self.dotpath = dotpath
self.variables = variables
self.dotfile_key_getter = dotfile_key_getter
self.dotfile_dst_getter = dotfile_dst_getter
self.dotfile_path_normalizer = dotfile_path_normalizer
self.dry = dry
self.safe = safe
self.debug = debug
self.ignore = ignore
self.showpatch = showpatch
self.templater = Templategen(variables=self.variables,
base=self.dotpath,
debug=self.debug)
# save template vars
self.tvars = self.templater.add_tmp_vars()
self.log = Logger()
def update_path(self, path):
"""update the dotfile installed on path"""
path = os.path.expanduser(path)
if not os.path.lexists(path):
self.log.err('\"{}\" does not exist!'.format(path))
return False
dotfile = self.dotfile_dst_getter(path)
if not dotfile:
return False
if self.debug:
self.log.dbg('updating {} from path \"{}\"'.format(dotfile, path))
return self._update(path, dotfile)
def update_key(self, key):
"""update the dotfile referenced by key"""
dotfile = self.dotfile_key_getter(key)
if not dotfile:
return False
if self.debug:
self.log.dbg('updating {} from key \"{}\"'.format(dotfile, key))
path = self.dotfile_path_normalizer(dotfile.dst)
return self._update(path, dotfile)
def _update(self, path, dotfile):
"""update dotfile from file pointed by path"""
ret = False
new_path = None
ignores = list(set(self.ignore + dotfile.upignore))
self.ignores = patch_ignores(ignores, dotfile.dst, debug=self.debug)
if self.debug:
self.log.dbg('ignore pattern(s): {}'.format(self.ignores))
path = os.path.expanduser(path)
dtpath = os.path.join(self.dotpath, dotfile.src)
dtpath = os.path.expanduser(dtpath)
if self._ignore([path, dtpath]):
self.log.sub('\"{}\" ignored'.format(dotfile.key))
return True
# apply write transformation if any
new_path = self._apply_trans_w(path, dotfile)
if not new_path:
return False
if os.path.isdir(new_path):
ret = self._handle_dir(new_path, dtpath)
else:
ret = self._handle_file(new_path, dtpath)
# clean temporary files
if new_path != path and os.path.exists(new_path):
remove(new_path)
return ret
def _apply_trans_w(self, path, dotfile):
"""apply write transformation to dotfile"""
trans = dotfile.get_trans_w()
if not trans:
return path
if self.debug:
self.log.dbg('executing write transformation {}'.format(trans))
tmp = get_unique_tmp_name()
self.templater.restore_vars(self.tvars)
newvars = dotfile.get_dotfile_variables()
self.templater.add_tmp_vars(newvars=newvars)
if not trans.transform(
path, tmp, templater=self.templater, debug=self.debug):
msg = 'transformation \"{}\" failed for {}'
self.log.err(msg.format(trans.key, dotfile.key))
if os.path.exists(tmp):
remove(tmp)
return None
return tmp
def _is_template(self, path):
if not Templategen.is_template(path):
if self.debug:
self.log.dbg('{} is NO template'.format(path))
return False
self.log.warn('{} uses template, update manually'.format(path))
return True
def _show_patch(self, fpath, tpath):
"""provide a way to manually patch the template"""
content = self._resolve_template(tpath)
tmp = write_to_tmpfile(content)
cmds = ['diff', '-u', tmp, fpath, '|', 'patch', tpath]
self.log.warn('try patching with: \"{}\"'.format(' '.join(cmds)))
return False
def _resolve_template(self, tpath):
"""resolve the template to a temporary file"""
self.templater.restore_vars(self.tvars)
return self.templater.generate(tpath)
def _handle_file(self, path, dtpath, compare=True):
"""sync path (deployed file) and dtpath (dotdrop dotfile path)"""
if self._ignore([path, dtpath]):
self.log.sub('\"{}\" ignored'.format(dtpath))
return True
if self.debug:
self.log.dbg('update for file {} and {}'.format(path, dtpath))
if self._is_template(dtpath):
# dotfile is a template
if self.debug:
self.log.dbg('{} is a template'.format(dtpath))
if self.showpatch:
self._show_patch(path, dtpath)
return False
if compare and filecmp.cmp(path, dtpath, shallow=True):
# no difference
if self.debug:
self.log.dbg('identical files: {} and {}'.format(path, dtpath))
return True
if not self._overwrite(path, dtpath):
return False
try:
if self.dry:
self.log.dry('would cp {} {}'.format(path, dtpath))
else:
if self.debug:
self.log.dbg('cp {} {}'.format(path, dtpath))
shutil.copyfile(path, dtpath)
self.log.sub('\"{}\" updated'.format(dtpath))
except IOError as e:
self.log.warn('{} update failed, do manually: {}'.format(path, e))
return False
return True
def _handle_dir(self, path, dtpath):
"""sync path (deployed dir) and dtpath (dotdrop dir path)"""
if self.debug:
self.log.dbg('handle update for dir {} to {}'.format(path, dtpath))
# paths must be absolute (no tildes)
path = os.path.expanduser(path)
dtpath = os.path.expanduser(dtpath)
if self._ignore([path, dtpath]):
self.log.sub('\"{}\" ignored'.format(dtpath))
return True
# find the differences
diff = filecmp.dircmp(path, dtpath, ignore=None)
# handle directories diff
return self._merge_dirs(diff)
def _merge_dirs(self, diff):
"""Synchronize directories recursively."""
left, right = diff.left, diff.right
if self.debug:
self.log.dbg('sync dir {} to {}'.format(left, right))
if self._ignore([left, right]):
return True
# create dirs that don't exist in dotdrop
for toadd in diff.left_only:
exist = os.path.join(left, toadd)
if not os.path.isdir(exist):
# ignore files for now
continue
# match to dotdrop dotpath
new = os.path.join(right, toadd)
if self._ignore([exist, new]):
self.log.sub('\"{}\" ignored'.format(exist))
continue
if self.dry:
self.log.dry('would cp -r {} {}'.format(exist, new))
continue
if self.debug:
self.log.dbg('cp -r {} {}'.format(exist, new))
# Newly created directory should be copied as is (for efficiency).
shutil.copytree(exist, new)
self.log.sub('\"{}\" dir added'.format(new))
# remove dirs that don't exist in deployed version
for toremove in diff.right_only:
old = os.path.join(right, toremove)
if not os.path.isdir(old):
# ignore files for now
continue
if self._ignore([old]):
continue
if self.dry:
self.log.dry('would rm -r {}'.format(old))
continue
if self.debug:
self.log.dbg('rm -r {}'.format(old))
if not self._confirm_rm_r(old):
continue
remove(old)
self.log.sub('\"{}\" dir removed'.format(old))
# handle files diff
# sync files that exist in both but are different
fdiff = diff.diff_files
fdiff.extend(diff.funny_files)
fdiff.extend(diff.common_funny)
for f in fdiff:
fleft = os.path.join(left, f)
fright = os.path.join(right, f)
if self._ignore([fleft, fright]):
continue
if self.dry:
self.log.dry('would cp {} {}'.format(fleft, fright))
continue
if self.debug:
self.log.dbg('cp {} {}'.format(fleft, fright))
self._handle_file(fleft, fright, compare=False)
# copy files that don't exist in dotdrop
for toadd in diff.left_only:
exist = os.path.join(left, toadd)
if os.path.isdir(exist):
# ignore dirs, done above
continue
new = os.path.join(right, toadd)
if self._ignore([exist, new]):
continue
if self.dry:
self.log.dry('would cp {} {}'.format(exist, new))
continue
if self.debug:
self.log.dbg('cp {} {}'.format(exist, new))
shutil.copyfile(exist, new)
self.log.sub('\"{}\" added'.format(new))
# remove files that don't exist in deployed version
for toremove in diff.right_only:
new = os.path.join(right, toremove)
if not os.path.exists(new):
continue
if os.path.isdir(new):
# ignore dirs, done above
continue
if self._ignore([new]):
continue
if self.dry:
self.log.dry('would rm {}'.format(new))
continue
if self.debug:
self.log.dbg('rm {}'.format(new))
remove(new)
self.log.sub('\"{}\" removed'.format(new))
# Recursively decent into common subdirectories.
for subdir in diff.subdirs.values():
self._merge_dirs(subdir)
# Nothing more to do here.
return True
def _overwrite(self, src, dst):
"""ask for overwritting"""
msg = 'Overwrite \"{}\" with \"{}\"?'.format(dst, src)
if self.safe and not self.log.ask(msg):
return False
return True
def _confirm_rm_r(self, directory):
"""ask for rm -r directory"""
msg = 'Recursively remove \"{}\"?'.format(directory)
if self.safe and not self.log.ask(msg):
return False
return True
def _ignore(self, paths):
if must_ignore(paths, self.ignores, debug=self.debug):
if self.debug:
self.log.dbg('ignoring update for {}'.format(paths))
return True
return False
class Templategen:
def __init__(self, base='.', debug=False):
self.base = base.rstrip(os.sep)
loader = FileSystemLoader(self.base)
self.env = Environment(loader=loader,
trim_blocks=True,
lstrip_blocks=True,
keep_trailing_newline=True,
block_start_string=BLOCK_START,
block_end_string=BLOCK_END,
variable_start_string=VAR_START,
variable_end_string=VAR_END,
comment_start_string=COMMENT_START,
comment_end_string=COMMENT_END)
self.log = Logger(debug=debug)
def generate(self, src, profile):
if not os.path.exists(src):
return ''
return self._handle_file(src, profile)
def _handle_file(self, src, profile):
"""generate the file content from template"""
filetype = utils.run(['file', '-b', src], raw=False).strip()
self.log.dbg('\"{}\" filetype: {}'.format(src, filetype))
istext = 'text' in filetype
self.log.dbg('\"{}\" is text: {}'.format(src, istext))
if not istext:
return self._handle_bin_file(src, profile)
return self._handle_text_file(src, profile)
def _handle_text_file(self, src, profile):
"""write text to file"""
template_rel_path = os.path.relpath(src, self.base)
try:
template = self.env.get_template(template_rel_path)
content = template.render(profile=profile, env=os.environ)
except UnicodeDecodeError:
data = self._read_bad_encoded_text(src)
template = self.env.from_string(data)
content = template.render(profile=profile, env=os.environ)
content = content.encode('UTF-8')
return content
def _handle_bin_file(self, src, profile):
"""write binary to file"""
# this is dirty
if not src.startswith(self.base):
src = os.path.join(self.base, src)
with open(src, 'rb') as f:
return f.read()
def _read_bad_encoded_text(self, path):
"""decode non utf-8 data"""
with open(path, 'rb') as f:
data = f.read()
return data.decode('utf-8', 'replace')
def get_marker():
"""return identifier for template dotfile"""
return BLOCK_START
class Importer:
def __init__(self,
profile,
conf,
dotpath,
diff_cmd,
dry=False,
safe=True,
debug=False,
keepdot=True,
ignore=[]):
"""constructor
@profile: the selected profile
@conf: configuration manager
@dotpath: dotfiles dotpath
@diff_cmd: diff command to use
@dry: simulate
@safe: ask for overwrite if True
@debug: enable debug
@keepdot: keep dot prefix
@ignore: patterns to ignore when importing
"""
self.profile = profile
self.conf = conf
self.dotpath = dotpath
self.diff_cmd = diff_cmd
self.dry = dry
self.safe = safe
self.debug = debug
self.keepdot = keepdot
self.ignore = ignore
self.umask = get_umask()
self.log = Logger()
def import_path(self,
path,
import_as=None,
import_link=LinkTypes.NOLINK,
import_mode=False):
"""
import a dotfile pointed by path
returns:
1: 1 dotfile imported
0: ignored
-1: error
"""
if self.debug:
self.log.dbg('import {}'.format(path))
if not os.path.exists(path):
self.log.err('\"{}\" does not exist, ignored!'.format(path))
return -1
return self._import(path,
import_as=import_as,
import_link=import_link,
import_mode=import_mode)
def _import(self,
path,
import_as=None,
import_link=LinkTypes.NOLINK,
import_mode=False):
"""
import path
returns:
1: 1 dotfile imported
0: ignored
-1: error
"""
# normalize path
dst = path.rstrip(os.sep)
dst = os.path.abspath(dst)
# test if must be ignored
if self._ignore(dst):
return 0
# ask confirmation for symlinks
if self.safe:
realdst = os.path.realpath(dst)
if dst != realdst:
msg = '\"{}\" is a symlink, dereference it and continue?'
if not self.log.ask(msg.format(dst)):
return 0
# create src path
src = strip_home(dst)
if import_as:
# handle import as
src = os.path.expanduser(import_as)
src = src.rstrip(os.sep)
src = os.path.abspath(src)
src = strip_home(src)
if self.debug:
self.log.dbg('import src for {} as {}'.format(dst, src))
# with or without dot prefix
strip = '.' + os.sep
if self.keepdot:
strip = os.sep
src = src.lstrip(strip)
# get the permission
perm = get_file_perm(dst)
# get the link attribute
linktype = import_link
if linktype == LinkTypes.LINK_CHILDREN and \
not os.path.isdir(path):
self.log.err('importing \"{}\" failed!'.format(path))
return -1
if self._already_exists(src, dst):
return -1
if self.debug:
self.log.dbg('import dotfile: src:{} dst:{}'.format(src, dst))
if not self._prepare_hierarchy(src, dst):
return -1
# handle file mode
chmod = None
dflperm = get_default_file_perms(dst, self.umask)
if self.debug:
self.log.dbg('import mode: {}'.format(import_mode))
if import_mode or perm != dflperm:
if self.debug:
msg = 'adopt mode {:o} (umask {:o})'
self.log.dbg(msg.format(perm, dflperm))
chmod = perm
# add file to config file
retconf = self.conf.new_dotfile(src, dst, linktype, chmod=chmod)
if not retconf:
self.log.warn('\"{}\" ignored during import'.format(path))
return 0
self.log.sub('\"{}\" imported'.format(path))
return 1
def _prepare_hierarchy(self, src, dst):
"""prepare hierarchy for dotfile"""
srcf = os.path.join(self.dotpath, src)
if self._ignore(srcf):
return False
srcfd = os.path.dirname(srcf)
if self._ignore(srcfd):
return False
# a dotfile in dotpath already exists at that spot
if os.path.exists(srcf):
if self.safe:
c = Comparator(debug=self.debug, diff_cmd=self.diff_cmd)
diff = c.compare(srcf, dst)
if diff != '':
# files are different, dunno what to do
self.log.log('diff \"{}\" VS \"{}\"'.format(dst, srcf))
self.log.emph(diff)
# ask user
msg = 'Dotfile \"{}\" already exists, overwrite?'
if not self.log.ask(msg.format(srcf)):
return False
if self.debug:
self.log.dbg('will overwrite existing file')
# create directory hierarchy
if self.dry:
cmd = 'mkdir -p {}'.format(srcfd)
self.log.dry('would run: {}'.format(cmd))
else:
try:
os.makedirs(srcfd, exist_ok=True)
except Exception:
self.log.err('importing \"{}\" failed!'.format(dst))
return False
if self.dry:
self.log.dry('would copy {} to {}'.format(dst, srcf))
else:
# copy the file to the dotpath
try:
if os.path.isdir(dst):
if os.path.exists(srcf):
shutil.rmtree(srcf)
ig = shutil.ignore_patterns(*self.ignore)
shutil.copytree(dst,
srcf,
copy_function=self._cp,
ignore=ig)
else:
shutil.copy2(dst, srcf)
except shutil.Error as e:
src = e.args[0][0][0]
why = e.args[0][0][2]
self.log.err('importing \"{}\" failed: {}'.format(src, why))
return True
def _cp(self, src, dst):
"""the copy function for copytree"""
# test if must be ignored
if self._ignore(src):
return
shutil.copy2(src, dst)
def _already_exists(self, src, dst):
"""
test no other dotfile exists with same
dst for this profile but different src
"""
dfs = self.conf.get_dotfile_by_dst(dst)
if not dfs:
return False
for df in dfs:
profiles = self.conf.get_profiles_by_dotfile_key(df.key)
profiles = [x.key for x in profiles]
if self.profile in profiles and \
not self.conf.get_dotfile_by_src_dst(src, dst):
# same profile
# different src
self.log.err('duplicate dotfile for this profile')
return True
return False
def _ignore(self, path):
if must_ignore([path], self.ignore, debug=self.debug):
if self.debug:
self.log.dbg('ignoring import of {}'.format(path))
self.log.warn('{} ignored'.format(path))
return True
return False
class Options(AttrMonitor):
"""dotdrop options manager"""
def __init__(self, args=None):
"""constructor
@args: argument dictionary (if None use sys)
"""
# attributes gotten from self.conf.get_settings()
self.banner = None
self.showdiff = None
self.default_actions = []
self.instignore = None
self.force_chmod = None
self.cmpignore = None
self.impignore = None
self.upignore = None
self.link_on_import = None
self.chmod_on_import = None
self.check_version = None
self.clear_workdir = None
self.key_prefix = None
self.key_separator = None
# args parsing
self.args = {}
if not args:
self.args = docopt(USAGE, version=VERSION)
if args:
self.args = args.copy()
self.debug = self.args['--verbose'] or ENV_DEBUG in os.environ
self.log = Logger(debug=self.debug)
self.dry = self.args['--dry']
if ENV_NODEBUG in os.environ:
# force disabling debugs
self.debug = False
self.profile = self.args['--profile']
self.confpath = self._get_config_path()
if not self.confpath:
raise YamlException('no config file found')
self.log.dbg('#################################################')
self.log.dbg('#################### DOTDROP ####################')
self.log.dbg('#################################################')
self.log.dbg('version: {}'.format(VERSION))
self.log.dbg('command: {}'.format(' '.join(sys.argv)))
self.log.dbg('config file: {}'.format(self.confpath))
self._read_config()
self._apply_args()
self._fill_attr()
if ENV_NOBANNER not in os.environ \
and self.banner \
and not self.args['--no-banner']:
self._header()
self._debug_attr()
# start monitoring for bad attribute
self._set_attr_err = True
@classmethod
def _get_config_from_fs(cls):
"""get config from filesystem"""
# look in ~/.config/dotdrop
cfg = os.path.expanduser(HOMECFG)
path = os.path.join(cfg, CONFIG)
if os.path.exists(path):
return path
# look in /etc/xdg/dotdrop
path = os.path.join(ETCXDGCFG, CONFIG)
if os.path.exists(path):
return path
# look in /etc/dotdrop
path = os.path.join(ETCCFG, CONFIG)
if os.path.exists(path):
return path
return ''
def _get_config_path(self):
"""get the config path"""
# cli provided
if self.args['--cfg']:
return os.path.expanduser(self.args['--cfg'])
# environment variable provided
if ENV_CONFIG in os.environ:
return os.path.expanduser(os.environ[ENV_CONFIG])
# look in current directory
if os.path.exists(CONFIG):
return CONFIG
# look in XDG_CONFIG_HOME
if ENV_XDG in os.environ:
cfg = os.path.expanduser(os.environ[ENV_XDG])
path = os.path.join(cfg, NAME, CONFIG)
if os.path.exists(path):
return path
return self._get_config_from_fs()
def _header(self):
"""display the header"""
self.log.log(BANNER)
self.log.log('')
def _read_config(self):
"""read the config file"""
self.conf = Cfg(self.confpath, self.profile, debug=self.debug,
dry=self.dry)
# transform the config settings to self attribute
settings = self.conf.get_settings()
debug_dict('effective settings', settings, self.debug)
for k, val in settings.items():
setattr(self, k, val)
def _apply_args_files(self):
"""files specifics"""
self.files_templateonly = self.args['--template']
self.files_grepable = self.args['--grepable']
def _apply_args_install(self):
"""install specifics"""
self.install_force_action = self.args['--force-actions']
self.install_temporary = self.args['--temp']
self.install_keys = self.args['<key>']
self.install_diff = not self.args['--nodiff']
self.install_showdiff = self.showdiff or self.args['--showdiff']
self.install_backup_suffix = BACKUP_SUFFIX
self.install_default_actions_pre = [a for a in self.default_actions
if a.kind == Action.pre]
self.install_default_actions_post = [a for a in self.default_actions
if a.kind == Action.post]
self.install_ignore = self.instignore
self.install_force_chmod = self.force_chmod
self.install_clear_workdir = self.args['--workdir-clear'] or \
self.clear_workdir
def _apply_args_compare(self):
"""compare specifics"""
self.compare_focus = self.args['--file']
self.compare_ignore = self.args['--ignore']
self.compare_ignore.extend(self.cmpignore)
self.compare_ignore.append('*{}'.format(self.install_backup_suffix))
self.compare_ignore = uniq_list(self.compare_ignore)
self.compare_fileonly = self.args['--file-only']
self.ignore_missing_in_dotdrop = self.ignore_missing_in_dotdrop or \
self.args['--ignore-missing']
def _apply_args_import(self):
"""import specifics"""
self.import_path = self.args['<path>']
self.import_as = self.args['--as']
self.import_mode = self.args['--preserve-mode'] or self.chmod_on_import
self.import_ignore = self.args['--ignore']
self.import_ignore.extend(self.impignore)
self.import_ignore.append('*{}'.format(self.install_backup_suffix))
self.import_ignore = uniq_list(self.import_ignore)
def _apply_args_update(self):
"""update specifics"""
self.update_path = self.args['<path>']
self.update_iskey = self.args['--key']
self.update_ignore = self.args['--ignore']
self.update_ignore.extend(self.upignore)
self.update_ignore.append('*{}'.format(self.install_backup_suffix))
self.update_ignore = uniq_list(self.update_ignore)
self.update_showpatch = self.args['--show-patch']
def _apply_args_profiles(self):
"""profiles specifics"""
self.profiles_grepable = self.args['--grepable']
def _apply_args_remove(self):
"""remove specifics"""
self.remove_path = self.args['<path>']
self.remove_iskey = self.args['--key']
def _apply_args_detail(self):
"""detail specifics"""
self.detail_keys = self.args['<key>']
def _apply_args(self):
"""apply cli args as attribute"""
# the commands
self.cmd_profiles = self.args['profiles']
self.cmd_files = self.args['files']
self.cmd_install = self.args['install']
self.cmd_compare = self.args['compare']
self.cmd_import = self.args['import']
self.cmd_update = self.args['update']
self.cmd_detail = self.args['detail']
self.cmd_remove = self.args['remove']
# adapt attributes based on arguments
self.safe = not self.args['--force']
try:
if ENV_WORKERS in os.environ:
workers = int(os.environ[ENV_WORKERS])
else:
workers = int(self.args['--workers'])
self.workers = workers
except ValueError:
self.log.err('bad option for --workers')
sys.exit(USAGE)
# import link default value
self.import_link = self.link_on_import
if self.args['--link']:
# overwrite default import link with cli switch
link = self.args['--link']
if link not in OPT_LINK:
self.log.err('bad option for --link: {}'.format(link))
sys.exit(USAGE)
self.import_link = OPT_LINK[link]
# "files" specifics
self._apply_args_files()
# "install" specifics
self._apply_args_install()
# "compare" specifics
self._apply_args_compare()
# "import" specifics
self._apply_args_import()
# "update" specifics
self._apply_args_update()
# "profiles" specifics
self._apply_args_profiles()
# "detail" specifics
self._apply_args_detail()
# "remove" specifics
self._apply_args_remove()
def _fill_attr(self):
"""create attributes from conf"""
# variables
self.variables = self.conf.get_variables()
# the dotfiles
self.dotfiles = self.conf.get_dotfiles()
# the profiles
self.profiles = self.conf.get_profiles()
def _debug_attr(self):
"""debug display all of this class attributes"""
if not self.debug:
return
self.log.dbg('effective options:')
for att in dir(self):
if att.startswith('_'):
continue
val = getattr(self, att)
if callable(val):
continue
if isinstance(val, list):
debug_list('-> {}'.format(att), val, self.debug)
elif isinstance(val, dict):
debug_dict('-> {}'.format(att), val, self.debug)
else:
self.log.dbg('-> {}: {}'.format(att, val))
def _attr_set(self, attr):
"""error when some inexistent attr is set"""
raise Exception('bad option: {}'.format(attr))
class Templategen:
def __init__(self, profile='', base='.', variables={}, debug=False):
self.base = base.rstrip(os.sep)
self.debug = debug
self.log = Logger()
loader = FileSystemLoader(self.base)
self.env = Environment(loader=loader,
trim_blocks=True,
lstrip_blocks=True,
keep_trailing_newline=True,
block_start_string=BLOCK_START,
block_end_string=BLOCK_END,
variable_start_string=VAR_START,
variable_end_string=VAR_END,
comment_start_string=COMMENT_START,
comment_end_string=COMMENT_END)
# adding variables
self.env.globals['env'] = os.environ
if profile:
self.env.globals['profile'] = profile
self.env.globals.update(variables)
# adding header method
self.env.globals['header'] = self._header
# adding helper methods
self.env.globals['exists'] = jhelpers.exists
def generate(self, src):
"""render template from path"""
if not os.path.exists(src):
return ''
return self._handle_file(src)
def generate_string(self, string):
"""render template from string"""
if not string:
return ''
return self.env.from_string(string).render()
def _header(self, prepend=''):
"""add a comment usually in the header of a dotfile"""
return '{}{}'.format(prepend, utils.header())
def _handle_file(self, src):
"""generate the file content from template"""
_, filetype = utils.run(['file', '-b', src],
raw=False,
debug=self.debug)
filetype = filetype.strip()
if self.debug:
self.log.dbg('\"{}\" filetype: {}'.format(src, filetype))
istext = 'text' in filetype
if self.debug:
self.log.dbg('\"{}\" is text: {}'.format(src, istext))
if not istext:
return self._handle_bin_file(src)
return self._handle_text_file(src)
def _handle_text_file(self, src):
"""write text to file"""
template_rel_path = os.path.relpath(src, self.base)
try:
template = self.env.get_template(template_rel_path)
content = template.render()
except UnicodeDecodeError:
data = self._read_bad_encoded_text(src)
template = self.env.from_string(data)
content = template.render()
content = content.encode('UTF-8')
return content
def _handle_bin_file(self, src):
"""write binary to file"""
# this is dirty
if not src.startswith(self.base):
src = os.path.join(self.base, src)
with open(src, 'rb') as f:
content = f.read()
return content
def _read_bad_encoded_text(self, path):
"""decode non utf-8 data"""
with open(path, 'rb') as f:
data = f.read()
return data.decode('utf-8', 'replace')
@staticmethod
def is_template(path):
"""recursively check if any file is a template within path"""
path = os.path.expanduser(path)
if not os.path.exists(path):
return False
if os.path.isfile(path):
# is file
return Templategen._is_template(path)
for entry in os.listdir(path):
fpath = os.path.join(path, entry)
if not os.path.isfile(fpath):
# rec explore dir
if Templategen.is_template(fpath):
return True
else:
# is file a template
if Templategen._is_template(fpath):
return True
return False
@staticmethod
def _is_template(path):
"""test if file pointed by path is a template"""
if not os.path.isfile(path):
return False
try:
with open(path, 'r') as f:
data = f.read()
except UnicodeDecodeError:
# is binary so surely no template
return False
markers = [BLOCK_START, VAR_START, COMMENT_START]
for marker in markers:
if marker in data:
return True
return False
class Updater:
def __init__(self,
dotpath,
variables,
conf,
dry=False,
safe=True,
debug=False,
ignore=[],
showpatch=False,
ignore_missing_in_dotdrop=False):
"""constructor
@dotpath: path where dotfiles are stored
@variables: dictionary of variables for the templates
@conf: configuration manager
@dry: simulate
@safe: ask for overwrite if True
@debug: enable debug
@ignore: pattern to ignore when updating
@showpatch: show patch if dotfile to update is a template
"""
self.dotpath = dotpath
self.variables = variables
self.conf = conf
self.dry = dry
self.safe = safe
self.debug = debug
self.ignore = ignore
self.ignores = None
self.showpatch = showpatch
self.ignore_missing_in_dotdrop = ignore_missing_in_dotdrop
self.templater = Templategen(variables=self.variables,
base=self.dotpath,
debug=self.debug)
# save template vars
self.tvars = self.templater.add_tmp_vars()
self.log = Logger()
def update_path(self, path):
"""update the dotfile installed on path"""
path = os.path.expanduser(path)
if not os.path.lexists(path):
self.log.err('\"{}\" does not exist!'.format(path))
return False
dotfiles = self.conf.get_dotfile_by_dst(path)
if not dotfiles:
return False
for dotfile in dotfiles:
if not dotfile:
msg = 'invalid dotfile for update: {}'
self.log.err(msg.format(dotfile.key))
return False
if self.debug:
msg = 'updating {} from path \"{}\"'
self.log.dbg(msg.format(dotfile, path))
if not self._update(path, dotfile):
return False
return True
def update_key(self, key):
"""update the dotfile referenced by key"""
dotfile = self.conf.get_dotfile(key)
if not dotfile:
return False
if self.debug:
self.log.dbg('updating {} from key \"{}\"'.format(dotfile, key))
path = self.conf.path_to_dotfile_dst(dotfile.dst)
return self._update(path, dotfile)
def _update(self, path, dotfile):
"""update dotfile from file pointed by path"""
ret = False
new_path = None
ignores = list(set(self.ignore + dotfile.upignore))
self.ignores = patch_ignores(ignores, dotfile.dst, debug=self.debug)
if self.debug:
self.log.dbg('ignore pattern(s): {}'.format(self.ignores))
deployed_path = os.path.expanduser(path)
local_path = os.path.join(self.dotpath, dotfile.src)
local_path = os.path.expanduser(local_path)
if not os.path.exists(deployed_path):
msg = '\"{}\" does not exist'
self.log.err(msg.format(deployed_path))
return False
if not os.path.exists(local_path):
msg = '\"{}\" does not exist, import it first'
self.log.err(msg.format(local_path))
return False
ignore_missing_in_dotdrop = self.ignore_missing_in_dotdrop or \
dotfile.ignore_missing_in_dotdrop
if (ignore_missing_in_dotdrop and not os.path.exists(local_path)) or \
self._ignore([deployed_path, local_path]):
self.log.sub('\"{}\" ignored'.format(dotfile.key))
return True
# apply write transformation if any
new_path = self._apply_trans_w(deployed_path, dotfile)
if not new_path:
return False
# save current rights
deployed_mode = get_file_perm(deployed_path)
local_mode = get_file_perm(local_path)
# handle the pointed file
if os.path.isdir(new_path):
ret = self._handle_dir(new_path, local_path, dotfile)
else:
ret = self._handle_file(new_path, local_path, dotfile)
if deployed_mode != local_mode:
# mirror rights
if self.debug:
m = 'adopt mode {:o} for {}'
self.log.dbg(m.format(deployed_mode, dotfile.key))
r = self.conf.update_dotfile(dotfile.key, deployed_mode)
if r:
ret = True
# clean temporary files
if new_path != deployed_path and os.path.exists(new_path):
removepath(new_path, logger=self.log)
return ret
def _apply_trans_w(self, path, dotfile):
"""apply write transformation to dotfile"""
trans = dotfile.get_trans_w()
if not trans:
return path
if self.debug:
self.log.dbg('executing write transformation {}'.format(trans))
tmp = get_unique_tmp_name()
self.templater.restore_vars(self.tvars)
newvars = dotfile.get_dotfile_variables()
self.templater.add_tmp_vars(newvars=newvars)
if not trans.transform(
path, tmp, templater=self.templater, debug=self.debug):
msg = 'transformation \"{}\" failed for {}'
self.log.err(msg.format(trans.key, dotfile.key))
if os.path.exists(tmp):
removepath(tmp, logger=self.log)
return None
return tmp
def _is_template(self, path):
if not Templategen.is_template(path, ignore=self.ignores):
if self.debug:
self.log.dbg('{} is NO template'.format(path))
return False
self.log.warn('{} uses template, update manually'.format(path))
return True
def _show_patch(self, fpath, tpath):
"""provide a way to manually patch the template"""
content = self._resolve_template(tpath)
tmp = write_to_tmpfile(content)
mirror_file_rights(tpath, tmp)
cmds = ['diff', '-u', tmp, fpath, '|', 'patch', tpath]
self.log.warn('try patching with: \"{}\"'.format(' '.join(cmds)))
return False
def _resolve_template(self, tpath):
"""resolve the template to a temporary file"""
self.templater.restore_vars(self.tvars)
return self.templater.generate(tpath)
def _same_rights(self, left, right):
"""return True if files have the same modes"""
try:
lefts = get_file_perm(left)
rights = get_file_perm(right)
return lefts == rights
except OSError as e:
self.log.err(e)
return False
def _mirror_rights(self, src, dst):
srcr = get_file_perm(src)
dstr = get_file_perm(dst)
if srcr == dstr:
return
if self.debug:
msg = 'copy rights from {} ({:o}) to {} ({:o})'
self.log.dbg(msg.format(src, srcr, dst, dstr))
try:
mirror_file_rights(src, dst)
except OSError as e:
self.log.err(e)
def _handle_file(self, deployed_path, local_path, dotfile, compare=True):
"""sync path (deployed file) and local_path (dotdrop dotfile path)"""
if self._ignore([deployed_path, local_path]):
self.log.sub('\"{}\" ignored'.format(local_path))
return True
if self.debug:
self.log.dbg('update for file {} and {}'.format(
deployed_path,
local_path,
))
if self._is_template(local_path):
# dotfile is a template
if self.debug:
self.log.dbg('{} is a template'.format(local_path))
if self.showpatch:
try:
self._show_patch(deployed_path, local_path)
except UndefinedException as e:
msg = 'unable to show patch for {}: {}'.format(
deployed_path,
e,
)
self.log.warn(msg)
return False
if compare and \
filecmp.cmp(deployed_path, local_path, shallow=False) and \
self._same_rights(deployed_path, local_path):
# no difference
if self.debug:
self.log.dbg('identical files: {} and {}'.format(
deployed_path,
local_path,
))
return True
if not self._overwrite(deployed_path, local_path):
return False
try:
if self.dry:
self.log.dry('would cp {} {}'.format(
deployed_path,
local_path,
))
else:
if self.debug:
self.log.dbg('cp {} {}'.format(deployed_path, local_path))
shutil.copyfile(deployed_path, local_path)
self._mirror_rights(deployed_path, local_path)
self.log.sub('\"{}\" updated'.format(local_path))
except IOError as e:
self.log.warn('{} update failed, do manually: {}'.format(
deployed_path, e))
return False
return True
def _handle_dir(self, deployed_path, local_path, dotfile):
"""sync path (local dir) and local_path (dotdrop dir path)"""
if self.debug:
self.log.dbg('handle update for dir {} to {}'.format(
deployed_path,
local_path,
))
# paths must be absolute (no tildes)
deployed_path = os.path.expanduser(deployed_path)
local_path = os.path.expanduser(local_path)
if self._ignore([deployed_path, local_path]):
self.log.sub('\"{}\" ignored'.format(local_path))
return True
# find the differences
diff = filecmp.dircmp(deployed_path, local_path, ignore=None)
# handle directories diff
ret = self._merge_dirs(diff, dotfile)
self._mirror_rights(deployed_path, local_path)
return ret
def _merge_dirs(self, diff, dotfile):
"""Synchronize directories recursively."""
left, right = diff.left, diff.right
if self.debug:
self.log.dbg('sync dir {} to {}'.format(left, right))
if self._ignore([left, right]):
return True
ignore_missing_in_dotdrop = self.ignore_missing_in_dotdrop or \
dotfile.ignore_missing_in_dotdrop
# create dirs that don't exist in dotdrop
for toadd in diff.left_only:
exist = os.path.join(left, toadd)
if not os.path.isdir(exist):
# ignore files for now
continue
# match to dotdrop dotpath
new = os.path.join(right, toadd)
if (ignore_missing_in_dotdrop and not os.path.exists(new)) or \
self._ignore([exist, new]):
self.log.sub('\"{}\" ignored'.format(exist))
continue
if self.dry:
self.log.dry('would cp -r {} {}'.format(exist, new))
continue
if self.debug:
self.log.dbg('cp -r {} {}'.format(exist, new))
# Newly created directory should be copied as is (for efficiency).
def ig(src, names):
whitelist, blacklist = set(), set()
for ignore in self.ignores:
for name in names:
path = os.path.join(src, name)
if ignore.startswith('!') and \
fnmatch.fnmatch(path, ignore[1:]):
# add to whitelist
whitelist.add(name)
elif fnmatch.fnmatch(path, ignore):
# add to blacklist
blacklist.add(name)
return blacklist - whitelist
shutil.copytree(exist, new, ignore=ig)
self.log.sub('\"{}\" dir added'.format(new))
# remove dirs that don't exist in deployed version
for toremove in diff.right_only:
old = os.path.join(right, toremove)
if not os.path.isdir(old):
# ignore files for now
continue
if self._ignore([old]):
continue
if self.dry:
self.log.dry('would rm -r {}'.format(old))
continue
if self.debug:
self.log.dbg('rm -r {}'.format(old))
if not self._confirm_rm_r(old):
continue
removepath(old, logger=self.log)
self.log.sub('\"{}\" dir removed'.format(old))
# handle files diff
# sync files that exist in both but are different
fdiff = diff.diff_files
fdiff.extend(diff.funny_files)
fdiff.extend(diff.common_funny)
for f in fdiff:
fleft = os.path.join(left, f)
fright = os.path.join(right, f)
if (ignore_missing_in_dotdrop and not os.path.exists(fright)) or \
self._ignore([fleft, fright]):
continue
if self.dry:
self.log.dry('would cp {} {}'.format(fleft, fright))
continue
if self.debug:
self.log.dbg('cp {} {}'.format(fleft, fright))
self._handle_file(fleft, fright, dotfile, compare=False)
# copy files that don't exist in dotdrop
for toadd in diff.left_only:
exist = os.path.join(left, toadd)
if os.path.isdir(exist):
# ignore dirs, done above
continue
new = os.path.join(right, toadd)
if (ignore_missing_in_dotdrop and not os.path.exists(new)) or \
self._ignore([exist, new]):
continue
if self.dry:
self.log.dry('would cp {} {}'.format(exist, new))
continue
if self.debug:
self.log.dbg('cp {} {}'.format(exist, new))
shutil.copyfile(exist, new)
self._mirror_rights(exist, new)
self.log.sub('\"{}\" added'.format(new))
# remove files that don't exist in deployed version
for toremove in diff.right_only:
new = os.path.join(right, toremove)
if not os.path.exists(new):
continue
if os.path.isdir(new):
# ignore dirs, done above
continue
if self._ignore([new]):
continue
if self.dry:
self.log.dry('would rm {}'.format(new))
continue
if self.debug:
self.log.dbg('rm {}'.format(new))
removepath(new, logger=self.log)
self.log.sub('\"{}\" removed'.format(new))
# compare rights
for common in diff.common_files:
leftf = os.path.join(left, common)
rightf = os.path.join(right, common)
if not self._same_rights(leftf, rightf):
self._mirror_rights(leftf, rightf)
# Recursively decent into common subdirectories.
for subdir in diff.subdirs.values():
self._merge_dirs(subdir, dotfile)
# Nothing more to do here.
return True
def _overwrite(self, src, dst):
"""ask for overwritting"""
msg = 'Overwrite \"{}\" with \"{}\"?'.format(dst, src)
if self.safe and not self.log.ask(msg):
return False
return True
def _confirm_rm_r(self, directory):
"""ask for rm -r directory"""
msg = 'Recursively remove \"{}\"?'.format(directory)
if self.safe and not self.log.ask(msg):
return False
return True
def _ignore(self, paths):
if must_ignore(paths, self.ignores, debug=self.debug):
if self.debug:
self.log.dbg('ignoring update for {}'.format(paths))
return True
return False
class Options(AttrMonitor):
def __init__(self, args=None):
"""constructor
@args: argument dictionary (if None use sys)
"""
self.args = {}
if not args:
self.args = docopt(USAGE, version=VERSION)
if args:
self.args = args.copy()
self.log = Logger()
self.debug = self.args['--verbose'] or ENV_DEBUG in os.environ
self.dry = self.args['--dry']
if ENV_NODEBUG in os.environ:
# force disabling debugs
self.debug = False
self.profile = self.args['--profile']
self.confpath = self._get_config_path()
if self.debug:
self.log.dbg('version: {}'.format(VERSION))
self.log.dbg('command: {}'.format(' '.join(sys.argv)))
self.log.dbg('config file: {}'.format(self.confpath))
self._read_config()
self._apply_args()
self._fill_attr()
if ENV_NOBANNER not in os.environ \
and self.banner \
and not self.args['--no-banner']:
self._header()
self._debug_attr()
# start monitoring for bad attribute
self._set_attr_err = True
def _get_config_path(self):
"""get the config path"""
# cli provided
if self.args['--cfg']:
return os.path.expanduser(self.args['--cfg'])
# environment variable provided
if ENV_CONFIG in os.environ:
return os.path.expanduser(os.environ[ENV_CONFIG])
# look in current directory
if os.path.exists(CONFIG):
return CONFIG
# look in XDG_CONFIG_HOME
if ENV_XDG in os.environ:
cfg = os.path.expanduser(os.environ[ENV_XDG])
path = os.path.join(cfg, NAME, CONFIG)
if os.path.exists(path):
return path
# look in ~/.config/dotdrop
cfg = os.path.expanduser(HOMECFG)
path = os.path.join(cfg, CONFIG)
if os.path.exists(path):
return path
# look in /etc/xdg/dotdrop
path = os.path.join(ETCXDGCFG, CONFIG)
if os.path.exists(path):
return path
# look in /etc/dotdrop
path = os.path.join(ETCCFG, CONFIG)
if os.path.exists(path):
return path
return ''
def _header(self):
"""display the header"""
self.log.log(BANNER)
self.log.log('')
def _read_config(self):
"""read the config file"""
self.conf = Cfg(self.confpath,
self.profile,
debug=self.debug,
dry=self.dry)
# transform the config settings to self attribute
self._debug_dict('effective settings', self.conf.get_settings())
for k, v in self.conf.get_settings().items():
setattr(self, k, v)
def _apply_args(self):
"""apply cli args as attribute"""
# the commands
self.cmd_profiles = self.args['profiles']
self.cmd_files = self.args['files']
self.cmd_install = self.args['install']
self.cmd_compare = self.args['compare']
self.cmd_import = self.args['import']
self.cmd_update = self.args['update']
self.cmd_detail = self.args['detail']
self.cmd_remove = self.args['remove']
# adapt attributes based on arguments
self.safe = not self.args['--force']
# import link default value
self.import_link = self.link_on_import
if self.args['--link']:
# overwrite default import link with cli switch
link = self.args['--link']
if link not in OPT_LINK.keys():
self.log.err('bad option for --link: {}'.format(link))
sys.exit(USAGE)
self.import_link = OPT_LINK[link]
# "files" specifics
self.files_templateonly = self.args['--template']
self.files_grepable = self.args['--grepable']
# "profiles" specifics
self.profiles_grepable = self.args['--grepable']
# "install" specifics
self.install_force_action = self.args['--force-actions']
self.install_temporary = self.args['--temp']
self.install_keys = self.args['<key>']
self.install_diff = not self.args['--nodiff']
self.install_showdiff = self.showdiff or self.args['--showdiff']
self.install_backup_suffix = BACKUP_SUFFIX
self.install_default_actions_pre = [
a for a in self.default_actions if a.kind == Action.pre
]
self.install_default_actions_post = [
a for a in self.default_actions if a.kind == Action.post
]
self.install_ignore = self.instignore
# "compare" specifics
self.compare_focus = self.args['--file']
self.compare_ignore = self.args['--ignore']
self.compare_ignore.extend(self.cmpignore)
self.compare_ignore.append('*{}'.format(self.install_backup_suffix))
self.compare_ignore = uniq_list(self.compare_ignore)
# "import" specifics
self.import_path = self.args['<path>']
self.import_as = self.args['--as']
# "update" specifics
self.update_path = self.args['<path>']
self.update_iskey = self.args['--key']
self.update_ignore = self.args['--ignore']
self.update_ignore.extend(self.upignore)
self.update_ignore.append('*{}'.format(self.install_backup_suffix))
self.update_ignore = uniq_list(self.update_ignore)
self.update_showpatch = self.args['--show-patch']
# "detail" specifics
self.detail_keys = self.args['<key>']
# "remove" specifics
self.remove_path = self.args['<path>']
self.remove_iskey = self.args['--key']
def _fill_attr(self):
"""create attributes from conf"""
# variables
self.variables = self.conf.get_variables()
# the dotfiles
self.dotfiles = self.conf.get_dotfiles()
# the profiles
self.profiles = self.conf.get_profiles()
def _debug_attr(self):
"""debug display all of this class attributes"""
if not self.debug:
return
self.log.dbg('effective options:')
for att in dir(self):
if att.startswith('_'):
continue
val = getattr(self, att)
if callable(val):
continue
if type(val) is list:
self._debug_list('-> {}'.format(att), val)
elif type(val) is dict:
self._debug_dict('-> {}'.format(att), val)
else:
self.log.dbg('-> {}: {}'.format(att, val))
def _attr_set(self, attr):
"""error when some inexistent attr is set"""
raise Exception('bad option: {}'.format(attr))
def _debug_list(self, title, elems):
"""pretty print list"""
if not self.debug:
return
self.log.dbg('{}:'.format(title))
for e in elems:
self.log.dbg('\t- {}'.format(e))
def _debug_dict(self, title, elems):
"""pretty print dict"""
if not self.debug:
return
self.log.dbg('{}:'.format(title))
for k, v in elems.items():
if type(v) is list:
self.log.dbg('\t- \"{}\":'.format(k))
for i in v:
self.log.dbg('\t\t- {}'.format(i))
else:
self.log.dbg('\t- \"{}\": {}'.format(k, v))
class Comparator:
def __init__(self, diffopts='', debug=False):
"""constructor
@diffopts: cli switches to pass to unix diff
@debug: enable debug
"""
self.diffopts = diffopts
self.debug = debug
self.log = Logger()
def compare(self, left, right, ignore=[]):
"""diff left (dotdrop dotfile) and right (deployed file)"""
left = os.path.expanduser(left)
right = os.path.expanduser(right)
if self.debug:
self.log.dbg('comparing {} and {}'.format(left, right))
self.log.dbg('ignore pattern(s): {}'.format(ignore))
if not os.path.isdir(left):
return self._comp_file(left, right, ignore)
return self._comp_dir(left, right, ignore)
def _comp_file(self, left, right, ignore):
"""compare a file"""
if utils.must_ignore([left, right], ignore, debug=self.debug):
if self.debug:
self.log.dbg('ignoring diff {} and {}'.format(left, right))
return ''
return self._diff(left, right)
def _comp_dir(self, left, right, ignore):
"""compare a directory"""
if not os.path.exists(right):
return ''
if utils.must_ignore([left, right], ignore, debug=self.debug):
if self.debug:
self.log.dbg('ignoring diff {} and {}'.format(left, right))
return ''
if self.debug:
self.log.dbg('compare {} and {}'.format(left, right))
ret = []
comp = filecmp.dircmp(left, right)
# handle files only in deployed file
for i in comp.left_only:
if utils.must_ignore([os.path.join(left, i)],
ignore,
debug=self.debug):
continue
ret.append('=> \"{}\" does not exist on local\n'.format(i))
for i in comp.right_only:
if utils.must_ignore([os.path.join(right, i)],
ignore,
debug=self.debug):
continue
ret.append('=> \"{}\" does not exist in dotdrop\n'.format(i))
# same left and right but different type
funny = comp.common_funny
for i in funny:
lfile = os.path.join(left, i)
rfile = os.path.join(right, i)
short = os.path.basename(lfile)
# file vs dir
ret.append('different type: \"{}\"\n'.format(short))
# content is different
funny = comp.diff_files
funny.extend(comp.funny_files)
funny = list(set(funny))
for i in funny:
lfile = os.path.join(left, i)
rfile = os.path.join(right, i)
diff = self._diff(lfile, rfile, header=True)
ret.append(diff)
return ''.join(ret)
def _diff(self, left, right, header=False):
"""diff using the unix tool diff"""
diff = utils.diff(left,
right,
raw=False,
opts=self.diffopts,
debug=self.debug)
if header:
lshort = os.path.basename(left)
rshort = os.path.basename(right)
diff = '=> diff \"{}\":\n{}'.format(lshort, diff)
return diff
class CfgAggregator:
file_prefix = 'f'
dir_prefix = 'd'
key_sep = '_'
def __init__(self, path, profile=None, debug=False):
"""
high level config parser
@path: path to the config file
@profile: selected profile
@debug: debug flag
"""
self.path = path
self.profile = profile
self.debug = debug
self.log = Logger()
self._load()
def _load(self):
"""load lower level config"""
self.cfgyaml = CfgYaml(self.path, self.profile, debug=self.debug)
# settings
self.settings = Settings.parse(None, self.cfgyaml.settings)
if self.debug:
self.log.dbg('settings: {}'.format(self.settings))
# dotfiles
self.dotfiles = Dotfile.parse_dict(self.cfgyaml.dotfiles)
if self.debug:
self.log.dbg('dotfiles: {}'.format(self.dotfiles))
# profiles
self.profiles = Profile.parse_dict(self.cfgyaml.profiles)
if self.debug:
self.log.dbg('profiles: {}'.format(self.profiles))
# actions
self.actions = Action.parse_dict(self.cfgyaml.actions)
if self.debug:
self.log.dbg('actions: {}'.format(self.actions))
# trans_r
self.trans_r = Transform.parse_dict(self.cfgyaml.trans_r)
if self.debug:
self.log.dbg('trans_r: {}'.format(self.trans_r))
# trans_w
self.trans_w = Transform.parse_dict(self.cfgyaml.trans_w)
if self.debug:
self.log.dbg('trans_w: {}'.format(self.trans_w))
# variables
self.variables = self.cfgyaml.get_variables()
if self.debug:
self.log.dbg('variables: {}'.format(self.variables))
# patch dotfiles in profiles
self._patch_keys_to_objs(self.profiles, "dotfiles", self.get_dotfile)
# patch action in dotfiles actions
self._patch_keys_to_objs(self.dotfiles, "actions",
self._get_action_w_args)
# patch action in profiles actions
self._patch_keys_to_objs(self.profiles, "actions",
self._get_action_w_args)
# patch actions in settings default_actions
self._patch_keys_to_objs([self.settings], "default_actions",
self._get_action_w_args)
if self.debug:
msg = 'default actions: {}'.format(self.settings.default_actions)
self.log.dbg(msg)
# patch trans_w/trans_r in dotfiles
self._patch_keys_to_objs(self.dotfiles,
"trans_r",
self._get_trans_r,
islist=False)
self._patch_keys_to_objs(self.dotfiles,
"trans_w",
self._get_trans_w,
islist=False)
def _patch_keys_to_objs(self, containers, keys, get_by_key, islist=True):
"""
map for each key in the attribute 'keys' in 'containers'
the returned object from the method 'get_by_key'
"""
if not containers:
return
if self.debug:
self.log.dbg('patching {} ...'.format(keys))
for c in containers:
objects = []
okeys = getattr(c, keys)
if not okeys:
continue
if not islist:
okeys = [okeys]
for k in okeys:
o = get_by_key(k)
if not o:
err = 'bad {} key for \"{}\": {}'.format(keys, c, k)
self.log.err(err)
raise Exception(err)
objects.append(o)
if not islist:
objects = objects[0]
if self.debug:
self.log.dbg('patching {}.{} with {}'.format(c, keys, objects))
setattr(c, keys, objects)
def del_dotfile(self, dotfile):
"""remove this dotfile from the config"""
return self.cfgyaml.del_dotfile(dotfile.key)
def del_dotfile_from_profile(self, dotfile, profile):
"""remove this dotfile from this profile"""
return self.cfgyaml.del_dotfile_from_profile(dotfile.key, profile.key)
def new(self, src, dst, link, profile_key):
"""
import a new dotfile
@src: path in dotpath
@dst: path in FS
@link: LinkType
@profile_key: to which profile
"""
dst = self.path_to_dotfile_dst(dst)
dotfile = self.get_dotfile_by_dst(dst)
if not dotfile:
# get a new dotfile with a unique key
key = self._get_new_dotfile_key(dst)
if self.debug:
self.log.dbg('new dotfile key: {}'.format(key))
# add the dotfile
self.cfgyaml.add_dotfile(key, src, dst, link)
dotfile = Dotfile(key, dst, src)
key = dotfile.key
ret = self.cfgyaml.add_dotfile_to_profile(key, profile_key)
if self.debug:
self.log.dbg('new dotfile {} to profile {}'.format(
key, profile_key))
# reload
self.cfgyaml.save()
if self.debug:
self.log.dbg('RELOADING')
self._load()
return ret
def _get_new_dotfile_key(self, dst):
"""return a new unique dotfile key"""
path = os.path.expanduser(dst)
existing_keys = [x.key for x in self.dotfiles]
if self.settings.longkey:
return self._get_long_key(path, existing_keys)
return self._get_short_key(path, existing_keys)
def _norm_key_elem(self, elem):
"""normalize path element for sanity"""
elem = elem.lstrip('.')
elem = elem.replace(' ', '-')
return elem.lower()
def _split_path_for_key(self, path):
"""return a list of path elements, excluded home path"""
p = strip_home(path)
dirs = []
while True:
p, f = os.path.split(p)
dirs.append(f)
if not p or not f:
break
dirs.reverse()
# remove empty entries
dirs = filter(None, dirs)
# normalize entries
return list(map(self._norm_key_elem, dirs))
def _get_long_key(self, path, keys):
"""
return a unique long key representing the
absolute path of path
"""
dirs = self._split_path_for_key(path)
prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix
key = self.key_sep.join([prefix] + dirs)
return self._uniq_key(key, keys)
def _get_short_key(self, path, keys):
"""
return a unique key where path
is known not to be an already existing dotfile
"""
dirs = self._split_path_for_key(path)
dirs.reverse()
prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix
entries = []
for d in dirs:
entries.insert(0, d)
key = self.key_sep.join([prefix] + entries)
if key not in keys:
return key
return self._uniq_key(key, keys)
def _uniq_key(self, key, keys):
"""unique dotfile key"""
newkey = key
cnt = 1
while newkey in keys:
# if unable to get a unique path
# get a random one
newkey = self.key_sep.join([key, str(cnt)])
cnt += 1
return newkey
def path_to_dotfile_dst(self, path):
"""normalize the path to match dotfile dst"""
path = os.path.expanduser(path)
path = os.path.expandvars(path)
path = os.path.abspath(path)
home = os.path.expanduser(TILD) + os.sep
# normalize the path
if path.startswith(home):
path = path[len(home):]
path = os.path.join(TILD, path)
return path
def get_dotfile_by_dst(self, dst):
"""get a dotfile by dst"""
try:
return next(d for d in self.dotfiles if d.dst == dst)
except StopIteration:
return None
def save(self):
"""save the config"""
return self.cfgyaml.save()
def dump(self):
"""dump the config dictionary"""
return self.cfgyaml.dump()
def get_settings(self):
"""return settings as a dict"""
return self.settings.serialize()[Settings.key_yaml]
def get_variables(self):
"""return variables"""
return self.variables
def get_profiles(self):
"""return profiles"""
return self.profiles
def get_profile(self, key):
"""return profile by key"""
try:
return next(x for x in self.profiles if x.key == key)
except StopIteration:
return None
def get_profiles_by_dotfile_key(self, key):
"""return all profiles having this dotfile"""
res = []
for p in self.profiles:
keys = [d.key for d in p.dotfiles]
if key in keys:
res.append(p)
return res
def get_dotfiles(self, profile=None):
"""return dotfiles dict for this profile key"""
if not profile:
return self.dotfiles
try:
pro = self.get_profile(profile)
if not pro:
return []
return pro.dotfiles
except StopIteration:
return []
def get_dotfile(self, key):
"""return dotfile by key"""
try:
return next(x for x in self.dotfiles if x.key == key)
except StopIteration:
return None
def _get_action(self, key):
"""return action by key"""
try:
return next(x for x in self.actions if x.key == key)
except StopIteration:
return None
def _get_action_w_args(self, key):
"""return action by key with the arguments"""
fields = shlex.split(key)
if len(fields) > 1:
# we have args
key, *args = fields
if self.debug:
self.log.dbg('action with parm: {} and {}'.format(key, args))
action = self._get_action(key).copy(args)
else:
action = self._get_action(key)
return action
def _get_trans_r(self, key):
"""return the trans_r with this key"""
try:
return next(x for x in self.trans_r if x.key == key)
except StopIteration:
return None
def _get_trans_w(self, key):
"""return the trans_w with this key"""
try:
return next(x for x in self.trans_w if x.key == key)
except StopIteration:
return None
class Comparator:
def __init__(self, diffopts='', ignore=[], debug=False):
self.diffopts = diffopts
self.ignore = [os.path.expanduser(i) for i in ignore]
self.debug = debug
self.log = Logger()
def compare(self, left, right):
"""diff left (dotdrop dotfile) and right (deployed file)"""
left = os.path.expanduser(left)
right = os.path.expanduser(right)
if not os.path.isdir(left):
return self._comp_file(left, right)
return self._comp_dir(left, right)
def _comp_file(self, left, right):
"""compare a file"""
if left in self.ignore or right in self.ignore:
if self.debug:
self.log.dbg('ignoring diff {} and {}'.format(left, right))
return ''
return self._diff(left, right)
def _comp_dir(self, left, right):
"""compare a directory"""
if left in self.ignore or right in self.ignore:
if self.debug:
self.log.dbg('ignoring diff {} and {}'.format(left, right))
return ''
if self.debug:
self.log.dbg('compare {} and {}'.format(left, right))
ret = []
comp = filecmp.dircmp(left, right, ignore=self.ignore)
# handle files only in deployed file
for i in comp.left_only:
if os.path.join(left, i) in self.ignore:
continue
ret.append('only in left: \"{}\"\n'.format(i))
for i in comp.right_only:
if os.path.join(right, i) in self.ignore:
continue
ret.append('only in right: \"{}\"\n'.format(i))
# same left and right but different type
funny = comp.common_funny
for i in funny:
lfile = os.path.join(left, i)
rfile = os.path.join(right, i)
short = os.path.basename(lfile)
# file vs dir
ret.append('different type: \"{}\"\n'.format(short))
# content is different
funny = comp.diff_files
funny.extend(comp.funny_files)
funny = list(set(funny))
for i in funny:
lfile = os.path.join(left, i)
rfile = os.path.join(right, i)
diff = self._diff(lfile, rfile, header=True)
ret.append(diff)
return ''.join(ret)
def _diff(self, left, right, header=False):
"""diff using the unix tool diff"""
diff = utils.diff(left,
right,
raw=False,
opts=self.diffopts,
debug=self.debug)
if header:
lshort = os.path.basename(left)
rshort = os.path.basename(right)
diff = 'diff \"{}\":\n{}'.format(lshort, diff)
return diff
class CfgYaml:
# global entries
key_settings = 'config'
key_dotfiles = 'dotfiles'
key_profiles = 'profiles'
key_actions = 'actions'
old_key_trans_r = 'trans'
key_trans_r = 'trans_read'
key_trans_w = 'trans_write'
key_variables = 'variables'
key_dvariables = 'dynvariables'
action_pre = 'pre'
action_post = 'post'
# profiles/dotfiles entries
key_dotfile_src = 'src'
key_dotfile_dst = 'dst'
key_dotfile_link = 'link'
key_dotfile_actions = 'actions'
key_dotfile_link_children = 'link_children'
key_dotfile_noempty = 'ignoreempty'
# profile
key_profile_dotfiles = 'dotfiles'
key_profile_include = 'include'
key_profile_variables = 'variables'
key_profile_dvariables = 'dynvariables'
key_profile_actions = 'actions'
key_all = 'ALL'
# import entries
key_import_actions = 'import_actions'
key_import_configs = 'import_configs'
key_import_variables = 'import_variables'
key_import_profile_dfs = 'import'
# settings
key_settings_dotpath = 'dotpath'
key_settings_workdir = 'workdir'
key_settings_link_dotfile_default = 'link_dotfile_default'
key_settings_noempty = 'ignoreempty'
key_settings_minversion = 'minversion'
key_imp_link = 'link_on_import'
# link values
lnk_nolink = LinkTypes.NOLINK.name.lower()
lnk_link = LinkTypes.LINK.name.lower()
lnk_children = LinkTypes.LINK_CHILDREN.name.lower()
def __init__(self, path, profile=None, debug=False):
"""
config parser
@path: config file path
@profile: the selected profile
@debug: debug flag
"""
self.path = os.path.abspath(path)
self.profile = profile
self.debug = debug
self.log = Logger()
# config needs to be written
self.dirty = False
# indicates the config has been updated
self.dirty_deprecated = False
if not os.path.exists(path):
err = 'invalid config path: \"{}\"'.format(path)
if self.debug:
self.log.dbg(err)
raise YamlException(err)
self.yaml_dict = self._load_yaml(self.path)
# live patch deprecated entries
self._fix_deprecated(self.yaml_dict)
# parse to self variables
self._parse_main_yaml(self.yaml_dict)
if self.debug:
self.log.dbg('before normalization: {}'.format(self.yaml_dict))
# resolve variables
self.variables, self.prokeys = self._merge_variables()
# apply variables
self._apply_variables()
# process imported variables (import_variables)
self._import_variables()
# process imported actions (import_actions)
self._import_actions()
# process imported profile dotfiles (import)
self._import_profiles_dotfiles()
# process imported configs (import_configs)
self._import_configs()
# process profile include
self._resolve_profile_includes()
# process profile ALL
self._resolve_profile_all()
# patch dotfiles paths
self._resolve_dotfile_paths()
if self.debug:
self.log.dbg('after normalization: {}'.format(self.yaml_dict))
def get_variables(self):
"""retrieve all variables"""
return self.variables
########################################################
# parsing
########################################################
def _parse_main_yaml(self, dic):
"""parse the different blocks"""
self.ori_settings = self._get_entry(dic, self.key_settings)
self.settings = Settings(None).serialize().get(self.key_settings)
self.settings.update(self.ori_settings)
# resolve minimum version
if self.key_settings_minversion in self.settings:
minversion = self.settings[self.key_settings_minversion]
self._check_minversion(minversion)
# resolve settings paths
p = self._norm_path(self.settings[self.key_settings_dotpath])
self.settings[self.key_settings_dotpath] = p
p = self._norm_path(self.settings[self.key_settings_workdir])
self.settings[self.key_settings_workdir] = p
if self.debug:
self.log.dbg('settings: {}'.format(self.settings))
# dotfiles
self.ori_dotfiles = self._get_entry(dic, self.key_dotfiles)
self.dotfiles = deepcopy(self.ori_dotfiles)
keys = self.dotfiles.keys()
if len(keys) != len(list(set(keys))):
dups = [x for x in keys if x not in list(set(keys))]
err = 'duplicate dotfile keys found: {}'.format(dups)
raise YamlException(err)
self.dotfiles = self._norm_dotfiles(self.dotfiles)
if self.debug:
self.log.dbg('dotfiles: {}'.format(self.dotfiles))
# profiles
self.ori_profiles = self._get_entry(dic, self.key_profiles)
self.profiles = deepcopy(self.ori_profiles)
self.profiles = self._norm_profiles(self.profiles)
if self.debug:
self.log.dbg('profiles: {}'.format(self.profiles))
# actions
self.ori_actions = self._get_entry(dic,
self.key_actions,
mandatory=False)
self.actions = deepcopy(self.ori_actions)
self.actions = self._norm_actions(self.actions)
if self.debug:
self.log.dbg('actions: {}'.format(self.actions))
# trans_r
key = self.key_trans_r
if self.old_key_trans_r in dic:
self.log.warn('\"trans\" is deprecated, please use \"trans_read\"')
dic[self.key_trans_r] = dic[self.old_key_trans_r]
del dic[self.old_key_trans_r]
self.ori_trans_r = self._get_entry(dic, key, mandatory=False)
self.trans_r = deepcopy(self.ori_trans_r)
if self.debug:
self.log.dbg('trans_r: {}'.format(self.trans_r))
# trans_w
self.ori_trans_w = self._get_entry(dic,
self.key_trans_w,
mandatory=False)
self.trans_w = deepcopy(self.ori_trans_w)
if self.debug:
self.log.dbg('trans_w: {}'.format(self.trans_w))
# variables
self.ori_variables = self._get_entry(dic,
self.key_variables,
mandatory=False)
if self.debug:
self.log.dbg('variables: {}'.format(self.ori_variables))
# dynvariables
self.ori_dvariables = self._get_entry(dic,
self.key_dvariables,
mandatory=False)
if self.debug:
self.log.dbg('dynvariables: {}'.format(self.ori_dvariables))
def _resolve_dotfile_paths(self):
"""resolve dotfile paths"""
t = Templategen(variables=self.variables)
for dotfile in self.dotfiles.values():
# src
src = dotfile[self.key_dotfile_src]
new = t.generate_string(src)
if new != src and self.debug:
self.log.dbg('dotfile: {} -> {}'.format(src, new))
src = new
src = os.path.join(self.settings[self.key_settings_dotpath], src)
dotfile[self.key_dotfile_src] = self._norm_path(src)
# dst
dst = dotfile[self.key_dotfile_dst]
new = t.generate_string(dst)
if new != dst and self.debug:
self.log.dbg('dotfile: {} -> {}'.format(dst, new))
dst = new
dotfile[self.key_dotfile_dst] = self._norm_path(dst)
def _rec_resolve_vars(self, variables):
"""recursive resolve variables"""
default = self._get_variables_dict(self.profile)
t = Templategen(variables=self._merge_dict(default, variables))
for k in variables.keys():
val = variables[k]
while Templategen.var_is_template(val):
val = t.generate_string(val)
variables[k] = val
t.update_variables(variables)
return variables
def _merge_variables(self):
"""
resolve all variables across the config
apply them to any needed entries
and return the full list of variables
"""
if self.debug:
self.log.dbg('get local variables')
# get all variables from local and resolve
var = self._get_variables_dict(self.profile)
# get all dynvariables from local and resolve
dvar = self._get_dvariables_dict()
# temporarly resolve all variables for "include"
merged = self._merge_dict(dvar, var)
merged = self._rec_resolve_vars(merged)
self._debug_vars(merged)
# exec dynvariables
self._shell_exec_dvars(dvar.keys(), merged)
if self.debug:
self.log.dbg('local variables resolved')
self._debug_vars(merged)
# resolve profile includes
t = Templategen(variables=merged)
for k, v in self.profiles.items():
if self.key_profile_include in v:
new = []
for k in v[self.key_profile_include]:
new.append(t.generate_string(k))
v[self.key_profile_include] = new
# now get the included ones
pro_var = self._get_included_variables(self.profile,
seen=[self.profile])
pro_dvar = self._get_included_dvariables(self.profile,
seen=[self.profile])
# exec incl dynvariables
self._shell_exec_dvars(pro_dvar.keys(), pro_dvar)
# merge all and resolve
merged = self._merge_dict(pro_var, merged)
merged = self._merge_dict(pro_dvar, merged)
merged = self._rec_resolve_vars(merged)
if self.debug:
self.log.dbg('resolve all uses of variables in config')
self._debug_vars(merged)
prokeys = list(pro_var.keys()) + list(pro_dvar.keys())
return merged, prokeys
def _apply_variables(self):
"""template any needed parts of the config"""
t = Templategen(variables=self.variables)
# import_actions
new = []
entries = self.settings.get(self.key_import_actions, [])
new = self._template_list(t, entries)
if new:
self.settings[self.key_import_actions] = new
# import_configs
entries = self.settings.get(self.key_import_configs, [])
new = self._template_list(t, entries)
if new:
self.settings[self.key_import_configs] = new
# import_variables
entries = self.settings.get(self.key_import_variables, [])
new = self._template_list(t, entries)
if new:
self.settings[self.key_import_variables] = new
# profile's import
for k, v in self.profiles.items():
entries = v.get(self.key_import_profile_dfs, [])
new = self._template_list(t, entries)
if new:
v[self.key_import_profile_dfs] = new
def _norm_actions(self, actions):
"""
ensure each action is either pre or post explicitely
action entry of the form {action_key: (pre|post, action)}
"""
if not actions:
return actions
new = {}
for k, v in actions.items():
if k == self.action_pre or k == self.action_post:
for key, action in v.items():
new[key] = (k, action)
else:
new[k] = (self.action_post, v)
return new
def _norm_profiles(self, profiles):
"""normalize profiles entries"""
if not profiles:
return profiles
new = {}
for k, v in profiles.items():
# add dotfiles entry if not present
if self.key_profile_dotfiles not in v:
v[self.key_profile_dotfiles] = []
new[k] = v
return new
def _norm_dotfiles(self, dotfiles):
"""normalize dotfiles entries"""
if not dotfiles:
return dotfiles
new = {}
for k, v in dotfiles.items():
# add 'src' as key' if not present
if self.key_dotfile_src not in v:
v[self.key_dotfile_src] = k
new[k] = v
else:
new[k] = v
# fix deprecated trans key
if self.old_key_trans_r in v:
msg = '\"trans\" is deprecated, please use \"trans_read\"'
self.log.warn(msg)
v[self.key_trans_r] = v[self.old_key_trans_r]
del v[self.old_key_trans_r]
new[k] = v
# apply link value
if self.key_dotfile_link not in v:
val = self.settings[self.key_settings_link_dotfile_default]
v[self.key_dotfile_link] = val
# apply noempty if undefined
if self.key_dotfile_noempty not in v:
val = self.settings.get(self.key_settings_noempty, False)
v[self.key_dotfile_noempty] = val
return new
def _get_variables_dict(self, profile):
"""return enriched variables"""
variables = deepcopy(self.ori_variables)
# add profile variable
if profile:
variables['profile'] = profile
# add some more variables
p = self.settings.get(self.key_settings_dotpath)
p = self._norm_path(p)
variables['_dotdrop_dotpath'] = p
variables['_dotdrop_cfgpath'] = self._norm_path(self.path)
p = self.settings.get(self.key_settings_workdir)
p = self._norm_path(p)
variables['_dotdrop_workdir'] = p
return variables
def _get_dvariables_dict(self):
"""return dynvariables"""
variables = deepcopy(self.ori_dvariables)
return variables
def _get_included_variables(self, profile, seen):
"""return included variables"""
variables = {}
if not profile or profile not in self.profiles.keys():
return variables
# profile entry
pentry = self.profiles.get(profile)
# inherite profile variables
for inherited_profile in pentry.get(self.key_profile_include, []):
if inherited_profile == profile or inherited_profile in seen:
raise YamlException('\"include\" loop')
seen.append(inherited_profile)
new = self._get_included_variables(inherited_profile, seen)
if self.debug:
msg = 'included vars from {}: {}'
self.log.dbg(msg.format(inherited_profile, new))
variables.update(new)
cur = pentry.get(self.key_profile_variables, {})
return self._merge_dict(cur, variables)
def _get_included_dvariables(self, profile, seen):
"""return included dynvariables"""
variables = {}
if not profile or profile not in self.profiles.keys():
return variables
# profile entry
pentry = self.profiles.get(profile)
# inherite profile dynvariables
for inherited_profile in pentry.get(self.key_profile_include, []):
if inherited_profile == profile or inherited_profile in seen:
raise YamlException('\"include loop\"')
seen.append(inherited_profile)
new = self._get_included_dvariables(inherited_profile, seen)
if self.debug:
msg = 'included dvars from {}: {}'
self.log.dbg(msg.format(inherited_profile, new))
variables.update(new)
cur = pentry.get(self.key_profile_dvariables, {})
return self._merge_dict(cur, variables)
def _resolve_profile_all(self):
"""resolve some other parts of the config"""
# profile -> ALL
for k, v in self.profiles.items():
dfs = v.get(self.key_profile_dotfiles, None)
if not dfs:
continue
if self.key_all in dfs:
if self.debug:
self.log.dbg('add ALL to profile {}'.format(k))
v[self.key_profile_dotfiles] = self.dotfiles.keys()
def _resolve_profile_includes(self):
# profiles -> include other profile
for k, v in self.profiles.items():
self._rec_resolve_profile_include(k)
def _rec_resolve_profile_include(self, profile):
"""
recursively resolve include of other profiles's:
* dotfiles
* actions
"""
this_profile = self.profiles[profile]
# include
dotfiles = this_profile.get(self.key_profile_dotfiles, [])
actions = this_profile.get(self.key_profile_actions, [])
includes = this_profile.get(self.key_profile_include, None)
if not includes:
# nothing to include
return dotfiles, actions
if self.debug:
self.log.dbg('{} includes: {}'.format(profile, ','.join(includes)))
self.log.dbg('{} dotfiles before include: {}'.format(
profile, dotfiles))
self.log.dbg('{} actions before include: {}'.format(
profile, actions))
seen = []
for i in uniq_list(includes):
# ensure no include loop occurs
if i in seen:
raise YamlException('\"include loop\"')
seen.append(i)
# included profile even exists
if i not in self.profiles.keys():
self.log.warn('include unknown profile: {}'.format(i))
continue
# recursive resolve
o_dfs, o_actions = self._rec_resolve_profile_include(i)
# merge dotfile keys
dotfiles.extend(o_dfs)
this_profile[self.key_profile_dotfiles] = uniq_list(dotfiles)
# merge actions keys
actions.extend(o_actions)
this_profile[self.key_profile_actions] = uniq_list(actions)
dotfiles = this_profile.get(self.key_profile_dotfiles, [])
actions = this_profile.get(self.key_profile_actions, [])
if self.debug:
self.log.dbg('{} dotfiles after include: {}'.format(
profile, dotfiles))
self.log.dbg('{} actions after include: {}'.format(
profile, actions))
# since dotfiles and actions are resolved here
# and variables have been already done at the beginning
# of the parsing, we can clear these include
self.profiles[profile][self.key_profile_include] = None
return dotfiles, actions
########################################################
# handle imported entries
########################################################
def _import_variables(self):
"""import external variables from paths"""
paths = self.settings.get(self.key_import_variables, None)
if not paths:
return
paths = self._glob_paths(paths)
for p in paths:
path = self._norm_path(p)
if self.debug:
self.log.dbg('import variables from {}'.format(path))
var = self._import_sub(path, self.key_variables, mandatory=False)
if self.debug:
self.log.dbg('import dynvariables from {}'.format(path))
dvar = self._import_sub(path, self.key_dvariables, mandatory=False)
merged = self._merge_dict(dvar, var)
merged = self._rec_resolve_vars(merged)
# execute dvar
self._shell_exec_dvars(dvar.keys(), merged)
self._clear_profile_vars(merged)
self.variables = self._merge_dict(merged, self.variables)
def _clear_profile_vars(self, dic):
"""remove profile variables from dic if found"""
[dic.pop(k, None) for k in self.prokeys]
def _import_actions(self):
"""import external actions from paths"""
paths = self.settings.get(self.key_import_actions, None)
if not paths:
return
paths = self._glob_paths(paths)
for p in paths:
path = self._norm_path(p)
if self.debug:
self.log.dbg('import actions from {}'.format(path))
new = self._import_sub(path,
self.key_actions,
mandatory=False,
patch_func=self._norm_actions)
self.actions = self._merge_dict(new, self.actions)
def _import_profiles_dotfiles(self):
"""import profile dotfiles"""
for k, v in self.profiles.items():
imp = v.get(self.key_import_profile_dfs, None)
if not imp:
continue
if self.debug:
self.log.dbg('import dotfiles for profile {}'.format(k))
paths = self._glob_paths(imp)
for p in paths:
current = v.get(self.key_dotfiles, [])
path = self._norm_path(p)
new = self._import_sub(path,
self.key_dotfiles,
mandatory=False)
v[self.key_dotfiles] = new + current
def _import_config(self, path):
"""import config from path"""
path = self._norm_path(path)
if self.debug:
self.log.dbg('import config from {}'.format(path))
sub = CfgYaml(path, profile=self.profile, debug=self.debug)
# settings is ignored
self.dotfiles = self._merge_dict(self.dotfiles, sub.dotfiles)
self.profiles = self._merge_dict(self.profiles, sub.profiles)
self.actions = self._merge_dict(self.actions, sub.actions)
self.trans_r = self._merge_dict(self.trans_r, sub.trans_r)
self.trans_w = self._merge_dict(self.trans_w, sub.trans_w)
self._clear_profile_vars(sub.variables)
if self.debug:
self.log.dbg('add import_configs var: {}'.format(sub.variables))
self.variables = self._merge_dict(sub.variables, self.variables)
def _import_configs(self):
"""import configs from external files"""
# settings -> import_configs
imp = self.settings.get(self.key_import_configs, None)
if not imp:
return
paths = self._glob_paths(imp)
for path in paths:
self._import_config(path)
def _import_sub(self, path, key, mandatory=False, patch_func=None):
"""
import the block "key" from "path"
patch_func is applied to each element if defined
"""
if self.debug:
self.log.dbg('import \"{}\" from \"{}\"'.format(key, path))
extdict = self._load_yaml(path)
new = self._get_entry(extdict, key, mandatory=mandatory)
if patch_func:
if self.debug:
self.log.dbg('calling patch: {}'.format(patch_func))
new = patch_func(new)
if not new and mandatory:
err = 'no \"{}\" imported from \"{}\"'.format(key, path)
self.log.warn(err)
raise YamlException(err)
if self.debug:
self.log.dbg('imported \"{}\": {}'.format(key, new))
return new
########################################################
# add/remove entries
########################################################
def _new_profile(self, key):
"""add a new profile if it doesn't exist"""
if key not in self.profiles.keys():
# update yaml_dict
self.yaml_dict[self.key_profiles][key] = {
self.key_profile_dotfiles: []
}
if self.debug:
self.log.dbg('adding new profile: {}'.format(key))
self.dirty = True
def add_dotfile_to_profile(self, dotfile_key, profile_key):
"""add an existing dotfile key to a profile_key"""
self._new_profile(profile_key)
profile = self.yaml_dict[self.key_profiles][profile_key]
if dotfile_key not in profile[self.key_profile_dotfiles]:
profile[self.key_profile_dotfiles].append(dotfile_key)
if self.debug:
msg = 'add \"{}\" to profile \"{}\"'.format(
dotfile_key, profile_key)
msg.format(dotfile_key, profile_key)
self.log.dbg(msg)
self.dirty = True
return self.dirty
def add_dotfile(self, key, src, dst, link):
"""add a new dotfile"""
if key in self.dotfiles.keys():
return False
if self.debug:
self.log.dbg('adding new dotfile: {}'.format(key))
df_dict = {
self.key_dotfile_src: src,
self.key_dotfile_dst: dst,
}
dfl = self.settings[self.key_settings_link_dotfile_default]
if str(link) != dfl:
df_dict[self.key_dotfile_link] = str(link)
self.yaml_dict[self.key_dotfiles][key] = df_dict
self.dirty = True
def del_dotfile(self, key):
"""remove this dotfile from config"""
if key not in self.yaml_dict[self.key_dotfiles]:
self.log.err('key not in dotfiles: {}'.format(key))
return False
if self.debug:
self.log.dbg('remove dotfile: {}'.format(key))
del self.yaml_dict[self.key_dotfiles][key]
if self.debug:
dfs = self.yaml_dict[self.key_dotfiles]
self.log.dbg('new dotfiles: {}'.format(dfs))
self.dirty = True
return True
def del_dotfile_from_profile(self, df_key, pro_key):
"""remove this dotfile from that profile"""
if df_key not in self.dotfiles.keys():
self.log.err('key not in dotfiles: {}'.format(df_key))
return False
if pro_key not in self.profiles.keys():
self.log.err('key not in profile: {}'.format(pro_key))
return False
# get the profile dictionary
profile = self.yaml_dict[self.key_profiles][pro_key]
if df_key not in profile[self.key_profile_dotfiles]:
return True
if self.debug:
dfs = profile[self.key_profile_dotfiles]
self.log.dbg('{} profile dotfiles: {}'.format(pro_key, dfs))
self.log.dbg('remove {} from profile {}'.format(df_key, pro_key))
profile[self.key_profile_dotfiles].remove(df_key)
if self.debug:
dfs = profile[self.key_profile_dotfiles]
self.log.dbg('{} profile dotfiles: {}'.format(pro_key, dfs))
self.dirty = True
return True
########################################################
# handle deprecated entries
########################################################
def _fix_deprecated(self, yamldict):
"""fix deprecated entries"""
self._fix_deprecated_link_by_default(yamldict)
self._fix_deprecated_dotfile_link(yamldict)
def _fix_deprecated_link_by_default(self, yamldict):
"""fix deprecated link_by_default"""
key = 'link_by_default'
newkey = self.key_imp_link
if self.key_settings not in yamldict:
return
if not yamldict[self.key_settings]:
return
config = yamldict[self.key_settings]
if key not in config:
return
if config[key]:
config[newkey] = self.lnk_link
else:
config[newkey] = self.lnk_nolink
del config[key]
self.log.warn('deprecated \"link_by_default\"')
self.dirty = True
self.dirty_deprecated = True
def _fix_deprecated_dotfile_link(self, yamldict):
"""fix deprecated link in dotfiles"""
if self.key_dotfiles not in yamldict:
return
if not yamldict[self.key_dotfiles]:
return
for k, dotfile in yamldict[self.key_dotfiles].items():
new = self.lnk_nolink
if self.key_dotfile_link in dotfile and \
type(dotfile[self.key_dotfile_link]) is bool:
# patch link: <bool>
cur = dotfile[self.key_dotfile_link]
new = self.lnk_nolink
if cur:
new = self.lnk_link
dotfile[self.key_dotfile_link] = new
self.dirty = True
self.dirty_deprecated = True
self.log.warn('deprecated \"link\" value')
elif self.key_dotfile_link_children in dotfile and \
type(dotfile[self.key_dotfile_link_children]) is bool:
# patch link_children: <bool>
cur = dotfile[self.key_dotfile_link_children]
new = self.lnk_nolink
if cur:
new = self.lnk_children
del dotfile[self.key_dotfile_link_children]
dotfile[self.key_dotfile_link] = new
self.dirty = True
self.dirty_deprecated = True
self.log.warn('deprecated \"link_children\" value')
########################################################
# yaml utils
########################################################
def save(self):
"""save this instance and return True if saved"""
if not self.dirty:
return False
content = self._clear_none(self.dump())
# make sure we have the base entries
if self.key_settings not in content:
content[self.key_settings] = None
if self.key_dotfiles not in content:
content[self.key_dotfiles] = None
if self.key_profiles not in content:
content[self.key_profiles] = None
if self.dirty_deprecated:
# add minversion
settings = content[self.key_settings]
settings[self.key_settings_minversion] = VERSION
# save to file
if self.debug:
self.log.dbg('saving to {}'.format(self.path))
try:
self._yaml_dump(content, self.path)
except Exception as e:
self.log.err(e)
raise YamlException('error saving config: {}'.format(self.path))
if self.dirty_deprecated:
warn = 'your config contained deprecated entries'
warn += ' and was updated'
self.log.warn(warn)
self.dirty = False
self.cfg_updated = False
return True
def dump(self):
"""dump the config dictionary"""
return self.yaml_dict
def _load_yaml(self, path):
"""load a yaml file to a dict"""
content = {}
if not os.path.exists(path):
raise YamlException('config path not found: {}'.format(path))
try:
content = self._yaml_load(path)
except Exception as e:
self.log.err(e)
raise YamlException('invalid config: {}'.format(path))
return content
def _yaml_load(self, path):
"""load from yaml"""
with open(path, 'r') as f:
y = yaml()
y.typ = 'rt'
content = y.load(f)
return content
def _yaml_dump(self, content, path):
"""dump to yaml"""
with open(self.path, 'w') as f:
y = yaml()
y.default_flow_style = False
y.indent = 2
y.typ = 'rt'
y.dump(content, f)
########################################################
# helpers
########################################################
def _merge_dict(self, high, low):
"""merge high and low dict"""
if not high:
high = {}
if not low:
low = {}
return {**low, **high}
def _get_entry(self, dic, key, mandatory=True):
"""return entry from yaml dictionary"""
if key not in dic:
if mandatory:
raise YamlException('invalid config: no {} found'.format(key))
dic[key] = {}
return dic[key]
if mandatory and not dic[key]:
# ensure is not none
dic[key] = {}
return dic[key]
def _clear_none(self, dic):
"""recursively delete all none/empty values in a dictionary."""
new = {}
for k, v in dic.items():
newv = v
if isinstance(v, dict):
newv = self._clear_none(v)
if not newv:
# no empty dict
continue
if newv is None:
# no None value
continue
if isinstance(newv, list) and not newv:
# no empty list
continue
new[k] = newv
return new
def _is_glob(self, path):
"""quick test if path is a glob"""
return '*' in path or '?' in path
def _glob_paths(self, paths):
"""glob a list of paths"""
if not isinstance(paths, list):
paths = [paths]
res = []
for p in paths:
if not self._is_glob(p):
res.append(p)
continue
p = os.path.expanduser(p)
new = glob.glob(p)
if not new:
raise YamlException('bad path: {}'.format(p))
res.extend(glob.glob(p))
return res
def _debug_vars(self, variables):
"""pretty print variables"""
if not self.debug:
return
self.log.dbg('variables:')
for k, v in variables.items():
self.log.dbg('\t\"{}\": {}'.format(k, v))
def _norm_path(self, path):
"""resolve a path either absolute or relative to config path"""
path = os.path.expanduser(path)
if not os.path.isabs(path):
d = os.path.dirname(self.path)
return os.path.join(d, path)
return os.path.normpath(path)
def _shell_exec_dvars(self, keys, variables):
"""shell execute dynvariables"""
for k in list(keys):
ret, out = shell(variables[k], debug=self.debug)
if not ret:
err = 'var \"{}: {}\" failed: {}'.format(k, variables[k], out)
self.log.err(err)
raise YamlException(err)
if self.debug:
self.log.dbg('\"{}\": {} -> {}'.format(k, variables[k], out))
variables[k] = out
def _template_list(self, t, entries):
"""template a list of entries"""
new = []
if not entries:
return new
for e in entries:
et = t.generate_string(e)
if self.debug and e != et:
self.log.dbg('resolved: {} -> {}'.format(e, et))
new.append(et)
return new
def _check_minversion(self, minversion):
if not minversion:
return
try:
cur = tuple([int(x) for x in VERSION.split('.')])
cfg = tuple([int(x) for x in minversion.split('.')])
except Exception:
err = 'bad version: \"{}\" VS \"{}\"'.format(VERSION, minversion)
raise YamlException(err)
if cur < cfg:
err = 'current dotdrop version is too old for that config file.'
err += ' Please update.'
raise YamlException(err)
class CfgYaml:
# global entries
key_settings = Settings.key_yaml
key_dotfiles = 'dotfiles'
key_profiles = 'profiles'
key_actions = 'actions'
old_key_trans_r = 'trans'
key_trans_r = 'trans_read'
key_trans_w = 'trans_write'
key_variables = 'variables'
key_dvariables = 'dynvariables'
action_pre = 'pre'
action_post = 'post'
# profiles/dotfiles entries
key_dotfile_src = 'src'
key_dotfile_dst = 'dst'
key_dotfile_link = 'link'
key_dotfile_actions = 'actions'
key_dotfile_link_children = 'link_children'
key_dotfile_noempty = 'ignoreempty'
# profile
key_profile_dotfiles = 'dotfiles'
key_profile_include = 'include'
key_profile_variables = 'variables'
key_profile_dvariables = 'dynvariables'
key_profile_actions = 'actions'
key_all = 'ALL'
# import entries
key_import_actions = 'import_actions'
key_import_configs = 'import_configs'
key_import_variables = 'import_variables'
key_import_profile_dfs = 'import'
key_import_sep = ':'
key_import_ignore_key = 'optional'
key_import_fatal_not_found = True
# settings
key_settings_dotpath = Settings.key_dotpath
key_settings_workdir = Settings.key_workdir
key_settings_link_dotfile_default = Settings.key_link_dotfile_default
key_settings_noempty = Settings.key_ignoreempty
key_settings_minversion = Settings.key_minversion
key_imp_link = Settings.key_link_on_import
# link values
lnk_nolink = LinkTypes.NOLINK.name.lower()
lnk_link = LinkTypes.LINK.name.lower()
lnk_children = LinkTypes.LINK_CHILDREN.name.lower()
def __init__(self, path, profile=None, addprofiles=[], debug=False):
"""
config parser
@path: config file path
@profile: the selected profile
@addprofiles: included profiles
@debug: debug flag
"""
self._path = os.path.abspath(path)
self._profile = profile
self._debug = debug
self._log = Logger()
# config needs to be written
self._dirty = False
# indicates the config has been updated
self._dirty_deprecated = False
# profile variables
self._profilevarskeys = []
# included profiles
self._inc_profiles = addprofiles
# init the dictionaries
self.settings = {}
self.dotfiles = {}
self.profiles = {}
self.actions = {}
self.trans_r = {}
self.trans_w = {}
self.variables = {}
if not os.path.exists(self._path):
err = 'invalid config path: \"{}\"'.format(path)
if self._debug:
self._dbg(err)
raise YamlException(err)
self._yaml_dict = self._load_yaml(self._path)
# live patch deprecated entries
self._fix_deprecated(self._yaml_dict)
##################################################
# parse the config and variables
##################################################
# parse the "config" block
self.settings = self._parse_blk_settings(self._yaml_dict)
# base templater (when no vars/dvars exist)
self.variables = self._enrich_vars(self.variables, self._profile)
self._redefine_templater()
# variables and dynvariables need to be first merged
# before being templated in order to allow cyclic
# references between them
# parse the "variables" block
var = self._parse_blk_variables(self._yaml_dict)
self._add_variables(var, template=False)
# parse the "dynvariables" block
dvariables = self._parse_blk_dynvariables(self._yaml_dict)
self._add_variables(dvariables, template=False)
# now template variables and dynvariables from the same pool
self._rec_resolve_variables(self.variables)
# and execute dvariables
# since this is done after recursively resolving variables
# and dynvariables this means that variables referencing
# dynvariables will result with the not executed value
if dvariables.keys():
self._shell_exec_dvars(self.variables, keys=dvariables.keys())
# finally redefine the template
self._redefine_templater()
if self._debug:
self._debug_dict('current variables defined', self.variables)
# parse the "profiles" block
self.profiles = self._parse_blk_profiles(self._yaml_dict)
# include the profile's variables/dynvariables last
# as it overwrites existing ones
self._inc_profiles, pv, pvd = self._get_profile_included_vars()
self._add_variables(pv, prio=True)
self._add_variables(pvd, shell=True, prio=True)
self._profilevarskeys.extend(pv.keys())
self._profilevarskeys.extend(pvd.keys())
# template variables
self.variables = self._template_dict(self.variables)
if self._debug:
self._debug_dict('current variables defined', self.variables)
##################################################
# template the "include" entries
##################################################
self._template_include_entry()
if self._debug:
self._debug_dict('current variables defined', self.variables)
##################################################
# parse the other blocks
##################################################
# parse the "dotfiles" block
self.dotfiles = self._parse_blk_dotfiles(self._yaml_dict)
# parse the "actions" block
self.actions = self._parse_blk_actions(self._yaml_dict)
# parse the "trans_r" block
self.trans_r = self._parse_blk_trans_r(self._yaml_dict)
# parse the "trans_w" block
self.trans_w = self._parse_blk_trans_w(self._yaml_dict)
##################################################
# import elements
##################################################
# process imported variables (import_variables)
newvars = self._import_variables()
self._clear_profile_vars(newvars)
self._add_variables(newvars)
# process imported actions (import_actions)
self._import_actions()
# process imported profile dotfiles (import)
self._import_profiles_dotfiles()
# process imported configs (import_configs)
self._import_configs()
# process profile include
self._resolve_profile_includes()
# add the current profile variables
_, pv, pvd = self._get_profile_included_vars()
self._add_variables(pv, prio=True)
self._add_variables(pvd, shell=True, prio=True)
self._profilevarskeys.extend(pv.keys())
self._profilevarskeys.extend(pvd.keys())
# resolve variables
self._clear_profile_vars(newvars)
self._add_variables(newvars)
# process profile ALL
self._resolve_profile_all()
# patch dotfiles paths
self._template_dotfiles_paths()
if self._debug:
self._dbg('########### {} ###########'.format('final config'))
self._debug_entries()
########################################################
# outside available methods
########################################################
def resolve_dotfile_src(self, src, templater=None):
"""resolve dotfile src path"""
newsrc = ''
if src:
new = src
if templater:
new = templater.generate_string(src)
if new != src and self._debug:
msg = 'dotfile src: \"{}\" -> \"{}\"'.format(src, new)
self._dbg(msg)
src = new
src = os.path.join(self.settings[self.key_settings_dotpath], src)
newsrc = self._norm_path(src)
return newsrc
def resolve_dotfile_dst(self, dst, templater=None):
"""resolve dotfile dst path"""
newdst = ''
if dst:
new = dst
if templater:
new = templater.generate_string(dst)
if new != dst and self._debug:
msg = 'dotfile dst: \"{}\" -> \"{}\"'.format(dst, new)
self._dbg(msg)
dst = new
newdst = self._norm_path(dst)
return newdst
def add_dotfile_to_profile(self, dotfile_key, profile_key):
"""add an existing dotfile key to a profile_key"""
self._new_profile(profile_key)
profile = self._yaml_dict[self.key_profiles][profile_key]
if self.key_profile_dotfiles not in profile or \
profile[self.key_profile_dotfiles] is None:
profile[self.key_profile_dotfiles] = []
pdfs = profile[self.key_profile_dotfiles]
if self.key_all not in pdfs and \
dotfile_key not in pdfs:
profile[self.key_profile_dotfiles].append(dotfile_key)
if self._debug:
msg = 'add \"{}\" to profile \"{}\"'.format(
dotfile_key, profile_key)
msg.format(dotfile_key, profile_key)
self._dbg(msg)
self._dirty = True
return self._dirty
def get_all_dotfile_keys(self):
"""return all existing dotfile keys"""
return self.dotfiles.keys()
def add_dotfile(self, key, src, dst, link):
"""add a new dotfile"""
if key in self.dotfiles.keys():
return False
if self._debug:
self._dbg('adding new dotfile: {}'.format(key))
self._dbg('new dotfile src: {}'.format(src))
self._dbg('new dotfile dst: {}'.format(dst))
df_dict = {
self.key_dotfile_src: src,
self.key_dotfile_dst: dst,
}
dfl = self.settings[self.key_settings_link_dotfile_default]
if str(link) != dfl:
df_dict[self.key_dotfile_link] = str(link)
self._yaml_dict[self.key_dotfiles][key] = df_dict
self._dirty = True
def del_dotfile(self, key):
"""remove this dotfile from config"""
if key not in self._yaml_dict[self.key_dotfiles]:
self._log.err('key not in dotfiles: {}'.format(key))
return False
if self._debug:
self._dbg('remove dotfile: {}'.format(key))
del self._yaml_dict[self.key_dotfiles][key]
if self._debug:
dfs = self._yaml_dict[self.key_dotfiles]
self._dbg('new dotfiles: {}'.format(dfs))
self._dirty = True
return True
def del_dotfile_from_profile(self, df_key, pro_key):
"""remove this dotfile from that profile"""
if df_key not in self.dotfiles.keys():
self._log.err('key not in dotfiles: {}'.format(df_key))
return False
if pro_key not in self.profiles.keys():
self._log.err('key not in profile: {}'.format(pro_key))
return False
# get the profile dictionary
profile = self._yaml_dict[self.key_profiles][pro_key]
if df_key not in profile[self.key_profile_dotfiles]:
return True
if self._debug:
dfs = profile[self.key_profile_dotfiles]
self._dbg('{} profile dotfiles: {}'.format(pro_key, dfs))
self._dbg('remove {} from profile {}'.format(df_key, pro_key))
profile[self.key_profile_dotfiles].remove(df_key)
if self._debug:
dfs = profile[self.key_profile_dotfiles]
self._dbg('{} profile dotfiles: {}'.format(pro_key, dfs))
self._dirty = True
return True
def save(self):
"""save this instance and return True if saved"""
if not self._dirty:
return False
content = self._prepare_to_save(self._yaml_dict)
if self._dirty_deprecated:
# add minversion
settings = content[self.key_settings]
settings[self.key_settings_minversion] = VERSION
# save to file
if self._debug:
self._dbg('saving to {}'.format(self._path))
try:
with open(self._path, 'w') as f:
self._yaml_dump(content, f)
except Exception as e:
self._log.err(e)
raise YamlException('error saving config: {}'.format(self._path))
if self._dirty_deprecated:
warn = 'your config contained deprecated entries'
warn += ' and was updated'
self._log.warn(warn)
self._dirty = False
self.cfg_updated = False
return True
def dump(self):
"""dump the config dictionary"""
output = io.StringIO()
content = self._prepare_to_save(self._yaml_dict.copy())
self._yaml_dump(content, output)
return output.getvalue()
########################################################
# block parsing
########################################################
def _parse_blk_settings(self, dic):
"""parse the "config" block"""
block = self._get_entry(dic, self.key_settings).copy()
# set defaults
settings = Settings(None).serialize().get(self.key_settings)
settings.update(block)
# resolve minimum version
if self.key_settings_minversion in settings:
minversion = settings[self.key_settings_minversion]
self._check_minversion(minversion)
# normalize paths
p = self._norm_path(settings[self.key_settings_dotpath])
settings[self.key_settings_dotpath] = p
p = self._norm_path(settings[self.key_settings_workdir])
settings[self.key_settings_workdir] = p
p = [self._norm_path(p) for p in settings[Settings.key_filter_file]]
settings[Settings.key_filter_file] = p
p = [self._norm_path(p) for p in settings[Settings.key_func_file]]
settings[Settings.key_func_file] = p
if self._debug:
self._debug_dict('settings block:', settings)
return settings
def _parse_blk_dotfiles(self, dic):
"""parse the "dotfiles" block"""
dotfiles = self._get_entry(dic, self.key_dotfiles).copy()
keys = dotfiles.keys()
if len(keys) != len(list(set(keys))):
dups = [x for x in keys if x not in list(set(keys))]
err = 'duplicate dotfile keys found: {}'.format(dups)
raise YamlException(err)
dotfiles = self._norm_dotfiles(dotfiles)
if self._debug:
self._debug_dict('dotfiles block', dotfiles)
return dotfiles
def _parse_blk_profiles(self, dic):
"""parse the "profiles" block"""
profiles = self._get_entry(dic, self.key_profiles).copy()
profiles = self._norm_profiles(profiles)
if self._debug:
self._debug_dict('profiles block', profiles)
return profiles
def _parse_blk_actions(self, dic):
"""parse the "actions" block"""
actions = self._get_entry(dic, self.key_actions, mandatory=False)
if actions:
actions = actions.copy()
actions = self._norm_actions(actions)
if self._debug:
self._debug_dict('actions block', actions)
return actions
def _parse_blk_trans_r(self, dic):
"""parse the "trans_r" block"""
key = self.key_trans_r
if self.old_key_trans_r in dic:
msg = '\"trans\" is deprecated, please use \"trans_read\"'
self._log.warn(msg)
dic[self.key_trans_r] = dic[self.old_key_trans_r]
del dic[self.old_key_trans_r]
trans_r = self._get_entry(dic, key, mandatory=False)
if trans_r:
trans_r = trans_r.copy()
if self._debug:
self._debug_dict('trans_r block', trans_r)
return trans_r
def _parse_blk_trans_w(self, dic):
"""parse the "trans_w" block"""
trans_w = self._get_entry(dic, self.key_trans_w, mandatory=False)
if trans_w:
trans_w = trans_w.copy()
if self._debug:
self._debug_dict('trans_w block', trans_w)
return trans_w
def _parse_blk_variables(self, dic):
"""parse the "variables" block"""
variables = self._get_entry(dic, self.key_variables, mandatory=False)
if variables:
variables = variables.copy()
if self._debug:
self._debug_dict('variables block', variables)
return variables
def _parse_blk_dynvariables(self, dic):
"""parse the "dynvariables" block"""
dvariables = self._get_entry(dic, self.key_dvariables, mandatory=False)
if dvariables:
dvariables = dvariables.copy()
if self._debug:
self._debug_dict('dynvariables block', dvariables)
return dvariables
########################################################
# parsing helpers
########################################################
def _template_include_entry(self):
"""template all "include" entries"""
# import_actions
new = []
entries = self.settings.get(self.key_import_actions, [])
new = self._template_list(entries)
if new:
self.settings[self.key_import_actions] = new
# import_configs
entries = self.settings.get(self.key_import_configs, [])
new = self._template_list(entries)
if new:
self.settings[self.key_import_configs] = new
# import_variables
entries = self.settings.get(self.key_import_variables, [])
new = self._template_list(entries)
if new:
self.settings[self.key_import_variables] = new
# profile's import
for k, v in self.profiles.items():
entries = v.get(self.key_import_profile_dfs, [])
new = self._template_list(entries)
if new:
v[self.key_import_profile_dfs] = new
def _norm_actions(self, actions):
"""
ensure each action is either pre or post explicitely
action entry of the form {action_key: (pre|post, action)}
"""
if not actions:
return actions
new = {}
for k, v in actions.items():
if k == self.action_pre or k == self.action_post:
for key, action in v.items():
new[key] = (k, action)
else:
new[k] = (self.action_post, v)
return new
def _norm_profiles(self, profiles):
"""normalize profiles entries"""
if not profiles:
return profiles
new = {}
for k, v in profiles.items():
if not v:
# no dotfiles
continue
# add dotfiles entry if not present
if self.key_profile_dotfiles not in v:
v[self.key_profile_dotfiles] = []
new[k] = v
return new
def _norm_dotfiles(self, dotfiles):
"""normalize dotfiles entries"""
if not dotfiles:
return dotfiles
new = {}
for k, v in dotfiles.items():
# add 'src' as key' if not present
if self.key_dotfile_src not in v:
v[self.key_dotfile_src] = k
new[k] = v
else:
new[k] = v
# fix deprecated trans key
if self.old_key_trans_r in v:
msg = '\"trans\" is deprecated, please use \"trans_read\"'
self._log.warn(msg)
v[self.key_trans_r] = v[self.old_key_trans_r]
del v[self.old_key_trans_r]
new[k] = v
# apply link value
if self.key_dotfile_link not in v:
val = self.settings[self.key_settings_link_dotfile_default]
v[self.key_dotfile_link] = val
# apply noempty if undefined
if self.key_dotfile_noempty not in v:
val = self.settings.get(self.key_settings_noempty, False)
v[self.key_dotfile_noempty] = val
return new
def _add_variables(self, new, shell=False, template=True, prio=False):
"""
add new variables
@shell: execute the variable through the shell
@template: template the variable
@prio: new takes priority over existing variables
"""
if not new:
return
# merge
if prio:
self.variables = self._merge_dict(new, self.variables)
else:
self.variables = self._merge_dict(self.variables, new)
# ensure enriched variables are relative to this config
self.variables = self._enrich_vars(self.variables, self._profile)
# re-create the templater
self._redefine_templater()
if template:
# rec resolve variables with new ones
self._rec_resolve_variables(self.variables)
if shell:
# shell exec
self._shell_exec_dvars(self.variables, keys=new.keys())
# re-create the templater
self._redefine_templater()
def _enrich_vars(self, variables, profile):
"""return enriched variables"""
# add profile variable
if profile:
variables['profile'] = profile
# add some more variables
p = self.settings.get(self.key_settings_dotpath)
p = self._norm_path(p)
variables['_dotdrop_dotpath'] = p
variables['_dotdrop_cfgpath'] = self._norm_path(self._path)
p = self.settings.get(self.key_settings_workdir)
p = self._norm_path(p)
variables['_dotdrop_workdir'] = p
return variables
def _get_profile_included_item(self, keyitem):
"""recursively get included <keyitem> in profile"""
profiles = [self._profile] + self._inc_profiles
items = {}
for profile in profiles:
seen = [self._profile]
i = self.__get_profile_included_item(profile, keyitem, seen)
items = self._merge_dict(i, items)
return items
def __get_profile_included_item(self, profile, keyitem, seen):
"""recursively get included <keyitem> from profile"""
items = {}
if not profile or profile not in self.profiles.keys():
return items
# considered profile entry
pentry = self.profiles.get(profile)
# recursively get <keyitem> from inherited profile
for inherited_profile in pentry.get(self.key_profile_include, []):
if inherited_profile == profile or inherited_profile in seen:
raise YamlException('\"include\" loop')
seen.append(inherited_profile)
new = self.__get_profile_included_item(inherited_profile, keyitem,
seen)
if self._debug:
msg = 'included {} from {}: {}'
self._dbg(msg.format(keyitem, inherited_profile, new))
items.update(new)
cur = pentry.get(keyitem, {})
return self._merge_dict(cur, items)
def _resolve_profile_all(self):
"""resolve some other parts of the config"""
# profile -> ALL
for k, v in self.profiles.items():
dfs = v.get(self.key_profile_dotfiles, None)
if not dfs:
continue
if self.key_all in dfs:
if self._debug:
self._dbg('add ALL to profile \"{}\"'.format(k))
v[self.key_profile_dotfiles] = self.dotfiles.keys()
def _resolve_profile_includes(self):
"""resolve profile(s) including other profiles"""
for k, v in self.profiles.items():
self._rec_resolve_profile_include(k)
def _rec_resolve_profile_include(self, profile):
"""
recursively resolve include of other profiles's:
* dotfiles
* actions
returns dotfiles, actions
"""
this_profile = self.profiles[profile]
# considered profile content
dotfiles = this_profile.get(self.key_profile_dotfiles, []) or []
actions = this_profile.get(self.key_profile_actions, []) or []
includes = this_profile.get(self.key_profile_include, []) or []
if not includes:
# nothing to include
return dotfiles, actions
if self._debug:
self._dbg('{} includes {}'.format(profile, ','.join(includes)))
self._dbg('{} dotfiles before include: {}'.format(
profile, dotfiles))
self._dbg('{} actions before include: {}'.format(profile, actions))
seen = []
for i in uniq_list(includes):
if self._debug:
self._dbg('resolving includes "{}" <- "{}"'.format(profile, i))
# ensure no include loop occurs
if i in seen:
raise YamlException('\"include loop\"')
seen.append(i)
# included profile even exists
if i not in self.profiles.keys():
self._log.warn('include unknown profile: {}'.format(i))
continue
# recursive resolve
if self._debug:
self._dbg(
'recursively resolving includes for profile "{}"'.format(
i))
o_dfs, o_actions = self._rec_resolve_profile_include(i)
# merge dotfile keys
if self._debug:
self._dbg('Merging dotfiles {} <- {}: {} <- {}'.format(
profile, i, dotfiles, o_dfs))
dotfiles.extend(o_dfs)
this_profile[self.key_profile_dotfiles] = uniq_list(dotfiles)
# merge actions keys
if self._debug:
self._dbg('Merging actions {} <- {}: {} <- {}'.format(
profile, i, actions, o_actions))
actions.extend(o_actions)
this_profile[self.key_profile_actions] = uniq_list(actions)
dotfiles = this_profile.get(self.key_profile_dotfiles, [])
actions = this_profile.get(self.key_profile_actions, [])
if self._debug:
self._dbg('{} dotfiles after include: {}'.format(
profile, dotfiles))
self._dbg('{} actions after include: {}'.format(profile, actions))
# since included items are resolved here
# we can clear these include
self.profiles[profile][self.key_profile_include] = []
return dotfiles, actions
########################################################
# handle imported entries
########################################################
def _import_variables(self):
"""import external variables from paths"""
paths = self.settings.get(self.key_import_variables, None)
if not paths:
return
paths = self._resolve_paths(paths)
newvars = {}
for path in paths:
if self._debug:
self._dbg('import variables from {}'.format(path))
var = self._import_sub(path, self.key_variables, mandatory=False)
if self._debug:
self._dbg('import dynvariables from {}'.format(path))
dvar = self._import_sub(path, self.key_dvariables, mandatory=False)
merged = self._merge_dict(dvar, var)
self._rec_resolve_variables(merged)
if dvar.keys():
self._shell_exec_dvars(merged, keys=dvar.keys())
self._clear_profile_vars(merged)
newvars = self._merge_dict(newvars, merged)
if self._debug:
self._debug_dict('imported variables', newvars)
return newvars
def _import_actions(self):
"""import external actions from paths"""
paths = self.settings.get(self.key_import_actions, None)
if not paths:
return
paths = self._resolve_paths(paths)
for path in paths:
if self._debug:
self._dbg('import actions from {}'.format(path))
new = self._import_sub(path,
self.key_actions,
mandatory=False,
patch_func=self._norm_actions)
self.actions = self._merge_dict(new, self.actions)
def _import_profiles_dotfiles(self):
"""import profile dotfiles"""
for k, v in self.profiles.items():
imp = v.get(self.key_import_profile_dfs, None)
if not imp:
continue
if self._debug:
self._dbg('import dotfiles for profile {}'.format(k))
paths = self._resolve_paths(imp)
for path in paths:
current = v.get(self.key_dotfiles, [])
new = self._import_sub(path,
self.key_dotfiles,
mandatory=False)
v[self.key_dotfiles] = new + current
def _import_config(self, path):
"""import config from path"""
if self._debug:
self._dbg('import config from {}'.format(path))
sub = CfgYaml(path,
profile=self._profile,
addprofiles=self._inc_profiles,
debug=self._debug)
# settings are ignored from external file
# except for filter_file and func_file
self.settings[Settings.key_func_file] += [
self._norm_path(func_file)
for func_file in sub.settings[Settings.key_func_file]
]
self.settings[Settings.key_filter_file] += [
self._norm_path(func_file)
for func_file in sub.settings[Settings.key_filter_file]
]
# merge top entries
self.dotfiles = self._merge_dict(self.dotfiles, sub.dotfiles)
self.profiles = self._merge_dict(self.profiles, sub.profiles)
self.actions = self._merge_dict(self.actions, sub.actions)
self.trans_r = self._merge_dict(self.trans_r, sub.trans_r)
self.trans_w = self._merge_dict(self.trans_w, sub.trans_w)
self._clear_profile_vars(sub.variables)
if self._debug:
self._debug_dict('add import_configs var', sub.variables)
self._add_variables(sub.variables, prio=True)
def _import_configs(self):
"""import configs from external files"""
# settings -> import_configs
imp = self.settings.get(self.key_import_configs, None)
if not imp:
return
paths = self._resolve_paths(imp)
for path in paths:
self._import_config(path)
def _import_sub(self, path, key, mandatory=False, patch_func=None):
"""
import the block "key" from "path"
patch_func is applied to each element if defined
"""
if self._debug:
self._dbg('import \"{}\" from \"{}\"'.format(key, path))
extdict = self._load_yaml(path)
new = self._get_entry(extdict, key, mandatory=mandatory)
if patch_func:
if self._debug:
self._dbg('calling patch: {}'.format(patch_func))
new = patch_func(new)
if not new and mandatory:
err = 'no \"{}\" imported from \"{}\"'.format(key, path)
self._log.warn(err)
raise YamlException(err)
if self._debug:
self._dbg('imported \"{}\": {}'.format(key, new))
return new
########################################################
# add/remove entries
########################################################
def _new_profile(self, key):
"""add a new profile if it doesn't exist"""
if key not in self.profiles.keys():
# update yaml_dict
self._yaml_dict[self.key_profiles][key] = {
self.key_profile_dotfiles: []
}
if self._debug:
self._dbg('adding new profile: {}'.format(key))
self._dirty = True
########################################################
# handle deprecated entries
########################################################
def _fix_deprecated(self, yamldict):
"""fix deprecated entries"""
if not yamldict:
return
self._fix_deprecated_link_by_default(yamldict)
self._fix_deprecated_dotfile_link(yamldict)
return yamldict
def _fix_deprecated_link_by_default(self, yamldict):
"""fix deprecated link_by_default"""
key = 'link_by_default'
newkey = self.key_imp_link
if self.key_settings not in yamldict:
return
if not yamldict[self.key_settings]:
return
config = yamldict[self.key_settings]
if key not in config:
return
if config[key]:
config[newkey] = self.lnk_link
else:
config[newkey] = self.lnk_nolink
del config[key]
self._log.warn('deprecated \"link_by_default\"')
self._dirty = True
self._dirty_deprecated = True
def _fix_deprecated_dotfile_link(self, yamldict):
"""fix deprecated link in dotfiles"""
if self.key_dotfiles not in yamldict:
return
if not yamldict[self.key_dotfiles]:
return
for k, dotfile in yamldict[self.key_dotfiles].items():
new = self.lnk_nolink
if self.key_dotfile_link in dotfile and \
type(dotfile[self.key_dotfile_link]) is bool:
# patch link: <bool>
cur = dotfile[self.key_dotfile_link]
new = self.lnk_nolink
if cur:
new = self.lnk_link
dotfile[self.key_dotfile_link] = new
self._dirty = True
self._dirty_deprecated = True
self._log.warn('deprecated \"link\" value')
elif self.key_dotfile_link_children in dotfile and \
type(dotfile[self.key_dotfile_link_children]) is bool:
# patch link_children: <bool>
cur = dotfile[self.key_dotfile_link_children]
new = self.lnk_nolink
if cur:
new = self.lnk_children
del dotfile[self.key_dotfile_link_children]
dotfile[self.key_dotfile_link] = new
self._dirty = True
self._dirty_deprecated = True
self._log.warn('deprecated \"link_children\" value')
########################################################
# yaml utils
########################################################
def _prepare_to_save(self, content):
content = self._clear_none(content)
# make sure we have the base entries
if self.key_settings not in content:
content[self.key_settings] = None
if self.key_dotfiles not in content:
content[self.key_dotfiles] = None
if self.key_profiles not in content:
content[self.key_profiles] = None
return content
def _load_yaml(self, path):
"""load a yaml file to a dict"""
content = {}
if self._debug:
self._dbg('----------start:{}----------'.format(path))
cfg = '\n'
with open(path, 'r') as f:
for line in f:
cfg += line
self._dbg(cfg.rstrip())
self._dbg('----------end:{}----------'.format(path))
try:
content = self._yaml_load(path)
except Exception as e:
self._log.err(e)
raise YamlException('invalid config: {}'.format(path))
return content
def _yaml_load(self, path):
"""load from yaml"""
with open(path, 'r') as f:
y = yaml()
y.typ = 'rt'
content = y.load(f)
return content
def _yaml_dump(self, content, where):
"""dump to yaml"""
y = yaml()
y.default_flow_style = False
y.indent = 2
y.typ = 'rt'
y.dump(content, where)
########################################################
# templating
########################################################
def _redefine_templater(self):
"""create templater based on current variables"""
fufile = self.settings[Settings.key_func_file]
fifile = self.settings[Settings.key_filter_file]
self._tmpl = Templategen(variables=self.variables,
func_file=fufile,
filter_file=fifile)
def _template_item(self, item, exc_if_fail=True):
"""
template an item using the templategen
will raise an exception if template failed and exc_if_fail
"""
if not Templategen.var_is_template(item):
return item
try:
val = item
while Templategen.var_is_template(val):
val = self._tmpl.generate_string(val)
except UndefinedException as e:
if exc_if_fail:
raise e
return val
def _template_list(self, entries):
"""template a list of entries"""
new = []
if not entries:
return new
for e in entries:
et = self._template_item(e)
if self._debug and e != et:
self._dbg('resolved: {} -> {}'.format(e, et))
new.append(et)
return new
def _template_dict(self, entries):
"""template a dictionary of entries"""
new = {}
if not entries:
return new
for k, v in entries.items():
vt = self._template_item(v)
if self._debug and v != vt:
self._dbg('resolved: {} -> {}'.format(v, vt))
new[k] = vt
return new
def _template_dotfiles_paths(self):
"""template dotfiles paths"""
if self._debug:
self._dbg('templating dotfiles paths')
dotfiles = self.dotfiles.copy()
# make sure no dotfiles path is None
for dotfile in dotfiles.values():
src = dotfile[self.key_dotfile_src]
if src is None:
dotfile[self.key_dotfile_src] = ''
dst = dotfile[self.key_dotfile_dst]
if dst is None:
dotfile[self.key_dotfile_dst] = ''
# only keep dotfiles related to the selected profile
pdfs = []
pro = self.profiles.get(self._profile, [])
if pro:
pdfs = list(pro.get(self.key_profile_dotfiles, []))
for addpro in self._inc_profiles:
pro = self.profiles.get(addpro, [])
if not pro:
continue
pdfsalt = pro.get(self.key_profile_dotfiles, [])
pdfs.extend(pdfsalt)
pdfs = uniq_list(pdfs)
if self.key_all not in pdfs:
# take a subset of the dotfiles
newdotfiles = {}
for k, v in dotfiles.items():
if k in pdfs:
newdotfiles[k] = v
dotfiles = newdotfiles
for dotfile in dotfiles.values():
# src
src = dotfile[self.key_dotfile_src]
newsrc = self.resolve_dotfile_src(src, templater=self._tmpl)
dotfile[self.key_dotfile_src] = newsrc
# dst
dst = dotfile[self.key_dotfile_dst]
newdst = self.resolve_dotfile_dst(dst, templater=self._tmpl)
dotfile[self.key_dotfile_dst] = newdst
def _rec_resolve_variables(self, variables):
"""recursive resolve variables"""
var = self._enrich_vars(variables, self._profile)
# use a separated templategen to handle variables
# resolved outside the main config
t = Templategen(variables=var,
func_file=self.settings[Settings.key_func_file],
filter_file=self.settings[Settings.key_filter_file])
for k in variables.keys():
val = variables[k]
while Templategen.var_is_template(val):
val = t.generate_string(val)
variables[k] = val
t.update_variables(variables)
if variables is self.variables:
self._redefine_templater()
def _get_profile_included_vars(self):
"""resolve profile included variables/dynvariables"""
for k, v in self.profiles.items():
if self.key_profile_include in v and v[self.key_profile_include]:
new = []
for x in v[self.key_profile_include]:
new.append(self._tmpl.generate_string(x))
v[self.key_profile_include] = new
# now get the included ones
pro_var = self._get_profile_included_item(self.key_profile_variables)
pro_dvar = self._get_profile_included_item(self.key_profile_dvariables)
# the included profiles
inc_profiles = []
if self._profile and self._profile in self.profiles.keys():
pentry = self.profiles.get(self._profile)
inc_profiles = pentry.get(self.key_profile_include, [])
# exec incl dynvariables
return inc_profiles, pro_var, pro_dvar
########################################################
# helpers
########################################################
def _clear_profile_vars(self, dic):
"""
remove profile variables from dic if found inplace
to avoid profile variables being overwriten
"""
if not dic:
return
[dic.pop(k, None) for k in self._profilevarskeys]
def _parse_extended_import_path(self, path_entry):
"""Parse an import path in a tuple (path, fatal_not_found)."""
if self._debug:
self._dbg('parsing path entry {}'.format(path_entry))
path, _, attribute = path_entry.rpartition(self.key_import_sep)
fatal_not_found = attribute != self.key_import_ignore_key
is_valid_attribute = attribute in ('', self.key_import_ignore_key)
if not is_valid_attribute:
# If attribute is not valid it can mean that:
# - path_entry doesn't contain the separator, and attribute is set
# to the whole path by str.rpartition
# - path_entry contains a separator, but it's in the file path, so
# attribute is set to whatever comes after the separator by
# str.rpartition
# In both cases, path_entry is the path we're looking for.
if self._debug:
self._dbg('using attribute default values for path {}'.format(
path_entry))
path = path_entry
fatal_not_found = self.key_import_fatal_not_found
elif self._debug:
self._dbg(
'path entry {} has fatal_not_found flag set to {}'.format(
path_entry, fatal_not_found))
return path, fatal_not_found
def _handle_non_existing_path(self, path, fatal_not_found=True):
"""Raise an exception or log a warning to handle non-existing paths."""
error = 'bad path {}'.format(path)
if fatal_not_found:
raise YamlException(error)
self._log.warn(error)
def _check_path_existence(self, path, fatal_not_found=True):
"""Check if a path exists, raising if necessary."""
if os.path.exists(path):
if self._debug:
self._dbg('path {} exists'.format(path))
return path
self._handle_non_existing_path(path, fatal_not_found)
# Explicit return for readability. Anything evaluating to false is ok.
return None
def _process_path(self, path_entry):
"""
This method processed a path entry. Namely it:
- Normalizes the path.
- Expands globs.
- Checks for path existence, taking in account fatal_not_found.
This method always returns a list containing only absolute paths
existing on the filesystem. If the input is not a glob, the list
contains at most one element, otheriwse it could hold more.
"""
path, fatal_not_found = self._parse_extended_import_path(path_entry)
path = self._norm_path(path)
paths = self._glob_path(path) if self._is_glob(path) else [path]
if not paths:
if self._debug:
self._dbg("glob path {} didn't expand".format(path))
self._handle_non_existing_path(path, fatal_not_found)
return []
checked_paths = (self._check_path_existence(p, fatal_not_found)
for p in paths)
return [p for p in checked_paths if p]
def _resolve_paths(self, paths):
"""
This function resolves a list of paths. This means normalizing,
expanding globs and checking for existence, taking in account
fatal_not_found flags.
"""
processed_paths = (self._process_path(p) for p in paths)
return list(chain.from_iterable(processed_paths))
def _merge_dict(self, high, low):
"""merge high and low dict"""
if not high:
high = {}
if not low:
low = {}
return {**low, **high}
def _get_entry(self, dic, key, mandatory=True):
"""return copy of entry from yaml dictionary"""
if key not in dic:
if mandatory:
raise YamlException('invalid config: no {} found'.format(key))
dic[key] = {}
return deepcopy(dic[key])
if mandatory and not dic[key]:
# ensure is not none
dic[key] = {}
return deepcopy(dic[key])
def _clear_none(self, dic):
"""recursively delete all none/empty values in a dictionary."""
new = {}
for k, v in dic.items():
if k == self.key_dotfile_src:
# allow empty dotfile src
new[k] = v
continue
if k == self.key_dotfile_dst:
# allow empty dotfile dst
new[k] = v
continue
newv = v
if isinstance(v, dict):
# recursive travers dict
newv = self._clear_none(v)
if not newv:
# no empty dict
continue
if newv is None:
# no None value
continue
if isinstance(newv, list) and not newv:
# no empty list
continue
new[k] = newv
return new
def _is_glob(self, path):
"""Quick test if path is a glob."""
return '*' in path or '?' in path
def _glob_path(self, path):
"""Expand a glob."""
if self._debug:
self._dbg('expanding glob {}'.format(path))
expanded_path = os.path.expanduser(path)
return glob.glob(expanded_path, recursive=True)
def _norm_path(self, path):
"""Resolve a path either absolute or relative to config path"""
if not path:
return path
path = os.path.expanduser(path)
if not os.path.isabs(path):
d = os.path.dirname(self._path)
ret = os.path.join(d, path)
if self._debug:
msg = 'normalizing relative to cfg: {} -> {}'
self._dbg(msg.format(path, ret))
return ret
ret = os.path.normpath(path)
if self._debug and path != ret:
self._dbg('normalizing: {} -> {}'.format(path, ret))
return ret
def _shell_exec_dvars(self, dic, keys=[]):
"""shell execute dynvariables in-place"""
if not keys:
keys = dic.keys()
for k in keys:
v = dic[k]
ret, out = shell(v, debug=self._debug)
if not ret:
err = 'var \"{}: {}\" failed: {}'.format(k, v, out)
self._log.err(err)
raise YamlException(err)
if self._debug:
self._dbg('{}: `{}` -> {}'.format(k, v, out))
dic[k] = out
def _check_minversion(self, minversion):
if not minversion:
return
try:
cur = tuple([int(x) for x in VERSION.split('.')])
cfg = tuple([int(x) for x in minversion.split('.')])
except Exception:
err = 'bad version: \"{}\" VS \"{}\"'.format(VERSION, minversion)
raise YamlException(err)
if cur < cfg:
err = 'current dotdrop version is too old for that config file.'
err += ' Please update.'
raise YamlException(err)
def _debug_entries(self):
"""debug print all interesting entries"""
if not self._debug:
return
self._dbg('Current entries')
self._debug_dict('entry settings', self.settings)
self._debug_dict('entry dotfiles', self.dotfiles)
self._debug_dict('entry profiles', self.profiles)
self._debug_dict('entry actions', self.actions)
self._debug_dict('entry trans_r', self.trans_r)
self._debug_dict('entry trans_w', self.trans_w)
self._debug_dict('entry variables', self.variables)
def _debug_dict(self, title, elems):
"""pretty print dict"""
if not self._debug:
return
self._dbg('{}:'.format(title))
if not elems:
return
for k, v in elems.items():
self._dbg('\t- \"{}\": {}'.format(k, v))
def _dbg(self, content):
pre = os.path.basename(self._path)
self._log.dbg('[{}] {}'.format(pre, content))
class CfgAggregator:
file_prefix = 'f'
dir_prefix = 'd'
key_sep = '_'
def __init__(self, path, profile_key, debug=False, dry=False):
"""
high level config parser
@path: path to the config file
@profile_key: profile key
@debug: debug flag
"""
self.path = path
self.profile_key = profile_key
self.debug = debug
self.dry = dry
self.log = Logger()
self._load()
def _load(self):
"""load lower level config"""
self.cfgyaml = CfgYaml(self.path,
self.profile_key,
debug=self.debug)
# settings
self.settings = Settings.parse(None, self.cfgyaml.settings)
# dotfiles
self.dotfiles = Dotfile.parse_dict(self.cfgyaml.dotfiles)
if self.debug:
self._debug_list('dotfiles', self.dotfiles)
# profiles
self.profiles = Profile.parse_dict(self.cfgyaml.profiles)
if self.debug:
self._debug_list('profiles', self.profiles)
# actions
self.actions = Action.parse_dict(self.cfgyaml.actions)
if self.debug:
self._debug_list('actions', self.actions)
# trans_r
self.trans_r = Transform.parse_dict(self.cfgyaml.trans_r)
if self.debug:
self._debug_list('trans_r', self.trans_r)
# trans_w
self.trans_w = Transform.parse_dict(self.cfgyaml.trans_w)
if self.debug:
self._debug_list('trans_w', self.trans_w)
# variables
self.variables = self.cfgyaml.variables
if self.debug:
self._debug_dict('variables', self.variables)
# patch dotfiles in profiles
self._patch_keys_to_objs(self.profiles,
"dotfiles", self.get_dotfile)
# patch action in dotfiles actions
self._patch_keys_to_objs(self.dotfiles,
"actions", self._get_action_w_args)
# patch action in profiles actions
self._patch_keys_to_objs(self.profiles,
"actions", self._get_action_w_args)
# patch actions in settings default_actions
self._patch_keys_to_objs([self.settings],
"default_actions", self._get_action_w_args)
if self.debug:
msg = 'default actions: {}'.format(self.settings.default_actions)
self.log.dbg(msg)
# patch trans_w/trans_r in dotfiles
self._patch_keys_to_objs(self.dotfiles,
"trans_r",
self._get_trans_w_args(self._get_trans_r),
islist=False)
self._patch_keys_to_objs(self.dotfiles,
"trans_w",
self._get_trans_w_args(self._get_trans_w),
islist=False)
def _patch_keys_to_objs(self, containers, keys, get_by_key, islist=True):
"""
map for each key in the attribute 'keys' in 'containers'
the returned object from the method 'get_by_key'
"""
if not containers:
return
if self.debug:
self.log.dbg('patching {} ...'.format(keys))
for c in containers:
objects = []
okeys = getattr(c, keys)
if not okeys:
continue
if not islist:
okeys = [okeys]
for k in okeys:
o = get_by_key(k)
if not o:
err = '{} does not contain'.format(c)
err += ' a {} entry named {}'.format(keys, k)
self.log.err(err)
raise Exception(err)
objects.append(o)
if not islist:
objects = objects[0]
# if self.debug:
# er = 'patching {}.{} with {}'
# self.log.dbg(er.format(c, keys, objects))
setattr(c, keys, objects)
def del_dotfile(self, dotfile):
"""remove this dotfile from the config"""
return self.cfgyaml.del_dotfile(dotfile.key)
def del_dotfile_from_profile(self, dotfile, profile):
"""remove this dotfile from this profile"""
return self.cfgyaml.del_dotfile_from_profile(dotfile.key, profile.key)
def _create_new_dotfile(self, src, dst, link):
"""create a new dotfile"""
# get a new dotfile with a unique key
key = self._get_new_dotfile_key(dst)
if self.debug:
self.log.dbg('new dotfile key: {}'.format(key))
# add the dotfile
self.cfgyaml.add_dotfile(key, src, dst, link)
return Dotfile(key, dst, src)
def new(self, src, dst, link):
"""
import a new dotfile
@src: path in dotpath
@dst: path in FS
@link: LinkType
"""
dst = self.path_to_dotfile_dst(dst)
dotfile = self.get_dotfile_by_src_dst(src, dst)
if not dotfile:
dotfile = self._create_new_dotfile(src, dst, link)
key = dotfile.key
ret = self.cfgyaml.add_dotfile_to_profile(key, self.profile_key)
if ret and self.debug:
msg = 'new dotfile {} to profile {}'
self.log.dbg(msg.format(key, self.profile_key))
self.save()
if ret and not self.dry:
# reload
if self.debug:
self.log.dbg('reloading config')
olddebug = self.debug
self.debug = False
self._load()
self.debug = olddebug
return ret
def _get_new_dotfile_key(self, dst):
"""return a new unique dotfile key"""
path = os.path.expanduser(dst)
existing_keys = self.cfgyaml.get_all_dotfile_keys()
if self.settings.longkey:
return self._get_long_key(path, existing_keys)
return self._get_short_key(path, existing_keys)
def _norm_key_elem(self, elem):
"""normalize path element for sanity"""
elem = elem.lstrip('.')
elem = elem.replace(' ', '-')
return elem.lower()
def _split_path_for_key(self, path):
"""return a list of path elements, excluded home path"""
p = strip_home(path)
dirs = []
while True:
p, f = os.path.split(p)
dirs.append(f)
if not p or not f:
break
dirs.reverse()
# remove empty entries
dirs = filter(None, dirs)
# normalize entries
return list(map(self._norm_key_elem, dirs))
def _get_long_key(self, path, keys):
"""
return a unique long key representing the
absolute path of path
"""
dirs = self._split_path_for_key(path)
prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix
key = self.key_sep.join([prefix] + dirs)
return self._uniq_key(key, keys)
def _get_short_key(self, path, keys):
"""
return a unique key where path
is known not to be an already existing dotfile
"""
dirs = self._split_path_for_key(path)
dirs.reverse()
prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix
entries = []
for d in dirs:
entries.insert(0, d)
key = self.key_sep.join([prefix] + entries)
if key not in keys:
return key
return self._uniq_key(key, keys)
def _uniq_key(self, key, keys):
"""unique dotfile key"""
newkey = key
cnt = 1
while newkey in keys:
# if unable to get a unique path
# get a random one
newkey = self.key_sep.join([key, str(cnt)])
cnt += 1
return newkey
def path_to_dotfile_dst(self, path):
"""normalize the path to match dotfile dst"""
path = self._norm_path(path)
# use tild for home
home = os.path.expanduser(TILD) + os.sep
if path.startswith(home):
path = path[len(home):]
path = os.path.join(TILD, path)
return path
def get_dotfile_by_dst(self, dst):
"""
get a list of dotfiles by dst
@dst: dotfile dst (on filesystem)
"""
dotfiles = []
dst = self._norm_path(dst)
for d in self.dotfiles:
left = self._norm_path(d.dst)
if left == dst:
dotfiles.append(d)
return dotfiles
def get_dotfile_by_src_dst(self, src, dst):
"""
get a dotfile by src and dst
@src: dotfile src (in dotpath)
@dst: dotfile dst (on filesystem)
"""
try:
src = self.cfgyaml.resolve_dotfile_src(src)
except UndefinedException as e:
err = 'unable to resolve {}: {}'
self.log.err(err.format(src, e))
return None
dotfiles = self.get_dotfile_by_dst(dst)
for d in dotfiles:
if d.src == src:
return d
return None
def save(self):
"""save the config"""
if self.dry:
return True
return self.cfgyaml.save()
def dump(self):
"""dump the config dictionary"""
return self.cfgyaml.dump()
def get_settings(self):
"""return settings as a dict"""
return self.settings.serialize()[Settings.key_yaml]
def get_variables(self):
"""return variables"""
return self.variables
def get_profiles(self):
"""return profiles"""
return self.profiles
def get_profile(self):
"""return profile object"""
try:
return next(x for x in self.profiles if x.key == self.profile_key)
except StopIteration:
return None
def get_profiles_by_dotfile_key(self, key):
"""return all profiles having this dotfile"""
res = []
for p in self.profiles:
keys = [d.key for d in p.dotfiles]
if key in keys:
res.append(p)
return res
def get_dotfiles(self):
"""get all dotfiles for this profile"""
dotfiles = []
profile = self.get_profile()
if not profile:
return dotfiles
return profile.dotfiles
def get_dotfile(self, key):
"""
return dotfile object by key
@key: the dotfile key to look for
"""
try:
return next(x for x in self.dotfiles
if x.key == key)
except StopIteration:
return None
def _get_action(self, key):
"""return action by key"""
try:
return next(x for x in self.actions if x.key == key)
except StopIteration:
return None
def _get_action_w_args(self, key):
"""return action by key with the arguments"""
fields = shlex.split(key)
if len(fields) > 1:
# we have args
key, *args = fields
if self.debug:
msg = 'action with parm: {} and {}'
self.log.dbg(msg.format(key, args))
action = self._get_action(key).copy(args)
else:
action = self._get_action(key)
return action
def _get_trans_w_args(self, getter):
"""return transformation by key with the arguments"""
def getit(key):
fields = shlex.split(key)
if len(fields) > 1:
# we have args
key, *args = fields
if self.debug:
msg = 'trans with parm: {} and {}'
self.log.dbg(msg.format(key, args))
trans = getter(key).copy(args)
else:
trans = getter(key)
return trans
return getit
def _get_trans_r(self, key):
"""return the trans_r with this key"""
try:
return next(x for x in self.trans_r if x.key == key)
except StopIteration:
return None
def _get_trans_w(self, key):
"""return the trans_w with this key"""
try:
return next(x for x in self.trans_w if x.key == key)
except StopIteration:
return None
def _norm_path(self, path):
if not path:
return path
path = os.path.expanduser(path)
path = os.path.expandvars(path)
path = os.path.abspath(path)
return path
def _debug_list(self, title, elems):
"""pretty print list"""
if not self.debug:
return
self.log.dbg('{}:'.format(title))
for e in elems:
self.log.dbg('\t- {}'.format(e))
def _debug_dict(self, title, elems):
"""pretty print dict"""
if not self.debug:
return
self.log.dbg('{}:'.format(title))
for k, v in elems.items():
self.log.dbg('\t- \"{}\": {}'.format(k, v))
class Installer:
BACKUP_SUFFIX = '.dotdropbak'
def __init__(self,
base='.',
create=True,
backup=True,
dry=False,
safe=False,
workdir='~/.config/dotdrop',
debug=False,
diff=True,
totemp=None,
showdiff=False):
self.create = create
self.backup = backup
self.dry = dry
self.safe = safe
self.workdir = os.path.expanduser(workdir)
self.base = base
self.debug = debug
self.diff = diff
self.totemp = totemp
self.showdiff = showdiff
self.comparing = False
self.action_executed = False
self.log = Logger()
def install(self, templater, src, dst, actions=[], noempty=False):
"""install the src to dst using a template"""
self.action_executed = False
src = os.path.join(self.base, os.path.expanduser(src))
if not os.path.exists(src):
self.log.err('source dotfile does not exist: {}'.format(src))
return []
dst = os.path.expanduser(dst)
if self.totemp:
dst = self._pivot_path(dst, self.totemp)
if utils.samefile(src, dst):
# symlink loop
self.log.err('dotfile points to itself: {}'.format(dst))
return []
if self.debug:
self.log.dbg('install {} to {}'.format(src, dst))
if os.path.isdir(src):
return self._handle_dir(templater,
src,
dst,
actions=actions,
noempty=noempty)
return self._handle_file(templater,
src,
dst,
actions=actions,
noempty=noempty)
def link(self, templater, src, dst, actions=[]):
"""set src as the link target of dst"""
self.action_executed = False
src = os.path.join(self.base, os.path.expanduser(src))
if not os.path.exists(src):
self.log.err('source dotfile does not exist: {}'.format(src))
return []
dst = os.path.expanduser(dst)
if self.totemp:
# ignore actions
return self.install(templater, src, dst, actions=[])
if Templategen.is_template(src):
if self.debug:
self.log.dbg('dotfile is a template')
self.log.dbg('install to {} and symlink'.format(self.workdir))
tmp = self._pivot_path(dst, self.workdir, striphome=True)
i = self.install(templater, src, tmp, actions=actions)
if not i and not os.path.exists(tmp):
return []
src = tmp
return self._link(src, dst, actions=actions)
def _link(self, src, dst, actions=[]):
"""set src as a link target of dst"""
if os.path.lexists(dst):
if os.path.realpath(dst) == os.path.realpath(src):
if self.debug:
self.log.dbg('ignoring "{}", link exists'.format(dst))
return []
if self.dry:
self.log.dry('would remove {} and link to {}'.format(dst, src))
return []
msg = 'Remove "{}" for link creation?'.format(dst)
if self.safe and not self.log.ask(msg):
msg = 'ignoring "{}", link was not created'
self.log.warn(msg.format(dst))
return []
try:
utils.remove(dst)
except OSError as e:
self.log.err('something went wrong with {}: {}'.format(src, e))
return []
if self.dry:
self.log.dry('would link {} to {}'.format(dst, src))
return []
base = os.path.dirname(dst)
if not self._create_dirs(base):
self.log.err('creating directory for {}'.format(dst))
return []
self._exec_pre_actions(actions)
os.symlink(src, dst)
self.log.sub('linked {} to {}'.format(dst, src))
return [(src, dst)]
def _handle_file(self, templater, src, dst, actions=[], noempty=False):
"""install src to dst when is a file"""
if self.debug:
self.log.dbg('generate template for {}'.format(src))
self.log.dbg('ignore empty: {}'.format(noempty))
if utils.samefile(src, dst):
# symlink loop
self.log.err('dotfile points to itself: {}'.format(dst))
return []
content = templater.generate(src)
if noempty and utils.content_empty(content):
self.log.warn('ignoring empty template: {}'.format(src))
return []
if content is None:
self.log.err('generate from template {}'.format(src))
return []
if not os.path.exists(src):
self.log.err('source dotfile does not exist: {}'.format(src))
return []
st = os.stat(src)
ret = self._write(src, dst, content, st.st_mode, actions=actions)
if ret < 0:
self.log.err('installing {} to {}'.format(src, dst))
return []
if ret > 0:
if self.debug:
self.log.dbg('ignoring {}'.format(dst))
return []
if ret == 0:
if not self.dry and not self.comparing:
self.log.sub('copied {} to {}'.format(src, dst))
return [(src, dst)]
return []
def _handle_dir(self, templater, src, dst, actions=[], noempty=False):
"""install src to dst when is a directory"""
if self.debug:
self.log.dbg('install dir {}'.format(src))
self.log.dbg('ignore empty: {}'.format(noempty))
ret = []
if not self._create_dirs(dst):
return []
# handle all files in dir
for entry in os.listdir(src):
f = os.path.join(src, entry)
if not os.path.isdir(f):
res = self._handle_file(templater,
f,
os.path.join(dst, entry),
actions=actions,
noempty=noempty)
ret.extend(res)
else:
res = self._handle_dir(templater,
f,
os.path.join(dst, entry),
actions=actions,
noempty=noempty)
ret.extend(res)
return ret
def _fake_diff(self, dst, content):
"""fake diff by comparing file content with content"""
cur = ''
with open(dst, 'br') as f:
cur = f.read()
return cur == content
def _write(self, src, dst, content, rights, actions=[]):
"""write content to file
return 0 for success,
1 when already exists
-1 when error"""
if self.dry:
self.log.dry('would install {}'.format(dst))
return 0
if os.path.lexists(dst):
samerights = os.stat(dst).st_mode == rights
if self.diff and self._fake_diff(dst, content) and samerights:
if self.debug:
self.log.dbg('{} is the same'.format(dst))
return 1
if self.safe:
if self.debug:
self.log.dbg('change detected for {}'.format(dst))
if self.showdiff:
self._diff_before_write(src, dst, content)
if not self.log.ask('Overwrite \"{}\"'.format(dst)):
self.log.warn('ignoring {}'.format(dst))
return 1
if self.backup and os.path.lexists(dst):
self._backup(dst)
base = os.path.dirname(dst)
if not self._create_dirs(base):
self.log.err('creating directory for {}'.format(dst))
return -1
if self.debug:
self.log.dbg('write content to {}'.format(dst))
self._exec_pre_actions(actions)
try:
with open(dst, 'wb') as f:
f.write(content)
except NotADirectoryError as e:
self.log.err('opening dest file: {}'.format(e))
return -1
os.chmod(dst, rights)
return 0
def _diff_before_write(self, src, dst, src_content):
"""diff before writing when using --showdiff - not efficient"""
# create tmp to diff for templates
tmpfile = utils.get_tmpfile()
with open(tmpfile, 'wb') as f:
f.write(src_content)
comp = Comparator(debug=self.debug)
diff = comp.compare(tmpfile, dst)
# fake the output for readability
self.log.log('diff \"{}\" VS \"{}\"'.format(src, dst))
self.log.emph(diff)
if tmpfile:
utils.remove(tmpfile)
def _create_dirs(self, directory):
"""mkdir -p <directory>"""
if not self.create and not os.path.exists(directory):
return False
if os.path.exists(directory):
return True
if self.dry:
self.log.dry('would mkdir -p {}'.format(directory))
return True
if self.debug:
self.log.dbg('mkdir -p {}'.format(directory))
os.makedirs(directory)
return os.path.exists(directory)
def _backup(self, path):
"""backup file pointed by path"""
if self.dry:
return
dst = path.rstrip(os.sep) + self.BACKUP_SUFFIX
self.log.log('backup {} to {}'.format(path, dst))
os.rename(path, dst)
def _pivot_path(self, path, newdir, striphome=False):
"""change path to be under newdir"""
if striphome:
home = os.path.expanduser('~')
path = path.lstrip(home)
sub = path.lstrip(os.sep)
return os.path.join(newdir, sub)
def _exec_pre_actions(self, actions):
"""execute pre-actions if any"""
if self.action_executed:
return
for action in actions:
if self.dry:
self.log.dry('would execute action: {}'.format(action))
else:
if self.debug:
self.log.dbg('executing pre action {}'.format(action))
action.execute()
self.action_executed = True
def _install_to_temp(self, templater, src, dst, tmpdir):
"""install a dotfile to a tempdir"""
tmpdst = self._pivot_path(dst, tmpdir)
return self.install(templater, src, tmpdst), tmpdst
def install_to_temp(self, templater, tmpdir, src, dst):
"""install a dotfile to a tempdir"""
ret = False
tmpdst = ''
# save some flags while comparing
self.comparing = True
drysaved = self.dry
self.dry = False
diffsaved = self.diff
self.diff = False
createsaved = self.create
self.create = True
# normalize src and dst
src = os.path.expanduser(src)
dst = os.path.expanduser(dst)
if self.debug:
self.log.dbg('tmp install {} to {}'.format(src, dst))
# install the dotfile to a temp directory for comparing
ret, tmpdst = self._install_to_temp(templater, src, dst, tmpdir)
if self.debug:
self.log.dbg('tmp installed in {}'.format(tmpdst))
# reset flags
self.dry = drysaved
self.diff = diffsaved
self.comparing = False
self.create = createsaved
return ret, tmpdst
class Cfg:
key_all = 'ALL'
# settings keys
key_settings = 'config'
key_dotpath = 'dotpath'
key_backup = 'backup'
key_create = 'create'
key_banner = 'banner'
key_long = 'longkey'
key_keepdot = 'keepdot'
key_ignoreempty = 'ignoreempty'
key_showdiff = 'showdiff'
key_deflink = 'link_by_default'
key_workdir = 'workdir'
# actions keys
key_actions = 'actions'
key_actions_pre = 'pre'
key_actions_post = 'post'
# transformations keys
key_trans_r = 'trans'
key_trans_w = 'trans_write'
# template variables
key_variables = 'variables'
# shell variables
key_dynvariables = 'dynvariables'
# dotfiles keys
key_dotfiles = 'dotfiles'
key_dotfiles_src = 'src'
key_dotfiles_dst = 'dst'
key_dotfiles_link = 'link'
key_dotfiles_link_children = 'link_children'
key_dotfiles_noempty = 'ignoreempty'
key_dotfiles_cmpignore = 'cmpignore'
key_dotfiles_actions = 'actions'
key_dotfiles_trans_r = 'trans'
key_dotfiles_trans_w = 'trans_write'
key_dotfiles_upignore = 'upignore'
# profiles keys
key_profiles = 'profiles'
key_profiles_dots = 'dotfiles'
key_profiles_incl = 'include'
# settings defaults
default_backup = True
default_create = True
default_banner = True
default_link = LinkTypes.NOLINK
default_longkey = False
default_keepdot = False
default_showdiff = False
default_ignoreempty = False
default_link_by_default = False
default_workdir = '~/.config/dotdrop'
def __init__(self, cfgpath, debug=False):
"""constructor
@cfgpath: path to the config file
@debug: enable debug
"""
if not os.path.exists(cfgpath):
raise ValueError('config file does not exist: {}'.format(cfgpath))
# make sure to have an absolute path to config file
self.cfgpath = os.path.abspath(cfgpath)
self.debug = debug
# init the logger
self.log = Logger()
# represents all entries under "config"
# linked inside the yaml dict (self.content)
self.lnk_settings = {}
# represents all entries under "profiles"
# linked inside the yaml dict (self.content)
self.lnk_profiles = {}
# represents all dotfiles
# NOT linked inside the yaml dict (self.content)
self.dotfiles = {}
# dict of all action objects by action key
# NOT linked inside the yaml dict (self.content)
self.actions = {}
# dict of all read transformation objects by trans key
# NOT linked inside the yaml dict (self.content)
self.trans_r = {}
# dict of all write transformation objects by trans key
# NOT linked inside the yaml dict (self.content)
self.trans_w = {}
# represents all dotfiles per profile by profile key
# NOT linked inside the yaml dict (self.content)
self.prodots = {}
if not self._load_file():
raise ValueError('config is not valid')
def eval_dotfiles(self, profile, variables, debug=False):
"""resolve dotfiles src/dst/actions templating for this profile"""
t = Templategen(variables=variables)
dotfiles = self._get_dotfiles(profile)
for d in dotfiles:
# src and dst path
d.src = t.generate_string(d.src)
d.dst = t.generate_string(d.dst)
# pre actions
if self.key_actions_pre in d.actions:
for action in d.actions[self.key_actions_pre]:
action.action = t.generate_string(action.action)
# post actions
if self.key_actions_post in d.actions:
for action in d.actions[self.key_actions_post]:
action.action = t.generate_string(action.action)
return dotfiles
def _load_file(self):
"""load the yaml file"""
with open(self.cfgpath, 'r') as f:
self.content = yaml.safe_load(f)
if not self._is_valid():
return False
return self._parse()
def _is_valid(self):
"""test the yaml dict (self.content) is valid"""
if self.key_profiles not in self.content:
self.log.err('missing \"{}\" in config'.format(self.key_profiles))
return False
if self.key_settings not in self.content:
self.log.err('missing \"{}\" in config'.format(self.key_settings))
return False
if self.key_dotfiles not in self.content:
self.log.err('missing \"{}\" in config'.format(self.key_dotfiles))
return False
return True
def _parse(self):
"""parse config file"""
# parse all actions
if self.key_actions in self.content:
if self.content[self.key_actions] is not None:
for k, v in self.content[self.key_actions].items():
# loop through all actions
if k in [self.key_actions_pre, self.key_actions_post]:
# parse pre/post actions
items = self.content[self.key_actions][k].items()
for k2, v2 in items:
if k not in self.actions:
self.actions[k] = {}
self.actions[k][k2] = Action(k2, k, v2)
else:
# parse naked actions as post actions
if self.key_actions_post not in self.actions:
self.actions[self.key_actions_post] = {}
self.actions[self.key_actions_post][k] = Action(
k, '', v)
# parse read transformations
if self.key_trans_r in self.content:
if self.content[self.key_trans_r] is not None:
for k, v in self.content[self.key_trans_r].items():
self.trans_r[k] = Transform(k, v)
# parse write transformations
if self.key_trans_w in self.content:
if self.content[self.key_trans_w] is not None:
for k, v in self.content[self.key_trans_w].items():
self.trans_w[k] = Transform(k, v)
# parse the profiles
self.lnk_profiles = self.content[self.key_profiles]
if self.lnk_profiles is None:
# ensures self.lnk_profiles is a dict
self.content[self.key_profiles] = {}
self.lnk_profiles = self.content[self.key_profiles]
for k, v in self.lnk_profiles.items():
if self.key_profiles_dots in v and \
v[self.key_profiles_dots] is None:
# if has the dotfiles entry but is empty
# ensures it's an empty list
v[self.key_profiles_dots] = []
# parse the settings
self.lnk_settings = self.content[self.key_settings]
self._complete_settings()
# parse the dotfiles
# and construct the dict of objects per dotfile key
if not self.content[self.key_dotfiles]:
# ensures the dotfiles entry is a dict
self.content[self.key_dotfiles] = {}
for k, v in self.content[self.key_dotfiles].items():
src = os.path.normpath(v[self.key_dotfiles_src])
dst = os.path.normpath(v[self.key_dotfiles_dst])
# Fail if both `link` and `link_children` present
if self.key_dotfiles_link in v \
and self.key_dotfiles_link_children in v:
msg = 'only one of `link` or `link_children` allowed per'
msg += ' dotfile, error on dotfile "{}".'
self.log.err(msg.format(k))
# Otherwise, get link type
link = LinkTypes.NOLINK
if self.key_dotfiles_link in v and v[self.key_dotfiles_link]:
link = LinkTypes.PARENTS
if self.key_dotfiles_link_children in v \
and v[self.key_dotfiles_link_children]:
link = LinkTypes.CHILDREN
noempty = v[self.key_dotfiles_noempty] if \
self.key_dotfiles_noempty \
in v else self.lnk_settings[self.key_ignoreempty]
itsactions = v[self.key_dotfiles_actions] if \
self.key_dotfiles_actions in v else []
actions = self._parse_actions(itsactions)
# parse read transformation
itstrans_r = v[self.key_dotfiles_trans_r] if \
self.key_dotfiles_trans_r in v else None
trans_r = None
if itstrans_r:
if type(itstrans_r) is list:
msg = 'One transformation allowed per dotfile'
msg += ', error on dotfile \"{}\"'
self.log.err(msg.format(k))
msg = 'Please modify your config file to: \"trans: {}\"'
self.log.err(msg.format(itstrans_r[0]))
msg = 'see https://github.com/deadc0de6/dotdrop/wiki/transformations#config-error-with-transformation-list' # noqa
self.log.err(msg)
return False
trans_r = self._parse_trans(itstrans_r, read=True)
if not trans_r:
msg = 'unknown trans \"{}\" for \"{}\"'
self.log.err(msg.format(itstrans_r, k))
return False
# parse write transformation
itstrans_w = v[self.key_dotfiles_trans_w] if \
self.key_dotfiles_trans_w in v else None
trans_w = None
if itstrans_w:
if type(itstrans_w) is list:
msg = 'One write transformation allowed per dotfile'
msg += ', error on dotfile \"{}\"'
self.log.err(msg.format(k))
msg = 'Please modify your config file: \"trans_write: {}\"'
self.log.err(msg.format(itstrans_w[0]))
msg = 'see https://github.com/deadc0de6/dotdrop/wiki/transformations#config-error-with-transformation-list' # noqa
self.log.err(msg)
return False
trans_w = self._parse_trans(itstrans_w, read=False)
if not trans_w:
msg = 'unknown trans_write \"{}\" for \"{}\"'
self.log.err(msg.format(itstrans_w, k))
return False
# disable transformation when link is true
if link != LinkTypes.NOLINK and (trans_r or trans_w):
msg = 'transformations disabled for \"{}\"'.format(dst)
msg += ' because link is True'
self.log.warn(msg)
trans_r = None
trans_w = None
# parse cmpignore pattern
cmpignores = v[self.key_dotfiles_cmpignore] if \
self.key_dotfiles_cmpignore in v else []
# parse upignore pattern
upignores = v[self.key_dotfiles_upignore] if \
self.key_dotfiles_upignore in v else []
# create new dotfile
self.dotfiles[k] = Dotfile(k,
dst,
src,
link=link,
actions=actions,
trans_r=trans_r,
trans_w=trans_w,
cmpignore=cmpignores,
noempty=noempty,
upignore=upignores)
# assign dotfiles to each profile
for k, v in self.lnk_profiles.items():
self.prodots[k] = []
if self.key_profiles_dots not in v:
# ensures is a list
v[self.key_profiles_dots] = []
if not v[self.key_profiles_dots]:
continue
dots = v[self.key_profiles_dots]
if self.key_all in dots:
# add all if key ALL is used
self.prodots[k] = list(self.dotfiles.values())
else:
# add the dotfiles
for d in dots:
if d not in self.dotfiles:
msg = 'unknown dotfile \"{}\" for {}'.format(d, k)
self.log.err(msg)
continue
self.prodots[k].append(self.dotfiles[d])
# handle "include" for each profile
for k in self.lnk_profiles.keys():
dots = self._get_included_dotfiles(k)
self.prodots[k].extend(dots)
# remove duplicates if any
self.prodots[k] = list(set(self.prodots[k]))
# make sure we have an absolute dotpath
self.curdotpath = self.lnk_settings[self.key_dotpath]
self.lnk_settings[self.key_dotpath] = \
self._abs_path(self.curdotpath)
# make sure we have an absolute workdir
self.curworkdir = self.lnk_settings[self.key_workdir]
self.lnk_settings[self.key_workdir] = \
self._abs_path(self.curworkdir)
return True
def _abs_path(self, path):
"""return absolute path of path relative to the confpath"""
path = os.path.expanduser(path)
if not os.path.isabs(path):
d = os.path.dirname(self.cfgpath)
return os.path.join(d, path)
return path
def _get_included_dotfiles(self, profile):
"""find all dotfiles for a specific profile
when using the include keyword"""
included = []
if self.key_profiles_incl not in self.lnk_profiles[profile]:
# no include found
return included
if not self.lnk_profiles[profile][self.key_profiles_incl]:
# empty include found
return included
for other in self.lnk_profiles[profile][self.key_profiles_incl]:
if other not in self.prodots:
# no such profile
self.log.warn('unknown included profile \"{}\"'.format(other))
continue
included.extend(self.prodots[other])
return included
def _parse_actions(self, entries):
"""parse actions specified for an element
where entries are the ones defined for this dotfile"""
res = {
self.key_actions_pre: [],
self.key_actions_post: [],
}
for line in entries:
fields = shlex.split(line)
entry = fields[0]
args = []
if len(fields) > 1:
args = fields[1:]
action = None
if self.key_actions_pre in self.actions and \
entry in self.actions[self.key_actions_pre]:
kind = self.key_actions_pre
if not args:
action = self.actions[self.key_actions_pre][entry]
else:
a = self.actions[self.key_actions_pre][entry].action
action = Action(entry, kind, a, *args)
elif self.key_actions_post in self.actions and \
entry in self.actions[self.key_actions_post]:
kind = self.key_actions_post
if not args:
action = self.actions[self.key_actions_post][entry]
else:
a = self.actions[self.key_actions_post][entry].action
action = Action(entry, kind, a, *args)
else:
self.log.warn('unknown action \"{}\"'.format(entry))
continue
res[kind].append(action)
return res
def _parse_trans(self, trans, read=True):
"""parse transformation key specified for a dotfile"""
transformations = self.trans_r
if not read:
transformations = self.trans_w
if trans not in transformations.keys():
return None
return transformations[trans]
def _complete_settings(self):
"""set settings defaults if not present"""
if self.key_backup not in self.lnk_settings:
self.lnk_settings[self.key_backup] = self.default_backup
if self.key_create not in self.lnk_settings:
self.lnk_settings[self.key_create] = self.default_create
if self.key_banner not in self.lnk_settings:
self.lnk_settings[self.key_banner] = self.default_banner
if self.key_long not in self.lnk_settings:
self.lnk_settings[self.key_long] = self.default_longkey
if self.key_keepdot not in self.lnk_settings:
self.lnk_settings[self.key_keepdot] = self.default_keepdot
if self.key_deflink not in self.lnk_settings:
self.lnk_settings[self.key_deflink] = self.default_link_by_default
if self.key_workdir not in self.lnk_settings:
self.lnk_settings[self.key_workdir] = self.default_workdir
if self.key_showdiff not in self.lnk_settings:
self.lnk_settings[self.key_showdiff] = self.default_showdiff
if self.key_ignoreempty not in self.lnk_settings:
self.lnk_settings[self.key_ignoreempty] = self.default_ignoreempty
def _save(self, content, path):
"""writes the config to file"""
ret = False
with open(path, 'w') as f:
ret = yaml.dump(content, f, default_flow_style=False, indent=2)
return ret
def _norm_key_elem(self, elem):
"""normalize path element for sanity"""
elem = elem.lstrip('.')
elem = elem.replace(' ', '-')
return elem.lower()
def _get_paths(self, path):
"""return a list of path elements, excluded home path"""
p = strip_home(path)
dirs = []
while True:
p, f = os.path.split(p)
dirs.append(f)
if not p or not f:
break
dirs.reverse()
# remove empty entries
dirs = filter(None, dirs)
# normalize entries
dirs = list(map(self._norm_key_elem, dirs))
return dirs
def _get_long_key(self, path):
"""return a long key representing the
absolute path of path"""
dirs = self._get_paths(path)
# prepend with indicator
if os.path.isdir(path):
key = 'd_{}'.format('_'.join(dirs))
else:
key = 'f_{}'.format('_'.join(dirs))
return key
def _get_short_key(self, path, keys):
"""return a unique key where path
is known not to be an already existing dotfile"""
dirs = self._get_paths(path)
dirs.reverse()
pre = 'f'
if os.path.isdir(path):
pre = 'd'
entries = []
for d in dirs:
entries.insert(0, d)
key = '_'.join(entries)
key = '{}_{}'.format(pre, key)
if key not in keys:
return key
okey = key
cnt = 1
while key in keys:
# if unable to get a unique path
# get a random one
key = '{}_{}'.format(okey, cnt)
cnt += 1
return key
def _dotfile_exists(self, dotfile):
"""return True and the existing dotfile key
if it already exists, False and a new unique key otherwise"""
dsts = [(k, d.dst) for k, d in self.dotfiles.items()]
if dotfile.dst in [x[1] for x in dsts]:
return True, [x[0] for x in dsts if x[1] == dotfile.dst][0]
# return key for this new dotfile
path = os.path.expanduser(dotfile.dst)
if self.lnk_settings[self.key_long]:
return False, self._get_long_key(path)
return False, self._get_short_key(path, self.dotfiles.keys())
def new(self, dotfile, profile, link=LinkTypes.NOLINK, debug=False):
"""import new dotfile
dotfile key will change and can be empty"""
# keep it short
home = os.path.expanduser('~')
dotfile.dst = dotfile.dst.replace(home, '~', 1)
# adding new profile if doesn't exist
if profile not in self.lnk_profiles:
if debug:
self.log.dbg('adding profile to config')
# in the yaml
self.lnk_profiles[profile] = {self.key_profiles_dots: []}
# in the global list of dotfiles per profile
self.prodots[profile] = []
exists, key = self._dotfile_exists(dotfile)
if exists:
if debug:
self.log.dbg('key already exists: {}'.format(key))
# retrieve existing dotfile
dotfile = self.dotfiles[key]
if dotfile in self.prodots[profile]:
self.log.err('\"{}\" already present'.format(dotfile.key))
return False, dotfile
# add for this profile
self.prodots[profile].append(dotfile)
# get a pointer in the yaml profiles->this_profile
# and complete it with the new entry
pro = self.content[self.key_profiles][profile]
if self.key_all not in pro[self.key_profiles_dots]:
pro[self.key_profiles_dots].append(dotfile.key)
return True, dotfile
if debug:
self.log.dbg('dotfile attributed key: {}'.format(key))
# adding the new dotfile
dotfile.key = key
dotfile.link = link
if debug:
self.log.dbg('adding new dotfile: {}'.format(dotfile))
# add the entry in the yaml file
dots = self.content[self.key_dotfiles]
dots[dotfile.key] = {
self.key_dotfiles_dst: dotfile.dst,
self.key_dotfiles_src: dotfile.src,
}
# set the link flag
if link == LinkTypes.PARENTS:
dots[dotfile.key][self.key_dotfiles_link] = True
elif link == LinkTypes.CHILDREN:
dots[dotfile.key][self.key_dotfiles_link_children] = True
# link it to this profile in the yaml file
pro = self.content[self.key_profiles][profile]
if self.key_all not in pro[self.key_profiles_dots]:
pro[self.key_profiles_dots].append(dotfile.key)
# add it to the global list of dotfiles
self.dotfiles[dotfile.key] = dotfile
# add it to this profile
self.prodots[profile].append(dotfile)
return True, dotfile
def _get_dotfiles(self, profile):
"""return a list of dotfiles for a specific profile"""
if profile not in self.prodots:
return []
return sorted(self.prodots[profile], key=lambda x: str(x.key))
def get_profiles(self):
"""return all defined profiles"""
return self.lnk_profiles.keys()
def get_settings(self):
"""return all defined settings"""
return self.lnk_settings.copy()
def get_variables(self, profile, debug=False):
"""return the variables for this profile"""
# get flat variables
variables = self._get_variables(profile)
# get interpreted variables
dvariables = self._get_dynvariables(profile)
# recursive resolve variables
allvars = variables.copy()
allvars.update(dvariables)
var = self._rec_resolve_vars(allvars)
# execute dynvariables
for k in dvariables.keys():
var[k] = shell(var[k])
if debug:
self.log.dbg('variables:')
for k, v in var.items():
self.log.dbg('\t\"{}\": {}'.format(k, v))
return var
def _rec_resolve_vars(self, variables):
"""recursive resolve all variables"""
t = Templategen(variables=variables)
for k in variables.keys():
val = variables[k]
while Templategen.var_is_template(val):
val = t.generate_string(val)
variables[k] = val
t.update_variables(variables)
return variables
def _get_variables(self, profile):
"""return the flat variables"""
variables = {}
# profile variable
variables['profile'] = profile
# global variables
if self.key_variables in self.content:
variables.update(self.content[self.key_variables])
if profile not in self.lnk_profiles:
return variables
# profile variables
var = self.lnk_profiles[profile]
if self.key_variables in var.keys():
for k, v in var[self.key_variables].items():
variables[k] = v
return variables
def _get_dynvariables(self, profile):
"""return the dyn variables"""
variables = {}
# global dynvariables
if self.key_dynvariables in self.content:
# interpret dynamic variables
variables.update(self.content[self.key_dynvariables])
if profile not in self.lnk_profiles:
return variables
# profile dynvariables
var = self.lnk_profiles[profile]
if self.key_dynvariables in var.keys():
variables.update(var[self.key_dynvariables])
return variables
def dump(self):
"""return a dump of the config"""
# temporary reset paths
dotpath = self.lnk_settings[self.key_dotpath]
workdir = self.lnk_settings[self.key_workdir]
self.lnk_settings[self.key_dotpath] = self.curdotpath
self.lnk_settings[self.key_workdir] = self.curworkdir
# dump
ret = yaml.dump(self.content, default_flow_style=False, indent=2)
ret = ret.replace('{}', '')
# restore paths
self.lnk_settings[self.key_dotpath] = dotpath
self.lnk_settings[self.key_workdir] = workdir
return ret
def save(self):
"""save the config to file"""
# temporary reset paths
dotpath = self.lnk_settings[self.key_dotpath]
workdir = self.lnk_settings[self.key_workdir]
self.lnk_settings[self.key_dotpath] = self.curdotpath
self.lnk_settings[self.key_workdir] = self.curworkdir
# save
ret = self._save(self.content, self.cfgpath)
# restore path
self.lnk_settings[self.key_dotpath] = dotpath
self.lnk_settings[self.key_workdir] = workdir
return ret
class Installer:
def __init__(self,
base='.',
create=True,
backup=True,
dry=False,
safe=False,
workdir='~/.config/dotdrop',
debug=False,
diff=True,
totemp=None,
showdiff=False,
backup_suffix='.dotdropbak',
diff_cmd=''):
"""
@base: directory path where to search for templates
@create: create directory hierarchy if missing when installing
@backup: backup existing dotfile when installing
@dry: just simulate
@safe: ask for any overwrite
@workdir: where to install template before symlinking
@debug: enable debug
@diff: diff when installing if True
@totemp: deploy to this path instead of dotfile dst if not None
@showdiff: show the diff before overwriting (or asking for)
@backup_suffix: suffix for dotfile backup file
@diff_cmd: diff command to use
"""
self.create = create
self.backup = backup
self.dry = dry
self.safe = safe
workdir = os.path.expanduser(workdir)
workdir = os.path.normpath(workdir)
self.workdir = workdir
base = os.path.expanduser(base)
base = os.path.normpath(base)
self.base = base
self.debug = debug
self.diff = diff
self.totemp = totemp
self.showdiff = showdiff
self.backup_suffix = backup_suffix
self.diff_cmd = diff_cmd
self.action_executed = False
# avoids printing file copied logs
# when using install_to_tmp for comparing
self.comparing = False
self.log = Logger()
########################################################
# public methods
########################################################
def install(self,
templater,
src,
dst,
linktype,
actionexec=None,
noempty=False,
ignore=[],
is_template=True,
chmod=None,
force_chmod=False):
"""
install src to dst
@templater: the templater object
@src: dotfile source path in dotpath
@dst: dotfile destination path in the FS
@linktype: linktypes.LinkTypes
@actionexec: action executor callback
@noempty: render empty template flag
@ignore: pattern to ignore when installing
@is_template: this dotfile is a template
@chmod: rights to apply if any
@force_chmod: do not ask user to chmod
return
- True, None : success
- False, error_msg : error
- False, None : ignored
"""
if not src or not dst:
# fake dotfile
if self.debug:
self.log.dbg('fake dotfile installed')
self._exec_pre_actions(actionexec)
return True, None
if self.debug:
msg = 'installing \"{}\" to \"{}\" (link: {})'
self.log.dbg(msg.format(src, dst, str(linktype)))
src, dst, cont, err = self._check_paths(src, dst, chmod)
if not cont:
return self._log_install(cont, err)
# check source file exists
src = os.path.join(self.base, src)
if not os.path.exists(src):
err = 'source dotfile does not exist: {}'.format(src)
return self._log_install(False, err)
self.action_executed = False
# install to temporary dir
# and ignore any actions
if self.totemp:
r, err, _ = self.install_to_temp(templater,
self.totemp,
src,
dst,
is_template=is_template,
chmod=chmod,
ignore=ignore)
return self._log_install(r, err)
isdir = os.path.isdir(src)
if self.debug:
self.log.dbg('install {} to {}'.format(src, dst))
self.log.dbg('\"{}\" is a directory: {}'.format(src, isdir))
if linktype == LinkTypes.NOLINK:
# normal file
if isdir:
r, err = self._copy_dir(templater,
src,
dst,
actionexec=actionexec,
noempty=noempty,
ignore=ignore,
is_template=is_template,
chmod=chmod)
else:
r, err = self._copy_file(templater,
src,
dst,
actionexec=actionexec,
noempty=noempty,
ignore=ignore,
is_template=is_template,
chmod=chmod)
elif linktype == LinkTypes.LINK:
# symlink
r, err = self._link(templater,
src,
dst,
actionexec=actionexec,
is_template=is_template)
elif linktype == LinkTypes.LINK_CHILDREN:
# symlink direct children
if not isdir:
if self.debug:
msg = 'symlink children of {} to {}'
self.log.dbg(msg.format(src, dst))
err = 'source dotfile is not a directory: {}'.format(src)
r = False
else:
r, err = self._link_children(templater,
src,
dst,
actionexec=actionexec,
is_template=is_template,
ignore=ignore)
if self.debug:
self.log.dbg('before chmod: {} err:{}'.format(r, err))
if self.dry:
return self._log_install(r, err)
# handle chmod
# - on success (r, not err)
# - no change (not r, not err)
# but not when
# - error (not r, err)
# - aborted (not r, err)
if (r or (not r and not err)):
if not chmod:
chmod = utils.get_file_perm(src)
dstperms = utils.get_file_perm(dst)
if dstperms != chmod:
# apply mode
msg = 'chmod {} to {:o}'.format(dst, chmod)
if not force_chmod and self.safe and not self.log.ask(msg):
r = False
err = 'aborted'
else:
if not self.comparing:
self.log.sub('chmod {} to {:o}'.format(dst, chmod))
if utils.chmod(dst, chmod, debug=self.debug):
r = True
else:
r = False
err = 'chmod failed'
return self._log_install(r, err)
def install_to_temp(self,
templater,
tmpdir,
src,
dst,
is_template=True,
chmod=None,
ignore=[]):
"""
install a dotfile to a tempdir
@templater: the templater object
@tmpdir: where to install
@src: dotfile source path in dotpath
@dst: dotfile destination path in the FS
@is_template: this dotfile is a template
@chmod: rights to apply if any
@ignore: patterns to ignore
return
- success, error-if-any, dotfile-installed-path
"""
if self.debug:
self.log.dbg('tmp install {} (defined dst: {})'.format(src, dst))
src, dst, cont, err = self._check_paths(src, dst, chmod)
if not cont:
return self._log_install(cont, err)
ret = False
tmpdst = ''
# save flags
self.comparing = True
drysaved = self.dry
self.dry = False
diffsaved = self.diff
self.diff = False
createsaved = self.create
self.create = True
totemp = self.totemp
self.totemp = None
# install the dotfile to a temp directory
tmpdst = self._pivot_path(dst, tmpdir)
ret, err = self.install(templater,
src,
tmpdst,
LinkTypes.NOLINK,
is_template=is_template,
chmod=chmod,
ignore=ignore)
if self.debug:
if ret:
self.log.dbg('tmp installed in {}'.format(tmpdst))
# restore flags
self.dry = drysaved
self.diff = diffsaved
self.create = createsaved
self.comparing = False
self.totemp = totemp
return ret, err, tmpdst
########################################################
# low level accessors for public methods
########################################################
def _link(self, templater, src, dst, actionexec=None, is_template=True):
"""
install link:link
return
- True, None : success
- False, error_msg : error
- False, None : ignored
- False, 'aborted' : user aborted
"""
if is_template:
if self.debug:
self.log.dbg('is a template')
self.log.dbg('install to {}'.format(self.workdir))
tmp = self._pivot_path(dst, self.workdir, striphome=True)
r, err = self.install(templater,
src,
tmp,
LinkTypes.NOLINK,
actionexec=actionexec,
is_template=is_template)
if not r and not os.path.exists(tmp):
return r, err
src = tmp
r, err = self._symlink(src, dst, actionexec=actionexec)
return r, err
def _link_children(self,
templater,
src,
dst,
actionexec=None,
is_template=True,
ignore=[]):
"""
install link:link_children
return
- True, None : success
- False, error_msg : error
- False, None : ignored
- False, 'aborted' : user aborted
"""
parent = os.path.join(self.base, src)
if not os.path.lexists(dst):
if self.dry:
self.log.dry('would create directory "{}"'.format(dst))
else:
if not self.comparing:
self.log.sub('creating directory "{}"'.format(dst))
self._create_dirs(dst)
if os.path.isfile(dst):
msg = ''.join([
'Remove regular file {} and ',
'replace with empty directory?',
]).format(dst)
if self.safe and not self.log.ask(msg):
return False, 'aborted'
os.unlink(dst)
self._create_dirs(dst)
children = os.listdir(parent)
srcs = [
os.path.normpath(os.path.join(parent, child)) for child in children
]
dsts = [
os.path.normpath(os.path.join(dst, child)) for child in children
]
installed = 0
for i in range(len(children)):
subsrc = srcs[i]
subdst = dsts[i]
if utils.must_ignore([subsrc, subdst], ignore, debug=self.debug):
if self.debug:
self.log.dbg(
'ignoring install of {} to {}'.format(src, dst), )
continue
if self.debug:
self.log.dbg('symlink child {} to {}'.format(subsrc, subdst))
if is_template:
if self.debug:
self.log.dbg('child is a template')
self.log.dbg('install to {} and symlink'.format(
self.workdir))
tmp = self._pivot_path(subdst, self.workdir, striphome=True)
r, e = self.install(templater,
subsrc,
tmp,
LinkTypes.NOLINK,
actionexec=actionexec,
is_template=is_template)
if not r and e and not os.path.exists(tmp):
continue
subsrc = tmp
ret, err = self._symlink(subsrc, subdst, actionexec=actionexec)
if ret:
installed += 1
# void actionexec if dotfile installed
# to prevent from running actions multiple times
actionexec = None
else:
if err:
return ret, err
return installed > 0, None
########################################################
# file operations
########################################################
def _symlink(self, src, dst, actionexec=None):
"""
set src as a link target of dst
return
- True, None : success
- False, error_msg : error
- False, None : ignored
- False, 'aborted' : user aborted
"""
overwrite = not self.safe
if os.path.lexists(dst):
if os.path.realpath(dst) == os.path.realpath(src):
msg = 'ignoring "{}", link already exists'.format(dst)
if self.debug:
self.log.dbg(msg)
return False, None
if self.dry:
self.log.dry('would remove {} and link to {}'.format(dst, src))
return True, None
if self.showdiff:
self._show_diff_before_write(src, dst)
msg = 'Remove "{}" for link creation?'.format(dst)
if self.safe and not self.log.ask(msg):
return False, 'aborted'
overwrite = True
try:
utils.removepath(dst)
except OSError as e:
err = 'something went wrong with {}: {}'.format(src, e)
return False, err
if self.dry:
self.log.dry('would link {} to {}'.format(dst, src))
return True, None
base = os.path.dirname(dst)
if not self._create_dirs(base):
err = 'error creating directory for {}'.format(dst)
return False, err
r, e = self._exec_pre_actions(actionexec)
if not r:
return False, e
# re-check in case action created the file
if os.path.lexists(dst):
msg = 'Remove "{}" for link creation?'.format(dst)
if self.safe and not overwrite and not self.log.ask(msg):
return False, 'aborted'
try:
utils.removepath(dst)
except OSError as e:
err = 'something went wrong with {}: {}'.format(src, e)
return False, err
os.symlink(src, dst)
if not self.comparing:
self.log.sub('linked {} to {}'.format(dst, src))
return True, None
def _copy_file(self,
templater,
src,
dst,
actionexec=None,
noempty=False,
ignore=[],
is_template=True,
chmod=None):
"""
install src to dst when is a file
return
- True, None : success
- False, error_msg : error
- False, None : ignored
- False, 'aborted' : user aborted
"""
if self.debug:
self.log.dbg('deploy file: {}'.format(src))
self.log.dbg('ignore empty: {}'.format(noempty))
self.log.dbg('ignore pattern: {}'.format(ignore))
self.log.dbg('is_template: {}'.format(is_template))
self.log.dbg('no empty: {}'.format(noempty))
# check no loop
if utils.samefile(src, dst):
err = 'dotfile points to itself: {}'.format(dst)
return False, err
if utils.must_ignore([src, dst], ignore, debug=self.debug):
if self.debug:
self.log.dbg('ignoring install of {} to {}'.format(src, dst))
return False, None
if utils.samefile(src, dst):
# loop
err = 'dotfile points to itself: {}'.format(dst)
return False, err
if not os.path.exists(src):
err = 'source dotfile does not exist: {}'.format(src)
return False, err
# handle the file
content = None
if is_template:
# template the file
saved = templater.add_tmp_vars(self._get_tmp_file_vars(src, dst))
try:
content = templater.generate(src)
except UndefinedException as e:
return False, str(e)
finally:
templater.restore_vars(saved)
# test is empty
if noempty and utils.content_empty(content):
if self.debug:
self.log.dbg('ignoring empty template: {}'.format(src))
return False, None
if content is None:
err = 'empty template {}'.format(src)
return False, err
# write the file
ret, err = self._write(src,
dst,
content=content,
actionexec=actionexec,
chmod=chmod)
if ret and not err:
if not self.dry and not self.comparing:
self.log.sub('install {} to {}'.format(src, dst))
return ret, err
def _copy_dir(self,
templater,
src,
dst,
actionexec=None,
noempty=False,
ignore=[],
is_template=True,
chmod=None):
"""
install src to dst when is a directory
return
- True, None : success
- False, error_msg : error
- False, None : ignored
- False, 'aborted' : user aborted
"""
if self.debug:
self.log.dbg('deploy dir {}'.format(src))
# default to nothing installed and no error
ret = False, None
# create the directory anyway
if not self._create_dirs(dst):
err = 'creating directory for {}'.format(dst)
return False, err
# handle all files in dir
for entry in os.listdir(src):
f = os.path.join(src, entry)
if self.debug:
self.log.dbg('deploy sub from {}: {}'.format(dst, entry))
if not os.path.isdir(f):
# is file
res, err = self._copy_file(templater,
f,
os.path.join(dst, entry),
actionexec=actionexec,
noempty=noempty,
ignore=ignore,
is_template=is_template,
chmod=None)
if not res and err:
# error occured
ret = res, err
break
elif res:
# something got installed
ret = True, None
else:
# is directory
res, err = self._copy_dir(templater,
f,
os.path.join(dst, entry),
actionexec=actionexec,
noempty=noempty,
ignore=ignore,
is_template=is_template,
chmod=None)
if not res and err:
# error occured
ret = res, err
break
elif res:
# something got installed
ret = True, None
return ret
def _write(self, src, dst, content=None, actionexec=None, chmod=None):
"""
copy dotfile / write content to file
return
- True, None : success
- False, error_msg : error
- False, None : ignored
- False, 'aborted' : user aborted
"""
overwrite = not self.safe
if self.dry:
self.log.dry('would install {}'.format(dst))
return True, None
if os.path.lexists(dst):
try:
os.stat(dst)
except OSError as e:
if e.errno == errno.ENOENT:
# broken symlink
err = 'broken symlink {}'.format(dst)
return False, err
if self.diff:
if not self._is_different(src, dst, content=content):
if self.debug:
self.log.dbg('{} is the same'.format(dst))
return False, None
if self.safe:
if self.debug:
self.log.dbg('change detected for {}'.format(dst))
if self.showdiff:
# get diff
self._show_diff_before_write(src, dst, content=content)
if not self.log.ask('Overwrite \"{}\"'.format(dst)):
return False, 'aborted'
overwrite = True
if self.backup and os.path.lexists(dst):
self._backup(dst)
base = os.path.dirname(dst)
if not self._create_dirs(base):
err = 'creating directory for {}'.format(dst)
return False, err
r, e = self._exec_pre_actions(actionexec)
if not r:
return False, e
if self.debug:
self.log.dbg('install file to \"{}\"'.format(dst))
# re-check in case action created the file
if self.safe and not overwrite and os.path.lexists(dst):
if not self.log.ask('Overwrite \"{}\"'.format(dst)):
self.log.warn('ignoring {}'.format(dst))
return False, 'aborted'
if content:
# write content the file
try:
with open(dst, 'wb') as f:
f.write(content)
shutil.copymode(src, dst)
except NotADirectoryError as e:
err = 'opening dest file: {}'.format(e)
return False, err
except Exception as e:
return False, str(e)
else:
# copy file
try:
shutil.copyfile(src, dst)
shutil.copymode(src, dst)
except Exception as e:
return False, str(e)
return True, None
########################################################
# helpers
########################################################
def _get_tmp_file_vars(self, src, dst):
tmp = {}
tmp['_dotfile_sub_abs_src'] = src
tmp['_dotfile_sub_abs_dst'] = dst
return tmp
def _is_different(self, src, dst, content=None):
"""
returns True if file is different and
needs to be installed
"""
# check file content
if content:
tmp = utils.write_to_tmpfile(content)
src = tmp
r = utils.fastdiff(src, dst)
if r:
if self.debug:
self.log.dbg('content differ')
return r
def _show_diff_before_write(self, src, dst, content=None):
"""
diff before writing
using a temp file if content is not None
returns diff string ('' if same)
"""
tmp = None
if content:
tmp = utils.write_to_tmpfile(content)
src = tmp
diff = utils.diff(modified=src, original=dst, diff_cmd=self.diff_cmd)
if tmp:
utils.removepath(tmp, logger=self.log)
if diff:
self._print_diff(src, dst, diff)
return diff
def _print_diff(self, src, dst, diff):
"""show diff to user"""
self.log.log('diff \"{}\" VS \"{}\"'.format(dst, src))
self.log.emph(diff)
def _create_dirs(self, directory):
"""mkdir -p <directory>"""
if not self.create and not os.path.exists(directory):
if self.debug:
self.log.dbg('no mkdir as \"create\" set to false in config')
return False
if os.path.exists(directory):
return True
if self.dry:
self.log.dry('would mkdir -p {}'.format(directory))
return True
if self.debug:
self.log.dbg('mkdir -p {}'.format(directory))
if not self.comparing:
self.log.sub('create directory {}'.format(directory))
os.makedirs(directory, exist_ok=True)
return os.path.exists(directory)
def _backup(self, path):
"""backup file pointed by path"""
if self.dry:
return
dst = path.rstrip(os.sep) + self.backup_suffix
self.log.log('backup {} to {}'.format(path, dst))
os.rename(path, dst)
def _pivot_path(self, path, newdir, striphome=False):
"""change path to be under newdir"""
if self.debug:
self.log.dbg('pivot new dir: \"{}\"'.format(newdir))
self.log.dbg('strip home: {}'.format(striphome))
if striphome:
path = utils.strip_home(path)
sub = path.lstrip(os.sep)
new = os.path.join(newdir, sub)
if self.debug:
self.log.dbg('pivot \"{}\" to \"{}\"'.format(path, new))
return new
def _exec_pre_actions(self, actionexec):
"""execute action executor"""
if self.action_executed:
return True, None
if not actionexec:
return True, None
ret, err = actionexec()
self.action_executed = True
return ret, err
def _log_install(self, boolean, err):
"""log installation process"""
if not self.debug:
return boolean, err
if boolean:
self.log.dbg('install: SUCCESS')
else:
if err:
self.log.dbg('install: ERROR: {}'.format(err))
else:
self.log.dbg('install: IGNORED')
return boolean, err
def _check_paths(self, src, dst, chmod):
"""
check and normalize param
returns <src>, <dst>, <continue>, <error>
"""
# check both path are valid
if not dst or not src:
err = 'empty dst or src for {}'.format(src)
if self.debug:
self.log.dbg(err)
return None, None, False, err
# normalize src and dst
src = os.path.expanduser(src)
src = os.path.normpath(src)
dst = os.path.expanduser(dst)
dst = os.path.normpath(dst)
return src, dst, True, None
class Templategen:
def __init__(self,
base='.',
variables={},
func_file=[],
filter_file=[],
debug=False):
"""constructor
@base: directory path where to search for templates
@variables: dictionary of variables for templates
@func_file: file path to load functions from
@filter_file: file path to load filters from
@debug: enable debug
"""
self.base = base.rstrip(os.sep)
self.debug = debug
self.log = Logger()
self.variables = {}
loader1 = FileSystemLoader(self.base)
loader2 = FunctionLoader(self._template_loader)
loader = ChoiceLoader([loader1, loader2])
self.env = Environment(loader=loader,
trim_blocks=True,
lstrip_blocks=True,
keep_trailing_newline=True,
block_start_string=BLOCK_START,
block_end_string=BLOCK_END,
variable_start_string=VAR_START,
variable_end_string=VAR_END,
comment_start_string=COMMENT_START,
comment_end_string=COMMENT_END,
undefined=StrictUndefined)
# adding variables
self.variables['env'] = os.environ
if variables:
self.variables.update(variables)
# adding header method
self.env.globals['header'] = self._header
# adding helper methods
if self.debug:
self.log.dbg('load global functions:')
self._load_funcs_to_dic(jhelpers, self.env.globals)
if func_file:
for f in func_file:
if self.debug:
self.log.dbg('load custom functions from {}'.format(f))
self._load_path_to_dic(f, self.env.globals)
if filter_file:
for f in filter_file:
if self.debug:
self.log.dbg('load custom filters from {}'.format(f))
self._load_path_to_dic(f, self.env.filters)
if self.debug:
self._debug_dict('template additional variables', variables)
def generate(self, src):
"""
render template from path
may raise a UndefinedException
in case a variable is undefined
"""
if not os.path.exists(src):
return ''
try:
return self._handle_file(src)
except UndefinedError as e:
err = 'undefined variable: {}'.format(e.message)
raise UndefinedException(err)
def generate_string(self, string):
"""
render template from string
may raise a UndefinedException
in case a variable is undefined
"""
if not string:
return ''
try:
return self.env.from_string(string).render(self.variables)
except UndefinedError as e:
err = 'undefined variable: {}'.format(e.message)
raise UndefinedException(err)
def add_tmp_vars(self, newvars={}):
"""add vars to the globals, make sure to call restore_vars"""
saved_variables = self.variables.copy()
if not newvars:
return saved_variables
self.variables.update(newvars)
return saved_variables
def restore_vars(self, saved_globals):
"""restore globals from add_tmp_vars"""
self.variables = saved_globals.copy()
def update_variables(self, variables):
"""update variables"""
self.variables.update(variables)
def _load_path_to_dic(self, path, dic):
mod = utils.get_module_from_path(path)
if not mod:
self.log.warn('cannot load module \"{}\"'.format(path))
return
self._load_funcs_to_dic(mod, dic)
def _load_funcs_to_dic(self, mod, dic):
"""dynamically load functions from module to dic"""
if not mod or not dic:
return
funcs = utils.get_module_functions(mod)
for name, func in funcs:
if self.debug:
self.log.dbg('load function \"{}\"'.format(name))
dic[name] = func
def _header(self, prepend=''):
"""add a comment usually in the header of a dotfile"""
return '{}{}'.format(prepend, utils.header())
def _handle_file(self, src):
"""generate the file content from template"""
try:
import magic
filetype = magic.from_file(src, mime=True)
if self.debug:
self.log.dbg('using \"magic\" for filetype identification')
except ImportError:
# fallback
_, filetype = utils.run(['file', '-b', '--mime-type', src],
debug=self.debug)
if self.debug:
self.log.dbg('using \"file\" for filetype identification')
filetype = filetype.strip()
istext = self._is_text(filetype)
if self.debug:
self.log.dbg('filetype \"{}\": {}'.format(src, filetype))
if self.debug:
self.log.dbg('is text \"{}\": {}'.format(src, istext))
if not istext:
return self._handle_bin_file(src)
return self._handle_text_file(src)
def _is_text(self, fileoutput):
"""return if `file -b` output is ascii text"""
out = fileoutput.lower()
if out.startswith('text'):
return True
if 'empty' in out:
return True
if 'json' in out:
return True
return False
def _template_loader(self, relpath):
"""manually load template when outside of base"""
path = os.path.join(self.base, relpath)
path = os.path.normpath(path)
if not os.path.exists(path):
raise TemplateNotFound(path)
with open(path, 'r') as f:
content = f.read()
return content
def _handle_text_file(self, src):
"""write text to file"""
template_rel_path = os.path.relpath(src, self.base)
try:
template = self.env.get_template(template_rel_path)
content = template.render(self.variables)
except UnicodeDecodeError:
data = self._read_bad_encoded_text(src)
content = self.generate_string(data)
return content.encode('utf-8')
def _handle_bin_file(self, src):
"""write binary to file"""
# this is dirty
if not src.startswith(self.base):
src = os.path.join(self.base, src)
with open(src, 'rb') as f:
content = f.read()
return content
def _read_bad_encoded_text(self, path):
"""decode non utf-8 data"""
with open(path, 'rb') as f:
data = f.read()
return data.decode('utf-8', 'replace')
@staticmethod
def is_template(path, ignore=[]):
"""recursively check if any file is a template within path"""
path = os.path.expanduser(path)
if utils.must_ignore([path], ignore, debug=False):
return False
if not os.path.exists(path):
return False
if os.path.isfile(path):
# is file
return Templategen._is_template(path, ignore=ignore)
for entry in os.listdir(path):
fpath = os.path.join(path, entry)
if not os.path.isfile(fpath):
# recursively explore directory
if Templategen.is_template(fpath, ignore=ignore):
return True
else:
# check if file is a template
if Templategen._is_template(fpath, ignore=ignore):
return True
return False
@staticmethod
def var_is_template(string):
"""check if variable contains template(s)"""
return VAR_START in str(string)
@staticmethod
def _is_template(path, ignore):
"""test if file pointed by path is a template"""
if utils.must_ignore([path], ignore, debug=False):
return False
if not os.path.isfile(path):
return False
if os.stat(path).st_size == 0:
return False
markers = [BLOCK_START, VAR_START, COMMENT_START]
patterns = [re.compile(marker.encode()) for marker in markers]
try:
with io.open(path, "r", encoding="utf-8") as f:
m = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
for pattern in patterns:
if pattern.search(m):
return True
except UnicodeDecodeError:
# is binary so surely no template
return False
return False
def _debug_dict(self, title, elems):
"""pretty print dict"""
if not self.debug:
return
self.log.dbg('{}:'.format(title))
if not elems:
return
for k, v in elems.items():
self.log.dbg(' - \"{}\": {}'.format(k, v))
class Installer:
def __init__(self,
base='.',
create=True,
backup=True,
dry=False,
safe=False,
workdir='~/.config/dotdrop',
debug=False,
diff=True,
totemp=None,
showdiff=False,
backup_suffix='.dotdropbak'):
"""constructor
@base: directory path where to search for templates
@create: create directory hierarchy if missing when installing
@backup: backup existing dotfile when installing
@dry: just simulate
@safe: ask for any overwrite
@workdir: where to install template before symlinking
@debug: enable debug
@diff: diff when installing if True
@totemp: deploy to this path instead of dotfile dst if not None
@showdiff: show the diff before overwriting (or asking for)
@backup_suffix: suffix for dotfile backup file
"""
self.create = create
self.backup = backup
self.dry = dry
self.safe = safe
self.workdir = os.path.expanduser(workdir)
self.base = base
self.debug = debug
self.diff = diff
self.totemp = totemp
self.showdiff = showdiff
self.backup_suffix = backup_suffix
self.comparing = False
self.action_executed = False
self.log = Logger()
def install(self, templater, src, dst, actions=[], noempty=False):
"""install the src to dst using a template"""
if self.debug:
self.log.dbg('install {} to {}'.format(src, dst))
self.action_executed = False
src = os.path.join(self.base, os.path.expanduser(src))
if not os.path.exists(src):
self.log.err('source dotfile does not exist: {}'.format(src))
return []
dst = os.path.expanduser(dst)
if self.totemp:
dst = self._pivot_path(dst, self.totemp)
if utils.samefile(src, dst):
# symlink loop
self.log.err('dotfile points to itself: {}'.format(dst))
return []
isdir = os.path.isdir(src)
if self.debug:
self.log.dbg('install {} to {}'.format(src, dst))
self.log.dbg('is \"{}\" a directory: {}'.format(src, isdir))
if isdir:
return self._handle_dir(templater,
src,
dst,
actions=actions,
noempty=noempty)
return self._handle_file(templater,
src,
dst,
actions=actions,
noempty=noempty)
def link(self, templater, src, dst, actions=[]):
"""set src as the link target of dst"""
if self.debug:
self.log.dbg('link {} to {}'.format(src, dst))
self.action_executed = False
src = os.path.normpath(os.path.join(self.base,
os.path.expanduser(src)))
if not os.path.exists(src):
self.log.err('source dotfile does not exist: {}'.format(src))
return []
dst = os.path.normpath(os.path.expanduser(dst))
if self.totemp:
# ignore actions
return self.install(templater, src, dst, actions=[])
if Templategen.is_template(src):
if self.debug:
self.log.dbg('dotfile is a template')
self.log.dbg('install to {} and symlink'.format(self.workdir))
tmp = self._pivot_path(dst, self.workdir, striphome=True)
i = self.install(templater, src, tmp, actions=actions)
if not i and not os.path.exists(tmp):
return []
src = tmp
return self._link(src, dst, actions=actions)
def link_children(self, templater, src, dst, actions=[]):
"""link all dotfiles in a given directory"""
if self.debug:
self.log.dbg('link_children {} to {}'.format(src, dst))
self.action_executed = False
parent = os.path.join(self.base, os.path.expanduser(src))
# Fail if source doesn't exist
if not os.path.exists(parent):
self.log.err('source dotfile does not exist: {}'.format(parent))
return []
# Fail if source not a directory
if not os.path.isdir(parent):
if self.debug:
self.log.dbg('symlink children of {} to {}'.format(src, dst))
self.log.err(
'source dotfile is not a directory: {}'.format(parent))
return []
dst = os.path.normpath(os.path.expanduser(dst))
if not os.path.lexists(dst):
self.log.sub('creating directory "{}"'.format(dst))
os.makedirs(dst)
if os.path.isfile(dst):
msg = ''.join([
'Remove regular file {} and ',
'replace with empty directory?',
]).format(dst)
if self.safe and not self.log.ask(msg):
msg = 'ignoring "{}", nothing installed'
self.log.warn(msg.format(dst))
return []
os.unlink(dst)
os.mkdir(dst)
children = os.listdir(parent)
srcs = [
os.path.normpath(os.path.join(parent, child)) for child in children
]
dsts = [
os.path.normpath(os.path.join(dst, child)) for child in children
]
for i in range(len(children)):
src = srcs[i]
dst = dsts[i]
if self.debug:
self.log.dbg('symlink child {} to {}'.format(src, dst))
if Templategen.is_template(src):
if self.debug:
self.log.dbg('dotfile is a template')
self.log.dbg('install to {} and symlink'.format(
self.workdir))
tmp = self._pivot_path(dst, self.workdir, striphome=True)
i = self.install(templater, src, tmp, actions=actions)
if not i and not os.path.exists(tmp):
continue
src = tmp
result = self._link(src, dst, actions)
# Empty actions if dotfile installed
# This prevents from running actions multiple times
if len(result):
actions = []
return (src, dst)
def _link(self, src, dst, actions=[]):
"""set src as a link target of dst"""
overwrite = not self.safe
if os.path.lexists(dst):
if os.path.realpath(dst) == os.path.realpath(src):
if self.debug:
self.log.dbg('ignoring "{}", link exists'.format(dst))
return []
if self.dry:
self.log.dry('would remove {} and link to {}'.format(dst, src))
return []
msg = 'Remove "{}" for link creation?'.format(dst)
if self.safe and not self.log.ask(msg):
msg = 'ignoring "{}", link was not created'
self.log.warn(msg.format(dst))
return []
overwrite = True
try:
utils.remove(dst)
except OSError as e:
self.log.err('something went wrong with {}: {}'.format(src, e))
return []
if self.dry:
self.log.dry('would link {} to {}'.format(dst, src))
return []
base = os.path.dirname(dst)
if not self._create_dirs(base):
self.log.err('creating directory for {}'.format(dst))
return []
self._exec_pre_actions(actions)
# re-check in case action created the file
if os.path.lexists(dst):
msg = 'Remove "{}" for link creation?'.format(dst)
if self.safe and not overwrite and not self.log.ask(msg):
msg = 'ignoring "{}", link was not created'
self.log.warn(msg.format(dst))
return []
try:
utils.remove(dst)
except OSError as e:
self.log.err('something went wrong with {}: {}'.format(src, e))
return []
os.symlink(src, dst)
self.log.sub('linked {} to {}'.format(dst, src))
return [(src, dst)]
def _handle_file(self, templater, src, dst, actions=[], noempty=False):
"""install src to dst when is a file"""
if self.debug:
self.log.dbg('generate template for {}'.format(src))
self.log.dbg('ignore empty: {}'.format(noempty))
if utils.samefile(src, dst):
# symlink loop
self.log.err('dotfile points to itself: {}'.format(dst))
return []
content = templater.generate(src)
if noempty and utils.content_empty(content):
self.log.warn('ignoring empty template: {}'.format(src))
return []
if content is None:
self.log.err('generate from template {}'.format(src))
return []
if not os.path.exists(src):
self.log.err('source dotfile does not exist: {}'.format(src))
return []
st = os.stat(src)
ret = self._write(src, dst, content, st.st_mode, actions=actions)
if ret < 0:
self.log.err('installing {} to {}'.format(src, dst))
return []
if ret > 0:
if self.debug:
self.log.dbg('ignoring {}'.format(dst))
return []
if ret == 0:
if not self.dry and not self.comparing:
self.log.sub('copied {} to {}'.format(src, dst))
return [(src, dst)]
return []
def _handle_dir(self, templater, src, dst, actions=[], noempty=False):
"""install src to dst when is a directory"""
if self.debug:
self.log.dbg('install dir {}'.format(src))
self.log.dbg('ignore empty: {}'.format(noempty))
ret = []
if not self._create_dirs(dst):
return []
# handle all files in dir
for entry in os.listdir(src):
f = os.path.join(src, entry)
if not os.path.isdir(f):
res = self._handle_file(templater,
f,
os.path.join(dst, entry),
actions=actions,
noempty=noempty)
ret.extend(res)
else:
res = self._handle_dir(templater,
f,
os.path.join(dst, entry),
actions=actions,
noempty=noempty)
ret.extend(res)
return ret
def _fake_diff(self, dst, content):
"""fake diff by comparing file content with content"""
cur = ''
with open(dst, 'br') as f:
cur = f.read()
return cur == content
def _write(self, src, dst, content, rights, actions=[]):
"""write content to file
return 0 for success,
1 when already exists
-1 when error"""
overwrite = not self.safe
if self.dry:
self.log.dry('would install {}'.format(dst))
return 0
if os.path.lexists(dst):
samerights = False
try:
samerights = os.stat(dst).st_mode == rights
except OSError as e:
if e.errno == errno.ENOENT:
# broken symlink
self.log.err('broken symlink {}'.format(dst))
return -1
if self.diff and self._fake_diff(dst, content) and samerights:
if self.debug:
self.log.dbg('{} is the same'.format(dst))
return 1
if self.safe:
if self.debug:
self.log.dbg('change detected for {}'.format(dst))
if self.showdiff:
self._diff_before_write(src, dst, content)
if not self.log.ask('Overwrite \"{}\"'.format(dst)):
self.log.warn('ignoring {}'.format(dst))
return 1
overwrite = True
if self.backup and os.path.lexists(dst):
self._backup(dst)
base = os.path.dirname(dst)
if not self._create_dirs(base):
self.log.err('creating directory for {}'.format(dst))
return -1
if self.debug:
self.log.dbg('write content to {}'.format(dst))
self._exec_pre_actions(actions)
# re-check in case action created the file
if self.safe and not overwrite and os.path.lexists(dst):
if not self.log.ask('Overwrite \"{}\"'.format(dst)):
self.log.warn('ignoring {}'.format(dst))
return 1
# write the file
try:
with open(dst, 'wb') as f:
f.write(content)
except NotADirectoryError as e:
self.log.err('opening dest file: {}'.format(e))
return -1
os.chmod(dst, rights)
return 0
def _diff_before_write(self, src, dst, src_content):
"""diff before writing when using --showdiff - not efficient"""
# create tmp to diff for templates
tmpfile = utils.get_tmpfile()
with open(tmpfile, 'wb') as f:
f.write(src_content)
comp = Comparator(debug=self.debug)
diff = comp.compare(tmpfile, dst)
# fake the output for readability
self.log.log('diff \"{}\" VS \"{}\"'.format(src, dst))
self.log.emph(diff)
if tmpfile:
utils.remove(tmpfile)
def _create_dirs(self, directory):
"""mkdir -p <directory>"""
if not self.create and not os.path.exists(directory):
return False
if os.path.exists(directory):
return True
if self.dry:
self.log.dry('would mkdir -p {}'.format(directory))
return True
if self.debug:
self.log.dbg('mkdir -p {}'.format(directory))
os.makedirs(directory)
return os.path.exists(directory)
def _backup(self, path):
"""backup file pointed by path"""
if self.dry:
return
dst = path.rstrip(os.sep) + self.backup_suffix
self.log.log('backup {} to {}'.format(path, dst))
os.rename(path, dst)
def _pivot_path(self, path, newdir, striphome=False):
"""change path to be under newdir"""
if self.debug:
self.log.dbg('pivot new dir: \"{}\"'.format(newdir))
self.log.dbg('strip home: {}'.format(striphome))
if striphome:
path = utils.strip_home(path)
sub = path.lstrip(os.sep)
new = os.path.join(newdir, sub)
if self.debug:
self.log.dbg('pivot \"{}\" to \"{}\"'.format(path, new))
return new
def _exec_pre_actions(self, actions):
"""execute pre-actions if any"""
if self.action_executed:
return
for action in actions:
if self.dry:
self.log.dry('would execute action: {}'.format(action))
else:
if self.debug:
self.log.dbg('executing pre action {}'.format(action))
action.execute()
self.action_executed = True
def _install_to_temp(self, templater, src, dst, tmpdir):
"""install a dotfile to a tempdir"""
tmpdst = self._pivot_path(dst, tmpdir)
return self.install(templater, src, tmpdst), tmpdst
def install_to_temp(self, templater, tmpdir, src, dst):
"""install a dotfile to a tempdir"""
ret = False
tmpdst = ''
# save some flags while comparing
self.comparing = True
drysaved = self.dry
self.dry = False
diffsaved = self.diff
self.diff = False
createsaved = self.create
self.create = True
# normalize src and dst
src = os.path.expanduser(src)
dst = os.path.expanduser(dst)
if self.debug:
self.log.dbg('tmp install {} to {}'.format(src, dst))
# install the dotfile to a temp directory for comparing
ret, tmpdst = self._install_to_temp(templater, src, dst, tmpdir)
if self.debug:
self.log.dbg('tmp installed in {}'.format(tmpdst))
# reset flags
self.dry = drysaved
self.diff = diffsaved
self.comparing = False
self.create = createsaved
return ret, tmpdst
