class Installer: BACKUP_SUFFIX = '.dotdropbak' def __init__(self, base='.', create=True, backup=True, dry=False, safe=False, debug=False, diff=True): self.create = create self.backup = backup self.dry = dry self.safe = safe self.base = base self.debug = debug self.diff = diff self.comparing = False self.log = Logger(debug=self.debug) def install(self, templater, profile, src, dst): """install the src to dst using a template""" src = os.path.join(self.base, os.path.expanduser(src)) dst = os.path.join(self.base, os.path.expanduser(dst)) self.log.dbg('install {} to {}'.format(src, dst)) if os.path.isdir(src): return self._handle_dir(templater, profile, src, dst) return self._handle_file(templater, profile, src, dst) def link(self, src, dst): """set src as the link target of dst""" src = os.path.join(self.base, os.path.expanduser(src)) dst = os.path.join(self.base, os.path.expanduser(dst)) if os.path.exists(dst): if os.path.realpath(dst) == os.path.realpath(src): self.log.sub('ignoring "{}", link exists'.format(dst)) return [] if self.dry: self.log.dry('would remove {} and link to {}'.format(dst, src)) return [] msg = 'Remove "{}" for link creation?'.format(dst) if self.safe and not self.log.ask(msg): msg = 'ignoring "{}", link was not created' self.log.warn(msg.format(dst)) return [] try: utils.remove(dst) except OSError: self.log.err('something went wrong with {}'.format(src)) return [] if self.dry: self.log.dry('would link {} to {}'.format(dst, src)) return [] base = os.path.dirname(dst) if not self._create_dirs(base): self.log.err('creating directory for \"{}\"'.format(dst)) return [] os.symlink(src, dst) self.log.sub('linked {} to {}'.format(dst, src)) return [(src, dst)] def _handle_file(self, templater, profile, src, dst): """install src to dst when is a file""" self.log.dbg('generate template for {}'.format(src)) content = templater.generate(src, profile) if content is None: self.log.err('generate from template \"{}\"'.format(src)) return [] if not os.path.exists(src): self.log.err('source dotfile does not exist: \"{}\"'.format(src)) return [] st = os.stat(src) ret = self._write(dst, content, st.st_mode) if ret < 0: self.log.err('installing \"{}\" to \"{}\"'.format(src, dst)) return [] if ret > 0: self.log.dbg('ignoring \"{}\", same content'.format(dst)) return [] if ret == 0: if not self.dry and not self.comparing: self.log.sub('copied \"{}\" to \"{}\"'.format(src, dst)) return [(src, dst)] return [] def _handle_dir(self, templater, profile, src, dst): """install src to dst when is a directory""" ret = [] for entry in os.listdir(src): f = os.path.join(src, entry) if not os.path.isdir(f): res = self._handle_file(templater, profile, f, os.path.join(dst, entry)) ret.extend(res) else: res = self._handle_dir(templater, profile, f, os.path.join(dst, entry)) ret.extend(res) return ret def _fake_diff(self, dst, content): """fake diff by comparing file content with content""" cur = '' with open(dst, 'br') as f: cur = f.read() return cur == content def _write(self, dst, content, rights): """write content to file return 0 for success, 1 when already exists -1 when error""" if self.dry: self.log.dry('would install {}'.format(dst)) return 0 if os.path.exists(dst): samerights = os.stat(dst).st_mode == rights if self.diff and self._fake_diff(dst, content) and samerights: self.log.dbg('{} is the same'.format(dst)) return 1 if self.safe and not self.log.ask('Overwrite \"{}\"'.format(dst)): self.log.warn('ignoring \"{}\", already present'.format(dst)) return 1 if self.backup and os.path.exists(dst): self._backup(dst) base = os.path.dirname(dst) if not self._create_dirs(base): self.log.err('creating directory for \"{}\"'.format(dst)) return -1 self.log.dbg('write content to {}'.format(dst)) try: with open(dst, 'wb') as f: f.write(content) except NotADirectoryError as e: self.log.err('opening dest file: {}'.format(e)) return -1 os.chmod(dst, rights) return 0 def _create_dirs(self, directory): """mkdir -p <directory>""" if not self.create and not os.path.exists(directory): return False if os.path.exists(directory): return True self.log.dbg('mkdir -p {}'.format(directory)) os.makedirs(directory) return os.path.exists(directory) def _backup(self, path): """backup file pointed by path""" if self.dry: return dst = path.rstrip(os.sep) + self.BACKUP_SUFFIX self.log.log('backup {} to {}'.format(path, dst)) os.rename(path, dst) def _install_to_temp(self, templater, profile, src, dst, tmpdir): """install a dotfile to a tempdir for comparing""" sub = dst if dst[0] == os.sep: sub = dst[1:] tmpdst = os.path.join(tmpdir, sub) return self.install(templater, profile, src, tmpdst), tmpdst def compare(self, templater, tmpdir, profile, src, dst, opts=''): """compare a temporary generated dotfile with the local one""" self.comparing = True retval = False, '' drysaved = self.dry self.dry = False diffsaved = self.diff self.diff = False createsaved = self.create self.create = True src = os.path.expanduser(src) dst = os.path.expanduser(dst) self.log.dbg('comparing {} and {}'.format(src, dst)) if not os.path.exists(dst): retval = False, '\"{}\" does not exist on local\n'.format(dst) else: ret, tmpdst = self._install_to_temp(templater, profile, src, dst, tmpdir) if ret: self.log.dbg('diffing {} and {}'.format(tmpdst, dst)) diff = utils.diff(tmpdst, dst, raw=False, opts=opts) if diff == '': retval = True, '' else: retval = False, diff self.dry = drysaved self.diff = diffsaved self.comparing = False self.create = createsaved return retval
class Comparator: def __init__(self, diff_cmd='', debug=False): """constructor @diff_cmd: diff command to use @debug: enable debug """ self.diff_cmd = diff_cmd self.debug = debug self.log = Logger() def compare(self, left, right, ignore=[]): """diff left (dotdrop dotfile) and right (deployed file)""" left = os.path.expanduser(left) right = os.path.expanduser(right) if self.debug: self.log.dbg('comparing {} and {}'.format(left, right)) self.log.dbg('ignore pattern(s): {}'.format(ignore)) # test type of file if os.path.isdir(left) and not os.path.isdir(right): return '\"{}\" is a dir while \"{}\" is a file\n'.format( left, right) if not os.path.isdir(left) and os.path.isdir(right): return '\"{}\" is a file while \"{}\" is a dir\n'.format( left, right) # test content if not os.path.isdir(left): if self.debug: self.log.dbg('is file') return self._comp_file(left, right, ignore) if self.debug: self.log.dbg('is directory') return self._comp_dir(left, right, ignore) def _comp_file(self, left, right, ignore): """compare a file""" if self.debug: self.log.dbg('compare file {} with {}'.format(left, right)) if must_ignore([left, right], ignore, debug=self.debug): if self.debug: self.log.dbg('ignoring diff {} and {}'.format(left, right)) return '' return self._diff(left, right) def _comp_dir(self, left, right, ignore): """compare a directory""" if self.debug: self.log.dbg('compare directory {} with {}'.format(left, right)) if not os.path.exists(right): return '' if must_ignore([left, right], ignore, debug=self.debug): if self.debug: self.log.dbg('ignoring diff {} and {}'.format(left, right)) return '' if not os.path.isdir(right): return '\"{}\" is a file\n'.format(right) if self.debug: self.log.dbg('compare {} and {}'.format(left, right)) ret = [] comp = filecmp.dircmp(left, right) # handle files only in deployed dir for i in comp.left_only: if must_ignore([os.path.join(left, i)], ignore, debug=self.debug): continue ret.append('=> \"{}\" does not exist on local\n'.format(i)) # handle files only in dotpath dir for i in comp.right_only: if must_ignore([os.path.join(right, i)], ignore, debug=self.debug): continue ret.append('=> \"{}\" does not exist in dotdrop\n'.format(i)) # same left and right but different type funny = comp.common_funny for i in funny: lfile = os.path.join(left, i) rfile = os.path.join(right, i) if must_ignore([lfile, rfile], ignore, debug=self.debug): continue short = os.path.basename(lfile) # file vs dir ret.append('=> different type: \"{}\"\n'.format(short)) # content is different funny = comp.diff_files funny.extend(comp.funny_files) funny = uniq_list(funny) for i in funny: lfile = os.path.join(left, i) rfile = os.path.join(right, i) if must_ignore([lfile, rfile], ignore, debug=self.debug): continue diff = self._diff(lfile, rfile, header=True) ret.append(diff) # recursively compare subdirs for i in comp.common_dirs: subleft = os.path.join(left, i) subright = os.path.join(right, i) ret.extend(self._comp_dir(subleft, subright, ignore)) return ''.join(ret) def _diff(self, left, right, header=False): """diff two files""" out = diff(modified=left, original=right, raw=False, diff_cmd=self.diff_cmd, debug=self.debug) if header: lshort = os.path.basename(left) out = '=> diff \"{}\":\n{}'.format(lshort, out) return out
class Comparator: def __init__(self, diff_cmd='', debug=False, ignore_missing_in_dotdrop=False): """constructor @diff_cmd: diff command to use @debug: enable debug """ self.diff_cmd = diff_cmd self.debug = debug self.log = Logger() self.ignore_missing_in_dotdrop = ignore_missing_in_dotdrop def compare(self, local_path, deployed_path, ignore=[]): """diff local_path (dotdrop dotfile) and deployed_path (destination file)""" local_path = os.path.expanduser(local_path) deployed_path = os.path.expanduser(deployed_path) if self.debug: self.log.dbg('comparing {} and {}'.format( local_path, deployed_path, )) self.log.dbg('ignore pattern(s): {}'.format(ignore)) # test type of file if os.path.isdir(local_path) and not os.path.isdir(deployed_path): return '\"{}\" is a dir while \"{}\" is a file\n'.format( local_path, deployed_path, ) if not os.path.isdir(local_path) and os.path.isdir(deployed_path): return '\"{}\" is a file while \"{}\" is a dir\n'.format( local_path, deployed_path, ) # test content if not os.path.isdir(local_path): if self.debug: self.log.dbg('{} is a file'.format(local_path)) if self.debug: self.log.dbg('is file') ret = self._comp_file(local_path, deployed_path, ignore) if not ret: ret = self._comp_mode(local_path, deployed_path) return ret if self.debug: self.log.dbg('{} is a directory'.format(local_path)) ret = self._comp_dir(local_path, deployed_path, ignore) if not ret: ret = self._comp_mode(local_path, deployed_path) return ret def _comp_mode(self, local_path, deployed_path): """compare mode""" local_mode = get_file_perm(local_path) deployed_mode = get_file_perm(deployed_path) if local_mode == deployed_mode: return '' if self.debug: msg = 'mode differ {} ({:o}) and {} ({:o})' self.log.dbg( msg.format(local_path, local_mode, deployed_path, deployed_mode)) ret = 'modes differ for {} ({:o}) vs {:o}\n' return ret.format(deployed_path, deployed_mode, local_mode) def _comp_file(self, local_path, deployed_path, ignore): """compare a file""" if self.debug: self.log.dbg('compare file {} with {}'.format( local_path, deployed_path, )) if (self.ignore_missing_in_dotdrop and not os.path.exists(local_path)) \ or must_ignore([local_path, deployed_path], ignore, debug=self.debug): if self.debug: self.log.dbg('ignoring diff {} and {}'.format( local_path, deployed_path, )) return '' return self._diff(local_path, deployed_path) def _comp_dir(self, local_path, deployed_path, ignore): """compare a directory""" if self.debug: self.log.dbg('compare directory {} with {}'.format( local_path, deployed_path, )) if not os.path.exists(deployed_path): return '' if (self.ignore_missing_in_dotdrop and not os.path.exists(local_path)) \ or must_ignore([local_path, deployed_path], ignore, debug=self.debug): if self.debug: self.log.dbg('ignoring diff {} and {}'.format( local_path, deployed_path, )) return '' if not os.path.isdir(deployed_path): return '\"{}\" is a file\n'.format(deployed_path) if self.debug: self.log.dbg('compare {} and {}'.format(local_path, deployed_path)) ret = [] comp = filecmp.dircmp(local_path, deployed_path) # handle files only in deployed dir for i in comp.left_only: if self.ignore_missing_in_dotdrop: continue if must_ignore([os.path.join(local_path, i)], ignore, debug=self.debug): continue ret.append('=> \"{}\" does not exist on destination\n'.format(i)) # handle files only in dotpath dir for i in comp.right_only: if must_ignore([os.path.join(deployed_path, i)], ignore, debug=self.debug): continue if not self.ignore_missing_in_dotdrop: ret.append('=> \"{}\" does not exist in dotdrop\n'.format(i)) # same local_path and deployed_path but different type funny = comp.common_funny for i in funny: source_file = os.path.join(local_path, i) deployed_file = os.path.join(deployed_path, i) if self.ignore_missing_in_dotdrop and \ not os.path.exists(source_file): continue if must_ignore([source_file, deployed_file], ignore, debug=self.debug): continue short = os.path.basename(source_file) # file vs dir ret.append('=> different type: \"{}\"\n'.format(short)) # content is different funny = comp.diff_files funny.extend(comp.funny_files) funny = uniq_list(funny) for i in funny: source_file = os.path.join(local_path, i) deployed_file = os.path.join(deployed_path, i) if self.ignore_missing_in_dotdrop and \ not os.path.exists(source_file): continue if must_ignore([source_file, deployed_file], ignore, debug=self.debug): continue ret.append(self._diff(source_file, deployed_file, header=True)) # recursively compare subdirs for i in comp.common_dirs: sublocal_path = os.path.join(local_path, i) subdeployed_path = os.path.join(deployed_path, i) ret.extend(self._comp_dir(sublocal_path, subdeployed_path, ignore)) return ''.join(ret) def _diff(self, local_path, deployed_path, header=False): """diff two files""" out = diff(modified=local_path, original=deployed_path, diff_cmd=self.diff_cmd, debug=self.debug) if header: lshort = os.path.basename(local_path) out = '=> diff \"{}\":\n{}'.format(lshort, out) return out
class Installer: def __init__(self, base='.', create=True, backup=True, dry=False, safe=False, workdir='~/.config/dotdrop', debug=False, diff=True, totemp=None, showdiff=False, backup_suffix='.dotdropbak', diff_cmd=''): """constructor @base: directory path where to search for templates @create: create directory hierarchy if missing when installing @backup: backup existing dotfile when installing @dry: just simulate @safe: ask for any overwrite @workdir: where to install template before symlinking @debug: enable debug @diff: diff when installing if True @totemp: deploy to this path instead of dotfile dst if not None @showdiff: show the diff before overwriting (or asking for) @backup_suffix: suffix for dotfile backup file @diff_cmd: diff command to use """ self.create = create self.backup = backup self.dry = dry self.safe = safe self.workdir = os.path.expanduser(workdir) self.base = base self.debug = debug self.diff = diff self.totemp = totemp self.showdiff = showdiff self.backup_suffix = backup_suffix self.diff_cmd = diff_cmd self.comparing = False self.action_executed = False self.log = Logger() def _log_install(self, boolean, err): if not self.debug: return boolean, err if boolean: self.log.dbg('install: SUCCESS') else: if err: self.log.dbg('install: ERROR: {}'.format(err)) else: self.log.dbg('install: IGNORED') return boolean, err def install(self, templater, src, dst, actionexec=None, noempty=False, ignore=[], template=True): """ install src to dst using a template @templater: the templater object @src: dotfile source path in dotpath @dst: dotfile destination path in the FS @actionexec: action executor callback @noempty: render empty template flag @ignore: pattern to ignore when installing @template: template this dotfile return - True, None : success - False, error_msg : error - False, None : ignored """ if self.debug: self.log.dbg('installing \"{}\" to \"{}\"'.format(src, dst)) if not dst or not src: if self.debug: self.log.dbg('empty dst for {}'.format(src)) return self._log_install(True, None) self.action_executed = False src = os.path.join(self.base, os.path.expanduser(src)) if not os.path.exists(src): err = 'source dotfile does not exist: {}'.format(src) return self._log_install(False, err) dst = os.path.expanduser(dst) if self.totemp: dst = self._pivot_path(dst, self.totemp) if utils.samefile(src, dst): # symlink loop err = 'dotfile points to itself: {}'.format(dst) return self._log_install(False, err) isdir = os.path.isdir(src) if self.debug: self.log.dbg('install {} to {}'.format(src, dst)) self.log.dbg('is a directory \"{}\": {}'.format(src, isdir)) if isdir: b, e = self._install_dir(templater, src, dst, actionexec=actionexec, noempty=noempty, ignore=ignore, template=template) return self._log_install(b, e) b, e = self._install_file(templater, src, dst, actionexec=actionexec, noempty=noempty, ignore=ignore, template=template) return self._log_install(b, e) def link(self, templater, src, dst, actionexec=None, template=True): """ set src as the link target of dst @templater: the templater @src: dotfile source path in dotpath @dst: dotfile destination path in the FS @actionexec: action executor callback @template: template this dotfile return - True, None : success - False, error_msg : error - False, None : ignored """ if self.debug: self.log.dbg('link \"{}\" to \"{}\"'.format(src, dst)) if not dst or not src: if self.debug: self.log.dbg('empty dst for {}'.format(src)) return self._log_install(True, None) self.action_executed = False src = os.path.normpath(os.path.join(self.base, os.path.expanduser(src))) if not os.path.exists(src): err = 'source dotfile does not exist: {}'.format(src) return self._log_install(False, err) dst = os.path.normpath(os.path.expanduser(dst)) if self.totemp: # ignore actions b, e = self.install(templater, src, dst, actionexec=None, template=template) return self._log_install(b, e) if template and Templategen.is_template(src): if self.debug: self.log.dbg('dotfile is a template') self.log.dbg('install to {} and symlink'.format(self.workdir)) tmp = self._pivot_path(dst, self.workdir, striphome=True) i, err = self.install(templater, src, tmp, actionexec=actionexec, template=template) if not i and not os.path.exists(tmp): return self._log_install(i, err) src = tmp b, e = self._link(src, dst, actionexec=actionexec) return self._log_install(b, e) def link_children(self, templater, src, dst, actionexec=None, template=True): """ link all files under a given directory @templater: the templater @src: dotfile source path in dotpath @dst: dotfile destination path in the FS @actionexec: action executor callback @template: template this dotfile return - True, None: success - False, error_msg: error - False, None, ignored """ if self.debug: self.log.dbg('link_children \"{}\" to \"{}\"'.format(src, dst)) if not dst or not src: if self.debug: self.log.dbg('empty dst for {}'.format(src)) return self._log_install(True, None) self.action_executed = False parent = os.path.join(self.base, os.path.expanduser(src)) # Fail if source doesn't exist if not os.path.exists(parent): err = 'source dotfile does not exist: {}'.format(parent) return self._log_install(False, err) # Fail if source not a directory if not os.path.isdir(parent): if self.debug: self.log.dbg('symlink children of {} to {}'.format(src, dst)) err = 'source dotfile is not a directory: {}'.format(parent) return self._log_install(False, err) dst = os.path.normpath(os.path.expanduser(dst)) if not os.path.lexists(dst): self.log.sub('creating directory "{}"'.format(dst)) os.makedirs(dst) if os.path.isfile(dst): msg = ''.join([ 'Remove regular file {} and ', 'replace with empty directory?', ]).format(dst) if self.safe and not self.log.ask(msg): err = 'ignoring "{}", nothing installed'.format(dst) return self._log_install(False, err) os.unlink(dst) os.mkdir(dst) children = os.listdir(parent) srcs = [os.path.normpath(os.path.join(parent, child)) for child in children] dsts = [os.path.normpath(os.path.join(dst, child)) for child in children] installed = 0 for i in range(len(children)): src = srcs[i] dst = dsts[i] if self.debug: self.log.dbg('symlink child {} to {}'.format(src, dst)) if template and Templategen.is_template(src): if self.debug: self.log.dbg('dotfile is a template') self.log.dbg('install to {} and symlink' .format(self.workdir)) tmp = self._pivot_path(dst, self.workdir, striphome=True) r, e = self.install(templater, src, tmp, actionexec=actionexec, template=template) if not r and e and not os.path.exists(tmp): continue src = tmp ret, err = self._link(src, dst, actionexec=actionexec) if ret: installed += 1 # void actionexec if dotfile installed # to prevent from running actions multiple times actionexec = None else: if err: return self._log_install(ret, err) return self._log_install(installed > 0, None) def _link(self, src, dst, actionexec=None): """ set src as a link target of dst return - True, None: success - False, error_msg: error - False, None, ignored """ overwrite = not self.safe if os.path.lexists(dst): if os.path.realpath(dst) == os.path.realpath(src): msg = 'ignoring "{}", link already exists'.format(dst) if self.debug: self.log.dbg(msg) return False, None if self.dry: self.log.dry('would remove {} and link to {}'.format(dst, src)) return True, None if self.showdiff: self._diff_before_write(src, dst, quiet=False) msg = 'Remove "{}" for link creation?'.format(dst) if self.safe and not self.log.ask(msg): err = 'ignoring "{}", link was not created'.format(dst) return False, err overwrite = True try: utils.removepath(dst) except OSError as e: err = 'something went wrong with {}: {}'.format(src, e) return False, err if self.dry: self.log.dry('would link {} to {}'.format(dst, src)) return True, None base = os.path.dirname(dst) if not self._create_dirs(base): err = 'error creating directory for {}'.format(dst) return False, err r, e = self._exec_pre_actions(actionexec) if not r: return False, e # re-check in case action created the file if os.path.lexists(dst): msg = 'Remove "{}" for link creation?'.format(dst) if self.safe and not overwrite and not self.log.ask(msg): err = 'ignoring "{}", link was not created'.format(dst) return False, err try: utils.removepath(dst) except OSError as e: err = 'something went wrong with {}: {}'.format(src, e) return False, err os.symlink(src, dst) self.log.sub('linked {} to {}'.format(dst, src)) return True, None def _get_tmp_file_vars(self, src, dst): tmp = {} tmp['_dotfile_sub_abs_src'] = src tmp['_dotfile_sub_abs_dst'] = dst return tmp def _install_file(self, templater, src, dst, actionexec=None, noempty=False, ignore=[], template=True): """install src to dst when is a file""" if self.debug: self.log.dbg('deploy file: {}'.format(src)) self.log.dbg('ignore empty: {}'.format(noempty)) self.log.dbg('ignore pattern: {}'.format(ignore)) self.log.dbg('template: {}'.format(template)) self.log.dbg('no empty: {}'.format(noempty)) if utils.must_ignore([src, dst], ignore, debug=self.debug): if self.debug: self.log.dbg('ignoring install of {} to {}'.format(src, dst)) return False, None if utils.samefile(src, dst): # symlink loop err = 'dotfile points to itself: {}'.format(dst) return False, err if not os.path.exists(src): err = 'source dotfile does not exist: {}'.format(src) return False, err # handle the file content = None if template: # template the file saved = templater.add_tmp_vars(self._get_tmp_file_vars(src, dst)) try: content = templater.generate(src) except UndefinedException as e: return False, str(e) finally: templater.restore_vars(saved) if noempty and utils.content_empty(content): if self.debug: self.log.dbg('ignoring empty template: {}'.format(src)) return False, None if content is None: err = 'empty template {}'.format(src) return False, err ret, err = self._write(src, dst, content=content, actionexec=actionexec, template=template) # build return values if ret < 0: # error return False, err if ret > 0: # already exists if self.debug: self.log.dbg('ignoring {}'.format(dst)) return False, None if ret == 0: # success if not self.dry and not self.comparing: self.log.sub('copied {} to {}'.format(src, dst)) return True, None # error err = 'installing {} to {}'.format(src, dst) return False, err def _install_dir(self, templater, src, dst, actionexec=None, noempty=False, ignore=[], template=True): """install src to dst when is a directory""" if self.debug: self.log.dbg('install dir {}'.format(src)) self.log.dbg('ignore empty: {}'.format(noempty)) # default to nothing installed and no error ret = False, None if not self._create_dirs(dst): err = 'creating directory for {}'.format(dst) return False, err # handle all files in dir for entry in os.listdir(src): f = os.path.join(src, entry) if not os.path.isdir(f): # is file res, err = self._install_file(templater, f, os.path.join(dst, entry), actionexec=actionexec, noempty=noempty, ignore=ignore, template=template) if not res and err: # error occured ret = res, err break elif res: # something got installed ret = True, None else: # is directory res, err = self._install_dir(templater, f, os.path.join(dst, entry), actionexec=actionexec, noempty=noempty, ignore=ignore, template=template) if not res and err: # error occured ret = res, err break elif res: # something got installed ret = True, None return ret def _fake_diff(self, dst, content): """ fake diff by comparing file content with content returns True if same """ cur = '' with open(dst, 'br') as f: cur = f.read() return cur == content def _write(self, src, dst, content=None, actionexec=None, template=True): """ copy dotfile / write content to file return 0, None: for success, 1, None: when already exists -1, err: when error content is always empty if template is False and is to be ignored """ overwrite = not self.safe if self.dry: self.log.dry('would install {}'.format(dst)) return 0, None if os.path.lexists(dst): rights = os.stat(src).st_mode samerights = False try: samerights = os.stat(dst).st_mode == rights except OSError as e: if e.errno == errno.ENOENT: # broken symlink err = 'broken symlink {}'.format(dst) return -1, err diff = None if self.diff: diff = self._diff_before_write(src, dst, content=content, quiet=True) if not diff and samerights: if self.debug: self.log.dbg('{} is the same'.format(dst)) return 1, None if self.safe: if self.debug: self.log.dbg('change detected for {}'.format(dst)) if self.showdiff: if diff is None: # get diff diff = self._diff_before_write(src, dst, content=content, quiet=True) if diff: self._print_diff(src, dst, diff) if not self.log.ask('Overwrite \"{}\"'.format(dst)): self.log.warn('ignoring {}'.format(dst)) return 1, None overwrite = True if self.backup and os.path.lexists(dst): self._backup(dst) base = os.path.dirname(dst) if not self._create_dirs(base): err = 'creating directory for {}'.format(dst) return -1, err r, e = self._exec_pre_actions(actionexec) if not r: return -1, e if self.debug: self.log.dbg('install dotfile to \"{}\"'.format(dst)) # re-check in case action created the file if self.safe and not overwrite and os.path.lexists(dst): if not self.log.ask('Overwrite \"{}\"'.format(dst)): self.log.warn('ignoring {}'.format(dst)) return 1, None if template: # write content the file try: with open(dst, 'wb') as f: f.write(content) shutil.copymode(src, dst) except NotADirectoryError as e: err = 'opening dest file: {}'.format(e) return -1, err except Exception as e: return -1, str(e) else: # copy file try: shutil.copyfile(src, dst) shutil.copymode(src, dst) except Exception as e: return -1, str(e) return 0, None def _diff_before_write(self, src, dst, content=None, quiet=False): """ diff before writing using a temp file if content is not None returns diff string ('' if same) """ tmp = None if content: tmp = utils.write_to_tmpfile(content) src = tmp diff = utils.diff(modified=src, original=dst, raw=False, diff_cmd=self.diff_cmd) if tmp: utils.removepath(tmp, logger=self.log) if not quiet and diff: self._print_diff(src, dst, diff) return diff def _print_diff(self, src, dst, diff): """show diff to user""" self.log.log('diff \"{}\" VS \"{}\"'.format(dst, src)) self.log.emph(diff) def _create_dirs(self, directory): """mkdir -p <directory>""" if not self.create and not os.path.exists(directory): if self.debug: self.log.dbg('no mkdir as \"create\" set to false in config') return False if os.path.exists(directory): return True if self.dry: self.log.dry('would mkdir -p {}'.format(directory)) return True if self.debug: self.log.dbg('mkdir -p {}'.format(directory)) os.makedirs(directory) return os.path.exists(directory) def _backup(self, path): """backup file pointed by path""" if self.dry: return dst = path.rstrip(os.sep) + self.backup_suffix self.log.log('backup {} to {}'.format(path, dst)) os.rename(path, dst) def _pivot_path(self, path, newdir, striphome=False): """change path to be under newdir""" if self.debug: self.log.dbg('pivot new dir: \"{}\"'.format(newdir)) self.log.dbg('strip home: {}'.format(striphome)) if striphome: path = utils.strip_home(path) sub = path.lstrip(os.sep) new = os.path.join(newdir, sub) if self.debug: self.log.dbg('pivot \"{}\" to \"{}\"'.format(path, new)) return new def _exec_pre_actions(self, actionexec): """execute action executor""" if self.action_executed: return True, None if not actionexec: return True, None ret, err = actionexec() self.action_executed = True return ret, err def _install_to_temp(self, templater, src, dst, tmpdir, template=True): """install a dotfile to a tempdir""" tmpdst = self._pivot_path(dst, tmpdir) r = self.install(templater, src, tmpdst, template=template) return r, tmpdst def install_to_temp(self, templater, tmpdir, src, dst, template=True): """install a dotfile to a tempdir""" ret = False tmpdst = '' # save some flags while comparing self.comparing = True drysaved = self.dry self.dry = False diffsaved = self.diff self.diff = False createsaved = self.create self.create = True # normalize src and dst src = os.path.expanduser(src) dst = os.path.expanduser(dst) if self.debug: self.log.dbg('tmp install {} (defined dst: {})'.format(src, dst)) # install the dotfile to a temp directory for comparing r, tmpdst = self._install_to_temp(templater, src, dst, tmpdir, template=template) ret, err = r if self.debug: self.log.dbg('tmp installed in {}'.format(tmpdst)) # reset flags self.dry = drysaved self.diff = diffsaved self.comparing = False self.create = createsaved return ret, err, tmpdst
class Updater: def __init__(self, conf, dotpath, profile, variables, dry, safe, iskey=False, debug=False, ignore=[], showpatch=False): """constructor @conf: configuration @dotpath: path where dotfiles are stored @profile: profile selected @variables: dictionary of variables for the templates @dry: simulate @safe: ask for overwrite if True @iskey: will the update be called on keys or path @debug: enable debug @ignore: pattern to ignore when updating @showpatch: show patch if dotfile to update is a template """ self.conf = conf self.dotpath = dotpath self.profile = profile self.variables = variables self.dry = dry self.safe = safe self.iskey = iskey self.debug = debug self.ignore = ignore self.showpatch = showpatch self.log = Logger() def update_path(self, path): """update the dotfile installed on path""" if not os.path.lexists(path): self.log.err('\"{}\" does not exist!'.format(path)) return False path = self._normalize(path) dotfile = self._get_dotfile_by_path(path) if not dotfile: return False path = os.path.expanduser(path) if self.debug: self.log.dbg('updating {} from path \"{}\"'.format(dotfile, path)) return self._update(path, dotfile) def update_key(self, key): """update the dotfile referenced by key""" dotfile = self._get_dotfile_by_key(key) if not dotfile: return False if self.debug: self.log.dbg('updating {} from key \"{}\"'.format(dotfile, key)) path = self._normalize(dotfile.dst) return self._update(path, dotfile) def _update(self, path, dotfile): """update dotfile from file pointed by path""" ret = False new_path = None self.ignores = list(set(self.ignore + dotfile.upignore)) if self.debug: self.log.dbg('ignore pattern(s): {}'.format(self.ignores)) left = os.path.expanduser(path) right = os.path.join(self.conf.abs_or_rel(self.dotpath), dotfile.src) right = os.path.expanduser(right) if self._ignore([left, right]): return True if dotfile.trans_w: # apply write transformation if any new_path = self._apply_trans_w(path, dotfile) if not new_path: return False left = new_path if os.path.isdir(left): ret = self._handle_dir(left, right) else: ret = self._handle_file(left, right) # clean temporary files if new_path and os.path.exists(new_path): utils.remove(new_path) return ret def _apply_trans_w(self, path, dotfile): """apply write transformation to dotfile""" trans = dotfile.trans_w if self.debug: self.log.dbg('executing write transformation {}'.format(trans)) tmp = utils.get_unique_tmp_name() if not trans.transform(path, tmp): msg = 'transformation \"{}\" failed for {}' self.log.err(msg.format(trans.key, dotfile.key)) if os.path.exists(tmp): utils.remove(tmp) return None return tmp def _normalize(self, path): """normalize the path to match dotfile""" path = os.path.expanduser(path) path = os.path.expandvars(path) path = os.path.abspath(path) home = os.path.expanduser(TILD) + os.sep # normalize the path if path.startswith(home): path = path[len(home):] path = os.path.join(TILD, path) return path def _get_dotfile_by_key(self, key): """get the dotfile matching this key""" dotfiles = self.conf.get_dotfiles(self.profile) subs = [d for d in dotfiles if d.key == key] if not subs: self.log.err('key \"{}\" not found!'.format(key)) return None if len(subs) > 1: found = ','.join([d.src for d in dotfiles]) self.log.err('multiple dotfiles found: {}'.format(found)) return None return subs[0] def _get_dotfile_by_path(self, path): """get the dotfile matching this path""" dotfiles = self.conf.get_dotfiles(self.profile) subs = [d for d in dotfiles if d.dst == path] if not subs: self.log.err('\"{}\" is not managed!'.format(path)) return None if len(subs) > 1: found = ','.join([d.src for d in dotfiles]) self.log.err('multiple dotfiles found: {}'.format(found)) return None return subs[0] def _is_template(self, path): if not Templategen.is_template(path): return False self.log.warn('{} uses template, update manually'.format(path)) return True def _show_patch(self, tpath, fpath): """provide a way to manually patch the template""" content = self._resolve_template(tpath) tmp = utils.write_to_tmpfile(content) cmds = ['diff', '-u', tmp, fpath, '|', 'patch', tpath] self.log.warn('try patching with: \"{}\"'.format(' '.join(cmds))) return False def _resolve_template(self, tpath): """resolve the template to a temporary file""" t = Templategen(variables=self.variables, base=self.dotpath, debug=self.debug) return t.generate(tpath) def _handle_file(self, left, right, compare=True): """sync left (deployed file) and right (dotdrop dotfile)""" if self._ignore([left, right]): return True if self.debug: self.log.dbg('update for file {} and {}'.format(left, right)) if self._is_template(right): # dotfile is a template if self.debug: self.log.dbg('{} is a template'.format(right)) if self.showpatch: self._show_patch(right, left) return False if compare and filecmp.cmp(left, right, shallow=True): # no difference if self.debug: self.log.dbg('identical files: {} and {}'.format(left, right)) return True if not self._overwrite(left, right): return False try: if self.dry: self.log.dry('would cp {} {}'.format(left, right)) else: if self.debug: self.log.dbg('cp {} {}'.format(left, right)) shutil.copyfile(left, right) except IOError as e: self.log.warn('{} update failed, do manually: {}'.format(left, e)) return False return True def _handle_dir(self, left, right): """sync left (deployed dir) and right (dotdrop dir)""" if self.debug: self.log.dbg('handle update for dir {} to {}'.format(left, right)) # paths must be absolute (no tildes) left = os.path.expanduser(left) right = os.path.expanduser(right) if self._ignore([left, right]): return True # find the differences diff = filecmp.dircmp(left, right, ignore=None) # handle directories diff return self._merge_dirs(diff) def _merge_dirs(self, diff): """Synchronize directories recursively.""" left, right = diff.left, diff.right if self.debug: self.log.dbg('sync dir {} to {}'.format(left, right)) if self._ignore([left, right]): return True # create dirs that don't exist in dotdrop for toadd in diff.left_only: exist = os.path.join(left, toadd) if not os.path.isdir(exist): # ignore files for now continue # match to dotdrop dotpath new = os.path.join(right, toadd) if self._ignore([exist, new]): continue if self.dry: self.log.dry('would cp -r {} {}'.format(exist, new)) continue if self.debug: self.log.dbg('cp -r {} {}'.format(exist, new)) # Newly created directory should be copied as is (for efficiency). shutil.copytree(exist, new) # remove dirs that don't exist in deployed version for toremove in diff.right_only: old = os.path.join(right, toremove) if not os.path.isdir(old): # ignore files for now continue if self._ignore([old]): continue if self.dry: self.log.dry('would rm -r {}'.format(old)) continue if self.debug: self.log.dbg('rm -r {}'.format(old)) if not self._confirm_rm_r(old): continue utils.remove(old) # handle files diff # sync files that exist in both but are different fdiff = diff.diff_files fdiff.extend(diff.funny_files) fdiff.extend(diff.common_funny) for f in fdiff: fleft = os.path.join(left, f) fright = os.path.join(right, f) if self._ignore([fleft, fright]): continue if self.dry: self.log.dry('would cp {} {}'.format(fleft, fright)) continue if self.debug: self.log.dbg('cp {} {}'.format(fleft, fright)) self._handle_file(fleft, fright, compare=False) # copy files that don't exist in dotdrop for toadd in diff.left_only: exist = os.path.join(left, toadd) if os.path.isdir(exist): # ignore dirs, done above continue new = os.path.join(right, toadd) if self._ignore([exist, new]): continue if self.dry: self.log.dry('would cp {} {}'.format(exist, new)) continue if self.debug: self.log.dbg('cp {} {}'.format(exist, new)) shutil.copyfile(exist, new) # remove files that don't exist in deployed version for toremove in diff.right_only: new = os.path.join(right, toremove) if not os.path.exists(new): continue if os.path.isdir(new): # ignore dirs, done above continue if self._ignore([new]): continue if self.dry: self.log.dry('would rm {}'.format(new)) continue if self.debug: self.log.dbg('rm {}'.format(new)) utils.remove(new) # Recursively decent into common subdirectories. for subdir in diff.subdirs.values(): self._merge_dirs(subdir) # Nothing more to do here. return True def _overwrite(self, src, dst): """ask for overwritting""" msg = 'Overwrite \"{}\" with \"{}\"?'.format(dst, src) if self.safe and not self.log.ask(msg): return False return True def _confirm_rm_r(self, directory): """ask for rm -r directory""" msg = 'Recursively remove \"{}\"?'.format(directory) if self.safe and not self.log.ask(msg): return False return True def _ignore(self, paths): if utils.must_ignore(paths, self.ignores, debug=self.debug): if self.debug: self.log.dbg('ignoring update for {}'.format(paths)) return True return False
class Updater: BACKUP_SUFFIX = '.dotdropbak' def __init__(self, conf, dotpath, dry, safe, debug): self.home = os.path.expanduser(TILD) self.conf = conf self.dotpath = dotpath self.dry = dry self.safe = safe self.debug = debug self.log = Logger() def _normalize(self, path): """normalize the path to match dotfile""" path = os.path.expanduser(path) path = os.path.expandvars(path) # normalize the path if path.startswith(self.home): path = path.lstrip(self.home) path = os.path.join(TILD, path) return path def _get_dotfile(self, path, profile): """get the dotfile matching this path""" dotfiles = self.conf.get_dotfiles(profile) subs = [d for d in dotfiles if d.dst == path] if not subs: self.log.err('\"{}\" is not managed!'.format(path)) return None if len(subs) > 1: found = ','.join([d.src for d in dotfiles]) self.log.err('multiple dotfiles found: {}'.format(found)) return None return subs[0] def update(self, path, profile): """update the dotfile installed on path""" if not os.path.lexists(path): self.log.err('\"{}\" does not exist!'.format(path)) return False left = self._normalize(path) dotfile = self._get_dotfile(path, profile) if not dotfile: return False if self.debug: self.log.dbg('updating {} from {}'.format(dotfile, path)) right = os.path.join(self.conf.abs_dotpath(self.dotpath), dotfile.src) # go through all files and update if os.path.isdir(path): return self._handle_dir(left, right) return self._handle_file(left, right) def _is_template(self, path): if Templategen.get_marker() not in open(path, 'r').read(): return False self.log.warn('{} uses template, update manually'.format(right)) return True def _handle_file(self, left, right, compare=True): """sync left (deployed file) and right (dotdrop dotfile)""" if self.debug: self.log.dbg('update for file {} and {}'.format(left, right)) if self._is_template(right): return False if compare and filecmp.cmp(left, right, shallow=True): # no difference if self.debug: self.log.dbg('identical files: {} and {}'.format(left, right)) return True if not self._overwrite(left, right): return False try: if self.dry: self.log.dry('would cp {} {}'.format(left, right)) else: if self.debug: self.log.dbg('cp {} {}'.format(left, right)) shutil.copyfile(left, right) except IOError as e: self.log.warn('{} update failed, do manually: {}'.format(left, e)) return False return True def _handle_dir(self, left, right): """sync left (deployed dir) and right (dotdrop dir)""" if self.debug: self.log.dbg('handle update for dir {} to {}'.format(left, right)) # find the difference diff = filecmp.dircmp(left, right, ignore=None) # handle directories diff # create dirs that don't exist in dotdrop if self.debug: self.log.dbg('handle dirs that do not exist in dotdrop') for toadd in diff.left_only: exist = os.path.join(left, toadd) if not os.path.isdir(exist): # ignore files for now continue # match to dotdrop dotpath new = os.path.join(right, toadd) if self.dry: self.log.dry('would mkdir -p {}'.format(new)) continue if self.debug: self.log.dbg('mkdir -p {}'.format(new)) self._create_dirs(new) # remove dirs that don't exist in deployed version if self.debug: self.log.dbg('remove dirs that do not exist in deployed version') for toremove in diff.right_only: new = os.path.join(right, toremove) if self.dry: self.log.dry('would rm -r {}'.format(new)) continue if self.debug: self.log.dbg('rm -r {}'.format(new)) if not self._confirm_rm_r(new): continue utils.remove(new) # handle files diff # sync files that exist in both but are different if self.debug: self.log.dbg('sync files that exist in both but are different') fdiff = diff.diff_files fdiff.extend(diff.funny_files) fdiff.extend(diff.common_funny) for f in fdiff: fleft = os.path.join(left, f) fright = os.path.join(right, f) if self.dry: self.log.dry('would cp {} {}'.format(fleft, fright)) continue if self.debug: self.log.dbg('cp {} {}'.format(fleft, fright)) self._handle_file(fleft, fright, compare=False) # copy files that don't exist in dotdrop if self.debug: self.log.dbg('copy files not existing in dotdrop') for toadd in diff.left_only: exist = os.path.join(left, toadd) if os.path.isdir(exist): # ignore dirs, done above continue new = os.path.join(right, toadd) if self.dry: self.log.dry('would cp {} {}'.format(exist, new)) continue if self.debug: self.log.dbg('cp {} {}'.format(exist, new)) shutil.copyfile(exist, new) # remove files that don't exist in deployed version if self.debug: self.log.dbg('remove files that do not exist in deployed version') for toremove in diff.right_only: new = os.path.join(right, toremove) if not os.path.exists(new): continue if os.path.isdir(new): # ignore dirs, done above continue if self.dry: self.log.dry('would rm {}'.format(new)) continue if self.debug: self.log.dbg('rm {}'.format(new)) utils.remove(new) return True def _create_dirs(self, directory): """mkdir -p <directory>""" if os.path.exists(directory): return True if self.dry: self.log.dry('would mkdir -p {}'.format(directory)) return True if self.debug: self.log.dbg('mkdir -p {}'.format(directory)) os.makedirs(directory) return os.path.exists(directory) def _overwrite(self, src, dst): """ask for overwritting""" msg = 'Overwrite \"{}\" with \"{}\"?'.format(dst, src) if self.safe and not self.log.ask(msg): return False return True def _confirm_rm_r(self, directory): """ask for rm -r directory""" msg = 'Recursively remove \"{}\"?'.format(directory) if self.safe and not self.log.ask(msg): return False return True
class Options(AttrMonitor): def __init__(self, args=None): """constructor @args: argument dictionary (if None use sys) """ self.args = args if not args: self.args = docopt(USAGE, version=VERSION) self.log = Logger() self.debug = self.args['--verbose'] if not self.debug and ENV_DEBUG in os.environ: self.debug = True if ENV_NODEBUG in os.environ: self.debug = False self.profile = self.args['--profile'] self.confpath = os.path.expanduser(self.args['--cfg']) if self.debug: self.log.dbg('config file: {}'.format(self.confpath)) self._read_config(self.profile) self._apply_args() self._fill_attr() if ENV_NOBANNER not in os.environ \ and self.banner \ and not self.args['--no-banner']: self._header() self._print_attr() # start monitoring for bad attribute self._set_attr_err = True def _header(self): """print the header""" self.log.log(BANNER) self.log.log('') def _read_config(self, profile=None): """read the config file""" self.conf = Cfg(self.confpath, profile=profile, debug=self.debug) # transform the configs in attribute for k, v in self.conf.get_settings().items(): setattr(self, k, v) def _apply_args(self): """apply cli args as attribute""" # the commands self.cmd_list = self.args['list'] self.cmd_listfiles = self.args['listfiles'] self.cmd_install = self.args['install'] self.cmd_compare = self.args['compare'] self.cmd_import = self.args['import'] self.cmd_update = self.args['update'] self.cmd_detail = self.args['detail'] # adapt attributes based on arguments self.dry = self.args['--dry'] self.safe = not self.args['--force'] self.link = LinkTypes.NOLINK if self.link_by_default: self.link = LinkTypes.PARENTS if self.args['--inv-link']: # Only invert link type from NOLINK to PARENTS and vice-versa if self.link == LinkTypes.NOLINK: self.link = LinkTypes.PARENTS elif self.link == LinkTypes.PARENTS: self.link = LinkTypes.NOLINK # "listfiles" specifics self.listfiles_templateonly = self.args['--template'] # "install" specifics self.install_temporary = self.args['--temp'] self.install_keys = self.args['<key>'] self.install_diff = not self.args['--nodiff'] self.install_showdiff = self.showdiff or self.args['--showdiff'] self.install_backup_suffix = BACKUP_SUFFIX # "compare" specifics self.compare_dopts = self.args['--dopts'] self.compare_focus = self.args['--file'] self.compare_ignore = self.args['--ignore'] self.compare_ignore.append('*{}'.format(self.install_backup_suffix)) # "import" specifics self.import_path = self.args['<path>'] # "update" specifics self.update_path = self.args['<path>'] self.update_iskey = self.args['--key'] self.update_ignore = self.args['--ignore'] self.update_ignore.append('*{}'.format(self.install_backup_suffix)) self.update_showpatch = self.args['--show-patch'] # "detail" specifics self.detail_keys = self.args['<key>'] def _fill_attr(self): """create attributes from conf""" # variables self.variables = self.conf.get_variables(self.profile, debug=self.debug).copy() # the dotfiles self.dotfiles = self.conf.eval_dotfiles(self.profile, self.variables, debug=self.debug).copy() # the profiles self.profiles = self.conf.get_profiles() def _print_attr(self): """print all of this class attributes""" if not self.debug: return self.log.dbg('options:') for att in dir(self): if att.startswith('_'): continue val = getattr(self, att) if callable(val): continue self.log.dbg('- {}: \"{}\"'.format(att, val)) def _attr_set(self, attr): """error when some inexistent attr is set""" raise Exception('bad option: {}'.format(attr))
class Templategen: def __init__(self, base='.', variables={}, func_file=[], filter_file=[], debug=False): """constructor @base: directory path where to search for templates @variables: dictionary of variables for templates @func_file: file path to load functions from @filter_file: file path to load filters from @debug: enable debug """ self.base = base.rstrip(os.sep) self.debug = debug self.log = Logger() loader1 = FileSystemLoader(self.base) loader2 = FunctionLoader(self._template_loader) loader = ChoiceLoader([loader1, loader2]) self.env = Environment(loader=loader, trim_blocks=True, lstrip_blocks=True, keep_trailing_newline=True, block_start_string=BLOCK_START, block_end_string=BLOCK_END, variable_start_string=VAR_START, variable_end_string=VAR_END, comment_start_string=COMMENT_START, comment_end_string=COMMENT_END) # adding variables self.env.globals['env'] = os.environ if variables: self.env.globals.update(variables) # adding header method self.env.globals['header'] = self._header # adding helper methods if self.debug: self.log.dbg('load global functions:') self._load_funcs_to_dic(jhelpers, self.env.globals) if func_file: for f in func_file: if self.debug: self.log.dbg('load custom functions from {}'.format(f)) self._load_path_to_dic(f, self.env.globals) if filter_file: for f in filter_file: if self.debug: self.log.dbg('load custom filters from {}'.format(f)) self._load_path_to_dic(f, self.env.filters) if self.debug: self.log.dbg('template additional variables: {}'.format(variables)) def generate(self, src): """render template from path""" if not os.path.exists(src): return '' return self._handle_file(src) def generate_string(self, string): """render template from string""" if not string: return '' return self.env.from_string(string).render() def add_tmp_vars(self, newvars={}): """add vars to the globals, make sure to call restore_vars""" saved_globals = self.env.globals.copy() if not newvars: return saved_globals self.env.globals.update(newvars) return saved_globals def restore_vars(self, saved_globals): """restore globals from add_tmp_vars""" self.env.globals = saved_globals.copy() def update_variables(self, variables): """update variables""" self.env.globals.update(variables) def _load_path_to_dic(self, path, dic): mod = utils.get_module_from_path(path) if not mod: self.log.warn('cannot load module \"{}\"'.format(path)) return self._load_funcs_to_dic(mod, dic) def _load_funcs_to_dic(self, mod, dic): """dynamically load functions from module to dic""" if not mod or not dic: return funcs = utils.get_module_functions(mod) for name, func in funcs: if self.debug: self.log.dbg('load function \"{}\"'.format(name)) dic[name] = func def _header(self, prepend=''): """add a comment usually in the header of a dotfile""" return '{}{}'.format(prepend, utils.header()) def _handle_file(self, src): """generate the file content from template""" _, filetype = utils.run(['file', '-b', src], raw=False, debug=self.debug) filetype = filetype.strip() if self.debug: self.log.dbg('\"{}\" filetype: {}'.format(src, filetype)) istext = self._is_text(filetype) if self.debug: self.log.dbg('\"{}\" is text: {}'.format(src, istext)) if not istext: return self._handle_bin_file(src) return self._handle_text_file(src) def _is_text(self, fileoutput): """return if `file -b` output is ascii text""" out = fileoutput.lower() if 'text' in out: return True if 'empty' in out: return True if 'json' in out: return True return False def _template_loader(self, relpath): """manually load template when outside of base""" path = os.path.join(self.base, relpath) path = os.path.normpath(path) if not os.path.exists(path): raise TemplateNotFound(path) with open(path, 'r') as f: content = f.read() return content def _handle_text_file(self, src): """write text to file""" template_rel_path = os.path.relpath(src, self.base) try: template = self.env.get_template(template_rel_path) content = template.render() except UnicodeDecodeError: data = self._read_bad_encoded_text(src) content = self.generate_string(data) return content.encode('utf-8') def _handle_bin_file(self, src): """write binary to file""" # this is dirty if not src.startswith(self.base): src = os.path.join(self.base, src) with open(src, 'rb') as f: content = f.read() return content def _read_bad_encoded_text(self, path): """decode non utf-8 data""" with open(path, 'rb') as f: data = f.read() return data.decode('utf-8', 'replace') @staticmethod def is_template(path): """recursively check if any file is a template within path""" path = os.path.expanduser(path) if not os.path.exists(path): return False if os.path.isfile(path): # is file return Templategen._is_template(path) for entry in os.listdir(path): fpath = os.path.join(path, entry) if not os.path.isfile(fpath): # recursively explore directory if Templategen.is_template(fpath): return True else: # check if file is a template if Templategen._is_template(fpath): return True return False @staticmethod def var_is_template(string): """check if variable contains template(s)""" return VAR_START in str(string) @staticmethod def _is_template(path): """test if file pointed by path is a template""" if not os.path.isfile(path): return False try: with open(path, 'r') as f: data = f.read() except UnicodeDecodeError: # is binary so surely no template return False markers = [BLOCK_START, VAR_START, COMMENT_START] for marker in markers: if marker in data: return True return False
class Updater: def __init__(self, dotpath, variables, dotfile_key_getter, dotfile_dst_getter, dotfile_path_normalizer, dry=False, safe=True, debug=False, ignore=[], showpatch=False): """constructor @dotpath: path where dotfiles are stored @variables: dictionary of variables for the templates @dotfile_key_getter: func to get a dotfile by key @dotfile_dst_getter: func to get a dotfile by dst @dotfile_path_normalizer: func to normalize dotfile dst @dry: simulate @safe: ask for overwrite if True @debug: enable debug @ignore: pattern to ignore when updating @showpatch: show patch if dotfile to update is a template """ self.dotpath = dotpath self.variables = variables self.dotfile_key_getter = dotfile_key_getter self.dotfile_dst_getter = dotfile_dst_getter self.dotfile_path_normalizer = dotfile_path_normalizer self.dry = dry self.safe = safe self.debug = debug self.ignore = ignore self.showpatch = showpatch self.templater = Templategen(variables=self.variables, base=self.dotpath, debug=self.debug) # save template vars self.tvars = self.templater.add_tmp_vars() self.log = Logger() def update_path(self, path): """update the dotfile installed on path""" path = os.path.expanduser(path) if not os.path.lexists(path): self.log.err('\"{}\" does not exist!'.format(path)) return False dotfile = self.dotfile_dst_getter(path) if not dotfile: return False if self.debug: self.log.dbg('updating {} from path \"{}\"'.format(dotfile, path)) return self._update(path, dotfile) def update_key(self, key): """update the dotfile referenced by key""" dotfile = self.dotfile_key_getter(key) if not dotfile: return False if self.debug: self.log.dbg('updating {} from key \"{}\"'.format(dotfile, key)) path = self.dotfile_path_normalizer(dotfile.dst) return self._update(path, dotfile) def _update(self, path, dotfile): """update dotfile from file pointed by path""" ret = False new_path = None ignores = list(set(self.ignore + dotfile.upignore)) self.ignores = patch_ignores(ignores, dotfile.dst, debug=self.debug) if self.debug: self.log.dbg('ignore pattern(s): {}'.format(self.ignores)) path = os.path.expanduser(path) dtpath = os.path.join(self.dotpath, dotfile.src) dtpath = os.path.expanduser(dtpath) if self._ignore([path, dtpath]): self.log.sub('\"{}\" ignored'.format(dotfile.key)) return True # apply write transformation if any new_path = self._apply_trans_w(path, dotfile) if not new_path: return False if os.path.isdir(new_path): ret = self._handle_dir(new_path, dtpath) else: ret = self._handle_file(new_path, dtpath) # clean temporary files if new_path != path and os.path.exists(new_path): remove(new_path) return ret def _apply_trans_w(self, path, dotfile): """apply write transformation to dotfile""" trans = dotfile.get_trans_w() if not trans: return path if self.debug: self.log.dbg('executing write transformation {}'.format(trans)) tmp = get_unique_tmp_name() self.templater.restore_vars(self.tvars) newvars = dotfile.get_dotfile_variables() self.templater.add_tmp_vars(newvars=newvars) if not trans.transform( path, tmp, templater=self.templater, debug=self.debug): msg = 'transformation \"{}\" failed for {}' self.log.err(msg.format(trans.key, dotfile.key)) if os.path.exists(tmp): remove(tmp) return None return tmp def _is_template(self, path): if not Templategen.is_template(path): if self.debug: self.log.dbg('{} is NO template'.format(path)) return False self.log.warn('{} uses template, update manually'.format(path)) return True def _show_patch(self, fpath, tpath): """provide a way to manually patch the template""" content = self._resolve_template(tpath) tmp = write_to_tmpfile(content) cmds = ['diff', '-u', tmp, fpath, '|', 'patch', tpath] self.log.warn('try patching with: \"{}\"'.format(' '.join(cmds))) return False def _resolve_template(self, tpath): """resolve the template to a temporary file""" self.templater.restore_vars(self.tvars) return self.templater.generate(tpath) def _handle_file(self, path, dtpath, compare=True): """sync path (deployed file) and dtpath (dotdrop dotfile path)""" if self._ignore([path, dtpath]): self.log.sub('\"{}\" ignored'.format(dtpath)) return True if self.debug: self.log.dbg('update for file {} and {}'.format(path, dtpath)) if self._is_template(dtpath): # dotfile is a template if self.debug: self.log.dbg('{} is a template'.format(dtpath)) if self.showpatch: self._show_patch(path, dtpath) return False if compare and filecmp.cmp(path, dtpath, shallow=True): # no difference if self.debug: self.log.dbg('identical files: {} and {}'.format(path, dtpath)) return True if not self._overwrite(path, dtpath): return False try: if self.dry: self.log.dry('would cp {} {}'.format(path, dtpath)) else: if self.debug: self.log.dbg('cp {} {}'.format(path, dtpath)) shutil.copyfile(path, dtpath) self.log.sub('\"{}\" updated'.format(dtpath)) except IOError as e: self.log.warn('{} update failed, do manually: {}'.format(path, e)) return False return True def _handle_dir(self, path, dtpath): """sync path (deployed dir) and dtpath (dotdrop dir path)""" if self.debug: self.log.dbg('handle update for dir {} to {}'.format(path, dtpath)) # paths must be absolute (no tildes) path = os.path.expanduser(path) dtpath = os.path.expanduser(dtpath) if self._ignore([path, dtpath]): self.log.sub('\"{}\" ignored'.format(dtpath)) return True # find the differences diff = filecmp.dircmp(path, dtpath, ignore=None) # handle directories diff return self._merge_dirs(diff) def _merge_dirs(self, diff): """Synchronize directories recursively.""" left, right = diff.left, diff.right if self.debug: self.log.dbg('sync dir {} to {}'.format(left, right)) if self._ignore([left, right]): return True # create dirs that don't exist in dotdrop for toadd in diff.left_only: exist = os.path.join(left, toadd) if not os.path.isdir(exist): # ignore files for now continue # match to dotdrop dotpath new = os.path.join(right, toadd) if self._ignore([exist, new]): self.log.sub('\"{}\" ignored'.format(exist)) continue if self.dry: self.log.dry('would cp -r {} {}'.format(exist, new)) continue if self.debug: self.log.dbg('cp -r {} {}'.format(exist, new)) # Newly created directory should be copied as is (for efficiency). shutil.copytree(exist, new) self.log.sub('\"{}\" dir added'.format(new)) # remove dirs that don't exist in deployed version for toremove in diff.right_only: old = os.path.join(right, toremove) if not os.path.isdir(old): # ignore files for now continue if self._ignore([old]): continue if self.dry: self.log.dry('would rm -r {}'.format(old)) continue if self.debug: self.log.dbg('rm -r {}'.format(old)) if not self._confirm_rm_r(old): continue remove(old) self.log.sub('\"{}\" dir removed'.format(old)) # handle files diff # sync files that exist in both but are different fdiff = diff.diff_files fdiff.extend(diff.funny_files) fdiff.extend(diff.common_funny) for f in fdiff: fleft = os.path.join(left, f) fright = os.path.join(right, f) if self._ignore([fleft, fright]): continue if self.dry: self.log.dry('would cp {} {}'.format(fleft, fright)) continue if self.debug: self.log.dbg('cp {} {}'.format(fleft, fright)) self._handle_file(fleft, fright, compare=False) # copy files that don't exist in dotdrop for toadd in diff.left_only: exist = os.path.join(left, toadd) if os.path.isdir(exist): # ignore dirs, done above continue new = os.path.join(right, toadd) if self._ignore([exist, new]): continue if self.dry: self.log.dry('would cp {} {}'.format(exist, new)) continue if self.debug: self.log.dbg('cp {} {}'.format(exist, new)) shutil.copyfile(exist, new) self.log.sub('\"{}\" added'.format(new)) # remove files that don't exist in deployed version for toremove in diff.right_only: new = os.path.join(right, toremove) if not os.path.exists(new): continue if os.path.isdir(new): # ignore dirs, done above continue if self._ignore([new]): continue if self.dry: self.log.dry('would rm {}'.format(new)) continue if self.debug: self.log.dbg('rm {}'.format(new)) remove(new) self.log.sub('\"{}\" removed'.format(new)) # Recursively decent into common subdirectories. for subdir in diff.subdirs.values(): self._merge_dirs(subdir) # Nothing more to do here. return True def _overwrite(self, src, dst): """ask for overwritting""" msg = 'Overwrite \"{}\" with \"{}\"?'.format(dst, src) if self.safe and not self.log.ask(msg): return False return True def _confirm_rm_r(self, directory): """ask for rm -r directory""" msg = 'Recursively remove \"{}\"?'.format(directory) if self.safe and not self.log.ask(msg): return False return True def _ignore(self, paths): if must_ignore(paths, self.ignores, debug=self.debug): if self.debug: self.log.dbg('ignoring update for {}'.format(paths)) return True return False
class Templategen: def __init__(self, base='.', debug=False): self.base = base.rstrip(os.sep) loader = FileSystemLoader(self.base) self.env = Environment(loader=loader, trim_blocks=True, lstrip_blocks=True, keep_trailing_newline=True, block_start_string=BLOCK_START, block_end_string=BLOCK_END, variable_start_string=VAR_START, variable_end_string=VAR_END, comment_start_string=COMMENT_START, comment_end_string=COMMENT_END) self.log = Logger(debug=debug) def generate(self, src, profile): if not os.path.exists(src): return '' return self._handle_file(src, profile) def _handle_file(self, src, profile): """generate the file content from template""" filetype = utils.run(['file', '-b', src], raw=False).strip() self.log.dbg('\"{}\" filetype: {}'.format(src, filetype)) istext = 'text' in filetype self.log.dbg('\"{}\" is text: {}'.format(src, istext)) if not istext: return self._handle_bin_file(src, profile) return self._handle_text_file(src, profile) def _handle_text_file(self, src, profile): """write text to file""" template_rel_path = os.path.relpath(src, self.base) try: template = self.env.get_template(template_rel_path) content = template.render(profile=profile, env=os.environ) except UnicodeDecodeError: data = self._read_bad_encoded_text(src) template = self.env.from_string(data) content = template.render(profile=profile, env=os.environ) content = content.encode('UTF-8') return content def _handle_bin_file(self, src, profile): """write binary to file""" # this is dirty if not src.startswith(self.base): src = os.path.join(self.base, src) with open(src, 'rb') as f: return f.read() def _read_bad_encoded_text(self, path): """decode non utf-8 data""" with open(path, 'rb') as f: data = f.read() return data.decode('utf-8', 'replace') def get_marker(): """return identifier for template dotfile""" return BLOCK_START
class Importer: def __init__(self, profile, conf, dotpath, diff_cmd, dry=False, safe=True, debug=False, keepdot=True, ignore=[]): """constructor @profile: the selected profile @conf: configuration manager @dotpath: dotfiles dotpath @diff_cmd: diff command to use @dry: simulate @safe: ask for overwrite if True @debug: enable debug @keepdot: keep dot prefix @ignore: patterns to ignore when importing """ self.profile = profile self.conf = conf self.dotpath = dotpath self.diff_cmd = diff_cmd self.dry = dry self.safe = safe self.debug = debug self.keepdot = keepdot self.ignore = ignore self.umask = get_umask() self.log = Logger() def import_path(self, path, import_as=None, import_link=LinkTypes.NOLINK, import_mode=False): """ import a dotfile pointed by path returns: 1: 1 dotfile imported 0: ignored -1: error """ if self.debug: self.log.dbg('import {}'.format(path)) if not os.path.exists(path): self.log.err('\"{}\" does not exist, ignored!'.format(path)) return -1 return self._import(path, import_as=import_as, import_link=import_link, import_mode=import_mode) def _import(self, path, import_as=None, import_link=LinkTypes.NOLINK, import_mode=False): """ import path returns: 1: 1 dotfile imported 0: ignored -1: error """ # normalize path dst = path.rstrip(os.sep) dst = os.path.abspath(dst) # test if must be ignored if self._ignore(dst): return 0 # ask confirmation for symlinks if self.safe: realdst = os.path.realpath(dst) if dst != realdst: msg = '\"{}\" is a symlink, dereference it and continue?' if not self.log.ask(msg.format(dst)): return 0 # create src path src = strip_home(dst) if import_as: # handle import as src = os.path.expanduser(import_as) src = src.rstrip(os.sep) src = os.path.abspath(src) src = strip_home(src) if self.debug: self.log.dbg('import src for {} as {}'.format(dst, src)) # with or without dot prefix strip = '.' + os.sep if self.keepdot: strip = os.sep src = src.lstrip(strip) # get the permission perm = get_file_perm(dst) # get the link attribute linktype = import_link if linktype == LinkTypes.LINK_CHILDREN and \ not os.path.isdir(path): self.log.err('importing \"{}\" failed!'.format(path)) return -1 if self._already_exists(src, dst): return -1 if self.debug: self.log.dbg('import dotfile: src:{} dst:{}'.format(src, dst)) if not self._prepare_hierarchy(src, dst): return -1 # handle file mode chmod = None dflperm = get_default_file_perms(dst, self.umask) if self.debug: self.log.dbg('import mode: {}'.format(import_mode)) if import_mode or perm != dflperm: if self.debug: msg = 'adopt mode {:o} (umask {:o})' self.log.dbg(msg.format(perm, dflperm)) chmod = perm # add file to config file retconf = self.conf.new_dotfile(src, dst, linktype, chmod=chmod) if not retconf: self.log.warn('\"{}\" ignored during import'.format(path)) return 0 self.log.sub('\"{}\" imported'.format(path)) return 1 def _prepare_hierarchy(self, src, dst): """prepare hierarchy for dotfile""" srcf = os.path.join(self.dotpath, src) if self._ignore(srcf): return False srcfd = os.path.dirname(srcf) if self._ignore(srcfd): return False # a dotfile in dotpath already exists at that spot if os.path.exists(srcf): if self.safe: c = Comparator(debug=self.debug, diff_cmd=self.diff_cmd) diff = c.compare(srcf, dst) if diff != '': # files are different, dunno what to do self.log.log('diff \"{}\" VS \"{}\"'.format(dst, srcf)) self.log.emph(diff) # ask user msg = 'Dotfile \"{}\" already exists, overwrite?' if not self.log.ask(msg.format(srcf)): return False if self.debug: self.log.dbg('will overwrite existing file') # create directory hierarchy if self.dry: cmd = 'mkdir -p {}'.format(srcfd) self.log.dry('would run: {}'.format(cmd)) else: try: os.makedirs(srcfd, exist_ok=True) except Exception: self.log.err('importing \"{}\" failed!'.format(dst)) return False if self.dry: self.log.dry('would copy {} to {}'.format(dst, srcf)) else: # copy the file to the dotpath try: if os.path.isdir(dst): if os.path.exists(srcf): shutil.rmtree(srcf) ig = shutil.ignore_patterns(*self.ignore) shutil.copytree(dst, srcf, copy_function=self._cp, ignore=ig) else: shutil.copy2(dst, srcf) except shutil.Error as e: src = e.args[0][0][0] why = e.args[0][0][2] self.log.err('importing \"{}\" failed: {}'.format(src, why)) return True def _cp(self, src, dst): """the copy function for copytree""" # test if must be ignored if self._ignore(src): return shutil.copy2(src, dst) def _already_exists(self, src, dst): """ test no other dotfile exists with same dst for this profile but different src """ dfs = self.conf.get_dotfile_by_dst(dst) if not dfs: return False for df in dfs: profiles = self.conf.get_profiles_by_dotfile_key(df.key) profiles = [x.key for x in profiles] if self.profile in profiles and \ not self.conf.get_dotfile_by_src_dst(src, dst): # same profile # different src self.log.err('duplicate dotfile for this profile') return True return False def _ignore(self, path): if must_ignore([path], self.ignore, debug=self.debug): if self.debug: self.log.dbg('ignoring import of {}'.format(path)) self.log.warn('{} ignored'.format(path)) return True return False
class Options(AttrMonitor): """dotdrop options manager""" def __init__(self, args=None): """constructor @args: argument dictionary (if None use sys) """ # attributes gotten from self.conf.get_settings() self.banner = None self.showdiff = None self.default_actions = [] self.instignore = None self.force_chmod = None self.cmpignore = None self.impignore = None self.upignore = None self.link_on_import = None self.chmod_on_import = None self.check_version = None self.clear_workdir = None self.key_prefix = None self.key_separator = None # args parsing self.args = {} if not args: self.args = docopt(USAGE, version=VERSION) if args: self.args = args.copy() self.debug = self.args['--verbose'] or ENV_DEBUG in os.environ self.log = Logger(debug=self.debug) self.dry = self.args['--dry'] if ENV_NODEBUG in os.environ: # force disabling debugs self.debug = False self.profile = self.args['--profile'] self.confpath = self._get_config_path() if not self.confpath: raise YamlException('no config file found') self.log.dbg('#################################################') self.log.dbg('#################### DOTDROP ####################') self.log.dbg('#################################################') self.log.dbg('version: {}'.format(VERSION)) self.log.dbg('command: {}'.format(' '.join(sys.argv))) self.log.dbg('config file: {}'.format(self.confpath)) self._read_config() self._apply_args() self._fill_attr() if ENV_NOBANNER not in os.environ \ and self.banner \ and not self.args['--no-banner']: self._header() self._debug_attr() # start monitoring for bad attribute self._set_attr_err = True @classmethod def _get_config_from_fs(cls): """get config from filesystem""" # look in ~/.config/dotdrop cfg = os.path.expanduser(HOMECFG) path = os.path.join(cfg, CONFIG) if os.path.exists(path): return path # look in /etc/xdg/dotdrop path = os.path.join(ETCXDGCFG, CONFIG) if os.path.exists(path): return path # look in /etc/dotdrop path = os.path.join(ETCCFG, CONFIG) if os.path.exists(path): return path return '' def _get_config_path(self): """get the config path""" # cli provided if self.args['--cfg']: return os.path.expanduser(self.args['--cfg']) # environment variable provided if ENV_CONFIG in os.environ: return os.path.expanduser(os.environ[ENV_CONFIG]) # look in current directory if os.path.exists(CONFIG): return CONFIG # look in XDG_CONFIG_HOME if ENV_XDG in os.environ: cfg = os.path.expanduser(os.environ[ENV_XDG]) path = os.path.join(cfg, NAME, CONFIG) if os.path.exists(path): return path return self._get_config_from_fs() def _header(self): """display the header""" self.log.log(BANNER) self.log.log('') def _read_config(self): """read the config file""" self.conf = Cfg(self.confpath, self.profile, debug=self.debug, dry=self.dry) # transform the config settings to self attribute settings = self.conf.get_settings() debug_dict('effective settings', settings, self.debug) for k, val in settings.items(): setattr(self, k, val) def _apply_args_files(self): """files specifics""" self.files_templateonly = self.args['--template'] self.files_grepable = self.args['--grepable'] def _apply_args_install(self): """install specifics""" self.install_force_action = self.args['--force-actions'] self.install_temporary = self.args['--temp'] self.install_keys = self.args['<key>'] self.install_diff = not self.args['--nodiff'] self.install_showdiff = self.showdiff or self.args['--showdiff'] self.install_backup_suffix = BACKUP_SUFFIX self.install_default_actions_pre = [a for a in self.default_actions if a.kind == Action.pre] self.install_default_actions_post = [a for a in self.default_actions if a.kind == Action.post] self.install_ignore = self.instignore self.install_force_chmod = self.force_chmod self.install_clear_workdir = self.args['--workdir-clear'] or \ self.clear_workdir def _apply_args_compare(self): """compare specifics""" self.compare_focus = self.args['--file'] self.compare_ignore = self.args['--ignore'] self.compare_ignore.extend(self.cmpignore) self.compare_ignore.append('*{}'.format(self.install_backup_suffix)) self.compare_ignore = uniq_list(self.compare_ignore) self.compare_fileonly = self.args['--file-only'] self.ignore_missing_in_dotdrop = self.ignore_missing_in_dotdrop or \ self.args['--ignore-missing'] def _apply_args_import(self): """import specifics""" self.import_path = self.args['<path>'] self.import_as = self.args['--as'] self.import_mode = self.args['--preserve-mode'] or self.chmod_on_import self.import_ignore = self.args['--ignore'] self.import_ignore.extend(self.impignore) self.import_ignore.append('*{}'.format(self.install_backup_suffix)) self.import_ignore = uniq_list(self.import_ignore) def _apply_args_update(self): """update specifics""" self.update_path = self.args['<path>'] self.update_iskey = self.args['--key'] self.update_ignore = self.args['--ignore'] self.update_ignore.extend(self.upignore) self.update_ignore.append('*{}'.format(self.install_backup_suffix)) self.update_ignore = uniq_list(self.update_ignore) self.update_showpatch = self.args['--show-patch'] def _apply_args_profiles(self): """profiles specifics""" self.profiles_grepable = self.args['--grepable'] def _apply_args_remove(self): """remove specifics""" self.remove_path = self.args['<path>'] self.remove_iskey = self.args['--key'] def _apply_args_detail(self): """detail specifics""" self.detail_keys = self.args['<key>'] def _apply_args(self): """apply cli args as attribute""" # the commands self.cmd_profiles = self.args['profiles'] self.cmd_files = self.args['files'] self.cmd_install = self.args['install'] self.cmd_compare = self.args['compare'] self.cmd_import = self.args['import'] self.cmd_update = self.args['update'] self.cmd_detail = self.args['detail'] self.cmd_remove = self.args['remove'] # adapt attributes based on arguments self.safe = not self.args['--force'] try: if ENV_WORKERS in os.environ: workers = int(os.environ[ENV_WORKERS]) else: workers = int(self.args['--workers']) self.workers = workers except ValueError: self.log.err('bad option for --workers') sys.exit(USAGE) # import link default value self.import_link = self.link_on_import if self.args['--link']: # overwrite default import link with cli switch link = self.args['--link'] if link not in OPT_LINK: self.log.err('bad option for --link: {}'.format(link)) sys.exit(USAGE) self.import_link = OPT_LINK[link] # "files" specifics self._apply_args_files() # "install" specifics self._apply_args_install() # "compare" specifics self._apply_args_compare() # "import" specifics self._apply_args_import() # "update" specifics self._apply_args_update() # "profiles" specifics self._apply_args_profiles() # "detail" specifics self._apply_args_detail() # "remove" specifics self._apply_args_remove() def _fill_attr(self): """create attributes from conf""" # variables self.variables = self.conf.get_variables() # the dotfiles self.dotfiles = self.conf.get_dotfiles() # the profiles self.profiles = self.conf.get_profiles() def _debug_attr(self): """debug display all of this class attributes""" if not self.debug: return self.log.dbg('effective options:') for att in dir(self): if att.startswith('_'): continue val = getattr(self, att) if callable(val): continue if isinstance(val, list): debug_list('-> {}'.format(att), val, self.debug) elif isinstance(val, dict): debug_dict('-> {}'.format(att), val, self.debug) else: self.log.dbg('-> {}: {}'.format(att, val)) def _attr_set(self, attr): """error when some inexistent attr is set""" raise Exception('bad option: {}'.format(attr))
class Templategen: def __init__(self, profile='', base='.', variables={}, debug=False): self.base = base.rstrip(os.sep) self.debug = debug self.log = Logger() loader = FileSystemLoader(self.base) self.env = Environment(loader=loader, trim_blocks=True, lstrip_blocks=True, keep_trailing_newline=True, block_start_string=BLOCK_START, block_end_string=BLOCK_END, variable_start_string=VAR_START, variable_end_string=VAR_END, comment_start_string=COMMENT_START, comment_end_string=COMMENT_END) # adding variables self.env.globals['env'] = os.environ if profile: self.env.globals['profile'] = profile self.env.globals.update(variables) # adding header method self.env.globals['header'] = self._header # adding helper methods self.env.globals['exists'] = jhelpers.exists def generate(self, src): """render template from path""" if not os.path.exists(src): return '' return self._handle_file(src) def generate_string(self, string): """render template from string""" if not string: return '' return self.env.from_string(string).render() def _header(self, prepend=''): """add a comment usually in the header of a dotfile""" return '{}{}'.format(prepend, utils.header()) def _handle_file(self, src): """generate the file content from template""" _, filetype = utils.run(['file', '-b', src], raw=False, debug=self.debug) filetype = filetype.strip() if self.debug: self.log.dbg('\"{}\" filetype: {}'.format(src, filetype)) istext = 'text' in filetype if self.debug: self.log.dbg('\"{}\" is text: {}'.format(src, istext)) if not istext: return self._handle_bin_file(src) return self._handle_text_file(src) def _handle_text_file(self, src): """write text to file""" template_rel_path = os.path.relpath(src, self.base) try: template = self.env.get_template(template_rel_path) content = template.render() except UnicodeDecodeError: data = self._read_bad_encoded_text(src) template = self.env.from_string(data) content = template.render() content = content.encode('UTF-8') return content def _handle_bin_file(self, src): """write binary to file""" # this is dirty if not src.startswith(self.base): src = os.path.join(self.base, src) with open(src, 'rb') as f: content = f.read() return content def _read_bad_encoded_text(self, path): """decode non utf-8 data""" with open(path, 'rb') as f: data = f.read() return data.decode('utf-8', 'replace') @staticmethod def is_template(path): """recursively check if any file is a template within path""" path = os.path.expanduser(path) if not os.path.exists(path): return False if os.path.isfile(path): # is file return Templategen._is_template(path) for entry in os.listdir(path): fpath = os.path.join(path, entry) if not os.path.isfile(fpath): # rec explore dir if Templategen.is_template(fpath): return True else: # is file a template if Templategen._is_template(fpath): return True return False @staticmethod def _is_template(path): """test if file pointed by path is a template""" if not os.path.isfile(path): return False try: with open(path, 'r') as f: data = f.read() except UnicodeDecodeError: # is binary so surely no template return False markers = [BLOCK_START, VAR_START, COMMENT_START] for marker in markers: if marker in data: return True return False
class Updater: def __init__(self, dotpath, variables, conf, dry=False, safe=True, debug=False, ignore=[], showpatch=False, ignore_missing_in_dotdrop=False): """constructor @dotpath: path where dotfiles are stored @variables: dictionary of variables for the templates @conf: configuration manager @dry: simulate @safe: ask for overwrite if True @debug: enable debug @ignore: pattern to ignore when updating @showpatch: show patch if dotfile to update is a template """ self.dotpath = dotpath self.variables = variables self.conf = conf self.dry = dry self.safe = safe self.debug = debug self.ignore = ignore self.ignores = None self.showpatch = showpatch self.ignore_missing_in_dotdrop = ignore_missing_in_dotdrop self.templater = Templategen(variables=self.variables, base=self.dotpath, debug=self.debug) # save template vars self.tvars = self.templater.add_tmp_vars() self.log = Logger() def update_path(self, path): """update the dotfile installed on path""" path = os.path.expanduser(path) if not os.path.lexists(path): self.log.err('\"{}\" does not exist!'.format(path)) return False dotfiles = self.conf.get_dotfile_by_dst(path) if not dotfiles: return False for dotfile in dotfiles: if not dotfile: msg = 'invalid dotfile for update: {}' self.log.err(msg.format(dotfile.key)) return False if self.debug: msg = 'updating {} from path \"{}\"' self.log.dbg(msg.format(dotfile, path)) if not self._update(path, dotfile): return False return True def update_key(self, key): """update the dotfile referenced by key""" dotfile = self.conf.get_dotfile(key) if not dotfile: return False if self.debug: self.log.dbg('updating {} from key \"{}\"'.format(dotfile, key)) path = self.conf.path_to_dotfile_dst(dotfile.dst) return self._update(path, dotfile) def _update(self, path, dotfile): """update dotfile from file pointed by path""" ret = False new_path = None ignores = list(set(self.ignore + dotfile.upignore)) self.ignores = patch_ignores(ignores, dotfile.dst, debug=self.debug) if self.debug: self.log.dbg('ignore pattern(s): {}'.format(self.ignores)) deployed_path = os.path.expanduser(path) local_path = os.path.join(self.dotpath, dotfile.src) local_path = os.path.expanduser(local_path) if not os.path.exists(deployed_path): msg = '\"{}\" does not exist' self.log.err(msg.format(deployed_path)) return False if not os.path.exists(local_path): msg = '\"{}\" does not exist, import it first' self.log.err(msg.format(local_path)) return False ignore_missing_in_dotdrop = self.ignore_missing_in_dotdrop or \ dotfile.ignore_missing_in_dotdrop if (ignore_missing_in_dotdrop and not os.path.exists(local_path)) or \ self._ignore([deployed_path, local_path]): self.log.sub('\"{}\" ignored'.format(dotfile.key)) return True # apply write transformation if any new_path = self._apply_trans_w(deployed_path, dotfile) if not new_path: return False # save current rights deployed_mode = get_file_perm(deployed_path) local_mode = get_file_perm(local_path) # handle the pointed file if os.path.isdir(new_path): ret = self._handle_dir(new_path, local_path, dotfile) else: ret = self._handle_file(new_path, local_path, dotfile) if deployed_mode != local_mode: # mirror rights if self.debug: m = 'adopt mode {:o} for {}' self.log.dbg(m.format(deployed_mode, dotfile.key)) r = self.conf.update_dotfile(dotfile.key, deployed_mode) if r: ret = True # clean temporary files if new_path != deployed_path and os.path.exists(new_path): removepath(new_path, logger=self.log) return ret def _apply_trans_w(self, path, dotfile): """apply write transformation to dotfile""" trans = dotfile.get_trans_w() if not trans: return path if self.debug: self.log.dbg('executing write transformation {}'.format(trans)) tmp = get_unique_tmp_name() self.templater.restore_vars(self.tvars) newvars = dotfile.get_dotfile_variables() self.templater.add_tmp_vars(newvars=newvars) if not trans.transform( path, tmp, templater=self.templater, debug=self.debug): msg = 'transformation \"{}\" failed for {}' self.log.err(msg.format(trans.key, dotfile.key)) if os.path.exists(tmp): removepath(tmp, logger=self.log) return None return tmp def _is_template(self, path): if not Templategen.is_template(path, ignore=self.ignores): if self.debug: self.log.dbg('{} is NO template'.format(path)) return False self.log.warn('{} uses template, update manually'.format(path)) return True def _show_patch(self, fpath, tpath): """provide a way to manually patch the template""" content = self._resolve_template(tpath) tmp = write_to_tmpfile(content) mirror_file_rights(tpath, tmp) cmds = ['diff', '-u', tmp, fpath, '|', 'patch', tpath] self.log.warn('try patching with: \"{}\"'.format(' '.join(cmds))) return False def _resolve_template(self, tpath): """resolve the template to a temporary file""" self.templater.restore_vars(self.tvars) return self.templater.generate(tpath) def _same_rights(self, left, right): """return True if files have the same modes""" try: lefts = get_file_perm(left) rights = get_file_perm(right) return lefts == rights except OSError as e: self.log.err(e) return False def _mirror_rights(self, src, dst): srcr = get_file_perm(src) dstr = get_file_perm(dst) if srcr == dstr: return if self.debug: msg = 'copy rights from {} ({:o}) to {} ({:o})' self.log.dbg(msg.format(src, srcr, dst, dstr)) try: mirror_file_rights(src, dst) except OSError as e: self.log.err(e) def _handle_file(self, deployed_path, local_path, dotfile, compare=True): """sync path (deployed file) and local_path (dotdrop dotfile path)""" if self._ignore([deployed_path, local_path]): self.log.sub('\"{}\" ignored'.format(local_path)) return True if self.debug: self.log.dbg('update for file {} and {}'.format( deployed_path, local_path, )) if self._is_template(local_path): # dotfile is a template if self.debug: self.log.dbg('{} is a template'.format(local_path)) if self.showpatch: try: self._show_patch(deployed_path, local_path) except UndefinedException as e: msg = 'unable to show patch for {}: {}'.format( deployed_path, e, ) self.log.warn(msg) return False if compare and \ filecmp.cmp(deployed_path, local_path, shallow=False) and \ self._same_rights(deployed_path, local_path): # no difference if self.debug: self.log.dbg('identical files: {} and {}'.format( deployed_path, local_path, )) return True if not self._overwrite(deployed_path, local_path): return False try: if self.dry: self.log.dry('would cp {} {}'.format( deployed_path, local_path, )) else: if self.debug: self.log.dbg('cp {} {}'.format(deployed_path, local_path)) shutil.copyfile(deployed_path, local_path) self._mirror_rights(deployed_path, local_path) self.log.sub('\"{}\" updated'.format(local_path)) except IOError as e: self.log.warn('{} update failed, do manually: {}'.format( deployed_path, e)) return False return True def _handle_dir(self, deployed_path, local_path, dotfile): """sync path (local dir) and local_path (dotdrop dir path)""" if self.debug: self.log.dbg('handle update for dir {} to {}'.format( deployed_path, local_path, )) # paths must be absolute (no tildes) deployed_path = os.path.expanduser(deployed_path) local_path = os.path.expanduser(local_path) if self._ignore([deployed_path, local_path]): self.log.sub('\"{}\" ignored'.format(local_path)) return True # find the differences diff = filecmp.dircmp(deployed_path, local_path, ignore=None) # handle directories diff ret = self._merge_dirs(diff, dotfile) self._mirror_rights(deployed_path, local_path) return ret def _merge_dirs(self, diff, dotfile): """Synchronize directories recursively.""" left, right = diff.left, diff.right if self.debug: self.log.dbg('sync dir {} to {}'.format(left, right)) if self._ignore([left, right]): return True ignore_missing_in_dotdrop = self.ignore_missing_in_dotdrop or \ dotfile.ignore_missing_in_dotdrop # create dirs that don't exist in dotdrop for toadd in diff.left_only: exist = os.path.join(left, toadd) if not os.path.isdir(exist): # ignore files for now continue # match to dotdrop dotpath new = os.path.join(right, toadd) if (ignore_missing_in_dotdrop and not os.path.exists(new)) or \ self._ignore([exist, new]): self.log.sub('\"{}\" ignored'.format(exist)) continue if self.dry: self.log.dry('would cp -r {} {}'.format(exist, new)) continue if self.debug: self.log.dbg('cp -r {} {}'.format(exist, new)) # Newly created directory should be copied as is (for efficiency). def ig(src, names): whitelist, blacklist = set(), set() for ignore in self.ignores: for name in names: path = os.path.join(src, name) if ignore.startswith('!') and \ fnmatch.fnmatch(path, ignore[1:]): # add to whitelist whitelist.add(name) elif fnmatch.fnmatch(path, ignore): # add to blacklist blacklist.add(name) return blacklist - whitelist shutil.copytree(exist, new, ignore=ig) self.log.sub('\"{}\" dir added'.format(new)) # remove dirs that don't exist in deployed version for toremove in diff.right_only: old = os.path.join(right, toremove) if not os.path.isdir(old): # ignore files for now continue if self._ignore([old]): continue if self.dry: self.log.dry('would rm -r {}'.format(old)) continue if self.debug: self.log.dbg('rm -r {}'.format(old)) if not self._confirm_rm_r(old): continue removepath(old, logger=self.log) self.log.sub('\"{}\" dir removed'.format(old)) # handle files diff # sync files that exist in both but are different fdiff = diff.diff_files fdiff.extend(diff.funny_files) fdiff.extend(diff.common_funny) for f in fdiff: fleft = os.path.join(left, f) fright = os.path.join(right, f) if (ignore_missing_in_dotdrop and not os.path.exists(fright)) or \ self._ignore([fleft, fright]): continue if self.dry: self.log.dry('would cp {} {}'.format(fleft, fright)) continue if self.debug: self.log.dbg('cp {} {}'.format(fleft, fright)) self._handle_file(fleft, fright, dotfile, compare=False) # copy files that don't exist in dotdrop for toadd in diff.left_only: exist = os.path.join(left, toadd) if os.path.isdir(exist): # ignore dirs, done above continue new = os.path.join(right, toadd) if (ignore_missing_in_dotdrop and not os.path.exists(new)) or \ self._ignore([exist, new]): continue if self.dry: self.log.dry('would cp {} {}'.format(exist, new)) continue if self.debug: self.log.dbg('cp {} {}'.format(exist, new)) shutil.copyfile(exist, new) self._mirror_rights(exist, new) self.log.sub('\"{}\" added'.format(new)) # remove files that don't exist in deployed version for toremove in diff.right_only: new = os.path.join(right, toremove) if not os.path.exists(new): continue if os.path.isdir(new): # ignore dirs, done above continue if self._ignore([new]): continue if self.dry: self.log.dry('would rm {}'.format(new)) continue if self.debug: self.log.dbg('rm {}'.format(new)) removepath(new, logger=self.log) self.log.sub('\"{}\" removed'.format(new)) # compare rights for common in diff.common_files: leftf = os.path.join(left, common) rightf = os.path.join(right, common) if not self._same_rights(leftf, rightf): self._mirror_rights(leftf, rightf) # Recursively decent into common subdirectories. for subdir in diff.subdirs.values(): self._merge_dirs(subdir, dotfile) # Nothing more to do here. return True def _overwrite(self, src, dst): """ask for overwritting""" msg = 'Overwrite \"{}\" with \"{}\"?'.format(dst, src) if self.safe and not self.log.ask(msg): return False return True def _confirm_rm_r(self, directory): """ask for rm -r directory""" msg = 'Recursively remove \"{}\"?'.format(directory) if self.safe and not self.log.ask(msg): return False return True def _ignore(self, paths): if must_ignore(paths, self.ignores, debug=self.debug): if self.debug: self.log.dbg('ignoring update for {}'.format(paths)) return True return False
class Options(AttrMonitor): def __init__(self, args=None): """constructor @args: argument dictionary (if None use sys) """ self.args = {} if not args: self.args = docopt(USAGE, version=VERSION) if args: self.args = args.copy() self.log = Logger() self.debug = self.args['--verbose'] or ENV_DEBUG in os.environ self.dry = self.args['--dry'] if ENV_NODEBUG in os.environ: # force disabling debugs self.debug = False self.profile = self.args['--profile'] self.confpath = self._get_config_path() if self.debug: self.log.dbg('version: {}'.format(VERSION)) self.log.dbg('command: {}'.format(' '.join(sys.argv))) self.log.dbg('config file: {}'.format(self.confpath)) self._read_config() self._apply_args() self._fill_attr() if ENV_NOBANNER not in os.environ \ and self.banner \ and not self.args['--no-banner']: self._header() self._debug_attr() # start monitoring for bad attribute self._set_attr_err = True def _get_config_path(self): """get the config path""" # cli provided if self.args['--cfg']: return os.path.expanduser(self.args['--cfg']) # environment variable provided if ENV_CONFIG in os.environ: return os.path.expanduser(os.environ[ENV_CONFIG]) # look in current directory if os.path.exists(CONFIG): return CONFIG # look in XDG_CONFIG_HOME if ENV_XDG in os.environ: cfg = os.path.expanduser(os.environ[ENV_XDG]) path = os.path.join(cfg, NAME, CONFIG) if os.path.exists(path): return path # look in ~/.config/dotdrop cfg = os.path.expanduser(HOMECFG) path = os.path.join(cfg, CONFIG) if os.path.exists(path): return path # look in /etc/xdg/dotdrop path = os.path.join(ETCXDGCFG, CONFIG) if os.path.exists(path): return path # look in /etc/dotdrop path = os.path.join(ETCCFG, CONFIG) if os.path.exists(path): return path return '' def _header(self): """display the header""" self.log.log(BANNER) self.log.log('') def _read_config(self): """read the config file""" self.conf = Cfg(self.confpath, self.profile, debug=self.debug, dry=self.dry) # transform the config settings to self attribute self._debug_dict('effective settings', self.conf.get_settings()) for k, v in self.conf.get_settings().items(): setattr(self, k, v) def _apply_args(self): """apply cli args as attribute""" # the commands self.cmd_profiles = self.args['profiles'] self.cmd_files = self.args['files'] self.cmd_install = self.args['install'] self.cmd_compare = self.args['compare'] self.cmd_import = self.args['import'] self.cmd_update = self.args['update'] self.cmd_detail = self.args['detail'] self.cmd_remove = self.args['remove'] # adapt attributes based on arguments self.safe = not self.args['--force'] # import link default value self.import_link = self.link_on_import if self.args['--link']: # overwrite default import link with cli switch link = self.args['--link'] if link not in OPT_LINK.keys(): self.log.err('bad option for --link: {}'.format(link)) sys.exit(USAGE) self.import_link = OPT_LINK[link] # "files" specifics self.files_templateonly = self.args['--template'] self.files_grepable = self.args['--grepable'] # "profiles" specifics self.profiles_grepable = self.args['--grepable'] # "install" specifics self.install_force_action = self.args['--force-actions'] self.install_temporary = self.args['--temp'] self.install_keys = self.args['<key>'] self.install_diff = not self.args['--nodiff'] self.install_showdiff = self.showdiff or self.args['--showdiff'] self.install_backup_suffix = BACKUP_SUFFIX self.install_default_actions_pre = [ a for a in self.default_actions if a.kind == Action.pre ] self.install_default_actions_post = [ a for a in self.default_actions if a.kind == Action.post ] self.install_ignore = self.instignore # "compare" specifics self.compare_focus = self.args['--file'] self.compare_ignore = self.args['--ignore'] self.compare_ignore.extend(self.cmpignore) self.compare_ignore.append('*{}'.format(self.install_backup_suffix)) self.compare_ignore = uniq_list(self.compare_ignore) # "import" specifics self.import_path = self.args['<path>'] self.import_as = self.args['--as'] # "update" specifics self.update_path = self.args['<path>'] self.update_iskey = self.args['--key'] self.update_ignore = self.args['--ignore'] self.update_ignore.extend(self.upignore) self.update_ignore.append('*{}'.format(self.install_backup_suffix)) self.update_ignore = uniq_list(self.update_ignore) self.update_showpatch = self.args['--show-patch'] # "detail" specifics self.detail_keys = self.args['<key>'] # "remove" specifics self.remove_path = self.args['<path>'] self.remove_iskey = self.args['--key'] def _fill_attr(self): """create attributes from conf""" # variables self.variables = self.conf.get_variables() # the dotfiles self.dotfiles = self.conf.get_dotfiles() # the profiles self.profiles = self.conf.get_profiles() def _debug_attr(self): """debug display all of this class attributes""" if not self.debug: return self.log.dbg('effective options:') for att in dir(self): if att.startswith('_'): continue val = getattr(self, att) if callable(val): continue if type(val) is list: self._debug_list('-> {}'.format(att), val) elif type(val) is dict: self._debug_dict('-> {}'.format(att), val) else: self.log.dbg('-> {}: {}'.format(att, val)) def _attr_set(self, attr): """error when some inexistent attr is set""" raise Exception('bad option: {}'.format(attr)) def _debug_list(self, title, elems): """pretty print list""" if not self.debug: return self.log.dbg('{}:'.format(title)) for e in elems: self.log.dbg('\t- {}'.format(e)) def _debug_dict(self, title, elems): """pretty print dict""" if not self.debug: return self.log.dbg('{}:'.format(title)) for k, v in elems.items(): if type(v) is list: self.log.dbg('\t- \"{}\":'.format(k)) for i in v: self.log.dbg('\t\t- {}'.format(i)) else: self.log.dbg('\t- \"{}\": {}'.format(k, v))
class Comparator: def __init__(self, diffopts='', debug=False): """constructor @diffopts: cli switches to pass to unix diff @debug: enable debug """ self.diffopts = diffopts self.debug = debug self.log = Logger() def compare(self, left, right, ignore=[]): """diff left (dotdrop dotfile) and right (deployed file)""" left = os.path.expanduser(left) right = os.path.expanduser(right) if self.debug: self.log.dbg('comparing {} and {}'.format(left, right)) self.log.dbg('ignore pattern(s): {}'.format(ignore)) if not os.path.isdir(left): return self._comp_file(left, right, ignore) return self._comp_dir(left, right, ignore) def _comp_file(self, left, right, ignore): """compare a file""" if utils.must_ignore([left, right], ignore, debug=self.debug): if self.debug: self.log.dbg('ignoring diff {} and {}'.format(left, right)) return '' return self._diff(left, right) def _comp_dir(self, left, right, ignore): """compare a directory""" if not os.path.exists(right): return '' if utils.must_ignore([left, right], ignore, debug=self.debug): if self.debug: self.log.dbg('ignoring diff {} and {}'.format(left, right)) return '' if self.debug: self.log.dbg('compare {} and {}'.format(left, right)) ret = [] comp = filecmp.dircmp(left, right) # handle files only in deployed file for i in comp.left_only: if utils.must_ignore([os.path.join(left, i)], ignore, debug=self.debug): continue ret.append('=> \"{}\" does not exist on local\n'.format(i)) for i in comp.right_only: if utils.must_ignore([os.path.join(right, i)], ignore, debug=self.debug): continue ret.append('=> \"{}\" does not exist in dotdrop\n'.format(i)) # same left and right but different type funny = comp.common_funny for i in funny: lfile = os.path.join(left, i) rfile = os.path.join(right, i) short = os.path.basename(lfile) # file vs dir ret.append('different type: \"{}\"\n'.format(short)) # content is different funny = comp.diff_files funny.extend(comp.funny_files) funny = list(set(funny)) for i in funny: lfile = os.path.join(left, i) rfile = os.path.join(right, i) diff = self._diff(lfile, rfile, header=True) ret.append(diff) return ''.join(ret) def _diff(self, left, right, header=False): """diff using the unix tool diff""" diff = utils.diff(left, right, raw=False, opts=self.diffopts, debug=self.debug) if header: lshort = os.path.basename(left) rshort = os.path.basename(right) diff = '=> diff \"{}\":\n{}'.format(lshort, diff) return diff
class CfgAggregator: file_prefix = 'f' dir_prefix = 'd' key_sep = '_' def __init__(self, path, profile=None, debug=False): """ high level config parser @path: path to the config file @profile: selected profile @debug: debug flag """ self.path = path self.profile = profile self.debug = debug self.log = Logger() self._load() def _load(self): """load lower level config""" self.cfgyaml = CfgYaml(self.path, self.profile, debug=self.debug) # settings self.settings = Settings.parse(None, self.cfgyaml.settings) if self.debug: self.log.dbg('settings: {}'.format(self.settings)) # dotfiles self.dotfiles = Dotfile.parse_dict(self.cfgyaml.dotfiles) if self.debug: self.log.dbg('dotfiles: {}'.format(self.dotfiles)) # profiles self.profiles = Profile.parse_dict(self.cfgyaml.profiles) if self.debug: self.log.dbg('profiles: {}'.format(self.profiles)) # actions self.actions = Action.parse_dict(self.cfgyaml.actions) if self.debug: self.log.dbg('actions: {}'.format(self.actions)) # trans_r self.trans_r = Transform.parse_dict(self.cfgyaml.trans_r) if self.debug: self.log.dbg('trans_r: {}'.format(self.trans_r)) # trans_w self.trans_w = Transform.parse_dict(self.cfgyaml.trans_w) if self.debug: self.log.dbg('trans_w: {}'.format(self.trans_w)) # variables self.variables = self.cfgyaml.get_variables() if self.debug: self.log.dbg('variables: {}'.format(self.variables)) # patch dotfiles in profiles self._patch_keys_to_objs(self.profiles, "dotfiles", self.get_dotfile) # patch action in dotfiles actions self._patch_keys_to_objs(self.dotfiles, "actions", self._get_action_w_args) # patch action in profiles actions self._patch_keys_to_objs(self.profiles, "actions", self._get_action_w_args) # patch actions in settings default_actions self._patch_keys_to_objs([self.settings], "default_actions", self._get_action_w_args) if self.debug: msg = 'default actions: {}'.format(self.settings.default_actions) self.log.dbg(msg) # patch trans_w/trans_r in dotfiles self._patch_keys_to_objs(self.dotfiles, "trans_r", self._get_trans_r, islist=False) self._patch_keys_to_objs(self.dotfiles, "trans_w", self._get_trans_w, islist=False) def _patch_keys_to_objs(self, containers, keys, get_by_key, islist=True): """ map for each key in the attribute 'keys' in 'containers' the returned object from the method 'get_by_key' """ if not containers: return if self.debug: self.log.dbg('patching {} ...'.format(keys)) for c in containers: objects = [] okeys = getattr(c, keys) if not okeys: continue if not islist: okeys = [okeys] for k in okeys: o = get_by_key(k) if not o: err = 'bad {} key for \"{}\": {}'.format(keys, c, k) self.log.err(err) raise Exception(err) objects.append(o) if not islist: objects = objects[0] if self.debug: self.log.dbg('patching {}.{} with {}'.format(c, keys, objects)) setattr(c, keys, objects) def del_dotfile(self, dotfile): """remove this dotfile from the config""" return self.cfgyaml.del_dotfile(dotfile.key) def del_dotfile_from_profile(self, dotfile, profile): """remove this dotfile from this profile""" return self.cfgyaml.del_dotfile_from_profile(dotfile.key, profile.key) def new(self, src, dst, link, profile_key): """ import a new dotfile @src: path in dotpath @dst: path in FS @link: LinkType @profile_key: to which profile """ dst = self.path_to_dotfile_dst(dst) dotfile = self.get_dotfile_by_dst(dst) if not dotfile: # get a new dotfile with a unique key key = self._get_new_dotfile_key(dst) if self.debug: self.log.dbg('new dotfile key: {}'.format(key)) # add the dotfile self.cfgyaml.add_dotfile(key, src, dst, link) dotfile = Dotfile(key, dst, src) key = dotfile.key ret = self.cfgyaml.add_dotfile_to_profile(key, profile_key) if self.debug: self.log.dbg('new dotfile {} to profile {}'.format( key, profile_key)) # reload self.cfgyaml.save() if self.debug: self.log.dbg('RELOADING') self._load() return ret def _get_new_dotfile_key(self, dst): """return a new unique dotfile key""" path = os.path.expanduser(dst) existing_keys = [x.key for x in self.dotfiles] if self.settings.longkey: return self._get_long_key(path, existing_keys) return self._get_short_key(path, existing_keys) def _norm_key_elem(self, elem): """normalize path element for sanity""" elem = elem.lstrip('.') elem = elem.replace(' ', '-') return elem.lower() def _split_path_for_key(self, path): """return a list of path elements, excluded home path""" p = strip_home(path) dirs = [] while True: p, f = os.path.split(p) dirs.append(f) if not p or not f: break dirs.reverse() # remove empty entries dirs = filter(None, dirs) # normalize entries return list(map(self._norm_key_elem, dirs)) def _get_long_key(self, path, keys): """ return a unique long key representing the absolute path of path """ dirs = self._split_path_for_key(path) prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix key = self.key_sep.join([prefix] + dirs) return self._uniq_key(key, keys) def _get_short_key(self, path, keys): """ return a unique key where path is known not to be an already existing dotfile """ dirs = self._split_path_for_key(path) dirs.reverse() prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix entries = [] for d in dirs: entries.insert(0, d) key = self.key_sep.join([prefix] + entries) if key not in keys: return key return self._uniq_key(key, keys) def _uniq_key(self, key, keys): """unique dotfile key""" newkey = key cnt = 1 while newkey in keys: # if unable to get a unique path # get a random one newkey = self.key_sep.join([key, str(cnt)]) cnt += 1 return newkey def path_to_dotfile_dst(self, path): """normalize the path to match dotfile dst""" path = os.path.expanduser(path) path = os.path.expandvars(path) path = os.path.abspath(path) home = os.path.expanduser(TILD) + os.sep # normalize the path if path.startswith(home): path = path[len(home):] path = os.path.join(TILD, path) return path def get_dotfile_by_dst(self, dst): """get a dotfile by dst""" try: return next(d for d in self.dotfiles if d.dst == dst) except StopIteration: return None def save(self): """save the config""" return self.cfgyaml.save() def dump(self): """dump the config dictionary""" return self.cfgyaml.dump() def get_settings(self): """return settings as a dict""" return self.settings.serialize()[Settings.key_yaml] def get_variables(self): """return variables""" return self.variables def get_profiles(self): """return profiles""" return self.profiles def get_profile(self, key): """return profile by key""" try: return next(x for x in self.profiles if x.key == key) except StopIteration: return None def get_profiles_by_dotfile_key(self, key): """return all profiles having this dotfile""" res = [] for p in self.profiles: keys = [d.key for d in p.dotfiles] if key in keys: res.append(p) return res def get_dotfiles(self, profile=None): """return dotfiles dict for this profile key""" if not profile: return self.dotfiles try: pro = self.get_profile(profile) if not pro: return [] return pro.dotfiles except StopIteration: return [] def get_dotfile(self, key): """return dotfile by key""" try: return next(x for x in self.dotfiles if x.key == key) except StopIteration: return None def _get_action(self, key): """return action by key""" try: return next(x for x in self.actions if x.key == key) except StopIteration: return None def _get_action_w_args(self, key): """return action by key with the arguments""" fields = shlex.split(key) if len(fields) > 1: # we have args key, *args = fields if self.debug: self.log.dbg('action with parm: {} and {}'.format(key, args)) action = self._get_action(key).copy(args) else: action = self._get_action(key) return action def _get_trans_r(self, key): """return the trans_r with this key""" try: return next(x for x in self.trans_r if x.key == key) except StopIteration: return None def _get_trans_w(self, key): """return the trans_w with this key""" try: return next(x for x in self.trans_w if x.key == key) except StopIteration: return None
class Comparator: def __init__(self, diffopts='', ignore=[], debug=False): self.diffopts = diffopts self.ignore = [os.path.expanduser(i) for i in ignore] self.debug = debug self.log = Logger() def compare(self, left, right): """diff left (dotdrop dotfile) and right (deployed file)""" left = os.path.expanduser(left) right = os.path.expanduser(right) if not os.path.isdir(left): return self._comp_file(left, right) return self._comp_dir(left, right) def _comp_file(self, left, right): """compare a file""" if left in self.ignore or right in self.ignore: if self.debug: self.log.dbg('ignoring diff {} and {}'.format(left, right)) return '' return self._diff(left, right) def _comp_dir(self, left, right): """compare a directory""" if left in self.ignore or right in self.ignore: if self.debug: self.log.dbg('ignoring diff {} and {}'.format(left, right)) return '' if self.debug: self.log.dbg('compare {} and {}'.format(left, right)) ret = [] comp = filecmp.dircmp(left, right, ignore=self.ignore) # handle files only in deployed file for i in comp.left_only: if os.path.join(left, i) in self.ignore: continue ret.append('only in left: \"{}\"\n'.format(i)) for i in comp.right_only: if os.path.join(right, i) in self.ignore: continue ret.append('only in right: \"{}\"\n'.format(i)) # same left and right but different type funny = comp.common_funny for i in funny: lfile = os.path.join(left, i) rfile = os.path.join(right, i) short = os.path.basename(lfile) # file vs dir ret.append('different type: \"{}\"\n'.format(short)) # content is different funny = comp.diff_files funny.extend(comp.funny_files) funny = list(set(funny)) for i in funny: lfile = os.path.join(left, i) rfile = os.path.join(right, i) diff = self._diff(lfile, rfile, header=True) ret.append(diff) return ''.join(ret) def _diff(self, left, right, header=False): """diff using the unix tool diff""" diff = utils.diff(left, right, raw=False, opts=self.diffopts, debug=self.debug) if header: lshort = os.path.basename(left) rshort = os.path.basename(right) diff = 'diff \"{}\":\n{}'.format(lshort, diff) return diff
class CfgYaml: # global entries key_settings = 'config' key_dotfiles = 'dotfiles' key_profiles = 'profiles' key_actions = 'actions' old_key_trans_r = 'trans' key_trans_r = 'trans_read' key_trans_w = 'trans_write' key_variables = 'variables' key_dvariables = 'dynvariables' action_pre = 'pre' action_post = 'post' # profiles/dotfiles entries key_dotfile_src = 'src' key_dotfile_dst = 'dst' key_dotfile_link = 'link' key_dotfile_actions = 'actions' key_dotfile_link_children = 'link_children' key_dotfile_noempty = 'ignoreempty' # profile key_profile_dotfiles = 'dotfiles' key_profile_include = 'include' key_profile_variables = 'variables' key_profile_dvariables = 'dynvariables' key_profile_actions = 'actions' key_all = 'ALL' # import entries key_import_actions = 'import_actions' key_import_configs = 'import_configs' key_import_variables = 'import_variables' key_import_profile_dfs = 'import' # settings key_settings_dotpath = 'dotpath' key_settings_workdir = 'workdir' key_settings_link_dotfile_default = 'link_dotfile_default' key_settings_noempty = 'ignoreempty' key_settings_minversion = 'minversion' key_imp_link = 'link_on_import' # link values lnk_nolink = LinkTypes.NOLINK.name.lower() lnk_link = LinkTypes.LINK.name.lower() lnk_children = LinkTypes.LINK_CHILDREN.name.lower() def __init__(self, path, profile=None, debug=False): """ config parser @path: config file path @profile: the selected profile @debug: debug flag """ self.path = os.path.abspath(path) self.profile = profile self.debug = debug self.log = Logger() # config needs to be written self.dirty = False # indicates the config has been updated self.dirty_deprecated = False if not os.path.exists(path): err = 'invalid config path: \"{}\"'.format(path) if self.debug: self.log.dbg(err) raise YamlException(err) self.yaml_dict = self._load_yaml(self.path) # live patch deprecated entries self._fix_deprecated(self.yaml_dict) # parse to self variables self._parse_main_yaml(self.yaml_dict) if self.debug: self.log.dbg('before normalization: {}'.format(self.yaml_dict)) # resolve variables self.variables, self.prokeys = self._merge_variables() # apply variables self._apply_variables() # process imported variables (import_variables) self._import_variables() # process imported actions (import_actions) self._import_actions() # process imported profile dotfiles (import) self._import_profiles_dotfiles() # process imported configs (import_configs) self._import_configs() # process profile include self._resolve_profile_includes() # process profile ALL self._resolve_profile_all() # patch dotfiles paths self._resolve_dotfile_paths() if self.debug: self.log.dbg('after normalization: {}'.format(self.yaml_dict)) def get_variables(self): """retrieve all variables""" return self.variables ######################################################## # parsing ######################################################## def _parse_main_yaml(self, dic): """parse the different blocks""" self.ori_settings = self._get_entry(dic, self.key_settings) self.settings = Settings(None).serialize().get(self.key_settings) self.settings.update(self.ori_settings) # resolve minimum version if self.key_settings_minversion in self.settings: minversion = self.settings[self.key_settings_minversion] self._check_minversion(minversion) # resolve settings paths p = self._norm_path(self.settings[self.key_settings_dotpath]) self.settings[self.key_settings_dotpath] = p p = self._norm_path(self.settings[self.key_settings_workdir]) self.settings[self.key_settings_workdir] = p if self.debug: self.log.dbg('settings: {}'.format(self.settings)) # dotfiles self.ori_dotfiles = self._get_entry(dic, self.key_dotfiles) self.dotfiles = deepcopy(self.ori_dotfiles) keys = self.dotfiles.keys() if len(keys) != len(list(set(keys))): dups = [x for x in keys if x not in list(set(keys))] err = 'duplicate dotfile keys found: {}'.format(dups) raise YamlException(err) self.dotfiles = self._norm_dotfiles(self.dotfiles) if self.debug: self.log.dbg('dotfiles: {}'.format(self.dotfiles)) # profiles self.ori_profiles = self._get_entry(dic, self.key_profiles) self.profiles = deepcopy(self.ori_profiles) self.profiles = self._norm_profiles(self.profiles) if self.debug: self.log.dbg('profiles: {}'.format(self.profiles)) # actions self.ori_actions = self._get_entry(dic, self.key_actions, mandatory=False) self.actions = deepcopy(self.ori_actions) self.actions = self._norm_actions(self.actions) if self.debug: self.log.dbg('actions: {}'.format(self.actions)) # trans_r key = self.key_trans_r if self.old_key_trans_r in dic: self.log.warn('\"trans\" is deprecated, please use \"trans_read\"') dic[self.key_trans_r] = dic[self.old_key_trans_r] del dic[self.old_key_trans_r] self.ori_trans_r = self._get_entry(dic, key, mandatory=False) self.trans_r = deepcopy(self.ori_trans_r) if self.debug: self.log.dbg('trans_r: {}'.format(self.trans_r)) # trans_w self.ori_trans_w = self._get_entry(dic, self.key_trans_w, mandatory=False) self.trans_w = deepcopy(self.ori_trans_w) if self.debug: self.log.dbg('trans_w: {}'.format(self.trans_w)) # variables self.ori_variables = self._get_entry(dic, self.key_variables, mandatory=False) if self.debug: self.log.dbg('variables: {}'.format(self.ori_variables)) # dynvariables self.ori_dvariables = self._get_entry(dic, self.key_dvariables, mandatory=False) if self.debug: self.log.dbg('dynvariables: {}'.format(self.ori_dvariables)) def _resolve_dotfile_paths(self): """resolve dotfile paths""" t = Templategen(variables=self.variables) for dotfile in self.dotfiles.values(): # src src = dotfile[self.key_dotfile_src] new = t.generate_string(src) if new != src and self.debug: self.log.dbg('dotfile: {} -> {}'.format(src, new)) src = new src = os.path.join(self.settings[self.key_settings_dotpath], src) dotfile[self.key_dotfile_src] = self._norm_path(src) # dst dst = dotfile[self.key_dotfile_dst] new = t.generate_string(dst) if new != dst and self.debug: self.log.dbg('dotfile: {} -> {}'.format(dst, new)) dst = new dotfile[self.key_dotfile_dst] = self._norm_path(dst) def _rec_resolve_vars(self, variables): """recursive resolve variables""" default = self._get_variables_dict(self.profile) t = Templategen(variables=self._merge_dict(default, variables)) for k in variables.keys(): val = variables[k] while Templategen.var_is_template(val): val = t.generate_string(val) variables[k] = val t.update_variables(variables) return variables def _merge_variables(self): """ resolve all variables across the config apply them to any needed entries and return the full list of variables """ if self.debug: self.log.dbg('get local variables') # get all variables from local and resolve var = self._get_variables_dict(self.profile) # get all dynvariables from local and resolve dvar = self._get_dvariables_dict() # temporarly resolve all variables for "include" merged = self._merge_dict(dvar, var) merged = self._rec_resolve_vars(merged) self._debug_vars(merged) # exec dynvariables self._shell_exec_dvars(dvar.keys(), merged) if self.debug: self.log.dbg('local variables resolved') self._debug_vars(merged) # resolve profile includes t = Templategen(variables=merged) for k, v in self.profiles.items(): if self.key_profile_include in v: new = [] for k in v[self.key_profile_include]: new.append(t.generate_string(k)) v[self.key_profile_include] = new # now get the included ones pro_var = self._get_included_variables(self.profile, seen=[self.profile]) pro_dvar = self._get_included_dvariables(self.profile, seen=[self.profile]) # exec incl dynvariables self._shell_exec_dvars(pro_dvar.keys(), pro_dvar) # merge all and resolve merged = self._merge_dict(pro_var, merged) merged = self._merge_dict(pro_dvar, merged) merged = self._rec_resolve_vars(merged) if self.debug: self.log.dbg('resolve all uses of variables in config') self._debug_vars(merged) prokeys = list(pro_var.keys()) + list(pro_dvar.keys()) return merged, prokeys def _apply_variables(self): """template any needed parts of the config""" t = Templategen(variables=self.variables) # import_actions new = [] entries = self.settings.get(self.key_import_actions, []) new = self._template_list(t, entries) if new: self.settings[self.key_import_actions] = new # import_configs entries = self.settings.get(self.key_import_configs, []) new = self._template_list(t, entries) if new: self.settings[self.key_import_configs] = new # import_variables entries = self.settings.get(self.key_import_variables, []) new = self._template_list(t, entries) if new: self.settings[self.key_import_variables] = new # profile's import for k, v in self.profiles.items(): entries = v.get(self.key_import_profile_dfs, []) new = self._template_list(t, entries) if new: v[self.key_import_profile_dfs] = new def _norm_actions(self, actions): """ ensure each action is either pre or post explicitely action entry of the form {action_key: (pre|post, action)} """ if not actions: return actions new = {} for k, v in actions.items(): if k == self.action_pre or k == self.action_post: for key, action in v.items(): new[key] = (k, action) else: new[k] = (self.action_post, v) return new def _norm_profiles(self, profiles): """normalize profiles entries""" if not profiles: return profiles new = {} for k, v in profiles.items(): # add dotfiles entry if not present if self.key_profile_dotfiles not in v: v[self.key_profile_dotfiles] = [] new[k] = v return new def _norm_dotfiles(self, dotfiles): """normalize dotfiles entries""" if not dotfiles: return dotfiles new = {} for k, v in dotfiles.items(): # add 'src' as key' if not present if self.key_dotfile_src not in v: v[self.key_dotfile_src] = k new[k] = v else: new[k] = v # fix deprecated trans key if self.old_key_trans_r in v: msg = '\"trans\" is deprecated, please use \"trans_read\"' self.log.warn(msg) v[self.key_trans_r] = v[self.old_key_trans_r] del v[self.old_key_trans_r] new[k] = v # apply link value if self.key_dotfile_link not in v: val = self.settings[self.key_settings_link_dotfile_default] v[self.key_dotfile_link] = val # apply noempty if undefined if self.key_dotfile_noempty not in v: val = self.settings.get(self.key_settings_noempty, False) v[self.key_dotfile_noempty] = val return new def _get_variables_dict(self, profile): """return enriched variables""" variables = deepcopy(self.ori_variables) # add profile variable if profile: variables['profile'] = profile # add some more variables p = self.settings.get(self.key_settings_dotpath) p = self._norm_path(p) variables['_dotdrop_dotpath'] = p variables['_dotdrop_cfgpath'] = self._norm_path(self.path) p = self.settings.get(self.key_settings_workdir) p = self._norm_path(p) variables['_dotdrop_workdir'] = p return variables def _get_dvariables_dict(self): """return dynvariables""" variables = deepcopy(self.ori_dvariables) return variables def _get_included_variables(self, profile, seen): """return included variables""" variables = {} if not profile or profile not in self.profiles.keys(): return variables # profile entry pentry = self.profiles.get(profile) # inherite profile variables for inherited_profile in pentry.get(self.key_profile_include, []): if inherited_profile == profile or inherited_profile in seen: raise YamlException('\"include\" loop') seen.append(inherited_profile) new = self._get_included_variables(inherited_profile, seen) if self.debug: msg = 'included vars from {}: {}' self.log.dbg(msg.format(inherited_profile, new)) variables.update(new) cur = pentry.get(self.key_profile_variables, {}) return self._merge_dict(cur, variables) def _get_included_dvariables(self, profile, seen): """return included dynvariables""" variables = {} if not profile or profile not in self.profiles.keys(): return variables # profile entry pentry = self.profiles.get(profile) # inherite profile dynvariables for inherited_profile in pentry.get(self.key_profile_include, []): if inherited_profile == profile or inherited_profile in seen: raise YamlException('\"include loop\"') seen.append(inherited_profile) new = self._get_included_dvariables(inherited_profile, seen) if self.debug: msg = 'included dvars from {}: {}' self.log.dbg(msg.format(inherited_profile, new)) variables.update(new) cur = pentry.get(self.key_profile_dvariables, {}) return self._merge_dict(cur, variables) def _resolve_profile_all(self): """resolve some other parts of the config""" # profile -> ALL for k, v in self.profiles.items(): dfs = v.get(self.key_profile_dotfiles, None) if not dfs: continue if self.key_all in dfs: if self.debug: self.log.dbg('add ALL to profile {}'.format(k)) v[self.key_profile_dotfiles] = self.dotfiles.keys() def _resolve_profile_includes(self): # profiles -> include other profile for k, v in self.profiles.items(): self._rec_resolve_profile_include(k) def _rec_resolve_profile_include(self, profile): """ recursively resolve include of other profiles's: * dotfiles * actions """ this_profile = self.profiles[profile] # include dotfiles = this_profile.get(self.key_profile_dotfiles, []) actions = this_profile.get(self.key_profile_actions, []) includes = this_profile.get(self.key_profile_include, None) if not includes: # nothing to include return dotfiles, actions if self.debug: self.log.dbg('{} includes: {}'.format(profile, ','.join(includes))) self.log.dbg('{} dotfiles before include: {}'.format( profile, dotfiles)) self.log.dbg('{} actions before include: {}'.format( profile, actions)) seen = [] for i in uniq_list(includes): # ensure no include loop occurs if i in seen: raise YamlException('\"include loop\"') seen.append(i) # included profile even exists if i not in self.profiles.keys(): self.log.warn('include unknown profile: {}'.format(i)) continue # recursive resolve o_dfs, o_actions = self._rec_resolve_profile_include(i) # merge dotfile keys dotfiles.extend(o_dfs) this_profile[self.key_profile_dotfiles] = uniq_list(dotfiles) # merge actions keys actions.extend(o_actions) this_profile[self.key_profile_actions] = uniq_list(actions) dotfiles = this_profile.get(self.key_profile_dotfiles, []) actions = this_profile.get(self.key_profile_actions, []) if self.debug: self.log.dbg('{} dotfiles after include: {}'.format( profile, dotfiles)) self.log.dbg('{} actions after include: {}'.format( profile, actions)) # since dotfiles and actions are resolved here # and variables have been already done at the beginning # of the parsing, we can clear these include self.profiles[profile][self.key_profile_include] = None return dotfiles, actions ######################################################## # handle imported entries ######################################################## def _import_variables(self): """import external variables from paths""" paths = self.settings.get(self.key_import_variables, None) if not paths: return paths = self._glob_paths(paths) for p in paths: path = self._norm_path(p) if self.debug: self.log.dbg('import variables from {}'.format(path)) var = self._import_sub(path, self.key_variables, mandatory=False) if self.debug: self.log.dbg('import dynvariables from {}'.format(path)) dvar = self._import_sub(path, self.key_dvariables, mandatory=False) merged = self._merge_dict(dvar, var) merged = self._rec_resolve_vars(merged) # execute dvar self._shell_exec_dvars(dvar.keys(), merged) self._clear_profile_vars(merged) self.variables = self._merge_dict(merged, self.variables) def _clear_profile_vars(self, dic): """remove profile variables from dic if found""" [dic.pop(k, None) for k in self.prokeys] def _import_actions(self): """import external actions from paths""" paths = self.settings.get(self.key_import_actions, None) if not paths: return paths = self._glob_paths(paths) for p in paths: path = self._norm_path(p) if self.debug: self.log.dbg('import actions from {}'.format(path)) new = self._import_sub(path, self.key_actions, mandatory=False, patch_func=self._norm_actions) self.actions = self._merge_dict(new, self.actions) def _import_profiles_dotfiles(self): """import profile dotfiles""" for k, v in self.profiles.items(): imp = v.get(self.key_import_profile_dfs, None) if not imp: continue if self.debug: self.log.dbg('import dotfiles for profile {}'.format(k)) paths = self._glob_paths(imp) for p in paths: current = v.get(self.key_dotfiles, []) path = self._norm_path(p) new = self._import_sub(path, self.key_dotfiles, mandatory=False) v[self.key_dotfiles] = new + current def _import_config(self, path): """import config from path""" path = self._norm_path(path) if self.debug: self.log.dbg('import config from {}'.format(path)) sub = CfgYaml(path, profile=self.profile, debug=self.debug) # settings is ignored self.dotfiles = self._merge_dict(self.dotfiles, sub.dotfiles) self.profiles = self._merge_dict(self.profiles, sub.profiles) self.actions = self._merge_dict(self.actions, sub.actions) self.trans_r = self._merge_dict(self.trans_r, sub.trans_r) self.trans_w = self._merge_dict(self.trans_w, sub.trans_w) self._clear_profile_vars(sub.variables) if self.debug: self.log.dbg('add import_configs var: {}'.format(sub.variables)) self.variables = self._merge_dict(sub.variables, self.variables) def _import_configs(self): """import configs from external files""" # settings -> import_configs imp = self.settings.get(self.key_import_configs, None) if not imp: return paths = self._glob_paths(imp) for path in paths: self._import_config(path) def _import_sub(self, path, key, mandatory=False, patch_func=None): """ import the block "key" from "path" patch_func is applied to each element if defined """ if self.debug: self.log.dbg('import \"{}\" from \"{}\"'.format(key, path)) extdict = self._load_yaml(path) new = self._get_entry(extdict, key, mandatory=mandatory) if patch_func: if self.debug: self.log.dbg('calling patch: {}'.format(patch_func)) new = patch_func(new) if not new and mandatory: err = 'no \"{}\" imported from \"{}\"'.format(key, path) self.log.warn(err) raise YamlException(err) if self.debug: self.log.dbg('imported \"{}\": {}'.format(key, new)) return new ######################################################## # add/remove entries ######################################################## def _new_profile(self, key): """add a new profile if it doesn't exist""" if key not in self.profiles.keys(): # update yaml_dict self.yaml_dict[self.key_profiles][key] = { self.key_profile_dotfiles: [] } if self.debug: self.log.dbg('adding new profile: {}'.format(key)) self.dirty = True def add_dotfile_to_profile(self, dotfile_key, profile_key): """add an existing dotfile key to a profile_key""" self._new_profile(profile_key) profile = self.yaml_dict[self.key_profiles][profile_key] if dotfile_key not in profile[self.key_profile_dotfiles]: profile[self.key_profile_dotfiles].append(dotfile_key) if self.debug: msg = 'add \"{}\" to profile \"{}\"'.format( dotfile_key, profile_key) msg.format(dotfile_key, profile_key) self.log.dbg(msg) self.dirty = True return self.dirty def add_dotfile(self, key, src, dst, link): """add a new dotfile""" if key in self.dotfiles.keys(): return False if self.debug: self.log.dbg('adding new dotfile: {}'.format(key)) df_dict = { self.key_dotfile_src: src, self.key_dotfile_dst: dst, } dfl = self.settings[self.key_settings_link_dotfile_default] if str(link) != dfl: df_dict[self.key_dotfile_link] = str(link) self.yaml_dict[self.key_dotfiles][key] = df_dict self.dirty = True def del_dotfile(self, key): """remove this dotfile from config""" if key not in self.yaml_dict[self.key_dotfiles]: self.log.err('key not in dotfiles: {}'.format(key)) return False if self.debug: self.log.dbg('remove dotfile: {}'.format(key)) del self.yaml_dict[self.key_dotfiles][key] if self.debug: dfs = self.yaml_dict[self.key_dotfiles] self.log.dbg('new dotfiles: {}'.format(dfs)) self.dirty = True return True def del_dotfile_from_profile(self, df_key, pro_key): """remove this dotfile from that profile""" if df_key not in self.dotfiles.keys(): self.log.err('key not in dotfiles: {}'.format(df_key)) return False if pro_key not in self.profiles.keys(): self.log.err('key not in profile: {}'.format(pro_key)) return False # get the profile dictionary profile = self.yaml_dict[self.key_profiles][pro_key] if df_key not in profile[self.key_profile_dotfiles]: return True if self.debug: dfs = profile[self.key_profile_dotfiles] self.log.dbg('{} profile dotfiles: {}'.format(pro_key, dfs)) self.log.dbg('remove {} from profile {}'.format(df_key, pro_key)) profile[self.key_profile_dotfiles].remove(df_key) if self.debug: dfs = profile[self.key_profile_dotfiles] self.log.dbg('{} profile dotfiles: {}'.format(pro_key, dfs)) self.dirty = True return True ######################################################## # handle deprecated entries ######################################################## def _fix_deprecated(self, yamldict): """fix deprecated entries""" self._fix_deprecated_link_by_default(yamldict) self._fix_deprecated_dotfile_link(yamldict) def _fix_deprecated_link_by_default(self, yamldict): """fix deprecated link_by_default""" key = 'link_by_default' newkey = self.key_imp_link if self.key_settings not in yamldict: return if not yamldict[self.key_settings]: return config = yamldict[self.key_settings] if key not in config: return if config[key]: config[newkey] = self.lnk_link else: config[newkey] = self.lnk_nolink del config[key] self.log.warn('deprecated \"link_by_default\"') self.dirty = True self.dirty_deprecated = True def _fix_deprecated_dotfile_link(self, yamldict): """fix deprecated link in dotfiles""" if self.key_dotfiles not in yamldict: return if not yamldict[self.key_dotfiles]: return for k, dotfile in yamldict[self.key_dotfiles].items(): new = self.lnk_nolink if self.key_dotfile_link in dotfile and \ type(dotfile[self.key_dotfile_link]) is bool: # patch link: <bool> cur = dotfile[self.key_dotfile_link] new = self.lnk_nolink if cur: new = self.lnk_link dotfile[self.key_dotfile_link] = new self.dirty = True self.dirty_deprecated = True self.log.warn('deprecated \"link\" value') elif self.key_dotfile_link_children in dotfile and \ type(dotfile[self.key_dotfile_link_children]) is bool: # patch link_children: <bool> cur = dotfile[self.key_dotfile_link_children] new = self.lnk_nolink if cur: new = self.lnk_children del dotfile[self.key_dotfile_link_children] dotfile[self.key_dotfile_link] = new self.dirty = True self.dirty_deprecated = True self.log.warn('deprecated \"link_children\" value') ######################################################## # yaml utils ######################################################## def save(self): """save this instance and return True if saved""" if not self.dirty: return False content = self._clear_none(self.dump()) # make sure we have the base entries if self.key_settings not in content: content[self.key_settings] = None if self.key_dotfiles not in content: content[self.key_dotfiles] = None if self.key_profiles not in content: content[self.key_profiles] = None if self.dirty_deprecated: # add minversion settings = content[self.key_settings] settings[self.key_settings_minversion] = VERSION # save to file if self.debug: self.log.dbg('saving to {}'.format(self.path)) try: self._yaml_dump(content, self.path) except Exception as e: self.log.err(e) raise YamlException('error saving config: {}'.format(self.path)) if self.dirty_deprecated: warn = 'your config contained deprecated entries' warn += ' and was updated' self.log.warn(warn) self.dirty = False self.cfg_updated = False return True def dump(self): """dump the config dictionary""" return self.yaml_dict def _load_yaml(self, path): """load a yaml file to a dict""" content = {} if not os.path.exists(path): raise YamlException('config path not found: {}'.format(path)) try: content = self._yaml_load(path) except Exception as e: self.log.err(e) raise YamlException('invalid config: {}'.format(path)) return content def _yaml_load(self, path): """load from yaml""" with open(path, 'r') as f: y = yaml() y.typ = 'rt' content = y.load(f) return content def _yaml_dump(self, content, path): """dump to yaml""" with open(self.path, 'w') as f: y = yaml() y.default_flow_style = False y.indent = 2 y.typ = 'rt' y.dump(content, f) ######################################################## # helpers ######################################################## def _merge_dict(self, high, low): """merge high and low dict""" if not high: high = {} if not low: low = {} return {**low, **high} def _get_entry(self, dic, key, mandatory=True): """return entry from yaml dictionary""" if key not in dic: if mandatory: raise YamlException('invalid config: no {} found'.format(key)) dic[key] = {} return dic[key] if mandatory and not dic[key]: # ensure is not none dic[key] = {} return dic[key] def _clear_none(self, dic): """recursively delete all none/empty values in a dictionary.""" new = {} for k, v in dic.items(): newv = v if isinstance(v, dict): newv = self._clear_none(v) if not newv: # no empty dict continue if newv is None: # no None value continue if isinstance(newv, list) and not newv: # no empty list continue new[k] = newv return new def _is_glob(self, path): """quick test if path is a glob""" return '*' in path or '?' in path def _glob_paths(self, paths): """glob a list of paths""" if not isinstance(paths, list): paths = [paths] res = [] for p in paths: if not self._is_glob(p): res.append(p) continue p = os.path.expanduser(p) new = glob.glob(p) if not new: raise YamlException('bad path: {}'.format(p)) res.extend(glob.glob(p)) return res def _debug_vars(self, variables): """pretty print variables""" if not self.debug: return self.log.dbg('variables:') for k, v in variables.items(): self.log.dbg('\t\"{}\": {}'.format(k, v)) def _norm_path(self, path): """resolve a path either absolute or relative to config path""" path = os.path.expanduser(path) if not os.path.isabs(path): d = os.path.dirname(self.path) return os.path.join(d, path) return os.path.normpath(path) def _shell_exec_dvars(self, keys, variables): """shell execute dynvariables""" for k in list(keys): ret, out = shell(variables[k], debug=self.debug) if not ret: err = 'var \"{}: {}\" failed: {}'.format(k, variables[k], out) self.log.err(err) raise YamlException(err) if self.debug: self.log.dbg('\"{}\": {} -> {}'.format(k, variables[k], out)) variables[k] = out def _template_list(self, t, entries): """template a list of entries""" new = [] if not entries: return new for e in entries: et = t.generate_string(e) if self.debug and e != et: self.log.dbg('resolved: {} -> {}'.format(e, et)) new.append(et) return new def _check_minversion(self, minversion): if not minversion: return try: cur = tuple([int(x) for x in VERSION.split('.')]) cfg = tuple([int(x) for x in minversion.split('.')]) except Exception: err = 'bad version: \"{}\" VS \"{}\"'.format(VERSION, minversion) raise YamlException(err) if cur < cfg: err = 'current dotdrop version is too old for that config file.' err += ' Please update.' raise YamlException(err)
class CfgYaml: # global entries key_settings = Settings.key_yaml key_dotfiles = 'dotfiles' key_profiles = 'profiles' key_actions = 'actions' old_key_trans_r = 'trans' key_trans_r = 'trans_read' key_trans_w = 'trans_write' key_variables = 'variables' key_dvariables = 'dynvariables' action_pre = 'pre' action_post = 'post' # profiles/dotfiles entries key_dotfile_src = 'src' key_dotfile_dst = 'dst' key_dotfile_link = 'link' key_dotfile_actions = 'actions' key_dotfile_link_children = 'link_children' key_dotfile_noempty = 'ignoreempty' # profile key_profile_dotfiles = 'dotfiles' key_profile_include = 'include' key_profile_variables = 'variables' key_profile_dvariables = 'dynvariables' key_profile_actions = 'actions' key_all = 'ALL' # import entries key_import_actions = 'import_actions' key_import_configs = 'import_configs' key_import_variables = 'import_variables' key_import_profile_dfs = 'import' key_import_sep = ':' key_import_ignore_key = 'optional' key_import_fatal_not_found = True # settings key_settings_dotpath = Settings.key_dotpath key_settings_workdir = Settings.key_workdir key_settings_link_dotfile_default = Settings.key_link_dotfile_default key_settings_noempty = Settings.key_ignoreempty key_settings_minversion = Settings.key_minversion key_imp_link = Settings.key_link_on_import # link values lnk_nolink = LinkTypes.NOLINK.name.lower() lnk_link = LinkTypes.LINK.name.lower() lnk_children = LinkTypes.LINK_CHILDREN.name.lower() def __init__(self, path, profile=None, addprofiles=[], debug=False): """ config parser @path: config file path @profile: the selected profile @addprofiles: included profiles @debug: debug flag """ self._path = os.path.abspath(path) self._profile = profile self._debug = debug self._log = Logger() # config needs to be written self._dirty = False # indicates the config has been updated self._dirty_deprecated = False # profile variables self._profilevarskeys = [] # included profiles self._inc_profiles = addprofiles # init the dictionaries self.settings = {} self.dotfiles = {} self.profiles = {} self.actions = {} self.trans_r = {} self.trans_w = {} self.variables = {} if not os.path.exists(self._path): err = 'invalid config path: \"{}\"'.format(path) if self._debug: self._dbg(err) raise YamlException(err) self._yaml_dict = self._load_yaml(self._path) # live patch deprecated entries self._fix_deprecated(self._yaml_dict) ################################################## # parse the config and variables ################################################## # parse the "config" block self.settings = self._parse_blk_settings(self._yaml_dict) # base templater (when no vars/dvars exist) self.variables = self._enrich_vars(self.variables, self._profile) self._redefine_templater() # variables and dynvariables need to be first merged # before being templated in order to allow cyclic # references between them # parse the "variables" block var = self._parse_blk_variables(self._yaml_dict) self._add_variables(var, template=False) # parse the "dynvariables" block dvariables = self._parse_blk_dynvariables(self._yaml_dict) self._add_variables(dvariables, template=False) # now template variables and dynvariables from the same pool self._rec_resolve_variables(self.variables) # and execute dvariables # since this is done after recursively resolving variables # and dynvariables this means that variables referencing # dynvariables will result with the not executed value if dvariables.keys(): self._shell_exec_dvars(self.variables, keys=dvariables.keys()) # finally redefine the template self._redefine_templater() if self._debug: self._debug_dict('current variables defined', self.variables) # parse the "profiles" block self.profiles = self._parse_blk_profiles(self._yaml_dict) # include the profile's variables/dynvariables last # as it overwrites existing ones self._inc_profiles, pv, pvd = self._get_profile_included_vars() self._add_variables(pv, prio=True) self._add_variables(pvd, shell=True, prio=True) self._profilevarskeys.extend(pv.keys()) self._profilevarskeys.extend(pvd.keys()) # template variables self.variables = self._template_dict(self.variables) if self._debug: self._debug_dict('current variables defined', self.variables) ################################################## # template the "include" entries ################################################## self._template_include_entry() if self._debug: self._debug_dict('current variables defined', self.variables) ################################################## # parse the other blocks ################################################## # parse the "dotfiles" block self.dotfiles = self._parse_blk_dotfiles(self._yaml_dict) # parse the "actions" block self.actions = self._parse_blk_actions(self._yaml_dict) # parse the "trans_r" block self.trans_r = self._parse_blk_trans_r(self._yaml_dict) # parse the "trans_w" block self.trans_w = self._parse_blk_trans_w(self._yaml_dict) ################################################## # import elements ################################################## # process imported variables (import_variables) newvars = self._import_variables() self._clear_profile_vars(newvars) self._add_variables(newvars) # process imported actions (import_actions) self._import_actions() # process imported profile dotfiles (import) self._import_profiles_dotfiles() # process imported configs (import_configs) self._import_configs() # process profile include self._resolve_profile_includes() # add the current profile variables _, pv, pvd = self._get_profile_included_vars() self._add_variables(pv, prio=True) self._add_variables(pvd, shell=True, prio=True) self._profilevarskeys.extend(pv.keys()) self._profilevarskeys.extend(pvd.keys()) # resolve variables self._clear_profile_vars(newvars) self._add_variables(newvars) # process profile ALL self._resolve_profile_all() # patch dotfiles paths self._template_dotfiles_paths() if self._debug: self._dbg('########### {} ###########'.format('final config')) self._debug_entries() ######################################################## # outside available methods ######################################################## def resolve_dotfile_src(self, src, templater=None): """resolve dotfile src path""" newsrc = '' if src: new = src if templater: new = templater.generate_string(src) if new != src and self._debug: msg = 'dotfile src: \"{}\" -> \"{}\"'.format(src, new) self._dbg(msg) src = new src = os.path.join(self.settings[self.key_settings_dotpath], src) newsrc = self._norm_path(src) return newsrc def resolve_dotfile_dst(self, dst, templater=None): """resolve dotfile dst path""" newdst = '' if dst: new = dst if templater: new = templater.generate_string(dst) if new != dst and self._debug: msg = 'dotfile dst: \"{}\" -> \"{}\"'.format(dst, new) self._dbg(msg) dst = new newdst = self._norm_path(dst) return newdst def add_dotfile_to_profile(self, dotfile_key, profile_key): """add an existing dotfile key to a profile_key""" self._new_profile(profile_key) profile = self._yaml_dict[self.key_profiles][profile_key] if self.key_profile_dotfiles not in profile or \ profile[self.key_profile_dotfiles] is None: profile[self.key_profile_dotfiles] = [] pdfs = profile[self.key_profile_dotfiles] if self.key_all not in pdfs and \ dotfile_key not in pdfs: profile[self.key_profile_dotfiles].append(dotfile_key) if self._debug: msg = 'add \"{}\" to profile \"{}\"'.format( dotfile_key, profile_key) msg.format(dotfile_key, profile_key) self._dbg(msg) self._dirty = True return self._dirty def get_all_dotfile_keys(self): """return all existing dotfile keys""" return self.dotfiles.keys() def add_dotfile(self, key, src, dst, link): """add a new dotfile""" if key in self.dotfiles.keys(): return False if self._debug: self._dbg('adding new dotfile: {}'.format(key)) self._dbg('new dotfile src: {}'.format(src)) self._dbg('new dotfile dst: {}'.format(dst)) df_dict = { self.key_dotfile_src: src, self.key_dotfile_dst: dst, } dfl = self.settings[self.key_settings_link_dotfile_default] if str(link) != dfl: df_dict[self.key_dotfile_link] = str(link) self._yaml_dict[self.key_dotfiles][key] = df_dict self._dirty = True def del_dotfile(self, key): """remove this dotfile from config""" if key not in self._yaml_dict[self.key_dotfiles]: self._log.err('key not in dotfiles: {}'.format(key)) return False if self._debug: self._dbg('remove dotfile: {}'.format(key)) del self._yaml_dict[self.key_dotfiles][key] if self._debug: dfs = self._yaml_dict[self.key_dotfiles] self._dbg('new dotfiles: {}'.format(dfs)) self._dirty = True return True def del_dotfile_from_profile(self, df_key, pro_key): """remove this dotfile from that profile""" if df_key not in self.dotfiles.keys(): self._log.err('key not in dotfiles: {}'.format(df_key)) return False if pro_key not in self.profiles.keys(): self._log.err('key not in profile: {}'.format(pro_key)) return False # get the profile dictionary profile = self._yaml_dict[self.key_profiles][pro_key] if df_key not in profile[self.key_profile_dotfiles]: return True if self._debug: dfs = profile[self.key_profile_dotfiles] self._dbg('{} profile dotfiles: {}'.format(pro_key, dfs)) self._dbg('remove {} from profile {}'.format(df_key, pro_key)) profile[self.key_profile_dotfiles].remove(df_key) if self._debug: dfs = profile[self.key_profile_dotfiles] self._dbg('{} profile dotfiles: {}'.format(pro_key, dfs)) self._dirty = True return True def save(self): """save this instance and return True if saved""" if not self._dirty: return False content = self._prepare_to_save(self._yaml_dict) if self._dirty_deprecated: # add minversion settings = content[self.key_settings] settings[self.key_settings_minversion] = VERSION # save to file if self._debug: self._dbg('saving to {}'.format(self._path)) try: with open(self._path, 'w') as f: self._yaml_dump(content, f) except Exception as e: self._log.err(e) raise YamlException('error saving config: {}'.format(self._path)) if self._dirty_deprecated: warn = 'your config contained deprecated entries' warn += ' and was updated' self._log.warn(warn) self._dirty = False self.cfg_updated = False return True def dump(self): """dump the config dictionary""" output = io.StringIO() content = self._prepare_to_save(self._yaml_dict.copy()) self._yaml_dump(content, output) return output.getvalue() ######################################################## # block parsing ######################################################## def _parse_blk_settings(self, dic): """parse the "config" block""" block = self._get_entry(dic, self.key_settings).copy() # set defaults settings = Settings(None).serialize().get(self.key_settings) settings.update(block) # resolve minimum version if self.key_settings_minversion in settings: minversion = settings[self.key_settings_minversion] self._check_minversion(minversion) # normalize paths p = self._norm_path(settings[self.key_settings_dotpath]) settings[self.key_settings_dotpath] = p p = self._norm_path(settings[self.key_settings_workdir]) settings[self.key_settings_workdir] = p p = [self._norm_path(p) for p in settings[Settings.key_filter_file]] settings[Settings.key_filter_file] = p p = [self._norm_path(p) for p in settings[Settings.key_func_file]] settings[Settings.key_func_file] = p if self._debug: self._debug_dict('settings block:', settings) return settings def _parse_blk_dotfiles(self, dic): """parse the "dotfiles" block""" dotfiles = self._get_entry(dic, self.key_dotfiles).copy() keys = dotfiles.keys() if len(keys) != len(list(set(keys))): dups = [x for x in keys if x not in list(set(keys))] err = 'duplicate dotfile keys found: {}'.format(dups) raise YamlException(err) dotfiles = self._norm_dotfiles(dotfiles) if self._debug: self._debug_dict('dotfiles block', dotfiles) return dotfiles def _parse_blk_profiles(self, dic): """parse the "profiles" block""" profiles = self._get_entry(dic, self.key_profiles).copy() profiles = self._norm_profiles(profiles) if self._debug: self._debug_dict('profiles block', profiles) return profiles def _parse_blk_actions(self, dic): """parse the "actions" block""" actions = self._get_entry(dic, self.key_actions, mandatory=False) if actions: actions = actions.copy() actions = self._norm_actions(actions) if self._debug: self._debug_dict('actions block', actions) return actions def _parse_blk_trans_r(self, dic): """parse the "trans_r" block""" key = self.key_trans_r if self.old_key_trans_r in dic: msg = '\"trans\" is deprecated, please use \"trans_read\"' self._log.warn(msg) dic[self.key_trans_r] = dic[self.old_key_trans_r] del dic[self.old_key_trans_r] trans_r = self._get_entry(dic, key, mandatory=False) if trans_r: trans_r = trans_r.copy() if self._debug: self._debug_dict('trans_r block', trans_r) return trans_r def _parse_blk_trans_w(self, dic): """parse the "trans_w" block""" trans_w = self._get_entry(dic, self.key_trans_w, mandatory=False) if trans_w: trans_w = trans_w.copy() if self._debug: self._debug_dict('trans_w block', trans_w) return trans_w def _parse_blk_variables(self, dic): """parse the "variables" block""" variables = self._get_entry(dic, self.key_variables, mandatory=False) if variables: variables = variables.copy() if self._debug: self._debug_dict('variables block', variables) return variables def _parse_blk_dynvariables(self, dic): """parse the "dynvariables" block""" dvariables = self._get_entry(dic, self.key_dvariables, mandatory=False) if dvariables: dvariables = dvariables.copy() if self._debug: self._debug_dict('dynvariables block', dvariables) return dvariables ######################################################## # parsing helpers ######################################################## def _template_include_entry(self): """template all "include" entries""" # import_actions new = [] entries = self.settings.get(self.key_import_actions, []) new = self._template_list(entries) if new: self.settings[self.key_import_actions] = new # import_configs entries = self.settings.get(self.key_import_configs, []) new = self._template_list(entries) if new: self.settings[self.key_import_configs] = new # import_variables entries = self.settings.get(self.key_import_variables, []) new = self._template_list(entries) if new: self.settings[self.key_import_variables] = new # profile's import for k, v in self.profiles.items(): entries = v.get(self.key_import_profile_dfs, []) new = self._template_list(entries) if new: v[self.key_import_profile_dfs] = new def _norm_actions(self, actions): """ ensure each action is either pre or post explicitely action entry of the form {action_key: (pre|post, action)} """ if not actions: return actions new = {} for k, v in actions.items(): if k == self.action_pre or k == self.action_post: for key, action in v.items(): new[key] = (k, action) else: new[k] = (self.action_post, v) return new def _norm_profiles(self, profiles): """normalize profiles entries""" if not profiles: return profiles new = {} for k, v in profiles.items(): if not v: # no dotfiles continue # add dotfiles entry if not present if self.key_profile_dotfiles not in v: v[self.key_profile_dotfiles] = [] new[k] = v return new def _norm_dotfiles(self, dotfiles): """normalize dotfiles entries""" if not dotfiles: return dotfiles new = {} for k, v in dotfiles.items(): # add 'src' as key' if not present if self.key_dotfile_src not in v: v[self.key_dotfile_src] = k new[k] = v else: new[k] = v # fix deprecated trans key if self.old_key_trans_r in v: msg = '\"trans\" is deprecated, please use \"trans_read\"' self._log.warn(msg) v[self.key_trans_r] = v[self.old_key_trans_r] del v[self.old_key_trans_r] new[k] = v # apply link value if self.key_dotfile_link not in v: val = self.settings[self.key_settings_link_dotfile_default] v[self.key_dotfile_link] = val # apply noempty if undefined if self.key_dotfile_noempty not in v: val = self.settings.get(self.key_settings_noempty, False) v[self.key_dotfile_noempty] = val return new def _add_variables(self, new, shell=False, template=True, prio=False): """ add new variables @shell: execute the variable through the shell @template: template the variable @prio: new takes priority over existing variables """ if not new: return # merge if prio: self.variables = self._merge_dict(new, self.variables) else: self.variables = self._merge_dict(self.variables, new) # ensure enriched variables are relative to this config self.variables = self._enrich_vars(self.variables, self._profile) # re-create the templater self._redefine_templater() if template: # rec resolve variables with new ones self._rec_resolve_variables(self.variables) if shell: # shell exec self._shell_exec_dvars(self.variables, keys=new.keys()) # re-create the templater self._redefine_templater() def _enrich_vars(self, variables, profile): """return enriched variables""" # add profile variable if profile: variables['profile'] = profile # add some more variables p = self.settings.get(self.key_settings_dotpath) p = self._norm_path(p) variables['_dotdrop_dotpath'] = p variables['_dotdrop_cfgpath'] = self._norm_path(self._path) p = self.settings.get(self.key_settings_workdir) p = self._norm_path(p) variables['_dotdrop_workdir'] = p return variables def _get_profile_included_item(self, keyitem): """recursively get included <keyitem> in profile""" profiles = [self._profile] + self._inc_profiles items = {} for profile in profiles: seen = [self._profile] i = self.__get_profile_included_item(profile, keyitem, seen) items = self._merge_dict(i, items) return items def __get_profile_included_item(self, profile, keyitem, seen): """recursively get included <keyitem> from profile""" items = {} if not profile or profile not in self.profiles.keys(): return items # considered profile entry pentry = self.profiles.get(profile) # recursively get <keyitem> from inherited profile for inherited_profile in pentry.get(self.key_profile_include, []): if inherited_profile == profile or inherited_profile in seen: raise YamlException('\"include\" loop') seen.append(inherited_profile) new = self.__get_profile_included_item(inherited_profile, keyitem, seen) if self._debug: msg = 'included {} from {}: {}' self._dbg(msg.format(keyitem, inherited_profile, new)) items.update(new) cur = pentry.get(keyitem, {}) return self._merge_dict(cur, items) def _resolve_profile_all(self): """resolve some other parts of the config""" # profile -> ALL for k, v in self.profiles.items(): dfs = v.get(self.key_profile_dotfiles, None) if not dfs: continue if self.key_all in dfs: if self._debug: self._dbg('add ALL to profile \"{}\"'.format(k)) v[self.key_profile_dotfiles] = self.dotfiles.keys() def _resolve_profile_includes(self): """resolve profile(s) including other profiles""" for k, v in self.profiles.items(): self._rec_resolve_profile_include(k) def _rec_resolve_profile_include(self, profile): """ recursively resolve include of other profiles's: * dotfiles * actions returns dotfiles, actions """ this_profile = self.profiles[profile] # considered profile content dotfiles = this_profile.get(self.key_profile_dotfiles, []) or [] actions = this_profile.get(self.key_profile_actions, []) or [] includes = this_profile.get(self.key_profile_include, []) or [] if not includes: # nothing to include return dotfiles, actions if self._debug: self._dbg('{} includes {}'.format(profile, ','.join(includes))) self._dbg('{} dotfiles before include: {}'.format( profile, dotfiles)) self._dbg('{} actions before include: {}'.format(profile, actions)) seen = [] for i in uniq_list(includes): if self._debug: self._dbg('resolving includes "{}" <- "{}"'.format(profile, i)) # ensure no include loop occurs if i in seen: raise YamlException('\"include loop\"') seen.append(i) # included profile even exists if i not in self.profiles.keys(): self._log.warn('include unknown profile: {}'.format(i)) continue # recursive resolve if self._debug: self._dbg( 'recursively resolving includes for profile "{}"'.format( i)) o_dfs, o_actions = self._rec_resolve_profile_include(i) # merge dotfile keys if self._debug: self._dbg('Merging dotfiles {} <- {}: {} <- {}'.format( profile, i, dotfiles, o_dfs)) dotfiles.extend(o_dfs) this_profile[self.key_profile_dotfiles] = uniq_list(dotfiles) # merge actions keys if self._debug: self._dbg('Merging actions {} <- {}: {} <- {}'.format( profile, i, actions, o_actions)) actions.extend(o_actions) this_profile[self.key_profile_actions] = uniq_list(actions) dotfiles = this_profile.get(self.key_profile_dotfiles, []) actions = this_profile.get(self.key_profile_actions, []) if self._debug: self._dbg('{} dotfiles after include: {}'.format( profile, dotfiles)) self._dbg('{} actions after include: {}'.format(profile, actions)) # since included items are resolved here # we can clear these include self.profiles[profile][self.key_profile_include] = [] return dotfiles, actions ######################################################## # handle imported entries ######################################################## def _import_variables(self): """import external variables from paths""" paths = self.settings.get(self.key_import_variables, None) if not paths: return paths = self._resolve_paths(paths) newvars = {} for path in paths: if self._debug: self._dbg('import variables from {}'.format(path)) var = self._import_sub(path, self.key_variables, mandatory=False) if self._debug: self._dbg('import dynvariables from {}'.format(path)) dvar = self._import_sub(path, self.key_dvariables, mandatory=False) merged = self._merge_dict(dvar, var) self._rec_resolve_variables(merged) if dvar.keys(): self._shell_exec_dvars(merged, keys=dvar.keys()) self._clear_profile_vars(merged) newvars = self._merge_dict(newvars, merged) if self._debug: self._debug_dict('imported variables', newvars) return newvars def _import_actions(self): """import external actions from paths""" paths = self.settings.get(self.key_import_actions, None) if not paths: return paths = self._resolve_paths(paths) for path in paths: if self._debug: self._dbg('import actions from {}'.format(path)) new = self._import_sub(path, self.key_actions, mandatory=False, patch_func=self._norm_actions) self.actions = self._merge_dict(new, self.actions) def _import_profiles_dotfiles(self): """import profile dotfiles""" for k, v in self.profiles.items(): imp = v.get(self.key_import_profile_dfs, None) if not imp: continue if self._debug: self._dbg('import dotfiles for profile {}'.format(k)) paths = self._resolve_paths(imp) for path in paths: current = v.get(self.key_dotfiles, []) new = self._import_sub(path, self.key_dotfiles, mandatory=False) v[self.key_dotfiles] = new + current def _import_config(self, path): """import config from path""" if self._debug: self._dbg('import config from {}'.format(path)) sub = CfgYaml(path, profile=self._profile, addprofiles=self._inc_profiles, debug=self._debug) # settings are ignored from external file # except for filter_file and func_file self.settings[Settings.key_func_file] += [ self._norm_path(func_file) for func_file in sub.settings[Settings.key_func_file] ] self.settings[Settings.key_filter_file] += [ self._norm_path(func_file) for func_file in sub.settings[Settings.key_filter_file] ] # merge top entries self.dotfiles = self._merge_dict(self.dotfiles, sub.dotfiles) self.profiles = self._merge_dict(self.profiles, sub.profiles) self.actions = self._merge_dict(self.actions, sub.actions) self.trans_r = self._merge_dict(self.trans_r, sub.trans_r) self.trans_w = self._merge_dict(self.trans_w, sub.trans_w) self._clear_profile_vars(sub.variables) if self._debug: self._debug_dict('add import_configs var', sub.variables) self._add_variables(sub.variables, prio=True) def _import_configs(self): """import configs from external files""" # settings -> import_configs imp = self.settings.get(self.key_import_configs, None) if not imp: return paths = self._resolve_paths(imp) for path in paths: self._import_config(path) def _import_sub(self, path, key, mandatory=False, patch_func=None): """ import the block "key" from "path" patch_func is applied to each element if defined """ if self._debug: self._dbg('import \"{}\" from \"{}\"'.format(key, path)) extdict = self._load_yaml(path) new = self._get_entry(extdict, key, mandatory=mandatory) if patch_func: if self._debug: self._dbg('calling patch: {}'.format(patch_func)) new = patch_func(new) if not new and mandatory: err = 'no \"{}\" imported from \"{}\"'.format(key, path) self._log.warn(err) raise YamlException(err) if self._debug: self._dbg('imported \"{}\": {}'.format(key, new)) return new ######################################################## # add/remove entries ######################################################## def _new_profile(self, key): """add a new profile if it doesn't exist""" if key not in self.profiles.keys(): # update yaml_dict self._yaml_dict[self.key_profiles][key] = { self.key_profile_dotfiles: [] } if self._debug: self._dbg('adding new profile: {}'.format(key)) self._dirty = True ######################################################## # handle deprecated entries ######################################################## def _fix_deprecated(self, yamldict): """fix deprecated entries""" if not yamldict: return self._fix_deprecated_link_by_default(yamldict) self._fix_deprecated_dotfile_link(yamldict) return yamldict def _fix_deprecated_link_by_default(self, yamldict): """fix deprecated link_by_default""" key = 'link_by_default' newkey = self.key_imp_link if self.key_settings not in yamldict: return if not yamldict[self.key_settings]: return config = yamldict[self.key_settings] if key not in config: return if config[key]: config[newkey] = self.lnk_link else: config[newkey] = self.lnk_nolink del config[key] self._log.warn('deprecated \"link_by_default\"') self._dirty = True self._dirty_deprecated = True def _fix_deprecated_dotfile_link(self, yamldict): """fix deprecated link in dotfiles""" if self.key_dotfiles not in yamldict: return if not yamldict[self.key_dotfiles]: return for k, dotfile in yamldict[self.key_dotfiles].items(): new = self.lnk_nolink if self.key_dotfile_link in dotfile and \ type(dotfile[self.key_dotfile_link]) is bool: # patch link: <bool> cur = dotfile[self.key_dotfile_link] new = self.lnk_nolink if cur: new = self.lnk_link dotfile[self.key_dotfile_link] = new self._dirty = True self._dirty_deprecated = True self._log.warn('deprecated \"link\" value') elif self.key_dotfile_link_children in dotfile and \ type(dotfile[self.key_dotfile_link_children]) is bool: # patch link_children: <bool> cur = dotfile[self.key_dotfile_link_children] new = self.lnk_nolink if cur: new = self.lnk_children del dotfile[self.key_dotfile_link_children] dotfile[self.key_dotfile_link] = new self._dirty = True self._dirty_deprecated = True self._log.warn('deprecated \"link_children\" value') ######################################################## # yaml utils ######################################################## def _prepare_to_save(self, content): content = self._clear_none(content) # make sure we have the base entries if self.key_settings not in content: content[self.key_settings] = None if self.key_dotfiles not in content: content[self.key_dotfiles] = None if self.key_profiles not in content: content[self.key_profiles] = None return content def _load_yaml(self, path): """load a yaml file to a dict""" content = {} if self._debug: self._dbg('----------start:{}----------'.format(path)) cfg = '\n' with open(path, 'r') as f: for line in f: cfg += line self._dbg(cfg.rstrip()) self._dbg('----------end:{}----------'.format(path)) try: content = self._yaml_load(path) except Exception as e: self._log.err(e) raise YamlException('invalid config: {}'.format(path)) return content def _yaml_load(self, path): """load from yaml""" with open(path, 'r') as f: y = yaml() y.typ = 'rt' content = y.load(f) return content def _yaml_dump(self, content, where): """dump to yaml""" y = yaml() y.default_flow_style = False y.indent = 2 y.typ = 'rt' y.dump(content, where) ######################################################## # templating ######################################################## def _redefine_templater(self): """create templater based on current variables""" fufile = self.settings[Settings.key_func_file] fifile = self.settings[Settings.key_filter_file] self._tmpl = Templategen(variables=self.variables, func_file=fufile, filter_file=fifile) def _template_item(self, item, exc_if_fail=True): """ template an item using the templategen will raise an exception if template failed and exc_if_fail """ if not Templategen.var_is_template(item): return item try: val = item while Templategen.var_is_template(val): val = self._tmpl.generate_string(val) except UndefinedException as e: if exc_if_fail: raise e return val def _template_list(self, entries): """template a list of entries""" new = [] if not entries: return new for e in entries: et = self._template_item(e) if self._debug and e != et: self._dbg('resolved: {} -> {}'.format(e, et)) new.append(et) return new def _template_dict(self, entries): """template a dictionary of entries""" new = {} if not entries: return new for k, v in entries.items(): vt = self._template_item(v) if self._debug and v != vt: self._dbg('resolved: {} -> {}'.format(v, vt)) new[k] = vt return new def _template_dotfiles_paths(self): """template dotfiles paths""" if self._debug: self._dbg('templating dotfiles paths') dotfiles = self.dotfiles.copy() # make sure no dotfiles path is None for dotfile in dotfiles.values(): src = dotfile[self.key_dotfile_src] if src is None: dotfile[self.key_dotfile_src] = '' dst = dotfile[self.key_dotfile_dst] if dst is None: dotfile[self.key_dotfile_dst] = '' # only keep dotfiles related to the selected profile pdfs = [] pro = self.profiles.get(self._profile, []) if pro: pdfs = list(pro.get(self.key_profile_dotfiles, [])) for addpro in self._inc_profiles: pro = self.profiles.get(addpro, []) if not pro: continue pdfsalt = pro.get(self.key_profile_dotfiles, []) pdfs.extend(pdfsalt) pdfs = uniq_list(pdfs) if self.key_all not in pdfs: # take a subset of the dotfiles newdotfiles = {} for k, v in dotfiles.items(): if k in pdfs: newdotfiles[k] = v dotfiles = newdotfiles for dotfile in dotfiles.values(): # src src = dotfile[self.key_dotfile_src] newsrc = self.resolve_dotfile_src(src, templater=self._tmpl) dotfile[self.key_dotfile_src] = newsrc # dst dst = dotfile[self.key_dotfile_dst] newdst = self.resolve_dotfile_dst(dst, templater=self._tmpl) dotfile[self.key_dotfile_dst] = newdst def _rec_resolve_variables(self, variables): """recursive resolve variables""" var = self._enrich_vars(variables, self._profile) # use a separated templategen to handle variables # resolved outside the main config t = Templategen(variables=var, func_file=self.settings[Settings.key_func_file], filter_file=self.settings[Settings.key_filter_file]) for k in variables.keys(): val = variables[k] while Templategen.var_is_template(val): val = t.generate_string(val) variables[k] = val t.update_variables(variables) if variables is self.variables: self._redefine_templater() def _get_profile_included_vars(self): """resolve profile included variables/dynvariables""" for k, v in self.profiles.items(): if self.key_profile_include in v and v[self.key_profile_include]: new = [] for x in v[self.key_profile_include]: new.append(self._tmpl.generate_string(x)) v[self.key_profile_include] = new # now get the included ones pro_var = self._get_profile_included_item(self.key_profile_variables) pro_dvar = self._get_profile_included_item(self.key_profile_dvariables) # the included profiles inc_profiles = [] if self._profile and self._profile in self.profiles.keys(): pentry = self.profiles.get(self._profile) inc_profiles = pentry.get(self.key_profile_include, []) # exec incl dynvariables return inc_profiles, pro_var, pro_dvar ######################################################## # helpers ######################################################## def _clear_profile_vars(self, dic): """ remove profile variables from dic if found inplace to avoid profile variables being overwriten """ if not dic: return [dic.pop(k, None) for k in self._profilevarskeys] def _parse_extended_import_path(self, path_entry): """Parse an import path in a tuple (path, fatal_not_found).""" if self._debug: self._dbg('parsing path entry {}'.format(path_entry)) path, _, attribute = path_entry.rpartition(self.key_import_sep) fatal_not_found = attribute != self.key_import_ignore_key is_valid_attribute = attribute in ('', self.key_import_ignore_key) if not is_valid_attribute: # If attribute is not valid it can mean that: # - path_entry doesn't contain the separator, and attribute is set # to the whole path by str.rpartition # - path_entry contains a separator, but it's in the file path, so # attribute is set to whatever comes after the separator by # str.rpartition # In both cases, path_entry is the path we're looking for. if self._debug: self._dbg('using attribute default values for path {}'.format( path_entry)) path = path_entry fatal_not_found = self.key_import_fatal_not_found elif self._debug: self._dbg( 'path entry {} has fatal_not_found flag set to {}'.format( path_entry, fatal_not_found)) return path, fatal_not_found def _handle_non_existing_path(self, path, fatal_not_found=True): """Raise an exception or log a warning to handle non-existing paths.""" error = 'bad path {}'.format(path) if fatal_not_found: raise YamlException(error) self._log.warn(error) def _check_path_existence(self, path, fatal_not_found=True): """Check if a path exists, raising if necessary.""" if os.path.exists(path): if self._debug: self._dbg('path {} exists'.format(path)) return path self._handle_non_existing_path(path, fatal_not_found) # Explicit return for readability. Anything evaluating to false is ok. return None def _process_path(self, path_entry): """ This method processed a path entry. Namely it: - Normalizes the path. - Expands globs. - Checks for path existence, taking in account fatal_not_found. This method always returns a list containing only absolute paths existing on the filesystem. If the input is not a glob, the list contains at most one element, otheriwse it could hold more. """ path, fatal_not_found = self._parse_extended_import_path(path_entry) path = self._norm_path(path) paths = self._glob_path(path) if self._is_glob(path) else [path] if not paths: if self._debug: self._dbg("glob path {} didn't expand".format(path)) self._handle_non_existing_path(path, fatal_not_found) return [] checked_paths = (self._check_path_existence(p, fatal_not_found) for p in paths) return [p for p in checked_paths if p] def _resolve_paths(self, paths): """ This function resolves a list of paths. This means normalizing, expanding globs and checking for existence, taking in account fatal_not_found flags. """ processed_paths = (self._process_path(p) for p in paths) return list(chain.from_iterable(processed_paths)) def _merge_dict(self, high, low): """merge high and low dict""" if not high: high = {} if not low: low = {} return {**low, **high} def _get_entry(self, dic, key, mandatory=True): """return copy of entry from yaml dictionary""" if key not in dic: if mandatory: raise YamlException('invalid config: no {} found'.format(key)) dic[key] = {} return deepcopy(dic[key]) if mandatory and not dic[key]: # ensure is not none dic[key] = {} return deepcopy(dic[key]) def _clear_none(self, dic): """recursively delete all none/empty values in a dictionary.""" new = {} for k, v in dic.items(): if k == self.key_dotfile_src: # allow empty dotfile src new[k] = v continue if k == self.key_dotfile_dst: # allow empty dotfile dst new[k] = v continue newv = v if isinstance(v, dict): # recursive travers dict newv = self._clear_none(v) if not newv: # no empty dict continue if newv is None: # no None value continue if isinstance(newv, list) and not newv: # no empty list continue new[k] = newv return new def _is_glob(self, path): """Quick test if path is a glob.""" return '*' in path or '?' in path def _glob_path(self, path): """Expand a glob.""" if self._debug: self._dbg('expanding glob {}'.format(path)) expanded_path = os.path.expanduser(path) return glob.glob(expanded_path, recursive=True) def _norm_path(self, path): """Resolve a path either absolute or relative to config path""" if not path: return path path = os.path.expanduser(path) if not os.path.isabs(path): d = os.path.dirname(self._path) ret = os.path.join(d, path) if self._debug: msg = 'normalizing relative to cfg: {} -> {}' self._dbg(msg.format(path, ret)) return ret ret = os.path.normpath(path) if self._debug and path != ret: self._dbg('normalizing: {} -> {}'.format(path, ret)) return ret def _shell_exec_dvars(self, dic, keys=[]): """shell execute dynvariables in-place""" if not keys: keys = dic.keys() for k in keys: v = dic[k] ret, out = shell(v, debug=self._debug) if not ret: err = 'var \"{}: {}\" failed: {}'.format(k, v, out) self._log.err(err) raise YamlException(err) if self._debug: self._dbg('{}: `{}` -> {}'.format(k, v, out)) dic[k] = out def _check_minversion(self, minversion): if not minversion: return try: cur = tuple([int(x) for x in VERSION.split('.')]) cfg = tuple([int(x) for x in minversion.split('.')]) except Exception: err = 'bad version: \"{}\" VS \"{}\"'.format(VERSION, minversion) raise YamlException(err) if cur < cfg: err = 'current dotdrop version is too old for that config file.' err += ' Please update.' raise YamlException(err) def _debug_entries(self): """debug print all interesting entries""" if not self._debug: return self._dbg('Current entries') self._debug_dict('entry settings', self.settings) self._debug_dict('entry dotfiles', self.dotfiles) self._debug_dict('entry profiles', self.profiles) self._debug_dict('entry actions', self.actions) self._debug_dict('entry trans_r', self.trans_r) self._debug_dict('entry trans_w', self.trans_w) self._debug_dict('entry variables', self.variables) def _debug_dict(self, title, elems): """pretty print dict""" if not self._debug: return self._dbg('{}:'.format(title)) if not elems: return for k, v in elems.items(): self._dbg('\t- \"{}\": {}'.format(k, v)) def _dbg(self, content): pre = os.path.basename(self._path) self._log.dbg('[{}] {}'.format(pre, content))
class CfgAggregator: file_prefix = 'f' dir_prefix = 'd' key_sep = '_' def __init__(self, path, profile_key, debug=False, dry=False): """ high level config parser @path: path to the config file @profile_key: profile key @debug: debug flag """ self.path = path self.profile_key = profile_key self.debug = debug self.dry = dry self.log = Logger() self._load() def _load(self): """load lower level config""" self.cfgyaml = CfgYaml(self.path, self.profile_key, debug=self.debug) # settings self.settings = Settings.parse(None, self.cfgyaml.settings) # dotfiles self.dotfiles = Dotfile.parse_dict(self.cfgyaml.dotfiles) if self.debug: self._debug_list('dotfiles', self.dotfiles) # profiles self.profiles = Profile.parse_dict(self.cfgyaml.profiles) if self.debug: self._debug_list('profiles', self.profiles) # actions self.actions = Action.parse_dict(self.cfgyaml.actions) if self.debug: self._debug_list('actions', self.actions) # trans_r self.trans_r = Transform.parse_dict(self.cfgyaml.trans_r) if self.debug: self._debug_list('trans_r', self.trans_r) # trans_w self.trans_w = Transform.parse_dict(self.cfgyaml.trans_w) if self.debug: self._debug_list('trans_w', self.trans_w) # variables self.variables = self.cfgyaml.variables if self.debug: self._debug_dict('variables', self.variables) # patch dotfiles in profiles self._patch_keys_to_objs(self.profiles, "dotfiles", self.get_dotfile) # patch action in dotfiles actions self._patch_keys_to_objs(self.dotfiles, "actions", self._get_action_w_args) # patch action in profiles actions self._patch_keys_to_objs(self.profiles, "actions", self._get_action_w_args) # patch actions in settings default_actions self._patch_keys_to_objs([self.settings], "default_actions", self._get_action_w_args) if self.debug: msg = 'default actions: {}'.format(self.settings.default_actions) self.log.dbg(msg) # patch trans_w/trans_r in dotfiles self._patch_keys_to_objs(self.dotfiles, "trans_r", self._get_trans_w_args(self._get_trans_r), islist=False) self._patch_keys_to_objs(self.dotfiles, "trans_w", self._get_trans_w_args(self._get_trans_w), islist=False) def _patch_keys_to_objs(self, containers, keys, get_by_key, islist=True): """ map for each key in the attribute 'keys' in 'containers' the returned object from the method 'get_by_key' """ if not containers: return if self.debug: self.log.dbg('patching {} ...'.format(keys)) for c in containers: objects = [] okeys = getattr(c, keys) if not okeys: continue if not islist: okeys = [okeys] for k in okeys: o = get_by_key(k) if not o: err = '{} does not contain'.format(c) err += ' a {} entry named {}'.format(keys, k) self.log.err(err) raise Exception(err) objects.append(o) if not islist: objects = objects[0] # if self.debug: # er = 'patching {}.{} with {}' # self.log.dbg(er.format(c, keys, objects)) setattr(c, keys, objects) def del_dotfile(self, dotfile): """remove this dotfile from the config""" return self.cfgyaml.del_dotfile(dotfile.key) def del_dotfile_from_profile(self, dotfile, profile): """remove this dotfile from this profile""" return self.cfgyaml.del_dotfile_from_profile(dotfile.key, profile.key) def _create_new_dotfile(self, src, dst, link): """create a new dotfile""" # get a new dotfile with a unique key key = self._get_new_dotfile_key(dst) if self.debug: self.log.dbg('new dotfile key: {}'.format(key)) # add the dotfile self.cfgyaml.add_dotfile(key, src, dst, link) return Dotfile(key, dst, src) def new(self, src, dst, link): """ import a new dotfile @src: path in dotpath @dst: path in FS @link: LinkType """ dst = self.path_to_dotfile_dst(dst) dotfile = self.get_dotfile_by_src_dst(src, dst) if not dotfile: dotfile = self._create_new_dotfile(src, dst, link) key = dotfile.key ret = self.cfgyaml.add_dotfile_to_profile(key, self.profile_key) if ret and self.debug: msg = 'new dotfile {} to profile {}' self.log.dbg(msg.format(key, self.profile_key)) self.save() if ret and not self.dry: # reload if self.debug: self.log.dbg('reloading config') olddebug = self.debug self.debug = False self._load() self.debug = olddebug return ret def _get_new_dotfile_key(self, dst): """return a new unique dotfile key""" path = os.path.expanduser(dst) existing_keys = self.cfgyaml.get_all_dotfile_keys() if self.settings.longkey: return self._get_long_key(path, existing_keys) return self._get_short_key(path, existing_keys) def _norm_key_elem(self, elem): """normalize path element for sanity""" elem = elem.lstrip('.') elem = elem.replace(' ', '-') return elem.lower() def _split_path_for_key(self, path): """return a list of path elements, excluded home path""" p = strip_home(path) dirs = [] while True: p, f = os.path.split(p) dirs.append(f) if not p or not f: break dirs.reverse() # remove empty entries dirs = filter(None, dirs) # normalize entries return list(map(self._norm_key_elem, dirs)) def _get_long_key(self, path, keys): """ return a unique long key representing the absolute path of path """ dirs = self._split_path_for_key(path) prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix key = self.key_sep.join([prefix] + dirs) return self._uniq_key(key, keys) def _get_short_key(self, path, keys): """ return a unique key where path is known not to be an already existing dotfile """ dirs = self._split_path_for_key(path) dirs.reverse() prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix entries = [] for d in dirs: entries.insert(0, d) key = self.key_sep.join([prefix] + entries) if key not in keys: return key return self._uniq_key(key, keys) def _uniq_key(self, key, keys): """unique dotfile key""" newkey = key cnt = 1 while newkey in keys: # if unable to get a unique path # get a random one newkey = self.key_sep.join([key, str(cnt)]) cnt += 1 return newkey def path_to_dotfile_dst(self, path): """normalize the path to match dotfile dst""" path = self._norm_path(path) # use tild for home home = os.path.expanduser(TILD) + os.sep if path.startswith(home): path = path[len(home):] path = os.path.join(TILD, path) return path def get_dotfile_by_dst(self, dst): """ get a list of dotfiles by dst @dst: dotfile dst (on filesystem) """ dotfiles = [] dst = self._norm_path(dst) for d in self.dotfiles: left = self._norm_path(d.dst) if left == dst: dotfiles.append(d) return dotfiles def get_dotfile_by_src_dst(self, src, dst): """ get a dotfile by src and dst @src: dotfile src (in dotpath) @dst: dotfile dst (on filesystem) """ try: src = self.cfgyaml.resolve_dotfile_src(src) except UndefinedException as e: err = 'unable to resolve {}: {}' self.log.err(err.format(src, e)) return None dotfiles = self.get_dotfile_by_dst(dst) for d in dotfiles: if d.src == src: return d return None def save(self): """save the config""" if self.dry: return True return self.cfgyaml.save() def dump(self): """dump the config dictionary""" return self.cfgyaml.dump() def get_settings(self): """return settings as a dict""" return self.settings.serialize()[Settings.key_yaml] def get_variables(self): """return variables""" return self.variables def get_profiles(self): """return profiles""" return self.profiles def get_profile(self): """return profile object""" try: return next(x for x in self.profiles if x.key == self.profile_key) except StopIteration: return None def get_profiles_by_dotfile_key(self, key): """return all profiles having this dotfile""" res = [] for p in self.profiles: keys = [d.key for d in p.dotfiles] if key in keys: res.append(p) return res def get_dotfiles(self): """get all dotfiles for this profile""" dotfiles = [] profile = self.get_profile() if not profile: return dotfiles return profile.dotfiles def get_dotfile(self, key): """ return dotfile object by key @key: the dotfile key to look for """ try: return next(x for x in self.dotfiles if x.key == key) except StopIteration: return None def _get_action(self, key): """return action by key""" try: return next(x for x in self.actions if x.key == key) except StopIteration: return None def _get_action_w_args(self, key): """return action by key with the arguments""" fields = shlex.split(key) if len(fields) > 1: # we have args key, *args = fields if self.debug: msg = 'action with parm: {} and {}' self.log.dbg(msg.format(key, args)) action = self._get_action(key).copy(args) else: action = self._get_action(key) return action def _get_trans_w_args(self, getter): """return transformation by key with the arguments""" def getit(key): fields = shlex.split(key) if len(fields) > 1: # we have args key, *args = fields if self.debug: msg = 'trans with parm: {} and {}' self.log.dbg(msg.format(key, args)) trans = getter(key).copy(args) else: trans = getter(key) return trans return getit def _get_trans_r(self, key): """return the trans_r with this key""" try: return next(x for x in self.trans_r if x.key == key) except StopIteration: return None def _get_trans_w(self, key): """return the trans_w with this key""" try: return next(x for x in self.trans_w if x.key == key) except StopIteration: return None def _norm_path(self, path): if not path: return path path = os.path.expanduser(path) path = os.path.expandvars(path) path = os.path.abspath(path) return path def _debug_list(self, title, elems): """pretty print list""" if not self.debug: return self.log.dbg('{}:'.format(title)) for e in elems: self.log.dbg('\t- {}'.format(e)) def _debug_dict(self, title, elems): """pretty print dict""" if not self.debug: return self.log.dbg('{}:'.format(title)) for k, v in elems.items(): self.log.dbg('\t- \"{}\": {}'.format(k, v))
class Installer: BACKUP_SUFFIX = '.dotdropbak' def __init__(self, base='.', create=True, backup=True, dry=False, safe=False, workdir='~/.config/dotdrop', debug=False, diff=True, totemp=None, showdiff=False): self.create = create self.backup = backup self.dry = dry self.safe = safe self.workdir = os.path.expanduser(workdir) self.base = base self.debug = debug self.diff = diff self.totemp = totemp self.showdiff = showdiff self.comparing = False self.action_executed = False self.log = Logger() def install(self, templater, src, dst, actions=[], noempty=False): """install the src to dst using a template""" self.action_executed = False src = os.path.join(self.base, os.path.expanduser(src)) if not os.path.exists(src): self.log.err('source dotfile does not exist: {}'.format(src)) return [] dst = os.path.expanduser(dst) if self.totemp: dst = self._pivot_path(dst, self.totemp) if utils.samefile(src, dst): # symlink loop self.log.err('dotfile points to itself: {}'.format(dst)) return [] if self.debug: self.log.dbg('install {} to {}'.format(src, dst)) if os.path.isdir(src): return self._handle_dir(templater, src, dst, actions=actions, noempty=noempty) return self._handle_file(templater, src, dst, actions=actions, noempty=noempty) def link(self, templater, src, dst, actions=[]): """set src as the link target of dst""" self.action_executed = False src = os.path.join(self.base, os.path.expanduser(src)) if not os.path.exists(src): self.log.err('source dotfile does not exist: {}'.format(src)) return [] dst = os.path.expanduser(dst) if self.totemp: # ignore actions return self.install(templater, src, dst, actions=[]) if Templategen.is_template(src): if self.debug: self.log.dbg('dotfile is a template') self.log.dbg('install to {} and symlink'.format(self.workdir)) tmp = self._pivot_path(dst, self.workdir, striphome=True) i = self.install(templater, src, tmp, actions=actions) if not i and not os.path.exists(tmp): return [] src = tmp return self._link(src, dst, actions=actions) def _link(self, src, dst, actions=[]): """set src as a link target of dst""" if os.path.lexists(dst): if os.path.realpath(dst) == os.path.realpath(src): if self.debug: self.log.dbg('ignoring "{}", link exists'.format(dst)) return [] if self.dry: self.log.dry('would remove {} and link to {}'.format(dst, src)) return [] msg = 'Remove "{}" for link creation?'.format(dst) if self.safe and not self.log.ask(msg): msg = 'ignoring "{}", link was not created' self.log.warn(msg.format(dst)) return [] try: utils.remove(dst) except OSError as e: self.log.err('something went wrong with {}: {}'.format(src, e)) return [] if self.dry: self.log.dry('would link {} to {}'.format(dst, src)) return [] base = os.path.dirname(dst) if not self._create_dirs(base): self.log.err('creating directory for {}'.format(dst)) return [] self._exec_pre_actions(actions) os.symlink(src, dst) self.log.sub('linked {} to {}'.format(dst, src)) return [(src, dst)] def _handle_file(self, templater, src, dst, actions=[], noempty=False): """install src to dst when is a file""" if self.debug: self.log.dbg('generate template for {}'.format(src)) self.log.dbg('ignore empty: {}'.format(noempty)) if utils.samefile(src, dst): # symlink loop self.log.err('dotfile points to itself: {}'.format(dst)) return [] content = templater.generate(src) if noempty and utils.content_empty(content): self.log.warn('ignoring empty template: {}'.format(src)) return [] if content is None: self.log.err('generate from template {}'.format(src)) return [] if not os.path.exists(src): self.log.err('source dotfile does not exist: {}'.format(src)) return [] st = os.stat(src) ret = self._write(src, dst, content, st.st_mode, actions=actions) if ret < 0: self.log.err('installing {} to {}'.format(src, dst)) return [] if ret > 0: if self.debug: self.log.dbg('ignoring {}'.format(dst)) return [] if ret == 0: if not self.dry and not self.comparing: self.log.sub('copied {} to {}'.format(src, dst)) return [(src, dst)] return [] def _handle_dir(self, templater, src, dst, actions=[], noempty=False): """install src to dst when is a directory""" if self.debug: self.log.dbg('install dir {}'.format(src)) self.log.dbg('ignore empty: {}'.format(noempty)) ret = [] if not self._create_dirs(dst): return [] # handle all files in dir for entry in os.listdir(src): f = os.path.join(src, entry) if not os.path.isdir(f): res = self._handle_file(templater, f, os.path.join(dst, entry), actions=actions, noempty=noempty) ret.extend(res) else: res = self._handle_dir(templater, f, os.path.join(dst, entry), actions=actions, noempty=noempty) ret.extend(res) return ret def _fake_diff(self, dst, content): """fake diff by comparing file content with content""" cur = '' with open(dst, 'br') as f: cur = f.read() return cur == content def _write(self, src, dst, content, rights, actions=[]): """write content to file return 0 for success, 1 when already exists -1 when error""" if self.dry: self.log.dry('would install {}'.format(dst)) return 0 if os.path.lexists(dst): samerights = os.stat(dst).st_mode == rights if self.diff and self._fake_diff(dst, content) and samerights: if self.debug: self.log.dbg('{} is the same'.format(dst)) return 1 if self.safe: if self.debug: self.log.dbg('change detected for {}'.format(dst)) if self.showdiff: self._diff_before_write(src, dst, content) if not self.log.ask('Overwrite \"{}\"'.format(dst)): self.log.warn('ignoring {}'.format(dst)) return 1 if self.backup and os.path.lexists(dst): self._backup(dst) base = os.path.dirname(dst) if not self._create_dirs(base): self.log.err('creating directory for {}'.format(dst)) return -1 if self.debug: self.log.dbg('write content to {}'.format(dst)) self._exec_pre_actions(actions) try: with open(dst, 'wb') as f: f.write(content) except NotADirectoryError as e: self.log.err('opening dest file: {}'.format(e)) return -1 os.chmod(dst, rights) return 0 def _diff_before_write(self, src, dst, src_content): """diff before writing when using --showdiff - not efficient""" # create tmp to diff for templates tmpfile = utils.get_tmpfile() with open(tmpfile, 'wb') as f: f.write(src_content) comp = Comparator(debug=self.debug) diff = comp.compare(tmpfile, dst) # fake the output for readability self.log.log('diff \"{}\" VS \"{}\"'.format(src, dst)) self.log.emph(diff) if tmpfile: utils.remove(tmpfile) def _create_dirs(self, directory): """mkdir -p <directory>""" if not self.create and not os.path.exists(directory): return False if os.path.exists(directory): return True if self.dry: self.log.dry('would mkdir -p {}'.format(directory)) return True if self.debug: self.log.dbg('mkdir -p {}'.format(directory)) os.makedirs(directory) return os.path.exists(directory) def _backup(self, path): """backup file pointed by path""" if self.dry: return dst = path.rstrip(os.sep) + self.BACKUP_SUFFIX self.log.log('backup {} to {}'.format(path, dst)) os.rename(path, dst) def _pivot_path(self, path, newdir, striphome=False): """change path to be under newdir""" if striphome: home = os.path.expanduser('~') path = path.lstrip(home) sub = path.lstrip(os.sep) return os.path.join(newdir, sub) def _exec_pre_actions(self, actions): """execute pre-actions if any""" if self.action_executed: return for action in actions: if self.dry: self.log.dry('would execute action: {}'.format(action)) else: if self.debug: self.log.dbg('executing pre action {}'.format(action)) action.execute() self.action_executed = True def _install_to_temp(self, templater, src, dst, tmpdir): """install a dotfile to a tempdir""" tmpdst = self._pivot_path(dst, tmpdir) return self.install(templater, src, tmpdst), tmpdst def install_to_temp(self, templater, tmpdir, src, dst): """install a dotfile to a tempdir""" ret = False tmpdst = '' # save some flags while comparing self.comparing = True drysaved = self.dry self.dry = False diffsaved = self.diff self.diff = False createsaved = self.create self.create = True # normalize src and dst src = os.path.expanduser(src) dst = os.path.expanduser(dst) if self.debug: self.log.dbg('tmp install {} to {}'.format(src, dst)) # install the dotfile to a temp directory for comparing ret, tmpdst = self._install_to_temp(templater, src, dst, tmpdir) if self.debug: self.log.dbg('tmp installed in {}'.format(tmpdst)) # reset flags self.dry = drysaved self.diff = diffsaved self.comparing = False self.create = createsaved return ret, tmpdst
class Cfg: key_all = 'ALL' # settings keys key_settings = 'config' key_dotpath = 'dotpath' key_backup = 'backup' key_create = 'create' key_banner = 'banner' key_long = 'longkey' key_keepdot = 'keepdot' key_ignoreempty = 'ignoreempty' key_showdiff = 'showdiff' key_deflink = 'link_by_default' key_workdir = 'workdir' # actions keys key_actions = 'actions' key_actions_pre = 'pre' key_actions_post = 'post' # transformations keys key_trans_r = 'trans' key_trans_w = 'trans_write' # template variables key_variables = 'variables' # shell variables key_dynvariables = 'dynvariables' # dotfiles keys key_dotfiles = 'dotfiles' key_dotfiles_src = 'src' key_dotfiles_dst = 'dst' key_dotfiles_link = 'link' key_dotfiles_link_children = 'link_children' key_dotfiles_noempty = 'ignoreempty' key_dotfiles_cmpignore = 'cmpignore' key_dotfiles_actions = 'actions' key_dotfiles_trans_r = 'trans' key_dotfiles_trans_w = 'trans_write' key_dotfiles_upignore = 'upignore' # profiles keys key_profiles = 'profiles' key_profiles_dots = 'dotfiles' key_profiles_incl = 'include' # settings defaults default_backup = True default_create = True default_banner = True default_link = LinkTypes.NOLINK default_longkey = False default_keepdot = False default_showdiff = False default_ignoreempty = False default_link_by_default = False default_workdir = '~/.config/dotdrop' def __init__(self, cfgpath, debug=False): """constructor @cfgpath: path to the config file @debug: enable debug """ if not os.path.exists(cfgpath): raise ValueError('config file does not exist: {}'.format(cfgpath)) # make sure to have an absolute path to config file self.cfgpath = os.path.abspath(cfgpath) self.debug = debug # init the logger self.log = Logger() # represents all entries under "config" # linked inside the yaml dict (self.content) self.lnk_settings = {} # represents all entries under "profiles" # linked inside the yaml dict (self.content) self.lnk_profiles = {} # represents all dotfiles # NOT linked inside the yaml dict (self.content) self.dotfiles = {} # dict of all action objects by action key # NOT linked inside the yaml dict (self.content) self.actions = {} # dict of all read transformation objects by trans key # NOT linked inside the yaml dict (self.content) self.trans_r = {} # dict of all write transformation objects by trans key # NOT linked inside the yaml dict (self.content) self.trans_w = {} # represents all dotfiles per profile by profile key # NOT linked inside the yaml dict (self.content) self.prodots = {} if not self._load_file(): raise ValueError('config is not valid') def eval_dotfiles(self, profile, variables, debug=False): """resolve dotfiles src/dst/actions templating for this profile""" t = Templategen(variables=variables) dotfiles = self._get_dotfiles(profile) for d in dotfiles: # src and dst path d.src = t.generate_string(d.src) d.dst = t.generate_string(d.dst) # pre actions if self.key_actions_pre in d.actions: for action in d.actions[self.key_actions_pre]: action.action = t.generate_string(action.action) # post actions if self.key_actions_post in d.actions: for action in d.actions[self.key_actions_post]: action.action = t.generate_string(action.action) return dotfiles def _load_file(self): """load the yaml file""" with open(self.cfgpath, 'r') as f: self.content = yaml.safe_load(f) if not self._is_valid(): return False return self._parse() def _is_valid(self): """test the yaml dict (self.content) is valid""" if self.key_profiles not in self.content: self.log.err('missing \"{}\" in config'.format(self.key_profiles)) return False if self.key_settings not in self.content: self.log.err('missing \"{}\" in config'.format(self.key_settings)) return False if self.key_dotfiles not in self.content: self.log.err('missing \"{}\" in config'.format(self.key_dotfiles)) return False return True def _parse(self): """parse config file""" # parse all actions if self.key_actions in self.content: if self.content[self.key_actions] is not None: for k, v in self.content[self.key_actions].items(): # loop through all actions if k in [self.key_actions_pre, self.key_actions_post]: # parse pre/post actions items = self.content[self.key_actions][k].items() for k2, v2 in items: if k not in self.actions: self.actions[k] = {} self.actions[k][k2] = Action(k2, k, v2) else: # parse naked actions as post actions if self.key_actions_post not in self.actions: self.actions[self.key_actions_post] = {} self.actions[self.key_actions_post][k] = Action( k, '', v) # parse read transformations if self.key_trans_r in self.content: if self.content[self.key_trans_r] is not None: for k, v in self.content[self.key_trans_r].items(): self.trans_r[k] = Transform(k, v) # parse write transformations if self.key_trans_w in self.content: if self.content[self.key_trans_w] is not None: for k, v in self.content[self.key_trans_w].items(): self.trans_w[k] = Transform(k, v) # parse the profiles self.lnk_profiles = self.content[self.key_profiles] if self.lnk_profiles is None: # ensures self.lnk_profiles is a dict self.content[self.key_profiles] = {} self.lnk_profiles = self.content[self.key_profiles] for k, v in self.lnk_profiles.items(): if self.key_profiles_dots in v and \ v[self.key_profiles_dots] is None: # if has the dotfiles entry but is empty # ensures it's an empty list v[self.key_profiles_dots] = [] # parse the settings self.lnk_settings = self.content[self.key_settings] self._complete_settings() # parse the dotfiles # and construct the dict of objects per dotfile key if not self.content[self.key_dotfiles]: # ensures the dotfiles entry is a dict self.content[self.key_dotfiles] = {} for k, v in self.content[self.key_dotfiles].items(): src = os.path.normpath(v[self.key_dotfiles_src]) dst = os.path.normpath(v[self.key_dotfiles_dst]) # Fail if both `link` and `link_children` present if self.key_dotfiles_link in v \ and self.key_dotfiles_link_children in v: msg = 'only one of `link` or `link_children` allowed per' msg += ' dotfile, error on dotfile "{}".' self.log.err(msg.format(k)) # Otherwise, get link type link = LinkTypes.NOLINK if self.key_dotfiles_link in v and v[self.key_dotfiles_link]: link = LinkTypes.PARENTS if self.key_dotfiles_link_children in v \ and v[self.key_dotfiles_link_children]: link = LinkTypes.CHILDREN noempty = v[self.key_dotfiles_noempty] if \ self.key_dotfiles_noempty \ in v else self.lnk_settings[self.key_ignoreempty] itsactions = v[self.key_dotfiles_actions] if \ self.key_dotfiles_actions in v else [] actions = self._parse_actions(itsactions) # parse read transformation itstrans_r = v[self.key_dotfiles_trans_r] if \ self.key_dotfiles_trans_r in v else None trans_r = None if itstrans_r: if type(itstrans_r) is list: msg = 'One transformation allowed per dotfile' msg += ', error on dotfile \"{}\"' self.log.err(msg.format(k)) msg = 'Please modify your config file to: \"trans: {}\"' self.log.err(msg.format(itstrans_r[0])) msg = 'see https://github.com/deadc0de6/dotdrop/wiki/transformations#config-error-with-transformation-list' # noqa self.log.err(msg) return False trans_r = self._parse_trans(itstrans_r, read=True) if not trans_r: msg = 'unknown trans \"{}\" for \"{}\"' self.log.err(msg.format(itstrans_r, k)) return False # parse write transformation itstrans_w = v[self.key_dotfiles_trans_w] if \ self.key_dotfiles_trans_w in v else None trans_w = None if itstrans_w: if type(itstrans_w) is list: msg = 'One write transformation allowed per dotfile' msg += ', error on dotfile \"{}\"' self.log.err(msg.format(k)) msg = 'Please modify your config file: \"trans_write: {}\"' self.log.err(msg.format(itstrans_w[0])) msg = 'see https://github.com/deadc0de6/dotdrop/wiki/transformations#config-error-with-transformation-list' # noqa self.log.err(msg) return False trans_w = self._parse_trans(itstrans_w, read=False) if not trans_w: msg = 'unknown trans_write \"{}\" for \"{}\"' self.log.err(msg.format(itstrans_w, k)) return False # disable transformation when link is true if link != LinkTypes.NOLINK and (trans_r or trans_w): msg = 'transformations disabled for \"{}\"'.format(dst) msg += ' because link is True' self.log.warn(msg) trans_r = None trans_w = None # parse cmpignore pattern cmpignores = v[self.key_dotfiles_cmpignore] if \ self.key_dotfiles_cmpignore in v else [] # parse upignore pattern upignores = v[self.key_dotfiles_upignore] if \ self.key_dotfiles_upignore in v else [] # create new dotfile self.dotfiles[k] = Dotfile(k, dst, src, link=link, actions=actions, trans_r=trans_r, trans_w=trans_w, cmpignore=cmpignores, noempty=noempty, upignore=upignores) # assign dotfiles to each profile for k, v in self.lnk_profiles.items(): self.prodots[k] = [] if self.key_profiles_dots not in v: # ensures is a list v[self.key_profiles_dots] = [] if not v[self.key_profiles_dots]: continue dots = v[self.key_profiles_dots] if self.key_all in dots: # add all if key ALL is used self.prodots[k] = list(self.dotfiles.values()) else: # add the dotfiles for d in dots: if d not in self.dotfiles: msg = 'unknown dotfile \"{}\" for {}'.format(d, k) self.log.err(msg) continue self.prodots[k].append(self.dotfiles[d]) # handle "include" for each profile for k in self.lnk_profiles.keys(): dots = self._get_included_dotfiles(k) self.prodots[k].extend(dots) # remove duplicates if any self.prodots[k] = list(set(self.prodots[k])) # make sure we have an absolute dotpath self.curdotpath = self.lnk_settings[self.key_dotpath] self.lnk_settings[self.key_dotpath] = \ self._abs_path(self.curdotpath) # make sure we have an absolute workdir self.curworkdir = self.lnk_settings[self.key_workdir] self.lnk_settings[self.key_workdir] = \ self._abs_path(self.curworkdir) return True def _abs_path(self, path): """return absolute path of path relative to the confpath""" path = os.path.expanduser(path) if not os.path.isabs(path): d = os.path.dirname(self.cfgpath) return os.path.join(d, path) return path def _get_included_dotfiles(self, profile): """find all dotfiles for a specific profile when using the include keyword""" included = [] if self.key_profiles_incl not in self.lnk_profiles[profile]: # no include found return included if not self.lnk_profiles[profile][self.key_profiles_incl]: # empty include found return included for other in self.lnk_profiles[profile][self.key_profiles_incl]: if other not in self.prodots: # no such profile self.log.warn('unknown included profile \"{}\"'.format(other)) continue included.extend(self.prodots[other]) return included def _parse_actions(self, entries): """parse actions specified for an element where entries are the ones defined for this dotfile""" res = { self.key_actions_pre: [], self.key_actions_post: [], } for line in entries: fields = shlex.split(line) entry = fields[0] args = [] if len(fields) > 1: args = fields[1:] action = None if self.key_actions_pre in self.actions and \ entry in self.actions[self.key_actions_pre]: kind = self.key_actions_pre if not args: action = self.actions[self.key_actions_pre][entry] else: a = self.actions[self.key_actions_pre][entry].action action = Action(entry, kind, a, *args) elif self.key_actions_post in self.actions and \ entry in self.actions[self.key_actions_post]: kind = self.key_actions_post if not args: action = self.actions[self.key_actions_post][entry] else: a = self.actions[self.key_actions_post][entry].action action = Action(entry, kind, a, *args) else: self.log.warn('unknown action \"{}\"'.format(entry)) continue res[kind].append(action) return res def _parse_trans(self, trans, read=True): """parse transformation key specified for a dotfile""" transformations = self.trans_r if not read: transformations = self.trans_w if trans not in transformations.keys(): return None return transformations[trans] def _complete_settings(self): """set settings defaults if not present""" if self.key_backup not in self.lnk_settings: self.lnk_settings[self.key_backup] = self.default_backup if self.key_create not in self.lnk_settings: self.lnk_settings[self.key_create] = self.default_create if self.key_banner not in self.lnk_settings: self.lnk_settings[self.key_banner] = self.default_banner if self.key_long not in self.lnk_settings: self.lnk_settings[self.key_long] = self.default_longkey if self.key_keepdot not in self.lnk_settings: self.lnk_settings[self.key_keepdot] = self.default_keepdot if self.key_deflink not in self.lnk_settings: self.lnk_settings[self.key_deflink] = self.default_link_by_default if self.key_workdir not in self.lnk_settings: self.lnk_settings[self.key_workdir] = self.default_workdir if self.key_showdiff not in self.lnk_settings: self.lnk_settings[self.key_showdiff] = self.default_showdiff if self.key_ignoreempty not in self.lnk_settings: self.lnk_settings[self.key_ignoreempty] = self.default_ignoreempty def _save(self, content, path): """writes the config to file""" ret = False with open(path, 'w') as f: ret = yaml.dump(content, f, default_flow_style=False, indent=2) return ret def _norm_key_elem(self, elem): """normalize path element for sanity""" elem = elem.lstrip('.') elem = elem.replace(' ', '-') return elem.lower() def _get_paths(self, path): """return a list of path elements, excluded home path""" p = strip_home(path) dirs = [] while True: p, f = os.path.split(p) dirs.append(f) if not p or not f: break dirs.reverse() # remove empty entries dirs = filter(None, dirs) # normalize entries dirs = list(map(self._norm_key_elem, dirs)) return dirs def _get_long_key(self, path): """return a long key representing the absolute path of path""" dirs = self._get_paths(path) # prepend with indicator if os.path.isdir(path): key = 'd_{}'.format('_'.join(dirs)) else: key = 'f_{}'.format('_'.join(dirs)) return key def _get_short_key(self, path, keys): """return a unique key where path is known not to be an already existing dotfile""" dirs = self._get_paths(path) dirs.reverse() pre = 'f' if os.path.isdir(path): pre = 'd' entries = [] for d in dirs: entries.insert(0, d) key = '_'.join(entries) key = '{}_{}'.format(pre, key) if key not in keys: return key okey = key cnt = 1 while key in keys: # if unable to get a unique path # get a random one key = '{}_{}'.format(okey, cnt) cnt += 1 return key def _dotfile_exists(self, dotfile): """return True and the existing dotfile key if it already exists, False and a new unique key otherwise""" dsts = [(k, d.dst) for k, d in self.dotfiles.items()] if dotfile.dst in [x[1] for x in dsts]: return True, [x[0] for x in dsts if x[1] == dotfile.dst][0] # return key for this new dotfile path = os.path.expanduser(dotfile.dst) if self.lnk_settings[self.key_long]: return False, self._get_long_key(path) return False, self._get_short_key(path, self.dotfiles.keys()) def new(self, dotfile, profile, link=LinkTypes.NOLINK, debug=False): """import new dotfile dotfile key will change and can be empty""" # keep it short home = os.path.expanduser('~') dotfile.dst = dotfile.dst.replace(home, '~', 1) # adding new profile if doesn't exist if profile not in self.lnk_profiles: if debug: self.log.dbg('adding profile to config') # in the yaml self.lnk_profiles[profile] = {self.key_profiles_dots: []} # in the global list of dotfiles per profile self.prodots[profile] = [] exists, key = self._dotfile_exists(dotfile) if exists: if debug: self.log.dbg('key already exists: {}'.format(key)) # retrieve existing dotfile dotfile = self.dotfiles[key] if dotfile in self.prodots[profile]: self.log.err('\"{}\" already present'.format(dotfile.key)) return False, dotfile # add for this profile self.prodots[profile].append(dotfile) # get a pointer in the yaml profiles->this_profile # and complete it with the new entry pro = self.content[self.key_profiles][profile] if self.key_all not in pro[self.key_profiles_dots]: pro[self.key_profiles_dots].append(dotfile.key) return True, dotfile if debug: self.log.dbg('dotfile attributed key: {}'.format(key)) # adding the new dotfile dotfile.key = key dotfile.link = link if debug: self.log.dbg('adding new dotfile: {}'.format(dotfile)) # add the entry in the yaml file dots = self.content[self.key_dotfiles] dots[dotfile.key] = { self.key_dotfiles_dst: dotfile.dst, self.key_dotfiles_src: dotfile.src, } # set the link flag if link == LinkTypes.PARENTS: dots[dotfile.key][self.key_dotfiles_link] = True elif link == LinkTypes.CHILDREN: dots[dotfile.key][self.key_dotfiles_link_children] = True # link it to this profile in the yaml file pro = self.content[self.key_profiles][profile] if self.key_all not in pro[self.key_profiles_dots]: pro[self.key_profiles_dots].append(dotfile.key) # add it to the global list of dotfiles self.dotfiles[dotfile.key] = dotfile # add it to this profile self.prodots[profile].append(dotfile) return True, dotfile def _get_dotfiles(self, profile): """return a list of dotfiles for a specific profile""" if profile not in self.prodots: return [] return sorted(self.prodots[profile], key=lambda x: str(x.key)) def get_profiles(self): """return all defined profiles""" return self.lnk_profiles.keys() def get_settings(self): """return all defined settings""" return self.lnk_settings.copy() def get_variables(self, profile, debug=False): """return the variables for this profile""" # get flat variables variables = self._get_variables(profile) # get interpreted variables dvariables = self._get_dynvariables(profile) # recursive resolve variables allvars = variables.copy() allvars.update(dvariables) var = self._rec_resolve_vars(allvars) # execute dynvariables for k in dvariables.keys(): var[k] = shell(var[k]) if debug: self.log.dbg('variables:') for k, v in var.items(): self.log.dbg('\t\"{}\": {}'.format(k, v)) return var def _rec_resolve_vars(self, variables): """recursive resolve all variables""" t = Templategen(variables=variables) for k in variables.keys(): val = variables[k] while Templategen.var_is_template(val): val = t.generate_string(val) variables[k] = val t.update_variables(variables) return variables def _get_variables(self, profile): """return the flat variables""" variables = {} # profile variable variables['profile'] = profile # global variables if self.key_variables in self.content: variables.update(self.content[self.key_variables]) if profile not in self.lnk_profiles: return variables # profile variables var = self.lnk_profiles[profile] if self.key_variables in var.keys(): for k, v in var[self.key_variables].items(): variables[k] = v return variables def _get_dynvariables(self, profile): """return the dyn variables""" variables = {} # global dynvariables if self.key_dynvariables in self.content: # interpret dynamic variables variables.update(self.content[self.key_dynvariables]) if profile not in self.lnk_profiles: return variables # profile dynvariables var = self.lnk_profiles[profile] if self.key_dynvariables in var.keys(): variables.update(var[self.key_dynvariables]) return variables def dump(self): """return a dump of the config""" # temporary reset paths dotpath = self.lnk_settings[self.key_dotpath] workdir = self.lnk_settings[self.key_workdir] self.lnk_settings[self.key_dotpath] = self.curdotpath self.lnk_settings[self.key_workdir] = self.curworkdir # dump ret = yaml.dump(self.content, default_flow_style=False, indent=2) ret = ret.replace('{}', '') # restore paths self.lnk_settings[self.key_dotpath] = dotpath self.lnk_settings[self.key_workdir] = workdir return ret def save(self): """save the config to file""" # temporary reset paths dotpath = self.lnk_settings[self.key_dotpath] workdir = self.lnk_settings[self.key_workdir] self.lnk_settings[self.key_dotpath] = self.curdotpath self.lnk_settings[self.key_workdir] = self.curworkdir # save ret = self._save(self.content, self.cfgpath) # restore path self.lnk_settings[self.key_dotpath] = dotpath self.lnk_settings[self.key_workdir] = workdir return ret
class Installer: def __init__(self, base='.', create=True, backup=True, dry=False, safe=False, workdir='~/.config/dotdrop', debug=False, diff=True, totemp=None, showdiff=False, backup_suffix='.dotdropbak', diff_cmd=''): """ @base: directory path where to search for templates @create: create directory hierarchy if missing when installing @backup: backup existing dotfile when installing @dry: just simulate @safe: ask for any overwrite @workdir: where to install template before symlinking @debug: enable debug @diff: diff when installing if True @totemp: deploy to this path instead of dotfile dst if not None @showdiff: show the diff before overwriting (or asking for) @backup_suffix: suffix for dotfile backup file @diff_cmd: diff command to use """ self.create = create self.backup = backup self.dry = dry self.safe = safe workdir = os.path.expanduser(workdir) workdir = os.path.normpath(workdir) self.workdir = workdir base = os.path.expanduser(base) base = os.path.normpath(base) self.base = base self.debug = debug self.diff = diff self.totemp = totemp self.showdiff = showdiff self.backup_suffix = backup_suffix self.diff_cmd = diff_cmd self.action_executed = False # avoids printing file copied logs # when using install_to_tmp for comparing self.comparing = False self.log = Logger() ######################################################## # public methods ######################################################## def install(self, templater, src, dst, linktype, actionexec=None, noempty=False, ignore=[], is_template=True, chmod=None, force_chmod=False): """ install src to dst @templater: the templater object @src: dotfile source path in dotpath @dst: dotfile destination path in the FS @linktype: linktypes.LinkTypes @actionexec: action executor callback @noempty: render empty template flag @ignore: pattern to ignore when installing @is_template: this dotfile is a template @chmod: rights to apply if any @force_chmod: do not ask user to chmod return - True, None : success - False, error_msg : error - False, None : ignored """ if not src or not dst: # fake dotfile if self.debug: self.log.dbg('fake dotfile installed') self._exec_pre_actions(actionexec) return True, None if self.debug: msg = 'installing \"{}\" to \"{}\" (link: {})' self.log.dbg(msg.format(src, dst, str(linktype))) src, dst, cont, err = self._check_paths(src, dst, chmod) if not cont: return self._log_install(cont, err) # check source file exists src = os.path.join(self.base, src) if not os.path.exists(src): err = 'source dotfile does not exist: {}'.format(src) return self._log_install(False, err) self.action_executed = False # install to temporary dir # and ignore any actions if self.totemp: r, err, _ = self.install_to_temp(templater, self.totemp, src, dst, is_template=is_template, chmod=chmod, ignore=ignore) return self._log_install(r, err) isdir = os.path.isdir(src) if self.debug: self.log.dbg('install {} to {}'.format(src, dst)) self.log.dbg('\"{}\" is a directory: {}'.format(src, isdir)) if linktype == LinkTypes.NOLINK: # normal file if isdir: r, err = self._copy_dir(templater, src, dst, actionexec=actionexec, noempty=noempty, ignore=ignore, is_template=is_template, chmod=chmod) else: r, err = self._copy_file(templater, src, dst, actionexec=actionexec, noempty=noempty, ignore=ignore, is_template=is_template, chmod=chmod) elif linktype == LinkTypes.LINK: # symlink r, err = self._link(templater, src, dst, actionexec=actionexec, is_template=is_template) elif linktype == LinkTypes.LINK_CHILDREN: # symlink direct children if not isdir: if self.debug: msg = 'symlink children of {} to {}' self.log.dbg(msg.format(src, dst)) err = 'source dotfile is not a directory: {}'.format(src) r = False else: r, err = self._link_children(templater, src, dst, actionexec=actionexec, is_template=is_template, ignore=ignore) if self.debug: self.log.dbg('before chmod: {} err:{}'.format(r, err)) if self.dry: return self._log_install(r, err) # handle chmod # - on success (r, not err) # - no change (not r, not err) # but not when # - error (not r, err) # - aborted (not r, err) if (r or (not r and not err)): if not chmod: chmod = utils.get_file_perm(src) dstperms = utils.get_file_perm(dst) if dstperms != chmod: # apply mode msg = 'chmod {} to {:o}'.format(dst, chmod) if not force_chmod and self.safe and not self.log.ask(msg): r = False err = 'aborted' else: if not self.comparing: self.log.sub('chmod {} to {:o}'.format(dst, chmod)) if utils.chmod(dst, chmod, debug=self.debug): r = True else: r = False err = 'chmod failed' return self._log_install(r, err) def install_to_temp(self, templater, tmpdir, src, dst, is_template=True, chmod=None, ignore=[]): """ install a dotfile to a tempdir @templater: the templater object @tmpdir: where to install @src: dotfile source path in dotpath @dst: dotfile destination path in the FS @is_template: this dotfile is a template @chmod: rights to apply if any @ignore: patterns to ignore return - success, error-if-any, dotfile-installed-path """ if self.debug: self.log.dbg('tmp install {} (defined dst: {})'.format(src, dst)) src, dst, cont, err = self._check_paths(src, dst, chmod) if not cont: return self._log_install(cont, err) ret = False tmpdst = '' # save flags self.comparing = True drysaved = self.dry self.dry = False diffsaved = self.diff self.diff = False createsaved = self.create self.create = True totemp = self.totemp self.totemp = None # install the dotfile to a temp directory tmpdst = self._pivot_path(dst, tmpdir) ret, err = self.install(templater, src, tmpdst, LinkTypes.NOLINK, is_template=is_template, chmod=chmod, ignore=ignore) if self.debug: if ret: self.log.dbg('tmp installed in {}'.format(tmpdst)) # restore flags self.dry = drysaved self.diff = diffsaved self.create = createsaved self.comparing = False self.totemp = totemp return ret, err, tmpdst ######################################################## # low level accessors for public methods ######################################################## def _link(self, templater, src, dst, actionexec=None, is_template=True): """ install link:link return - True, None : success - False, error_msg : error - False, None : ignored - False, 'aborted' : user aborted """ if is_template: if self.debug: self.log.dbg('is a template') self.log.dbg('install to {}'.format(self.workdir)) tmp = self._pivot_path(dst, self.workdir, striphome=True) r, err = self.install(templater, src, tmp, LinkTypes.NOLINK, actionexec=actionexec, is_template=is_template) if not r and not os.path.exists(tmp): return r, err src = tmp r, err = self._symlink(src, dst, actionexec=actionexec) return r, err def _link_children(self, templater, src, dst, actionexec=None, is_template=True, ignore=[]): """ install link:link_children return - True, None : success - False, error_msg : error - False, None : ignored - False, 'aborted' : user aborted """ parent = os.path.join(self.base, src) if not os.path.lexists(dst): if self.dry: self.log.dry('would create directory "{}"'.format(dst)) else: if not self.comparing: self.log.sub('creating directory "{}"'.format(dst)) self._create_dirs(dst) if os.path.isfile(dst): msg = ''.join([ 'Remove regular file {} and ', 'replace with empty directory?', ]).format(dst) if self.safe and not self.log.ask(msg): return False, 'aborted' os.unlink(dst) self._create_dirs(dst) children = os.listdir(parent) srcs = [ os.path.normpath(os.path.join(parent, child)) for child in children ] dsts = [ os.path.normpath(os.path.join(dst, child)) for child in children ] installed = 0 for i in range(len(children)): subsrc = srcs[i] subdst = dsts[i] if utils.must_ignore([subsrc, subdst], ignore, debug=self.debug): if self.debug: self.log.dbg( 'ignoring install of {} to {}'.format(src, dst), ) continue if self.debug: self.log.dbg('symlink child {} to {}'.format(subsrc, subdst)) if is_template: if self.debug: self.log.dbg('child is a template') self.log.dbg('install to {} and symlink'.format( self.workdir)) tmp = self._pivot_path(subdst, self.workdir, striphome=True) r, e = self.install(templater, subsrc, tmp, LinkTypes.NOLINK, actionexec=actionexec, is_template=is_template) if not r and e and not os.path.exists(tmp): continue subsrc = tmp ret, err = self._symlink(subsrc, subdst, actionexec=actionexec) if ret: installed += 1 # void actionexec if dotfile installed # to prevent from running actions multiple times actionexec = None else: if err: return ret, err return installed > 0, None ######################################################## # file operations ######################################################## def _symlink(self, src, dst, actionexec=None): """ set src as a link target of dst return - True, None : success - False, error_msg : error - False, None : ignored - False, 'aborted' : user aborted """ overwrite = not self.safe if os.path.lexists(dst): if os.path.realpath(dst) == os.path.realpath(src): msg = 'ignoring "{}", link already exists'.format(dst) if self.debug: self.log.dbg(msg) return False, None if self.dry: self.log.dry('would remove {} and link to {}'.format(dst, src)) return True, None if self.showdiff: self._show_diff_before_write(src, dst) msg = 'Remove "{}" for link creation?'.format(dst) if self.safe and not self.log.ask(msg): return False, 'aborted' overwrite = True try: utils.removepath(dst) except OSError as e: err = 'something went wrong with {}: {}'.format(src, e) return False, err if self.dry: self.log.dry('would link {} to {}'.format(dst, src)) return True, None base = os.path.dirname(dst) if not self._create_dirs(base): err = 'error creating directory for {}'.format(dst) return False, err r, e = self._exec_pre_actions(actionexec) if not r: return False, e # re-check in case action created the file if os.path.lexists(dst): msg = 'Remove "{}" for link creation?'.format(dst) if self.safe and not overwrite and not self.log.ask(msg): return False, 'aborted' try: utils.removepath(dst) except OSError as e: err = 'something went wrong with {}: {}'.format(src, e) return False, err os.symlink(src, dst) if not self.comparing: self.log.sub('linked {} to {}'.format(dst, src)) return True, None def _copy_file(self, templater, src, dst, actionexec=None, noempty=False, ignore=[], is_template=True, chmod=None): """ install src to dst when is a file return - True, None : success - False, error_msg : error - False, None : ignored - False, 'aborted' : user aborted """ if self.debug: self.log.dbg('deploy file: {}'.format(src)) self.log.dbg('ignore empty: {}'.format(noempty)) self.log.dbg('ignore pattern: {}'.format(ignore)) self.log.dbg('is_template: {}'.format(is_template)) self.log.dbg('no empty: {}'.format(noempty)) # check no loop if utils.samefile(src, dst): err = 'dotfile points to itself: {}'.format(dst) return False, err if utils.must_ignore([src, dst], ignore, debug=self.debug): if self.debug: self.log.dbg('ignoring install of {} to {}'.format(src, dst)) return False, None if utils.samefile(src, dst): # loop err = 'dotfile points to itself: {}'.format(dst) return False, err if not os.path.exists(src): err = 'source dotfile does not exist: {}'.format(src) return False, err # handle the file content = None if is_template: # template the file saved = templater.add_tmp_vars(self._get_tmp_file_vars(src, dst)) try: content = templater.generate(src) except UndefinedException as e: return False, str(e) finally: templater.restore_vars(saved) # test is empty if noempty and utils.content_empty(content): if self.debug: self.log.dbg('ignoring empty template: {}'.format(src)) return False, None if content is None: err = 'empty template {}'.format(src) return False, err # write the file ret, err = self._write(src, dst, content=content, actionexec=actionexec, chmod=chmod) if ret and not err: if not self.dry and not self.comparing: self.log.sub('install {} to {}'.format(src, dst)) return ret, err def _copy_dir(self, templater, src, dst, actionexec=None, noempty=False, ignore=[], is_template=True, chmod=None): """ install src to dst when is a directory return - True, None : success - False, error_msg : error - False, None : ignored - False, 'aborted' : user aborted """ if self.debug: self.log.dbg('deploy dir {}'.format(src)) # default to nothing installed and no error ret = False, None # create the directory anyway if not self._create_dirs(dst): err = 'creating directory for {}'.format(dst) return False, err # handle all files in dir for entry in os.listdir(src): f = os.path.join(src, entry) if self.debug: self.log.dbg('deploy sub from {}: {}'.format(dst, entry)) if not os.path.isdir(f): # is file res, err = self._copy_file(templater, f, os.path.join(dst, entry), actionexec=actionexec, noempty=noempty, ignore=ignore, is_template=is_template, chmod=None) if not res and err: # error occured ret = res, err break elif res: # something got installed ret = True, None else: # is directory res, err = self._copy_dir(templater, f, os.path.join(dst, entry), actionexec=actionexec, noempty=noempty, ignore=ignore, is_template=is_template, chmod=None) if not res and err: # error occured ret = res, err break elif res: # something got installed ret = True, None return ret def _write(self, src, dst, content=None, actionexec=None, chmod=None): """ copy dotfile / write content to file return - True, None : success - False, error_msg : error - False, None : ignored - False, 'aborted' : user aborted """ overwrite = not self.safe if self.dry: self.log.dry('would install {}'.format(dst)) return True, None if os.path.lexists(dst): try: os.stat(dst) except OSError as e: if e.errno == errno.ENOENT: # broken symlink err = 'broken symlink {}'.format(dst) return False, err if self.diff: if not self._is_different(src, dst, content=content): if self.debug: self.log.dbg('{} is the same'.format(dst)) return False, None if self.safe: if self.debug: self.log.dbg('change detected for {}'.format(dst)) if self.showdiff: # get diff self._show_diff_before_write(src, dst, content=content) if not self.log.ask('Overwrite \"{}\"'.format(dst)): return False, 'aborted' overwrite = True if self.backup and os.path.lexists(dst): self._backup(dst) base = os.path.dirname(dst) if not self._create_dirs(base): err = 'creating directory for {}'.format(dst) return False, err r, e = self._exec_pre_actions(actionexec) if not r: return False, e if self.debug: self.log.dbg('install file to \"{}\"'.format(dst)) # re-check in case action created the file if self.safe and not overwrite and os.path.lexists(dst): if not self.log.ask('Overwrite \"{}\"'.format(dst)): self.log.warn('ignoring {}'.format(dst)) return False, 'aborted' if content: # write content the file try: with open(dst, 'wb') as f: f.write(content) shutil.copymode(src, dst) except NotADirectoryError as e: err = 'opening dest file: {}'.format(e) return False, err except Exception as e: return False, str(e) else: # copy file try: shutil.copyfile(src, dst) shutil.copymode(src, dst) except Exception as e: return False, str(e) return True, None ######################################################## # helpers ######################################################## def _get_tmp_file_vars(self, src, dst): tmp = {} tmp['_dotfile_sub_abs_src'] = src tmp['_dotfile_sub_abs_dst'] = dst return tmp def _is_different(self, src, dst, content=None): """ returns True if file is different and needs to be installed """ # check file content if content: tmp = utils.write_to_tmpfile(content) src = tmp r = utils.fastdiff(src, dst) if r: if self.debug: self.log.dbg('content differ') return r def _show_diff_before_write(self, src, dst, content=None): """ diff before writing using a temp file if content is not None returns diff string ('' if same) """ tmp = None if content: tmp = utils.write_to_tmpfile(content) src = tmp diff = utils.diff(modified=src, original=dst, diff_cmd=self.diff_cmd) if tmp: utils.removepath(tmp, logger=self.log) if diff: self._print_diff(src, dst, diff) return diff def _print_diff(self, src, dst, diff): """show diff to user""" self.log.log('diff \"{}\" VS \"{}\"'.format(dst, src)) self.log.emph(diff) def _create_dirs(self, directory): """mkdir -p <directory>""" if not self.create and not os.path.exists(directory): if self.debug: self.log.dbg('no mkdir as \"create\" set to false in config') return False if os.path.exists(directory): return True if self.dry: self.log.dry('would mkdir -p {}'.format(directory)) return True if self.debug: self.log.dbg('mkdir -p {}'.format(directory)) if not self.comparing: self.log.sub('create directory {}'.format(directory)) os.makedirs(directory, exist_ok=True) return os.path.exists(directory) def _backup(self, path): """backup file pointed by path""" if self.dry: return dst = path.rstrip(os.sep) + self.backup_suffix self.log.log('backup {} to {}'.format(path, dst)) os.rename(path, dst) def _pivot_path(self, path, newdir, striphome=False): """change path to be under newdir""" if self.debug: self.log.dbg('pivot new dir: \"{}\"'.format(newdir)) self.log.dbg('strip home: {}'.format(striphome)) if striphome: path = utils.strip_home(path) sub = path.lstrip(os.sep) new = os.path.join(newdir, sub) if self.debug: self.log.dbg('pivot \"{}\" to \"{}\"'.format(path, new)) return new def _exec_pre_actions(self, actionexec): """execute action executor""" if self.action_executed: return True, None if not actionexec: return True, None ret, err = actionexec() self.action_executed = True return ret, err def _log_install(self, boolean, err): """log installation process""" if not self.debug: return boolean, err if boolean: self.log.dbg('install: SUCCESS') else: if err: self.log.dbg('install: ERROR: {}'.format(err)) else: self.log.dbg('install: IGNORED') return boolean, err def _check_paths(self, src, dst, chmod): """ check and normalize param returns <src>, <dst>, <continue>, <error> """ # check both path are valid if not dst or not src: err = 'empty dst or src for {}'.format(src) if self.debug: self.log.dbg(err) return None, None, False, err # normalize src and dst src = os.path.expanduser(src) src = os.path.normpath(src) dst = os.path.expanduser(dst) dst = os.path.normpath(dst) return src, dst, True, None
class Templategen: def __init__(self, base='.', variables={}, func_file=[], filter_file=[], debug=False): """constructor @base: directory path where to search for templates @variables: dictionary of variables for templates @func_file: file path to load functions from @filter_file: file path to load filters from @debug: enable debug """ self.base = base.rstrip(os.sep) self.debug = debug self.log = Logger() self.variables = {} loader1 = FileSystemLoader(self.base) loader2 = FunctionLoader(self._template_loader) loader = ChoiceLoader([loader1, loader2]) self.env = Environment(loader=loader, trim_blocks=True, lstrip_blocks=True, keep_trailing_newline=True, block_start_string=BLOCK_START, block_end_string=BLOCK_END, variable_start_string=VAR_START, variable_end_string=VAR_END, comment_start_string=COMMENT_START, comment_end_string=COMMENT_END, undefined=StrictUndefined) # adding variables self.variables['env'] = os.environ if variables: self.variables.update(variables) # adding header method self.env.globals['header'] = self._header # adding helper methods if self.debug: self.log.dbg('load global functions:') self._load_funcs_to_dic(jhelpers, self.env.globals) if func_file: for f in func_file: if self.debug: self.log.dbg('load custom functions from {}'.format(f)) self._load_path_to_dic(f, self.env.globals) if filter_file: for f in filter_file: if self.debug: self.log.dbg('load custom filters from {}'.format(f)) self._load_path_to_dic(f, self.env.filters) if self.debug: self._debug_dict('template additional variables', variables) def generate(self, src): """ render template from path may raise a UndefinedException in case a variable is undefined """ if not os.path.exists(src): return '' try: return self._handle_file(src) except UndefinedError as e: err = 'undefined variable: {}'.format(e.message) raise UndefinedException(err) def generate_string(self, string): """ render template from string may raise a UndefinedException in case a variable is undefined """ if not string: return '' try: return self.env.from_string(string).render(self.variables) except UndefinedError as e: err = 'undefined variable: {}'.format(e.message) raise UndefinedException(err) def add_tmp_vars(self, newvars={}): """add vars to the globals, make sure to call restore_vars""" saved_variables = self.variables.copy() if not newvars: return saved_variables self.variables.update(newvars) return saved_variables def restore_vars(self, saved_globals): """restore globals from add_tmp_vars""" self.variables = saved_globals.copy() def update_variables(self, variables): """update variables""" self.variables.update(variables) def _load_path_to_dic(self, path, dic): mod = utils.get_module_from_path(path) if not mod: self.log.warn('cannot load module \"{}\"'.format(path)) return self._load_funcs_to_dic(mod, dic) def _load_funcs_to_dic(self, mod, dic): """dynamically load functions from module to dic""" if not mod or not dic: return funcs = utils.get_module_functions(mod) for name, func in funcs: if self.debug: self.log.dbg('load function \"{}\"'.format(name)) dic[name] = func def _header(self, prepend=''): """add a comment usually in the header of a dotfile""" return '{}{}'.format(prepend, utils.header()) def _handle_file(self, src): """generate the file content from template""" try: import magic filetype = magic.from_file(src, mime=True) if self.debug: self.log.dbg('using \"magic\" for filetype identification') except ImportError: # fallback _, filetype = utils.run(['file', '-b', '--mime-type', src], debug=self.debug) if self.debug: self.log.dbg('using \"file\" for filetype identification') filetype = filetype.strip() istext = self._is_text(filetype) if self.debug: self.log.dbg('filetype \"{}\": {}'.format(src, filetype)) if self.debug: self.log.dbg('is text \"{}\": {}'.format(src, istext)) if not istext: return self._handle_bin_file(src) return self._handle_text_file(src) def _is_text(self, fileoutput): """return if `file -b` output is ascii text""" out = fileoutput.lower() if out.startswith('text'): return True if 'empty' in out: return True if 'json' in out: return True return False def _template_loader(self, relpath): """manually load template when outside of base""" path = os.path.join(self.base, relpath) path = os.path.normpath(path) if not os.path.exists(path): raise TemplateNotFound(path) with open(path, 'r') as f: content = f.read() return content def _handle_text_file(self, src): """write text to file""" template_rel_path = os.path.relpath(src, self.base) try: template = self.env.get_template(template_rel_path) content = template.render(self.variables) except UnicodeDecodeError: data = self._read_bad_encoded_text(src) content = self.generate_string(data) return content.encode('utf-8') def _handle_bin_file(self, src): """write binary to file""" # this is dirty if not src.startswith(self.base): src = os.path.join(self.base, src) with open(src, 'rb') as f: content = f.read() return content def _read_bad_encoded_text(self, path): """decode non utf-8 data""" with open(path, 'rb') as f: data = f.read() return data.decode('utf-8', 'replace') @staticmethod def is_template(path, ignore=[]): """recursively check if any file is a template within path""" path = os.path.expanduser(path) if utils.must_ignore([path], ignore, debug=False): return False if not os.path.exists(path): return False if os.path.isfile(path): # is file return Templategen._is_template(path, ignore=ignore) for entry in os.listdir(path): fpath = os.path.join(path, entry) if not os.path.isfile(fpath): # recursively explore directory if Templategen.is_template(fpath, ignore=ignore): return True else: # check if file is a template if Templategen._is_template(fpath, ignore=ignore): return True return False @staticmethod def var_is_template(string): """check if variable contains template(s)""" return VAR_START in str(string) @staticmethod def _is_template(path, ignore): """test if file pointed by path is a template""" if utils.must_ignore([path], ignore, debug=False): return False if not os.path.isfile(path): return False if os.stat(path).st_size == 0: return False markers = [BLOCK_START, VAR_START, COMMENT_START] patterns = [re.compile(marker.encode()) for marker in markers] try: with io.open(path, "r", encoding="utf-8") as f: m = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) for pattern in patterns: if pattern.search(m): return True except UnicodeDecodeError: # is binary so surely no template return False return False def _debug_dict(self, title, elems): """pretty print dict""" if not self.debug: return self.log.dbg('{}:'.format(title)) if not elems: return for k, v in elems.items(): self.log.dbg(' - \"{}\": {}'.format(k, v))
class Installer: def __init__(self, base='.', create=True, backup=True, dry=False, safe=False, workdir='~/.config/dotdrop', debug=False, diff=True, totemp=None, showdiff=False, backup_suffix='.dotdropbak'): """constructor @base: directory path where to search for templates @create: create directory hierarchy if missing when installing @backup: backup existing dotfile when installing @dry: just simulate @safe: ask for any overwrite @workdir: where to install template before symlinking @debug: enable debug @diff: diff when installing if True @totemp: deploy to this path instead of dotfile dst if not None @showdiff: show the diff before overwriting (or asking for) @backup_suffix: suffix for dotfile backup file """ self.create = create self.backup = backup self.dry = dry self.safe = safe self.workdir = os.path.expanduser(workdir) self.base = base self.debug = debug self.diff = diff self.totemp = totemp self.showdiff = showdiff self.backup_suffix = backup_suffix self.comparing = False self.action_executed = False self.log = Logger() def install(self, templater, src, dst, actions=[], noempty=False): """install the src to dst using a template""" if self.debug: self.log.dbg('install {} to {}'.format(src, dst)) self.action_executed = False src = os.path.join(self.base, os.path.expanduser(src)) if not os.path.exists(src): self.log.err('source dotfile does not exist: {}'.format(src)) return [] dst = os.path.expanduser(dst) if self.totemp: dst = self._pivot_path(dst, self.totemp) if utils.samefile(src, dst): # symlink loop self.log.err('dotfile points to itself: {}'.format(dst)) return [] isdir = os.path.isdir(src) if self.debug: self.log.dbg('install {} to {}'.format(src, dst)) self.log.dbg('is \"{}\" a directory: {}'.format(src, isdir)) if isdir: return self._handle_dir(templater, src, dst, actions=actions, noempty=noempty) return self._handle_file(templater, src, dst, actions=actions, noempty=noempty) def link(self, templater, src, dst, actions=[]): """set src as the link target of dst""" if self.debug: self.log.dbg('link {} to {}'.format(src, dst)) self.action_executed = False src = os.path.normpath(os.path.join(self.base, os.path.expanduser(src))) if not os.path.exists(src): self.log.err('source dotfile does not exist: {}'.format(src)) return [] dst = os.path.normpath(os.path.expanduser(dst)) if self.totemp: # ignore actions return self.install(templater, src, dst, actions=[]) if Templategen.is_template(src): if self.debug: self.log.dbg('dotfile is a template') self.log.dbg('install to {} and symlink'.format(self.workdir)) tmp = self._pivot_path(dst, self.workdir, striphome=True) i = self.install(templater, src, tmp, actions=actions) if not i and not os.path.exists(tmp): return [] src = tmp return self._link(src, dst, actions=actions) def link_children(self, templater, src, dst, actions=[]): """link all dotfiles in a given directory""" if self.debug: self.log.dbg('link_children {} to {}'.format(src, dst)) self.action_executed = False parent = os.path.join(self.base, os.path.expanduser(src)) # Fail if source doesn't exist if not os.path.exists(parent): self.log.err('source dotfile does not exist: {}'.format(parent)) return [] # Fail if source not a directory if not os.path.isdir(parent): if self.debug: self.log.dbg('symlink children of {} to {}'.format(src, dst)) self.log.err( 'source dotfile is not a directory: {}'.format(parent)) return [] dst = os.path.normpath(os.path.expanduser(dst)) if not os.path.lexists(dst): self.log.sub('creating directory "{}"'.format(dst)) os.makedirs(dst) if os.path.isfile(dst): msg = ''.join([ 'Remove regular file {} and ', 'replace with empty directory?', ]).format(dst) if self.safe and not self.log.ask(msg): msg = 'ignoring "{}", nothing installed' self.log.warn(msg.format(dst)) return [] os.unlink(dst) os.mkdir(dst) children = os.listdir(parent) srcs = [ os.path.normpath(os.path.join(parent, child)) for child in children ] dsts = [ os.path.normpath(os.path.join(dst, child)) for child in children ] for i in range(len(children)): src = srcs[i] dst = dsts[i] if self.debug: self.log.dbg('symlink child {} to {}'.format(src, dst)) if Templategen.is_template(src): if self.debug: self.log.dbg('dotfile is a template') self.log.dbg('install to {} and symlink'.format( self.workdir)) tmp = self._pivot_path(dst, self.workdir, striphome=True) i = self.install(templater, src, tmp, actions=actions) if not i and not os.path.exists(tmp): continue src = tmp result = self._link(src, dst, actions) # Empty actions if dotfile installed # This prevents from running actions multiple times if len(result): actions = [] return (src, dst) def _link(self, src, dst, actions=[]): """set src as a link target of dst""" overwrite = not self.safe if os.path.lexists(dst): if os.path.realpath(dst) == os.path.realpath(src): if self.debug: self.log.dbg('ignoring "{}", link exists'.format(dst)) return [] if self.dry: self.log.dry('would remove {} and link to {}'.format(dst, src)) return [] msg = 'Remove "{}" for link creation?'.format(dst) if self.safe and not self.log.ask(msg): msg = 'ignoring "{}", link was not created' self.log.warn(msg.format(dst)) return [] overwrite = True try: utils.remove(dst) except OSError as e: self.log.err('something went wrong with {}: {}'.format(src, e)) return [] if self.dry: self.log.dry('would link {} to {}'.format(dst, src)) return [] base = os.path.dirname(dst) if not self._create_dirs(base): self.log.err('creating directory for {}'.format(dst)) return [] self._exec_pre_actions(actions) # re-check in case action created the file if os.path.lexists(dst): msg = 'Remove "{}" for link creation?'.format(dst) if self.safe and not overwrite and not self.log.ask(msg): msg = 'ignoring "{}", link was not created' self.log.warn(msg.format(dst)) return [] try: utils.remove(dst) except OSError as e: self.log.err('something went wrong with {}: {}'.format(src, e)) return [] os.symlink(src, dst) self.log.sub('linked {} to {}'.format(dst, src)) return [(src, dst)] def _handle_file(self, templater, src, dst, actions=[], noempty=False): """install src to dst when is a file""" if self.debug: self.log.dbg('generate template for {}'.format(src)) self.log.dbg('ignore empty: {}'.format(noempty)) if utils.samefile(src, dst): # symlink loop self.log.err('dotfile points to itself: {}'.format(dst)) return [] content = templater.generate(src) if noempty and utils.content_empty(content): self.log.warn('ignoring empty template: {}'.format(src)) return [] if content is None: self.log.err('generate from template {}'.format(src)) return [] if not os.path.exists(src): self.log.err('source dotfile does not exist: {}'.format(src)) return [] st = os.stat(src) ret = self._write(src, dst, content, st.st_mode, actions=actions) if ret < 0: self.log.err('installing {} to {}'.format(src, dst)) return [] if ret > 0: if self.debug: self.log.dbg('ignoring {}'.format(dst)) return [] if ret == 0: if not self.dry and not self.comparing: self.log.sub('copied {} to {}'.format(src, dst)) return [(src, dst)] return [] def _handle_dir(self, templater, src, dst, actions=[], noempty=False): """install src to dst when is a directory""" if self.debug: self.log.dbg('install dir {}'.format(src)) self.log.dbg('ignore empty: {}'.format(noempty)) ret = [] if not self._create_dirs(dst): return [] # handle all files in dir for entry in os.listdir(src): f = os.path.join(src, entry) if not os.path.isdir(f): res = self._handle_file(templater, f, os.path.join(dst, entry), actions=actions, noempty=noempty) ret.extend(res) else: res = self._handle_dir(templater, f, os.path.join(dst, entry), actions=actions, noempty=noempty) ret.extend(res) return ret def _fake_diff(self, dst, content): """fake diff by comparing file content with content""" cur = '' with open(dst, 'br') as f: cur = f.read() return cur == content def _write(self, src, dst, content, rights, actions=[]): """write content to file return 0 for success, 1 when already exists -1 when error""" overwrite = not self.safe if self.dry: self.log.dry('would install {}'.format(dst)) return 0 if os.path.lexists(dst): samerights = False try: samerights = os.stat(dst).st_mode == rights except OSError as e: if e.errno == errno.ENOENT: # broken symlink self.log.err('broken symlink {}'.format(dst)) return -1 if self.diff and self._fake_diff(dst, content) and samerights: if self.debug: self.log.dbg('{} is the same'.format(dst)) return 1 if self.safe: if self.debug: self.log.dbg('change detected for {}'.format(dst)) if self.showdiff: self._diff_before_write(src, dst, content) if not self.log.ask('Overwrite \"{}\"'.format(dst)): self.log.warn('ignoring {}'.format(dst)) return 1 overwrite = True if self.backup and os.path.lexists(dst): self._backup(dst) base = os.path.dirname(dst) if not self._create_dirs(base): self.log.err('creating directory for {}'.format(dst)) return -1 if self.debug: self.log.dbg('write content to {}'.format(dst)) self._exec_pre_actions(actions) # re-check in case action created the file if self.safe and not overwrite and os.path.lexists(dst): if not self.log.ask('Overwrite \"{}\"'.format(dst)): self.log.warn('ignoring {}'.format(dst)) return 1 # write the file try: with open(dst, 'wb') as f: f.write(content) except NotADirectoryError as e: self.log.err('opening dest file: {}'.format(e)) return -1 os.chmod(dst, rights) return 0 def _diff_before_write(self, src, dst, src_content): """diff before writing when using --showdiff - not efficient""" # create tmp to diff for templates tmpfile = utils.get_tmpfile() with open(tmpfile, 'wb') as f: f.write(src_content) comp = Comparator(debug=self.debug) diff = comp.compare(tmpfile, dst) # fake the output for readability self.log.log('diff \"{}\" VS \"{}\"'.format(src, dst)) self.log.emph(diff) if tmpfile: utils.remove(tmpfile) def _create_dirs(self, directory): """mkdir -p <directory>""" if not self.create and not os.path.exists(directory): return False if os.path.exists(directory): return True if self.dry: self.log.dry('would mkdir -p {}'.format(directory)) return True if self.debug: self.log.dbg('mkdir -p {}'.format(directory)) os.makedirs(directory) return os.path.exists(directory) def _backup(self, path): """backup file pointed by path""" if self.dry: return dst = path.rstrip(os.sep) + self.backup_suffix self.log.log('backup {} to {}'.format(path, dst)) os.rename(path, dst) def _pivot_path(self, path, newdir, striphome=False): """change path to be under newdir""" if self.debug: self.log.dbg('pivot new dir: \"{}\"'.format(newdir)) self.log.dbg('strip home: {}'.format(striphome)) if striphome: path = utils.strip_home(path) sub = path.lstrip(os.sep) new = os.path.join(newdir, sub) if self.debug: self.log.dbg('pivot \"{}\" to \"{}\"'.format(path, new)) return new def _exec_pre_actions(self, actions): """execute pre-actions if any""" if self.action_executed: return for action in actions: if self.dry: self.log.dry('would execute action: {}'.format(action)) else: if self.debug: self.log.dbg('executing pre action {}'.format(action)) action.execute() self.action_executed = True def _install_to_temp(self, templater, src, dst, tmpdir): """install a dotfile to a tempdir""" tmpdst = self._pivot_path(dst, tmpdir) return self.install(templater, src, tmpdst), tmpdst def install_to_temp(self, templater, tmpdir, src, dst): """install a dotfile to a tempdir""" ret = False tmpdst = '' # save some flags while comparing self.comparing = True drysaved = self.dry self.dry = False diffsaved = self.diff self.diff = False createsaved = self.create self.create = True # normalize src and dst src = os.path.expanduser(src) dst = os.path.expanduser(dst) if self.debug: self.log.dbg('tmp install {} to {}'.format(src, dst)) # install the dotfile to a temp directory for comparing ret, tmpdst = self._install_to_temp(templater, src, dst, tmpdir) if self.debug: self.log.dbg('tmp installed in {}'.format(tmpdst)) # reset flags self.dry = drysaved self.diff = diffsaved self.comparing = False self.create = createsaved return ret, tmpdst