class Updater: def __init__(self, dotpath, variables, dotfile_key_getter, dotfile_dst_getter, dotfile_path_normalizer, dry=False, safe=True, debug=False, ignore=[], showpatch=False): """constructor @dotpath: path where dotfiles are stored @variables: dictionary of variables for the templates @dotfile_key_getter: func to get a dotfile by key @dotfile_dst_getter: func to get a dotfile by dst @dotfile_path_normalizer: func to normalize dotfile dst @dry: simulate @safe: ask for overwrite if True @debug: enable debug @ignore: pattern to ignore when updating @showpatch: show patch if dotfile to update is a template """ self.dotpath = dotpath self.variables = variables self.dotfile_key_getter = dotfile_key_getter self.dotfile_dst_getter = dotfile_dst_getter self.dotfile_path_normalizer = dotfile_path_normalizer self.dry = dry self.safe = safe self.debug = debug self.ignore = ignore self.showpatch = showpatch self.templater = Templategen(variables=self.variables, base=self.dotpath, debug=self.debug) # save template vars self.tvars = self.templater.add_tmp_vars() self.log = Logger() def update_path(self, path): """update the dotfile installed on path""" path = os.path.expanduser(path) if not os.path.lexists(path): self.log.err('\"{}\" does not exist!'.format(path)) return False dotfile = self.dotfile_dst_getter(path) if not dotfile: return False if self.debug: self.log.dbg('updating {} from path \"{}\"'.format(dotfile, path)) return self._update(path, dotfile) def update_key(self, key): """update the dotfile referenced by key""" dotfile = self.dotfile_key_getter(key) if not dotfile: return False if self.debug: self.log.dbg('updating {} from key \"{}\"'.format(dotfile, key)) path = self.dotfile_path_normalizer(dotfile.dst) return self._update(path, dotfile) def _update(self, path, dotfile): """update dotfile from file pointed by path""" ret = False new_path = None ignores = list(set(self.ignore + dotfile.upignore)) self.ignores = patch_ignores(ignores, dotfile.dst, debug=self.debug) if self.debug: self.log.dbg('ignore pattern(s): {}'.format(self.ignores)) path = os.path.expanduser(path) dtpath = os.path.join(self.dotpath, dotfile.src) dtpath = os.path.expanduser(dtpath) if self._ignore([path, dtpath]): self.log.sub('\"{}\" ignored'.format(dotfile.key)) return True # apply write transformation if any new_path = self._apply_trans_w(path, dotfile) if not new_path: return False if os.path.isdir(new_path): ret = self._handle_dir(new_path, dtpath) else: ret = self._handle_file(new_path, dtpath) # clean temporary files if new_path != path and os.path.exists(new_path): remove(new_path) return ret def _apply_trans_w(self, path, dotfile): """apply write transformation to dotfile""" trans = dotfile.get_trans_w() if not trans: return path if self.debug: self.log.dbg('executing write transformation {}'.format(trans)) tmp = get_unique_tmp_name() self.templater.restore_vars(self.tvars) newvars = dotfile.get_dotfile_variables() self.templater.add_tmp_vars(newvars=newvars) if not trans.transform( path, tmp, templater=self.templater, debug=self.debug): msg = 'transformation \"{}\" failed for {}' self.log.err(msg.format(trans.key, dotfile.key)) if os.path.exists(tmp): remove(tmp) return None return tmp def _is_template(self, path): if not Templategen.is_template(path): if self.debug: self.log.dbg('{} is NO template'.format(path)) return False self.log.warn('{} uses template, update manually'.format(path)) return True def _show_patch(self, fpath, tpath): """provide a way to manually patch the template""" content = self._resolve_template(tpath) tmp = write_to_tmpfile(content) cmds = ['diff', '-u', tmp, fpath, '|', 'patch', tpath] self.log.warn('try patching with: \"{}\"'.format(' '.join(cmds))) return False def _resolve_template(self, tpath): """resolve the template to a temporary file""" self.templater.restore_vars(self.tvars) return self.templater.generate(tpath) def _handle_file(self, path, dtpath, compare=True): """sync path (deployed file) and dtpath (dotdrop dotfile path)""" if self._ignore([path, dtpath]): self.log.sub('\"{}\" ignored'.format(dtpath)) return True if self.debug: self.log.dbg('update for file {} and {}'.format(path, dtpath)) if self._is_template(dtpath): # dotfile is a template if self.debug: self.log.dbg('{} is a template'.format(dtpath)) if self.showpatch: self._show_patch(path, dtpath) return False if compare and filecmp.cmp(path, dtpath, shallow=True): # no difference if self.debug: self.log.dbg('identical files: {} and {}'.format(path, dtpath)) return True if not self._overwrite(path, dtpath): return False try: if self.dry: self.log.dry('would cp {} {}'.format(path, dtpath)) else: if self.debug: self.log.dbg('cp {} {}'.format(path, dtpath)) shutil.copyfile(path, dtpath) self.log.sub('\"{}\" updated'.format(dtpath)) except IOError as e: self.log.warn('{} update failed, do manually: {}'.format(path, e)) return False return True def _handle_dir(self, path, dtpath): """sync path (deployed dir) and dtpath (dotdrop dir path)""" if self.debug: self.log.dbg('handle update for dir {} to {}'.format(path, dtpath)) # paths must be absolute (no tildes) path = os.path.expanduser(path) dtpath = os.path.expanduser(dtpath) if self._ignore([path, dtpath]): self.log.sub('\"{}\" ignored'.format(dtpath)) return True # find the differences diff = filecmp.dircmp(path, dtpath, ignore=None) # handle directories diff return self._merge_dirs(diff) def _merge_dirs(self, diff): """Synchronize directories recursively.""" left, right = diff.left, diff.right if self.debug: self.log.dbg('sync dir {} to {}'.format(left, right)) if self._ignore([left, right]): return True # create dirs that don't exist in dotdrop for toadd in diff.left_only: exist = os.path.join(left, toadd) if not os.path.isdir(exist): # ignore files for now continue # match to dotdrop dotpath new = os.path.join(right, toadd) if self._ignore([exist, new]): self.log.sub('\"{}\" ignored'.format(exist)) continue if self.dry: self.log.dry('would cp -r {} {}'.format(exist, new)) continue if self.debug: self.log.dbg('cp -r {} {}'.format(exist, new)) # Newly created directory should be copied as is (for efficiency). shutil.copytree(exist, new) self.log.sub('\"{}\" dir added'.format(new)) # remove dirs that don't exist in deployed version for toremove in diff.right_only: old = os.path.join(right, toremove) if not os.path.isdir(old): # ignore files for now continue if self._ignore([old]): continue if self.dry: self.log.dry('would rm -r {}'.format(old)) continue if self.debug: self.log.dbg('rm -r {}'.format(old)) if not self._confirm_rm_r(old): continue remove(old) self.log.sub('\"{}\" dir removed'.format(old)) # handle files diff # sync files that exist in both but are different fdiff = diff.diff_files fdiff.extend(diff.funny_files) fdiff.extend(diff.common_funny) for f in fdiff: fleft = os.path.join(left, f) fright = os.path.join(right, f) if self._ignore([fleft, fright]): continue if self.dry: self.log.dry('would cp {} {}'.format(fleft, fright)) continue if self.debug: self.log.dbg('cp {} {}'.format(fleft, fright)) self._handle_file(fleft, fright, compare=False) # copy files that don't exist in dotdrop for toadd in diff.left_only: exist = os.path.join(left, toadd) if os.path.isdir(exist): # ignore dirs, done above continue new = os.path.join(right, toadd) if self._ignore([exist, new]): continue if self.dry: self.log.dry('would cp {} {}'.format(exist, new)) continue if self.debug: self.log.dbg('cp {} {}'.format(exist, new)) shutil.copyfile(exist, new) self.log.sub('\"{}\" added'.format(new)) # remove files that don't exist in deployed version for toremove in diff.right_only: new = os.path.join(right, toremove) if not os.path.exists(new): continue if os.path.isdir(new): # ignore dirs, done above continue if self._ignore([new]): continue if self.dry: self.log.dry('would rm {}'.format(new)) continue if self.debug: self.log.dbg('rm {}'.format(new)) remove(new) self.log.sub('\"{}\" removed'.format(new)) # Recursively decent into common subdirectories. for subdir in diff.subdirs.values(): self._merge_dirs(subdir) # Nothing more to do here. return True def _overwrite(self, src, dst): """ask for overwritting""" msg = 'Overwrite \"{}\" with \"{}\"?'.format(dst, src) if self.safe and not self.log.ask(msg): return False return True def _confirm_rm_r(self, directory): """ask for rm -r directory""" msg = 'Recursively remove \"{}\"?'.format(directory) if self.safe and not self.log.ask(msg): return False return True def _ignore(self, paths): if must_ignore(paths, self.ignores, debug=self.debug): if self.debug: self.log.dbg('ignoring update for {}'.format(paths)) return True return False
def _resolve_template(self, tpath): """resolve the template to a temporary file""" t = Templategen(variables=self.variables, base=self.dotpath, debug=self.debug) return t.generate(tpath)
class Updater: def __init__(self, dotpath, variables, conf, dry=False, safe=True, debug=False, ignore=[], showpatch=False, ignore_missing_in_dotdrop=False): """constructor @dotpath: path where dotfiles are stored @variables: dictionary of variables for the templates @conf: configuration manager @dry: simulate @safe: ask for overwrite if True @debug: enable debug @ignore: pattern to ignore when updating @showpatch: show patch if dotfile to update is a template """ self.dotpath = dotpath self.variables = variables self.conf = conf self.dry = dry self.safe = safe self.debug = debug self.ignore = ignore self.ignores = None self.showpatch = showpatch self.ignore_missing_in_dotdrop = ignore_missing_in_dotdrop self.templater = Templategen(variables=self.variables, base=self.dotpath, debug=self.debug) # save template vars self.tvars = self.templater.add_tmp_vars() self.log = Logger() def update_path(self, path): """update the dotfile installed on path""" path = os.path.expanduser(path) if not os.path.lexists(path): self.log.err('\"{}\" does not exist!'.format(path)) return False dotfiles = self.conf.get_dotfile_by_dst(path) if not dotfiles: return False for dotfile in dotfiles: if not dotfile: msg = 'invalid dotfile for update: {}' self.log.err(msg.format(dotfile.key)) return False if self.debug: msg = 'updating {} from path \"{}\"' self.log.dbg(msg.format(dotfile, path)) if not self._update(path, dotfile): return False return True def update_key(self, key): """update the dotfile referenced by key""" dotfile = self.conf.get_dotfile(key) if not dotfile: return False if self.debug: self.log.dbg('updating {} from key \"{}\"'.format(dotfile, key)) path = self.conf.path_to_dotfile_dst(dotfile.dst) return self._update(path, dotfile) def _update(self, path, dotfile): """update dotfile from file pointed by path""" ret = False new_path = None ignores = list(set(self.ignore + dotfile.upignore)) self.ignores = patch_ignores(ignores, dotfile.dst, debug=self.debug) if self.debug: self.log.dbg('ignore pattern(s): {}'.format(self.ignores)) deployed_path = os.path.expanduser(path) local_path = os.path.join(self.dotpath, dotfile.src) local_path = os.path.expanduser(local_path) if not os.path.exists(deployed_path): msg = '\"{}\" does not exist' self.log.err(msg.format(deployed_path)) return False if not os.path.exists(local_path): msg = '\"{}\" does not exist, import it first' self.log.err(msg.format(local_path)) return False ignore_missing_in_dotdrop = self.ignore_missing_in_dotdrop or \ dotfile.ignore_missing_in_dotdrop if (ignore_missing_in_dotdrop and not os.path.exists(local_path)) or \ self._ignore([deployed_path, local_path]): self.log.sub('\"{}\" ignored'.format(dotfile.key)) return True # apply write transformation if any new_path = self._apply_trans_w(deployed_path, dotfile) if not new_path: return False # save current rights deployed_mode = get_file_perm(deployed_path) local_mode = get_file_perm(local_path) # handle the pointed file if os.path.isdir(new_path): ret = self._handle_dir(new_path, local_path, dotfile) else: ret = self._handle_file(new_path, local_path, dotfile) if deployed_mode != local_mode: # mirror rights if self.debug: m = 'adopt mode {:o} for {}' self.log.dbg(m.format(deployed_mode, dotfile.key)) r = self.conf.update_dotfile(dotfile.key, deployed_mode) if r: ret = True # clean temporary files if new_path != deployed_path and os.path.exists(new_path): removepath(new_path, logger=self.log) return ret def _apply_trans_w(self, path, dotfile): """apply write transformation to dotfile""" trans = dotfile.get_trans_w() if not trans: return path if self.debug: self.log.dbg('executing write transformation {}'.format(trans)) tmp = get_unique_tmp_name() self.templater.restore_vars(self.tvars) newvars = dotfile.get_dotfile_variables() self.templater.add_tmp_vars(newvars=newvars) if not trans.transform( path, tmp, templater=self.templater, debug=self.debug): msg = 'transformation \"{}\" failed for {}' self.log.err(msg.format(trans.key, dotfile.key)) if os.path.exists(tmp): removepath(tmp, logger=self.log) return None return tmp def _is_template(self, path): if not Templategen.is_template(path, ignore=self.ignores): if self.debug: self.log.dbg('{} is NO template'.format(path)) return False self.log.warn('{} uses template, update manually'.format(path)) return True def _show_patch(self, fpath, tpath): """provide a way to manually patch the template""" content = self._resolve_template(tpath) tmp = write_to_tmpfile(content) mirror_file_rights(tpath, tmp) cmds = ['diff', '-u', tmp, fpath, '|', 'patch', tpath] self.log.warn('try patching with: \"{}\"'.format(' '.join(cmds))) return False def _resolve_template(self, tpath): """resolve the template to a temporary file""" self.templater.restore_vars(self.tvars) return self.templater.generate(tpath) def _same_rights(self, left, right): """return True if files have the same modes""" try: lefts = get_file_perm(left) rights = get_file_perm(right) return lefts == rights except OSError as e: self.log.err(e) return False def _mirror_rights(self, src, dst): srcr = get_file_perm(src) dstr = get_file_perm(dst) if srcr == dstr: return if self.debug: msg = 'copy rights from {} ({:o}) to {} ({:o})' self.log.dbg(msg.format(src, srcr, dst, dstr)) try: mirror_file_rights(src, dst) except OSError as e: self.log.err(e) def _handle_file(self, deployed_path, local_path, dotfile, compare=True): """sync path (deployed file) and local_path (dotdrop dotfile path)""" if self._ignore([deployed_path, local_path]): self.log.sub('\"{}\" ignored'.format(local_path)) return True if self.debug: self.log.dbg('update for file {} and {}'.format( deployed_path, local_path, )) if self._is_template(local_path): # dotfile is a template if self.debug: self.log.dbg('{} is a template'.format(local_path)) if self.showpatch: try: self._show_patch(deployed_path, local_path) except UndefinedException as e: msg = 'unable to show patch for {}: {}'.format( deployed_path, e, ) self.log.warn(msg) return False if compare and \ filecmp.cmp(deployed_path, local_path, shallow=False) and \ self._same_rights(deployed_path, local_path): # no difference if self.debug: self.log.dbg('identical files: {} and {}'.format( deployed_path, local_path, )) return True if not self._overwrite(deployed_path, local_path): return False try: if self.dry: self.log.dry('would cp {} {}'.format( deployed_path, local_path, )) else: if self.debug: self.log.dbg('cp {} {}'.format(deployed_path, local_path)) shutil.copyfile(deployed_path, local_path) self._mirror_rights(deployed_path, local_path) self.log.sub('\"{}\" updated'.format(local_path)) except IOError as e: self.log.warn('{} update failed, do manually: {}'.format( deployed_path, e)) return False return True def _handle_dir(self, deployed_path, local_path, dotfile): """sync path (local dir) and local_path (dotdrop dir path)""" if self.debug: self.log.dbg('handle update for dir {} to {}'.format( deployed_path, local_path, )) # paths must be absolute (no tildes) deployed_path = os.path.expanduser(deployed_path) local_path = os.path.expanduser(local_path) if self._ignore([deployed_path, local_path]): self.log.sub('\"{}\" ignored'.format(local_path)) return True # find the differences diff = filecmp.dircmp(deployed_path, local_path, ignore=None) # handle directories diff ret = self._merge_dirs(diff, dotfile) self._mirror_rights(deployed_path, local_path) return ret def _merge_dirs(self, diff, dotfile): """Synchronize directories recursively.""" left, right = diff.left, diff.right if self.debug: self.log.dbg('sync dir {} to {}'.format(left, right)) if self._ignore([left, right]): return True ignore_missing_in_dotdrop = self.ignore_missing_in_dotdrop or \ dotfile.ignore_missing_in_dotdrop # create dirs that don't exist in dotdrop for toadd in diff.left_only: exist = os.path.join(left, toadd) if not os.path.isdir(exist): # ignore files for now continue # match to dotdrop dotpath new = os.path.join(right, toadd) if (ignore_missing_in_dotdrop and not os.path.exists(new)) or \ self._ignore([exist, new]): self.log.sub('\"{}\" ignored'.format(exist)) continue if self.dry: self.log.dry('would cp -r {} {}'.format(exist, new)) continue if self.debug: self.log.dbg('cp -r {} {}'.format(exist, new)) # Newly created directory should be copied as is (for efficiency). def ig(src, names): whitelist, blacklist = set(), set() for ignore in self.ignores: for name in names: path = os.path.join(src, name) if ignore.startswith('!') and \ fnmatch.fnmatch(path, ignore[1:]): # add to whitelist whitelist.add(name) elif fnmatch.fnmatch(path, ignore): # add to blacklist blacklist.add(name) return blacklist - whitelist shutil.copytree(exist, new, ignore=ig) self.log.sub('\"{}\" dir added'.format(new)) # remove dirs that don't exist in deployed version for toremove in diff.right_only: old = os.path.join(right, toremove) if not os.path.isdir(old): # ignore files for now continue if self._ignore([old]): continue if self.dry: self.log.dry('would rm -r {}'.format(old)) continue if self.debug: self.log.dbg('rm -r {}'.format(old)) if not self._confirm_rm_r(old): continue removepath(old, logger=self.log) self.log.sub('\"{}\" dir removed'.format(old)) # handle files diff # sync files that exist in both but are different fdiff = diff.diff_files fdiff.extend(diff.funny_files) fdiff.extend(diff.common_funny) for f in fdiff: fleft = os.path.join(left, f) fright = os.path.join(right, f) if (ignore_missing_in_dotdrop and not os.path.exists(fright)) or \ self._ignore([fleft, fright]): continue if self.dry: self.log.dry('would cp {} {}'.format(fleft, fright)) continue if self.debug: self.log.dbg('cp {} {}'.format(fleft, fright)) self._handle_file(fleft, fright, dotfile, compare=False) # copy files that don't exist in dotdrop for toadd in diff.left_only: exist = os.path.join(left, toadd) if os.path.isdir(exist): # ignore dirs, done above continue new = os.path.join(right, toadd) if (ignore_missing_in_dotdrop and not os.path.exists(new)) or \ self._ignore([exist, new]): continue if self.dry: self.log.dry('would cp {} {}'.format(exist, new)) continue if self.debug: self.log.dbg('cp {} {}'.format(exist, new)) shutil.copyfile(exist, new) self._mirror_rights(exist, new) self.log.sub('\"{}\" added'.format(new)) # remove files that don't exist in deployed version for toremove in diff.right_only: new = os.path.join(right, toremove) if not os.path.exists(new): continue if os.path.isdir(new): # ignore dirs, done above continue if self._ignore([new]): continue if self.dry: self.log.dry('would rm {}'.format(new)) continue if self.debug: self.log.dbg('rm {}'.format(new)) removepath(new, logger=self.log) self.log.sub('\"{}\" removed'.format(new)) # compare rights for common in diff.common_files: leftf = os.path.join(left, common) rightf = os.path.join(right, common) if not self._same_rights(leftf, rightf): self._mirror_rights(leftf, rightf) # Recursively decent into common subdirectories. for subdir in diff.subdirs.values(): self._merge_dirs(subdir, dotfile) # Nothing more to do here. return True def _overwrite(self, src, dst): """ask for overwritting""" msg = 'Overwrite \"{}\" with \"{}\"?'.format(dst, src) if self.safe and not self.log.ask(msg): return False return True def _confirm_rm_r(self, directory): """ask for rm -r directory""" msg = 'Recursively remove \"{}\"?'.format(directory) if self.safe and not self.log.ask(msg): return False return True def _ignore(self, paths): if must_ignore(paths, self.ignores, debug=self.debug): if self.debug: self.log.dbg('ignoring update for {}'.format(paths)) return True return False