def forget(identifier):
'''
Tells homely to forget about a dotfiles repository that was previously
added. You can then run `homely update` to have homely perform automatic
cleanup of anything that was installed by that dotfiles repo.
REPO
This should be the path to a local dotfiles repository that has already
been registered using `homely add`. You may specify multiple REPOs to
remove at once.
'''
errors = False
for one in identifier:
cfg = RepoListConfig()
info = cfg.find_by_any(one, "ilc")
if not info:
warn("No repos matching %r" % one)
errors = True
continue
# update the config ...
note("Removing record of repo [%s] at %s" %
(info.shortid(), info.localrepo.repo_path))
with saveconfig(RepoListConfig()) as cfg:
cfg.remove_repo(info.repoid)
# if there were errors, then don't try and do an update
if errors:
sys.exit(1)
def cleanup(self, conflicts):
assert conflicts in (self.RAISE, self.WARN, self.POSTPONE, self.ASK)
note("CLEANING UP %d items ..." % (
len(self._old_cleaners) + len(self._created)))
stack = list(self._old_cleaners)
affected = []
while len(stack):
deferred = []
for cleaner in stack:
# TODO: do we still need this complexity?
self._removecleaner(cleaner)
self._tryclean(cleaner, conflicts, affected)
self._savecfg()
assert len(deferred) < len(stack), "Every cleaner wants to go last"
stack = deferred
# all old cleaners should now be finished, or delayed
assert len(self._old_cleaners) == 0
# re-run any helpers that touch the affected files
for path in affected:
for helper in self._helpers:
if helper.affectspath(path) and not helper.isdone():
note("REDO: %s" % helper.description)
helper.makechanges()
# now, clean up the old paths we found
while len(self._old_paths_owned):
before = len(self._old_paths_owned)
for path in list(self._old_paths_owned.keys()):
type_ = self._old_paths_owned[path]
self._trycleanpath(path, type_, conflicts)
if len(self._old_paths_owned) >= before:
raise Exception("All paths want to delay cleaning")
def _remove():
# remove the thing
if type_ == self.TYPE_FOLDER_ONLY:
# TODO: what do we do if the folder isn't empty?
with note("Removing dir %s" % path):
try:
os.rmdir(path)
except OSError as err:
from errno import ENOTEMPTY
if err.errno == ENOTEMPTY:
warn("Directory not empty: {}".format(path))
else:
raise
elif type_ == self.TYPE_FILE_ALL:
note("Removing {}".format(path))
os.unlink(path)
elif type_ == self.TYPE_FILE_PART:
if os.stat(path).st_size == 0:
note("Removing empty {}".format(path))
os.unlink(path)
else:
note("Refusing to remove non-empty {}".format(path))
else:
note("Removing link {}".format(path))
os.unlink(path)
_discard()
def _postpone():
with note("Postponing cleanup of path: {}".format(path)):
self._postponed.add(path)
assert path not in self._new_paths_owned
self._new_paths_owned[path] = type_
self._old_paths_owned.pop(path)
self._savecfg()
def _tryclean(self, cleaner, conflicts, affected):
# if the cleaner is not needed, we get rid of it
# FIXME try/except around the isneeded() call
if not cleaner.isneeded():
note("{}: Not needed".format(cleaner.description))
return
# run the cleaner now
with note("Cleaning: {}".format(cleaner.description)):
# if there are still helpers with claims over things the cleaner
# wants to remove, then the cleaner needs to wait
for claim in cleaner.needsclaims():
if claim in self._claims:
note("Postponed: Something else claimed %r" % claim)
self._addcleaner(cleaner)
return
try:
affected.extend(cleaner.makechanges())
except CleanupObstruction as err:
why = err.args[0]
if conflicts == self.RAISE:
raise
if conflicts == self.POSTPONE:
note("Postponed: %s" % why)
# add the cleaner back in
self._addcleaner(cleaner)
return
# NOTE: eventually we'd like to ask the user what to do, but
# for now we just issue a warning
assert conflicts in (self.WARN, self.ASK)
warn("Aborted: %s" % err.why)
def outputhandler(data, isend, prefix):
# FIXME: if we only get part of a stream, then we have a potential bug
# where we only get part of a multi-byte utf-8 character.
while len(data):
pos = data.find(b"\n")
if pos < 0:
break
# write out the line
note(data[0:pos].decode('utf-8'), dash=prefix)
data = data[pos + 1:]
if isend:
if len(data):
note(data.decode('utf-8'), dash=prefix)
else:
# return any remaining data so it can be included at the start of
# the next run
return data
def outputhandler(data, isend, prefix):
# FIXME: if we only get part of a stream, then we have a potential bug
# where we only get part of a multi-byte utf-8 character.
while len(data):
pos = data.find(b"\n")
if pos < 0:
break
# write out the line
note(data[0:pos].decode('utf-8'), dash=prefix)
data = data[pos+1:]
if isend:
if len(data):
note(data.decode('utf-8'), dash=prefix)
else:
# return any remaining data so it can be included at the start of
# the next run
return data
def _pull(self, origin, local):
# delete every local file except the special ones
for thing in os.listdir(local):
if thing not in (ORIGINFILE, MARKERFILE, DIRTYFILE):
destroyme = os.path.join(local, thing)
if os.path.isdir(destroyme):
note('rmtree %s' % destroyme)
shutil.rmtree(destroyme)
else:
note('rm -f %s' % destroyme)
os.unlink(destroyme)
# now copy stuff across
for thing in os.listdir(origin):
if thing in (ORIGINFILE, DIRTYFILE):
continue
src = os.path.join(origin, thing)
dst = os.path.join(local, thing)
if os.path.isdir(src):
shutil.copytree(src, dst)
else:
shutil.copy2(src, dst)
def _pull(self, origin, local):
# delete every local file except the special ones
for thing in os.listdir(local):
if thing not in (ORIGINFILE, MARKERFILE, DIRTYFILE):
destroyme = os.path.join(local, thing)
if os.path.isdir(destroyme):
note('rmtree %s' % destroyme)
shutil.rmtree(destroyme)
else:
note('rm -f %s' % destroyme)
os.unlink(destroyme)
# now copy stuff across
for thing in os.listdir(origin):
if thing in (ORIGINFILE, DIRTYFILE):
continue
src = os.path.join(origin, thing)
dst = os.path.join(local, thing)
if os.path.isdir(src):
shutil.copytree(src, dst)
else:
shutil.copy2(src, dst)
def execute(cmd, stdout=None, stderr=None, expectexit=0, **kwargs):
# Executes `cmd` in a subprocess. Raises a SystemError if the exit code
# is different to `expecterror`.
#
# The stdout and stderr arguments for the most part work just like
# homely._ui.run(), with the main difference being that when stdout=None or
# stderr=None, these two streams will be filtered through the homely's
# logging functions instead of being sent directly to the python process's
# stdout/stderr. Also, the stderr argument will default to "STDOUT" so that
# the timing of the two streams is recorded more accurately.
#
# If the process absolutely _must_ talk to a TTY, you can use stdout="TTY",
# and a SystemError will be raised if homely is being run in
# non-interactive mode. When using stdout="TTY", you should omit the stderr
# argument.
def outputhandler(data, isend, prefix):
# FIXME: if we only get part of a stream, then we have a potential bug
# where we only get part of a multi-byte utf-8 character.
while len(data):
pos = data.find(b"\n")
if pos < 0:
break
# write out the line
note(data[0:pos].decode('utf-8'), dash=prefix)
data = data[pos + 1:]
if isend:
if len(data):
note(data.decode('utf-8'), dash=prefix)
else:
# return any remaining data so it can be included at the start of
# the next run
return data
if stdout == "TTY":
if not allowinteractive():
raise SystemError("cmd wants interactive mode")
assert stderr is None
stdout = None
else:
if stdout is None:
prefix = "1> " if stderr is False else "&> "
stdout = partial(outputhandler, prefix=prefix)
if stderr is None:
if stdout in (False, True):
stderr = partial(outputhandler, prefix="2> ")
else:
stderr = "STDOUT"
outredir = ' 1> /dev/null' if stdout is False else ''
if stderr is None:
errredir = ' 2>&1'
else:
errredir = ' 2> /dev/null' if stderr is False else ''
with note('{}$ {}{}{}'.format(kwargs.get('cwd', ''),
' '.join(map(shellquote,
cmd)), outredir, errredir)):
returncode, out, err = run(cmd, stdout=stdout, stderr=stderr, **kwargs)
if type(expectexit) is int:
exitok = returncode == expectexit
else:
exitok = returncode in expectexit
if exitok:
return returncode, out, err
# still need to dump the stdout/stderr if they were captured
if out is not None:
outputhandler(out, True, '1> ')
if err is not None:
outputhandler(err, True, '1> ')
message = "Unexpected exit code {}. Expected {}".format(
returncode, expectexit)
warn(message)
raise SystemError(message)
def _discard():
note(" Forgetting about %s %s" % (type_, path))
self._old_paths_owned.pop(path)
self._postponed.discard(path)
self._created.discard(path)
self._savecfg()
def _trycleanpath(self, path, type_, conflicts):
def _discard():
note(" Forgetting about %s %s" % (type_, path))
self._old_paths_owned.pop(path)
self._postponed.discard(path)
self._created.discard(path)
self._savecfg()
def _remove():
# remove the thing
if type_ == self.TYPE_FOLDER_ONLY:
# TODO: what do we do if the folder isn't empty?
with note("Removing dir %s" % path):
try:
os.rmdir(path)
except OSError as err:
from errno import ENOTEMPTY
if err.errno == ENOTEMPTY:
warn("Directory not empty: {}".format(path))
else:
raise
elif type_ == self.TYPE_FILE_ALL:
note("Removing {}".format(path))
os.unlink(path)
elif type_ == self.TYPE_FILE_PART:
if os.stat(path).st_size == 0:
note("Removing empty {}".format(path))
os.unlink(path)
else:
note("Refusing to remove non-empty {}".format(path))
else:
note("Removing link {}".format(path))
os.unlink(path)
_discard()
def _postpone():
with note("Postponing cleanup of path: {}".format(path)):
self._postponed.add(path)
assert path not in self._new_paths_owned
self._new_paths_owned[path] = type_
self._old_paths_owned.pop(path)
self._savecfg()
# if we didn't create the path, then we don't need to clean it up
if path not in self._created:
return _discard()
# if the path no longer exists, we have nothing to do
if not _exists(path):
return _discard()
# if the thing has the wrong type, we'll issue an note() and just skip
if type_ in (self.TYPE_FILE_PART, self.TYPE_FILE_ALL):
correcttype = os.path.isfile(path)
elif type_ in (self.TYPE_FOLDER_ONLY, self.TYPE_FOLDER_ALL):
correcttype = os.path.isdir(path)
else:
assert type_ == self.TYPE_LINK
correcttype = os.path.islink(path)
if not correcttype:
with note("Ignoring: Won't remove {} as it is no longer a {}"
.format(path, type_)):
return _discard()
# work out if there is another path we need to remove first
for otherpath in self._old_paths_owned:
if otherpath != path and isnecessarypath(path, otherpath):
# If there's another path we need to do first, then don't do
# anything just yet. NOTE: there is an assertion to ensure that
# we can't get stuck in an infinite loop repeatedly not
# removing things.
return
# if any helpers want the path, don't delete it
wantedby = None
for c in self._new_cleaners:
if c.wantspath(path):
wantedby = c
break
if not wantedby:
for otherpath in self._new_paths_owned:
if isnecessarypath(path, otherpath):
wantedby = otherpath
if wantedby:
# if we previously postponed this path, keep postponing it
# while there are still things hanging around that want it
if path in self._postponed:
return _postpone()
if conflicts == self.ASK:
raise Exception("TODO: ask the user what to do") # noqa
# NOTE: options are:
# A) postpone until it can run later
# B) discard it
if conflicts == self.RAISE:
raise CleanupConflict(conflictpath=path, pathwanter=wantedby)
if conflicts == self.POSTPONE:
return _postpone()
assert conflicts == self.WARN
warn("Couldn't clean up path {}; it is still needed for {}"
.format(path, wantedby))
return _discard()
# if nothing else wants this path, clean it up now
return _remove()
def run(self, helper):
assert isinstance(helper, Helper)
cfg_modified = False
# what claims does this helper make?
self._claims.update(*helper.getclaims())
# take ownership of paths
for path, type_ in helper.pathsownable().items():
cfg_modified = True
self._new_paths_owned[path] = type_
self._old_paths_owned.pop(path, None)
# get a cleaner for this helper
cleaner = helper.getcleaner()
if helper.isdone():
# if there is already a cleaner for this thing, add and remove it
# so it hangs around. If there is no cleaner but the thing is
# already done, it means we shouldn't be cleaning it up
if cleaner is not None:
cfg_modified = True
if self._removecleaner(cleaner):
self._addcleaner(cleaner)
note("{}: Already done".format(helper.description))
else:
# remove and add the cleaner so that we know it will try to clean
# up, since we know we will be making the change
if cleaner is not None:
cfg_modified = True
self._removecleaner(cleaner)
self._addcleaner(cleaner)
# if the helper isn't already done, tell it to do its thing now
with note("{}: Running ...".format(helper.description)):
# take ownership of any paths that don't exist yet!
for path, type_ in helper.pathsownable().items():
if type_ in (self.TYPE_FILE_ALL, self.TYPE_FOLDER_ALL):
exists = path in self._created
elif type_ in (self.TYPE_FILE_PART, self.TYPE_FOLDER_ONLY):
exists = os.path.exists(path)
else:
exists = os.path.islink(path)
if not exists:
self._created.add(path)
cfg_modified = True
if cfg_modified:
# save the updated config before we try anything
self._savecfg()
cfg_modified = False
try:
helper.makechanges()
except HelperError as err:
warn("Failed: %s" % err.args[0])
self._helpers.append(helper)
# save the config now if we were successful
if cfg_modified:
# save the updated config before we try anything
self._savecfg()
def execute(cmd, stdout=None, stderr=None, expectexit=0, **kwargs):
# Executes `cmd` in a subprocess. Raises a SystemError if the exit code
# is different to `expecterror`.
#
# The stdout and stderr arguments for the most part work just like
# homely._ui.run(), with the main difference being that when stdout=None or
# stderr=None, these two streams will be filtered through the homely's
# logging functions instead of being sent directly to the python process's
# stdout/stderr. Also, the stderr argument will default to "STDOUT" so that
# the timing of the two streams is recorded more accurately.
#
# If the process absolutely _must_ talk to a TTY, you can use stdout="TTY",
# and a SystemError will be raised if homely is being run in
# non-interactive mode. When using stdout="TTY", you should omit the stderr
# argument.
def outputhandler(data, isend, prefix):
# FIXME: if we only get part of a stream, then we have a potential bug
# where we only get part of a multi-byte utf-8 character.
while len(data):
pos = data.find(b"\n")
if pos < 0:
break
# write out the line
note(data[0:pos].decode('utf-8'), dash=prefix)
data = data[pos+1:]
if isend:
if len(data):
note(data.decode('utf-8'), dash=prefix)
else:
# return any remaining data so it can be included at the start of
# the next run
return data
if stdout == "TTY":
if not allowinteractive():
raise SystemError("cmd wants interactive mode")
assert stderr is None
stdout = None
else:
if stdout is None:
prefix = "1> " if stderr is False else "&> "
stdout = partial(outputhandler, prefix=prefix)
if stderr is None:
if stdout in (False, True):
stderr = partial(outputhandler, prefix="2> ")
else:
stderr = "STDOUT"
outredir = ' 1> /dev/null' if stdout is False else ''
if stderr is None:
errredir = ' 2>&1'
else:
errredir = ' 2> /dev/null' if stderr is False else ''
with note('{}$ {}{}{}'.format(kwargs.get('cwd', ''),
' '.join(map(shellquote, cmd)),
outredir,
errredir)):
returncode, out, err = run(cmd, stdout=stdout, stderr=stderr, **kwargs)
if type(expectexit) is int:
exitok = returncode == expectexit
else:
exitok = returncode in expectexit
if exitok:
return returncode, out, err
# still need to dump the stdout/stderr if they were captured
if out is not None:
outputhandler(out, True, '1> ')
if err is not None:
outputhandler(err, True, '1> ')
message = "Unexpected exit code {}. Expected {}".format(
returncode, expectexit)
warn(message)
raise SystemError(message)
def makechanges(self):
assert self._source_repo is not None
assert self._clone_to is not None
if not os.path.exists(self._real_clone_to):
note("Cloning %s" % self._source_repo)
pull_needed = False
execute(['git', 'clone', self._source_repo, self._real_clone_to])
else:
pull_needed = True
if not os.path.exists(os.path.join(self._real_clone_to, '.git')):
raise HelperError("%s is not a git repo" % self._real_clone_to)
# do we want a particular branch?
if self._branch:
execute(['git', 'checkout', self._branch], cwd=self._real_clone_to)
if pull_needed and allowpull():
note("Updating %s from %s" %
(self._clone_to, self._source_repo))
execute(['git', 'pull'], cwd=self._real_clone_to)
# check the branch fact to see if we need to compile again
factname = self._branchfact
else:
assert self._tag is not None
if pull_needed and allowpull():
note("Updating %s from %s" %
(self._clone_to, self._source_repo))
execute(['git', 'fetch', '--tags'], cwd=self._real_clone_to)
execute(['git', 'checkout', self._tag], cwd=self._real_clone_to)
# if we used a tag name, create a 'fact' to prevent us re-compiling
# each time we run
factname = '{}:compile-tag:{}:{}'.format(self.__class__.__name__,
self._real_clone_to,
self._tag)
docompile = False
if self._compile:
last_compile, prev_cmds = self._getfact(factname, (0, None))
what = ("Branch {}".format(self._branch)
if self._branch else "Tag {}".format(self._tag))
if last_compile == 0:
note("{} has never been compiled".format(what))
docompile = True
elif (self._expiry is not None
and ((last_compile + self._expiry) < time.time())):
note("{} is due to be compiled again".format(what))
docompile = True
elif prev_cmds != self._compile:
note("{} needs to be compiled again with new commands".format(
what))
docompile = True
# run any compilation commands
if docompile:
# FIXME: we probably need to delete all the symlink targets before
# compiling, as this is our best way of determining that the
# compilation has failed ...
stdout = "TTY" if self._needs_tty else None
for cmd in self._compile:
execute(cmd, cwd=self._real_clone_to, stdout=stdout)
self._setfact(factname, (time.time(), self._compile))
# create new symlinks
for source, dest in self._symlinks:
with note("Ensure symlink exists: %s -> %s" % (source, dest)):
if os.path.islink(dest):
target = os.readlink(dest)
if os.path.realpath(target) != os.path.realpath(source):
raise HelperError("Symlink %s is not pointing at %s" %
(dest, source))
continue
if os.path.exists(dest):
raise HelperError("%s already exists" % dest)
os.symlink(source, dest)
def makechanges(self):
assert self._source_repo is not None
assert self._clone_to is not None
if not os.path.exists(self._real_clone_to):
note("Cloning %s" % self._source_repo)
pull_needed = False
execute(['git', 'clone', self._source_repo, self._real_clone_to])
else:
pull_needed = True
if not os.path.exists(os.path.join(self._real_clone_to, '.git')):
raise HelperError("%s is not a git repo" % self._real_clone_to)
# do we want a particular branch?
if self._branch:
execute(['git', 'checkout', self._branch], cwd=self._real_clone_to)
if pull_needed and allowpull():
note("Updating %s from %s" %
(self._clone_to, self._source_repo))
execute(['git', 'pull'], cwd=self._real_clone_to)
# check the branch fact to see if we need to compile again
factname = self._branchfact
else:
assert self._tag is not None
if pull_needed and allowpull():
note("Updating %s from %s" %
(self._clone_to, self._source_repo))
# NOTE: we use --force for projects like neovim that have a
# rolling 'nightly' tag
execute(['git', 'fetch', '--tags', '--force'],
cwd=self._real_clone_to)
execute(['git', 'checkout', self._tag], cwd=self._real_clone_to)
# if we used a tag name, create a 'fact' to prevent us re-compiling
# each time we run
factname = '{}:compile-tag:{}:{}'.format(
self.__class__.__name__,
self._real_clone_to,
self._tag)
docompile = False
if self._compile:
last_compile, prev_cmds = self._getfact(factname, (0, None))
what = ("Branch {}".format(self._branch) if self._branch
else "Tag {}".format(self._tag))
if last_compile == 0:
note("{} has never been compiled".format(what))
docompile = True
elif (self._expiry is not None
and ((last_compile + self._expiry) < time.time())):
note("{} is due to be compiled again".format(what))
docompile = True
elif prev_cmds != self._compile:
note("{} needs to be compiled again with new commands"
.format(what))
docompile = True
# run any compilation commands
if docompile:
# FIXME: we probably need to delete all the symlink targets before
# compiling, as this is our best way of determining that the
# compilation has failed ...
stdout = "TTY" if self._needs_tty else None
for cmd in self._compile:
if cmd[0] == "sudo" and not _ALLOW_INSTALL:
raise HelperError(
"%s is not allowed to run commands as root"
", as per setallowinstall()")
execute(cmd, cwd=self._real_clone_to, stdout=stdout)
self._setfact(factname, (time.time(), self._compile))
# create new symlinks
for source, dest in self._symlinks:
with note("Ensure symlink exists: %s -> %s" % (source, dest)):
if os.path.islink(dest):
target = os.readlink(dest)
if os.path.realpath(target) != os.path.realpath(source):
raise HelperError("Symlink %s is not pointing at %s" %
(dest, source))
continue
if os.path.exists(dest):
raise HelperError("%s already exists" % dest)
os.symlink(source, dest)