def __maybeDeleteDirectory(self, entrydir, changeset): from os.path import join, exists from vcpx.repository.cvs import CvsEntries if not entrydir: return absentrydir = join(self.repository.basedir, entrydir) if not exists(absentrydir) or CvsEntries(absentrydir).isEmpty(): # Oh, the directory is empty: if there are no other added entries # in the directory, insert a REMOVE event against it. for added in changeset.addedEntries(): if added.name.startswith(entrydir): # entrydir got empty, but only temporarily return False return True return False
def _checkoutUpstreamRevision(self, revision): """ Concretely do the checkout of the upstream sources. Use `revision` as the name of the tag to get, or as a date if it starts with a number. Return the last applied changeset. """ from os.path import join, exists, split from time import sleep from vcpx.repository.cvs import CvsEntries, compare_cvs_revs from vcpx.changes import ChangesetEntry if not self.repository.module: raise InvocationError("Must specify a module name") timestamp = None if revision is not None: # If the revision contains a space, assume it really # specify a branch and a timestamp. If it starts with # a digit, assume it's a timestamp. Otherwise, it must # be a branch name if revision[0] in '0123456789' or revision == 'INITIAL': timestamp = revision revision = None elif ' ' in revision: revision, timestamp = revision.split(' ', 1) csets = self.getPendingChangesets(revision) if not csets: raise TargetInitializationFailure( "Something went wrong: there are no changesets since " "revision '%s'" % revision) if timestamp == 'INITIAL': initialcset = csets.next() timestamp = initialcset.date.replace(tzinfo=None).isoformat( sep=' ') else: initialcset = None if not exists(join(self.repository.basedir, 'CVS')): # CVS does not handle "checkout -d multi/level/subdir", so # split the basedir and use it's parentdir as cwd below. parentdir, subdir = split(self.repository.basedir) cmd = self.repository.command("-f", "-q", "-d", self.repository.repository, "checkout", "-d", subdir) if revision: cmd.extend(["-r", revision]) if timestamp: cmd.extend(["-D", "%s UTC" % timestamp]) if self.repository.freeze_keywords: cmd.append('-kk') checkout = ExternalCommand(cwd=parentdir, command=cmd) retry = 0 while True: checkout.execute(self.repository.module) if checkout.exit_status: retry += 1 if retry > 3: break delay = 2**retry self.log.warning( "%s returned status %s, " "retrying in %d seconds...", str(checkout), checkout.exit_status, delay) sleep(retry) else: break if checkout.exit_status: raise TargetInitializationFailure( "%s returned status %s" % (str(checkout), checkout.exit_status)) else: self.log.info("Using existing %s", self.repository.basedir) if self.repository.tag_entries: self.__forceTagOnEachEntry() entries = CvsEntries(self.repository.basedir) youngest_entry = entries.getYoungestEntry() if youngest_entry is None: raise EmptyRepositoriesFoolsMe("The working copy '%s' of the " "CVS repository seems empty, " "don't know how to deal with " "that." % self.repository.basedir) # loop over the changesets and find the last applied, to find # out the actual cvsps revision found = False def already_applied(cs, entries=entries): "Loop over changeset entries to determine if it's already applied." applied = False # applied become True when an entry is DELETED *and* there is # no metainfo for it: thus, a changeset that removes a few entries # very late in history would be assumed as applied. Prevent that # by checking for at least one explicit match on an existing entry. onepositive = False for m in cs.entries: info = entries.getFileInfo(m.name) # If the entry's info exists, compare the on-disk # version with what we have: the revision is already # applied if the former is greater or equal than the # latter. The same if the info does not exist and it's # a delete event. if info: odversion = info.cvs_version applied = compare_cvs_revs(odversion, m.new_revision) >= 0 # If only one "hunk" is not yet applied, the whole # changeset is new. if not applied: break else: onepositive = True elif m.action_kind == ChangesetEntry.DELETED: applied = True return applied and onepositive # We cannot stop at the first not-applied cset, because it may # old enough to trick already_applied(): an entry may have # been moved in the meantime, and thus the getFileInfo() # method would return None, for example... So we really have # to loop over the whole queue. for cset in self.state_file: applied = already_applied(cset) found = found or applied if applied: last = cset if not found and initialcset: found = already_applied(initialcset) if found: last = initialcset if not found: raise TargetInitializationFailure( "Something went wrong: unable to determine the exact upstream " "revision of the checked out tree in '%s'. Either you specified " "the wrong initial timestamp, or you are checking out a " "composition of 'CVS modules' and Tailor does not support them; " "see the option 'trim-module-components' for a possible " "workaround." % self.repository.basedir) else: self.log.info("Working copy up to revision %s", last.revision) return last
def _applyChangeset(self, changeset): from os.path import join, exists, split from os import listdir from shutil import rmtree from time import sleep from vcpx.repository.cvs import CvsEntries, compare_cvs_revs entries = CvsEntries(self.repository.basedir) # Collect added and deleted directories addeddirs = [] deleteddirs = [] # Group entries at the same revision reventries = {} for e in changeset.entries: if e.action_kind == e.UPDATED: info = entries.getFileInfo(e.name) if not info: self.log.debug('promoting "%s" to ADDED at ' 'revision %s', e.name, e.new_revision) e.action_kind = e.ADDED for dir in self.__createParentCVSDirectories( changeset, e.name): addeddirs.append((e, dir)) elif info.cvs_version == e.new_revision: self.log.debug( 'skipping "%s" since it is already ' 'at revision %s', e.name, e.new_revision) continue elif e.action_kind == e.DELETED: if not exists(join(self.repository.basedir, e.name)): self.log.debug( 'skipping "%s" since it is already ' 'deleted', e.name) entrydir = split(e.name)[0] if self.__maybeDeleteDirectory(entrydir, changeset): deleteddirs.append(entrydir) continue elif e.action_kind == e.ADDED: if e.new_revision is None: # This is a new directory entry, there is no need to update it continue else: for dir in self.__createParentCVSDirectories( changeset, e.name): addeddirs.append((e, dir)) # If this is a directory (CVS does not version directories, # and thus new_revision is always None for them), and it's # going to be deleted, do not execute a 'cvs update', that # in some cases does not do what one would expect. Instead, # remove it with everything it contains (that should be # just a single "CVS" subdir, btw) if e.action_kind == e.DELETED and e.new_revision is None: assert CvsEntries( join(self.repository.basedir, e.name)).isEmpty(), '%s should be empty' % e.name rmtree(join(self.repository.basedir, e.name)) else: names = reventries.setdefault(e.new_revision, []) names.append(e.name) if e.action_kind == e.DELETED: entrydir = split(e.name)[0] if self.__maybeDeleteDirectory(entrydir, changeset): deleteddirs.append(entrydir) revs = reventries.keys() revs.sort(compare_cvs_revs) cmd = self.repository.command("-f", "-d", "%(repository)s", "-q", "update", "-d", "-r", "%(revision)s") if self.repository.freeze_keywords: cmd.append('-kk') cvsup = ExternalCommand(cwd=self.repository.basedir, command=cmd) for rev in revs: names = reventries[rev] retry = 0 while True: cvsup.execute(names, repository=self.repository.repository, revision=rev) if cvsup.exit_status: retry += 1 if retry > 3: break delay = 2**retry self.log.warning( "%s returned status %s, " "retrying in %d seconds...", str(cvsup), cvsup.exit_status, delay) sleep(retry) else: break if cvsup.exit_status: raise ChangesetApplicationFailure( "%s returned status %s" % (str(cvsup), cvsup.exit_status)) self.log.debug("%s updated to %s", ','.join(names), e.new_revision) # Fake up ADD and DEL events for the directories implicitly # added/removed, so that the replayer gets their name. for entry, path in addeddirs: entry = changeset.addEntry(path, None, before=entry) entry.action_kind = entry.ADDED self.log.info("registering new %s directory", entry.name) for path in deleteddirs: deldir = changeset.addEntry(path, None) deldir.action_kind = deldir.DELETED self.log.info("registering %s directory deletion", path) # Make sure all files are present on disk: CVS update does not # create them nor reports an error if the files have been # completely removed from the cvs repository. So loop over the # entries and verify the presence of added/changed ones. for entry in changeset.entries: if (entry.action_kind in (entry.ADDED, entry.UPDATED) and not exists(join(self.repository.basedir, entry.name))): self.log.warning( "Ignoring entry %s, CVS source no " "longer knows about it.", entry.name) changeset.entries.remove(entry)
def _checkoutUpstreamRevision(self, revision): """ Concretely do the checkout of the upstream sources. Use `revision` as the name of the tag to get, or as a date if it starts with a number. Return the last applied changeset. """ from os.path import join, exists, split from time import sleep from vcpx.repository.cvs import CvsEntries, compare_cvs_revs from vcpx.changes import ChangesetEntry if not self.repository.module: raise InvocationError("Must specify a module name") timestamp = None if revision is not None: # If the revision contains a space, assume it really # specify a branch and a timestamp. If it starts with # a digit, assume it's a timestamp. Otherwise, it must # be a branch name if revision[0] in '0123456789' or revision == 'INITIAL': timestamp = revision revision = None elif ' ' in revision: revision, timestamp = revision.split(' ', 1) csets = self.getPendingChangesets(revision) if not csets: raise TargetInitializationFailure( "Something went wrong: there are no changesets since " "revision '%s'" % revision) if timestamp == 'INITIAL': initialcset = csets.next() timestamp = initialcset.date.replace(tzinfo=None).isoformat(sep=' ') else: initialcset = None if not exists(join(self.repository.basedir, 'CVS')): # CVS does not handle "checkout -d multi/level/subdir", so # split the basedir and use it's parentdir as cwd below. parentdir, subdir = split(self.repository.basedir) cmd = self.repository.command("-f", "-q", "-d", self.repository.repository, "checkout", "-d", subdir) if revision: cmd.extend(["-r", revision]) if timestamp: cmd.extend(["-D", "%s UTC" % timestamp]) if self.repository.freeze_keywords: cmd.append('-kk') checkout = ExternalCommand(cwd=parentdir, command=cmd) retry = 0 while True: checkout.execute(self.repository.module) if checkout.exit_status: retry += 1 if retry>3: break delay = 2**retry self.log.warning("%s returned status %s, " "retrying in %d seconds...", str(checkout), checkout.exit_status, delay) sleep(retry) else: break if checkout.exit_status: raise TargetInitializationFailure( "%s returned status %s" % (str(checkout), checkout.exit_status)) else: self.log.info("Using existing %s", self.repository.basedir) if self.repository.tag_entries: self.__forceTagOnEachEntry() entries = CvsEntries(self.repository.basedir) youngest_entry = entries.getYoungestEntry() if youngest_entry is None: raise EmptyRepositoriesFoolsMe("The working copy '%s' of the " "CVS repository seems empty, " "don't know how to deal with " "that." % self.repository.basedir) # loop over the changesets and find the last applied, to find # out the actual cvsps revision found = False def already_applied(cs, entries=entries): "Loop over changeset entries to determine if it's already applied." applied = False # applied become True when an entry is DELETED *and* there is # no metainfo for it: thus, a changeset that removes a few entries # very late in history would be assumed as applied. Prevent that # by checking for at least one explicit match on an existing entry. onepositive = False for m in cs.entries: info = entries.getFileInfo(m.name) # If the entry's info exists, compare the on-disk # version with what we have: the revision is already # applied if the former is greater or equal than the # latter. The same if the info does not exist and it's # a delete event. if info: odversion = info.cvs_version applied = compare_cvs_revs(odversion, m.new_revision) >= 0 # If only one "hunk" is not yet applied, the whole # changeset is new. if not applied: break else: onepositive = True elif m.action_kind == ChangesetEntry.DELETED: applied = True return applied and onepositive # We cannot stop at the first not-applied cset, because it may # old enough to trick already_applied(): an entry may have # been moved in the meantime, and thus the getFileInfo() # method would return None, for example... So we really have # to loop over the whole queue. for cset in self.state_file: applied = already_applied(cset) found = found or applied if applied: last = cset if not found and initialcset: found = already_applied(initialcset) if found: last = initialcset if not found: raise TargetInitializationFailure( "Something went wrong: unable to determine the exact upstream " "revision of the checked out tree in '%s'. Either you specified " "the wrong initial timestamp, or you are checking out a " "composition of 'CVS modules' and Tailor does not support them; " "see the option 'trim-module-components' for a possible " "workaround." % self.repository.basedir) else: self.log.info("Working copy up to revision %s", last.revision) return last
def _applyChangeset(self, changeset): from os.path import join, exists, split from os import listdir from shutil import rmtree from time import sleep from vcpx.repository.cvs import CvsEntries, compare_cvs_revs entries = CvsEntries(self.repository.basedir) # Collect added and deleted directories addeddirs = [] deleteddirs = [] # Group entries at the same revision reventries = {} for e in changeset.entries: if e.action_kind == e.UPDATED: info = entries.getFileInfo(e.name) if not info: self.log.debug('promoting "%s" to ADDED at ' 'revision %s', e.name, e.new_revision) e.action_kind = e.ADDED for dir in self.__createParentCVSDirectories(changeset, e.name): addeddirs.append((e, dir)) elif info.cvs_version == e.new_revision: self.log.debug('skipping "%s" since it is already ' 'at revision %s', e.name, e.new_revision) continue elif e.action_kind == e.DELETED: if not exists(join(self.repository.basedir, e.name)): self.log.debug('skipping "%s" since it is already ' 'deleted', e.name) entrydir = split(e.name)[0] if self.__maybeDeleteDirectory(entrydir, changeset): deleteddirs.append(entrydir) continue elif e.action_kind == e.ADDED: if e.new_revision is None: # This is a new directory entry, there is no need to update it continue else: for dir in self.__createParentCVSDirectories(changeset, e.name): addeddirs.append((e, dir)) # If this is a directory (CVS does not version directories, # and thus new_revision is always None for them), and it's # going to be deleted, do not execute a 'cvs update', that # in some cases does not do what one would expect. Instead, # remove it with everything it contains (that should be # just a single "CVS" subdir, btw) if e.action_kind == e.DELETED and e.new_revision is None: assert CvsEntries(join(self.repository.basedir, e.name)).isEmpty(), '%s should be empty' % e.name rmtree(join(self.repository.basedir, e.name)) else: names = reventries.setdefault(e.new_revision, []) names.append(e.name) if e.action_kind == e.DELETED: entrydir = split(e.name)[0] if self.__maybeDeleteDirectory(entrydir, changeset): deleteddirs.append(entrydir) revs = reventries.keys() revs.sort(compare_cvs_revs) cmd = self.repository.command("-f", "-d", "%(repository)s", "-q", "update", "-d", "-r", "%(revision)s") if self.repository.freeze_keywords: cmd.append('-kk') cvsup = ExternalCommand(cwd=self.repository.basedir, command=cmd) for rev in revs: names = reventries[rev] retry = 0 while True: cvsup.execute(names, repository=self.repository.repository, revision=rev) if cvsup.exit_status: retry += 1 if retry>3: break delay = 2**retry self.log.warning("%s returned status %s, " "retrying in %d seconds...", str(cvsup), cvsup.exit_status, delay) sleep(retry) else: break if cvsup.exit_status: raise ChangesetApplicationFailure( "%s returned status %s" % (str(cvsup), cvsup.exit_status)) self.log.debug("%s updated to %s", ','.join(names), e.new_revision) # Fake up ADD and DEL events for the directories implicitly # added/removed, so that the replayer gets their name. for entry,path in addeddirs: entry = changeset.addEntry(path, None, before=entry) entry.action_kind = entry.ADDED self.log.info("registering new %s directory", entry.name) for path in deleteddirs: deldir = changeset.addEntry(path, None) deldir.action_kind = deldir.DELETED self.log.info("registering %s directory deletion", path) # Make sure all files are present on disk: CVS update does not # create them nor reports an error if the files have been # completely removed from the cvs repository. So loop over the # entries and verify the presence of added/changed ones. for entry in changeset.entries: if (entry.action_kind in (entry.ADDED, entry.UPDATED) and not exists(join(self.repository.basedir, entry.name))): self.log.warning("Ignoring entry %s, CVS source no " "longer knows about it.", entry.name) changeset.entries.remove(entry)