Beispiel #1
0
    def __maybeDeleteDirectory(self, entrydir, changeset):
        from os.path import join, exists
        from vcpx.repository.cvs import CvsEntries

        if not entrydir:
            return

        absentrydir = join(self.repository.basedir, entrydir)
        if not exists(absentrydir) or CvsEntries(absentrydir).isEmpty():
            # Oh, the directory is empty: if there are no other added entries
            # in the directory, insert a REMOVE event against it.
            for added in changeset.addedEntries():
                if added.name.startswith(entrydir):
                    # entrydir got empty, but only temporarily
                    return False
            return True
        return False
Beispiel #2
0
    def _checkoutUpstreamRevision(self, revision):
        """
        Concretely do the checkout of the upstream sources. Use
        `revision` as the name of the tag to get, or as a date if it
        starts with a number.

        Return the last applied changeset.
        """

        from os.path import join, exists, split
        from time import sleep
        from vcpx.repository.cvs import CvsEntries, compare_cvs_revs
        from vcpx.changes import ChangesetEntry

        if not self.repository.module:
            raise InvocationError("Must specify a module name")

        timestamp = None
        if revision is not None:
            # If the revision contains a space, assume it really
            # specify a branch and a timestamp. If it starts with
            # a digit, assume it's a timestamp. Otherwise, it must
            # be a branch name
            if revision[0] in '0123456789' or revision == 'INITIAL':
                timestamp = revision
                revision = None
            elif ' ' in revision:
                revision, timestamp = revision.split(' ', 1)

        csets = self.getPendingChangesets(revision)
        if not csets:
            raise TargetInitializationFailure(
                "Something went wrong: there are no changesets since "
                "revision '%s'" % revision)
        if timestamp == 'INITIAL':
            initialcset = csets.next()
            timestamp = initialcset.date.replace(tzinfo=None).isoformat(
                sep=' ')
        else:
            initialcset = None

        if not exists(join(self.repository.basedir, 'CVS')):
            # CVS does not handle "checkout -d multi/level/subdir", so
            # split the basedir and use it's parentdir as cwd below.
            parentdir, subdir = split(self.repository.basedir)
            cmd = self.repository.command("-f", "-q", "-d",
                                          self.repository.repository,
                                          "checkout", "-d", subdir)
            if revision:
                cmd.extend(["-r", revision])
            if timestamp:
                cmd.extend(["-D", "%s UTC" % timestamp])
            if self.repository.freeze_keywords:
                cmd.append('-kk')

            checkout = ExternalCommand(cwd=parentdir, command=cmd)
            retry = 0
            while True:
                checkout.execute(self.repository.module)
                if checkout.exit_status:
                    retry += 1
                    if retry > 3:
                        break
                    delay = 2**retry
                    self.log.warning(
                        "%s returned status %s, "
                        "retrying in %d seconds...", str(checkout),
                        checkout.exit_status, delay)
                    sleep(retry)
                else:
                    break

            if checkout.exit_status:
                raise TargetInitializationFailure(
                    "%s returned status %s" %
                    (str(checkout), checkout.exit_status))
        else:
            self.log.info("Using existing %s", self.repository.basedir)

        if self.repository.tag_entries:
            self.__forceTagOnEachEntry()

        entries = CvsEntries(self.repository.basedir)
        youngest_entry = entries.getYoungestEntry()
        if youngest_entry is None:
            raise EmptyRepositoriesFoolsMe("The working copy '%s' of the "
                                           "CVS repository seems empty, "
                                           "don't know how to deal with "
                                           "that." % self.repository.basedir)

        # loop over the changesets and find the last applied, to find
        # out the actual cvsps revision

        found = False

        def already_applied(cs, entries=entries):
            "Loop over changeset entries to determine if it's already applied."

            applied = False

            # applied become True when an entry is DELETED *and* there is
            # no metainfo for it: thus, a changeset that removes a few entries
            # very late in history would be assumed as applied. Prevent that
            # by checking for at least one explicit match on an existing entry.
            onepositive = False

            for m in cs.entries:
                info = entries.getFileInfo(m.name)

                # If the entry's info exists, compare the on-disk
                # version with what we have: the revision is already
                # applied if the former is greater or equal than the
                # latter. The same if the info does not exist and it's
                # a delete event.

                if info:
                    odversion = info.cvs_version
                    applied = compare_cvs_revs(odversion, m.new_revision) >= 0

                    # If only one "hunk" is not yet applied, the whole
                    # changeset is new.
                    if not applied:
                        break
                    else:
                        onepositive = True
                elif m.action_kind == ChangesetEntry.DELETED:
                    applied = True
            return applied and onepositive

        # We cannot stop at the first not-applied cset, because it may
        # old enough to trick already_applied(): an entry may have
        # been moved in the meantime, and thus the getFileInfo()
        # method would return None, for example... So we really have
        # to loop over the whole queue.

        for cset in self.state_file:
            applied = already_applied(cset)
            found = found or applied
            if applied:
                last = cset

        if not found and initialcset:
            found = already_applied(initialcset)
            if found:
                last = initialcset

        if not found:
            raise TargetInitializationFailure(
                "Something went wrong: unable to determine the exact upstream "
                "revision of the checked out tree in '%s'. Either you specified "
                "the wrong initial timestamp, or you are checking out a "
                "composition of 'CVS modules' and Tailor does not support them; "
                "see the option 'trim-module-components' for a possible "
                "workaround." % self.repository.basedir)
        else:
            self.log.info("Working copy up to revision %s", last.revision)

        return last
Beispiel #3
0
    def _applyChangeset(self, changeset):
        from os.path import join, exists, split
        from os import listdir
        from shutil import rmtree
        from time import sleep
        from vcpx.repository.cvs import CvsEntries, compare_cvs_revs

        entries = CvsEntries(self.repository.basedir)

        # Collect added and deleted directories
        addeddirs = []
        deleteddirs = []

        # Group entries at the same revision
        reventries = {}

        for e in changeset.entries:
            if e.action_kind == e.UPDATED:
                info = entries.getFileInfo(e.name)
                if not info:
                    self.log.debug('promoting "%s" to ADDED at '
                                   'revision %s', e.name, e.new_revision)
                    e.action_kind = e.ADDED
                    for dir in self.__createParentCVSDirectories(
                            changeset, e.name):
                        addeddirs.append((e, dir))
                elif info.cvs_version == e.new_revision:
                    self.log.debug(
                        'skipping "%s" since it is already '
                        'at revision %s', e.name, e.new_revision)
                    continue
            elif e.action_kind == e.DELETED:
                if not exists(join(self.repository.basedir, e.name)):
                    self.log.debug(
                        'skipping "%s" since it is already '
                        'deleted', e.name)
                    entrydir = split(e.name)[0]
                    if self.__maybeDeleteDirectory(entrydir, changeset):
                        deleteddirs.append(entrydir)
                    continue
            elif e.action_kind == e.ADDED:
                if e.new_revision is None:
                    # This is a new directory entry, there is no need to update it
                    continue
                else:
                    for dir in self.__createParentCVSDirectories(
                            changeset, e.name):
                        addeddirs.append((e, dir))

            # If this is a directory (CVS does not version directories,
            # and thus new_revision is always None for them), and it's
            # going to be deleted, do not execute a 'cvs update', that
            # in some cases does not do what one would expect. Instead,
            # remove it with everything it contains (that should be
            # just a single "CVS" subdir, btw)

            if e.action_kind == e.DELETED and e.new_revision is None:
                assert CvsEntries(
                    join(self.repository.basedir,
                         e.name)).isEmpty(), '%s should be empty' % e.name
                rmtree(join(self.repository.basedir, e.name))
            else:
                names = reventries.setdefault(e.new_revision, [])
                names.append(e.name)

            if e.action_kind == e.DELETED:
                entrydir = split(e.name)[0]
                if self.__maybeDeleteDirectory(entrydir, changeset):
                    deleteddirs.append(entrydir)

        revs = reventries.keys()
        revs.sort(compare_cvs_revs)

        cmd = self.repository.command("-f", "-d", "%(repository)s", "-q",
                                      "update", "-d", "-r", "%(revision)s")
        if self.repository.freeze_keywords:
            cmd.append('-kk')
        cvsup = ExternalCommand(cwd=self.repository.basedir, command=cmd)
        for rev in revs:
            names = reventries[rev]
            retry = 0
            while True:
                cvsup.execute(names,
                              repository=self.repository.repository,
                              revision=rev)
                if cvsup.exit_status:
                    retry += 1
                    if retry > 3:
                        break
                    delay = 2**retry
                    self.log.warning(
                        "%s returned status %s, "
                        "retrying in %d seconds...", str(cvsup),
                        cvsup.exit_status, delay)
                    sleep(retry)
                else:
                    break

            if cvsup.exit_status:
                raise ChangesetApplicationFailure(
                    "%s returned status %s" % (str(cvsup), cvsup.exit_status))

            self.log.debug("%s updated to %s", ','.join(names), e.new_revision)

        # Fake up ADD and DEL events for the directories implicitly
        # added/removed, so that the replayer gets their name.

        for entry, path in addeddirs:
            entry = changeset.addEntry(path, None, before=entry)
            entry.action_kind = entry.ADDED
            self.log.info("registering new %s directory", entry.name)

        for path in deleteddirs:
            deldir = changeset.addEntry(path, None)
            deldir.action_kind = deldir.DELETED
            self.log.info("registering %s directory deletion", path)

        # Make sure all files are present on disk: CVS update does not
        # create them nor reports an error if the files have been
        # completely removed from the cvs repository. So loop over the
        # entries and verify the presence of added/changed ones.

        for entry in changeset.entries:
            if (entry.action_kind in (entry.ADDED, entry.UPDATED)
                    and not exists(join(self.repository.basedir, entry.name))):
                self.log.warning(
                    "Ignoring entry %s, CVS source no "
                    "longer knows about it.", entry.name)
                changeset.entries.remove(entry)
Beispiel #4
0
    def _checkoutUpstreamRevision(self, revision):
        """
        Concretely do the checkout of the upstream sources. Use
        `revision` as the name of the tag to get, or as a date if it
        starts with a number.

        Return the last applied changeset.
        """

        from os.path import join, exists, split
        from time import sleep
        from vcpx.repository.cvs import CvsEntries, compare_cvs_revs
        from vcpx.changes import ChangesetEntry

        if not self.repository.module:
            raise InvocationError("Must specify a module name")

        timestamp = None
        if revision is not None:
            # If the revision contains a space, assume it really
            # specify a branch and a timestamp. If it starts with
            # a digit, assume it's a timestamp. Otherwise, it must
            # be a branch name
            if revision[0] in '0123456789' or revision == 'INITIAL':
                timestamp = revision
                revision = None
            elif ' ' in revision:
                revision, timestamp = revision.split(' ', 1)

        csets = self.getPendingChangesets(revision)
        if not csets:
            raise TargetInitializationFailure(
                "Something went wrong: there are no changesets since "
                "revision '%s'" % revision)
        if timestamp == 'INITIAL':
            initialcset = csets.next()
            timestamp = initialcset.date.replace(tzinfo=None).isoformat(sep=' ')
        else:
            initialcset = None

        if not exists(join(self.repository.basedir, 'CVS')):
            # CVS does not handle "checkout -d multi/level/subdir", so
            # split the basedir and use it's parentdir as cwd below.
            parentdir, subdir = split(self.repository.basedir)
            cmd = self.repository.command("-f", "-q",
                                          "-d", self.repository.repository,
                                          "checkout",
                                          "-d", subdir)
            if revision:
                cmd.extend(["-r", revision])
            if timestamp:
                cmd.extend(["-D", "%s UTC" % timestamp])
            if self.repository.freeze_keywords:
                cmd.append('-kk')

            checkout = ExternalCommand(cwd=parentdir, command=cmd)
            retry = 0
            while True:
                checkout.execute(self.repository.module)
                if checkout.exit_status:
                    retry += 1
                    if retry>3:
                        break
                    delay = 2**retry
                    self.log.warning("%s returned status %s, "
                                     "retrying in %d seconds...",
                                     str(checkout), checkout.exit_status,
                                     delay)
                    sleep(retry)
                else:
                    break

            if checkout.exit_status:
                raise TargetInitializationFailure(
                    "%s returned status %s" % (str(checkout),
                                               checkout.exit_status))
        else:
            self.log.info("Using existing %s", self.repository.basedir)

        if self.repository.tag_entries:
            self.__forceTagOnEachEntry()

        entries = CvsEntries(self.repository.basedir)
        youngest_entry = entries.getYoungestEntry()
        if youngest_entry is None:
            raise EmptyRepositoriesFoolsMe("The working copy '%s' of the "
                                           "CVS repository seems empty, "
                                           "don't know how to deal with "
                                           "that." % self.repository.basedir)

        # loop over the changesets and find the last applied, to find
        # out the actual cvsps revision

        found = False

        def already_applied(cs, entries=entries):
            "Loop over changeset entries to determine if it's already applied."

            applied = False

            # applied become True when an entry is DELETED *and* there is
            # no metainfo for it: thus, a changeset that removes a few entries
            # very late in history would be assumed as applied. Prevent that
            # by checking for at least one explicit match on an existing entry.
            onepositive = False

            for m in cs.entries:
                info = entries.getFileInfo(m.name)

                # If the entry's info exists, compare the on-disk
                # version with what we have: the revision is already
                # applied if the former is greater or equal than the
                # latter. The same if the info does not exist and it's
                # a delete event.

                if info:
                    odversion = info.cvs_version
                    applied = compare_cvs_revs(odversion, m.new_revision) >= 0

                    # If only one "hunk" is not yet applied, the whole
                    # changeset is new.
                    if not applied:
                        break
                    else:
                        onepositive = True
                elif m.action_kind == ChangesetEntry.DELETED:
                    applied = True
            return applied and onepositive

        # We cannot stop at the first not-applied cset, because it may
        # old enough to trick already_applied(): an entry may have
        # been moved in the meantime, and thus the getFileInfo()
        # method would return None, for example... So we really have
        # to loop over the whole queue.

        for cset in self.state_file:
            applied = already_applied(cset)
            found = found or applied
            if applied:
                last = cset

        if not found and initialcset:
            found = already_applied(initialcset)
            if found:
                last = initialcset

        if not found:
            raise TargetInitializationFailure(
                "Something went wrong: unable to determine the exact upstream "
                "revision of the checked out tree in '%s'. Either you specified "
                "the wrong initial timestamp, or you are checking out a "
                "composition of 'CVS modules' and Tailor does not support them; "
                "see the option 'trim-module-components' for a possible "
                "workaround." % self.repository.basedir)
        else:
            self.log.info("Working copy up to revision %s", last.revision)

        return last
Beispiel #5
0
    def _applyChangeset(self, changeset):
        from os.path import join, exists, split
        from os import listdir
        from shutil import rmtree
        from time import sleep
        from vcpx.repository.cvs import CvsEntries, compare_cvs_revs

        entries = CvsEntries(self.repository.basedir)

        # Collect added and deleted directories
        addeddirs = []
        deleteddirs = []

        # Group entries at the same revision
        reventries = {}

        for e in changeset.entries:
            if e.action_kind == e.UPDATED:
                info = entries.getFileInfo(e.name)
                if not info:
                    self.log.debug('promoting "%s" to ADDED at '
                                   'revision %s', e.name, e.new_revision)
                    e.action_kind = e.ADDED
                    for dir in self.__createParentCVSDirectories(changeset, e.name):
                        addeddirs.append((e, dir))
                elif info.cvs_version == e.new_revision:
                    self.log.debug('skipping "%s" since it is already '
                                   'at revision %s', e.name, e.new_revision)
                    continue
            elif e.action_kind == e.DELETED:
                if not exists(join(self.repository.basedir, e.name)):
                    self.log.debug('skipping "%s" since it is already '
                                   'deleted', e.name)
                    entrydir = split(e.name)[0]
                    if self.__maybeDeleteDirectory(entrydir, changeset):
                        deleteddirs.append(entrydir)
                    continue
            elif e.action_kind == e.ADDED:
                if e.new_revision is None:
                    # This is a new directory entry, there is no need to update it
                    continue
                else:
                    for dir in self.__createParentCVSDirectories(changeset, e.name):
                        addeddirs.append((e, dir))

            # If this is a directory (CVS does not version directories,
            # and thus new_revision is always None for them), and it's
            # going to be deleted, do not execute a 'cvs update', that
            # in some cases does not do what one would expect. Instead,
            # remove it with everything it contains (that should be
            # just a single "CVS" subdir, btw)

            if e.action_kind == e.DELETED and e.new_revision is None:
                assert CvsEntries(join(self.repository.basedir, e.name)).isEmpty(), '%s should be empty' % e.name
                rmtree(join(self.repository.basedir, e.name))
            else:
                names = reventries.setdefault(e.new_revision, [])
                names.append(e.name)

            if e.action_kind == e.DELETED:
                entrydir = split(e.name)[0]
                if self.__maybeDeleteDirectory(entrydir, changeset):
                    deleteddirs.append(entrydir)

        revs = reventries.keys()
        revs.sort(compare_cvs_revs)

        cmd = self.repository.command("-f", "-d", "%(repository)s",
                                      "-q", "update", "-d",
                                      "-r", "%(revision)s")
        if self.repository.freeze_keywords:
            cmd.append('-kk')
        cvsup = ExternalCommand(cwd=self.repository.basedir, command=cmd)
        for rev in revs:
            names = reventries[rev]
            retry = 0
            while True:
                cvsup.execute(names, repository=self.repository.repository,
                              revision=rev)
                if cvsup.exit_status:
                    retry += 1
                    if retry>3:
                        break
                    delay = 2**retry
                    self.log.warning("%s returned status %s, "
                                     "retrying in %d seconds...",
                                     str(cvsup), cvsup.exit_status, delay)
                    sleep(retry)
                else:
                    break

            if cvsup.exit_status:
                raise ChangesetApplicationFailure(
                    "%s returned status %s" % (str(cvsup),
                                               cvsup.exit_status))

            self.log.debug("%s updated to %s", ','.join(names), e.new_revision)

        # Fake up ADD and DEL events for the directories implicitly
        # added/removed, so that the replayer gets their name.

        for entry,path in addeddirs:
            entry = changeset.addEntry(path, None, before=entry)
            entry.action_kind = entry.ADDED
            self.log.info("registering new %s directory", entry.name)

        for path in deleteddirs:
            deldir = changeset.addEntry(path, None)
            deldir.action_kind = deldir.DELETED
            self.log.info("registering %s directory deletion", path)

        # Make sure all files are present on disk: CVS update does not
        # create them nor reports an error if the files have been
        # completely removed from the cvs repository. So loop over the
        # entries and verify the presence of added/changed ones.

        for entry in changeset.entries:
            if (entry.action_kind in (entry.ADDED, entry.UPDATED)
                and not exists(join(self.repository.basedir, entry.name))):
                self.log.warning("Ignoring entry %s, CVS source no "
                                 "longer knows about it.", entry.name)
                changeset.entries.remove(entry)