Example #1
0
    def _setup_sources(self):
        """
        Create a copy of the git source for the project at the point in time
        our build tag was created.

        Created in the temporary rpmbuild SOURCES directory.
        """
        self._create_build_dirs()

        debug("Creating %s from git tag: %s..." % (self.tgz_filename,
            self.git_commit_id))
        create_tgz(self.git_root, self.tgz_dir, self.git_commit_id,
                self.relative_project_dir,
                os.path.join(self.rpmbuild_sourcedir, self.tgz_filename))

        # Extract the source so we can get at the spec file, etc.
        debug("Copying git source to: %s" % self.rpmbuild_gitcopy)
        run_command("cd %s/ && tar xzf %s" % (self.rpmbuild_sourcedir,
            self.tgz_filename))

        # Show contents of the directory structure we just extracted.
        debug('', 'ls -lR %s/' % self.rpmbuild_gitcopy)

        # NOTE: The spec file we actually use is the one exported by git
        # archive into the temp build directory. This is done so we can
        # modify the version/release on the fly when building test rpms
        # that use a git SHA1 for their version.
        self.spec_file_name = os.path.basename(find_spec_like_file(self.rpmbuild_gitcopy))
        self.spec_file = os.path.join(
            self.rpmbuild_gitcopy, self.spec_file_name)
Example #2
0
    def _setup_sources(self):
        """
        Create a copy of the git source for the project at the point in time
        our build tag was created.

        Created in the temporary rpmbuild SOURCES directory.
        """
        self._create_build_dirs()

        debug("Creating %s from git tag: %s..." % (self.tgz_filename,
            self.git_commit_id))
        create_tgz(self.git_root, self.tgz_dir, self.git_commit_id,
                self.relative_project_dir,
                os.path.join(self.rpmbuild_sourcedir, self.tgz_filename))

        # Extract the source so we can get at the spec file, etc.
        debug("Copying git source to: %s" % self.rpmbuild_gitcopy)
        run_command("cd %s/ && tar xzf %s" % (self.rpmbuild_sourcedir,
            self.tgz_filename))

        # Show contents of the directory structure we just extracted.
        debug('', 'ls -lR %s/' % self.rpmbuild_gitcopy)

        # NOTE: The spec file we actually use is the one exported by git
        # archive into the temp build directory. This is done so we can
        # modify the version/release on the fly when building test rpms
        # that use a git SHA1 for their version.
        self.spec_file_name = os.path.basename(find_spec_like_file(self.rpmbuild_gitcopy))
        self.spec_file = os.path.join(
            self.rpmbuild_gitcopy, self.spec_file_name)
Example #3
0
    def tgz(self):
        """
        Override parent behavior, we need a tgz from the upstream spacewalk
        project we're based on.
        """
        # TODO: Wasteful step here, all we really need is a way to look for a
        # spec file at the point in time this release was tagged.
        NoTgzBuilder._setup_sources(self)
        # If we knew what it was named at that point in time we could just do:
        # Export a copy of our spec file at the revision to be built:
#        cmd = "git show %s:%s%s > %s" % (self.git_commit_id,
#                self.relative_project_dir, self.spec_file_name,
#                self.spec_file)
#        debug(cmd)
        self._create_build_dirs()

        self.upstream_version = self._get_upstream_version()
        self.upstream_tag = "%s-%s-1" % (self.upstream_name,
                self.upstream_version)

        print("Building upstream tgz for tag [%s]" % (self.upstream_tag))
        if self.upstream_tag != self.build_tag:
            check_tag_exists(self.upstream_tag, offline=self.offline)

        self.spec_file = os.path.join(self.rpmbuild_sourcedir,
                self.spec_file_name)
        command = "cp %s %s" % (os.path.join(self.rpmbuild_gitcopy,
            self.spec_file_name), self.spec_file)
        run_command(command)

        # Create the upstream tgz:
        prefix = "%s-%s" % (self.upstream_name, self.upstream_version)
        tgz_filename = "%s.tar.gz" % prefix
        commit = get_build_commit(tag=self.upstream_tag)
        relative_dir = get_relative_project_dir(
            project_name=self.upstream_name, commit=commit)
        tgz_fullpath = os.path.join(self.rpmbuild_sourcedir, tgz_filename)
        print("Creating %s from git tag: %s..." % (tgz_filename, commit))
        create_tgz(self.git_root, prefix, commit, relative_dir,
                tgz_fullpath)
        self.ran_tgz = True
        self.sources.append(tgz_fullpath)

        # If these are equal then the tag we're building was likely created in
        # Spacewalk and thus we don't need to do any patching.
        if (self.upstream_tag == self.build_tag and not self.test):
            return

        self.patch_upstream()
Example #4
0
    def tgz(self):
        """
        Override parent behavior, we need a tgz from the upstream spacewalk
        project we're based on.
        """
        # TODO: Wasteful step here, all we really need is a way to look for a
        # spec file at the point in time this release was tagged.
        NoTgzBuilder._setup_sources(self)
        # If we knew what it was named at that point in time we could just do:
        # Export a copy of our spec file at the revision to be built:
#        cmd = "git show %s:%s%s > %s" % (self.git_commit_id,
#                self.relative_project_dir, self.spec_file_name,
#                self.spec_file)
#        debug(cmd)
        self._create_build_dirs()

        self.upstream_version = self._get_upstream_version()
        self.upstream_tag = "%s-%s-1" % (self.upstream_name,
                self.upstream_version)

        print("Building upstream tgz for tag [%s]" % (self.upstream_tag))
        if self.upstream_tag != self.build_tag:
            check_tag_exists(self.upstream_tag, offline=self.offline)

        self.spec_file = os.path.join(self.rpmbuild_sourcedir,
                self.spec_file_name)
        command = "cp %s %s" % (os.path.join(self.rpmbuild_gitcopy,
            self.spec_file_name), self.spec_file)
        run_command(command)

        # Create the upstream tgz:
        prefix = "%s-%s" % (self.upstream_name, self.upstream_version)
        tgz_filename = "%s.tar.gz" % prefix
        commit = get_build_commit(tag=self.upstream_tag)
        relative_dir = get_relative_project_dir(
            project_name=self.upstream_name, commit=commit)
        tgz_fullpath = os.path.join(self.rpmbuild_sourcedir, tgz_filename)
        print("Creating %s from git tag: %s..." % (tgz_filename, commit))
        create_tgz(self.git_root, prefix, commit, relative_dir,
                tgz_fullpath)
        self.ran_tgz = True
        self.sources.append(tgz_fullpath)

        # If these are equal then the tag we're building was likely created in
        # Spacewalk and thus we don't need to do any patching.
        if (self.upstream_tag == self.build_tag and not self.test):
            return

        self.patch_upstream()
Example #5
0
    def _setup_sources(self):
        """
        Create a copy of the git source for the project at the point in time
        our build tag was created.

        Created in the temporary rpmbuild SOURCES directory.
        """
        self._create_build_dirs()

        debug("Creating %s from git tag: %s..." %
              (self.tgz_filename, self.git_commit_id))
        create_tgz(self.git_root, self.tgz_dir, self.git_commit_id,
                   self.relative_project_dir,
                   os.path.join(self.rpmbuild_sourcedir, self.tgz_filename))

        # Extract the source so we can get at the spec file, etc.
        debug("Copying git source to: %s" % self.rpmbuild_gitcopy)
        run_command("cd %s/ && tar xzf %s" %
                    (self.rpmbuild_sourcedir, self.tgz_filename))

        # Find the gemspec
        gemspec_filename = find_gemspec_file(in_dir=self.rpmbuild_gitcopy)

        debug("Building gem: %s in %s" %
              (gemspec_filename, self.rpmbuild_gitcopy))
        # FIXME - this is ugly and should probably be handled better
        cmd = "gem_name=$(cd %s/ && gem build %s | awk '/File/ {print $2}'); \
            cp %s/$gem_name %s/" % (self.rpmbuild_gitcopy, gemspec_filename,
                                    self.rpmbuild_gitcopy,
                                    self.rpmbuild_sourcedir)

        run_command(cmd)

        # NOTE: The spec file we actually use is the one exported by git
        # archive into the temp build directory. This is done so we can
        # modify the version/release on the fly when building test rpms
        # that use a git SHA1 for their version.
        self.spec_file_name = find_spec_file(in_dir=self.rpmbuild_gitcopy)
        self.spec_file = os.path.join(self.rpmbuild_gitcopy,
                                      self.spec_file_name)
Example #6
0
    def _setup_sources(self):
        """
        Create a copy of the git source for the project at the point in time
        our build tag was created.

        Created in the temporary rpmbuild SOURCES directory.
        """
        self._create_build_dirs()

        debug("Creating %s from git tag: %s..." % (self.tgz_filename,
            self.git_commit_id))
        create_tgz(self.git_root, self.tgz_dir, self.git_commit_id,
                self.relative_project_dir,
                os.path.join(self.rpmbuild_sourcedir, self.tgz_filename))

        # Extract the source so we can get at the spec file, etc.
        debug("Copying git source to: %s" % self.rpmbuild_gitcopy)
        run_command("cd %s/ && tar xzf %s" % (self.rpmbuild_sourcedir,
            self.tgz_filename))

        # Find the gemspec
        gemspec_filename = find_gemspec_file(self.rpmbuild_gitcopy)

        debug("Building gem: %s in %s" % (gemspec_filename,
            self.rpmbuild_gitcopy))
        # FIXME - this is ugly and should probably be handled better
        cmd = "gem_name=$(cd %s/ && gem build %s | awk '/File/ {print $2}'); \
            cp %s/$gem_name %s/" % (self.rpmbuild_gitcopy, gemspec_filename,
            self.rpmbuild_gitcopy, self.rpmbuild_sourcedir)

        run_command(cmd)

        # NOTE: The spec file we actually use is the one exported by git
        # archive into the temp build directory. This is done so we can
        # modify the version/release on the fly when building test rpms
        # that use a git SHA1 for their version.
        self.spec_file_name = find_spec_file(self.rpmbuild_gitcopy)
        self.spec_file = os.path.join(
            self.rpmbuild_gitcopy, self.spec_file_name)
Example #7
0
    def tgz(self):
        destination_file = os.path.join(self.rpmbuild_basedir, self.tgz_filename)
        formatted_properties = ["-D%s" % x for x in self.maven_properties]

        run_command("git clone --no-hardlinks %s %s" % (find_git_root(), self.maven_clone_dir))
        with chdir(self.maven_clone_dir):
            run_command("git checkout %s" % self.git_commit_id)

            try:
                info_out("Running Maven build...")
                # We always want to deploy to a tito controlled location during local builds
                local_properties = formatted_properties + [
                    "-DaltDeploymentRepository=local-output::default::file://%s" % self.deploy_dir]
                run_command("mvn %s %s deploy" % (
                    " ".join(self.maven_args),
                    " ".join(local_properties)))
            except RunCommandException as e:
                error_out("Maven build failed! %s" % e.output)

        self._create_build_dirs()

        full_path = self._find_tarball()
        if full_path:
            fh = gzip.open(full_path, 'rb')
            fixed_tar = os.path.join(os.path.splitext(full_path)[0])
            fixed_tar_fh = open(fixed_tar, 'wb')
            timestamp = get_commit_timestamp(self.git_commit_id)
            try:
                tarfixer = TarFixer(fh, fixed_tar_fh, timestamp, self.git_commit_id, maven_built=True)
                tarfixer.fix()
            finally:
                fixed_tar_fh.close()

            # It's a pity we can't use Python's gzip, but it doesn't offer an equivalent of -n
            run_command("gzip -n -c < %s > %s" % (fixed_tar, destination_file))
        else:
            warn_out([
                "No Maven generated tarball found.",
                "Please set up the assembly plugin in your pom.xml to generate a .tar.gz"])
            full_path = os.path.join(self.rpmbuild_sourcedir, self.tgz_filename)
            create_tgz(self.git_root, self.tgz_dir, self.git_commit_id, self.relative_project_dir, full_path)
            print("Creating %s from git tag: %s..." % (self.tgz_filename, self.build_tag))
            shutil.copy(full_path, destination_file)

        debug("Copying git source to: %s" % self.rpmbuild_gitcopy)
        shutil.copy(destination_file, self.rpmbuild_gitcopy)

        # Extract the source so we can get at the spec file, etc.
        with chdir(self.rpmbuild_gitcopy):
            run_command("tar --strip-components=1 -xvf %s" % os.path.join(self.rpmbuild_gitcopy, self.tgz_filename))

        if self.local_build:
            artifacts = {}
            all_artifacts = []
            all_artifacts_with_path = []

            for directory, unused, filenames in os.walk(self.deploy_dir):
                for f in filenames:
                    artifacts.setdefault(os.path.splitext(f)[1], []).append(f)
                dir_artifacts_with_path = [os.path.join(directory, f) for f in filenames]

                # Place the Maven artifacts in the SOURCES directory for rpmbuild to use
                for artifact in dir_artifacts_with_path:
                    shutil.copy(artifact, self.rpmbuild_sourcedir)

                dir_artifacts_with_path = map(lambda x: os.path.relpath(x, self.deploy_dir), dir_artifacts_with_path)
                all_artifacts_with_path.extend(dir_artifacts_with_path)
                all_artifacts.extend([os.path.basename(f) for f in filenames])

            cheetah_input = {
                'name': self.project_name,
                'version': self.spec_version,
                'release': self.spec_release,
                'epoch': None,  # TODO: May need to support this at some point
                'artifacts': artifacts,
                'all_artifacts': all_artifacts,
                'all_artifacts_with_path': all_artifacts_with_path,
            }
            debug("Cheetah input: %s" % cheetah_input)
            render_cheetah(find_cheetah_template_file(self.start_dir), self.rpmbuild_gitcopy, cheetah_input)
            self.spec_file_name = find_spec_file(self.rpmbuild_gitcopy)
        else:
            self.spec_file_name = find_cheetah_template_file(self.rpmbuild_gitcopy)

        # NOTE: The spec file we actually use is the one exported by git
        # archive into the temp build directory. This is done so we can
        # modify the version/release on the fly when building test rpms
        # that use a git SHA1 for their version.
        self.spec_file = os.path.join(self.rpmbuild_gitcopy, self.spec_file_name)

        info_out("Wrote: %s" % destination_file)
        self.sources.append(destination_file)
        self.artifacts.append(destination_file)
        self.ran_tgz = True
Example #8
0
    def tgz(self):
        destination_file = os.path.join(self.rpmbuild_basedir, self.tgz_filename)
        formatted_properties = ["-D%s" % x for x in self.maven_properties]

        run_command("git clone --no-hardlinks %s %s" % (find_git_root(), self.maven_clone_dir))
        with chdir(self.maven_clone_dir):
            run_command("git checkout %s" % self.git_commit_id)

            try:
                info_out("Running Maven build...")
                # We always want to deploy to a tito controlled location during local builds
                local_properties = formatted_properties + [
                    "-DaltDeploymentRepository=local-output::default::file://%s" % self.deploy_dir]
                run_command("mvn %s %s deploy" % (
                    " ".join(self.maven_args),
                    " ".join(local_properties)))
            except RunCommandException as e:
                error_out("Maven build failed! %s" % e.output)

        self._create_build_dirs()

        full_path = self._find_tarball()
        if full_path:
            fh = gzip.open(full_path, 'rb')
            fixed_tar = os.path.join(os.path.splitext(full_path)[0])
            fixed_tar_fh = open(fixed_tar, 'wb')
            timestamp = get_commit_timestamp(self.git_commit_id)
            try:
                tarfixer = TarFixer(fh, fixed_tar_fh, timestamp, self.git_commit_id, maven_built=True)
                tarfixer.fix()
            finally:
                fixed_tar_fh.close()

            # It's a pity we can't use Python's gzip, but it doesn't offer an equivalent of -n
            run_command("gzip -n -c < %s > %s" % (fixed_tar, destination_file))
        else:
            warn_out([
                "No Maven generated tarball found.",
                "Please set up the assembly plugin in your pom.xml to generate a .tar.gz"])
            full_path = os.path.join(self.rpmbuild_sourcedir, self.tgz_filename)
            create_tgz(self.git_root, self.tgz_dir, self.git_commit_id, self.relative_project_dir, full_path)
            print("Creating %s from git tag: %s..." % (self.tgz_filename, self.build_tag))
            shutil.copy(full_path, destination_file)

        debug("Copying git source to: %s" % self.rpmbuild_gitcopy)
        shutil.copy(destination_file, self.rpmbuild_gitcopy)

        # Extract the source so we can get at the spec file, etc.
        with chdir(self.rpmbuild_gitcopy):
            run_command("tar --strip-components=1 -xvf %s" % os.path.join(self.rpmbuild_gitcopy, self.tgz_filename))

        if self.local_build:
            artifacts = {}
            all_artifacts = []
            all_artifacts_with_path = []

            for directory, unused, filenames in os.walk(self.deploy_dir):
                for f in filenames:
                    artifacts.setdefault(os.path.splitext(f)[1], []).append(f)
                dir_artifacts_with_path = [os.path.join(directory, f) for f in filenames]

                # Place the Maven artifacts in the SOURCES directory for rpmbuild to use
                for artifact in dir_artifacts_with_path:
                    shutil.copy(artifact, self.rpmbuild_sourcedir)

                dir_artifacts_with_path = map(lambda x: os.path.relpath(x, self.deploy_dir), dir_artifacts_with_path)
                all_artifacts_with_path.extend(dir_artifacts_with_path)
                all_artifacts.extend([os.path.basename(f) for f in filenames])

            cheetah_input = {
                'name': self.project_name,
                'version': self.spec_version,
                'release': self.spec_release,
                'epoch': None,  # TODO: May need to support this at some point
                'artifacts': artifacts,
                'all_artifacts': all_artifacts,
                'all_artifacts_with_path': all_artifacts_with_path,
            }
            debug("Cheetah input: %s" % cheetah_input)
            render_cheetah(find_cheetah_template_file(self.start_dir), self.rpmbuild_gitcopy, cheetah_input)
            self.spec_file_name = find_spec_file(self.rpmbuild_gitcopy)
        else:
            self.spec_file_name = find_cheetah_template_file(self.rpmbuild_gitcopy)

        # NOTE: The spec file we actually use is the one exported by git
        # archive into the temp build directory. This is done so we can
        # modify the version/release on the fly when building test rpms
        # that use a git SHA1 for their version.
        self.spec_file = os.path.join(self.rpmbuild_gitcopy, self.spec_file_name)

        info_out("Wrote: %s" % destination_file)
        self.sources.append(destination_file)
        self.artifacts.append(destination_file)
        self.ran_tgz = True
Example #9
0
    def create_tgz(self, git_root, prefix, commit, relative_dir,
                   dest_tgz):
        """
        Create a .tar.gz from a projects source in git.
        And include submodules
        """

        git_root_abspath = os.path.abspath(git_root)
        gitmodules_path = os.path.join(git_root_abspath, '.gitmodules')

        # if .gitmodules does not exist, just call the existing create_tgz function
        # as there is nothing to see here.
        if not os.path.exists(gitmodules_path):
            return create_tgz(git_root, prefix, commit, relative_dir, dest_tgz)

        os.chdir(git_root_abspath)
        timestamp = get_commit_timestamp(commit)

        # Accommodate standalone projects with specfile in root of git repo:
        relative_git_dir = "%s" % relative_dir
        if relative_git_dir in ['/', './']:
            relative_git_dir = ""

        basename = os.path.splitext(dest_tgz)[0]
        initial_tar = "%s.initial" % basename

        # We need to tar up the following:
        # 1. the current repo
        self.run_git_archive(relative_git_dir, prefix, commit, initial_tar, None)

        # 2. all of the submodules
        # then combine those into a single archive.
        submodules_cmd = 'git submodule--helper list'
        submodules_output = run_command(submodules_cmd)

        # split submodules output on newline
        # then on tab, and the directory is the last entry
        submodules_list = [line.split('\t')[-1] for line in submodules_output.split('\n')]

        submodule_tar_files = [initial_tar]
        # We ignore the hash in the sub modules list as we'll have to get the correct one
        # from the commit id in commit
        for submodule in submodules_list:
            # to find the submodule shars:
            # git rev-parse <commit>:./<submodule>
            rev_parse_cmd = 'git rev-parse %s:./%s' % (commit, submodule)
            submodule_commit = run_command(rev_parse_cmd)
            submodule_tar_file = '%s.%s' % (initial_tar, submodule)
            # prefix should be <prefix>/<submodule>
            submodule_prefix = '%s/%s' % (prefix, submodule)

            self.run_git_archive(relative_git_dir, submodule_prefix, submodule_commit,
                                 submodule_tar_file, submodule)
            submodule_tar_files.append(submodule_tar_file)

        # we need to append all of the submodule tar files onto the initial
        tarfiles = ' '.join(submodule_tar_files)
        run_command("tar -Af %s" % tarfiles)

        fixed_tar = "%s.tar" % basename
        fixed_tar_fh = open(fixed_tar, 'wb')
        try:
            tarfixer = TarFixer(open(initial_tar, 'rb'), fixed_tar_fh, timestamp, commit)
            tarfixer.fix()
        finally:
            fixed_tar_fh.close()

        # It's a pity we can't use Python's gzip, but it doesn't offer an equivalent of -n
        return run_command("gzip -n -c < %s > %s" % (fixed_tar, dest_tgz))