def __init__(self, global_config=None, keep_version=False, offline=False, user_config=None):
self.git_root = find_git_root()
self.rel_eng_dir = os.path.join(self.git_root, "rel-eng")
self.config = global_config
self.user_config = user_config
self.full_project_dir = os.getcwd()
self.spec_file_name = find_spec_file()
self.project_name = get_project_name(tag=None)
self.relative_project_dir = self._get_relative_project_dir(
self.git_root) # i.e. java/
self.spec_file = os.path.join(self.full_project_dir,
self.spec_file_name)
self.keep_version = keep_version
self.today = strftime("%a %b %d %Y")
(self.git_user, self.git_email) = self._get_git_user_info()
git_email = self.git_email
if git_email is None:
git_email = ''
self.changelog_regex = re.compile('\\*\s%s\s%s(\s<%s>)?' % (self.today,
self.git_user, git_email.replace("+", "\+").replace(".", "\.")))
self._no_auto_changelog = False
self._accept_auto_changelog = False
self._new_changelog_msg = "new package built with tito"
self.offline = offline
def _get_build_version(self):
"""
Figure out the git tag and version-release we're building.
"""
# Determine which package version we should build:
build_version = None
if self.build_tag:
build_version = self.build_tag[len(self.project_name + "-"):]
else:
build_version = get_latest_tagged_version(self.project_name)
if build_version is None:
if not self.test:
error_out([
"Unable to lookup latest package info.",
"Perhaps you need to tag first?"
])
sys.stderr.write("WARNING: unable to lookup latest package "
"tag, building untagged test project\n")
build_version = get_spec_version_and_release(
self.start_dir, find_spec_file(in_dir=self.start_dir))
self.build_tag = "v%s" % (build_version)
if not self.test:
check_tag_exists(self.build_tag, offline=self.offline)
return build_version
def _setup_sources(self):
"""
Create a copy of the git source for the project at the point in time
our build tag was created.
Created in the temporary rpmbuild SOURCES directory.
"""
self._create_build_dirs()
debug("Creating %s from git tag: %s..." % (self.tgz_filename,
self.git_commit_id))
create_tgz(self.git_root, self.tgz_dir, self.git_commit_id,
self.relative_project_dir,
os.path.join(self.rpmbuild_sourcedir, self.tgz_filename))
# Extract the source so we can get at the spec file, etc.
debug("Copying git source to: %s" % self.rpmbuild_gitcopy)
run_command("cd %s/ && tar xzf %s" % (self.rpmbuild_sourcedir,
self.tgz_filename))
# Show contents of the directory structure we just extracted.
debug('', 'ls -lR %s/' % self.rpmbuild_gitcopy)
# NOTE: The spec file we actually use is the one exported by git
# archive into the temp build directory. This is done so we can
# modify the version/release on the fly when building test rpms
# that use a git SHA1 for their version.
self.spec_file_name = find_spec_file(in_dir=self.rpmbuild_gitcopy)
self.spec_file = os.path.join(
self.rpmbuild_gitcopy, self.spec_file_name)
def _setup_sources(self):
"""Create .tar.gz for this package.
Returns: absolute path to the archive.
"""
self._create_build_dirs()
target_archive = Path(self.rpmbuild_sourcedir, self.tgz_filename)
worktree = TemporaryWorktree(prefix="scl.org.build-",
commit=self.git_commit_id)
with worktree as workdir:
debug("Copy source directory to {}".format(self.rpmbuild_gitcopy))
rmtree(self.rpmbuild_gitcopy)
copytree(workdir, self.rpmbuild_gitcopy)
debug("Make sdist archive")
run_command("python3 ./setup.py sdist")
source_archive, = workdir.joinpath("dist").glob("*.tar.gz")
debug("Recreate sdist archive in {target}".format(
target=target_archive))
self._recreate_tgz(source_archive, target_archive)
self.spec_file_name = find_spec_file(self.rpmbuild_gitcopy)
self.spec_file = Path(self.rpmbuild_gitcopy, self.spec_file_name)
def _setup_sources(self):
"""
Create a copy of the git source for the project at the point in time
our build tag was created.
Created in the temporary rpmbuild SOURCES directory.
"""
self._create_build_dirs()
debug("Creating %s from git tag: %s..." %
(self.tgz_filename, self.git_commit_id))
create_tgz(self.git_root, self.tgz_dir, self.git_commit_id,
self.relative_project_dir,
os.path.join(self.rpmbuild_sourcedir, self.tgz_filename))
# Extract the source so we can get at the spec file, etc.
debug("Copying git source to: %s" % self.rpmbuild_gitcopy)
run_command("cd %s/ && tar xzf %s" %
(self.rpmbuild_sourcedir, self.tgz_filename))
# Show contents of the directory structure we just extracted.
debug('', 'ls -lR %s/' % self.rpmbuild_gitcopy)
# NOTE: The spec file we actually use is the one exported by git
# archive into the temp build directory. This is done so we can
# modify the version/release on the fly when building test rpms
# that use a git SHA1 for their version.
self.spec_file_name = find_spec_file(in_dir=self.rpmbuild_gitcopy)
self.spec_file = os.path.join(self.rpmbuild_gitcopy,
self.spec_file_name)
def _get_build_version(self):
"""
Figure out the git tag and version-release we're building.
"""
# Determine which package version we should build:
build_version = None
if self.build_tag:
build_version = self.build_tag[len(self.project_name + "-") :]
else:
build_version = get_latest_tagged_version(self.project_name)
if build_version is None:
if not self.test:
error_out(["Unable to lookup latest package info.", "Perhaps you need to tag first?"])
sys.stderr.write("WARNING: unable to lookup latest package " "tag, building untagged test project\n")
build_version = get_spec_version_and_release(self.start_dir, find_spec_file(in_dir=self.start_dir))
self.build_tag = "v{0}".format(build_version)
if not self.test:
check_tag_exists(self.build_tag, offline=self.offline)
return build_version
def rpm(self):
""" Build an RPM. """
self._create_build_dirs()
if not self.ran_tgz:
self.tgz()
define_dist = ""
if self.dist:
define_dist = "--define 'dist %s'" % self.dist
rpmbuild_options = self.rpmbuild_options + self._scl_to_rpmbuild_option()
cmd = ('rpmbuild --define "_source_filedigest_algorithm md5" '
'--define "_binary_filedigest_algorithm md5" %s %s %s --clean '
'-ba %s' % (rpmbuild_options,
self._get_rpmbuild_dir_options(), define_dist, self.spec_file))
debug(cmd)
try:
output = run_command_print(cmd)
except (KeyboardInterrupt, SystemExit):
print("")
exit(1)
except RunCommandException:
err = sys.exc_info()[1]
msg = str(err)
if re.search('Failed build dependencies', err.output):
cmd = "dnf builddep %s" if package_manager() == "dnf" else "yum-builddep %s"
msg = "Please run '%s' as root." % \
cmd % find_spec_file(self.relative_project_dir)
error_out('%s' % msg)
except Exception:
err = sys.exc_info()[1]
error_out('%s' % str(err))
files_written = find_wrote_in_rpmbuild_output(output)
if len(files_written) < 2:
error_out("Error parsing rpmbuild output")
self.srpm_location = files_written[0]
self.artifacts.extend(files_written)
print
info_out("Successfully built: %s" % ' '.join(files_written))
def rpm(self):
""" Build an RPM. """
self._create_build_dirs()
if not self.ran_tgz:
self.tgz()
define_dist = ""
if self.dist:
define_dist = "--define 'dist %s'" % self.dist
rpmbuild_options = self.rpmbuild_options + self._scl_to_rpmbuild_option(
)
cmd = ('rpmbuild --define "_source_filedigest_algorithm md5" '
'--define "_binary_filedigest_algorithm md5" %s %s %s --clean '
'-ba %s' % (rpmbuild_options, self._get_rpmbuild_dir_options(),
define_dist, self.spec_file))
debug(cmd)
try:
output = run_command_print(cmd)
except (KeyboardInterrupt, SystemExit):
print("")
exit(1)
except RunCommandException:
err = sys.exc_info()[1]
msg = str(err)
if (re.search('Failed build dependencies', err.output)):
msg = "Please run 'yum-builddep %s' as root." % \
find_spec_file(self.relative_project_dir)
error_out('%s' % msg)
except Exception:
err = sys.exc_info()[1]
error_out('%s' % str(err))
files_written = find_wrote_in_rpmbuild_output(output)
if len(files_written) < 2:
error_out("Error parsing rpmbuild output")
self.srpm_location = files_written[0]
self.artifacts.extend(files_written)
print
print("Successfully built: %s" % ' '.join(files_written))
def _setup_sources(self):
"""
Create the autotools tarball using "make dist" command. Works only
with tar.gz tarballs.
Created in the temporary rpmbuild SOURCES directory.
"""
self._create_build_dirs()
debug("Creating dist tarball")
run_command("make dist VERSION=%s" % self.display_version)
full_path = os.path.join(self.rpmbuild_basedir, self.tgz_filename)
debug("Moving %s to %s" % (self.tgz_filename, self.rpmbuild_sourcedir))
run_command("mv %s %s" % (self.tgz_filename, self.rpmbuild_sourcedir))
# Extract the source so we can get at the spec file, etc.
debug("Copying git source to: %s" % self.rpmbuild_sourcedir)
run_command("cd %s/ && tar xzf %s" % (self.rpmbuild_sourcedir, self.tgz_filename))
self.spec_file_name = find_spec_file(in_dir=self.rpmbuild_gitcopy)
self.spec_file = os.path.join(self.rpmbuild_gitcopy, self.spec_file_name)
def _setup_sources(self):
"""
Create a copy of the git source for the project at the point in time
our build tag was created.
Created in the temporary rpmbuild SOURCES directory.
"""
self._create_build_dirs()
debug("Creating %s from git tag: %s..." %
(self.tgz_filename, self.git_commit_id))
create_tgz(self.git_root, self.tgz_dir, self.git_commit_id,
self.relative_project_dir,
os.path.join(self.rpmbuild_sourcedir, self.tgz_filename))
# Extract the source so we can get at the spec file, etc.
debug("Copying git source to: %s" % self.rpmbuild_gitcopy)
run_command("cd %s/ && tar xzf %s" %
(self.rpmbuild_sourcedir, self.tgz_filename))
# Find the gemspec
gemspec_filename = find_gemspec_file(in_dir=self.rpmbuild_gitcopy)
debug("Building gem: %s in %s" %
(gemspec_filename, self.rpmbuild_gitcopy))
# FIXME - this is ugly and should probably be handled better
cmd = "gem_name=$(cd %s/ && gem build %s | awk '/File/ {print $2}'); \
cp %s/$gem_name %s/" % (self.rpmbuild_gitcopy, gemspec_filename,
self.rpmbuild_gitcopy,
self.rpmbuild_sourcedir)
run_command(cmd)
# NOTE: The spec file we actually use is the one exported by git
# archive into the temp build directory. This is done so we can
# modify the version/release on the fly when building test rpms
# that use a git SHA1 for their version.
self.spec_file_name = find_spec_file(in_dir=self.rpmbuild_gitcopy)
self.spec_file = os.path.join(self.rpmbuild_gitcopy,
self.spec_file_name)
def _setup_sources(self):
"""
Create a copy of the git source for the project at the point in time
our build tag was created.
Created in the temporary rpmbuild SOURCES directory.
"""
self._create_build_dirs()
debug("Creating %s from git tag: %s..." % (self.tgz_filename,
self.git_commit_id))
create_tgz(self.git_root, self.tgz_dir, self.git_commit_id,
self.relative_project_dir,
os.path.join(self.rpmbuild_sourcedir, self.tgz_filename))
# Extract the source so we can get at the spec file, etc.
debug("Copying git source to: %s" % self.rpmbuild_gitcopy)
run_command("cd %s/ && tar xzf %s" % (self.rpmbuild_sourcedir,
self.tgz_filename))
# Find the gemspec
gemspec_filename = find_gemspec_file(self.rpmbuild_gitcopy)
debug("Building gem: %s in %s" % (gemspec_filename,
self.rpmbuild_gitcopy))
# FIXME - this is ugly and should probably be handled better
cmd = "gem_name=$(cd %s/ && gem build %s | awk '/File/ {print $2}'); \
cp %s/$gem_name %s/" % (self.rpmbuild_gitcopy, gemspec_filename,
self.rpmbuild_gitcopy, self.rpmbuild_sourcedir)
run_command(cmd)
# NOTE: The spec file we actually use is the one exported by git
# archive into the temp build directory. This is done so we can
# modify the version/release on the fly when building test rpms
# that use a git SHA1 for their version.
self.spec_file_name = find_spec_file(self.rpmbuild_gitcopy)
self.spec_file = os.path.join(
self.rpmbuild_gitcopy, self.spec_file_name)
def tgz(self):
destination_file = os.path.join(self.rpmbuild_basedir, self.tgz_filename)
formatted_properties = ["-D%s" % x for x in self.maven_properties]
run_command("git clone --no-hardlinks %s %s" % (find_git_root(), self.maven_clone_dir))
with chdir(self.maven_clone_dir):
run_command("git checkout %s" % self.git_commit_id)
try:
info_out("Running Maven build...")
# We always want to deploy to a tito controlled location during local builds
local_properties = formatted_properties + [
"-DaltDeploymentRepository=local-output::default::file://%s" % self.deploy_dir]
run_command("mvn %s %s deploy" % (
" ".join(self.maven_args),
" ".join(local_properties)))
except RunCommandException as e:
error_out("Maven build failed! %s" % e.output)
self._create_build_dirs()
full_path = self._find_tarball()
if full_path:
fh = gzip.open(full_path, 'rb')
fixed_tar = os.path.join(os.path.splitext(full_path)[0])
fixed_tar_fh = open(fixed_tar, 'wb')
timestamp = get_commit_timestamp(self.git_commit_id)
try:
tarfixer = TarFixer(fh, fixed_tar_fh, timestamp, self.git_commit_id, maven_built=True)
tarfixer.fix()
finally:
fixed_tar_fh.close()
# It's a pity we can't use Python's gzip, but it doesn't offer an equivalent of -n
run_command("gzip -n -c < %s > %s" % (fixed_tar, destination_file))
else:
warn_out([
"No Maven generated tarball found.",
"Please set up the assembly plugin in your pom.xml to generate a .tar.gz"])
full_path = os.path.join(self.rpmbuild_sourcedir, self.tgz_filename)
create_tgz(self.git_root, self.tgz_dir, self.git_commit_id, self.relative_project_dir, full_path)
print("Creating %s from git tag: %s..." % (self.tgz_filename, self.build_tag))
shutil.copy(full_path, destination_file)
debug("Copying git source to: %s" % self.rpmbuild_gitcopy)
shutil.copy(destination_file, self.rpmbuild_gitcopy)
# Extract the source so we can get at the spec file, etc.
with chdir(self.rpmbuild_gitcopy):
run_command("tar --strip-components=1 -xvf %s" % os.path.join(self.rpmbuild_gitcopy, self.tgz_filename))
if self.local_build:
artifacts = {}
all_artifacts = []
all_artifacts_with_path = []
for directory, unused, filenames in os.walk(self.deploy_dir):
for f in filenames:
artifacts.setdefault(os.path.splitext(f)[1], []).append(f)
dir_artifacts_with_path = [os.path.join(directory, f) for f in filenames]
# Place the Maven artifacts in the SOURCES directory for rpmbuild to use
for artifact in dir_artifacts_with_path:
shutil.copy(artifact, self.rpmbuild_sourcedir)
dir_artifacts_with_path = map(lambda x: os.path.relpath(x, self.deploy_dir), dir_artifacts_with_path)
all_artifacts_with_path.extend(dir_artifacts_with_path)
all_artifacts.extend([os.path.basename(f) for f in filenames])
cheetah_input = {
'name': self.project_name,
'version': self.spec_version,
'release': self.spec_release,
'epoch': None, # TODO: May need to support this at some point
'artifacts': artifacts,
'all_artifacts': all_artifacts,
'all_artifacts_with_path': all_artifacts_with_path,
}
debug("Cheetah input: %s" % cheetah_input)
render_cheetah(find_cheetah_template_file(self.start_dir), self.rpmbuild_gitcopy, cheetah_input)
self.spec_file_name = find_spec_file(self.rpmbuild_gitcopy)
else:
self.spec_file_name = find_cheetah_template_file(self.rpmbuild_gitcopy)
# NOTE: The spec file we actually use is the one exported by git
# archive into the temp build directory. This is done so we can
# modify the version/release on the fly when building test rpms
# that use a git SHA1 for their version.
self.spec_file = os.path.join(self.rpmbuild_gitcopy, self.spec_file_name)
info_out("Wrote: %s" % destination_file)
self.sources.append(destination_file)
self.artifacts.append(destination_file)
self.ran_tgz = True
def tgz(self):
destination_file = os.path.join(self.rpmbuild_basedir, self.tgz_filename)
formatted_properties = ["-D%s" % x for x in self.maven_properties]
run_command("git clone --no-hardlinks %s %s" % (find_git_root(), self.maven_clone_dir))
with chdir(self.maven_clone_dir):
run_command("git checkout %s" % self.git_commit_id)
try:
info_out("Running Maven build...")
# We always want to deploy to a tito controlled location during local builds
local_properties = formatted_properties + [
"-DaltDeploymentRepository=local-output::default::file://%s" % self.deploy_dir]
run_command("mvn %s %s deploy" % (
" ".join(self.maven_args),
" ".join(local_properties)))
except RunCommandException as e:
error_out("Maven build failed! %s" % e.output)
self._create_build_dirs()
full_path = self._find_tarball()
if full_path:
fh = gzip.open(full_path, 'rb')
fixed_tar = os.path.join(os.path.splitext(full_path)[0])
fixed_tar_fh = open(fixed_tar, 'wb')
timestamp = get_commit_timestamp(self.git_commit_id)
try:
tarfixer = TarFixer(fh, fixed_tar_fh, timestamp, self.git_commit_id, maven_built=True)
tarfixer.fix()
finally:
fixed_tar_fh.close()
# It's a pity we can't use Python's gzip, but it doesn't offer an equivalent of -n
run_command("gzip -n -c < %s > %s" % (fixed_tar, destination_file))
else:
warn_out([
"No Maven generated tarball found.",
"Please set up the assembly plugin in your pom.xml to generate a .tar.gz"])
full_path = os.path.join(self.rpmbuild_sourcedir, self.tgz_filename)
create_tgz(self.git_root, self.tgz_dir, self.git_commit_id, self.relative_project_dir, full_path)
print("Creating %s from git tag: %s..." % (self.tgz_filename, self.build_tag))
shutil.copy(full_path, destination_file)
debug("Copying git source to: %s" % self.rpmbuild_gitcopy)
shutil.copy(destination_file, self.rpmbuild_gitcopy)
# Extract the source so we can get at the spec file, etc.
with chdir(self.rpmbuild_gitcopy):
run_command("tar --strip-components=1 -xvf %s" % os.path.join(self.rpmbuild_gitcopy, self.tgz_filename))
if self.local_build:
artifacts = {}
all_artifacts = []
all_artifacts_with_path = []
for directory, unused, filenames in os.walk(self.deploy_dir):
for f in filenames:
artifacts.setdefault(os.path.splitext(f)[1], []).append(f)
dir_artifacts_with_path = [os.path.join(directory, f) for f in filenames]
# Place the Maven artifacts in the SOURCES directory for rpmbuild to use
for artifact in dir_artifacts_with_path:
shutil.copy(artifact, self.rpmbuild_sourcedir)
dir_artifacts_with_path = map(lambda x: os.path.relpath(x, self.deploy_dir), dir_artifacts_with_path)
all_artifacts_with_path.extend(dir_artifacts_with_path)
all_artifacts.extend([os.path.basename(f) for f in filenames])
cheetah_input = {
'name': self.project_name,
'version': self.spec_version,
'release': self.spec_release,
'epoch': None, # TODO: May need to support this at some point
'artifacts': artifacts,
'all_artifacts': all_artifacts,
'all_artifacts_with_path': all_artifacts_with_path,
}
debug("Cheetah input: %s" % cheetah_input)
render_cheetah(find_cheetah_template_file(self.start_dir), self.rpmbuild_gitcopy, cheetah_input)
self.spec_file_name = find_spec_file(self.rpmbuild_gitcopy)
else:
self.spec_file_name = find_cheetah_template_file(self.rpmbuild_gitcopy)
# NOTE: The spec file we actually use is the one exported by git
# archive into the temp build directory. This is done so we can
# modify the version/release on the fly when building test rpms
# that use a git SHA1 for their version.
self.spec_file = os.path.join(self.rpmbuild_gitcopy, self.spec_file_name)
info_out("Wrote: %s" % destination_file)
self.sources.append(destination_file)
self.artifacts.append(destination_file)
self.ran_tgz = True
