def _run_untagged_report(self, config): """ Display a report of all packages with differences between HEAD and their most recent tag, as well as a patch for that diff. Used to determine which packages are in need of a rebuild. """ print("Scanning for packages that may need to be tagged...") print("") git_root = find_git_root() rel_eng_dir = os.path.join(git_root, tito_config_dir()) os.chdir(git_root) package_metadata_dir = os.path.join(rel_eng_dir, "packages") for root, dirs, files in os.walk(package_metadata_dir): for md_file in files: if md_file[0] == '.': continue f = open(os.path.join(package_metadata_dir, md_file)) (version, relative_dir) = f.readline().strip().split(" ") # Hack for single project git repos: if relative_dir == '/': relative_dir = "" project_dir = os.path.join(git_root, relative_dir) self._print_diff(config, md_file, version, project_dir, relative_dir)
def main(self, argv): (self.options, args) = self.parser.parse_args(argv) self._validate_options() if len(argv) < 1: print(self.parser.error("Must supply an argument. " "Try -h for help.")) self.global_config = self._read_global_config() if self.global_config.has_option(GLOBALCONFIG_SECTION, "offline"): self.options.offline = True if self.options.debug: os.environ['DEBUG'] = "true" # Check if global config defines a custom lib dir: if self.global_config.has_option(GLOBALCONFIG_SECTION, "lib_dir"): lib_dir = self.global_config.get(GLOBALCONFIG_SECTION, "lib_dir") if lib_dir[0] != '/': # Looks like a relative path, assume from the git root: lib_dir = os.path.join(find_git_root(), lib_dir) if os.path.exists(lib_dir): sys.path.append(lib_dir) debug("Added lib dir to PYTHONPATH: %s" % lib_dir) else: print("WARNING: lib_dir specified but does not exist: %s" % lib_dir)
def __init__(self, global_config=None, keep_version=False, offline=False, user_config=None): self.git_root = find_git_root() self.rel_eng_dir = os.path.join(self.git_root, "rel-eng") self.config = global_config self.user_config = user_config self.full_project_dir = os.getcwd() self.spec_file_name = find_spec_file() self.project_name = get_project_name(tag=None) self.relative_project_dir = self._get_relative_project_dir( self.git_root) # i.e. java/ self.spec_file = os.path.join(self.full_project_dir, self.spec_file_name) self.keep_version = keep_version self.today = strftime("%a %b %d %Y") (self.git_user, self.git_email) = self._get_git_user_info() git_email = self.git_email if git_email is None: git_email = '' self.changelog_regex = re.compile('\\*\s%s\s%s(\s<%s>)?' % (self.today, self.git_user, git_email.replace("+", "\+").replace(".", "\."))) self._no_auto_changelog = False self._accept_auto_changelog = False self._new_changelog_msg = "new package built with tito" self.offline = offline
def main(): """The main function allows converting a git repo the is stup for "normal" tito to use classed defined here instead """ options = parse_args() rel_eng_dir = os.path.join(find_git_root(), tito_config_dir()) releasers_filename = os.path.join(rel_eng_dir, RELEASERS_CONF_FILENAME) releasers_config = RawConfigParser() releasers_config.read(releasers_filename) print("Read configuration file: {0}".format(releasers_filename)) for section in releasers_config.sections(): print(" found section: {0}".format(section)) if releasers_config.has_option(section, 'releaser'): old_releaser = releasers_config.get(section, 'releaser') print(" releaser is set to: {0}".format(old_releaser)) if old_releaser.startswith('tito.release.'): new_releaser = old_releaser.replace('tito.release.', 'mocktito.', 1) print(" replaced with: {0}".format(new_releaser)) releasers_config.set(section, 'releaser', new_releaser) elif old_releaser.startswith('mocktito.'): pass else: stderr.write("Found a releaser type I don't know, aborting\n") exit(1) if options.extra_yum_repos: print(" adding extra yum repos") releasers_config.set(section, EXTRA_YUM_REPOS, "\n".join(options.extra_yum_repos)) if options.extra_yum_repos_for: print(" adding extra yum repos for branches") releasers_config.set(section, EXTRA_YUM_REPOS_FOR, json.dumps(options.extra_yum_repos_for)) with open(releasers_filename, 'w') as rfp: releasers_config.write(rfp)
def load_config(self, package_name, build_dir, tag): self.config = ConfigLoader(package_name, build_dir, tag).load() if self.config.has_option(BUILDCONFIG_SECTION, "offline"): self.options.offline = True # TODO: Not ideal: if self.options.debug: os.environ['DEBUG'] = "true" # Check if config defines a custom lib dir, if so we add it # to the python path allowing users to specify custom builders/taggers # in their config: if self.config.has_option(BUILDCONFIG_SECTION, "lib_dir"): lib_dir = self.config.get(BUILDCONFIG_SECTION, "lib_dir") if lib_dir[0] != '/': # Looks like a relative path, assume from the git root: lib_dir = os.path.join(find_git_root(), lib_dir) if os.path.exists(lib_dir): sys.path.append(lib_dir) debug("Added lib dir to PYTHONPATH: %s" % lib_dir) else: warn_out("lib_dir specified but does not exist: %s" % lib_dir)
def main(self, argv): # DO NOT CALL BaseCliModule.main(self) # we are initializing tito to work in this module and # calling main will result in a configuration error. (self.options, self.args) = self.parser.parse_args(argv) should_commit = False rel_eng_dir = os.path.join(find_git_root(), '.tito') if not os.path.exists(rel_eng_dir): print("Creating tito metadata in: %s" % rel_eng_dir) os.makedirs(rel_eng_dir) print(" - created %s" % rel_eng_dir) else: print("Reinitializing existing tito metadata in %s" % rel_eng_dir) propsfile = os.path.join(rel_eng_dir, TITO_PROPS) if not os.path.exists(propsfile): # write out tito.props out_f = open(propsfile, 'w') out_f.write("[buildconfig]\n") out_f.write("builder = %s\n" % 'tito.builder.Builder') out_f.write("tagger = %s\n" % 'tito.tagger.VersionTagger') out_f.write("changelog_do_not_remove_cherrypick = 0\n") out_f.write("changelog_format = %s (%ae)\n") out_f.close() print(" - wrote %s" % TITO_PROPS) getoutput('git add %s' % propsfile) should_commit = True # prep the packages metadata directory pkg_dir = os.path.join(rel_eng_dir, "packages") readme = os.path.join(pkg_dir, '.readme') if not os.path.exists(readme): if not os.path.exists(pkg_dir): os.makedirs(pkg_dir) print(" - created %s" % pkg_dir) # write out readme file explaining what pkg_dir is for readme = os.path.join(pkg_dir, '.readme') out_f = open(readme, 'w') out_f.write( "the .tito/packages directory contains metadata files\n") out_f.write( "named after their packages. Each file has the latest tagged\n" ) out_f.write("version and the project's relative directory.\n") out_f.close() print(" - wrote %s" % readme) getoutput('git add %s' % readme) should_commit = True if should_commit: getoutput('git commit -m "Initialized to use tito. "') print(" - committed to git") info_out("Done!") return []
def main(): """The main function allows converting a git repo the is stup for "normal" tito to use classed defined here instead """ options = parse_args() rel_eng_dir = os.path.join(find_git_root(), tito_config_dir()) releasers_filename = os.path.join(rel_eng_dir, RELEASERS_CONF_FILENAME) releasers_config = RawConfigParser() releasers_config.read(releasers_filename) print("Read configuration file: {0}".format(releasers_filename)) for section in releasers_config.sections(): print(" found section: {0}".format(section)) if releasers_config.has_option(section, "releaser"): old_releaser = releasers_config.get(section, "releaser") print(" releaser is set to: {0}".format(old_releaser)) if old_releaser.startswith("tito.release."): new_releaser = old_releaser.replace("tito.release.", "mocktito.", 1) print(" replaced with: {0}".format(new_releaser)) releasers_config.set(section, "releaser", new_releaser) elif old_releaser.startswith("mocktito."): pass else: stderr.write("Found a releaser type I don't know, aborting\n") exit(1) if options.extra_yum_repos: print(" adding extra yum repos") releasers_config.set(section, EXTRA_YUM_REPOS, "\n".join(options.extra_yum_repos)) if options.extra_yum_repos_for: print(" adding extra yum repos for branches") releasers_config.set(section, EXTRA_YUM_REPOS_FOR, json.dumps(options.extra_yum_repos_for)) with open(releasers_filename, "w") as rfp: releasers_config.write(rfp)
def _read_global_config(self): """ Read global build.py configuration from the rel-eng dir of the git repository we're being run from. """ rel_eng_dir = os.path.join(find_git_root(), "rel-eng") filename = os.path.join(rel_eng_dir, GLOBAL_BUILD_PROPS_FILENAME) if not os.path.exists(filename): # HACK: Try the old filename location, pre-tito rename: oldfilename = os.path.join(rel_eng_dir, "global.build.py.props") if not os.path.exists(oldfilename): error_out("Unable to locate branch configuration: %s" "\nPlease run 'tito init'" % filename) config = ConfigParser.ConfigParser() config.read(filename) # Verify the config contains what we need from it: required_global_config = [ (GLOBALCONFIG_SECTION, DEFAULT_BUILDER), (GLOBALCONFIG_SECTION, DEFAULT_TAGGER), ] for section, option in required_global_config: if not config.has_section(section) or not \ config.has_option(section, option): error_out("%s missing required config: %s %s" % ( filename, section, option)) return config
def _read_releaser_config(self): """ Read the releaser targets from .tito/releasers.conf. """ rel_eng_dir = os.path.join(find_git_root(), tito_config_dir()) filename = os.path.join(rel_eng_dir, RELEASERS_CONF_FILENAME) config = RawConfigParser() config.read(filename) return config
def main(self, argv): # DO NOT CALL BaseCliModule.main(self) # we are initializing tito to work in this module and # calling main will result in a configuration error. (self.options, self.args) = self.parser.parse_args(argv) should_commit = False rel_eng_dir = os.path.join(find_git_root(), '.tito') print("Creating tito metadata in: %s" % rel_eng_dir) propsfile = os.path.join(rel_eng_dir, TITO_PROPS) if not os.path.exists(propsfile): if not os.path.exists(rel_eng_dir): getoutput("mkdir -p %s" % rel_eng_dir) print(" - created %s" % rel_eng_dir) # write out tito.props out_f = open(propsfile, 'w') out_f.write("[buildconfig]\n") out_f.write("builder = %s\n" % 'tito.builder.Builder') out_f.write( "tagger = %s\n" % 'tito.tagger.VersionTagger') out_f.write("changelog_do_not_remove_cherrypick = 0\n") out_f.write("changelog_format = %s (%ae)\n") out_f.close() print(" - wrote %s" % TITO_PROPS) getoutput('git add %s' % propsfile) should_commit = True # prep the packages metadata directory pkg_dir = os.path.join(rel_eng_dir, "packages") readme = os.path.join(pkg_dir, '.readme') if not os.path.exists(readme): if not os.path.exists(pkg_dir): getoutput("mkdir -p %s" % pkg_dir) print(" - created %s" % pkg_dir) # write out readme file explaining what pkg_dir is for readme = os.path.join(pkg_dir, '.readme') out_f = open(readme, 'w') out_f.write("the .tito/packages directory contains metadata files\n") out_f.write("named after their packages. Each file has the latest tagged\n") out_f.write("version and the project's relative directory.\n") out_f.close() print(" - wrote %s" % readme) getoutput('git add %s' % readme) should_commit = True if should_commit: getoutput('git commit -m "Initialized to use tito. "') print(" - committed to git") info_out("Done!") return []
def main(self, argv): # DO NOT CALL BaseCliModule.main(self) # we are initializing tito to work in this module and # calling main will result in a configuration error. should_commit = False rel_eng_dir = os.path.join(find_git_root(), "rel-eng") print("Creating tito metadata in: %s" % rel_eng_dir) propsfile = os.path.join(rel_eng_dir, GLOBAL_BUILD_PROPS_FILENAME) if not os.path.exists(propsfile): if not os.path.exists(rel_eng_dir): commands.getoutput("mkdir -p %s" % rel_eng_dir) print(" - created %s" % rel_eng_dir) # write out tito.props out_f = open(propsfile, 'w') out_f.write("[globalconfig]\n") out_f.write("default_builder = %s\n" % 'tito.builder.Builder') out_f.write( "default_tagger = %s\n" % 'tito.tagger.VersionTagger') out_f.write("changelog_do_not_remove_cherrypick = 0\n") out_f.write("changelog_with_email = 1\n") out_f.close() print(" - wrote %s" % GLOBAL_BUILD_PROPS_FILENAME) commands.getoutput('git add %s' % propsfile) should_commit = True # prep the packages metadata directory pkg_dir = os.path.join(rel_eng_dir, "packages") readme = os.path.join(pkg_dir, '.readme') if not os.path.exists(readme): if not os.path.exists(pkg_dir): commands.getoutput("mkdir -p %s" % pkg_dir) print(" - created %s" % pkg_dir) # write out readme file explaining what pkg_dir is for readme = os.path.join(pkg_dir, '.readme') out_f = open(readme, 'w') out_f.write("the rel-eng/packages directory contains metadata files\n") out_f.write("named after their packages. Each file has the latest tagged\n") out_f.write("version and the project's relative directory.\n") out_f.close() print(" - wrote %s" % readme) commands.getoutput('git add %s' % readme) should_commit = True if should_commit: commands.getoutput('git commit -m "Initialized to use tito. "') print(" - committed to git") print("Done!") return []
def __init__(self, config=None): """ config - Merged configuration. (global plus package specific) """ self.config = config # Override global configurations using local configurations for section in config.sections(): for options in config.options(section): if not self.config.has_section(section): self.config.add_section(section) self.config.set(section, options, config.get(section, options)) self.git_root = find_git_root() self.rel_eng_dir = os.path.join(self.git_root, "rel-eng")
def __init__(self, config=None): """ config - Merged configuration. (global plus package specific) """ self.config = config # Override global configurations using local configurations for section in config.sections(): for options in config.options(section): if not self.config.has_section(section): self.config.add_section(section) self.config.set(section, options, config.get(section, options)) self.git_root = find_git_root() self.rel_eng_dir = os.path.join(self.git_root, tito_config_dir())
def __init__(self, pkg_config=None, global_config=None): """ pkg_config - Package specific configuration. global_config - Global configuration from rel-eng/tito.props. """ self.config = global_config # Override global configurations using local configurations for section in pkg_config.sections(): for options in pkg_config.options(section): if not self.config.has_section(section): self.config.add_section(section) self.config.set(section, options, pkg_config.get(section, options)) self.git_root = find_git_root() self.rel_eng_dir = os.path.join(self.git_root, "rel-eng")
def get_latest_tagged_version(package_name): """ Return the latest git tag for this package in the current branch. Uses the info in .tito/packages/package-name. Returns None if file does not exist. """ git_root = find_git_root() rel_eng_dir = os.path.join(git_root, tito_config_dir()) file_path = "{0}/packages/{1}".format(rel_eng_dir, package_name) debug("Getting latest package info from: {0}".format(file_path)) if not os.path.exists(file_path): return None output = run_command("awk '{ print $1 ; exit }' {0}".format(file_path)) if output is None or output.strip() == "": error_out("Error looking up latest tagged version in: {0}".format(file_path)) return output
def get_latest_tagged_version(package_name): """ Return the latest git tag for this package in the current branch. Uses the info in .tito/packages/package-name. Returns None if file does not exist. """ git_root = find_git_root() rel_eng_dir = os.path.join(git_root, tito_config_dir()) file_path = "{0}/packages/{1}".format(rel_eng_dir, package_name) debug("Getting latest package info from: {0}".format(file_path)) if not os.path.exists(file_path): return None output = run_command("awk '{ print $1 ; exit }' {0}".format(file_path)) if output is None or output.strip() == "": error_out("Error looking up latest tagged version in: {0}".format( file_path)) return output
def _read_config(self): """ Read global build.py configuration from the .tito dir of the git repository we're being run from. NOTE: We always load the latest config file, not tito.props as it was for the tag being operated on. """ # List of filepaths to config files we'll be loading: rel_eng_dir = os.path.join(find_git_root(), tito_config_dir()) filename = os.path.join(rel_eng_dir, TITO_PROPS) if not os.path.exists(filename): error_out("Unable to locate branch configuration: %s" "\nPlease run 'tito init'" % filename) # Load the global config. Later, when we know what tag/package we're # building, we may also load that and potentially override some global # settings. config = RawConfigParser() config.read(filename) self._check_legacy_globalconfig(config) return config
def __init__(self, name=None, tag=None, build_dir=None, config=None, user_config=None, args=None, **kwargs): """ name - Package name that is being built. version - Version and release being built. tag - The git tag being built. build_dir - Temporary build directory where we can safely work. config - Merged configuration. (global plus package specific) user_config - User configuration from ~/.titorc. args - Optional arguments specific to each builder. Can be passed in explicitly by user on the CLI, or via a release target config entry. Only for things which vary on invocations of the builder, avoid using these if possible. *Given in the format of a dictionary of lists.* """ ConfigObject.__init__(self, config=config) BuilderBase.__init__(self, name=name, build_dir=build_dir, config=config, user_config=user_config, args=args, **kwargs) self.build_tag = tag self.build_version = self._get_build_version() if kwargs and 'options' in kwargs: warn_out("'options' no longer a supported builder constructor argument.") if self.config.has_section("requirements"): if self.config.has_option("requirements", "tito"): if loose_version(self.config.get("requirements", "tito")) > \ loose_version(require('tito')[0].version): error_out([ "tito version %s or later is needed to build this project." % self.config.get("requirements", "tito"), "Your version: %s" % require('tito')[0].version ]) self.display_version = self._get_display_version() with chdir(find_git_root()): self.git_commit_id = get_build_commit(tag=self.build_tag, test=self.test) self.relative_project_dir = get_relative_project_dir( project_name=self.project_name, commit=self.git_commit_id) if self.relative_project_dir is None and self.test: warn_out(".tito/packages/%s doesn't exist " "in git, using current directory" % self.project_name) self.relative_project_dir = get_relative_project_dir_cwd( self.git_root) tgz_base = self._get_tgz_name_and_ver() self.tgz_filename = tgz_base + ".tar.gz" self.tgz_dir = tgz_base self.artifacts = [] # A copy of the git code from commit we're building: self.rpmbuild_gitcopy = os.path.join(self.rpmbuild_sourcedir, self.tgz_dir) # Used to make sure we only modify the spec file for a test build # once. The srpm method may be called multiple times during koji # releases to create the proper disttags, but we only want to modify # the spec file once. self.ran_setup_test_specfile = False # NOTE: These are defined later when/if we actually dump a copy of the # project source at the tag we're building. Only then can we search for # a spec file. self.spec_file_name = None self.spec_file = None # Set to path to srpm once we build one. self.srpm_location = None
def tgz(self): destination_file = os.path.join(self.rpmbuild_basedir, self.tgz_filename) formatted_properties = ["-D%s" % x for x in self.maven_properties] run_command("git clone --no-hardlinks %s %s" % (find_git_root(), self.maven_clone_dir)) with chdir(self.maven_clone_dir): run_command("git checkout %s" % self.git_commit_id) try: info_out("Running Maven build...") # We always want to deploy to a tito controlled location during local builds local_properties = formatted_properties + [ "-DaltDeploymentRepository=local-output::default::file://%s" % self.deploy_dir] run_command("mvn %s %s deploy" % ( " ".join(self.maven_args), " ".join(local_properties))) except RunCommandException as e: error_out("Maven build failed! %s" % e.output) self._create_build_dirs() full_path = self._find_tarball() if full_path: fh = gzip.open(full_path, 'rb') fixed_tar = os.path.join(os.path.splitext(full_path)[0]) fixed_tar_fh = open(fixed_tar, 'wb') timestamp = get_commit_timestamp(self.git_commit_id) try: tarfixer = TarFixer(fh, fixed_tar_fh, timestamp, self.git_commit_id, maven_built=True) tarfixer.fix() finally: fixed_tar_fh.close() # It's a pity we can't use Python's gzip, but it doesn't offer an equivalent of -n run_command("gzip -n -c < %s > %s" % (fixed_tar, destination_file)) else: warn_out([ "No Maven generated tarball found.", "Please set up the assembly plugin in your pom.xml to generate a .tar.gz"]) full_path = os.path.join(self.rpmbuild_sourcedir, self.tgz_filename) create_tgz(self.git_root, self.tgz_dir, self.git_commit_id, self.relative_project_dir, full_path) print("Creating %s from git tag: %s..." % (self.tgz_filename, self.build_tag)) shutil.copy(full_path, destination_file) debug("Copying git source to: %s" % self.rpmbuild_gitcopy) shutil.copy(destination_file, self.rpmbuild_gitcopy) # Extract the source so we can get at the spec file, etc. with chdir(self.rpmbuild_gitcopy): run_command("tar --strip-components=1 -xvf %s" % os.path.join(self.rpmbuild_gitcopy, self.tgz_filename)) if self.local_build: artifacts = {} all_artifacts = [] all_artifacts_with_path = [] for directory, unused, filenames in os.walk(self.deploy_dir): for f in filenames: artifacts.setdefault(os.path.splitext(f)[1], []).append(f) dir_artifacts_with_path = [os.path.join(directory, f) for f in filenames] # Place the Maven artifacts in the SOURCES directory for rpmbuild to use for artifact in dir_artifacts_with_path: shutil.copy(artifact, self.rpmbuild_sourcedir) dir_artifacts_with_path = map(lambda x: os.path.relpath(x, self.deploy_dir), dir_artifacts_with_path) all_artifacts_with_path.extend(dir_artifacts_with_path) all_artifacts.extend([os.path.basename(f) for f in filenames]) cheetah_input = { 'name': self.project_name, 'version': self.spec_version, 'release': self.spec_release, 'epoch': None, # TODO: May need to support this at some point 'artifacts': artifacts, 'all_artifacts': all_artifacts, 'all_artifacts_with_path': all_artifacts_with_path, } debug("Cheetah input: %s" % cheetah_input) render_cheetah(find_cheetah_template_file(self.start_dir), self.rpmbuild_gitcopy, cheetah_input) self.spec_file_name = find_spec_file(self.rpmbuild_gitcopy) else: self.spec_file_name = find_cheetah_template_file(self.rpmbuild_gitcopy) # NOTE: The spec file we actually use is the one exported by git # archive into the temp build directory. This is done so we can # modify the version/release on the fly when building test rpms # that use a git SHA1 for their version. self.spec_file = os.path.join(self.rpmbuild_gitcopy, self.spec_file_name) info_out("Wrote: %s" % destination_file) self.sources.append(destination_file) self.artifacts.append(destination_file) self.ran_tgz = True