def register_source_alias(self, alias, path): self.logger.info("Registering source alias %s: %s" % (alias, path)) path = os.path.abspath(path) assertion.isdir(path, "Error registering source alias %s" % alias) self.source_paths[alias] = path with Dir(path): origin_url = "?" rc1, out_origin, err_origin = exectools.cmd_gather( ["git", "config", "--get", "remote.origin.url"]) if rc1 == 0: origin_url = out_origin.strip() # Usually something like "[email protected]:openshift/origin.git" # But we want an https hyperlink like http://github.com/openshift/origin if origin_url.startswith("git@"): origin_url = origin_url[4:] # remove git@ origin_url = origin_url[:-4] # remove .git origin_url = origin_url.replace(":", "/", 1) # replace first colon with / origin_url = "https://%s" % origin_url else: self.logger.error("Failed acquiring origin url for source alias %s: %s" % (alias, err_origin)) branch = "?" rc2, out_branch, err_branch = exectools.cmd_gather( ["git", "rev-parse", "--abbrev-ref", "HEAD"]) if rc2 == 0: branch = out_branch.strip() else: self.logger.error("Failed acquiring origin branch for source alias %s: %s" % (alias, err_branch)) self.add_record("source_alias", alias=alias, origin_url=origin_url, branch=branch, path=path)
def _build_rpm(self, scratch, record, terminate_event): """ The part of `build_container` which actually starts the build, separated for clarity. """ with Dir(self.source_path): self.logger.info("Building rpm: %s" % self.rpm_name) cmd_list = ['tito', 'release', '--debug', '--yes', '--test'] if scratch: cmd_list.append('--scratch') cmd_list.append('aos') # Run the build with --nowait so that we can immediately get information about the brew task rc, out, err = exectools.cmd_gather(cmd_list) if rc != 0: # Probably no point in continuing.. can't contact brew? self.logger.info("Unable to create brew task: out={} ; err={}".format(out, err)) return False # Otherwise, we should have a brew task we can monitor listed in the stdout. out_lines = out.splitlines() # Look for a line like: "Created task: 13949050" . Extract the identifier. task_id = next((created_line.split(":")[1]).strip() for created_line in out_lines if created_line.startswith("Created task:")) record["task_id"] = task_id # Look for a line like: "Task info: https://brewweb.engineering.redhat.com/brew/taskinfo?taskID=13948942" task_url = next((info_line.split(":", 1)[1]).strip() for info_line in out_lines if info_line.startswith("Task info:")) self.logger.info("Build running: {} - {}".format(self.rpm_name, task_url)) record["task_url"] = task_url # Now that we have the basics about the task, wait for it to complete error = watch_task(self.logger.info, task_id, terminate_event) # Gather brew-logs logs_dir = "%s/%s" % (self.runtime.brew_logs_dir, self.name) logs_rc, _, logs_err = exectools.cmd_gather( ["brew", "download-logs", "-d", logs_dir, task_id]) if logs_rc != 0: self.logger.info("Error downloading build logs from brew for task %s: %s" % (task_id, logs_err)) if error is not None: # An error occurred. We don't have a viable build. self.logger.info("Error building rpm: {}, {}".format(task_url, error)) return False self.logger.info("Successfully built rpm: {} ; {}".format(self.rpm_name, task_url)) return True
def auto_version(self, repo_type): """ Find and return the version of the atomic-openshift package in the OCP RPM repository. This repository is the primary input for OCP images. The group_config for a group specifies the location for both signed and unsigned rpms. The caller must indicate which to use. """ repo_url = self.repos['rhel-server-ose-rpms'].baseurl( repo_type, 'x86_64') self.logger.info( "Getting version from atomic-openshift package in {}".format( repo_url)) # create a randomish repo name to avoid erroneous cache hits repoid = "oit" + datetime.datetime.now().strftime("%s") version_query = [ "/usr/bin/repoquery", "--quiet", "--tempcache", "--repoid", repoid, "--repofrompath", repoid + "," + repo_url, "--queryformat", "%{VERSION}", "atomic-openshift" ] rc, auto_version, err = exectools.cmd_gather(version_query) if rc != 0: raise RuntimeError( "Unable to get OCP version from RPM repository: {}".format( err)) version = "v" + auto_version.strip() self.logger.info("Auto-detected OCP version: {}".format(version)) return version
def get_latest_build_info(self): """ Queries brew to determine the most recently built release of the component associated with this image. This method does not rely on the "release" label needing to be present in the Dockerfile. :return: A tuple: (component name, version, release); e.g. ("registry-console-docker", "v3.6.173.0.75", "1") """ component_name = self.get_component_name() tag = "{}-candidate".format(self.branch()) rc, stdout, stderr = exectools.cmd_gather( ["brew", "latest-build", tag, component_name]) assertion.success(rc, "Unable to search brew builds: %s" % stderr) latest = stdout.strip().splitlines()[-1].split(' ')[0] if not latest.startswith(component_name): # If no builds found, `brew latest-build` output will appear as: # Build Tag Built by # ---------------------------------------- -------------------- ---------------- raise IOError("No builds detected for %s using tag: %s" % (self.qualified_name, tag)) # latest example: "registry-console-docker-v3.6.173.0.75-1"" name, version, release = latest.rsplit( "-", 2) # [ "registry-console-docker", "v3.6.173.0.75", "1"] return name, version, release
def create_tag(self, scratch): if not self.tag: raise ValueError('Must run set_nvr() before calling!') with Dir(self.source_path): if not scratch: exectools.cmd_assert('git tag {}'.format(self.tag)) rc, sha, err = exectools.cmd_gather('git rev-parse HEAD') self.commit_sha = sha.strip()
def get_brew_buildinfo(build): """Get the buildinfo of a brew build from brew. :param str|int build: A build NVR or numeric ID :return dict buildinfo: A dict of the build info. Certain fields may be transformed such as 'Tags' turned into a list, and 'Extra' into a proper dict object instead of a JSON string :return str buildinfo: The raw unparsed buildinfo as a string Note: This is different from get_brew_build in that this function queries brew directly using the 'brew buildinfo' command. Whereas, get_brew_build queries the Errata Tool API for other information. This function will give information not provided by ET: build tags, finished date, built by, etc. """ query_string = "brew buildinfo {nvr}".format(nvr=build.nvr) rc, stdout, stderr = exectools.cmd_gather(shlex.split(query_string)) buildinfo = {} if as_string and rc == 0: return stdout else: buildinfo = {} # These keys indicate content which can be whitespace split into # an array, ex: the value of the "Tags" key comes as a string, but # we can split it on space characters and have a list of tags # instead. split_keys = ['Tags'] # The value of these keys contain json strings which we can # attempt to load into a datastructure json_keys = ['Extra'] for line in stdout.splitlines(): key, token, rest = line.partition(': ') if key in split_keys: buildinfo[key] = rest.split(' ') continue elif key in json_keys: try: # Why would we use the ast module for this? Is # json.loads not enough? Actually, no, it isn't # enough. The <airquotes>JSON</airquotes> returned # from 'brew buildinfo' uses single-quotes to wrap # strings, that is not valid json. Whereas, # ast.literal_eval() can handle that and load the # string into a structure we can work with buildinfo[key] = ast.literal_eval(rest) except Exception as e: buildinfo[key] = rest continue else: buildinfo[key] = rest return buildinfo
def get_tagged_image_builds(tag, latest=True): """Wrapper around shelling out to run 'brew list-tagged' for a given tag. :param str tag: The tag to list builds from :param bool latest: Only show the single latest build of a package """ if latest: latest_option = '--latest' else: latest_option = '' query_string = "brew list-tagged {tag} {latest} --type=image --quiet".format( tag=tag, latest=latest_option) # --latest - Only the last build for that package # --type=image - Only show container images builds # --quiet - Omit field headers in output return exectools.cmd_gather(shlex.split(query_string))
def test_gather_fail(self): """ """ (status, stdout, stderr) = exectools.cmd_gather( ["/usr/bin/sed", "-e", "f"]) status_expected = 1 stdout_expected = "" stderr_expected = "/usr/bin/sed: -e expression #1, char 1: unknown command: `f'\n" self.assertEquals(status_expected, status) self.assertEquals(stdout, stdout_expected) self.assertEquals(stderr, stderr_expected) # check that the log file has all of the tests. log_file = open(self.test_file, 'r') lines = log_file.readlines() self.assertEquals(len(lines), 6)
def test_gather_success(self): """ """ (status, stdout, stderr) = exectools.cmd_gather( "/usr/bin/echo hello there") status_expected = 0 stdout_expected = "hello there\n" stderr_expected = "" self.assertEquals(status_expected, status) self.assertEquals(stdout, stdout_expected) self.assertEquals(stderr, stderr_expected) # check that the log file has all of the tests. log_file = open(self.test_file, 'r') lines = log_file.readlines() self.assertEquals(len(lines), 6)
def get_tagged_rpm_builds(tag, arch='src', latest=True): """Wrapper around shelling out to run 'brew list-tagged' for a given tag. :param str tag: The tag to list builds from :param str arch: Filter results to only this architecture :param bool latest: Only show the single latest build of a package """ if latest is True: latest_flag = "--latest" else: latest_flag = "" query_string = "brew list-tagged {tag} {latest} --rpm --quiet --arch {arch}".format( tag=tag, latest=latest_flag, arch=arch) # --latest - Only the last build for that package # --rpm - Only show RPM builds # --quiet - Omit field headers in output # --arch {arch} - Only show builds of this architecture return exectools.cmd_gather(shlex.split(query_string))
def get_tagged_rpm_names(branch, arch='x86_64'): query_string = "brew list-tagged --inherit --latest --rpms --arch {arch} {branch}-container-build" # --latest - Only the last build for that package # --rpm - Only show RPM builds # --quiet - Omit field headers in output # --arch {arch} - Only show builds of this architecture rpms = [] arches = ['noarch', arch] # always gather noarch for a in arches: rc, stdout, stderr = exectools.cmd_gather( shlex.split(query_string.format(branch=branch, arch=a))) if rc == 0: rpms.extend(stdout.splitlines()) else: raise ValueError(stderr) result = set([splitRPMFilename(line.strip())[0] for line in rpms]) return result
def check_rpm_buildroot(name, branch, arch='x86_64'): """ Query the buildroot used by ODCS to determine if a given RPM name is provided by ODCS for the given arch. :param str name: RPM name :param str branch: Current building branch, such as rhaos-3.10-rhel-7 :param str arch: CPU architecture to search """ args = locals() query = 'repoquery --repofrompath foo,"http://download-node-02.eng.bos.redhat.com/brewroot/repos/{branch}-ppc64le-container-build/latest/{arch}" --repoid=foo --arch {arch},noarch --whatprovides {name}' rc, stdout, stderr = exectools.cmd_gather(query.format(**args)) if rc == 0: result = [] stdout = stdout.strip() for rpm in stdout.strip().splitlines(): n = rpm.split(':')[0] n = '-'.join(n.split('-')[0:-1]) result.append(n) return result else: raise ValueError(stderr)
def resolve_metadata(self): """ The group control data can be on a local filesystem, in a git repository that can be checked out, or some day in a database If the scheme is empty, assume file:///... Allow http, https, ssh and ssh+git (all valid git clone URLs) """ if self.metadata_dir is None: self.metadata_dir = constants.OCP_BUILD_DATA_RW schemes = ['ssh', 'ssh+git', "http", "https"] self.logger.info('Using {} for metadata'.format(self.metadata_dir)) md_url = urlparse.urlparse(self.metadata_dir) if md_url.scheme in schemes or (md_url.scheme == '' and ':' in md_url.path): # Assume this is a git repo to clone # # An empty scheme with a colon in the path is likely an "scp" style # path: ala [email protected]:owner/path # determine where to put it md_name = os.path.splitext(os.path.basename(md_url.path))[0] md_destination = os.path.join(self.working_dir, md_name) clone_data = True if os.path.isdir(md_destination): self.logger.info( 'Metadata clone directory already exists, checking commit sha' ) with Dir(md_destination): rc, out, err = exectools.cmd_gather( ["git", "ls-remote", self.metadata_dir, "HEAD"]) if rc: raise IOError( 'Unable to check remote sha: {}'.format(err)) remote = out.strip().split('\t')[0] try: exectools.cmd_assert( 'git branch --contains {}'.format(remote)) self.logger.info( '{} is already cloned and latest'.format( self.metadata_dir)) clone_data = False except: rc, out, err = exectools.cmd_gather( 'git log origin/HEAD..HEAD') out = out.strip() if len(out): msg = """ Local config is out of sync with remote and you have unpushed commits. {} You must either clear your local config repo with `./oit.py cleanup` or manually rebase from latest remote to continue """.format(md_destination) raise IOError(msg) if clone_data: if os.path.isdir(md_destination): # delete if already there shutil.rmtree(md_destination) self.logger.info('Cloning config data from {}'.format( self.metadata_dir)) if not os.path.isdir(md_destination): cmd = "git clone --depth 1 {} {}".format( self.metadata_dir, md_destination) try: exectools.cmd_assert(cmd.split(' ')) except: if self.metadata_dir == constants.OCP_BUILD_DATA_RW: self.logger.warn( 'Failed to clone {}, falling back to {}'. format(constants.OCP_BUILD_DATA_RW, constants.OCP_BUILD_DATA_RO)) self.metadata_dir = constants.OCP_BUILD_DATA_RO return self.resolve_metadata() else: raise self.metadata_dir = md_destination elif md_url.scheme in ['', 'file']: # no scheme, assume the path is a local file self.metadata_dir = md_url.path if not os.path.isdir(self.metadata_dir): raise ValueError( "Invalid metadata_dir: {} - Not a directory".format( self.metadata_dir)) else: # invalid scheme: not '' or any of the valid list raise ValueError( "Invalid metadata_dir: {} - invalid scheme: {}".format( self.metadata_dir, md_url.scheme))
def resolve_source(self, alias, required=True): """ Looks up a source alias and returns a path to the directory containing that source. Sources can be specified on the command line, or, failing that, in group.yml. If a source specified in group.yaml has not be resolved before, this method will clone that source to checkout the group's desired branch before returning a path to the cloned repo. :param alias: The source alias to resolve :param required: If True, thrown an exception if not found :return: Returns the source path or None (if required=False) """ self.logger.debug( "Resolving local source directory for alias {}".format(alias)) if alias in self.source_paths: self.logger.debug( "returning previously resolved path for alias {}: {}".format( alias, self.source_paths[alias])) return self.source_paths[alias] # Check if the group config specs the "alias" for the source location if (self.group_config.sources is Missing or alias not in self.group_config.sources): if required: raise IOError( "Source alias not found in specified sources or in the current group: %s" % alias) else: return None # Where the source will land source_dir = os.path.join(self.sources_dir, alias) self.logger.debug( "checking for source directory in source_dir: {}".format( source_dir)) # If this source has already been extracted for this working directory if os.path.isdir(source_dir): # Store so that the next attempt to resolve the source hits the map self.source_paths[alias] = source_dir self.logger.info( "Source '{}' already exists in (skipping clone): {}".format( alias, source_dir)) return source_dir source_config = self.group_config.sources[alias] url = source_config["url"] branches = source_config['branch'] self.logger.info( "Cloning source '%s' from %s as specified by group into: %s" % (alias, url, source_dir)) exectools.cmd_assert( cmd=["git", "clone", url, source_dir], retries=3, on_retry=["rm", "-rf", source_dir], ) stage_branch = branches.get('stage', None) fallback_branch = branches.get("fallback", None) found = False with Dir(source_dir): if self.stage and stage_branch: self.logger.info( 'Normal branch overridden by --stage option, using "{}"'. format(stage_branch)) branch = stage_branch else: branch = branches["target"] self.logger.info( "Attempting to checkout source '%s' branch %s in: %s" % (alias, branch, source_dir)) if branch != "master": rc, out, err = exectools.cmd_gather( ["git", "checkout", "-b", branch, "origin/%s" % branch]) else: rc = 0 if rc == 0: found = True else: if self.stage and stage_branch: raise IOError( '--stage option specified and no stage branch named "{}" exists for {}|{}' .format(stage_branch, alias, url)) elif fallback_branch is not None: self.logger.info( "Unable to checkout branch %s ; trying fallback %s" % (branch, fallback_branch)) self.logger.info( "Attempting to checkout source '%s' fallback-branch %s in: %s" % (alias, fallback_branch, source_dir)) if fallback_branch != "master": rc2, out, err = exectools.cmd_gather([ "git", "checkout", "-b", fallback_branch, "origin/%s" % fallback_branch ], ) else: rc2 = 0 if rc2 == 0: found = True else: self.logger.error( "Failed checking out fallback-branch %s: %s" % (branch, err)) else: self.logger.error("Failed checking out branch %s: %s" % (branch, err)) if found: # Store so that the next attempt to resolve the source hits the map self.register_source_alias(alias, source_dir) return source_dir else: if required: raise IOError( "Error checking out target branch of source '%s' in: %s" % (alias, source_dir)) else: return None
def _cmd(self, cmd): """Run a docker related command""" return exectools.cmd_gather(shlex.split(cmd))