def push_tag(self): if not self.tag: raise ValueError('Must run set_nvr() before calling!') with Dir(self.source_path): exectools.cmd_assert('git push origin {}'.format(self.tag), retries=3)
def tito_setup(self): if self.config.content.build.use_source_tito_config: return # rely on tito already set up in source tito_dir = os.path.join(self.source_path, '.tito') tito_target = self.config.content.build.tito_target tito_target = tito_target if tito_target else 'aos' tito_dist = self.config.content.build.tito_dist tito_dist = tito_dist if tito_dist else '.el7aos' with Dir(self.source_path): if not os.path.isdir(tito_dir): exectools.cmd_assert('tito init') # roll the init changes into the others exectools.cmd_assert('git reset HEAD~') with open(os.path.join(tito_dir, 'releasers.conf'), 'w') as r: r.write( RELEASERS_CONF.format( branch=self.runtime.branch, name=self.name, target=tito_target, dist=tito_dist, )) r.flush() # fix for tito 0.6.10 which looks like remote_git_name in wrong place with open(os.path.join(tito_dir, 'tito.props'), 'a') as props: props.write( TITO_PROPS.format(name=self.name, target=tito_target)) props.flush()
def commit(self, msg): """ Commit outstanding metadata config changes """ self.runtime.logger.info('Commit config: {}'.format(msg)) with Dir(self.runtime.metadata_dir): exectools.cmd_assert(["git", "add", "."]) exectools.cmd_assert(["git", "commit", "--allow-empty", "-m", msg])
def push(self): """ Push changes back to config repo. Will of course fail if user does not have write access. """ self.runtime.logger.info('Pushing config...') with Dir(self.runtime.metadata_dir): exectools.cmd_assert(["git", "push"])
def create_tag(self, scratch): if not self.tag: raise ValueError('Must run set_nvr() before calling!') with Dir(self.source_path): if not scratch: exectools.cmd_assert('git tag {}'.format(self.tag)) rc, sha, err = exectools.cmd_gather('git rev-parse HEAD') self.commit_sha = sha.strip()
def test_cmd_assert_fail(self): """ """ # Try a failing command 3 times, at 1 sec intervals with self.assertRaises(IOError): exectools.cmd_assert("/usr/bin/false", 3, 1) # check that the log file has all of the tests. log_file = open(self.test_file, 'r') lines = log_file.readlines() log_file.close() self.assertEquals(len(lines), 12)
def test_cmd_assert_success(self): """ """ try: exectools.cmd_assert("/bin/true") except IOError as error: self.Fail("/bin/truereturned failure: {}".format(error)) # check that the log file has all of the tests. log_file = open(self.test_file, 'r') lines = log_file.readlines() log_file.close() self.assertEquals(len(lines), 4)
def tito_setup(self): tito_dir = os.path.join(self.source_path, '.tito') with Dir(self.source_path): if not os.path.isdir(tito_dir): exectools.cmd_assert('tito init') with open(os.path.join(tito_dir, 'releasers.conf'), 'w') as r: r.write(RELEASERS_CONF.format(branch=self.runtime.branch, name=self.name)) r.flush() # fix for tito 0.6.10 which looks like remote_git_name in wrong place with open(os.path.join(tito_dir, 'tito.props'), 'a') as props: props.write(TITO_PROPS.format(name=self.name)) props.flush()
def builds_for_group_branch(self): # return a dict of all the latest builds for this group, according to # the branch's candidate tag in brew. each entry is name => tuple(version, release). output, _ = exectools.cmd_assert( "brew list-tagged --quiet --latest {}-candidate".format( self.branch), retries=3, ) builds = [ # each line like "build tag owner" split into build NVR line.split()[0].rsplit("-", 2) for line in output.strip().split("\n") if line.strip() ] return {n: (v, r) for n, v, r in builds}
def commit_changes(self, scratch): if not self.tag: raise ValueError('Must run set_nvr() before calling!') with Dir(self.source_path): if self.config.content.build.use_source_tito_config: # just use the tito tagger to change spec and tag exectools.cmd_assert([ "tito", "tag", "--no-auto-changelog", "--use-version", self.version, "--use-release", "{}%{{?dist}}".format(self.release) ]) if self.config.content.build.push_release_commit and not scratch: exectools.cmd_assert("git push origin") return exectools.cmd_assert( ["git", "tag", "-am", "Release with doozer", self.tag]) exectools.cmd_assert("git add .") commit_msg = "Automatic commit of package [{name}] release [{version}-{release}].".format( name=self.config.name, version=self.version, release=self.release) exectools.cmd_assert(['git', 'commit', '-m', commit_msg])
def _get_remote_branch_ref(self, git_url, branch): """Detect whether a single branch exists on a remote repo; returns git hash if found""" self.logger.info('Checking if target branch {} exists in {}'.format( branch, git_url)) try: out, _ = exectools.cmd_assert('git ls-remote --heads {} {}'.format( git_url, branch), retries=3) except Exception as err: self.logger.error( 'Unable to check if target branch {} exists: {}'.format( branch, err)) return None result = out.strip() # any result means the branch is found return result.split()[0] if result else None
def post_build(self, scratch): build_spec = self.config.content.build with Dir(self.source_path): if self.build_status and not scratch: # success; push the tag try: self.push_tag() except Exception: raise RuntimeError( 'Build succeeded but failure pushing RPM tag for {}'. format(self.qualified_name)) elif build_spec.use_source_tito_config: # don't leave the tag/commit lying around if not a valid build exectools.cmd_assert("tito tag --undo") if not build_spec.push_release_commit and not build_spec.use_source_tito_config: # temporary tag/commit; never leave lying around exectools.cmd_assert("git reset --hard HEAD~") exectools.cmd_assert("git tag -d {}".format(self.tag))
def resolve_source(self, parent, meta): """ Looks up a source alias and returns a path to the directory containing that source. Sources can be specified on the command line, or, failing that, in group.yml. If a source specified in group.yaml has not be resolved before, this method will clone that source to checkout the group's desired branch before returning a path to the cloned repo. :param parent: Name of parent the source belongs to :param meta: The MetaData object to resolve source for :return: Returns the source path """ source = meta.config.content.source # This allows passing `--source <distgit_key> path` to # override any source to something local without it # having been configured for an alias if self.local and meta.distgit_key in self.source_paths: source['alias'] = meta.distgit_key if 'git' in source: del source['git'] source_details = None if 'git' in source: git_url = urlparse.urlparse(source.git.url) name = os.path.splitext(os.path.basename(git_url.path))[0] alias = '{}_{}'.format(parent, name) source_details = dict(source.git) elif 'alias' in source: alias = source.alias else: raise DoozerFatalError( 'Error while processing source for {}'.format(parent)) self.logger.debug( "Resolving local source directory for alias {}".format(alias)) if alias in self.source_paths: self.logger.debug( "returning previously resolved path for alias {}: {}".format( alias, self.source_paths[alias])) return self.source_paths[alias] # Where the source will land, check early so we know if old or new style sub_path = '{}{}'.format('global_' if source_details is None else '', alias) source_dir = os.path.join(self.sources_dir, sub_path) if not source_details: # old style alias was given if (self.group_config.sources is Missing or alias not in self.group_config.sources): raise DoozerFatalError( "Source alias not found in specified sources or in the current group: %s" % alias) source_details = self.group_config.sources[alias] self.logger.debug( "checking for source directory in source_dir: {}".format( source_dir)) # If this source has already been extracted for this working directory if os.path.isdir(source_dir): # Store so that the next attempt to resolve the source hits the map self.source_paths[alias] = source_dir self.logger.info( "Source '{}' already exists in (skipping clone): {}".format( alias, source_dir)) return source_dir clone_branch, _ = self.detect_remote_source_branch(source_details) url = source_details["url"] try: self.logger.info( "Attempting to checkout source '%s' branch %s in: %s" % (url, clone_branch, source_dir)) exectools.cmd_assert( # get a little history to enable finding a recent Dockerfile change, but not too much. "git clone -b {} --single-branch {} --depth 50 {}".format( clone_branch, url, source_dir), retries=3, on_retry=["rm", "-rf", source_dir], ) except IOError as e: self.logger.info("Unable to checkout branch {}: {}".format( clone_branch, e.message)) raise DoozerFatalError( "Error checking out target branch of source '%s' in: %s" % (alias, source_dir)) # Store so that the next attempt to resolve the source hits the map self.register_source_alias(alias, source_dir) return source_dir
def resolve_metadata(self): """ The group control data can be on a local filesystem, in a git repository that can be checked out, or some day in a database If the scheme is empty, assume file:///... Allow http, https, ssh and ssh+git (all valid git clone URLs) """ if self.metadata_dir is None: self.metadata_dir = constants.OCP_BUILD_DATA_RW schemes = ['ssh', 'ssh+git', "http", "https"] self.logger.info('Using {} for metadata'.format(self.metadata_dir)) md_url = urlparse.urlparse(self.metadata_dir) if md_url.scheme in schemes or (md_url.scheme == '' and ':' in md_url.path): # Assume this is a git repo to clone # # An empty scheme with a colon in the path is likely an "scp" style # path: ala [email protected]:owner/path # determine where to put it md_name = os.path.splitext(os.path.basename(md_url.path))[0] md_destination = os.path.join(self.working_dir, md_name) clone_data = True if os.path.isdir(md_destination): self.logger.info( 'Metadata clone directory already exists, checking commit sha' ) with Dir(md_destination): rc, out, err = exectools.cmd_gather( ["git", "ls-remote", self.metadata_dir, "HEAD"]) if rc: raise IOError( 'Unable to check remote sha: {}'.format(err)) remote = out.strip().split('\t')[0] try: exectools.cmd_assert( 'git branch --contains {}'.format(remote)) self.logger.info( '{} is already cloned and latest'.format( self.metadata_dir)) clone_data = False except: rc, out, err = exectools.cmd_gather( 'git log origin/HEAD..HEAD') out = out.strip() if len(out): msg = """ Local config is out of sync with remote and you have unpushed commits. {} You must either clear your local config repo with `./oit.py cleanup` or manually rebase from latest remote to continue """.format(md_destination) raise IOError(msg) if clone_data: if os.path.isdir(md_destination): # delete if already there shutil.rmtree(md_destination) self.logger.info('Cloning config data from {}'.format( self.metadata_dir)) if not os.path.isdir(md_destination): cmd = "git clone --depth 1 {} {}".format( self.metadata_dir, md_destination) try: exectools.cmd_assert(cmd.split(' ')) except: if self.metadata_dir == constants.OCP_BUILD_DATA_RW: self.logger.warn( 'Failed to clone {}, falling back to {}'. format(constants.OCP_BUILD_DATA_RW, constants.OCP_BUILD_DATA_RO)) self.metadata_dir = constants.OCP_BUILD_DATA_RO return self.resolve_metadata() else: raise self.metadata_dir = md_destination elif md_url.scheme in ['', 'file']: # no scheme, assume the path is a local file self.metadata_dir = md_url.path if not os.path.isdir(self.metadata_dir): raise ValueError( "Invalid metadata_dir: {} - Not a directory".format( self.metadata_dir)) else: # invalid scheme: not '' or any of the valid list raise ValueError( "Invalid metadata_dir: {} - invalid scheme: {}".format( self.metadata_dir, md_url.scheme))
def resolve_source(self, alias, required=True): """ Looks up a source alias and returns a path to the directory containing that source. Sources can be specified on the command line, or, failing that, in group.yml. If a source specified in group.yaml has not be resolved before, this method will clone that source to checkout the group's desired branch before returning a path to the cloned repo. :param alias: The source alias to resolve :param required: If True, thrown an exception if not found :return: Returns the source path or None (if required=False) """ self.logger.debug( "Resolving local source directory for alias {}".format(alias)) if alias in self.source_paths: self.logger.debug( "returning previously resolved path for alias {}: {}".format( alias, self.source_paths[alias])) return self.source_paths[alias] # Check if the group config specs the "alias" for the source location if (self.group_config.sources is Missing or alias not in self.group_config.sources): if required: raise IOError( "Source alias not found in specified sources or in the current group: %s" % alias) else: return None # Where the source will land source_dir = os.path.join(self.sources_dir, alias) self.logger.debug( "checking for source directory in source_dir: {}".format( source_dir)) # If this source has already been extracted for this working directory if os.path.isdir(source_dir): # Store so that the next attempt to resolve the source hits the map self.source_paths[alias] = source_dir self.logger.info( "Source '{}' already exists in (skipping clone): {}".format( alias, source_dir)) return source_dir source_config = self.group_config.sources[alias] url = source_config["url"] branches = source_config['branch'] self.logger.info( "Cloning source '%s' from %s as specified by group into: %s" % (alias, url, source_dir)) exectools.cmd_assert( cmd=["git", "clone", url, source_dir], retries=3, on_retry=["rm", "-rf", source_dir], ) stage_branch = branches.get('stage', None) fallback_branch = branches.get("fallback", None) found = False with Dir(source_dir): if self.stage and stage_branch: self.logger.info( 'Normal branch overridden by --stage option, using "{}"'. format(stage_branch)) branch = stage_branch else: branch = branches["target"] self.logger.info( "Attempting to checkout source '%s' branch %s in: %s" % (alias, branch, source_dir)) if branch != "master": rc, out, err = exectools.cmd_gather( ["git", "checkout", "-b", branch, "origin/%s" % branch]) else: rc = 0 if rc == 0: found = True else: if self.stage and stage_branch: raise IOError( '--stage option specified and no stage branch named "{}" exists for {}|{}' .format(stage_branch, alias, url)) elif fallback_branch is not None: self.logger.info( "Unable to checkout branch %s ; trying fallback %s" % (branch, fallback_branch)) self.logger.info( "Attempting to checkout source '%s' fallback-branch %s in: %s" % (alias, fallback_branch, source_dir)) if fallback_branch != "master": rc2, out, err = exectools.cmd_gather([ "git", "checkout", "-b", fallback_branch, "origin/%s" % fallback_branch ], ) else: rc2 = 0 if rc2 == 0: found = True else: self.logger.error( "Failed checking out fallback-branch %s: %s" % (branch, err)) else: self.logger.error("Failed checking out branch %s: %s" % (branch, err)) if found: # Store so that the next attempt to resolve the source hits the map self.register_source_alias(alias, source_dir) return source_dir else: if required: raise IOError( "Error checking out target branch of source '%s' in: %s" % (alias, source_dir)) else: return None
def commit_changes(self): with Dir(self.source_path): exectools.cmd_assert("git add .") exectools.cmd_assert( ['git', 'commit', '-m', "Local commit for dist-git build"])
def update_spec(self): if self.config.content.build.use_source_tito_config: # tito tag will handle updating specfile return # otherwise, make changes similar to tito tagging replace = { 'Name:': 'Name: {}\n'.format(self.config.name), 'Version:': 'Version: {}\n'.format(self.version), 'Release:': 'Release: {}%{{?dist}}\n'.format(self.release), } # self.version example: 3.9.0 # Extract the major, minor, patch major, minor, patch = self.version.split('.') full = "v{}".format(self.version) # If this is a pre-release RPM, the include the release field in # the full version. # pre-release full version: v3.9.0-0.20.1 # release full version: v3.9.0 if self.release.startswith("0."): full += "-{}".format(self.release) replace_keys = replace.keys() with Dir(self.source_path): commit_sha = exectools.cmd_assert('git rev-parse HEAD')[0].strip() # run generic modifications first if self.config.content.source.modifications is not Missing: self._run_modifications() # second, update with NVR with open(self.specfile, 'r+') as sf: lines = sf.readlines() for i in range(len(lines)): if "%global os_git_vars " in lines[i]: lines[ i] = "%global os_git_vars OS_GIT_VERSION={version} OS_GIT_MAJOR={major} OS_GIT_MINOR={minor} OS_GIT_PATCH={patch} OS_GIT_COMMIT={commit} OS_GIT_TREE_STATE=clean\n".format( version=full, major=major, minor=minor, patch=patch, commit=commit_sha) elif "%global commit" in lines[i]: lines[i] = re.sub(r'commit\s+\w+', "commit {}".format(commit_sha), lines[i]) elif replace_keys: # If there are keys left to replace for k in replace_keys: v = replace[k] if lines[i].startswith(k): lines[i] = v replace_keys.remove(k) break # truncate the original file sf.seek(0) sf.truncate() # write back new lines sf.writelines(lines)