Example #1
0
def new_sources(branch=None, fedpkg=FEDPKG, new_sources=False):
    _ensure_branch(branch)
    if not new_sources:
        return
    sources = specfile.Spec().get_source_fns()
    cmd = fedpkg + ['new-sources'] + sources
    run(*cmd, direct=True)
Example #2
0
def new_sources(branch=None, fedpkg=FEDPKG, new_sources=False):
    _ensure_branch(branch)
    if not new_sources:
        return
    sources = specfile.Spec().get_source_fns()
    cmd = fedpkg + ['new-sources'] + sources
    run(*cmd, direct=True)
Example #3
0
def rpmlint_check(*args):
    # run rpmlint and return linting hints it found
    hints = []
    cmd = ['rpmlint']
    if args:
        cmd += args
    try:
        out = run(*cmd, fatal=False, log_fail=False)
    except exception.CommandNotFound:
        raise exception.CommandNotFound(
            msg="Unable to run rpmlint checks because rpmlint is missing.")
    for line in out.splitlines():
        m = re.match(RE_RPMLINT_HINT, line)
        if m:
            hints.append(LintHint(location=m.group(1),
                                  level=m.group(2),
                                  msg=m.group(3)))
            continue
        m = re.match(RE_RPMLINT_SUMMARY, line)
        if m:
            # ignore final rpmlint summary
            continue
        hints.append(LintHint(location='rpmlint', level='W', msg=(
            'Failed to parse rpmlint output: %s' % line)))
    return hints
Example #4
0
 def new_update(self, update, check_availability=True):
     update_id, upfile_path = self._get_new_update_id()
     updir_path, _ = os.path.split(upfile_path)
     helpers.ensure_dir(updir_path)
     upfile = file(upfile_path, 'wt')
     upfile.write(update.update_file())
     upfile.close()
     helpers.edit(upfile_path)
     cmd.run('/bin/sed', '-i', '-e', '/^#/d', upfile_path, shell=False)
     parsed_update = None
     while not parsed_update:
         try:
             parsed_update = rdoupdate.actions.check_file(upfile_path)
         except Exception as ex:
             print("\n{t.important}Error parsing update file: {t.normal}"
                   "{t.warn}{ex}{t.normal}".format(t=log.term, ex=ex))
             again = raw_input("Do you want to edit? [Yn] ")
             if again and again.lower() != 'y':
                 os.remove(upfile_path)
                 raise exception.UserAbort()
             helpers.edit(upfile_path)
             continue
         if check_availability:
             log.info("Checking availability of updated builds...")
             bad_builds = [x for x in parsed_update.builds
                           if not x.is_available(verbose=True)]
             if bad_builds:
                 builds_str = "\n".join(map(str, bad_builds))
                 print("\n{t.important}Builds below doesn't seem to be "
                       "available:{t.normal}\n{builds}\n".format(
                       t=log.term, builds=builds_str))
                 print("Options:")
                 print("  e: Edit update")
                 print("  u: Update anyway")
                 print("  a: Abort")
                 resp = raw_input("What do you want to do? [euA] ").lower()
                 if resp == 'e':
                     parsed_update = None
                     helpers.edit(upfile_path)
                 elif resp == 'u':
                     pass
                 else:
                     raise exception.UserAbort()
     print("\nUpdate:\n%s\n" % parsed_update)
     self.submit_update_file(update_id, msg=str(parsed_update))
Example #5
0
 def __call__(self, *params, **kwargs):
     # allows us to run git in isolated mode, avoiding interaction with user
     # specific config, like ~/.git-templates/hooks (used for testing)
     if kwargs.get('isolated', False):
         env = kwargs.get('env', os.environ.copy())
         env['GIT_CONFIG_NOSYSTEM'] = '1'
         env['GIT_CONFIG_NOGLOBAL'] = '1'
         kwargs['env'] = env
     return run(self.command, *params, **kwargs)
Example #6
0
 def __call__(self, *params, **kwargs):
     # allows us to run git in isolated mode, avoiding interaction with user
     # specific config, like ~/.git-templates/hooks (used for testing)
     if kwargs.get('isolated', False):
         env = kwargs.get('env', os.environ.copy())
         env['GIT_CONFIG_NOSYSTEM'] = '1'
         env['GIT_CONFIG_NOGLOBAL'] = '1'
         kwargs['env'] = env
     return run(self.command, *params, **kwargs)
Example #7
0
def repoquery(repo_url, repo_name, package, verbose=False):
    repo_id = "rdopkg_%s" % re.sub('[^\w]', '_', repo_name)
    cmd = ["repoquery", "--nvr",
           "--repofrompath=%s,%s" % (repo_id, repo_url),
           "--repoid=%s" % repo_id, "-q", package]
    try:
        o = run(*cmd, log_cmd=verbose, log_fail=verbose)
    except Exception:
        return None
    lines = o.strip().split("\n")
    return lines[0] or None
Example #8
0
    def sign_packages(self):
        repos = defaultdict(set)

        def _sign_pkg(upf):
            update = self._load_update_file(upf)
            for build in update.builds:
                build_path = self._build_tmp_path(upf, build)
                repos[build.repo].add(build_path)

        self._run_on_each(_sign_pkg, 'sign')

        for repo, build_paths in repos.items():
            key = "rdo-%s" % repo
            log.info("Signing with %s..." % log.term.bold(key))
            rpms = set()
            for path in build_paths:
                build_rpms = helpers.find_files(path, ext='.rpm')
                rpms = rpms.union(build_rpms)
            cmd = [self.sign_tool_path, key] + list(rpms)
            run(*cmd, direct=True)
        return self.update_files
Example #9
0
def edit(path):
    editor = os.environ.get('EDITOR')
    if not editor:
        editor = 'vim'
        log.info("$EDITOR not set. Falling back to %s." % editor)
    try:
        r = run(editor, path, direct=True)
    except exception.CommandNotFound:
        raise exception.CommandNotFound(
            msg='Failed to find suitable text editor ({0}).  '
            'Please set $EDITOR environment variable.'.format(editor))
    return r.success
Example #10
0
def edit(path):
    editor = os.environ.get('EDITOR')
    if not editor:
        editor = 'vim'
        log.info("$EDITOR not set. Falling back to %s." % editor)
    try:
        r = run(editor, path, direct=True)
    except exception.CommandNotFound:
        raise exception.CommandNotFound(
            msg='Failed to find suitable text editor ({0}).  '
                'Please set $EDITOR environment variable.'.format(editor))
    return r.success
Example #11
0
    def sign_packages(self):
        repos = defaultdict(set)

        def _sign_pkg(upf):
            update = self._load_update_file(upf)
            for build in update.builds:
                build_path = self._build_tmp_path(upf, build)
                repos[build.repo].add(build_path)

        self._run_on_each(_sign_pkg, 'sign')

        for repo, build_paths in repos.items():
            key = "rdo-%s" % repo
            log.info("Signing with %s..." % log.term.bold(key))
            rpms = set()
            for path in build_paths:
                build_rpms = helpers.find_files(path, ext='.rpm')
                rpms = rpms.union(build_rpms)
            cmd = [self.sign_tool_path, key] + list(rpms)
            run(*cmd, direct=True)
        return self.update_files
Example #12
0
def repoquery(repo_url, repo_name, package, verbose=False):
    repo_id = "rdopkg_%s" % re.sub('[^\w]', '_', repo_name)
    cmd = [
        "repoquery", "--nvr",
        "--repofrompath=%s,%s" % (repo_id, repo_url),
        "--repoid=%s" % repo_id, "-q", package
    ]
    try:
        o = run(*cmd, log_cmd=verbose, log_fail=verbose)
    except Exception:
        return None
    lines = o.strip().split("\n")
    return lines[0] or None
Example #13
0
def make_srpm(package, dist=None, fedpkg=FEDPKG):
    cmd = list(fedpkg)
    if dist:
        dname, _, drls = dist.partition('-')
        if dname == 'epel' and drls:
            cmd += ['--dist', 'el' + drls]
    cmd.append('srpm')
    out = run(*cmd)
    m = re.search(r'/([^/\\]+\.src.rpm)\b', out)
    if not m:
        raise exception.CommandOutputParseError(tool=cmd[0], output=out)
    srpm = m.group(1)
    if not os.path.isfile(srpm):
        raise exception.FileNotFound(path=srpm)
    return {'srpm': srpm}
Example #14
0
def make_srpm(package, dist=None, fedpkg=FEDPKG):
    cmd = list(fedpkg)
    if dist:
        dname, _, drls = dist.partition('-')
        if dname == 'epel' and drls:
            cmd += ['--dist', 'el' + drls]
    cmd.append('srpm')
    out = run(*cmd)
    m = re.search(r'/([^/\\]+\.src.rpm)\b', out)
    if not m:
        raise exception.CommandOutputParseError(tool=cmd[0], output=out)
    srpm = m.group(1)
    if not os.path.isfile(srpm):
        raise exception.FileNotFound(path=srpm)
    return {'srpm': srpm}
Example #15
0
def upload_fpo(pkg, user):
    dst_host = user + '@fedorapeople.org'
    dst_path = '~/public_html/copr'
    dst = '%s:%s/%s' % (dst_host, dst_path, pkg)
    _cmd = ['scp', pkg, dst]
    url = fpo_url(pkg, user)
    try:
        cmd.run(*_cmd)
    except exception.CommandFailed as ex:
        err = ex.kwargs['out'].stderr
        # TODO: fragile, use paramiko instead?
        if not re.match('scp: .*No such file or directory', err):
            raise
        log.info("Creating remote dir: %s:%s" % (dst_host, dst_path))
        cmd.run('ssh', dst_host, 'mkdir -p ' + dst_path)
        cmd.run(*_cmd)
    return url
Example #16
0
def upload_fpo(pkg, user):
    dst_host = user + '@fedorapeople.org'
    dst_path = '~/public_html/copr'
    dst = '%s:%s/%s' % (dst_host, dst_path, pkg)
    _cmd = ['scp', pkg, dst]
    url = fpo_url(pkg, user)
    try:
        cmd.run(*_cmd)
    except exception.CommandFailed as ex:
        err = ex.kwargs['out'].stderr
        # TODO: fragile, use paramiko instead?
        if not re.match('scp: .*No such file or directory', err):
            raise
        log.info("Creating remote dir: %s:%s" % (dst_host, dst_path))
        cmd.run('ssh', dst_host, 'mkdir -p ' + dst_path)
        cmd.run(*_cmd)
    return url
Example #17
0
    def push_packages(self):
        updated_repo_bases = set()
        updated_repos = set()

        def _push_pkg(upf):
            log.info("\nPushing update {t.bold}{upf}{t.normal}".format(
                t=log.term, upf=upf))
            update = self._load_update_file(upf)
            pushed_rpms = []
            try:
                _updated_repos = set()
                _updated_repo_bases = set()
                _pushed_build_tmp_paths = []
                for build in update.builds:
                    src_path = self._build_tmp_path(upf, build)
                    if src_path in _pushed_build_tmp_paths:
                        continue
                    build_rpms = helpers.find_files(src_path, ext='.rpm')
                    dest_repo_base_path = self._dest_repo_base_path(build.repo)
                    if not os.path.isdir(dest_repo_base_path):
                        raise exception.NotADirectory(path=dest_repo_base_path)
                    dest_path = self._build_dest_path(build)
                    for rpm in build_rpms:
                        pushed_path = copy_package(rpm,
                                                   dest_path,
                                                   overwrite=self.overwrite)
                        pushed_rpms.append(pushed_path)
                    _pushed_build_tmp_paths.append(src_path)
                    _updated_repo_bases.add(dest_repo_base_path)
                    _updated_repos.add(
                        self._dest_repo_path(build.repo, build.dist))

                with helpers.cdir(self.update_repo_path):
                    helpers.ensure_dir(self.pushed_dir)
                    upf_base = os.path.basename(upf)
                    pushed_upf = os.path.join(self.pushed_dir, upf_base)
                    pushed_files_fn = pushed_upf + self.pushed_files_ext
                    git('mv', upf, pushed_upf)
                    pushed_files_f = open(pushed_files_fn, 'w')
                    pushed_files_f.writelines(
                        map(lambda x: "%s\n" % x, pushed_rpms))
                    pushed_files_f.close()
                    git('add', pushed_files_fn)
                    try:
                        git('commit', '-m',
                            "Push %s" % rdoupdate.core.pp_update(upf))
                    except Exception:
                        git('git', 'reset', '--hard')
                        raise
                updated_repos.update(_updated_repos)
                updated_repo_bases.update(_updated_repo_bases)
            except Exception as ex:
                if pushed_rpms:
                    log.warn("Final push failed for %s, cleaning copied "
                             "packages" % upf)
                    for rpm in pushed_rpms:
                        log.info("{t.warn}remove{t.normal} {rpm}".format(
                            t=log.term, rpm=rpm))
                        os.remove(rpm)
                raise

        self._run_on_each(_push_pkg, 'final push')

        if updated_repos:
            log.info("\nRunning {t.cmd}createrepo{t.normal} on updated repos".
                     format(t=log.term))
            for repo in sorted(updated_repos):
                run('createrepo', repo, direct=True)

        return sorted(updated_repo_bases)
Example #18
0
def fedpkg_mockbuild(fedpkg=FEDPKG):
    cmd = list(fedpkg) + ['mockbuild']
    run(*cmd, direct=True)
Example #19
0
def fedpkg_mockbuild(fedpkg=FEDPKG):
    cmd = list(fedpkg) + ['mockbuild']
    run(*cmd, direct=True)
Example #20
0
def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument(
        '--patches-branch',
        help='Specify another local "patches" branch, like "ceph-5.0-rhel-patches-bz12345"',
    )
    args = parser.parse_args()
    spec = specfile.Spec()

    name = spec.get_tag('Name', expand_macros=True)  # "ceph"
    version = spec.get_tag('Version', expand_macros=True)  # "12.2.8"
    orig_commit = spec.get_macro('commit')  # "9e20ef1b14ac70dea53123"

    branch = git.current_branch()  # "ceph-3.2-rhel-7"
    tag_style = guess.version_tag_style(version=version)  # "vX.Y.Z"
    base_tag = guess.version2tag(version, tag_style)  # "v12.2.8"
    osdist = guess.osdist()

    # "ceph-3.2-rhel-patches"
    if args.patches_branch:
        patches_branch = args.patches_branch
    else:
        remote_patches_branch = guess.patches_branch(branch, pkg=name,
                                                     osdist=osdist)
        patches_branch = remote_patches_branch.partition('/')[2]

    patches_sha = git('rev-parse', patches_branch)  # "9e20ef1b14ac70dea53123"

    archive_basename = '%s-%s' % (name, version)  # "ceph-12.2.8"
    patches_base, patches_base_commits = spec.get_patches_base()
    if patches_base_commits != 0:
        # We don't yet support the "+n_commits" syntax for patches_base.
        raise NotImplementedError('use a plain ref in patches_base')
    if patches_base is None:
        patches_base = base_tag
    filenames = diff_filenames(patches_base, patches_branch)
    if not filenames:
        # todo: make this a silent no-op eventually
        log.warning('%s identical to %s' % (patches_branch, patches_base))
        raise RuntimeError(patches_base)

    tarball = archive_files(archive_basename, patches_sha, filenames)
    log.info('wrote %s' % tarball)

    # Ensure our spec file will reference this tarball.
    spec.set_macro('commit', patches_sha)
    set_source1(spec)
    spec.save()

    # Find the changelog entries from the Git -patches branch.
    changes = check_new_commits(patches_base, orig_commit, patches_sha)

    if not changes:
        log.info('no changes. exiting')
        raise SystemExit(1)

    # Bump the release and add the %changelog entries.

    # Insert %changelog.
    rdopkg.actions.distgit.actions.update_spec(branch=branch, changes=changes)

    # add + upload this new tarball.
    if guess.new_sources():
        fedpkg = 'fedpkg'
        if osdist.startswith('RH'):
            fedpkg = 'rhpkg'
        clear_old_changes_sources()
        run(fedpkg, 'upload', tarball, direct=True)

    # Commit everything to dist-git
    rdopkg.actions.distgit.actions.commit_distgit_update(branch=branch,
                                                         local_patches_branch=patches_branch)
    # Show the final commit
    rdopkg.actions.distgit.actions.final_spec_diff(branch=branch)
Example #21
0
    def push_packages(self):
        updated_repo_bases = set()
        updated_repos = set()

        def _push_pkg(upf):
            log.info("\nPushing update {t.bold}{upf}{t.normal}".format(
                t=log.term, upf=upf))
            update = self._load_update_file(upf)
            pushed_rpms = []
            try:
                _updated_repos = set()
                _updated_repo_bases = set()
                _pushed_build_tmp_paths = []
                for build in update.builds:
                    src_path = self._build_tmp_path(upf, build)
                    if src_path in _pushed_build_tmp_paths:
                        continue
                    build_rpms = helpers.find_files(src_path, ext='.rpm')
                    dest_repo_base_path = self._dest_repo_base_path(build.repo)
                    if not os.path.isdir(dest_repo_base_path):
                        raise exception.NotADirectory(path=dest_repo_base_path)
                    dest_path = self._build_dest_path(build)
                    for rpm in build_rpms:
                        pushed_path = copy_package(rpm, dest_path,
                                                   overwrite=self.overwrite)
                        pushed_rpms.append(pushed_path)
                    _pushed_build_tmp_paths.append(src_path)
                    _updated_repo_bases.add(dest_repo_base_path)
                    _updated_repos.add(self._dest_repo_path(build.repo,
                                                            build.dist))

                with helpers.cdir(self.update_repo_path):
                    helpers.ensure_dir(self.pushed_dir)
                    upf_base = os.path.basename(upf)
                    pushed_upf = os.path.join(self.pushed_dir, upf_base)
                    pushed_files_fn = pushed_upf + self.pushed_files_ext
                    git('mv', upf, pushed_upf)
                    pushed_files_f = open(pushed_files_fn, 'w')
                    pushed_files_f.writelines(
                        map(lambda x: "%s\n" % x, pushed_rpms))
                    pushed_files_f.close()
                    git('add', pushed_files_fn)
                    try:
                        git('commit', '-m',
                            "Push %s" % rdoupdate.core.pp_update(upf))
                    except Exception:
                        git('git', 'reset', '--hard')
                        raise
                updated_repos.update(_updated_repos)
                updated_repo_bases.update(_updated_repo_bases)
            except Exception as ex:
                if pushed_rpms:
                    log.warn("Final push failed for %s, cleaning copied "
                             "packages" % upf)
                    for rpm in pushed_rpms:
                        log.info("{t.warn}remove{t.normal} {rpm}".format(
                            t=log.term, rpm=rpm))
                        os.remove(rpm)
                raise

        self._run_on_each(_push_pkg, 'final push')

        if updated_repos:
            log.info("\nRunning {t.cmd}createrepo{t.normal} on updated repos"
                     .format(t=log.term))
            for repo in sorted(updated_repos):
                run('createrepo', repo, direct=True)

        return sorted(updated_repo_bases)
Example #22
0
def download_file(url):
    run('curl', '-L', '-f', '-O', url, direct=True)
Example #23
0
def download_file(url):
    run('curl', '-L', '-f', '-O', url, direct=True)