def new_sources(branch=None, fedpkg=FEDPKG, new_sources=False):
_ensure_branch(branch)
if not new_sources:
return
sources = specfile.Spec().get_source_fns()
cmd = fedpkg + ['new-sources'] + sources
run(*cmd, direct=True)
def new_sources(branch=None, fedpkg=FEDPKG, new_sources=False):
_ensure_branch(branch)
if not new_sources:
return
sources = specfile.Spec().get_source_fns()
cmd = fedpkg + ['new-sources'] + sources
run(*cmd, direct=True)
def rpmlint_check(*args):
# run rpmlint and return linting hints it found
hints = []
cmd = ['rpmlint']
if args:
cmd += args
try:
out = run(*cmd, fatal=False, log_fail=False)
except exception.CommandNotFound:
raise exception.CommandNotFound(
msg="Unable to run rpmlint checks because rpmlint is missing.")
for line in out.splitlines():
m = re.match(RE_RPMLINT_HINT, line)
if m:
hints.append(LintHint(location=m.group(1),
level=m.group(2),
msg=m.group(3)))
continue
m = re.match(RE_RPMLINT_SUMMARY, line)
if m:
# ignore final rpmlint summary
continue
hints.append(LintHint(location='rpmlint', level='W', msg=(
'Failed to parse rpmlint output: %s' % line)))
return hints
def new_update(self, update, check_availability=True):
update_id, upfile_path = self._get_new_update_id()
updir_path, _ = os.path.split(upfile_path)
helpers.ensure_dir(updir_path)
upfile = file(upfile_path, 'wt')
upfile.write(update.update_file())
upfile.close()
helpers.edit(upfile_path)
cmd.run('/bin/sed', '-i', '-e', '/^#/d', upfile_path, shell=False)
parsed_update = None
while not parsed_update:
try:
parsed_update = rdoupdate.actions.check_file(upfile_path)
except Exception as ex:
print("\n{t.important}Error parsing update file: {t.normal}"
"{t.warn}{ex}{t.normal}".format(t=log.term, ex=ex))
again = raw_input("Do you want to edit? [Yn] ")
if again and again.lower() != 'y':
os.remove(upfile_path)
raise exception.UserAbort()
helpers.edit(upfile_path)
continue
if check_availability:
log.info("Checking availability of updated builds...")
bad_builds = [x for x in parsed_update.builds
if not x.is_available(verbose=True)]
if bad_builds:
builds_str = "\n".join(map(str, bad_builds))
print("\n{t.important}Builds below doesn't seem to be "
"available:{t.normal}\n{builds}\n".format(
t=log.term, builds=builds_str))
print("Options:")
print(" e: Edit update")
print(" u: Update anyway")
print(" a: Abort")
resp = raw_input("What do you want to do? [euA] ").lower()
if resp == 'e':
parsed_update = None
helpers.edit(upfile_path)
elif resp == 'u':
pass
else:
raise exception.UserAbort()
print("\nUpdate:\n%s\n" % parsed_update)
self.submit_update_file(update_id, msg=str(parsed_update))
def __call__(self, *params, **kwargs):
# allows us to run git in isolated mode, avoiding interaction with user
# specific config, like ~/.git-templates/hooks (used for testing)
if kwargs.get('isolated', False):
env = kwargs.get('env', os.environ.copy())
env['GIT_CONFIG_NOSYSTEM'] = '1'
env['GIT_CONFIG_NOGLOBAL'] = '1'
kwargs['env'] = env
return run(self.command, *params, **kwargs)
def __call__(self, *params, **kwargs):
# allows us to run git in isolated mode, avoiding interaction with user
# specific config, like ~/.git-templates/hooks (used for testing)
if kwargs.get('isolated', False):
env = kwargs.get('env', os.environ.copy())
env['GIT_CONFIG_NOSYSTEM'] = '1'
env['GIT_CONFIG_NOGLOBAL'] = '1'
kwargs['env'] = env
return run(self.command, *params, **kwargs)
def repoquery(repo_url, repo_name, package, verbose=False):
repo_id = "rdopkg_%s" % re.sub('[^\w]', '_', repo_name)
cmd = ["repoquery", "--nvr",
"--repofrompath=%s,%s" % (repo_id, repo_url),
"--repoid=%s" % repo_id, "-q", package]
try:
o = run(*cmd, log_cmd=verbose, log_fail=verbose)
except Exception:
return None
lines = o.strip().split("\n")
return lines[0] or None
def sign_packages(self):
repos = defaultdict(set)
def _sign_pkg(upf):
update = self._load_update_file(upf)
for build in update.builds:
build_path = self._build_tmp_path(upf, build)
repos[build.repo].add(build_path)
self._run_on_each(_sign_pkg, 'sign')
for repo, build_paths in repos.items():
key = "rdo-%s" % repo
log.info("Signing with %s..." % log.term.bold(key))
rpms = set()
for path in build_paths:
build_rpms = helpers.find_files(path, ext='.rpm')
rpms = rpms.union(build_rpms)
cmd = [self.sign_tool_path, key] + list(rpms)
run(*cmd, direct=True)
return self.update_files
def edit(path):
editor = os.environ.get('EDITOR')
if not editor:
editor = 'vim'
log.info("$EDITOR not set. Falling back to %s." % editor)
try:
r = run(editor, path, direct=True)
except exception.CommandNotFound:
raise exception.CommandNotFound(
msg='Failed to find suitable text editor ({0}). '
'Please set $EDITOR environment variable.'.format(editor))
return r.success
def edit(path):
editor = os.environ.get('EDITOR')
if not editor:
editor = 'vim'
log.info("$EDITOR not set. Falling back to %s." % editor)
try:
r = run(editor, path, direct=True)
except exception.CommandNotFound:
raise exception.CommandNotFound(
msg='Failed to find suitable text editor ({0}). '
'Please set $EDITOR environment variable.'.format(editor))
return r.success
def sign_packages(self):
repos = defaultdict(set)
def _sign_pkg(upf):
update = self._load_update_file(upf)
for build in update.builds:
build_path = self._build_tmp_path(upf, build)
repos[build.repo].add(build_path)
self._run_on_each(_sign_pkg, 'sign')
for repo, build_paths in repos.items():
key = "rdo-%s" % repo
log.info("Signing with %s..." % log.term.bold(key))
rpms = set()
for path in build_paths:
build_rpms = helpers.find_files(path, ext='.rpm')
rpms = rpms.union(build_rpms)
cmd = [self.sign_tool_path, key] + list(rpms)
run(*cmd, direct=True)
return self.update_files
def repoquery(repo_url, repo_name, package, verbose=False):
repo_id = "rdopkg_%s" % re.sub('[^\w]', '_', repo_name)
cmd = [
"repoquery", "--nvr",
"--repofrompath=%s,%s" % (repo_id, repo_url),
"--repoid=%s" % repo_id, "-q", package
]
try:
o = run(*cmd, log_cmd=verbose, log_fail=verbose)
except Exception:
return None
lines = o.strip().split("\n")
return lines[0] or None
def make_srpm(package, dist=None, fedpkg=FEDPKG):
cmd = list(fedpkg)
if dist:
dname, _, drls = dist.partition('-')
if dname == 'epel' and drls:
cmd += ['--dist', 'el' + drls]
cmd.append('srpm')
out = run(*cmd)
m = re.search(r'/([^/\\]+\.src.rpm)\b', out)
if not m:
raise exception.CommandOutputParseError(tool=cmd[0], output=out)
srpm = m.group(1)
if not os.path.isfile(srpm):
raise exception.FileNotFound(path=srpm)
return {'srpm': srpm}
def make_srpm(package, dist=None, fedpkg=FEDPKG):
cmd = list(fedpkg)
if dist:
dname, _, drls = dist.partition('-')
if dname == 'epel' and drls:
cmd += ['--dist', 'el' + drls]
cmd.append('srpm')
out = run(*cmd)
m = re.search(r'/([^/\\]+\.src.rpm)\b', out)
if not m:
raise exception.CommandOutputParseError(tool=cmd[0], output=out)
srpm = m.group(1)
if not os.path.isfile(srpm):
raise exception.FileNotFound(path=srpm)
return {'srpm': srpm}
def upload_fpo(pkg, user):
dst_host = user + '@fedorapeople.org'
dst_path = '~/public_html/copr'
dst = '%s:%s/%s' % (dst_host, dst_path, pkg)
_cmd = ['scp', pkg, dst]
url = fpo_url(pkg, user)
try:
cmd.run(*_cmd)
except exception.CommandFailed as ex:
err = ex.kwargs['out'].stderr
# TODO: fragile, use paramiko instead?
if not re.match('scp: .*No such file or directory', err):
raise
log.info("Creating remote dir: %s:%s" % (dst_host, dst_path))
cmd.run('ssh', dst_host, 'mkdir -p ' + dst_path)
cmd.run(*_cmd)
return url
def upload_fpo(pkg, user):
dst_host = user + '@fedorapeople.org'
dst_path = '~/public_html/copr'
dst = '%s:%s/%s' % (dst_host, dst_path, pkg)
_cmd = ['scp', pkg, dst]
url = fpo_url(pkg, user)
try:
cmd.run(*_cmd)
except exception.CommandFailed as ex:
err = ex.kwargs['out'].stderr
# TODO: fragile, use paramiko instead?
if not re.match('scp: .*No such file or directory', err):
raise
log.info("Creating remote dir: %s:%s" % (dst_host, dst_path))
cmd.run('ssh', dst_host, 'mkdir -p ' + dst_path)
cmd.run(*_cmd)
return url
def push_packages(self):
updated_repo_bases = set()
updated_repos = set()
def _push_pkg(upf):
log.info("\nPushing update {t.bold}{upf}{t.normal}".format(
t=log.term, upf=upf))
update = self._load_update_file(upf)
pushed_rpms = []
try:
_updated_repos = set()
_updated_repo_bases = set()
_pushed_build_tmp_paths = []
for build in update.builds:
src_path = self._build_tmp_path(upf, build)
if src_path in _pushed_build_tmp_paths:
continue
build_rpms = helpers.find_files(src_path, ext='.rpm')
dest_repo_base_path = self._dest_repo_base_path(build.repo)
if not os.path.isdir(dest_repo_base_path):
raise exception.NotADirectory(path=dest_repo_base_path)
dest_path = self._build_dest_path(build)
for rpm in build_rpms:
pushed_path = copy_package(rpm,
dest_path,
overwrite=self.overwrite)
pushed_rpms.append(pushed_path)
_pushed_build_tmp_paths.append(src_path)
_updated_repo_bases.add(dest_repo_base_path)
_updated_repos.add(
self._dest_repo_path(build.repo, build.dist))
with helpers.cdir(self.update_repo_path):
helpers.ensure_dir(self.pushed_dir)
upf_base = os.path.basename(upf)
pushed_upf = os.path.join(self.pushed_dir, upf_base)
pushed_files_fn = pushed_upf + self.pushed_files_ext
git('mv', upf, pushed_upf)
pushed_files_f = open(pushed_files_fn, 'w')
pushed_files_f.writelines(
map(lambda x: "%s\n" % x, pushed_rpms))
pushed_files_f.close()
git('add', pushed_files_fn)
try:
git('commit', '-m',
"Push %s" % rdoupdate.core.pp_update(upf))
except Exception:
git('git', 'reset', '--hard')
raise
updated_repos.update(_updated_repos)
updated_repo_bases.update(_updated_repo_bases)
except Exception as ex:
if pushed_rpms:
log.warn("Final push failed for %s, cleaning copied "
"packages" % upf)
for rpm in pushed_rpms:
log.info("{t.warn}remove{t.normal} {rpm}".format(
t=log.term, rpm=rpm))
os.remove(rpm)
raise
self._run_on_each(_push_pkg, 'final push')
if updated_repos:
log.info("\nRunning {t.cmd}createrepo{t.normal} on updated repos".
format(t=log.term))
for repo in sorted(updated_repos):
run('createrepo', repo, direct=True)
return sorted(updated_repo_bases)
def fedpkg_mockbuild(fedpkg=FEDPKG):
cmd = list(fedpkg) + ['mockbuild']
run(*cmd, direct=True)
def fedpkg_mockbuild(fedpkg=FEDPKG):
cmd = list(fedpkg) + ['mockbuild']
run(*cmd, direct=True)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
'--patches-branch',
help='Specify another local "patches" branch, like "ceph-5.0-rhel-patches-bz12345"',
)
args = parser.parse_args()
spec = specfile.Spec()
name = spec.get_tag('Name', expand_macros=True) # "ceph"
version = spec.get_tag('Version', expand_macros=True) # "12.2.8"
orig_commit = spec.get_macro('commit') # "9e20ef1b14ac70dea53123"
branch = git.current_branch() # "ceph-3.2-rhel-7"
tag_style = guess.version_tag_style(version=version) # "vX.Y.Z"
base_tag = guess.version2tag(version, tag_style) # "v12.2.8"
osdist = guess.osdist()
# "ceph-3.2-rhel-patches"
if args.patches_branch:
patches_branch = args.patches_branch
else:
remote_patches_branch = guess.patches_branch(branch, pkg=name,
osdist=osdist)
patches_branch = remote_patches_branch.partition('/')[2]
patches_sha = git('rev-parse', patches_branch) # "9e20ef1b14ac70dea53123"
archive_basename = '%s-%s' % (name, version) # "ceph-12.2.8"
patches_base, patches_base_commits = spec.get_patches_base()
if patches_base_commits != 0:
# We don't yet support the "+n_commits" syntax for patches_base.
raise NotImplementedError('use a plain ref in patches_base')
if patches_base is None:
patches_base = base_tag
filenames = diff_filenames(patches_base, patches_branch)
if not filenames:
# todo: make this a silent no-op eventually
log.warning('%s identical to %s' % (patches_branch, patches_base))
raise RuntimeError(patches_base)
tarball = archive_files(archive_basename, patches_sha, filenames)
log.info('wrote %s' % tarball)
# Ensure our spec file will reference this tarball.
spec.set_macro('commit', patches_sha)
set_source1(spec)
spec.save()
# Find the changelog entries from the Git -patches branch.
changes = check_new_commits(patches_base, orig_commit, patches_sha)
if not changes:
log.info('no changes. exiting')
raise SystemExit(1)
# Bump the release and add the %changelog entries.
# Insert %changelog.
rdopkg.actions.distgit.actions.update_spec(branch=branch, changes=changes)
# add + upload this new tarball.
if guess.new_sources():
fedpkg = 'fedpkg'
if osdist.startswith('RH'):
fedpkg = 'rhpkg'
clear_old_changes_sources()
run(fedpkg, 'upload', tarball, direct=True)
# Commit everything to dist-git
rdopkg.actions.distgit.actions.commit_distgit_update(branch=branch,
local_patches_branch=patches_branch)
# Show the final commit
rdopkg.actions.distgit.actions.final_spec_diff(branch=branch)
def push_packages(self):
updated_repo_bases = set()
updated_repos = set()
def _push_pkg(upf):
log.info("\nPushing update {t.bold}{upf}{t.normal}".format(
t=log.term, upf=upf))
update = self._load_update_file(upf)
pushed_rpms = []
try:
_updated_repos = set()
_updated_repo_bases = set()
_pushed_build_tmp_paths = []
for build in update.builds:
src_path = self._build_tmp_path(upf, build)
if src_path in _pushed_build_tmp_paths:
continue
build_rpms = helpers.find_files(src_path, ext='.rpm')
dest_repo_base_path = self._dest_repo_base_path(build.repo)
if not os.path.isdir(dest_repo_base_path):
raise exception.NotADirectory(path=dest_repo_base_path)
dest_path = self._build_dest_path(build)
for rpm in build_rpms:
pushed_path = copy_package(rpm, dest_path,
overwrite=self.overwrite)
pushed_rpms.append(pushed_path)
_pushed_build_tmp_paths.append(src_path)
_updated_repo_bases.add(dest_repo_base_path)
_updated_repos.add(self._dest_repo_path(build.repo,
build.dist))
with helpers.cdir(self.update_repo_path):
helpers.ensure_dir(self.pushed_dir)
upf_base = os.path.basename(upf)
pushed_upf = os.path.join(self.pushed_dir, upf_base)
pushed_files_fn = pushed_upf + self.pushed_files_ext
git('mv', upf, pushed_upf)
pushed_files_f = open(pushed_files_fn, 'w')
pushed_files_f.writelines(
map(lambda x: "%s\n" % x, pushed_rpms))
pushed_files_f.close()
git('add', pushed_files_fn)
try:
git('commit', '-m',
"Push %s" % rdoupdate.core.pp_update(upf))
except Exception:
git('git', 'reset', '--hard')
raise
updated_repos.update(_updated_repos)
updated_repo_bases.update(_updated_repo_bases)
except Exception as ex:
if pushed_rpms:
log.warn("Final push failed for %s, cleaning copied "
"packages" % upf)
for rpm in pushed_rpms:
log.info("{t.warn}remove{t.normal} {rpm}".format(
t=log.term, rpm=rpm))
os.remove(rpm)
raise
self._run_on_each(_push_pkg, 'final push')
if updated_repos:
log.info("\nRunning {t.cmd}createrepo{t.normal} on updated repos"
.format(t=log.term))
for repo in sorted(updated_repos):
run('createrepo', repo, direct=True)
return sorted(updated_repo_bases)
def download_file(url):
run('curl', '-L', '-f', '-O', url, direct=True)
def download_file(url):
run('curl', '-L', '-f', '-O', url, direct=True)