def run_action(self, action, args=None):
if not args:
args = {}
if action.continuable:
log.info(log.term.bold("## %s" % action.name))
for carg in action.const_args:
args[carg] = action.const_args[carg]
action_fun = action.action_fun
if not action_fun:
action_fun = self._get_action_fun(action)
if not action_fun:
raise exception.ActionFunctionNotAvailable(
action=action.name, module=action.module)
action.action_fun = action_fun
argspec = inspect_getargspec(action_fun)
fun_args = []
if argspec.defaults:
n_defaults = len(argspec.defaults)
else:
n_defaults = 0
n_required = len(argspec.args) - n_defaults
for i, arg in enumerate(argspec.args):
if arg in args:
fun_args.append(args[arg])
else:
if i < n_required:
raise exception.RequiredActionArgumentNotAvailable(
action=action.name, arg=arg)
else:
fun_args.append(argspec.defaults[i - n_required])
return action_fun(*fun_args)
def __init__(self, base_path=None, url=None, local_repo_path=None,
verbose=False, user=None):
# remote repo (base_path, url) XOR local repo (local_repo_path)
assert bool(base_path and url) != bool(local_repo_path)
self.user = user
if not self.user:
# we need a user, so pick the current user by default
env = os.environ.copy()
# USERNAME is an env var used by gerrit
self.user = env.get('USERNAME') or env.get('USER')
if verbose:
log.info('Using user %s with repo %s' % (self.user,
url))
self.url = tidy_ssh_user(url, self.user)
self.verbose = verbose
if local_repo_path:
self.repo_path = local_repo_path
self.base_path, self.repo_name = \
os.path.split(os.path.abspath(local_repo_path))
else:
self.base_path = base_path
self.repo_name = repo_name_from_url(self.url)
if not self.repo_name:
raise exception.RepoError(
what='Failed to parse %s repo URL: %s' % (self.repo_desc,
self.url))
self.repo_path = os.path.join(self.base_path, self.repo_name)
def get_updates_info(verbose=False):
gitdir = tempfile.mkdtemp(prefix='rdopkg-list-updates')
uinfos = []
prev_cwd = os.getcwd()
os.chdir(gitdir)
try:
cmd.git('init', log_cmd=False)
f_project = filters.OrFilter()
f_project.add_items('project', 'rdo-update')
f_other = filters.Items()
f_other.add_items('is', 'open')
query = reviews.Query(cfg['RDO_UPDATE_GERRIT_HOST'])
for review in query.filter(f_project, f_other):
try:
url = review.get('url', '???')
if verbose:
log.info("Processing update review: %s" % url)
uinfo = get_review_update_info(review, gitdir)
uinfos.append(uinfo)
except Exception as ex:
if verbose:
log.warn("Error processing update review: %s: %s",
type(ex).__name__, str(ex))
pass
finally:
os.chdir(prev_cwd)
shutil.rmtree(gitdir)
return uinfos
def rebase_patches_branch(new_version, local_patches_branch,
patches_branch=None, local_patches=False,
patches_style=None, version_tag_style=None,
bump_only=False):
if bump_only:
return
git.checkout(local_patches_branch)
new_version_tag = guess.version2tag(new_version, version_tag_style)
git('rebase', new_version_tag, direct=True)
if patches_style != 'review':
if local_patches or not patches_branch:
return
if _is_same_commit(local_patches_branch, patches_branch):
log.info("%s is up to date, no need for push." % patches_branch)
return
try:
remote, branch = git.remote_branch_split(patches_branch)
helpers.confirm("Push %s to %s / %s (with --force)?" % (
local_patches_branch, remote, branch))
git('push', '--force', remote,
'%s:%s' % (local_patches_branch, branch))
# push the tag
git('push', '--force', remote, new_version_tag)
except exception.UserAbort:
pass
def new_build(self, srpm_url, release, dist, watch=False):
copr = rdo_copr_name(release, dist)
url = self.copr_api_url('coprs/%s/%s/new_build/' % (self.owner, copr))
data = {
'pkgs': srpm_url,
}
req = requests.post(url,
auth=(self.user['login'], self.user['token']),
data=data)
output = _get_copr_data(req, self.user, type='new_build')
build_ids = output.get('ids')
if not build_ids:
raise exception.CoprError(
error="copr didn't return id of new build.(?!)")
build_id = build_ids[0]
if watch:
log.info("\nWatching build (may be safely interrupted)...")
prevstatus = None
try:
while True:
try:
status = self._fetch_build_status(build_id)
except exception.CoprError as ex:
log.warn("Failed to get build status: %s" % ex)
break
if prevstatus != status:
now = datetime.datetime.now()
if status in ['pending', 'waiting', 'running']:
cstatus = log.term.bold(status)
elif status == 'succeeded':
cstatus = log.term.good(status)
elif status == 'failed':
cstatus = log.term.error(status)
elif status == 'skipped':
cstatus = ("{t.magenta}{st}{t.normal} (build "
"already done)".format(t=log.term,
st=status))
else:
cstatus = log.term.warn(status)
log.info("[%s] %s" % (now.strftime('%H:%M:%S'),
cstatus))
prevstatus = status
if status in [
'succeeded',
'failed',
'canceled',
'skipped']:
break
time.sleep(60)
except KeyboardInterrupt:
pass
except Exception as ex:
log.warn("Error during copr build monitoring: %s" % ex)
return build_id
def copy_package(pkg_path, dest_dir, overwrite=False):
helpers.ensure_dir(dest_dir)
pkg_name = os.path.basename(pkg_path)
dst_path = os.path.join(dest_dir, pkg_name)
if not overwrite and os.path.exists(dst_path):
raise exception.NewPackageAlreadyPresent(path=dst_path)
log.info("{t.bold}copy{t.normal} {src} {t.bold}->{t.normal} {dst}".format(
src=pkg_path, dst=dst_path, t=log.term))
shutil.copyfile(pkg_path, dst_path)
return dst_path
def _get_new_update_id(self):
looking = True
while looking:
update_id = generate_id()
upfile_path = self._upfile_path_abs(update_id)
if os.path.exists(upfile_path):
log.info("Generated colliding ID %s. Weird. Generating new ID.")
else:
looking = False
return update_id, upfile_path
def _push_pkg(upf):
log.info("\nPushing update {t.bold}{upf}{t.normal}".format(
t=log.term, upf=upf))
update = self._load_update_file(upf)
pushed_rpms = []
try:
_updated_repos = set()
_updated_repo_bases = set()
_pushed_build_tmp_paths = []
for build in update.builds:
src_path = self._build_tmp_path(upf, build)
if src_path in _pushed_build_tmp_paths:
continue
build_rpms = helpers.find_files(src_path, ext='.rpm')
dest_repo_base_path = self._dest_repo_base_path(build.repo)
if not os.path.isdir(dest_repo_base_path):
raise exception.NotADirectory(path=dest_repo_base_path)
dest_path = self._build_dest_path(build)
for rpm in build_rpms:
pushed_path = copy_package(rpm, dest_path,
overwrite=self.overwrite)
pushed_rpms.append(pushed_path)
_pushed_build_tmp_paths.append(src_path)
_updated_repo_bases.add(dest_repo_base_path)
_updated_repos.add(self._dest_repo_path(build.repo,
build.dist))
with helpers.cdir(self.update_repo_path):
helpers.ensure_dir(self.pushed_dir)
upf_base = os.path.basename(upf)
pushed_upf = os.path.join(self.pushed_dir, upf_base)
pushed_files_fn = pushed_upf + self.pushed_files_ext
git('mv', upf, pushed_upf)
pushed_files_f = open(pushed_files_fn, 'w')
pushed_files_f.writelines(
map(lambda x: "%s\n" % x, pushed_rpms))
pushed_files_f.close()
git('add', pushed_files_fn)
try:
git('commit', '-m',
"Push %s" % rdoupdate.core.pp_update(upf))
except Exception:
git('git', 'reset', '--hard')
raise
updated_repos.update(_updated_repos)
updated_repo_bases.update(_updated_repo_bases)
except Exception as ex:
if pushed_rpms:
log.warn("Final push failed for %s, cleaning copied "
"packages" % upf)
for rpm in pushed_rpms:
log.info("{t.warn}remove{t.normal} {rpm}".format(
t=log.term, rpm=rpm))
os.remove(rpm)
raise
def _clone(self):
if self.verbose:
log.info("Cloning {desc} repo: {url}\n"
" {space} into: {path}".format(
desc=self.repo_desc,
space=len(self.repo_desc) * ' ',
url=self.url,
path=self.repo_path))
with helpers.cdir(self.base_path):
cmd.git('clone', self.url, self.repo_name, log_cmd=self.verbose)
def edit(path):
editor = os.environ.get('EDITOR')
if not editor:
editor = 'vim'
log.info("$EDITOR not set. Falling back to %s." % editor)
try:
r = run(editor, path, direct=True)
except exception.CommandNotFound:
raise exception.CommandNotFound(
msg='Failed to find suitable text editor ({0}). '
'Please set $EDITOR environment variable.'.format(editor))
return r.success
def query_repos(distrepos, package, verbose=False):
if not distrepos:
return []
versions = []
for repo in distrepos:
repo_name = repo['name']
repo_url = repo['url']
version = repoquery(repo_url, repo_name, package, verbose=verbose)
if version:
versions.append((repo_name, version))
if verbose:
log.info("%s: %s", repo_name, version or 'N/A')
versions.sort(cmp=lambda x, y: nvrcmp(x[0], y[0]), reverse=True)
return versions
def submit_existing_update(self, upfile_path, check_availability=True):
update = ensure_update_notes(upfile_path)
update_rdoinfo_check(update)
if check_availability:
log.info("Checking availability of updated builds...")
for b in update.builds:
r = b.is_available(verbose=True)
if not r:
raise rdoupdate.exception.BuildNotAvailable(
build_id=b.id, source=b.source, detail=r.err)
update_id, tmp_upfile_path = self._get_new_update_id()
tmp_updir_path, _ = os.path.split(tmp_upfile_path)
helpers.ensure_dir(tmp_updir_path)
shutil.copyfile(upfile_path, tmp_upfile_path)
print("\nUpdate:\n%s\n" % update)
self.submit_update_file(update_id, msg=str(update))
def conf():
if cfg_files:
log.info("Following config files were read:")
helpers.print_list(cfg_files)
else:
log.info("No rdopkg config files found, using default config:")
log.info("")
for item in cfg.items():
log.info("%s: %s" % item)
def upload_fpo(pkg, user):
dst_host = user + '@fedorapeople.org'
dst_path = '~/public_html/copr'
dst = '%s:%s/%s' % (dst_host, dst_path, pkg)
_cmd = ['scp', pkg, dst]
url = fpo_url(pkg, user)
try:
cmd.run(*_cmd)
except exception.CommandFailed as ex:
err = ex.kwargs['out'].stderr
# TODO: fragile, use paramiko instead?
if not re.match('scp: .*No such file or directory', err):
raise
log.info("Creating remote dir: %s:%s" % (dst_host, dst_path))
cmd.run('ssh', dst_host, 'mkdir -p ' + dst_path)
cmd.run(*_cmd)
return url
def new_update(self, update, check_availability=True):
update_id, upfile_path = self._get_new_update_id()
updir_path, _ = os.path.split(upfile_path)
helpers.ensure_dir(updir_path)
upfile = file(upfile_path, 'wt')
upfile.write(update.update_file())
upfile.close()
helpers.edit(upfile_path)
cmd.run('/bin/sed', '-i', '-e', '/^#/d', upfile_path, shell=False)
parsed_update = None
while not parsed_update:
try:
parsed_update = rdoupdate.actions.check_file(upfile_path)
except Exception as ex:
print("\n{t.important}Error parsing update file: {t.normal}"
"{t.warn}{ex}{t.normal}".format(t=log.term, ex=ex))
again = raw_input("Do you want to edit? [Yn] ")
if again and again.lower() != 'y':
os.remove(upfile_path)
raise exception.UserAbort()
helpers.edit(upfile_path)
continue
if check_availability:
log.info("Checking availability of updated builds...")
bad_builds = [x for x in parsed_update.builds
if not x.is_available(verbose=True)]
if bad_builds:
builds_str = "\n".join(map(str, bad_builds))
print("\n{t.important}Builds below doesn't seem to be "
"available:{t.normal}\n{builds}\n".format(
t=log.term, builds=builds_str))
print("Options:")
print(" e: Edit update")
print(" u: Update anyway")
print(" a: Abort")
resp = raw_input("What do you want to do? [euA] ").lower()
if resp == 'e':
parsed_update = None
helpers.edit(upfile_path)
elif resp == 'u':
pass
else:
raise exception.UserAbort()
print("\nUpdate:\n%s\n" % parsed_update)
self.submit_update_file(update_id, msg=str(parsed_update))
def get_package_env(version=None, release=None, dist=None, branch=None,
patches_branch=None, local_patches_branch=None,
patches_style=None, gerrit_patches_chain=None,
release_bump_index=None):
if not branch:
branch = git.current_branch()
if branch.endswith('-patches'):
branch = branch[:-8]
if git.branch_exists(branch):
log.info(
"This looks like -patches branch. Assuming distgit branch: "
"%s" % branch)
git.checkout(branch)
else:
raise exception.InvalidUsage(
why="This action must be run on a distgit branch.")
args = {
'package': guess.package(),
'branch': branch,
}
osdist = guess.osdist()
if osdist.startswith('RH'):
log.info("RH package detected.")
args['fedpkg'] = ['rhpkg']
if not patches_branch:
patches_branch = guess.patches_branch(branch, pkg=args['package'],
osdist=osdist)
if not patches_style:
patches_style = guess.patches_style(gerrit_patches_chain)
args['patches_style'] = patches_style
args['patches_branch'] = patches_branch
if release_bump_index is None:
args['release_bump_index'] = guess.release_bump_index()
if not local_patches_branch:
args['local_patches_branch'] = patches_branch.partition('/')[2]
if not version:
base_ref = guess.patches_base_ref()
version, _ = guess.tag2version(base_ref)
args['version'] = version
args['version_tag_style'] = guess.version_tag_style(version=version)
return args
def get_source(new_sources=False):
if not new_sources:
# due to new-version changes, this function does nothing by default :)
# TODO: factor this into get_source() and new_version_get_source()
return
# TODO: consider using `spectool --gf --source`
source_urls = specfile.Spec().get_source_urls()
# So far, only Source/Source0 is a tarball to download
source_url = source_urls[0]
source_fn = os.path.basename(source_url)
if os.path.isfile(source_fn):
log.info("%s already present" % source_fn)
return
try:
helpers.download_file(source_url)
except exception.CommandFailed:
# only valid for new-sources, not for get-source
raise exception.ActionRequired(
msg="Failed to download source tarball. Please update Source0 in "
".spec file.", rerun=True)
def _fetch(self, force=False):
need_fetch = True
with self.repo_dir():
if not force:
try:
t_fetch = os.path.getmtime('.git/FETCH_HEAD')
t_now = int(time.time())
delta = t_now - t_fetch
if delta < cfg['FETCH_PERIOD']:
need_fetch = False
except Exception:
pass
if need_fetch:
if self.verbose:
log.info("Fetching %s repo: %s" % (
self.repo_desc, self.repo_path))
cmd.git('fetch', 'origin', log_cmd=self.verbose)
cmd.git('checkout', '-f', 'master', log_cmd=self.verbose)
cmd.git('reset', '--hard', 'origin/master',
log_cmd=self.verbose)
def __init__(self, base_path=None, url=None, local_repo_path=None,
verbose=False, user=None):
# remote repo (base_path, url) XOR local repo (local_repo_path)
assert bool(base_path and url) != bool(local_repo_path)
self.user = user
self.url = url
if self.url and self.url.startswith('ssh://'):
# is there a user already?
match = re.compile('ssh://([^@]+)@.+').match(self.url)
if match:
ssh_user = match.groups()[0]
if ssh_user != self.user:
# assume prevalence of argument
self.url.replace(ssh_user + '@',
self.user + '@')
else:
if not self.user:
# we need a user, so pick the current user by default
env = os.environ.copy()
# USERNAME is an env var used by gerrit
self.user = env.get('USERNAME') or env.get('USER')
if verbose:
log.info('Using user %s with %s' % (self.user,
self.url))
self.url = 'ssh://' +\
self.user + '@' +\
self.url[len('ssh://'):]
self.verbose = verbose
if local_repo_path:
self.repo_path = local_repo_path
self.base_path, self.repo_name = \
os.path.split(os.path.abspath(local_repo_path))
else:
self.base_path = base_path
self.repo_name = repo_name_from_url(self.url)
if not self.repo_name:
raise exception.RepoError(
what='Failed to parse %s repo URL: %s' % (self.repo_desc,
self.url))
self.repo_path = os.path.join(self.base_path, self.repo_name)
def sign_packages(self):
repos = defaultdict(set)
def _sign_pkg(upf):
update = self._load_update_file(upf)
for build in update.builds:
build_path = self._build_tmp_path(upf, build)
repos[build.repo].add(build_path)
self._run_on_each(_sign_pkg, 'sign')
for repo, build_paths in repos.items():
key = "rdo-%s" % repo
log.info("Signing with %s..." % log.term.bold(key))
rpms = set()
for path in build_paths:
build_rpms = helpers.find_files(path, ext='.rpm')
rpms = rpms.union(build_rpms)
cmd = [self.sign_tool_path, key] + list(rpms)
run(*cmd, direct=True)
return self.update_files
def init(self, force_fetch=False):
if not self.url:
if not os.path.isdir(self.repo_path):
raise exception.NotADirectory(path=self.repo_path)
return
if self.base_path and not os.path.isdir(self.base_path):
if self.verbose:
log.info("Creating base directory: %s" % self.base_path)
os.makedirs(self.base_path)
if not os.path.isdir(self.repo_path):
self._clone()
else:
try:
self.git_check_remote()
except exception.RepoError as e:
if self.verbose:
log.warn("%s repo didn't pass the checks, renewing: %s"
% (self.repo_desc, e))
self._nuke()
self._clone()
else:
self._fetch(force=force_fetch)
def submit_update_file(self, id, msg=''):
upfile_path = self._upfile_path(id)
with self.repo_dir():
if not os.path.isfile(upfile_path):
raise exception.UpdateFileNotFound(path=upfile_path)
update = rdoupdate.actions.check_file(upfile_path)
branch = update_summary(update)
commit_msg = "New %s" % id
if msg:
commit_msg += "\n\n%s\n" % msg
log.info("Setting up gerrit.")
cmd.git('review', '-s', direct=True)
cmd.git('branch', branch, 'origin/master')
try:
cmd.git('checkout', '-f', branch)
cmd.git('add', upfile_path)
cmd.git('commit', '-F', '-', input=commit_msg)
log.info("Submitting update for review.")
cmd.git('review', direct=True)
finally:
pass
cmd.git('checkout', '-f', 'master')
cmd.git('branch', '-D', branch)
def reqcheck(version, spec=False):
if version.upper() == 'XXX':
if 'upstream' in git.remotes():
current_branch = git.current_branch()
branch = current_branch.replace('rpm-', '')
if branch != 'master':
branch = 'stable/{}'.format(branch)
version = 'upstream/{}'.format(branch)
check = _reqs.reqcheck_spec(ref=version)
else:
m = re.search(r'/([^/]+)_distro', os.getcwd())
if not m:
raise exception.CantGuess(what="requirements.txt location",
why="failed to parse current path")
path = '../%s/requirements.txt' % m.group(1)
log.info("Delorean detected. Using %s" % path)
check = _reqs.reqcheck_spec(reqs_txt=path)
else:
check = _reqs.reqcheck_spec(ref=version)
format = None
if spec:
format = 'spec'
_reqs.print_reqcheck(*check, format=format)
def update_rdoinfo_check(update):
fail_msg = None
warn = False
rls = defaultdict(set)
for b in update.builds:
rls[b.repo].add(b)
log.info("Checking update using rdoinfo...".format(
t=log.term))
for r, builds in rls.items():
rbuilds = guess.builds(r)
rbuilds_ = list(rbuilds)
if not rbuilds:
msg = 'Unknown OpenStack release: %s' % r
log.warn(msg)
if not fail_msg:
fail_msg = msg
continue
for b in builds:
found = False
for dist_, src_ in rbuilds:
if b.dist == dist_:
found = True
rbuilds_ = filter(lambda x: x[0] != b.dist, rbuilds_)
break
if not found:
msg = 'Unexpected %s build: %s' % (r, b.dist)
log.warn(msg)
if not fail_msg:
fail_msg = msg
for dist_, src_ in rbuilds_:
log.info('Missing %s build: %s' % (r, dist_))
warn = True
if fail_msg:
raise exception.UpdateCheckFailed(fail=fail_msg)
if warn:
helpers.confirm("Submit anyway?")
def dump_build(build, update_file):
if os.path.isfile(update_file):
update = rdoupdate.actions.check_file(update_file)
found = False
for b in update.builds:
if b.__dict__ == build.__dict__:
found = True
break
if found:
log.info("\nBuild already present in update file: %s" % update_file)
else:
log.info("\nAppending build to update file: %s" % update_file)
update.builds.append(build)
else:
log.info("\nSaving build to new update file: %s" % update_file)
update = rdoupdate.core.Update(builds=[build],
notes=FILLME)
helpers.ensure_new_file_dirs(update_file)
file(update_file, 'w').write(update.update_file(hints=False))
def update_spec(branch=None, changes=None,
new_rpm_version=None, new_release=None,
new_milestone=None, new_patches_base=None,
no_bump=False, changelog_user=None,
changelog_email=None, release_bump_index=None):
if no_bump:
return
if not changes:
changes = []
_ensure_branch(branch)
spec = specfile.Spec()
if new_rpm_version:
old_version = spec.get_tag('Version')
if specfile.has_macros(old_version):
log.info('Version contains macro - not touching that.')
else:
spec.set_tag('Version', new_rpm_version)
if new_release is not None:
if spec.recognized_release():
spec.set_release(new_release, milestone=new_milestone)
else:
log.info('Custom Release format detected '
'- assuming custom milestone management.')
spec.set_release(new_release)
else:
spec.bump_release(milestone=new_milestone,
index=release_bump_index)
if new_patches_base:
new_patches_base_version, _ = guess.tag2version(new_patches_base)
if new_patches_base_version == new_rpm_version:
new_patches_base = None
changed = spec.set_patches_base_version(new_patches_base)
if not changed:
log.info("Macro detected in patches_base - not touching that.")
user = (changelog_user or guess.user())
email = (changelog_email or guess.email())
spec.new_changelog_entry(user=user, email=email,
changes=changes)
spec.save()
def reqquery(reqs_file=None,
reqs_ref=None,
spec=False,
filter=None,
dump=None,
dump_file=None,
load=None,
load_file=None,
verbose=False):
if not (reqs_ref or reqs_file or spec or load or load_file):
reqs_ref = guess.patches_base_ref()
if not (bool(reqs_ref) ^ bool(reqs_file) ^ bool(spec) ^ bool(load)
^ bool(load_file)):
raise exception.InvalidUsage(
why="Only one requirements source (-r/-R/-s/-l/-L) can be "
"selected.")
if dump and dump_file:
raise exception.InvalidUsage(
why="Only one dump method (-d/-D) can be selected.")
if dump:
dump_file = 'requirements.yml'
if load:
load_file = 'requirements.yml'
# get query results as requested
if load_file:
log.info("Loading query results from file: %s" % load_file)
r = yaml.load(open(load_file))
else:
release, dist = None, None
if not filter:
try:
release, dist = guess.osreleasedist()
log.info('Autodetected filter: %s/%s' % (release, dist))
except exception.CantGuess as ex:
raise exception.CantGuess(
msg='%s\n\nPlease select RELEASE(/DIST) filter to query.' %
str(ex))
else:
release, _, dist = filter.partition('/')
module2pkg = True
if reqs_file:
log.info("Querying requirements file: %s" % reqs_file)
reqs = _reqs.get_reqs_from_path(reqs_file)
elif reqs_ref:
log.info("Querying requirements file from git: "
"%s -- requirements.txt" % reqs_ref)
reqs = _reqs.get_reqs_from_ref(reqs_ref)
else:
log.info("Querying .spec file")
module2pkg = False
reqs = _reqs.get_reqs_from_spec(as_objects=True)
log.info('')
r = _reqs.reqquery(reqs,
release=release,
dist=dist,
module2pkg=module2pkg,
verbose=verbose)
if dump_file:
log.info("Saving query results to file: %s" % dump_file)
yaml.dump(r, open(dump_file, 'w'))
_reqs.print_reqquery(r)
def gerrit_patches_chain(project=None, verbose=True):
if not project:
project = project_from_repo()
gerrit_host, gerrit_port = gerrit_from_repo()
gerrit_query = GerritQuery(gerrit_host, gerrit_port, log_cmd=verbose)
candidate = None
number = None
subject = None
_last_patch = last_patch()
if _last_patch:
log.info('Assumed last patch is %s' % _last_patch)
with open(_last_patch, 'r') as f:
patch_content = f.read()
subject_regex = re.compile("^Subject: (?:\[PATCH\]\s*)?(.+)$",
re.MULTILINE | re.IGNORECASE)
s = subject_regex.findall(patch_content)
if s:
subject = s[0]
# commit id tells us which revision we need
from_regex = re.compile(r"^From ([0-9a-f]{40})",
re.MULTILINE | re.IGNORECASE)
for from_commit in from_regex.findall(patch_content):
query = "project:%s commit:%s" % (project, from_commit)
q = gerrit_query('--patch-sets', query)
if q:
candidate = q
numbers = [ps.get('number') for ps
in candidate.get('patchSets')
if ps.get('revision') == from_commit]
if numbers:
number = numbers[0]
break
# If this fails (we're dealing with migration residues),
# look for changeIDs and use last revision.
if not candidate:
log.info("Revision number not found, the latest patchset "
"(if found) will be downloaded.")
cid_regex = re.compile(r"^Change-Id: (I[0-9a-f]{40})",
re.MULTILINE | re.IGNORECASE)
for cid in cid_regex.findall(patch_content):
query = "project:%s change:%s" % (project, cid)
if subject:
query += " message:%s" % subject
query += " limit:1"
q = gerrit_query(query)
if q:
candidate = q
break
# last chance, with the commit message
if not candidate:
query = "project:%s status:open" % project
if subject:
query += " message:%s" % subject
query += " limit:1"
candidate = gerrit_query(query)
if not candidate:
return None
patchset = candidate.get('number')
if number:
patchset = "%s/%s" % (patchset, number)
return patchset
def clone(package,
force_fetch=False,
use_master_distgit=False,
gerrit_remotes=False,
review_user=None):
inforepo = rdoinfo.get_default_inforepo()
inforepo.init(force_fetch=force_fetch)
pkg = inforepo.get_package(package)
if not pkg:
raise exception.InvalidRDOPackage(package=package)
if use_master_distgit:
try:
distgit = pkg['master-distgit']
distgit_str = 'master-distgit'
except KeyError:
raise exception.InvalidUsage(
msg="-m/--use-master-distgit used but 'master-distgit' "
"missing in rdoinfo for package: %s" % package)
else:
distgit = pkg['distgit']
distgit_str = 'distgit'
log.info("Cloning {dg} into ./{t.bold}{pkg}{t.normal}/".format(
t=log.term, dg=distgit_str, pkg=package))
patches = pkg.get('patches')
upstream = pkg.get('upstream')
review_patches = pkg.get('review-patches')
review_origin = pkg.get('review-origin')
git('clone', distgit, package)
with helpers.cdir(package):
if gerrit_remotes:
log.info('Adding gerrit-origin remote...')
git('remote', 'add', 'gerrit-origin', distgit)
if patches:
log.info('Adding patches remote...')
git('remote', 'add', 'patches', patches)
if gerrit_remotes:
log.info('Adding gerrit-patches remote...')
git('remote', 'add', 'gerrit-patches', patches)
else:
log.warn("'patches' remote information not available in rdoinfo.")
if upstream:
log.info('Adding upstream remote...')
git('remote', 'add', 'upstream', upstream)
else:
log.warn("'upstream' remote information not available in rdoinfo.")
if patches or upstream:
git('fetch', '--all')
if not review_user:
# USERNAME is an env var used by gerrit
review_user = os.environ.get('USERNAME') or os.environ.get('USER')
msg_user = ('Using {t.bold}{u}{t.normal} as gerrit username, '
'you can change it with '
'{t.cmd}git remote set-url {r} ...{t.normal}')
if review_patches:
log.info('Adding gerrit remote for patch chains reviews...')
r = tidy_ssh_user(review_patches, review_user)
log.info(
msg_user.format(u=review_user, r='review-patches', t=log.term))
git('remote', 'add', 'review-patches', r)
else:
log.warn("'review-patches' remote information not available"
" in rdoinfo.")
if review_origin:
log.info('Adding gerrit remote for reviews...')
r = tidy_ssh_user(review_origin, review_user)
log.info(
msg_user.format(u=review_user, r='review-origin', t=log.term))
git('remote', 'add', 'review-origin', r)
else:
log.warn("'review-origin' remote information not available"
" in rdoinfo.")
git('remote', '-v', direct=True)
def push_packages(self):
updated_repo_bases = set()
updated_repos = set()
def _push_pkg(upf):
log.info("\nPushing update {t.bold}{upf}{t.normal}".format(
t=log.term, upf=upf))
update = self._load_update_file(upf)
pushed_rpms = []
try:
_updated_repos = set()
_updated_repo_bases = set()
_pushed_build_tmp_paths = []
for build in update.builds:
src_path = self._build_tmp_path(upf, build)
if src_path in _pushed_build_tmp_paths:
continue
build_rpms = helpers.find_files(src_path, ext='.rpm')
dest_repo_base_path = self._dest_repo_base_path(build.repo)
if not os.path.isdir(dest_repo_base_path):
raise exception.NotADirectory(path=dest_repo_base_path)
dest_path = self._build_dest_path(build)
for rpm in build_rpms:
pushed_path = copy_package(rpm,
dest_path,
overwrite=self.overwrite)
pushed_rpms.append(pushed_path)
_pushed_build_tmp_paths.append(src_path)
_updated_repo_bases.add(dest_repo_base_path)
_updated_repos.add(
self._dest_repo_path(build.repo, build.dist))
with helpers.cdir(self.update_repo_path):
helpers.ensure_dir(self.pushed_dir)
upf_base = os.path.basename(upf)
pushed_upf = os.path.join(self.pushed_dir, upf_base)
pushed_files_fn = pushed_upf + self.pushed_files_ext
git('mv', upf, pushed_upf)
pushed_files_f = open(pushed_files_fn, 'w')
pushed_files_f.writelines(
map(lambda x: "%s\n" % x, pushed_rpms))
pushed_files_f.close()
git('add', pushed_files_fn)
try:
git('commit', '-m',
"Push %s" % rdoupdate.core.pp_update(upf))
except Exception:
git('git', 'reset', '--hard')
raise
updated_repos.update(_updated_repos)
updated_repo_bases.update(_updated_repo_bases)
except Exception as ex:
if pushed_rpms:
log.warn("Final push failed for %s, cleaning copied "
"packages" % upf)
for rpm in pushed_rpms:
log.info("{t.warn}remove{t.normal} {rpm}".format(
t=log.term, rpm=rpm))
os.remove(rpm)
raise
self._run_on_each(_push_pkg, 'final push')
if updated_repos:
log.info("\nRunning {t.cmd}createrepo{t.normal} on updated repos".
format(t=log.term))
for repo in sorted(updated_repos):
run('createrepo', repo, direct=True)
return sorted(updated_repo_bases)
def gerrit_patches_chain(project=None, verbose=True):
if not project:
project = project_from_repo()
gerrit_host, gerrit_port = gerrit_from_repo()
gerrit_query = GerritQuery(gerrit_host, gerrit_port, log_cmd=verbose)
candidate = None
number = None
subject = None
_last_patch = last_patch()
if _last_patch:
log.info('Assumed last patch is %s' % _last_patch)
with open(_last_patch, 'r') as f:
patch_content = f.read()
subject_regex = re.compile("^Subject: (?:\[PATCH\]\s*)?(.+)$",
re.MULTILINE | re.IGNORECASE)
s = subject_regex.findall(patch_content)
if s:
subject = s[0]
# commit id tells us which revision we need
from_regex = re.compile(r"^From ([0-9a-f]{40})",
re.MULTILINE | re.IGNORECASE)
for from_commit in from_regex.findall(patch_content):
query = "project:%s commit:%s" % (project, from_commit)
q = gerrit_query('--patch-sets', query)
if q:
candidate = q
numbers = [
ps.get('number') for ps in candidate.get('patchSets')
if ps.get('revision') == from_commit
]
if numbers:
number = numbers[0]
break
# If this fails (we're dealing with migration residues),
# look for changeIDs and use last revision.
if not candidate:
log.info("Revision number not found, the latest patchset "
"(if found) will be downloaded.")
cid_regex = re.compile(r"^Change-Id: (I[0-9a-f]{40})",
re.MULTILINE | re.IGNORECASE)
for cid in cid_regex.findall(patch_content):
query = "project:%s change:%s" % (project, cid)
if subject:
query += " message:%s" % subject
query += " limit:1"
q = gerrit_query(query)
if q:
candidate = q
break
# last chance, with the commit message
if not candidate:
query = "project:%s status:open" % project
if subject:
query += " message:%s" % subject
query += " limit:1"
candidate = gerrit_query(query)
if not candidate:
return None
patchset = candidate.get('number')
if number:
patchset = "%s/%s" % (patchset, number)
return patchset
def reqdiff(version_tag_from, version_tag_to):
fmt = "\n{t.bold}requirements.txt diff{t.normal} between " \
"{t.bold}{old}{t.normal} and {t.bold}{new}{t.normal}:"
log.info(fmt.format(t=log.term, old=version_tag_from, new=version_tag_to))
rdiff = _reqs.reqdiff_from_refs(version_tag_from, version_tag_to)
_reqs.print_reqdiff(*rdiff)
def _nuke(self):
log.info("Removing %s repo: %s" % (self.repo_desc, self.repo_path))
shutil.rmtree(self.repo_path, ignore_errors=True)
def new_version_setup(patches_branch=None,
local_patches=False,
version=None,
new_version=None,
version_tag_style=None,
new_sources=None,
no_new_sources=None,
unattended=False,
bug=None,
bump_only=False):
args = {}
if new_version:
# support both version and tag
ver, _ = guess.tag2version(new_version)
if ver != new_version:
new_version = ver
args['new_version'] = new_version
new_version_tag = guess.version2tag(new_version, version_tag_style)
if not bump_only and not git.object_type(new_version_tag):
raise exception.InvalidGitRef(ref=new_version_tag)
else:
ub = guess.upstream_branch()
if not git.ref_exists('refs/remotes/%s' % ub):
msg = ("Upstream branch not found: %s\n"
"Can't guess latest version.\n\n"
"a) provide new version (git tag) yourself\n"
" $ rdopkg new-version 1.2.3\n\n"
"b) add upstream git remote:\n"
" $ git remote add -f upstream GIT_URL\n" % ub)
raise exception.CantGuess(msg=msg)
new_version_tag = git.get_latest_tag(ub)
new_version, _ = guess.tag2version(new_version_tag)
args['new_version'] = new_version
log.info("Latest version detected from %s: %s" % (ub, new_version))
if version == new_version:
if unattended:
log.info("Package is already at version %s\n" % version)
raise exception.UserAbort(exit_code=0)
helpers.confirm(msg="It seems the package is already at version %s\n\n"
"Run new-version anyway?" % version,
default_yes=False)
changelog = 'Update to %s' % new_version
if bug:
changelog += ' (%s)' % bug
args['changes'] = [changelog]
args['new_patches_base'] = new_version_tag
spec = specfile.Spec()
rpm_version = spec.get_tag('Version')
rpm_milestone = spec.get_milestone()
new_rpm_version, new_milestone = specfile.version_parts(new_version)
args['new_rpm_version'] = new_rpm_version
if new_milestone:
args['new_milestone'] = new_milestone
if (rpm_version != new_rpm_version
or bool(new_milestone) != bool(rpm_milestone)):
if new_milestone:
args['new_release'] = '0.1'
else:
args['new_release'] = '1'
if not local_patches and not bump_only:
if not patches_branch or \
not git.ref_exists('refs/remotes/' + patches_branch):
log.warn("Patches branch '%s' not found. Running in --bump-only "
"mode." % patches_branch)
args['bump_only'] = True
if new_sources or no_new_sources:
if new_sources and no_new_sources:
raise exception.InvalidUsage(
msg="DOES NOT COMPUTE: both -n and -N don't make sense.")
# new_sources == not no_new_sources
else:
new_sources = guess.new_sources()
args['new_sources'] = new_sources
return args
def reqdiff(version_tag_from, version_tag_to):
fmt = "\n{t.bold}requirements.txt diff{t.normal} between " \
"{t.bold}{old}{t.normal} and {t.bold}{new}{t.normal}:"
log.info(fmt.format(t=log.term, old=version_tag_from, new=version_tag_to))
rdiff = _reqs.reqdiff_from_refs(version_tag_from, version_tag_to)
_reqs.print_reqdiff(*rdiff)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
'--patches-branch',
help='Specify another local "patches" branch, like "ceph-5.0-rhel-patches-bz12345"',
)
args = parser.parse_args()
spec = specfile.Spec()
name = spec.get_tag('Name', expand_macros=True) # "ceph"
version = spec.get_tag('Version', expand_macros=True) # "12.2.8"
orig_commit = spec.get_macro('commit') # "9e20ef1b14ac70dea53123"
branch = git.current_branch() # "ceph-3.2-rhel-7"
tag_style = guess.version_tag_style(version=version) # "vX.Y.Z"
base_tag = guess.version2tag(version, tag_style) # "v12.2.8"
osdist = guess.osdist()
# "ceph-3.2-rhel-patches"
if args.patches_branch:
patches_branch = args.patches_branch
else:
remote_patches_branch = guess.patches_branch(branch, pkg=name,
osdist=osdist)
patches_branch = remote_patches_branch.partition('/')[2]
patches_sha = git('rev-parse', patches_branch) # "9e20ef1b14ac70dea53123"
archive_basename = '%s-%s' % (name, version) # "ceph-12.2.8"
patches_base, patches_base_commits = spec.get_patches_base()
if patches_base_commits != 0:
# We don't yet support the "+n_commits" syntax for patches_base.
raise NotImplementedError('use a plain ref in patches_base')
if patches_base is None:
patches_base = base_tag
filenames = diff_filenames(patches_base, patches_branch)
if not filenames:
# todo: make this a silent no-op eventually
log.warning('%s identical to %s' % (patches_branch, patches_base))
raise RuntimeError(patches_base)
tarball = archive_files(archive_basename, patches_sha, filenames)
log.info('wrote %s' % tarball)
# Ensure our spec file will reference this tarball.
spec.set_macro('commit', patches_sha)
set_source1(spec)
spec.save()
# Find the changelog entries from the Git -patches branch.
changes = check_new_commits(patches_base, orig_commit, patches_sha)
if not changes:
log.info('no changes. exiting')
raise SystemExit(1)
# Bump the release and add the %changelog entries.
# Insert %changelog.
rdopkg.actions.distgit.actions.update_spec(branch=branch, changes=changes)
# add + upload this new tarball.
if guess.new_sources():
fedpkg = 'fedpkg'
if osdist.startswith('RH'):
fedpkg = 'rhpkg'
clear_old_changes_sources()
run(fedpkg, 'upload', tarball, direct=True)
# Commit everything to dist-git
rdopkg.actions.distgit.actions.commit_distgit_update(branch=branch,
local_patches_branch=patches_branch)
# Show the final commit
rdopkg.actions.distgit.actions.final_spec_diff(branch=branch)
def _nuke(self):
log.info("Removing %s repo: %s" % (self.repo_desc, self.repo_path))
shutil.rmtree(self.repo_path, ignore_errors=True)
def reqquery(reqs_file=None, reqs_ref=None, spec=False, filter=None,
dump=None, dump_file=None, load=None, load_file=None,
verbose=False):
if not (reqs_ref or reqs_file or spec or load or load_file):
reqs_ref = guess.patches_base_ref()
if not (bool(reqs_ref) ^ bool(reqs_file) ^ bool(spec)
^ bool(load) ^ bool(load_file)):
raise exception.InvalidUsage(
why="Only one requirements source (-r/-R/-s/-l/-L) can be "
"selected.")
if dump and dump_file:
raise exception.InvalidUsage(
why="Only one dump method (-d/-D) can be selected.")
if dump:
dump_file = 'requirements.yml'
if load:
load_file = 'requirements.yml'
# get query results as requested
if load_file:
log.info("Loading query results from file: %s" % load_file)
r = yaml.load(open(load_file))
else:
release, dist = None, None
if not filter:
try:
release, dist = guess.osreleasedist()
log.info('Autodetected filter: %s/%s'
% (release, dist))
except exception.CantGuess as ex:
raise exception.CantGuess(
msg='%s\n\nPlease select RELEASE(/DIST) filter to query.' %
str(ex))
else:
release, _, dist = filter.partition('/')
module2pkg = True
if reqs_file:
log.info("Querying requirements file: %s" % reqs_file)
reqs = _reqs.get_reqs_from_path(reqs_file)
elif reqs_ref:
log.info("Querying requirements file from git: "
"%s -- requirements.txt" % reqs_ref)
reqs = _reqs.get_reqs_from_ref(reqs_ref)
else:
log.info("Querying .spec file")
module2pkg = False
reqs = _reqs.get_reqs_from_spec(as_objects=True)
log.info('')
r = _reqs.reqquery(reqs, release=release, dist=dist,
module2pkg=module2pkg, verbose=verbose)
if dump_file:
log.info("Saving query results to file: %s" % dump_file)
yaml.dump(r, open(dump_file, 'w'))
_reqs.print_reqquery(r)
def update_patches(branch,
local_patches_branch,
bump_only=False,
version=None,
new_version=None,
version_tag_style=None):
if bump_only:
return
target_version = new_version or version
if not target_version:
raise exception.RequiredActionArgumentNotAvailable(
action='update_patches', arg='version or new_version')
tag = guess.version2tag(target_version, version_tag_style)
_ensure_branch(local_patches_branch)
patches = list(git.get_commits(tag, local_patches_branch))
n_patches = len(patches)
_ensure_branch(branch)
spec = specfile.Spec()
spec.sanity_check()
patches_base, n_excluded = spec.get_patches_base()
ignore_regex = spec.get_patches_ignore_regex()
if ignore_regex and patches_base is None:
# TODO: patches_base and patches_ignore should be independent
# patches_ignore feature tests should help with this a lot
raise exception.OnlyPatchesIgnoreUsed()
pass
patch_fns = spec.get_patch_fns()
for pfn in patch_fns:
git('rm', '-f', '--ignore-unmatch', pfn)
patch_fns = []
if n_excluded > 0:
patches = patches[:-n_excluded]
patches.reverse()
ranges = [patches]
filtered_patches = patches
if ignore_regex:
ranges = _partition_patches(patches, ignore_regex)
filtered_patches = flatten(ranges)
n_filtered_out = len(patches) - len(filtered_patches)
if ignore_regex:
fmt = (
'\nUsing {t.bold}patches_ignore={t.normal}{t.magenta}%s'
'{t.normal} regexp to filter out patches.') % ignore_regex.pattern
else:
fmt = ('\nNo valid {t.bold}patches_ignore{t.normal} '
'filtering regex found in the .spec file.')
log.info(fmt.format(t=log.term))
log.info(
"\n{t.bold}{n} patches{t.normal} on top of {t.bold}{tag}{t.normal}"
", {t.bold}{ne}{t.normal} excluded by base"
", {t.bold}{nf}{t.normal} filtered out by regex.".format(
t=log.term, n=n_patches, tag=tag, ne=n_excluded,
nf=n_filtered_out))
if patches and filtered_patches:
for hsh, title in reversed(filtered_patches):
log.info("%s %s" % (log.term.green(hsh), title))
log.info("")
patch_fns = []
for patch_range in ranges:
start_commit, _title = patch_range[0]
end_commit, _title = patch_range[-1]
start_number = len(patch_fns) + 1
rng = git.rev_range(start_commit + '~', end_commit)
format_patch_cmd = [
'-c', 'core.abbrev=7', 'format-patch', '--no-renames',
'--no-signature', '-N', '--ignore-submodules',
'--start-number',
str(start_number), rng
]
o = git(*format_patch_cmd)
range_files = git._parse_output(o)
patch_fns.extend(range_files)
for pfn in patch_fns:
git('add', pfn)
spec.set_new_patches(patch_fns)
patches_branch_ref = git('rev-parse', local_patches_branch, log_cmd=False)
spec.set_commit_ref_macro(patches_branch_ref)
spec.save()
def run(cmd, *params, **kwargs):
fatal = kwargs.get('fatal', True)
direct = kwargs.get('direct', False)
log_cmd = kwargs.get('log_cmd', True)
log_fail = kwargs.get('log_fail', True)
input = kwargs.get('input')
print_stdout = kwargs.get('print_stdout', False)
print_stderr = kwargs.get('print_stderr', False)
print_output = kwargs.get('print_output', False)
env = kwargs.get('env', None)
cmd = [cmd] + list(params)
cmd_str = ' '.join(cmd)
if log_cmd:
log.command(log.term.cmd(cmd_str))
if print_output:
print_stdout = True
print_stderr = True
if input:
stdin = subprocess.PIPE
input = input.encode()
else:
stdin = None
if direct:
stdout = None
stderr = None
else:
stdout = subprocess.PIPE
stderr = subprocess.PIPE
try:
prc = subprocess.Popen(cmd, stdin=stdin, stdout=stdout,
stderr=stderr, env=env)
except OSError:
raise exception.CommandNotFound(cmd=cmd[0])
out, err = prc.communicate(input=input)
if out:
out = out.rstrip()
if print_stdout:
log.info(out)
else:
out = b''
if err:
err = err.rstrip()
if print_stderr:
log.info(err)
else:
err = b''
cout = _CommandOutput(out.decode('utf-8'))
cout.stderr = err
cout.return_code = prc.returncode
cout.cmd = cmd_str
if prc.returncode != 0:
if log_fail:
log_cmd_fail(cmd_str, cout)
if fatal:
raise exception.CommandFailed(cmd=cmd, out=cout)
return cout
