def ensure_tagged(buildinfo, tags, session, dryrun):
"""
Ensure this build is tagged into Koji.
:param dict buildinfo: dict from getBuild with this name/version/release
:param list tags: list of tags for this build.
:param session: Koji session
:param bool dryrun: show what would happen, but don't do it.
"""
task_ids = []
nvr = '%(name)s-%(version)s-%(release)s' % buildinfo
for tag in sorted(tags):
tagged_builds = session.listTagged(tag,
package=buildinfo['name'],
type='debian')
tagged_nvrs = [tagged_build['nvr'] for tagged_build in tagged_builds]
if nvr in tagged_nvrs:
log.info('%s is already tagged into %s' % (nvr, tag))
continue
log.info('tagging %s into %s' % (nvr, tag))
if dryrun:
continue
task_id = session.tagBuild(tag, nvr)
task_ids.append(task_id)
if not task_ids:
# This build is already tagged into all the necessary tags.
return
task_result = watch_tasks(session, task_ids, poll_interval=15)
if task_result != 0 and not dryrun:
raise RuntimeError('failed to tag build %s' % nvr)
def main(args):
# Pre-flight checks
directory = args.directory
assert os.path.isdir(directory)
session = misoctl.session.get_session(args.profile)
# TODO: verify this session is authorized to import to the debian CG.
# Needs https://pagure.io/koji/pull-request/1160
owner = args.owner
verify_user(owner, session)
tag = args.tag
if tag:
verify_tag(tag, session)
buildinfo = import_from_directory(directory, session, args.owner,
args.skip_log, args.scm_url, args.dryrun)
log.info('imported %(name)s-%(version)s-%(release)s' % buildinfo)
if tag:
tag_build(buildinfo, tag, session)
else:
log.info('not tagging %(name)s-%(version)s-%(release)s' % buildinfo)
def ensure_uploaded(nvr, chacra_url, rsession, session, owner, scm_template,
dryrun):
"""
Ensure this build is uploaded into Koji.
:param nvr: build's name_versionrelease in chacra
:param chacra_url: base url to chacra instance
:param rsession: requests.Session object
:param session: Koji session
:param owner: Koji user name that will own this build
:param scm_template: format string for this build's scm_url
:param dryrun: if True, show what would have happened, but don't do it
"""
koji_nvr = get_koji_nvr(nvr)
# Check if this build exists in Koji
buildinfo = session.getBuild(koji_nvr)
if buildinfo:
return buildinfo
if dryrun:
log.info('would download chacra build %s' % nvr)
return
directory = chacra.download_build(nvr, chacra_url, rsession)
skip_log = True
(name, version) = chacra.name_version(nvr)
scm_url = scm_template.format(name=name)
buildinfo = upload.import_from_directory(directory, session, owner,
skip_log, scm_url, dryrun)
return buildinfo
def tag_build(buildinfo, tag, session):
""" Tag this build in Koji. """
nvr = '%(name)s-%(version)s-%(release)s' % buildinfo
log.info('tagging %s into %s' % (nvr, tag))
task_id = session.tagBuild(tag, nvr)
task_result = watch_tasks(session, [task_id], poll_interval=15)
if task_result != 0:
raise RuntimeError('failed to tag builds')
def upload(all_files, session):
"""
Upload all files to a remote directory in Koji.
"""
remote_directory = unique_path('cli-import')
log.info('uploading files to %s' % remote_directory)
for filename in all_files:
callback = _progress_callback
log.info("Uploading %s" % filename)
session.uploadWrapper(filename, remote_directory, callback=callback)
if callback:
print('')
return remote_directory
def get_session(profile):
"""
Return an authenticated Koji session
"""
# Return a cached session, if available.
mykoji = koji.get_profile_module(profile)
opts = mykoji.grab_session_options(mykoji.config)
session = mykoji.ClientSession(mykoji.config.server, opts)
# Log in ("activate") this sesssion:
# Note: this can raise SystemExit if there is a problem, eg with Kerberos:
activate_session(session, mykoji.config)
assert session.logged_in
userinfo = session.getLoggedInUser()
username = userinfo['name']
log.info('authenticated to %s as %s' % (mykoji.config.server, username))
return session
def main(args):
rsession = requests.Session()
session = misoctl.session.get_session(args.profile)
upload.verify_user(args.owner, session)
nvrs = find_all_nvrs(args.directory)
sorted_nvrs = sort_nvrs(nvrs.keys())
for nvr in sorted_nvrs:
log.info('nvr: "%s"' % nvr)
buildinfo = ensure_uploaded(nvr, args.chacra_url, rsession, session,
args.owner, args.scm_template, args.dryrun)
if args.dryrun and not buildinfo:
# Minimally fake the buildinfo we would have generated above.
(name, version, release) = chacra.name_version_release(nvr)
buildinfo = {'name': name, 'version': version, 'release': release}
tags = nvrs[nvr]
ensure_tagged(buildinfo, tags, session, args.dryrun)
def download_build(nvr, base_url, session):
"""
Download an NVR from chacra to a nvr-named directory.
:param nvr: build NVR to download, eg ceph-ansible_3.2.0~rc3-2redhat1
:param base_url: chacra base URL
:param session: persistent requests.Session() to use for HTTPS requests
:returns: destination directory for this build
"""
(pkg, version) = name_version(nvr)
dest_dir = os.path.join('downloads', nvr)
ensure_directory(dest_dir)
build_url = posixpath.join(base_url, 'binaries/', pkg, version, 'ubuntu',
'all')
log.info('searching %s for builds' % build_url)
build_response = session.get(build_url)
build_response.raise_for_status()
payload = build_response.json()
for arch, binaries in payload.items():
metadata_url = posixpath.join(build_url, arch)
metadata_response = session.get(metadata_url)
metadata_response.raise_for_status()
metadata = metadata_response.json()
for binary in binaries:
output_path = os.path.join(dest_dir, binary)
if os.path.isfile(output_path):
checksum = metadata[binary]['checksum']
if verify_checksum(output_path, checksum):
log.info('skipping %s' % binary)
continue
else:
log.warning('checksum mismatch on %s' % binary)
log.info('downloading %s' % binary)
binary_url = posixpath.join(build_url, arch, binary) + '/'
r = session.get(binary_url, stream=True)
r.raise_for_status()
with open(output_path, 'wb') as f:
for chunk in r.iter_content(4096):
f.write(chunk)
return dest_dir
def import_from_directory(directory, session, owner, skip_log, scm_url,
dryrun):
"""
Import the build artifacts in this directory into a Koji CG build.
:param directory: dir containing the build artifacts, with one dsc file.
:param session: Koji session.
:param owner: Koji user to own this imported build.
:param skip_log: Don't try to import log files for this build.
:param scm_url: SCM (dist-git) url for this build.
:param dryrun: show what would be done, but don't do it.
"""
# Discover our files on disk
dsc_file = filemanager.find_dsc_file(directory)
dsc = filemanager.parse_dsc(dsc_file)
source_files = filemanager.find_source_files(dsc, directory)
deb_files = filemanager.find_deb_files(directory)
log_files = set()
log_file = filemanager.find_log_file(directory, fatal=not skip_log)
if log_file:
log_file = rename_log_file(log_file)
log_files.add(log_file)
# Bail early if this build already exists
nvr = '%(Source)s-deb-%(Version)s' % dsc
if session.getBuild(nvr):
raise RuntimeError('%s build exists in koji' % nvr)
# Determine build metadata
if log_file:
(start_time, end_time) = filemanager.get_build_times(log_file)
else:
# This is not optimial, because the start and end times are the same,
# so it looks as if the build took zero seconds.
changes_file = filemanager.find_changes_file(directory)
changes_time = filemanager.get_changes_time(changes_file)
start_time = changes_time
end_time = changes_time
build = get_build_data(dsc, start_time, end_time, scm_url, owner)
# Determine buildroot metadata
buildroots = get_buildroots()
# Determine output metadata
dsc_files = set([dsc_file])
all_files = set.union(dsc_files, source_files, deb_files, log_files)
output = get_output_data(all_files)
# Generate the main metdata JSON
metadata = get_metadata(build, buildroots, output)
with open('metadata.json', 'w') as f:
json.dump(metadata, f)
all_files.add('metadata.json')
# TODO: check if this build already exists in Koji before uploading here
if dryrun:
log.info('dryrun: would upload')
for filename in all_files:
log.info(filename)
return {}
buildinfo = cg_import(all_files, metadata, session)
return buildinfo