def build_complete(self):
# use the repo search results to get a ref and a sha1; the
# input to teuthology-suite doesn't contain both
try:
self.assert_result()
except VersionNotFoundError:
return False
search_result = self._result.json()[0]
# now look for the build complete status
path = '/'.join(
('builds/ceph', search_result['ref'], search_result['sha1']))
build_url = urljoin(self.query_url, path)
try:
resp = requests.get(build_url)
resp.raise_for_status()
except requests.HttpError:
return False
for build in resp.json():
if (build['distro'] == search_result['distro'] and
build['distro_version'] == search_result['distro_version']
and build['flavor'] == search_result['flavor']
and build['distro_arch'] == 'x86_64'):
return build['status'] == 'completed'
return False
def get_ceph_binary_url(package=None,
branch=None,
tag=None,
sha1=None,
dist=None,
flavor=None,
format=None,
arch=None):
"""
return the url of the ceph binary found on gitbuildder.
"""
BASE = 'http://{host}/{package}-{format}-{dist}-{arch}-{flavor}/'.format(
host=config.gitbuilder_host,
package=package,
flavor=flavor,
arch=arch,
format=format,
dist=dist)
if sha1 is not None:
assert branch is None, "cannot set both sha1 and branch"
assert tag is None, "cannot set both sha1 and tag"
else:
# gitbuilder uses remote-style ref names for branches, mangled to
# have underscores instead of slashes; e.g. origin_master
if tag is not None:
ref = tag
assert branch is None, "cannot set both branch and tag"
else:
if branch is None:
branch = 'master'
ref = branch
sha1_url = urljoin(BASE, 'ref/{ref}/sha1'.format(ref=ref))
log.debug('Translating ref to sha1 using url %s', sha1_url)
try:
sha1_fp = urlopen(sha1_url)
sha1 = sha1_fp.read().rstrip('\n')
sha1_fp.close()
except HTTPError as e:
log.error('Failed to get url %s', sha1_url)
raise e
log.debug('Using %s %s sha1 %s', package, format, sha1)
bindir_url = urljoin(BASE, 'sha1/{sha1}/'.format(sha1=sha1))
return (sha1, bindir_url)
def _search_uri(self):
flavor = self.flavor
req_obj = OrderedDict()
req_obj['status'] = 'ready'
req_obj['project'] = self.project
req_obj['flavor'] = flavor
arch = "noarch" if self.force_noarch else self.arch
req_obj['distros'] = '%s/%s' % (self.distro, arch)
ref_name, ref_val = list(self._choose_reference().items())[0]
if ref_name == 'tag':
req_obj['sha1'] = self._sha1 = self._tag_to_sha1()
elif ref_name == 'sha1':
req_obj['sha1'] = ref_val
else:
req_obj['ref'] = ref_val
req_str = urlencode(req_obj)
uri = urljoin(
self.query_url,
'search',
) + '?%s' % req_str
return uri
def build_complete(self):
# use the repo search results to get a ref and a sha1; the
# input to teuthology-suite doesn't contain both
try:
self.assert_result()
except VersionNotFoundError:
return False
# self._result has status, project, flavor, distros, arch, and sha1
# restrictions, so the only reason for multiples should be "multiple
# builds of the same sha1 etc."; the first entry is the newest
search_result = self._result.json()[0]
# now look for the build complete status
path = '/'.join(
('builds/ceph', search_result['ref'], search_result['sha1']))
build_url = urljoin(self.query_url, path)
try:
resp = requests.get(build_url)
resp.raise_for_status()
except requests.HttpError:
return False
log.debug(f'looking for {self.distro} {self.arch} {self.flavor}')
for build in resp.json():
log.debug(
f'build: {build["distro"]}/{build["distro_version"]} {build["distro_arch"]} {build["flavor"]}'
)
if (
# we must compare build arch to self.arch, since shaman's
# results can have multiple arches but we're searching
# for precisely one here
build['distro'] == search_result['distro'] and
build['distro_version'] == search_result['distro_version']
and build['flavor'] == search_result['flavor']
and build['distro_arch'] == self.arch):
return build['status'] == 'completed'
return False
def __init__(self, hosts, time_from, time_until='now'):
super(PCPGrapher, self).__init__(hosts, time_from, time_until)
self.base_url = urljoin(teuth_config.pcp_host, self._endpoint)
def repo_url(self):
self.assert_result()
return urljoin(
self._result.json()[0]['chacra_url'],
'repo',
)
def download_kernel(ctx, config):
"""
Supply each remote with a kernel package:
- local kernels are copied over
- gitbuilder kernels are downloaded
- nothing is done for distro kernels
:param ctx: Context
:param config: Configuration
"""
procs = {}
for role, src in config.items():
needs_download = False
if src == 'distro':
# don't need to download distro kernels
log.debug("src is distro, skipping download");
continue
(role_remote,) = ctx.cluster.only(role).remotes.keys()
if isinstance(src, dict):
# we're downloading a kernel from koji, the src dict here
# is the build_info retrieved from koji using get_koji_build_info
if src.get("id"):
build_id = src["id"]
log.info("Downloading kernel with build_id {build_id} on {role}...".format(
build_id=build_id,
role=role
))
needs_download = True
baseurl = get_kojiroot_base_url(src)
pkg_name = get_koji_package_name("kernel", src)
elif src.get("task_id"):
needs_download = True
log.info("Downloading kernel with task_id {task_id} on {role}...".format(
task_id=src["task_id"],
role=role
))
baseurl = src["base_url"]
# this var is also poorly named as it's not the package name,
# but the full name of the rpm file to download.
pkg_name = src["rpm_name"]
elif src.find('/') >= 0:
# local package - src is path
log.info('Copying kernel package {path} to {role}...'.format(
path=src, role=role))
role_remote.put_file(src,remote_pkg_path(role_remote))
else:
# gitbuilder package - src is sha1
log.info('Downloading kernel {sha1} on {role}...'.format(
sha1=src,
role=role,
))
needs_download = True
builder = get_builder_project()(
'kernel',
{'sha1': src},
ctx=ctx,
remote=role_remote,
)
if teuth_config.use_shaman:
if role_remote.os.package_type == 'rpm':
arch = builder.arch
baseurl = urljoin(
builder.base_url,
'/'.join([arch, ''])
)
pkg_name = "kernel-%s.%s.rpm" % (
builder.version,
arch,
)
elif role_remote.os.package_type == 'deb':
arch = 'amd64' # FIXME
baseurl = urljoin(
builder.base_url,
'/'.join([
'pool', 'main', 'l',
'linux-%s' % builder.scm_version, ''
])
)
pkg_name = 'linux-image-%s_%s_%s.deb' % (
builder.scm_version,
builder.version,
arch,
)
else:
baseurl = builder.base_url + "/"
pkg_name = gitbuilder_pkg_name(role_remote)
log.info("fetching, builder baseurl is %s", baseurl)
if needs_download:
proc = role_remote.run(
args=[
'rm', '-f', remote_pkg_path(role_remote),
run.Raw('&&'),
'echo',
pkg_name,
run.Raw('|'),
'wget',
'-nv',
'-O',
remote_pkg_path(role_remote),
'--base={url}'.format(url=baseurl),
'--input-file=-',
],
wait=False)
procs[role_remote.name] = proc
for name, proc in procs.items():
log.debug('Waiting for download/copy to %s to complete...', name)
proc.wait()
