def build_complete(self): # use the repo search results to get a ref and a sha1; the # input to teuthology-suite doesn't contain both try: self.assert_result() except VersionNotFoundError: return False search_result = self._result.json()[0] # now look for the build complete status path = '/'.join( ('builds/ceph', search_result['ref'], search_result['sha1'])) build_url = urljoin(self.query_url, path) try: resp = requests.get(build_url) resp.raise_for_status() except requests.HttpError: return False for build in resp.json(): if (build['distro'] == search_result['distro'] and build['distro_version'] == search_result['distro_version'] and build['flavor'] == search_result['flavor'] and build['distro_arch'] == 'x86_64'): return build['status'] == 'completed' return False
def get_ceph_binary_url(package=None, branch=None, tag=None, sha1=None, dist=None, flavor=None, format=None, arch=None): """ return the url of the ceph binary found on gitbuildder. """ BASE = 'http://{host}/{package}-{format}-{dist}-{arch}-{flavor}/'.format( host=config.gitbuilder_host, package=package, flavor=flavor, arch=arch, format=format, dist=dist) if sha1 is not None: assert branch is None, "cannot set both sha1 and branch" assert tag is None, "cannot set both sha1 and tag" else: # gitbuilder uses remote-style ref names for branches, mangled to # have underscores instead of slashes; e.g. origin_master if tag is not None: ref = tag assert branch is None, "cannot set both branch and tag" else: if branch is None: branch = 'master' ref = branch sha1_url = urljoin(BASE, 'ref/{ref}/sha1'.format(ref=ref)) log.debug('Translating ref to sha1 using url %s', sha1_url) try: sha1_fp = urlopen(sha1_url) sha1 = sha1_fp.read().rstrip('\n') sha1_fp.close() except HTTPError as e: log.error('Failed to get url %s', sha1_url) raise e log.debug('Using %s %s sha1 %s', package, format, sha1) bindir_url = urljoin(BASE, 'sha1/{sha1}/'.format(sha1=sha1)) return (sha1, bindir_url)
def _search_uri(self): flavor = self.flavor req_obj = OrderedDict() req_obj['status'] = 'ready' req_obj['project'] = self.project req_obj['flavor'] = flavor arch = "noarch" if self.force_noarch else self.arch req_obj['distros'] = '%s/%s' % (self.distro, arch) ref_name, ref_val = list(self._choose_reference().items())[0] if ref_name == 'tag': req_obj['sha1'] = self._sha1 = self._tag_to_sha1() elif ref_name == 'sha1': req_obj['sha1'] = ref_val else: req_obj['ref'] = ref_val req_str = urlencode(req_obj) uri = urljoin( self.query_url, 'search', ) + '?%s' % req_str return uri
def build_complete(self): # use the repo search results to get a ref and a sha1; the # input to teuthology-suite doesn't contain both try: self.assert_result() except VersionNotFoundError: return False # self._result has status, project, flavor, distros, arch, and sha1 # restrictions, so the only reason for multiples should be "multiple # builds of the same sha1 etc."; the first entry is the newest search_result = self._result.json()[0] # now look for the build complete status path = '/'.join( ('builds/ceph', search_result['ref'], search_result['sha1'])) build_url = urljoin(self.query_url, path) try: resp = requests.get(build_url) resp.raise_for_status() except requests.HttpError: return False log.debug(f'looking for {self.distro} {self.arch} {self.flavor}') for build in resp.json(): log.debug( f'build: {build["distro"]}/{build["distro_version"]} {build["distro_arch"]} {build["flavor"]}' ) if ( # we must compare build arch to self.arch, since shaman's # results can have multiple arches but we're searching # for precisely one here build['distro'] == search_result['distro'] and build['distro_version'] == search_result['distro_version'] and build['flavor'] == search_result['flavor'] and build['distro_arch'] == self.arch): return build['status'] == 'completed' return False
def __init__(self, hosts, time_from, time_until='now'): super(PCPGrapher, self).__init__(hosts, time_from, time_until) self.base_url = urljoin(teuth_config.pcp_host, self._endpoint)
def repo_url(self): self.assert_result() return urljoin( self._result.json()[0]['chacra_url'], 'repo', )
def download_kernel(ctx, config): """ Supply each remote with a kernel package: - local kernels are copied over - gitbuilder kernels are downloaded - nothing is done for distro kernels :param ctx: Context :param config: Configuration """ procs = {} for role, src in config.items(): needs_download = False if src == 'distro': # don't need to download distro kernels log.debug("src is distro, skipping download"); continue (role_remote,) = ctx.cluster.only(role).remotes.keys() if isinstance(src, dict): # we're downloading a kernel from koji, the src dict here # is the build_info retrieved from koji using get_koji_build_info if src.get("id"): build_id = src["id"] log.info("Downloading kernel with build_id {build_id} on {role}...".format( build_id=build_id, role=role )) needs_download = True baseurl = get_kojiroot_base_url(src) pkg_name = get_koji_package_name("kernel", src) elif src.get("task_id"): needs_download = True log.info("Downloading kernel with task_id {task_id} on {role}...".format( task_id=src["task_id"], role=role )) baseurl = src["base_url"] # this var is also poorly named as it's not the package name, # but the full name of the rpm file to download. pkg_name = src["rpm_name"] elif src.find('/') >= 0: # local package - src is path log.info('Copying kernel package {path} to {role}...'.format( path=src, role=role)) role_remote.put_file(src,remote_pkg_path(role_remote)) else: # gitbuilder package - src is sha1 log.info('Downloading kernel {sha1} on {role}...'.format( sha1=src, role=role, )) needs_download = True builder = get_builder_project()( 'kernel', {'sha1': src}, ctx=ctx, remote=role_remote, ) if teuth_config.use_shaman: if role_remote.os.package_type == 'rpm': arch = builder.arch baseurl = urljoin( builder.base_url, '/'.join([arch, '']) ) pkg_name = "kernel-%s.%s.rpm" % ( builder.version, arch, ) elif role_remote.os.package_type == 'deb': arch = 'amd64' # FIXME baseurl = urljoin( builder.base_url, '/'.join([ 'pool', 'main', 'l', 'linux-%s' % builder.scm_version, '' ]) ) pkg_name = 'linux-image-%s_%s_%s.deb' % ( builder.scm_version, builder.version, arch, ) else: baseurl = builder.base_url + "/" pkg_name = gitbuilder_pkg_name(role_remote) log.info("fetching, builder baseurl is %s", baseurl) if needs_download: proc = role_remote.run( args=[ 'rm', '-f', remote_pkg_path(role_remote), run.Raw('&&'), 'echo', pkg_name, run.Raw('|'), 'wget', '-nv', '-O', remote_pkg_path(role_remote), '--base={url}'.format(url=baseurl), '--input-file=-', ], wait=False) procs[role_remote.name] = proc for name, proc in procs.items(): log.debug('Waiting for download/copy to %s to complete...', name) proc.wait()