class ClientCommandContainer(CommandContainer): def __init__(self, conf, **kwargs): self.conf = PyConfigParser() self.conf.load_from_conf(conf) self.conf.load_from_dict(kwargs) def set_hub(self, username=None, password=None, auto_login=True, proxy_user=None): if username: if password is None: password = password_prompt(default_value=password) self.conf["AUTH_METHOD"] = "password" self.conf["USERNAME"] = username self.conf["PASSWORD"] = password if proxy_user: self.conf["PROXY_USER"] = proxy_user cacert = self.conf.get('CA_CERT') if cacert and not os.path.exists(cacert): self.error( 'CA_CERT configuration points to non-existing file: %s' % cacert) self.hub = HubProxy(conf=self.conf, auto_login=auto_login)
class ClientCommandContainer(CommandContainer): def __init__(self, conf, **kwargs): self.conf = PyConfigParser() self.conf.load_from_conf(conf) self.conf.load_from_dict(kwargs) def set_hub(self, username=None, password=None, auto_login=True, proxy_user=None): if username: if password is None: password = password_prompt(default_value=password) self.conf["AUTH_METHOD"] = "password" self.conf["USERNAME"] = username self.conf["PASSWORD"] = password if proxy_user: self.conf["PROXY_USER"] = proxy_user cacert = self.conf.get('CA_CERT') if cacert and not os.path.exists(cacert): raise BeakerClientConfigurationError('CA_CERT configuration points to non-existing file: %s' % cacert) self.hub = HubProxy(conf=self.conf, auto_login=auto_login)
class HubProxy(object): """A Hub client (thin ServerProxy wrapper).""" def __init__(self, conf, client_type=None, logger=None, transport=None, auto_login=True, timeout=120, **kwargs): self._conf = PyConfigParser() self._hub = None # load default config default_config = os.path.abspath(os.path.join(os.path.dirname(__file__), "default.conf")) self._conf.load_from_file(default_config) # update config with another one if conf is not None: self._conf.load_from_conf(conf) # update config with kwargs self._conf.load_from_dict(kwargs) # initialize properties self._client_type = client_type or "client" self._hub_url = self._conf["HUB_URL"] self._auth_method = self._conf["AUTH_METHOD"] self._logger = logger self._logged_in = False if transport is not None: self._transport = transport else: transport_args = {'timeout': timeout} if self._hub_url.startswith("https://"): TransportClass = retry_request_decorator(SafeCookieTransport) if hasattr(ssl, 'create_default_context') and self._conf.get('CA_CERT'): ssl_context = ssl.create_default_context() ssl_context.load_verify_locations(cafile=self._conf['CA_CERT']) transport_args['context'] = ssl_context else: TransportClass = retry_request_decorator(CookieTransport) self._transport = TransportClass(**transport_args) self._hub = xmlrpclib.ServerProxy( "%s/%s/" % (self._hub_url, self._client_type), allow_none=True, transport=self._transport, verbose=self._conf.get("DEBUG_XMLRPC")) if auto_login: self._login() def __del__(self): if hasattr(self._transport, "retry_count"): self._transport.retry_count = 0 def __getattr__(self, name): try: return getattr(self._hub, name) except: raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, name)) def _login(self, force=False): """Login to the hub. - self._hub instance is created in this method - session information is stored in a cookie in self._transport """ if self._auth_method == "none" or not self._auth_method: return login_method_name = "_login_%s" % self._auth_method if not hasattr(self, login_method_name): raise ImproperlyConfigured("Unknown authentication method: %s" % self._auth_method) self._logger and self._logger.info("Creating new session...") try: login_method = getattr(self, login_method_name) login_method() self._logged_in = True except KeyboardInterrupt: raise except Exception, ex: self._logger and self._logger.error("Failed to create new session: %s" % ex) raise else:
class BeakerTargets(object): def __init__(self, params={}, logger=None): self.__dict__ = params.copy() self.conf = PyConfigParser() default_config = os.path.expanduser(BEAKER_CONF) self.conf.load_from_file(default_config) self.hub = HubProxy(logger=logger, conf=self.conf) def _get_url(self, bkr_id): """ Constructs the Beaker URL for the job related to the provided Beaker ID. That ID should be all numeric, unless the structure of Beaker changes in the future. If that's the case, then the ID should be appropriately URL encoded to be appended to the end of a URL properly. """ base = self.conf.get('HUB_URL', '') if base == '': raise Exception("Unable to construct URL") if base[-1] != '/': base += '/' return base + 'jobs/' + bkr_id def get_system_statuses(self): """ Checks on the status of a set of Beaker jobs (ids) and returns their hostname once the jobs have reached their defined status. """ attempts = 0 pass_count = 0 all_count = len(self.ids) while attempts < self.max_attempts: job_results = self._check_jobs(self.ids) pass_count = 0 for resource in job_results['resources']: result = resource['result'] status = resource['status'] print >> stderr, "status: %s, result: %s" % (status, result) if status not in ['Cancelled', 'Aborted']: if (result == 'Pass' or (result == 'Warn' and self.skip_no_system)): pass_count += 1 elif result in ['Fail', 'Warn', 'Panic', 'Completed']: raise Exception("System failed with state" " '{0}'".format(result)) elif status == 'Aborted': if result == 'Warn' and self.skip_no_system: pass_count += 1 else: raise Exception("System aborted") elif status == 'Cancelled': raise Exception("System canceled") attempts += 1 if pass_count == all_count: return job_results['resources'] sleep(WAIT_TIME) raise Exception("{0} system(s) never completed in {1}" " polling attempts. {2}".format( all_count - pass_count, attempts, dumps(job_results))) def _check_jobs(self, ids): """ Get state of a job in Beaker """ jobs = ["J:" + _id for _id in ids] results = {} resources = [] bkrcmd = BeakerCommand('BeakerCommand') bkrcmd.check_taskspec_args(jobs) for task in jobs: myxml = self.hub.taskactions.to_xml(task) myxml = myxml.encode('utf8') root = eT.fromstring(myxml) # TODO: Using getiterator() since its backward compatible # with Python 2.6 # This is deprectated in 2.7 and we should be using iter() for job in root.getiterator('job'): results.update({ 'job_id': job.get('id'), 'results': job.get('result') }) for recipe in root.getiterator('recipe'): resources.append({ 'family': recipe.get('family'), 'distro': recipe.get('distro'), 'arch': recipe.get('arch'), 'variant': recipe.get('variant'), 'system': recipe.get('system'), 'status': recipe.get('status'), 'result': recipe.get('result'), 'id': recipe.get('job_id') }) results.update({'resources': resources}) return results
class HubProxy(object): """A Hub client (thin ServerProxy wrapper).""" def __init__(self, conf, client_type=None, logger=None, transport=None, auto_login=True, timeout=120, **kwargs): self._conf = PyConfigParser() self._hub = None # load default config default_config = os.path.abspath( os.path.join(os.path.dirname(__file__), "default.conf")) self._conf.load_from_file(default_config) # update config with another one if conf is not None: self._conf.load_from_conf(conf) # update config with kwargs self._conf.load_from_dict(kwargs) # initialize properties self._client_type = client_type or "client" self._hub_url = self._conf["HUB_URL"] self._auth_method = self._conf["AUTH_METHOD"] self._logger = logger self._logged_in = False if transport is not None: self._transport = transport else: transport_args = {'timeout': timeout} if self._hub_url.startswith("https://"): TransportClass = retry_request_decorator(SafeCookieTransport) if hasattr(ssl, 'create_default_context') and self._conf.get( 'CA_CERT'): ssl_context = ssl.create_default_context() ssl_context.load_verify_locations( cafile=self._conf['CA_CERT']) transport_args['context'] = ssl_context else: TransportClass = retry_request_decorator(CookieTransport) self._transport = TransportClass(**transport_args) self._hub = xmlrpclib.ServerProxy( "%s/%s/" % (self._hub_url, self._client_type), allow_none=True, transport=self._transport, verbose=self._conf.get("DEBUG_XMLRPC")) if auto_login: self._login() def __del__(self): if hasattr(self._transport, "retry_count"): self._transport.retry_count = 0 def __getattr__(self, name): try: return getattr(self._hub, name) except: raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, name)) def _login(self, force=False): """Login to the hub. - self._hub instance is created in this method - session information is stored in a cookie in self._transport """ if self._auth_method == "none" or not self._auth_method: return login_method_name = "_login_%s" % self._auth_method if not hasattr(self, login_method_name): raise ImproperlyConfigured("Unknown authentication method: %s" % self._auth_method) self._logger and self._logger.info("Creating new session...") try: login_method = getattr(self, login_method_name) login_method() self._logged_in = True except KeyboardInterrupt: raise except Exception, ex: self._logger and self._logger.error( "Failed to create new session: %s" % ex) raise else:
class HubProxy(object): """ A Hub client (thin ServerProxy wrapper). """ def __init__(self, conf, client_type=None, logger=None, transport=None, auto_login=True, timeout=120, **kwargs): self._conf = PyConfigParser() self._hub = None # load default config default_config = os.path.abspath( os.path.join(os.path.dirname(__file__), "default.conf")) self._conf.load_from_file(default_config) # update config with another one if conf is not None: self._conf.load_from_conf(conf) # update config with kwargs self._conf.load_from_dict(kwargs) # initialize properties self._client_type = client_type or "client" self._hub_url = self._conf["HUB_URL"] self._auth_method = self._conf["AUTH_METHOD"] self._logger = logger self._logged_in = False if transport is not None: self._transport = transport else: transport_args = {'timeout': timeout} if self._hub_url.startswith("https://"): TransportClass = retry_request_decorator(SafeCookieTransport) if hasattr(ssl, 'create_default_context') and self._conf.get( 'CA_CERT'): ssl_context = ssl.create_default_context() ssl_context.load_verify_locations( cafile=self._conf['CA_CERT']) transport_args['context'] = ssl_context elif (hasattr(ssl, '_create_unverified_context') and not self._conf.get('SSL_VERIFY', True)): # Python 2.6 doesn't have context argument for xmlrpclib.ServerProxy # therefore transport needs to be modified ssl_context = ssl._create_unverified_context() transport_args['context'] = ssl_context else: TransportClass = retry_request_decorator(CookieTransport) self._transport = TransportClass(**transport_args) self._hub = xmlrpc_client.ServerProxy( "%s/%s/" % (self._hub_url, self._client_type), allow_none=True, transport=self._transport, verbose=self._conf.get("DEBUG_XMLRPC")) if auto_login: self._login() def __del__(self): if hasattr(self._transport, "retry_count"): self._transport.retry_count = 0 def __getattr__(self, name): try: return getattr(self._hub, name) except: raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, name)) def _login(self, force=False): """Login to the hub. - self._hub instance is created in this method - session information is stored in a cookie in self._transport """ if self._auth_method == "none" or not self._auth_method: return login_method_name = "_login_%s" % self._auth_method if not hasattr(self, login_method_name): raise ImproperlyConfigured("Unknown authentication method: %s" % self._auth_method) self._logger and self._logger.info("Creating new session...") try: login_method = getattr(self, login_method_name) login_method() self._logged_in = True except KeyboardInterrupt: raise except Exception as ex: self._logger and self._logger.error( "Failed to create new session: %s" % ex) raise else: self._logger and self._logger.info("New session created.") def _logout(self): """No-op for backwards compatibility.""" pass def _login_password(self): """Login using username and password.""" username = self._conf.get("USERNAME") password = self._conf.get("PASSWORD") proxyuser = self._conf.get("PROXY_USER") if not username: raise AuthenticationError("USERNAME is not set") self._hub.auth.login_password(username, password, proxyuser) def _login_oauth2(self): """Login using OAuth2 access token.""" access_token = self._conf.get("ACCESS_TOKEN") if not access_token: raise AuthenticationError("ACCESS_TOKEN is not set") self._hub.auth.login_oauth2(access_token) def _login_krbv(self): """ Login using kerberos credentials (uses python-gssapi). """ def get_server_principal(service=None, realm=None): """ Convert hub url to kerberos principal. """ hostname = urlparse.urlparse(self._hub_url)[1] # remove port from hostname hostname = hostname.split(":")[0] if realm is None: # guess realm: last two parts from hostname realm = ".".join(hostname.split(".")[-2:]).upper() if service is None: service = "HTTP" return '%s/%s@%s' % (service, hostname, realm) # read default values from settings principal = self._conf.get("KRB_PRINCIPAL") keytab = self._conf.get("KRB_KEYTAB") service = self._conf.get("KRB_SERVICE") realm = self._conf.get("KRB_REALM") ccache = self._conf.get("KRB_CCACHE") proxyuser = self._conf.get("PROXY_USER") krb5kdc_err_s_principal_unknown = 2529638919 # Server not found in Kerberos database name = None if principal: name = gssapi.Name(principal, gssapi.NameType.kerberos_principal) store = None # Default ccache if keytab: # Make sure we are using always APP ccache or user specified ccache # instead of MIT krb5 default one with keytabs. Default ccache can be occupied by # system application store = { 'client_keytab': keytab, 'ccache': ccache or tempfile.NamedTemporaryFile(prefix='krb5cc_bkr_').name } elif ccache: store = {'ccache': ccache} creds = gssapi.Credentials(name=name, store=store, usage='initiate') target_name = gssapi.Name(get_server_principal(service, realm)) try: res = gssapi.raw.init_sec_context( target_name, creds, flags=( gssapi.RequirementFlag.out_of_sequence_detection | gssapi.RequirementFlag.replay_detection | gssapi.RequirementFlag.mutual_authentication | # This is a hack which causes GSSAPI to give us back a raw # KRB_AP_REQ token value, without GSSAPI header wrapping, which # is what the Beaker server is expecting in auth.login_krbv. gssapi.RequirementFlag.dce_style)) except gssapi.raw.GSSError as ex: if ex.min_code == krb5kdc_err_s_principal_unknown: # pylint: disable=no-member ex.message += ". Make sure you correctly set KRB_REALM (current value: %s)." % realm ex.args = (ex.message, ) raise ex if six.PY2: req_enc = base64.encodestring(res.token) else: req_enc = base64.encodebytes(res.token) # pylint: disable=maybe-no-member try: req_enc = str(req_enc, 'utf-8') # bytes to string except TypeError: pass self._hub.auth.login_krbv(req_enc, proxyuser)
class BeakerTargets(object): def __init__(self, params={}, logger=None): self.__dict__ = params.copy() self.conf = PyConfigParser() default_config = os.path.expanduser(BEAKER_CONF) self.conf.load_from_file(default_config) self.hub = HubProxy(logger=logger, conf=self.conf) def _get_url(self, bkr_id): """ Constructs the Beaker URL for the job related to the provided Beaker ID. That ID should be all numeric, unless the structure of Beaker changes in the future. If that's the case, then the ID should be appropriately URL encoded to be appended to the end of a URL properly. """ base = self.conf.get('HUB_URL', '') if base == '': raise Exception("Unable to construct URL") if base[-1] != '/': base += '/' return base + 'jobs/' + bkr_id def get_system_statuses(self): """ Checks on the status of a set of Beaker jobs (ids) and returns their hostname once the jobs have reached their defined status. """ attempts = 0 pass_count = 0 all_count = len(self.ids) while attempts < self.max_attempts: job_results = self._check_jobs(self.ids) pass_count = 0 for resource in job_results['resources']: result = resource['result'] status = resource['status'] print >> stderr, "status: %s, result: %s" % (status, result) if status not in ['Cancelled', 'Aborted']: if result == 'Pass' or (result == 'Warn' and self.skip_no_system): pass_count += 1 elif result in ['Fail', 'Warn', 'Panic', 'Completed']: raise Exception("System failed with state '{0}'"\ .format(result)) elif status == 'Aborted': if result == 'Warn' and self.skip_no_system: pass_count += 1 else: raise Exception("System aborted") elif status == 'Cancelled': raise Exception("System canceled") attempts += 1 if pass_count == all_count: return job_results['resources'] sleep(WAIT_TIME) raise Exception("{0} system(s) never completed in {1} polling attempts. {2}"\ .format(all_count - pass_count, attempts, dumps(job_results))) def _check_jobs(self, ids): """ Get state of a job in Beaker """ jobs = ["J:" + _id for _id in ids] results = {} resources = [] bkrcmd = BeakerCommand('BeakerCommand') bkrcmd.check_taskspec_args(jobs) for task in jobs: myxml = self.hub.taskactions.to_xml(task) myxml = myxml.encode('utf8') root = eT.fromstring(myxml) # TODO: Using getiterator() since its backward compatible # with Python 2.6 # This is deprectated in 2.7 and we should be using iter() for job in root.getiterator('job'): results.update({'job_id': job.get('id'), 'results': job.get('result')}) for recipe in root.getiterator('recipe'): resources.append({'family': recipe.get('family'), 'distro': recipe.get('distro'), 'arch': recipe.get('arch'), 'variant': recipe.get('variant'), 'system': recipe.get('system'), 'status': recipe.get('status'), 'result': recipe.get('result'), 'id': recipe.get('job_id')}) results.update({'resources': resources}) return results