def __init__(self, cfg, response_handler=None, pulp_host=None, local=False): """Initialize this object with needed instance attributes.""" # How do we make requests? if not pulp_host: if cfg.pulp_version < Version("3"): pulp_host = cfg.get_hosts("pulp cli")[0] else: pulp_host = cfg.get_hosts("shell")[0] if local: pulp_host = collections.namedtuple("Host", "hostname roles") pulp_host.hostname = "localhost" pulp_host.roles = {"shell": {"transport": "local"}} self.pulp_host = pulp_host self.response_handler = response_handler or code_handler self.cfg = cfg self._is_root_cache = None self._machine = None self._transport = None self._podname = None logger.debug("New %s", self)
def run(self, args, sudo=False, **kwargs): """Run a command and ``return self.response_handler(result)``. This method is a thin wrapper around Plumbum's `BaseCommand.run`_ method, which is itself a thin wrapper around the standard library's `subprocess.Popen`_ class. See their documentation for detailed usage instructions. See :class:`pulp_smash.cli.Client` for a usage example. :param args: Any arguments to be passed to the process (a tuple). :param sudo: If the command should run as superuser (a boolean). :param kwargs: Extra named arguments passed to plumbumBaseCommand.run. .. _BaseCommand.run: http://plumbum.readthedocs.io/en/latest/api/commands.html#plumbum.commands.base.BaseCommand.run .. _subprocess.Popen: https://docs.python.org/3/library/subprocess.html#subprocess.Popen """ # Let self.response_handler check return codes. See: # https://plumbum.readthedocs.io/en/latest/api/commands.html#plumbum.commands.base.BaseCommand.run kwargs.setdefault("retcode") logger.debug("Running %s cmd (sudo:%s) - %s", args, sudo, kwargs) if self._podname: args = ("kubectl", "exec", self._podname, "--") + tuple(args) if sudo and args[0] != "sudo" and not self.is_superuser: args = ("sudo", ) + tuple(args) code, stdout, stderr = self.machine[args[0]].run(args[1:], **kwargs) completed_process = CompletedProcess(args, code, stdout, stderr) logger.debug("Finished %s command: %s", args, (code, stdout, stderr)) return self.response_handler(completed_process)
def page_handler(client, response): """Call :meth:`json_handler`, optionally collect results, and return. Do the following: 1. If ``response`` has an HTTP No Content (204) `status code`_, return ``response``. 2. Call :meth:`json_handler`. 3. If the response appears to be paginated, walk through each page of results, and collect them into a single list. Otherwise, do nothing. Return either the list of results or the single decoded response. :raises: ``ValueError`` if the target Pulp application under test is older than version 3 or at least version 4. .. _status code: https://en.wikipedia.org/wiki/List_of_HTTP_status_codes """ check_pulp3_restriction(client) maybe_page = json_handler(client, response) if not isinstance(maybe_page, dict): return maybe_page # HTTP 204 No Content if "results" not in maybe_page: return maybe_page # Content isn't a page. collected_results = [] for result in _walk_pages(client._cfg, maybe_page, client.pulp_host): collected_results.extend(result) logger.debug("paginated %s result pages", len(collected_results)) return collected_results
def request(self, method, url, **kwargs): """Send an HTTP request. Arguments passed directly in to this method override (but do not overwrite!) arguments specified in ``self.request_kwargs``. """ # The `self.request_kwargs` dict should *always* have a "url" argument. # This is enforced by `self.__init__`. This allows us to call the # `requests.request` function and satisfy its signature: # # request(method, url, **kwargs) # intended_host = self.pulp_host.hostname request_kwargs = self.request_kwargs.copy() request_kwargs["url"] = urljoin(request_kwargs["url"], url) request_kwargs.update(kwargs) actual_host = urlparse(request_kwargs["url"]).hostname if intended_host != actual_host: warnings.warn( "This client should be used to communicate with {0}, but a " "request is being made to {1}. The request will be made, but " "beware that information intended for {0} (such as " "authentication tokens) may now be sent to {1}. Here's the " "list of options being sent with this request: {2}".format( intended_host, actual_host, request_kwargs), RuntimeWarning, ) logger.debug("Making a %s request with %s", method, request_kwargs) response = self.response_handler( self, requests.request(method, **request_kwargs)) logger.debug("Finished %s request with response: %s", method, response) return response
def machine(self): """Initialize the plumbum machine lazily.""" if self._machine is None: if self.transport == "local": self._machine = plumbum.machines.local elif self.transport == "kubectl": self._machine = plumbum.machines.local chain = (self._machine["sudo"]["kubectl", "get", "pods"] | self._machine["grep"]["-E", "-o", r"pulp-api-(\w+)-(\w+)"]) self._podname = chain().replace("\n", "") elif self.transport in ["docker", "podman"]: self._machine = plumbum.machines.local self._podname = self.pulp_host.roles.get("shell", {}).get( "container", "pulp") elif self.transport == "ssh": # The SshMachine is a wrapper around the host's "ssh" binary. # Thus, it uses ~/.ssh/config, ~/.ssh/known_hosts, etc. hostname = self.pulp_host.hostname self._machine = plumbum.machines.SshMachine(hostname) else: raise NotImplementedError( "Transport ({}) is not implemented.".format( self.transport)) logger.debug("Initialized plumbum machine %s", self._machine) return self._machine
def poll_task(cfg, href, pulp_host=None): """Wait for a task and its children to complete. Yield response bodies. Poll the task at ``href``, waiting for the task to complete. When a response is received indicating that the task is complete, yield that response body and recursively poll each child task. :param cfg: A :class:`pulp_smash.config.PulpSmashConfig` object. :param href: The path to a task you'd like to monitor recursively. :param pulp_host: The host to poll. If ``None``, a host will automatically be selected by :class:`Client`. :returns: An generator yielding response bodies. :raises pulp_smash.exceptions.TaskTimedOutError: If a task takes too long to complete. """ # Read the timeout in seconds from the cfg, and divide by the sleep_time # to see how many times we query Pulp. # An example: Assuming timeout = 1800s, and sleep_time = 0.3s # 1800s/0.3s = 6000 if cfg.pulp_version < Version("3"): sleep_time = 2 else: sleep_time = 0.3 poll_limit = int(cfg.timeout / sleep_time) poll_counter = 0 json_client = Client(cfg, json_handler, pulp_host=pulp_host) logger.debug("Polling task %s with poll_limit %s", href, poll_limit) while True: task = json_client.get(href) if cfg.pulp_version < Version("3"): task_end_states = _TASK_END_STATES else: task_end_states = _P3_TASK_END_STATES if task["state"] in task_end_states: # This task has completed. Yield its final state, then recursively # iterate through children and yield their final states. yield task if "spawned_tasks" in task: for spawned_task in task["spawned_tasks"]: key = ( "_href" if cfg.pulp_version < Version("3") else "pulp_href" ) for descendant_tsk in poll_task( cfg, spawned_task[key], pulp_host ): yield descendant_tsk break poll_counter += 1 if poll_counter > poll_limit: raise exceptions.TaskTimedOutError( "Task {} is ongoing after {} polls.".format(href, poll_limit) ) logger.debug( "Polling %s progress %s/%s", href, poll_counter, poll_limit ) sleep(sleep_time)
def code_handler(completed_proc): """Check the process for a non-zero return code. Return the process. Check the return code by calling ``completed_proc.check_returncode()``. See: :meth:`pulp_smash.cli.CompletedProcess.check_returncode`. """ completed_proc.check_returncode() logger.debug("Process return code: %s", completed_proc.returncode) return completed_proc
def download_content_unit(_cfg, distribution, unit_path, **kwargs): """Download the content unit distribution using pulp-smash config. :param pulp_smash.config.PulpSmashConfig cfg: Information about the Pulp host. :param distribution: A dict of information about the distribution. :param unit_path: A string path to the unit to be downloaded. :param kwargs: Extra arguments passed to requests.get. """ unit_url = build_unit_url(_cfg, distribution, unit_path) logger.debug("Downloading content %s", unit_url) return utils.http_get(unit_url, **kwargs)
def code_handler(client, response): """Check the response status code, and return the response. Unlike :meth:`safe_handler`, this method doesn't wait for asynchronous tasks to complete if ``response`` has an HTTP 202 status code. :raises: ``requests.exceptions.HTTPError`` if the response status code is in the 4XX or 5XX range. """ response.raise_for_status() logger.debug("response status: %s", response.status_code) return response
def _handle_202(cfg, response, pulp_host): """Check for an HTTP 202 response and handle it appropriately.""" if response.status_code == 202: # "Accepted" _check_http_202_content_type(response) call_report = response.json() tasks = tuple(poll_spawned_tasks(cfg, call_report, pulp_host)) logger.debug("Task call report: %s", call_report) if cfg.pulp_version < Version("3"): _check_call_report(call_report) _check_tasks(cfg, tasks, ("error", "exception", "traceback")) else: _check_tasks(cfg, tasks, ("error", ))
def json_handler(client, response): """Like ``safe_handler``, but also return a JSON-decoded response body. Do what :func:`pulp_smash.api.safe_handler` does. In addition, decode the response body as JSON and return the result. """ response.raise_for_status() logger.debug("response status: %s", response.status_code) if response.status_code == 204: return response _handle_202(client._cfg, response, client.pulp_host) return response.json()
def __init__( self, cfg, response_handler=None, request_kwargs=None, pulp_host=None ): """Initialize this object with needed instance attributes.""" self._cfg = cfg self.response_handler = response_handler or smart_handler self.pulp_host = pulp_host or self._cfg.get_hosts("api")[0] self.request_kwargs = self._cfg.get_requests_kwargs(self.pulp_host) self.request_kwargs["url"] = self._cfg.get_base_url(self.pulp_host) if request_kwargs: self.request_kwargs.update(request_kwargs) self._using_handler_cache = {} logger.debug("New %s", self)
def is_superuser(self): """Check if the current client is root. If the current client is in root mode it stores the status as a cache to avoid it to be called again. This property is named `is_supersuser` to avoid conflict with existing `is_root` function. """ if self._is_root_cache is None: self._is_root_cache = is_root(self.cfg, self.pulp_host) logger.debug("Is Superuser: %s", self._is_root_cache) return self._is_root_cache
def http_get(url, **kwargs): """Issue a HTTP request to the ``url`` and return the response content. This is useful for downloading file contents over HTTP[S]. :param url: the URL where the content should be get. :param kwargs: additional kwargs to be passed to ``requests.get``. :returns: the response content of a GET request to ``url``. """ response = requests.get(url, **kwargs) response.raise_for_status() logger.debug("GET Request to %s finished with %s", url, response) return response.content
def download_content_unit(cfg, distribution, unit_path, **kwargs): """Download the content unit from distribution base url. :param pulp_smash.config.PulpSmashConfig cfg: Information about the Pulp host. :param distribution: A dict of information about the distribution. :param unit_path: A string path to the unit to be downloaded. :param kwargs: Extra arguments passed to requests.get. """ client = api.Client(cfg, api.safe_handler) unit_url = urljoin(get_served_content_url(cfg, distribution), unit_path) logger.debug("Downloading content %s", unit_url) return client.get(unit_url, **kwargs).content
def __init__(self, cfg, response_handler=None, pulp_host=None): """Initialize this object with needed instance attributes.""" # How do we make requests? if not pulp_host: if cfg.pulp_version < Version("3"): pulp_host = cfg.get_hosts("pulp cli")[0] else: pulp_host = cfg.get_hosts("shell")[0] self.pulp_host = pulp_host self.response_handler = response_handler or code_handler self.cfg = cfg self._is_root_cache = None self._machine = None logger.debug("New %s", self)
def machine(self): """Initialize the plumbum machine lazily.""" if self._machine is None: hostname = self.pulp_host.hostname transport = self.pulp_host.roles.get("shell", {}).get("transport") if transport is None: transport = "local" if hostname == socket.getfqdn() else "ssh" if transport == "local": self._machine = plumbum.machines.local else: # transport == 'ssh' # The SshMachine is a wrapper around the host's "ssh" binary. # Thus, it uses ~/.ssh/config, ~/.ssh/known_hosts, etc. self._machine = plumbum.machines.SshMachine(hostname) logger.debug("Initialized plumbum machine %s", self._machine) return self._machine
def safe_handler(client, response): """Check status code, wait for tasks to complete, and check tasks. Inspect the response's HTTP status code. If the response has an HTTP Accepted status code, inspect the returned call report, wait for each task to complete, and inspect each completed task. :raises: ``requests.exceptions.HTTPError`` if the response status code is in the 4XX or 5XX range. :raises pulp_smash.exceptions.CallReportError: If the call report contains an error. :raises pulp_smash.exceptions.TaskReportError: If the task report contains an error. """ response.raise_for_status() logger.debug("response status: %s", response.status_code) _handle_202(client._cfg, response, client.pulp_host) return response
def download_content_unit_return_requests_response(_cfg, distribution, unit_path, **kwargs): """Download the content unit distribution using pulp-smash config, returning the raw response. :param pulp_smash.config.PulpSmashConfig cfg: Information about the Pulp host. :param distribution: A dict of information about the distribution. :param unit_path: A string path to the unit to be downloaded. :param kwargs: Extra arguments passed to requests.get. """ unit_url = build_unit_url(_cfg, distribution, unit_path) logger.debug("Downloading content %s", unit_url) if "verify" not in kwargs: kwargs["verify"] = False response = requests.get(unit_url, **kwargs) response.raise_for_status() logger.debug("GET Request to %s finished with %s", unit_url, response) return response
def http_get_sha256(url, **kwargs): """Issue a HTTP request to the ``url`` and return the SHA256 hexdigest of the content. :param url: URL where the content should be get. :param kwargs: additional kwargs to be passed to ``requests.get``. :returns: SHA256 hexdigest of the body of the GET response to ``url``. """ # GET the raw data by streaming (requests will try to decode according to # "Content-Encoding" header by default) resp = requests.get(url, stream=True, **kwargs) resp.raise_for_status() logger.debug("GET Request to %s finished with %s", url, resp) sha256 = hashlib.sha256() while True: content = resp.raw.read(decode_content=False) if len(content): sha256.update(content) else: break return sha256.hexdigest()
def using_handler(self, response_handler): """Return a copy this same client changing specific handler dependency. This method clones and injects a new handler dependency in to the existing client instance and then returns it. This method is offered just as a 'syntax-sugar' for:: from pulp_smash import api, config def function(client): # This function needs to use a different handler other_client = api.Client(config.get_config(), other_handler) other_client.get(url) with this method the above can be done in fewer lines:: def function(client): # already receives a client here client.using_handler(other_handler).get(url) """ logger.debug("Switching %s to %s", self.response_handler, response_handler) try: existing_client = self._using_handler_cache[response_handler] logger.debug("Reusing Existing Client: %s", existing_client) return existing_client except KeyError: # EAFP new = copy.copy(self) new.response_handler = response_handler self._using_handler_cache[response_handler] = new logger.debug("Creating a new copy of Client %s", new) return new
def smart_handler(client, response): """Decides which handler to call based on response content. Do the following: 1. Pass response through safe_handler to handle 202 and raise_for_status. 2. Return the response if it is not Pulp 3. 3. Return the response if it is not application/json type. 4. Pass response through task_handler if is JSON 202 with 'task'. 5. Pass response through page_handler if is JSON but not 202 with 'task'. """ # safe_handler Will raise_for_Status, handle 202 and pool tasks response = safe_handler(client, response) try: check_pulp3_restriction(client) except ValueError: # If pulp is not 3+ return the result of safe_handler by default return response if response.headers.get("Content-Type") != "application/json": # Not a valid JSON, return pure response logger.debug("Response is not JSON") return response # We got JSON is that a task call report? if response.status_code == 202 and "task" in response.json(): logger.debug("Response is a task") return task_handler(client, response) # Its JSON, it is not a Task, default to page_handler logger.debug("Response is a JSON") return page_handler(client, response)
def machine(self): """Initialize the plumbum machine lazily.""" if self._machine is None: hostname = self.pulp_host.hostname transport = self.pulp_host.roles.get("shell", {}).get("transport") if transport is None: transport = "local" if hostname == socket.getfqdn() else "ssh" if transport == "local": self._machine = plumbum.machines.local elif transport == "kubectl": self._machine = plumbum.machines.local chain = (self._machine["sudo"]["kubectl", "get", "pods"] | self._machine["grep"]["-E", "-o", r"pulp-api-(\w+)-(\w+)"]) self._podname = chain().replace("\n", "") else: # transport == 'ssh' # The SshMachine is a wrapper around the host's "ssh" binary. # Thus, it uses ~/.ssh/config, ~/.ssh/known_hosts, etc. self._machine = plumbum.machines.SshMachine(hostname) logger.debug("Initialized plumbum machine %s", self._machine) return self._machine
def task_handler(client, response): """Wait for tasks to complete and then collect resources. Do the following: 1. Call :meth:`json_handler` to handle 202 and get call_report. 2. Raise error if response is not a task. 3. Re-read the task by its _href to get the final state and metadata. 4. Return the task's created or updated resource or task final state. :raises: ``ValueError`` if the target Pulp application under test is older than version 3 or at least version 4. Usage examples: Create a distribution using meth:`json_handler`:: client = Client(cfg, api.json_handler) spawned_task = client.post(DISTRIBUTION_PATH, body) # json_handler returns the task call report not the created entity spawned_task == {'task': ...} # to have the distribution it is needed to get the task's resources Create a distribution using meth:`task_handler`:: client = Client(cfg, api.task_handler) distribution = client.post(DISTRIBUTION_PATH, body) # task_handler resolves the created entity and returns its data distribution == {'_href': ..., 'base_path': ...} Having an existent client it is possible to use the shortcut:: client.using_handler(api.task_handler).post(DISTRIBUTION_PATH, body) """ check_pulp3_restriction(client) # JSON handler takes care of pooling tasks until it is done # If task errored then json_handler will raise the error response_dict = json_handler(client, response) if "task" not in response_dict: raise exceptions.CallReportError( "Response does not contains a task call_report: {}".format( response_dict)) # Get the final state of the done task done_task = client.using_handler(json_handler).get(response_dict["task"]) if response.request.method == "POST": # Task might have created new resources if "created_resources" in done_task: created = done_task["created_resources"] logger.debug("Task created resources: %s", created) if len(created) == 1: # Single resource href return client.using_handler(json_handler).get(created[0]) if len(created) > 1: # Multiple resource hrefs return [ client.using_handler(json_handler).get(resource_href) for resource_href in created ] else: return [] if response.request.method in ["PUT", "PATCH"]: # Task might have updated resource so re-read and return it back logger.debug("Task updated resource: %s", response.request.url) return client.using_handler(json_handler).get(response.request.url) # response.request.method is one of ['DELETE', 'GET', 'HEAD', 'OPTION'] # Returns the final state of the done task logger.debug("Task finished: %s", done_task) return done_task
def echo_handler(client, response): """Immediately return ``response``.""" logger.debug("response status: %s", response.status_code) return response
def echo_handler(completed_proc): """Immediately return ``completed_proc``.""" logger.debug("Process return code: %s", completed_proc.returncode) return completed_proc