def __init__(self,
cfg,
response_handler=None,
pulp_host=None,
local=False):
"""Initialize this object with needed instance attributes."""
# How do we make requests?
if not pulp_host:
if cfg.pulp_version < Version("3"):
pulp_host = cfg.get_hosts("pulp cli")[0]
else:
pulp_host = cfg.get_hosts("shell")[0]
if local:
pulp_host = collections.namedtuple("Host", "hostname roles")
pulp_host.hostname = "localhost"
pulp_host.roles = {"shell": {"transport": "local"}}
self.pulp_host = pulp_host
self.response_handler = response_handler or code_handler
self.cfg = cfg
self._is_root_cache = None
self._machine = None
self._transport = None
self._podname = None
logger.debug("New %s", self)
def run(self, args, sudo=False, **kwargs):
"""Run a command and ``return self.response_handler(result)``.
This method is a thin wrapper around Plumbum's `BaseCommand.run`_
method, which is itself a thin wrapper around the standard library's
`subprocess.Popen`_ class. See their documentation for detailed usage
instructions. See :class:`pulp_smash.cli.Client` for a usage example.
:param args: Any arguments to be passed to the process (a tuple).
:param sudo: If the command should run as superuser (a boolean).
:param kwargs: Extra named arguments passed to plumbumBaseCommand.run.
.. _BaseCommand.run:
http://plumbum.readthedocs.io/en/latest/api/commands.html#plumbum.commands.base.BaseCommand.run
.. _subprocess.Popen:
https://docs.python.org/3/library/subprocess.html#subprocess.Popen
"""
# Let self.response_handler check return codes. See:
# https://plumbum.readthedocs.io/en/latest/api/commands.html#plumbum.commands.base.BaseCommand.run
kwargs.setdefault("retcode")
logger.debug("Running %s cmd (sudo:%s) - %s", args, sudo, kwargs)
if self._podname:
args = ("kubectl", "exec", self._podname, "--") + tuple(args)
if sudo and args[0] != "sudo" and not self.is_superuser:
args = ("sudo", ) + tuple(args)
code, stdout, stderr = self.machine[args[0]].run(args[1:], **kwargs)
completed_process = CompletedProcess(args, code, stdout, stderr)
logger.debug("Finished %s command: %s", args, (code, stdout, stderr))
return self.response_handler(completed_process)
def page_handler(client, response):
"""Call :meth:`json_handler`, optionally collect results, and return.
Do the following:
1. If ``response`` has an HTTP No Content (204) `status code`_, return
``response``.
2. Call :meth:`json_handler`.
3. If the response appears to be paginated, walk through each page of
results, and collect them into a single list. Otherwise, do nothing.
Return either the list of results or the single decoded response.
:raises: ``ValueError`` if the target Pulp application under test is older
than version 3 or at least version 4.
.. _status code: https://en.wikipedia.org/wiki/List_of_HTTP_status_codes
"""
check_pulp3_restriction(client)
maybe_page = json_handler(client, response)
if not isinstance(maybe_page, dict):
return maybe_page # HTTP 204 No Content
if "results" not in maybe_page:
return maybe_page # Content isn't a page.
collected_results = []
for result in _walk_pages(client._cfg, maybe_page, client.pulp_host):
collected_results.extend(result)
logger.debug("paginated %s result pages", len(collected_results))
return collected_results
def request(self, method, url, **kwargs):
"""Send an HTTP request.
Arguments passed directly in to this method override (but do not
overwrite!) arguments specified in ``self.request_kwargs``.
"""
# The `self.request_kwargs` dict should *always* have a "url" argument.
# This is enforced by `self.__init__`. This allows us to call the
# `requests.request` function and satisfy its signature:
#
# request(method, url, **kwargs)
#
intended_host = self.pulp_host.hostname
request_kwargs = self.request_kwargs.copy()
request_kwargs["url"] = urljoin(request_kwargs["url"], url)
request_kwargs.update(kwargs)
actual_host = urlparse(request_kwargs["url"]).hostname
if intended_host != actual_host:
warnings.warn(
"This client should be used to communicate with {0}, but a "
"request is being made to {1}. The request will be made, but "
"beware that information intended for {0} (such as "
"authentication tokens) may now be sent to {1}. Here's the "
"list of options being sent with this request: {2}".format(
intended_host, actual_host, request_kwargs),
RuntimeWarning,
)
logger.debug("Making a %s request with %s", method, request_kwargs)
response = self.response_handler(
self, requests.request(method, **request_kwargs))
logger.debug("Finished %s request with response: %s", method, response)
return response
def machine(self):
"""Initialize the plumbum machine lazily."""
if self._machine is None:
if self.transport == "local":
self._machine = plumbum.machines.local
elif self.transport == "kubectl":
self._machine = plumbum.machines.local
chain = (self._machine["sudo"]["kubectl", "get", "pods"]
| self._machine["grep"]["-E", "-o",
r"pulp-api-(\w+)-(\w+)"])
self._podname = chain().replace("\n", "")
elif self.transport in ["docker", "podman"]:
self._machine = plumbum.machines.local
self._podname = self.pulp_host.roles.get("shell", {}).get(
"container", "pulp")
elif self.transport == "ssh":
# The SshMachine is a wrapper around the host's "ssh" binary.
# Thus, it uses ~/.ssh/config, ~/.ssh/known_hosts, etc.
hostname = self.pulp_host.hostname
self._machine = plumbum.machines.SshMachine(hostname)
else:
raise NotImplementedError(
"Transport ({}) is not implemented.".format(
self.transport))
logger.debug("Initialized plumbum machine %s", self._machine)
return self._machine
def poll_task(cfg, href, pulp_host=None):
"""Wait for a task and its children to complete. Yield response bodies.
Poll the task at ``href``, waiting for the task to complete. When a
response is received indicating that the task is complete, yield that
response body and recursively poll each child task.
:param cfg: A :class:`pulp_smash.config.PulpSmashConfig` object.
:param href: The path to a task you'd like to monitor recursively.
:param pulp_host: The host to poll. If ``None``, a host will automatically
be selected by :class:`Client`.
:returns: An generator yielding response bodies.
:raises pulp_smash.exceptions.TaskTimedOutError: If a task takes too
long to complete.
"""
# Read the timeout in seconds from the cfg, and divide by the sleep_time
# to see how many times we query Pulp.
# An example: Assuming timeout = 1800s, and sleep_time = 0.3s
# 1800s/0.3s = 6000
if cfg.pulp_version < Version("3"):
sleep_time = 2
else:
sleep_time = 0.3
poll_limit = int(cfg.timeout / sleep_time)
poll_counter = 0
json_client = Client(cfg, json_handler, pulp_host=pulp_host)
logger.debug("Polling task %s with poll_limit %s", href, poll_limit)
while True:
task = json_client.get(href)
if cfg.pulp_version < Version("3"):
task_end_states = _TASK_END_STATES
else:
task_end_states = _P3_TASK_END_STATES
if task["state"] in task_end_states:
# This task has completed. Yield its final state, then recursively
# iterate through children and yield their final states.
yield task
if "spawned_tasks" in task:
for spawned_task in task["spawned_tasks"]:
key = (
"_href"
if cfg.pulp_version < Version("3")
else "pulp_href"
)
for descendant_tsk in poll_task(
cfg, spawned_task[key], pulp_host
):
yield descendant_tsk
break
poll_counter += 1
if poll_counter > poll_limit:
raise exceptions.TaskTimedOutError(
"Task {} is ongoing after {} polls.".format(href, poll_limit)
)
logger.debug(
"Polling %s progress %s/%s", href, poll_counter, poll_limit
)
sleep(sleep_time)
def code_handler(completed_proc):
"""Check the process for a non-zero return code. Return the process.
Check the return code by calling ``completed_proc.check_returncode()``.
See: :meth:`pulp_smash.cli.CompletedProcess.check_returncode`.
"""
completed_proc.check_returncode()
logger.debug("Process return code: %s", completed_proc.returncode)
return completed_proc
def download_content_unit(_cfg, distribution, unit_path, **kwargs):
"""Download the content unit distribution using pulp-smash config.
:param pulp_smash.config.PulpSmashConfig cfg: Information about the Pulp host.
:param distribution: A dict of information about the distribution.
:param unit_path: A string path to the unit to be downloaded.
:param kwargs: Extra arguments passed to requests.get.
"""
unit_url = build_unit_url(_cfg, distribution, unit_path)
logger.debug("Downloading content %s", unit_url)
return utils.http_get(unit_url, **kwargs)
def code_handler(client, response):
"""Check the response status code, and return the response.
Unlike :meth:`safe_handler`, this method doesn't wait for asynchronous
tasks to complete if ``response`` has an HTTP 202 status code.
:raises: ``requests.exceptions.HTTPError`` if the response status code is
in the 4XX or 5XX range.
"""
response.raise_for_status()
logger.debug("response status: %s", response.status_code)
return response
def _handle_202(cfg, response, pulp_host):
"""Check for an HTTP 202 response and handle it appropriately."""
if response.status_code == 202: # "Accepted"
_check_http_202_content_type(response)
call_report = response.json()
tasks = tuple(poll_spawned_tasks(cfg, call_report, pulp_host))
logger.debug("Task call report: %s", call_report)
if cfg.pulp_version < Version("3"):
_check_call_report(call_report)
_check_tasks(cfg, tasks, ("error", "exception", "traceback"))
else:
_check_tasks(cfg, tasks, ("error", ))
def json_handler(client, response):
"""Like ``safe_handler``, but also return a JSON-decoded response body.
Do what :func:`pulp_smash.api.safe_handler` does. In addition, decode the
response body as JSON and return the result.
"""
response.raise_for_status()
logger.debug("response status: %s", response.status_code)
if response.status_code == 204:
return response
_handle_202(client._cfg, response, client.pulp_host)
return response.json()
def __init__(
self, cfg, response_handler=None, request_kwargs=None, pulp_host=None
):
"""Initialize this object with needed instance attributes."""
self._cfg = cfg
self.response_handler = response_handler or smart_handler
self.pulp_host = pulp_host or self._cfg.get_hosts("api")[0]
self.request_kwargs = self._cfg.get_requests_kwargs(self.pulp_host)
self.request_kwargs["url"] = self._cfg.get_base_url(self.pulp_host)
if request_kwargs:
self.request_kwargs.update(request_kwargs)
self._using_handler_cache = {}
logger.debug("New %s", self)
def is_superuser(self):
"""Check if the current client is root.
If the current client is in root mode it stores the status as a cache
to avoid it to be called again.
This property is named `is_supersuser` to avoid conflict with existing
`is_root` function.
"""
if self._is_root_cache is None:
self._is_root_cache = is_root(self.cfg, self.pulp_host)
logger.debug("Is Superuser: %s", self._is_root_cache)
return self._is_root_cache
def http_get(url, **kwargs):
"""Issue a HTTP request to the ``url`` and return the response content.
This is useful for downloading file contents over HTTP[S].
:param url: the URL where the content should be get.
:param kwargs: additional kwargs to be passed to ``requests.get``.
:returns: the response content of a GET request to ``url``.
"""
response = requests.get(url, **kwargs)
response.raise_for_status()
logger.debug("GET Request to %s finished with %s", url, response)
return response.content
def download_content_unit(cfg, distribution, unit_path, **kwargs):
"""Download the content unit from distribution base url.
:param pulp_smash.config.PulpSmashConfig cfg: Information about the Pulp
host.
:param distribution: A dict of information about the distribution.
:param unit_path: A string path to the unit to be downloaded.
:param kwargs: Extra arguments passed to requests.get.
"""
client = api.Client(cfg, api.safe_handler)
unit_url = urljoin(get_served_content_url(cfg, distribution), unit_path)
logger.debug("Downloading content %s", unit_url)
return client.get(unit_url, **kwargs).content
def __init__(self, cfg, response_handler=None, pulp_host=None):
"""Initialize this object with needed instance attributes."""
# How do we make requests?
if not pulp_host:
if cfg.pulp_version < Version("3"):
pulp_host = cfg.get_hosts("pulp cli")[0]
else:
pulp_host = cfg.get_hosts("shell")[0]
self.pulp_host = pulp_host
self.response_handler = response_handler or code_handler
self.cfg = cfg
self._is_root_cache = None
self._machine = None
logger.debug("New %s", self)
def machine(self):
"""Initialize the plumbum machine lazily."""
if self._machine is None:
hostname = self.pulp_host.hostname
transport = self.pulp_host.roles.get("shell", {}).get("transport")
if transport is None:
transport = "local" if hostname == socket.getfqdn() else "ssh"
if transport == "local":
self._machine = plumbum.machines.local
else: # transport == 'ssh'
# The SshMachine is a wrapper around the host's "ssh" binary.
# Thus, it uses ~/.ssh/config, ~/.ssh/known_hosts, etc.
self._machine = plumbum.machines.SshMachine(hostname)
logger.debug("Initialized plumbum machine %s", self._machine)
return self._machine
def safe_handler(client, response):
"""Check status code, wait for tasks to complete, and check tasks.
Inspect the response's HTTP status code. If the response has an HTTP
Accepted status code, inspect the returned call report, wait for each task
to complete, and inspect each completed task.
:raises: ``requests.exceptions.HTTPError`` if the response status code is
in the 4XX or 5XX range.
:raises pulp_smash.exceptions.CallReportError: If the call report contains
an error.
:raises pulp_smash.exceptions.TaskReportError: If the task report contains
an error.
"""
response.raise_for_status()
logger.debug("response status: %s", response.status_code)
_handle_202(client._cfg, response, client.pulp_host)
return response
def download_content_unit_return_requests_response(_cfg, distribution,
unit_path, **kwargs):
"""Download the content unit distribution using pulp-smash config, returning the raw response.
:param pulp_smash.config.PulpSmashConfig cfg: Information about the Pulp host.
:param distribution: A dict of information about the distribution.
:param unit_path: A string path to the unit to be downloaded.
:param kwargs: Extra arguments passed to requests.get.
"""
unit_url = build_unit_url(_cfg, distribution, unit_path)
logger.debug("Downloading content %s", unit_url)
if "verify" not in kwargs:
kwargs["verify"] = False
response = requests.get(unit_url, **kwargs)
response.raise_for_status()
logger.debug("GET Request to %s finished with %s", unit_url, response)
return response
def http_get_sha256(url, **kwargs):
"""Issue a HTTP request to the ``url`` and return the SHA256 hexdigest of the content.
:param url: URL where the content should be get.
:param kwargs: additional kwargs to be passed to ``requests.get``.
:returns: SHA256 hexdigest of the body of the GET response to ``url``.
"""
# GET the raw data by streaming (requests will try to decode according to
# "Content-Encoding" header by default)
resp = requests.get(url, stream=True, **kwargs)
resp.raise_for_status()
logger.debug("GET Request to %s finished with %s", url, resp)
sha256 = hashlib.sha256()
while True:
content = resp.raw.read(decode_content=False)
if len(content):
sha256.update(content)
else:
break
return sha256.hexdigest()
def using_handler(self, response_handler):
"""Return a copy this same client changing specific handler dependency.
This method clones and injects a new handler dependency in to the
existing client instance and then returns it.
This method is offered just as a 'syntax-sugar' for::
from pulp_smash import api, config
def function(client):
# This function needs to use a different handler
other_client = api.Client(config.get_config(), other_handler)
other_client.get(url)
with this method the above can be done in fewer lines::
def function(client): # already receives a client here
client.using_handler(other_handler).get(url)
"""
logger.debug("Switching %s to %s", self.response_handler,
response_handler)
try:
existing_client = self._using_handler_cache[response_handler]
logger.debug("Reusing Existing Client: %s", existing_client)
return existing_client
except KeyError: # EAFP
new = copy.copy(self)
new.response_handler = response_handler
self._using_handler_cache[response_handler] = new
logger.debug("Creating a new copy of Client %s", new)
return new
def smart_handler(client, response):
"""Decides which handler to call based on response content.
Do the following:
1. Pass response through safe_handler to handle 202 and raise_for_status.
2. Return the response if it is not Pulp 3.
3. Return the response if it is not application/json type.
4. Pass response through task_handler if is JSON 202 with 'task'.
5. Pass response through page_handler if is JSON but not 202 with 'task'.
"""
# safe_handler Will raise_for_Status, handle 202 and pool tasks
response = safe_handler(client, response)
try:
check_pulp3_restriction(client)
except ValueError:
# If pulp is not 3+ return the result of safe_handler by default
return response
if response.headers.get("Content-Type") != "application/json":
# Not a valid JSON, return pure response
logger.debug("Response is not JSON")
return response
# We got JSON is that a task call report?
if response.status_code == 202 and "task" in response.json():
logger.debug("Response is a task")
return task_handler(client, response)
# Its JSON, it is not a Task, default to page_handler
logger.debug("Response is a JSON")
return page_handler(client, response)
def machine(self):
"""Initialize the plumbum machine lazily."""
if self._machine is None:
hostname = self.pulp_host.hostname
transport = self.pulp_host.roles.get("shell", {}).get("transport")
if transport is None:
transport = "local" if hostname == socket.getfqdn() else "ssh"
if transport == "local":
self._machine = plumbum.machines.local
elif transport == "kubectl":
self._machine = plumbum.machines.local
chain = (self._machine["sudo"]["kubectl", "get", "pods"]
| self._machine["grep"]["-E", "-o",
r"pulp-api-(\w+)-(\w+)"])
self._podname = chain().replace("\n", "")
else: # transport == 'ssh'
# The SshMachine is a wrapper around the host's "ssh" binary.
# Thus, it uses ~/.ssh/config, ~/.ssh/known_hosts, etc.
self._machine = plumbum.machines.SshMachine(hostname)
logger.debug("Initialized plumbum machine %s", self._machine)
return self._machine
def task_handler(client, response):
"""Wait for tasks to complete and then collect resources.
Do the following:
1. Call :meth:`json_handler` to handle 202 and get call_report.
2. Raise error if response is not a task.
3. Re-read the task by its _href to get the final state and metadata.
4. Return the task's created or updated resource or task final state.
:raises: ``ValueError`` if the target Pulp application under test is older
than version 3 or at least version 4.
Usage examples:
Create a distribution using meth:`json_handler`::
client = Client(cfg, api.json_handler)
spawned_task = client.post(DISTRIBUTION_PATH, body)
# json_handler returns the task call report not the created entity
spawned_task == {'task': ...}
# to have the distribution it is needed to get the task's resources
Create a distribution using meth:`task_handler`::
client = Client(cfg, api.task_handler)
distribution = client.post(DISTRIBUTION_PATH, body)
# task_handler resolves the created entity and returns its data
distribution == {'_href': ..., 'base_path': ...}
Having an existent client it is possible to use the shortcut::
client.using_handler(api.task_handler).post(DISTRIBUTION_PATH, body)
"""
check_pulp3_restriction(client)
# JSON handler takes care of pooling tasks until it is done
# If task errored then json_handler will raise the error
response_dict = json_handler(client, response)
if "task" not in response_dict:
raise exceptions.CallReportError(
"Response does not contains a task call_report: {}".format(
response_dict))
# Get the final state of the done task
done_task = client.using_handler(json_handler).get(response_dict["task"])
if response.request.method == "POST":
# Task might have created new resources
if "created_resources" in done_task:
created = done_task["created_resources"]
logger.debug("Task created resources: %s", created)
if len(created) == 1: # Single resource href
return client.using_handler(json_handler).get(created[0])
if len(created) > 1: # Multiple resource hrefs
return [
client.using_handler(json_handler).get(resource_href)
for resource_href in created
]
else:
return []
if response.request.method in ["PUT", "PATCH"]:
# Task might have updated resource so re-read and return it back
logger.debug("Task updated resource: %s", response.request.url)
return client.using_handler(json_handler).get(response.request.url)
# response.request.method is one of ['DELETE', 'GET', 'HEAD', 'OPTION']
# Returns the final state of the done task
logger.debug("Task finished: %s", done_task)
return done_task
def echo_handler(client, response):
"""Immediately return ``response``."""
logger.debug("response status: %s", response.status_code)
return response
def echo_handler(completed_proc):
"""Immediately return ``completed_proc``."""
logger.debug("Process return code: %s", completed_proc.returncode)
return completed_proc
