def test_get_user(self):
"""Test that when a client is created, it logs in just once."""
with mock.patch.object(config, "_CONFIG", self.config):
self.assertEqual(config.get_config(), self.config)
client = api.Client
client.login = MagicMock()
response = MagicMock(json=MagicMock(
return_value={"username": "******"}))
client.request = MagicMock(return_value=response)
cl = client()
u = cl.get_user().json()["username"]
assert u == config.get_config()["qpc"]["username"]
client.request.assert_called_once_with(
"GET", urljoin(cl.url, "users/current/"))
def yupana_config():
"""Return all of the yupana config values."""
try:
yupana_configs = get_config().get("yupana", [])
except ConfigFileNotFoundError:
yupana_configs = []
return yupana_configs
def test_create_no_config(self):
"""If a base url is specified we use it."""
with mock.patch.object(config, "_CONFIG", {}):
self.assertEqual(config.get_config(), {})
other_host = "http://hostname.com"
client = api.Client(url=other_host, authenticate=False)
self.assertNotEqual("http://example.com/api/v1/", client.url)
self.assertEqual(other_host, client.url)
def test_cache_empty(self):
"""A config is read from disk if the cache is empty."""
with mock.patch.object(config, "_CONFIG", None), mock.patch(
"camayoc.config.yaml.load"
) as load, mock.patch("camayoc.config._get_config_file_path"):
load.return_value = self.config
self.assertEqual(config.get_config(), self.config)
self.assertEqual(load.call_count, 1)
def test_login(self):
"""Test that when a client is created, it logs in just once."""
with mock.patch.object(config, "_CONFIG", self.config):
self.assertEqual(config.get_config(), self.config)
client = api.Client
client.login = MagicMock()
cl = client()
assert client.login.call_count == 1
cl.token = uuid4()
assert cl.default_headers() != {}
def login(self):
"""Login to the server to receive an authorization token."""
cfg = config.get_config().get("qpc", {})
server_username = cfg.get("username", "admin")
server_password = cfg.get("password", "pass")
login_request = self.request(
"POST",
urljoin(self.url, QPC_TOKEN_PATH),
json={"username": server_username, "password": server_password},
)
self.token = login_request.json()["token"]
return login_request
def get_config():
"""Gather existing config file or create one.
If a config file is found by camayoc, use that.
Otherwise, use BASE_CONFIG as auth.
"""
try:
cfg = config.get_config()
except ConfigFileNotFoundError:
cfg = yaml.load(BASE_CONFIG, Loader=yaml.FullLoader)
return cfg, os.path.join(os.getcwd(), "config.yaml")
def __init__(self, response_handler=None, url=None, authenticate=True):
"""Initialize this object, collecting base URL from config file.
If no response handler is specified, use the `code_handler` which will
raise an exception for 'bad' return codes.
If no URL is specified, then the config file will be parsed and the URL
will be built by reading the hostname, port and https values. You can
configure the default URL by including the following on your Camayoc
configuration file::
qpc:
hostname: <machine_hostname_or_ip_address>
port: <port> # if not defined will take the default port
# depending on the https config: 80 if https is
# false and 443 if https is true.
https: false # change to true if server is published over
# https. Defaults to false if not defined
"""
self.url = url
self.token = None
cfg = config.get_config().get("qpc", {})
self.verify = cfg.get("ssl-verify", False)
if not self.url:
hostname = cfg.get("hostname")
if not hostname:
raise exceptions.QPCBaseUrlNotFound(
"\n'qpc' section specified in camayoc config file, but"
"no 'hostname' key found."
)
scheme = "https" if cfg.get("https", False) else "http"
port = str(cfg.get("port", ""))
netloc = hostname + ":{}".format(port) if port else hostname
self.url = urlunparse((scheme, netloc, QPC_API_VERSION, "", "", ""))
if not self.url:
raise exceptions.QPCBaseUrlNotFound(
"No base url was specified to the client either with the "
'url="host" option or with the camayoc config file.'
)
if response_handler is None:
self.response_handler = code_handler
else:
self.response_handler = response_handler
if authenticate:
self.login()
def get_qpc_url():
"""Return the base url for the qpc server."""
cfg = get_config().get("qpc", {})
hostname = cfg.get("hostname")
if not hostname:
raise exceptions.QPCBaseUrlNotFound(
'Make sure you have a "qpc" section and `hostname`is specified in '
"the camayoc config file")
scheme = "https" if cfg.get("https", False) else "http"
port = str(cfg.get("port", ""))
netloc = hostname + ":{}".format(port) if port else hostname
return urlunparse((scheme, netloc, "", "", "", ""))
def test_logout(self):
"""Test that when we log out, all credentials are cleared."""
with mock.patch.object(config, "_CONFIG", self.config):
self.assertEqual(config.get_config(), self.config)
client = api.Client
client.login = MagicMock()
cl = client()
assert client.login.call_count == 1
cl.token = uuid4()
assert cl.default_headers() is not {}
client.request = MagicMock()
cl.logout()
assert client.request.call_count == 1
assert cl.token is None
assert cl.default_headers() == {}
def config_sources():
"""Return all sources available on configuration file for CLI scans."""
try:
config_sources = get_config().get("qpc", {}).get("sources", [])
except ConfigFileNotFoundError:
config_sources = []
if not config_sources:
return []
scan_sources = list(
itertools.chain(*[scan["sources"] for scan in config_scans()]))
return [
source for source in config_sources if source["name"] in scan_sources
]
def scan_permutations():
"""Generate a tuple of hostname and auth matching expected scans."""
try:
config = get_config()
auths = [
auth for auth in config["credentials"]
if auth["type"] == "network" and auth.get("rho", True)
]
inventory = {
machine["hostname"]: machine
for machine in config["inventory"]
if machine.get("hypervisor") == "vcenter"
}
return list(itertools.product(inventory, auths))
except (ConfigFileNotFoundError, KeyError):
return []
def test_response_handler(self):
"""Test that when we get a 4xx or 5xx response, an error is raised."""
with mock.patch.object(config, "_CONFIG", self.config):
self.assertEqual(config.get_config(), self.config)
client = api.Client(authenticate=False)
mock_request = mock.Mock(
body='{"Test Body"}',
path_url="/example/path/",
headers='{"Test Header"}',
text="Some text",
)
mock_response = mock.Mock(status_code=404)
mock_response.request = mock_request
mock_response.json = MagicMock(return_value=json.dumps(
'{"The resource you requested was not found"}'))
with self.subTest(msg="Test code handler"):
client.response_handler = api.code_handler
with self.assertRaises(requests.exceptions.HTTPError):
client.response_handler(mock_response)
with self.subTest(msg="Test json handler"):
client.response_handler = api.json_handler
with self.assertRaises(requests.exceptions.HTTPError):
client.response_handler(mock_response)
with self.subTest(msg="Test echo handler"):
client.response_handler = api.echo_handler
# no error should be raised with the echo handler
client.response_handler(mock_response)
# not all responses have valid json
mock_response.json = MagicMock(return_value="Not valid json")
with self.subTest(msg="Test code handler without json available"):
client.response_handler = api.code_handler
with self.assertRaises(requests.exceptions.HTTPError):
client.response_handler(mock_response)
with self.subTest(msg="Test json handler without json available"):
client.response_handler = api.json_handler
with self.assertRaises(requests.exceptions.HTTPError):
client.response_handler(mock_response)
with self.subTest(msg="Test echo handler without json available"):
client.response_handler = api.echo_handler
# no error should be raised with the echo handler
client.response_handler(mock_response)
def config_credentials():
"""Return all credentials available on configuration file for CLI scans."""
try:
config_credentials = get_config().get("credentials", [])
except ConfigFileNotFoundError:
config_credentials = []
if not config_credentials:
return []
scan_credentials = list(
itertools.chain(
*[source["credentials"] for source in config_sources()]))
return [
credential for credential in config_credentials
if credential["name"] in scan_credentials
]
def isolated_filesystem(request):
"""Fixture that creates a temporary directory.
Changes the current working directory to the created temporary directory
for isolated filesystem tests.
"""
# Create isolated filesystem directory in the ssh_keyfile_path
# configuration location if marked with `ssh_keyfile_path`.
mark = request.node.get_closest_marker('ssh_keyfile_path')
ssh_keyfile_path = None
if mark:
cfg = get_config().get("qpc", {})
ssh_keyfile_path = cfg.get("ssh_keyfile_path")
if not ssh_keyfile_path:
pytest.fail("QPC configuration 'ssh_keyfile_path' not provided or "
"found")
with utils.isolated_filesystem(ssh_keyfile_path) as path:
yield path
def vcenter_client():
"""Create a vCenter client.
Get the client confifuration from environment variables and Camayoc's
configuration file in this order. Raise a KeyError if can't find the
expecting configuration.
The expected environment variables are VCHOSTNAME, VCUSER and VCPASS for
vcenter's hostname, username and password respectively.
The configuration in the Camayoc's configuration file should be as the
following::
vcenter:
hostname: vcenter.domain.example.com
username: gandalf
password: YouShallNotPass
The vcenter config can be mixed by using both environement variables and
the configuration file but environment variable takes precedence.
"""
config = get_config()
vcenter_host = os.getenv("VCHOSTNAME", config["vcenter"]["hostname"])
vcenter_user = os.getenv("VCUSER", config["vcenter"]["username"])
vcenter_pwd = os.getenv("VCPASS", config["vcenter"]["password"])
try:
c = SmartConnect(
host=vcenter_host,
user=vcenter_user,
pwd=vcenter_pwd,
connectionPoolTimeout=-1,
)
except ssl.SSLError:
c = SmartConnect(
host=vcenter_host,
user=vcenter_user,
pwd=vcenter_pwd,
sslContext=ssl._create_unverified_context(),
connectionPoolTimeout=-1,
)
yield c
Disconnect(c)
def run_all_scans(vcenter_client):
"""Run all configured scans caching the report id associated with each.
Run each scan defined in the ``qpc`` section of the configuration file.
Cache the report id of the scan, associating it with its scan.
The expected configuration in the Camayoc's configuration file is as
follows::
qpc:
# other needed qpc config data
#
# specific for scans:
- scans:
- name: network-vcenter-sat-mix
sources:
- sample-none-rhel-5-vc
- sample-sat-6
- sample-vcenter
disabled_optional_products: {'jboss_fuse': True}
type: 'connect'
enabled_extended_product_search: {'jboss_eap': True,
'search_directories': ['/foo/bar']}
inventory:
- hostname: sample-none-rhel-5-vc
ipv4: 10.10.10.1
hypervisor: vcenter
distribution:
name: rhel
version: '5.9'
products: {}
credentials:
- root
- hostname: sample-sat-6
type: 'vcenter'
options:
ssl_cert_verify: false
credentials:
- sat6_admin
- hostname: sample-vcenter
type: 'vcenter'
credentials:
- vcenter_admin
credentials:
- name: root
sshkeyfile: /path/to/.ssh/id_rsa
username: root
type: network
- name: sat6_admin
password: foo
username: admin
type: satellite
- name: vcenter_admin
password: foo
username: admin
type: vcenter
In the sample configuration file above, one machine will be turned on,
and one scan will be run against the three sources each created with
their own credential.
"""
cleanup = []
config = get_config()
config_scans = config.get("qpc", {}).get("scans", [])
if not config_scans:
# if no scans are defined, no need to go any further
return
config_creds = {
credential["name"]: credential for credential in config.get("credentials", [])
}
inventory = {machine["hostname"]: machine for machine in config["inventory"]}
vcenter_inventory = {
machine["hostname"]: machine
for machine in config["inventory"]
if machine.get("hypervisor") == "vcenter"
}
if not config_creds or not inventory:
raise ValueError(
"Make sure to have credentials and inventory" " items in the config file"
)
credential_ids = {}
source_ids = {}
expected_products = {}
for scan in config_scans:
scan_vms = {
vm.name: vm
for vm in get_vcenter_vms(vcenter_client)
if (vm.name in vcenter_inventory) and (vm.name in scan["sources"])
}
with vcenter_vms(scan_vms.values()):
for source_name in scan["sources"]:
if source_name not in source_ids:
machine = inventory[source_name]
for credential_name in machine["credentials"]:
if credential_name not in credential_ids:
credential = config_creds[credential_name].copy()
credential.pop("rho", None)
credential["cred_type"] = credential.pop("type")
credential["ssh_keyfile"] = credential.pop(
"sshkeyfile", None
)
credential_ids[credential_name] = create_cred(
credential, cleanup
)
if machine.get("type", "network") == "network":
vm = scan_vms[machine["hostname"]]
machine["ipv4"] = vm.guest.ipAddress
source_info = {
"source_type": machine.get("type", "network"),
"hosts": [machine.get("ipv4") or machine["hostname"]],
"credential_ids": [
credential_ids[credential]
for credential in machine["credentials"]
],
}
if "options" in machine:
source_info["options"] = machine["options"].copy()
source_ids[source_name] = create_source(source_info, cleanup)
expected_products[source_name] = machine.get("products", {})
expected_products[source_name].update(
{"distribution": source_info.get("distribution", {})}
)
scan_info = {
"expected_products": [],
"name": scan["name"],
"source_id_to_hostname": {},
"source_ids": [],
}
for source_name in scan["sources"]:
source_id = source_ids[source_name]
scan_info["expected_products"].append(
{source_id: expected_products[source_name]}
)
scan_info["source_id_to_hostname"][source_id] = source_name
scan_info["source_ids"].append(source_id)
run_scan(
scan_info,
scan.get("disabled_optional_products", {}),
scan.get("enabled_extended_product_search", {}),
cleanup=cleanup,
scan_type=scan.get("type", "inspect"),
)
def get_source(source_type, cleanup):
"""Retrieve a single network source if available from config file.
:param source_type: The type of source to be created. This function
retreives one source of matching type from the config file.
:param cleanup: The "cleanup" list that tests use to destroy objects after
a test has run. The "cleanup" list is provided by the py.test fixture
of the same name defined in camayoc/tests/qpc/conftest.py.
Sources are retreived from the following section and are assumed to be
available on demand and do not require their power state to be managed.
The expected configuration in the Camayoc's configuration file is as
follows::
qpc:
# other needed qpc config data
sources:
- hosts:
- '127.0.0.1'
name: local
type: network
credentials:
- root
The credential listed is assumed to be in the top level 'credentials'
section.
This source is meant to be used for tests where we do not care about the
results of the scan, for example tests that assert we can pause and restart
a scan.
:returns: camayoc.qpc_models.Source of the same type that was requested
with the 'source_type' parameter. The returned source has been created
on server and has all credentials listed in config file created and
associtated with it.
"""
cfg = config.get_config()
cred_list = cfg.get("credentials", [])
src_list = cfg.get("qpc", {}).get("sources", [])
config_src = {}
if not (src_list and cred_list):
return
for src in src_list:
if src.get("type") == source_type:
config_src = src
if config_src:
config_src.setdefault("credential_ids", [])
src_creds = config_src.get("credentials")
for cred in src_creds:
for config_cred in cred_list:
if cred == config_cred["name"]:
server_cred = Credential(cred_type=source_type,
username=config_cred["username"])
if config_cred.get("password"):
server_cred.password = config_cred["password"]
else:
server_cred.ssh_keyfile = config_cred["sshkeyfile"]
server_cred.create()
cleanup.append(server_cred)
config_src["credential_ids"].append(server_cred._id)
server_src = Source(
hosts=config_src["hosts"],
credential_ids=config_src["credential_ids"],
source_type=source_type,
)
if config_src.get("options"):
server_src.options = config_src.get("options")
server_src.create()
cleanup.append(server_src)
return server_src
def scan_machines(vcenter_client, isolated_filesystem):
"""Scan all machines caching the results.
Scan each machine found in the configuration's file inventory using each
auth defined in the same file.
Cache the report path of each scan or ``None`` if the scan can't be
completed.
The expected configuration in the Camayoc's configuration file is as
follows::
inventory:
- hostname: sample-none-rhel-5-vc
ipv4: 10.10.10.1
hypervisor: vcenter
distribution:
name: rhel
version: '5.9'
products: {}
- hostname: sample-none-rhel-6-vc
ipv4: 10.10.10.2
hypervisor: vcenter
distribution:
name: rhel
version: '6.9'
products: {}
credentials:
- name: root
sshkeyfile: /path/to/.ssh/id_rsa
username: root
type: network
- name: admin
sshkeyfile: /path/to/.ssh/id_rsa
username: admin
type: network
If you need rho to skip using a network credential, add the field as
follows:
- name: admin
sshkeyfile: /path/to/.ssh/id_rsa
username: admin
type: network
rho: false
In the sample configuration file above will be performed four scans, one
for each auth and machine combination.
"""
config = get_config()
auths = [
auth for auth in config["credentials"]
if auth["type"] == "network" and auth.get("rho", True)
]
inventory = {
machine["hostname"]: machine
for machine in config["inventory"]
if machine.get("hypervisor") == "vcenter"
}
if not auths or not inventory:
raise ValueError("Make sure to have credentials and inventory"
" items in the config file")
hostnames = [machine["hostname"] for machine in inventory.values()]
vms = [
vm for vm in get_vcenter_vms(vcenter_client) if vm.name in hostnames
]
for auth in auths:
if auth.get("sshkeyfile"):
auth_add({
"name": auth["name"],
"username": auth["username"],
"sshkeyfile": auth["sshkeyfile"],
})
elif auth.get("password"):
auth_add(
{
"name": auth["name"],
"username": auth["username"],
"password": None
},
[(CONNECTION_PASSWORD_INPUT, auth["password"])],
)
chunk_size = 5
chunks = [vms[i:i + chunk_size] for i in range(0, len(vms), chunk_size)]
for chunk in chunks:
machines_to_wait = []
for vm in chunk:
if vm.runtime.powerState == "poweredOff":
vm.PowerOnVM_Task()
machines_to_wait.append(inventory[vm.name])
wait_until_live(machines_to_wait)
for vm in chunk:
if vm.runtime.powerState == "poweredOn":
machine = inventory[vm.name]
for auth in auths:
profile_name = machine["hostname"] + "-" + auth["name"]
try:
result = scan_machine(machine, auth, profile_name)
except (AssertionError, pexpect.exceptions.EOF) as err:
with open(profile_name, "w") as handler:
handler.write(str(err))
result = None
SCAN_RESULTS[profile_name] = result
vm.PowerOffVM_Task()
def config_scans():
"""Return all CLI scans available on the configuration file."""
try:
return get_config().get("qpc", {}).get("cli-scans", [])
except ConfigFileNotFoundError:
return []
def setup_qpc():
"""Configure and login qpc with Camayoc's configuration info.
The minimum required configuration is both ``hostname`` and ``port``, for
example::
qpc:
hostname: localhost
port: 8000
If not specified ``https``, ``ssl-verify``, ``username`` and ``password``
will use their default values: ``false``, ``false``, ``admin`` and ``pass``
respectively.
See below an example with all fields being defined::
qpc:
hostname: quipucords.example.com
https: true
password: youshallnotpass
port: 443
ssl-verify: /path/to/custom/certificate
username: gandalf
"""
qpc_config = get_config().get("qpc", {})
hostname = qpc_config.get("hostname")
port = qpc_config.get("port")
if not all([hostname, port]):
raise ValueError(
"Both hostname and port must be defined under the qpc section on "
"the Camayoc's configuration file.")
https = qpc_config.get("https", False)
if not https:
https = " --use-http"
else:
https = ""
ssl_verify = qpc_config.get("ssl-verify", False)
if ssl_verify not in (True, False):
ssl_verify = " --ssl-verify={}".format(ssl_verify)
else:
ssl_verify = ""
command = "{} server config --host {} --port {}{}{}".format(
client_cmd, hostname, port, https, ssl_verify)
output, exitstatus = pexpect.run(command,
encoding="utf8",
withexitstatus=True)
assert exitstatus == 0, output
# now login to the server
username = qpc_config.get("username", "admin")
password = qpc_config.get("password", "pass")
command = "{} server login --username {}".format(client_cmd, username)
output, exitstatus = pexpect.run(
command,
encoding="utf8",
events=[("Password: "******"\n")],
withexitstatus=True,
)
assert exitstatus == 0, output
def test_invalid_hostname(self):
"""Raise an error if no config entry is found and no url specified."""
with mock.patch.object(config, "_CONFIG", self.invalid_config):
self.assertEqual(config.get_config(), self.invalid_config)
with self.assertRaises(exceptions.QPCBaseUrlNotFound):
api.Client(authenticate=False)
def test_create_with_config(self):
"""If a hostname is specified in the config file, we use it."""
with mock.patch.object(config, "_CONFIG", self.config):
self.assertEqual(config.get_config(), self.config)
client = api.Client(authenticate=False)
self.assertEqual(client.url, "http://example.com/api/v1/")
def test_cache_full(self):
"""No config is read from disk if the cache is populated."""
with mock.patch.object(config, "_CONFIG", self.config):
with mock.patch("camayoc.config.yaml.load") as load:
self.assertEqual(config.get_config(), self.config)
self.assertEqual(load.call_count, 0)
def scan_list():
"""Generate list of scan dict objects found in config file."""
try:
return get_config().get("qpc", {}).get("scans", [])
except (ConfigFileNotFoundError, KeyError):
return []
