def _get_client():
api_version = TAP_core_services[
TapComponent.container_broker]["api_version"]
configuration = K8sServiceConfigurationProvider.get(
TapComponent.container_broker,
api_endpoint="api/{}".format(api_version))
return HttpClientFactory.get(configuration)
def k8s_get_pods_metrics(pod_ip_port):
"""This method does not call k8s api, but particular pod"""
client = HttpClientFactory.get(
ProxiedConfigurationProvider.get(url="https://{}".format(pod_ip_port)))
return client.request(method=HttpMethod.GET,
path="metrics",
msg="POD: get metrics for {}".format(pod_ip_port))
def k8s_scale_pod(pod_name,
number_of_replicas,
rest_prefix="apis",
api_version="extensions/v1beta1"):
"""PUT /apis/extensions/v1beta1/namespaces/{namespace}/deployments/{name}/scale"""
body = {
"metadata": {
"name": pod_name,
"namespace": DEFAULT_NAMESPACE
},
"spec": {
"replicas": number_of_replicas
}
}
client = HttpClientFactory.get(
KubernetesConfigurationProvider.get(rest_prefix, api_version))
return client.request(
method=HttpMethod.PUT,
body=body,
path="namespaces/{DEFAULT_NAMESPACE}/deployments/{pod_name}/scale",
path_params={
'DEFAULT_NAMESPACE': DEFAULT_NAMESPACE,
'pod_name': pod_name
},
msg="POD {} scaled to {} replicas".format(pod_name,
number_of_replicas))
def basic_auth_client(self):
credentials = config.ng_k8s_service_credentials()
configuration = HttpClientConfiguration(HttpClientType.BASIC_AUTH,
url=config.api_url_full,
username=credentials[0],
password=credentials[1])
return HttpClientFactory.get(configuration)
def test_api_indicator_returns_platform_header(self, component):
"""
<b>Description:</b>
Checks if api indicator returns platform header.
<b>Input data:</b>
1. Component name: api service.
<b>Expected results:</b>
Test passes when component api indicator HTTP response header "x-platform" contains "TAP" string.
<b>Steps:</b>
1. Create HTTP client.
2. Send GET request to component api endpoint.
3. Verify that HTTP response header "x-platform" contains "TAP" string.
4. Verify that HTTP response text is empty.
"""
step("Check get /api endpoint")
url = "http://{}.{}".format(component, config.tap_domain)
client = HttpClientFactory.get(
ServiceConfigurationProvider.get(url, username=None,
password=None))
response = client.request(method=HttpMethod.GET,
path="api",
raw_response=True)
step('Check if request contains "x-platform" header')
assert response.headers['x-platform'] == 'TAP'
assert response.text == ''
def _get_client(client_type=HttpClientType.BASIC_AUTH,
username=None,
password=None):
configuration = HttpClientConfiguration(client_type=client_type,
url=config.api_url_full,
username=username,
password=password)
return HttpClientFactory.get(configuration)
def k8s_get_pods():
"""GET /namespaces/{namespace}/pods"""
return HttpClientFactory.get(
KubernetesConfigurationProvider.get()).request(
method=HttpMethod.GET,
path="namespaces/{DEFAULT_NAMESPACE}/pods",
path_params={'DEFAULT_NAMESPACE': DEFAULT_NAMESPACE},
msg="KUBERNETES: get pods")
def k8s_get_ingresses(rest_prefix="apis", api_version="extensions/v1beta1"):
"""PUT /apis/extensions/v1beta1/namespaces/{namespace}/ingresses"""
client = HttpClientFactory.get(
KubernetesConfigurationProvider.get(rest_prefix, api_version))
return client.request(method=HttpMethod.GET,
path="namespaces/{DEFAULT_NAMESPACE}/ingresses",
path_params={'DEFAULT_NAMESPACE': DEFAULT_NAMESPACE},
msg="KUBERNETES: get ingresses")
def k8s_get_services(*, namespace=DEFAULT_NAMESPACE):
"""GET /namespaces/{namespace}/services"""
return HttpClientFactory.get(
KubernetesConfigurationProvider.get()).request(
method=HttpMethod.GET,
path="namespaces/{namespace}/services",
path_params={'namespace': namespace},
msg="KUBERNETES: get services")
def test_login_to_console_with_empty_credentials(self):
"""
<b>Description:</b>
Log in to console with empty credentials
<b>Input data:</b>
None
<b>Expected results:</b>
It's impossible to log in to platform with empty credentials
<b>Steps:</b>
Log in to console with empty credentials
"""
step("Login using uaa endpoint")
configuration = ConsoleConfigurationProvider.get(username="",
password="")
with pytest.raises(UnexpectedResponseError):
HttpClientFactory.get(configuration)
def test_login_to_console_without_password(self):
"""
<b>Description:</b>
Log in to console without password
<b>Input data:</b>
Credentials without password
<b>Expected results:</b>
It's impossible to log in to platform without password
<b>Steps:</b>
Log in to console without password
"""
step("Login using uaa endpoint")
configuration = ConsoleConfigurationProvider.get(
username=config.admin_username, password=None)
with pytest.raises(UnexpectedResponseError):
HttpClientFactory.get(configuration)
def _get_client():
image_repository_config = third_party_services[TAP.image_repository]
service_url = image_repository_config.get("url")
if not service_url:
service_url = K8sSecureServiceConfigurationProvider.get_service_url(
service_name=image_repository_config["kubernetes_service_name"],
namespace=image_repository_config["kubernetes_namespace"])
configuration = K8sSecureServiceConfigurationProvider.get(
service_url=service_url,
api_version=image_repository_config["api_version"])
return HttpClientFactory.get(configuration)
def test_login_to_console_with_invalid_credentials(self):
"""
<b>Description:</b>
Log in to console with invalid credentials
<b>Input data:</b>
Invalid credentials
<b>Expected results:</b>
It's impossible to log in to platform with invalid credentials
<b>Steps:</b>
Log in to console with invalid credentials
"""
step("Login using uaa endpoint")
configuration = ConsoleConfigurationProvider.get(
username=generate_test_object_name(separator=""),
password=generate_test_object_name(separator=""))
with pytest.raises(UnexpectedResponseError):
HttpClientFactory.get(configuration)
def k8s_get_configmap(configmap_name):
"""GET /namespaces/{namespace}/configmaps/{configmap_name}"""
return HttpClientFactory.get(
KubernetesConfigurationProvider.get()).request(
method=HttpMethod.GET,
path="namespaces/{DEFAULT_NAMESPACE}/configmaps/{configmap_name}",
path_params={
'DEFAULT_NAMESPACE': DEFAULT_NAMESPACE,
'configmap_name': configmap_name
},
msg="KUBERNETES: get configmap {}".format(configmap_name))
def ensure_app_is_ready(self):
url = self.get_details()[self.FIELD_URLS][0]
client = HttpClientFactory.get(ServiceToolConfigurationProvider.get(url=url))
response = client.request(
method=HttpMethod.GET,
path="",
raw_response=True,
raise_exception=True,
msg="Send GET to application {}".format(self.name)
)
assert response.status_code == HttpStatus.CODE_OK
def k8s_logs(application_name, params):
"""GET /namespaces/{namespace}/pods/{application_name}/log"""
return HttpClientFactory.get(
KubernetesConfigurationProvider.get()).request(
method=HttpMethod.GET,
path="namespaces/{DEFAULT_NAMESPACE}/pods/{application_name}/log",
path_params={
'DEFAULT_NAMESPACE': DEFAULT_NAMESPACE,
'application_name': application_name
},
params=params,
msg="KUBERNETES: get logs {}".format(application_name))
def get_metrics(self):
proxy = "socks5://localhost:{}".format(config.ng_socks_proxy_port)
configuration = HttpClientConfiguration(
client_type=HttpClientType.NO_AUTH,
url="https://{}".format(self.host),
proxies={
"http": proxy,
"https": proxy
})
client = HttpClientFactory.get(configuration)
response = client.request(method=HttpMethod.GET,
path="metrics",
msg="NODE: get metrics from {}".format(
self.name))
return response
def _go_to_dashboard(instance, offering_name):
step("Check if instance has a dashboard url")
if instance.url is None:
pytest.skip(
"{} instance doesn't have an url".format(offering_name))
step("Go to dashboard")
client = HttpClientFactory.get(ConsoleConfigurationProvider.get())
url, path = instance.url.split(config.tap_domain + "/", 1)
try:
client.url = url + config.tap_domain
response = client.request(method=HttpMethod.GET,
path=path,
msg="Go to dashboard",
raw_response=True)
assert response.status_code // 100 == 2, "{} dashboard can't be reached".format(
offering_name)
finally:
client.url = config.console_url
def test_login_to_console_with_valid_credentials(self):
"""
<b>Description:</b>
Log in to console with valid credentials
<b>Input data:</b>
Valid credentials
<b>Expected results:</b>
It's possible to log in to platform with valid credentials
<b>Steps:</b>
Log in to console with valid credentials
"""
step("Login using uaa endpoint")
client = HttpClientFactory.get(
ConsoleConfigurationProvider.get(config.admin_username,
config.admin_password))
client.request(method=HttpMethod.GET, path="users/current")
def test_components_root_endpoint(self, component):
"""
<b>Description:</b>
Checks if console root endpoint returns OK status to HTTP GET request.
<b>Input data:</b>
Component root endpoint.
<b>Expected results:</b>
Test passes when console root endpoint returns OK status to HTTP GET request.
<b>Steps:</b>
1. Check console root endpoint.
2. Verify that HTTP response status code is 200.
"""
step("Check get / endpoint")
url = "http://{}.{}".format(component, config.tap_domain)
client = HttpClientFactory.get(ServiceConfigurationProvider.get(url))
response = client.request(method=HttpMethod.GET,
path="",
raw_response=True)
assert response.status_code == HttpStatus.CODE_OK
def test_components_check_healthz(self, component):
"""
<b>Description:</b>
Checks if api service and uaa return status OK on healthz endpoint.
<b>Input data:</b>
1. Component names: api service, uaa.
<b>Expected results:</b>
Test passes when api service and uaa healthz endpoint return status OK to HTTP GET request.
<b>Steps:</b>
1. Check healthz endpoint for components: api service and uaa.
2. Verify that response status is OK.
"""
step("Check healthz endpoint")
url = "http://{}.{}".format(component, config.tap_domain)
client = HttpClientFactory.get(
ApplicationConfigurationProvider.get(url))
response = client.request(method=HttpMethod.GET,
path="healthz",
raw_response=True)
assert response.status_code == HttpStatus.CODE_OK
def test_3rd_party_component_check_availability(self, service,
service_params):
"""
<b>Description:</b>
Checks if 3rd party component is available.
<b>Input data:</b>
1. List of third party components (currently: image repository).
<b>Expected results:</b>
Test passes when 3rd party component api service returns OK status to HTTP GET request.
<b>Steps:</b>
1. Check api get endpoint of 3rd party component.
2. Verify that response status is OK.
"""
if service_params["get_endpoint"] is None or service_params[
"api_version"] is None:
pytest.skip(
"Service {} does not have get endpoint or api version configured"
.format(service))
step("Check 3rd party component get api endpoint for {}".format(
service))
service_url = service_params.get("url")
if not service_url:
service_url = K8sSecureServiceConfigurationProvider.get_service_url(
service_name=service_params["kubernetes_service_name"],
namespace=service_params["kubernetes_namespace"])
client_config = K8sSecureServiceConfigurationProvider.get(
service_url=service_url, api_version=service_params["api_version"])
health_client = HttpClientFactory.get(client_config)
response = health_client.request(HttpMethod.GET,
path=service_params["get_endpoint"],
raw_response=True,
raise_exception=True,
msg="get")
assert response.status_code == HttpStatus.CODE_OK
def test_calling_healthz_on_non_existing_service_returns_404(self):
"""
<b>Description:</b>
Checks if calling healthz endpoint on not existing service returns error status code 404.
<b>Input data:</b>
False service URL.
<b>Expected results:</b>
Test passes when not existing service healthz endpoint returns status code 404 to HTTP GET request.
<b>Steps:</b>
1. Check healthz endpoint for not existing service.
2. Verify that response status code is 404.
"""
expected_status_code = HttpStatus.CODE_NOT_FOUND
url = "http://anything.{}".format(config.tap_domain)
step("Check that calling {} returns {}".format(url,
expected_status_code))
client = HttpClientFactory.get(
ApplicationConfigurationProvider.get(url))
with pytest.raises(UnexpectedResponseError) as e:
client.request(method=HttpMethod.GET, path="healthz")
assert e.value.status == expected_status_code
def test_3_submit_java_job_design(self, admin_user):
"""
<b>Description:</b>
Check that submitting a new job works.
<b>Input data:</b>
1. user admin id
2. job.properties file
3. workflow.xml file
<b>Expected results:</b>
Test passes when java job is submitted to platform.
<b>Steps:</b>
1. Create cdh master client.
2. Create job.properties and workflow.xml.
3. Create hdfs directory.
4. Copy workflow.xml to hdfs directory.
5. Submit java job to platform.
6. Check that job was successfully submited.
"""
step("Create application hdfs path name")
app_hdfs_path = generate_test_object_name(
prefix="hdfs://nameservice1/user/hue/oozie/workspaces/_{}_-oozie-".
format(admin_user.guid))
step("Create Cdh master 2 client")
self.__class__.client = CdhMasterClient(
cdh_host_name=config.cdh_master_2_hostname)
step("Create directory to store oozie data")
properties_path = "/tmp/{}".format(
datetime.now().strftime("%Y%m%d_%H%M%S_%f"))
step("Get oozie url")
app_name = TAP.cdh_broker
cdh_broker = next(
(app for app in Application.get_list() if app_name == app.name),
None)
assert cdh_broker is not None, "{} not available".format(app_name)
oozie_url = str(json.loads(cdh_broker.cf_api_env()["ENVIRONMENT_JSON"]["CREDENTIALS"])["resource_manager"]). \
replace("8088", "11000/oozie")
step("Prepare commands needed before job design execution")
cmds = [
["mkdir", properties_path],
["echo", "non-kerberos"],
[
"echo",
self.generate_job_properties(admin_user=admin_user,
app_hdfs_path=app_hdfs_path)
],
[
"cp", "{}{}".format(self.client._output_path, "/2_1"),
"{}{}".format(properties_path, "/job.properties")
],
[
"echo",
self.generate_workflow(job_name=self.job_name,
main_class=self.main_class,
csv_hdfs_path=self.csv_hdfs_path,
output_name=self.output_name)
],
[
"cp", "{}{}".format(self.client._output_path, "/4_1"),
"{}{}".format(properties_path, "/workflow.xml")
],
["hadoop ", "fs", "-mkdir", app_hdfs_path],
[
"hadoop ", "fs", "-copyFromLocal",
"{}{}".format(properties_path, "/workflow.xml"), app_hdfs_path
],
]
step("Check if this is kerberos environment")
if kerberos.is_kerberos_environment():
step(
"If kerberos environment: add ktinit with oauth token to commands"
)
client = HttpClientFactory.get(
UaaConfigurationProvider.get(config.admin_username,
config.admin_password))
cmds[1] = ["ktinit", "-t", client.auth.token]
step(
"Execute commands: Create job.properties and workflow.xml files at cdh-master-2; Create hdfs directory;"
"Copy workflow.xml to hdfs directory")
self.client.exec_commands(cmds)
step("Run java job design")
submited_job = self.client.exec_commands([[
"oozie", "job", "-oozie", oozie_url, "-config",
"{}{}".format(properties_path, "/job.properties"), "-run"
]])
assert "job" in submited_job[0][0], "Job was not created"
step("Get job workflow id")
self.__class__.workflow_id = submited_job[0][0].strip("job: ")
def get_client(self, service_name, endpoint=None):
configuration = K8sServiceConfigurationProvider.get(
service_name, api_endpoint=endpoint)
return HttpClientFactory.get(configuration)
def k8s_get_nodes():
"""GET /nodes"""
return HttpClientFactory.get(
KubernetesConfigurationProvider.get()).request(
method=HttpMethod.GET, path="nodes", msg="KUBERNETES: get nodes")
def _get_default_client(cls):
return HttpClientFactory.get(ConsoleConfigurationProvider.get())