def test_endpoints():
foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME)
# check that we can reach the scheduler via admin router, and that returned endpoints are sanitized:
core_site = etree.fromstring(sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'endpoints core-site.xml'))
check_properties(core_site, {
'ha.zookeeper.parent-znode': '/{}/hadoop-ha'.format(sdk_utils.get_zk_path(
foldered_name))
})
hdfs_site = etree.fromstring(sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'endpoints hdfs-site.xml'))
expect = {
'dfs.namenode.shared.edits.dir': 'qjournal://{}/hdfs'.format(';'.join([
sdk_hosts.autoip_host(
foldered_name,
'journal-{}-node'.format(i),
8485
) for i in range(3)])),
}
for i in range(2):
name_node = 'name-{}-node'.format(i)
expect['dfs.namenode.rpc-address.hdfs.{}'.format(name_node)] = sdk_hosts.autoip_host(
foldered_name, name_node, 9001)
expect['dfs.namenode.http-address.hdfs.{}'.format(name_node)] = sdk_hosts.autoip_host(
foldered_name, name_node, 9002)
check_properties(hdfs_site, expect)
def test_endpoints():
# check that we can reach the scheduler via admin router, and that returned endpoints are sanitized:
core_site = etree.fromstring(
cmd.run_cli('hdfs --name={} endpoints core-site.xml'.format(
FOLDERED_SERVICE_NAME)))
check_properties(
core_site, {
'ha.zookeeper.parent-znode':
'/dcos-service-{}/hadoop-ha'.format(ZK_SERVICE_PATH)
})
hdfs_site = etree.fromstring(
cmd.run_cli('hdfs --name={} endpoints hdfs-site.xml'.format(
FOLDERED_SERVICE_NAME)))
expect = {
'dfs.namenode.shared.edits.dir':
'qjournal://{}/hdfs'.format(';'.join([
sdk_hosts.autoip_host(FOLDERED_SERVICE_NAME,
'journal-{}-node'.format(i), 8485)
for i in range(3)
])),
}
for i in range(2):
name_node = 'name-{}-node'.format(i)
expect['dfs.namenode.rpc-address.hdfs.{}'.format(
name_node)] = sdk_hosts.autoip_host(FOLDERED_SERVICE_NAME,
name_node, 9001)
expect['dfs.namenode.http-address.hdfs.{}'.format(
name_node)] = sdk_hosts.autoip_host(FOLDERED_SERVICE_NAME,
name_node, 9002)
check_properties(hdfs_site, expect)
def test_endpoints():
foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME)
# check that we can reach the scheduler via admin router, and that returned endpoints are sanitized:
core_site = etree.fromstring(sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'endpoints core-site.xml'))
check_properties(core_site, {
'ha.zookeeper.parent-znode': '/{}/hadoop-ha'.format(sdk_utils.get_zk_path(
foldered_name))
})
hdfs_site = etree.fromstring(sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'endpoints hdfs-site.xml'))
expect = {
'dfs.namenode.shared.edits.dir': 'qjournal://{}/hdfs'.format(';'.join([
sdk_hosts.autoip_host(
foldered_name,
'journal-{}-node'.format(i),
8485
) for i in range(3)])),
}
for i in range(2):
name_node = 'name-{}-node'.format(i)
expect['dfs.namenode.rpc-address.hdfs.{}'.format(name_node)] = sdk_hosts.autoip_host(
foldered_name, name_node, 9001)
expect['dfs.namenode.http-address.hdfs.{}'.format(name_node)] = sdk_hosts.autoip_host(
foldered_name, name_node, 9002)
check_properties(hdfs_site, expect)
def test_endpoints():
# check that we can reach the scheduler via admin router, and that returned endpoints are sanitized:
core_site = etree.fromstring(
sdk_networks.get_endpoint_string(config.PACKAGE_NAME, foldered_name, "core-site.xml")
)
check_properties(
core_site,
{"ha.zookeeper.parent-znode": "/{}/hadoop-ha".format(sdk_utils.get_zk_path(foldered_name))},
)
hdfs_site = etree.fromstring(
sdk_networks.get_endpoint_string(config.PACKAGE_NAME, foldered_name, "hdfs-site.xml")
)
expect = {
"dfs.namenode.shared.edits.dir": "qjournal://{}/hdfs".format(
";".join(
[
sdk_hosts.autoip_host(foldered_name, "journal-{}-node".format(i), 8485)
for i in range(3)
]
)
)
}
for i in range(2):
name_node = "name-{}-node".format(i)
expect["dfs.namenode.rpc-address.hdfs.{}".format(name_node)] = sdk_hosts.autoip_host(
foldered_name, name_node, 9001
)
expect["dfs.namenode.http-address.hdfs.{}".format(name_node)] = sdk_hosts.autoip_host(
foldered_name, name_node, 9002
)
check_properties(hdfs_site, expect)
def get_service_principals(service_name: str,
realm: str,
custom_domain: str = None) -> list:
"""
Sets up the appropriate principals needed for a kerberized deployment of HDFS.
:return: A list of said principals
"""
primaries = [
"kafka",
]
tasks = [
"kafka-0-broker",
"kafka-1-broker",
"kafka-2-broker",
]
if custom_domain:
instances = map(
lambda task: sdk_hosts.custom_host(service_name, task,
custom_domain), tasks)
else:
instances = map(lambda task: sdk_hosts.autoip_host(service_name, task),
tasks)
principals = kerberos.generate_principal_list(primaries, instances, realm)
principals.extend(kerberos.generate_principal_list(USERS, [
None,
], realm))
return principals
def test_verify_https_ports(hdfs_client, node_type, port):
"""
Verify that HTTPS port is open name, journal and data node types.
"""
task_id = "{}-0-node".format(node_type)
host = sdk_hosts.autoip_host(config.SERVICE_NAME, task_id, port)
cmd = [
"curl",
"-v",
"--cacert",
hdfs_client["dcos_ca_bundle"],
"https://{host}".format(host=host),
]
rc, stdout, stderr = sdk_cmd.marathon_task_exec(hdfs_client["id"],
" ".join(cmd))
assert not rc
assert "SSL connection using TLS1.2 / ECDHE_RSA_AES_128_GCM_SHA256" in stderr
assert "server certificate verification OK" in stderr
assert "common name: {}.{} (matched)".format(task_id,
config.SERVICE_NAME) in stderr
# In the Kerberos case we expect a 401 error
assert "401 Authentication required" in stdout
def test_endpoints_address():
foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME)
@retrying.retry(wait_fixed=1000,
stop_max_delay=120 * 1000,
retry_on_result=lambda res: not res)
def wait():
_, ret, _ = sdk_cmd.svc_cli(
config.PACKAGE_NAME,
foldered_name,
"endpoints {}".format(config.DEFAULT_TASK_NAME),
parse_json=True,
)
if len(ret["address"]) == config.DEFAULT_BROKER_COUNT:
return ret
return False
endpoints = wait()
# NOTE: do NOT closed-to-extension assert len(endpoints) == _something_
assert len(endpoints["address"]) == config.DEFAULT_BROKER_COUNT
assert len(endpoints["dns"]) == config.DEFAULT_BROKER_COUNT
for i in range(len(endpoints["dns"])):
assert (sdk_hosts.autoip_host(foldered_name,
"kafka-{}-broker".format(i))
in endpoints["dns"][i])
assert endpoints["vip"] == sdk_hosts.vip_host(foldered_name, "broker",
9092)
def test_tls_basic_artifacts():
# Load end-entity certificate from keystore and root CA cert from truststore
stdout = sdk_cmd.service_task_exec(
config.SERVICE_NAME, "artifacts-0-node", "cat secure-tls-pod.crt"
)[1].encode("ascii")
end_entity_cert = x509.load_pem_x509_certificate(stdout, DEFAULT_BACKEND)
root_ca_cert_in_truststore = _export_cert_from_task_keystore(
"artifacts-0-node", "keystore.truststore", "dcos-root"
)
# Check that certificate subject maches the service name
common_name = end_entity_cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value
assert common_name in sdk_hosts.autoip_host(config.SERVICE_NAME, "artifacts-0-node")
san_extension = end_entity_cert.extensions.get_extension_for_oid(
ExtensionOID.SUBJECT_ALTERNATIVE_NAME
)
sans = san_extension.value._general_names._general_names
assert len(sans) == 1
cluster_root_ca_cert = x509.load_pem_x509_certificate(
transport_encryption.fetch_dcos_ca_bundle_contents(), DEFAULT_BACKEND
)
assert root_ca_cert_in_truststore.signature == cluster_root_ca_cert.signature
def get_service_principals(service_name: str, realm: str) -> list:
"""
Sets up the appropriate principals needed for a kerberized deployment of HDFS.
:return: A list of said principals
"""
primaries = ["hdfs", "HTTP"]
tasks = [
"name-0-node",
"name-0-zkfc",
"name-1-node",
"name-1-zkfc",
"journal-0-node",
"journal-1-node",
"journal-2-node",
"data-0-node",
"data-1-node",
"data-2-node",
]
instances = map(lambda task: sdk_hosts.autoip_host(service_name, task),
tasks)
principals = kerberos.generate_principal_list(primaries, instances, realm)
principals.extend(kerberos.generate_principal_list(USERS, [
None,
], realm))
http_instance = sdk_hosts.vip_host("marathon",
".".join(["api", service_name]))
http_principal = kerberos.genererate_principal("HTTP", http_instance,
realm)
principals.append(http_principal)
return principals
def test_endpoints():
foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME)
# check that we can reach the scheduler via admin router, and that returned endpoints are sanitized:
for endpoint in config.ENDPOINT_TYPES:
# TODO: if an endpoint isn't present this call fails w/ the following:
# 》dcos beta-elastic --name=/test/integration/elastic endpoints ingest-http
# *** snip ***
# Could not reach the service scheduler with name '/test/integration/elastic'.capitalizeDid you provide the correct service name? Specify a different name with '--name=<name>'.absWas the service recently installed or updated? It may still be initializing, wait a bit and try again.abs 1
# *** snip ***
# Please consider using the CLI endpoints (list) command to determine which endpoints are present
# rather than a hardcoded set in config.py .
# Also please consider either fixing the CLI+API to emit an error message that is more meaningful.
# The service is up, the error condition is that the user requested an endpoint that isn't present
# that should be a 404, (specific resource) not found, not that the service is down or that the
# service name is not found.
#
# further, since we expect the endpoints to differ if ingest nodes A) is zero ; or B) positive integer, we
# should have a test for each case.
endpoints = sdk_cmd.svc_cli(config.PACKAGE_NAME,
foldered_name,
'endpoints {}'.format(endpoint),
json=True)
host = endpoint.split('-')[0] # 'coordinator-http' => 'coordinator'
assert endpoints['dns'][0].startswith(
sdk_hosts.autoip_host(foldered_name, host + '-0-node'))
assert endpoints['vip'].startswith(
sdk_hosts.vip_host(foldered_name, host))
def test_tls_basic_artifacts(hello_world_service):
task_id = sdk_tasks.get_task_ids(PACKAGE_NAME, 'artifacts')[0]
assert task_id
# Load end-entity certificate from keystore and root CA cert from truststore
end_entity_cert = x509.load_pem_x509_certificate(
task_exec(task_id, 'cat secure-tls-pod.crt').encode('ascii'),
DEFAULT_BACKEND)
root_ca_cert_in_truststore = _export_cert_from_task_keystore(
task_id, 'keystore.truststore', 'dcos-root')
# Check that certificate subject maches the service name
common_name = end_entity_cert.subject.get_attributes_for_oid(
NameOID.COMMON_NAME)[0].value
assert common_name in sdk_hosts.autoip_host(PACKAGE_NAME, 'artifacts-0-node')
san_extension = end_entity_cert.extensions.get_extension_for_oid(
ExtensionOID.SUBJECT_ALTERNATIVE_NAME)
sans = san_extension.value._general_names._general_names
assert len(sans) == 1
cluster_root_ca_cert = x509.load_pem_x509_certificate(
sdk_cmd.request(
'get', shakedown.dcos_url_path('/ca/dcos-ca.crt')).content,
DEFAULT_BACKEND)
assert root_ca_cert_in_truststore.signature == cluster_root_ca_cert.signature
def test_tls_basic_artifacts(hello_world_service):
task_id = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'artifacts')[0]
assert task_id
# Load end-entity certificate from keystore and root CA cert from truststore
end_entity_cert = x509.load_pem_x509_certificate(
task_exec(task_id, 'cat secure-tls-pod.crt').encode('ascii'),
DEFAULT_BACKEND)
root_ca_cert_in_truststore = _export_cert_from_task_keystore(
task_id, 'keystore.truststore', 'dcos-root')
# Check that certificate subject maches the service name
common_name = end_entity_cert.subject.get_attributes_for_oid(
NameOID.COMMON_NAME)[0].value
assert common_name in sdk_hosts.autoip_host(config.SERVICE_NAME, 'artifacts-0-node')
san_extension = end_entity_cert.extensions.get_extension_for_oid(
ExtensionOID.SUBJECT_ALTERNATIVE_NAME)
sans = san_extension.value._general_names._general_names
assert len(sans) == 1
cluster_root_ca_cert = x509.load_pem_x509_certificate(
sdk_cmd.request(
'get', shakedown.dcos_url_path('/ca/dcos-ca.crt')).content,
DEFAULT_BACKEND)
assert root_ca_cert_in_truststore.signature == cluster_root_ca_cert.signature
def test_tls_basic_artifacts():
# Load end-entity certificate from keystore and root CA cert from truststore
stdout = sdk_cmd.service_task_exec(
config.SERVICE_NAME, 'artifacts-0-node',
'cat secure-tls-pod.crt')[1].encode('ascii')
end_entity_cert = x509.load_pem_x509_certificate(stdout, DEFAULT_BACKEND)
root_ca_cert_in_truststore = _export_cert_from_task_keystore(
'artifacts-0-node', 'keystore.truststore', 'dcos-root')
# Check that certificate subject maches the service name
common_name = end_entity_cert.subject.get_attributes_for_oid(
NameOID.COMMON_NAME)[0].value
assert common_name in sdk_hosts.autoip_host(config.SERVICE_NAME,
'artifacts-0-node')
san_extension = end_entity_cert.extensions.get_extension_for_oid(
ExtensionOID.SUBJECT_ALTERNATIVE_NAME)
sans = san_extension.value._general_names._general_names
assert len(sans) == 1
cluster_root_ca_cert = x509.load_pem_x509_certificate(
sdk_cmd.cluster_request('GET', '/ca/dcos-ca.crt').content,
DEFAULT_BACKEND)
assert root_ca_cert_in_truststore.signature == cluster_root_ca_cert.signature
def get_service_principals(service_name: str, realm: str) -> list:
"""
Sets up the appropriate principals needed for a kerberized deployment of HDFS.
:return: A list of said principals
"""
primaries = ["hdfs", "HTTP"]
tasks = [
"name-0-node",
"name-0-zkfc",
"name-1-node",
"name-1-zkfc",
"journal-0-node",
"journal-1-node",
"journal-2-node",
"data-0-node",
"data-1-node",
"data-2-node",
]
instances = map(lambda task: sdk_hosts.autoip_host(service_name, task), tasks)
principals = kerberos.generate_principal_list(primaries, instances, realm)
principals.extend(kerberos.generate_principal_list(USERS, [None], realm))
http_instance = sdk_hosts.vip_host("marathon", ".".join(["api", service_name]))
http_principal = kerberos.genererate_principal("HTTP", http_instance, realm)
principals.append(http_principal)
return principals
def _curl_query(
service_name,
method,
endpoint,
json_body=None,
task="master-0-node",
https=False,
return_json=True,
http_user=DEFAULT_ELASTICSEARCH_USER,
http_password=DEFAULT_ELASTICSEARCH_PASSWORD,
):
protocol = "https" if https else "http"
if http_password and not http_user:
raise Exception(
"HTTP authentication won't work with just a password. Needs at least user, or both user AND password"
)
credentials = ""
if http_user:
credentials = "-u {}".format(http_user)
if http_password:
credentials = "{}:{}".format(credentials, http_password)
host = sdk_hosts.autoip_host(service_name, task,
_master_zero_http_port(service_name))
curl_cmd = "/opt/mesosphere/bin/curl -sS {} -X{} '{}://{}/{}'".format(
credentials, method, protocol, host, endpoint)
if json_body:
curl_cmd += " -H 'Content-type: application/json' -d '{}'".format(
json.dumps(json_body))
task_name = "master-0-node"
exit_code, stdout, stderr = sdk_cmd.service_task_exec(
service_name, task_name, curl_cmd)
def build_errmsg(msg):
return "{}\nCommand:\n{}\nstdout:\n{}\nstderr:\n{}".format(
msg, curl_cmd, stdout, stderr)
if exit_code:
log.warning(
build_errmsg(
"Failed to run command on {}, retrying or giving up.".format(
task_name)))
return None
if not return_json:
return stdout
try:
return json.loads(stdout)
except Exception:
log.warning(
build_errmsg(
"Failed to parse stdout as JSON, retrying or giving up."))
return None
def get_zookeeper_principals(service_name: str, realm: str) -> list:
primaries = ["zookeeper"]
tasks = ["zookeeper-0-server", "zookeeper-1-server", "zookeeper-2-server"]
instances = map(lambda task: sdk_hosts.autoip_host(service_name, task), tasks)
principals = krb5.generate_principal_list(primaries, instances, realm)
return principals
def test_endpoints():
# check that we can reach the scheduler via admin router, and that returned endpoints are sanitized:
endpoints = cmd.svc_cli(
config.PACKAGE_NAME, config.get_foldered_service_name(),
'endpoints native-client', json=True)
assert endpoints['dns'][0] == sdk_hosts.autoip_host(
config.get_foldered_service_name(), 'node-0-server', 9042)
assert not 'vip' in endpoints
def test_endpoints():
# check that we can reach the scheduler via admin router, and that returned endpoints are sanitized:
endpoints = sdk_networks.get_endpoint(config.PACKAGE_NAME,
config.get_foldered_service_name(),
"native-client")
assert endpoints["dns"][0] == sdk_hosts.autoip_host(
config.get_foldered_service_name(), "node-0-server", 9042)
assert "vip" not in endpoints
def test_endpoints():
# check that we can reach the scheduler via admin router, and that returned endpoints are sanitized:
endpoints = json.loads(
cmd.run_cli('cassandra --name={} endpoints node'.format(
FOLDERED_SERVICE_NAME)))
assert endpoints['dns'][0] == hosts.autoip_host(FOLDERED_SERVICE_NAME,
'node-0-server', 9042)
assert endpoints['vips'][0] == hosts.vip_host(FOLDERED_SERVICE_NAME,
'node', 9042)
def test_endpoints():
# check that we can reach the scheduler via admin router, and that returned endpoints are sanitized:
endpoints = json.loads(
cmd.run_cli('cassandra --name={} endpoints node'.format(
config.get_foldered_service_name())))
assert endpoints['dns'][0] == sdk_hosts.autoip_host(
config.get_foldered_service_name(), 'node-0-server', 9042)
assert endpoints['vip'] == sdk_hosts.vip_host(
config.get_foldered_service_name(), 'node', 9042)
def _curl_query(
service_name: str,
method: str,
endpoint: str,
json_body: Optional[Dict[str, Any]] = None,
task: str = "master-0-node",
https: bool = False,
return_json: bool = True,
http_user: Optional[str] = DEFAULT_ELASTICSEARCH_USER,
http_password: Optional[str] = DEFAULT_ELASTICSEARCH_PASSWORD,
) -> Optional[Union[str, Dict[str, Any]]]:
protocol = "https" if https else "http"
if http_password:
if not http_user:
http_user = DEFAULT_ELASTICSEARCH_USER
log.info("Using default basic HTTP user: '******'", http_user)
credentials = "-u {}:{}".format(http_user, http_password)
else:
if http_user:
raise Exception(
"HTTP authentication won't work with just a user. Needs both user AND password"
)
credentials = ""
host = sdk_hosts.autoip_host(service_name, task, _master_zero_http_port(service_name))
curl_cmd = "/opt/mesosphere/bin/curl -sS {} -X{} '{}://{}/{}'".format(
credentials, method, protocol, host, endpoint
)
if json_body:
curl_cmd += " -H 'Content-type: application/json' -d '{}'".format(json.dumps(json_body))
task_name = "master-0-node"
exit_code, stdout, stderr = sdk_cmd.service_task_exec(service_name, task_name, curl_cmd)
def build_errmsg(msg: str) -> str:
return "{}\nCommand:\n{}\nstdout:\n{}\nstderr:\n{}".format(msg, curl_cmd, stdout, stderr)
if exit_code:
log.warning(
build_errmsg("Failed to run command on {}, retrying or giving up.".format(task_name))
)
return None
if not return_json:
return stdout
try:
result = json.loads(stdout)
assert isinstance(result, dict)
return result
except Exception:
log.warning(build_errmsg("Failed to parse stdout as JSON, retrying or giving up."))
return None
def test_endpoints():
# check that we can reach the scheduler via admin router, and that returned endpoints are sanitized:
for endpoint in config.ENDPOINT_TYPES:
endpoints = cmd.svc_cli(
config.PACKAGE_NAME, FOLDERED_SERVICE_NAME,
'endpoints {}'.format(endpoint), json=True)
host = endpoint.split('-')[0] # 'coordinator-http' => 'coordinator'
assert endpoints['dns'][0].startswith(sdk_hosts.autoip_host(FOLDERED_SERVICE_NAME, host + '-0-node'))
assert endpoints['vip'].startswith(sdk_hosts.vip_host(FOLDERED_SERVICE_NAME, host))
def test_endpoints():
# check that we can reach the scheduler via admin router, and that returned endpoints are sanitized:
for endpoint in config.ENDPOINT_TYPES:
endpoints = sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'endpoints {}'.format(endpoint), json=True)
host = endpoint.split('-')[0] # 'coordinator-http' => 'coordinator'
assert endpoints['dns'][0].startswith(sdk_hosts.autoip_host(foldered_name, host + '-0-node'))
assert endpoints['vip'].startswith(sdk_hosts.vip_host(foldered_name, host))
sdk_plan.wait_for_completed_deployment(foldered_name)
sdk_plan.wait_for_completed_recovery(foldered_name)
def test_endpoints():
# check that we can reach the scheduler via admin router, and that returned endpoints are sanitized:
for nodetype in ('coordinator', 'data', 'ingest', 'master'):
endpoints = json.loads(
cmd.run_cli('elastic --name={} endpoints {}'.format(
FOLDERED_SERVICE_NAME, nodetype)))
assert endpoints['dns'][0].startswith(
hosts.autoip_host(FOLDERED_SERVICE_NAME, nodetype + '-0-node'))
assert endpoints['vips'][0].startswith(
hosts.vip_host(FOLDERED_SERVICE_NAME, nodetype))
def test_verify_https_ports(hdfs_client, node_type, port):
"""
Verify that HTTPS port is open name, journal and data node types.
"""
task_id = "{}-0-node".format(node_type)
host = sdk_hosts.autoip_host(config.SERVICE_NAME, task_id, port)
ca_bundle = transport_encryption.fetch_dcos_ca_bundle(hdfs_client["id"])
config.verify_https_ports(ca_bundle, host, task_id)
def test_endpoints() -> None:
# Check that we can reach the scheduler via admin router, and that returned endpoints are
# sanitized.
for endpoint in config.ENDPOINT_TYPES:
endpoints = sdk_networks.get_endpoint(package_name, service_name, endpoint)
host = endpoint.split("-")[0] # 'coordinator-http' => 'coordinator'
assert endpoints["dns"][0].startswith(sdk_hosts.autoip_host(service_name, host + "-0-node"))
assert endpoints["vip"].startswith(sdk_hosts.vip_host(service_name, host))
sdk_plan.wait_for_completed_deployment(service_name)
sdk_plan.wait_for_completed_recovery(service_name)
def test_endpoints():
# check that we can reach the scheduler via admin router, and that returned endpoints are sanitized:
for endpoint in config.ENDPOINT_TYPES:
endpoints = json.loads(
cmd.run_cli('elastic --name={} endpoints {}'.format(
FOLDERED_SERVICE_NAME, endpoint)))
host = endpoint.split('-')[0] # 'coordinator-http' => 'coordinator'
assert endpoints['dns'][0].startswith(
sdk_hosts.autoip_host(FOLDERED_SERVICE_NAME, host + '-0-node'))
assert endpoints['vip'].startswith(
sdk_hosts.vip_host(FOLDERED_SERVICE_NAME, host))
def test_verify_https_ports(node_type, port, hdfs_service_tls):
"""
Verify that HTTPS port is open name, journal and data node types.
"""
host = sdk_hosts.autoip_host(config.SERVICE_NAME,
"{}-0-node".format(node_type), port)
exit_status, output = shakedown.run_command_on_master(
_curl_https_get_code(host))
assert exit_status
assert output == '200'
def get_zookeeper_principals(service_name: str, realm: str) -> list:
primaries = ["zookeeper", ]
tasks = [
"zookeeper-0-server",
"zookeeper-1-server",
"zookeeper-2-server",
]
instances = map(lambda task: sdk_hosts.autoip_host(service_name, task), tasks)
principals = krb5.generate_principal_list(primaries, instances, realm)
return principals
def test_endpoints():
# check that we can reach the scheduler via admin router, and that returned endpoints are sanitized:
for endpoint in config.ENDPOINT_TYPES:
endpoints = sdk_networks.get_endpoint(config.PACKAGE_NAME,
foldered_name, endpoint)
host = endpoint.split("-")[0] # 'coordinator-http' => 'coordinator'
assert endpoints["dns"][0].startswith(
sdk_hosts.autoip_host(foldered_name, host + "-0-node"))
assert endpoints["vip"].startswith(
sdk_hosts.vip_host(foldered_name, host))
sdk_plan.wait_for_completed_deployment(foldered_name)
sdk_plan.wait_for_completed_recovery(foldered_name)
def test_endpoints_address():
endpoints = sdk_networks.get_endpoint(config.PACKAGE_NAME, FOLDERED_NAME,
"broker")
# NOTE: do NOT closed-to-extension assert len(endpoints) == _something_
assert len(endpoints["address"]) == config.DEFAULT_BROKER_COUNT
assert len(endpoints["dns"]) == config.DEFAULT_BROKER_COUNT
for i in range(len(endpoints["dns"])):
assert (sdk_hosts.autoip_host(FOLDERED_NAME,
"kafka-{}-broker".format(i))
in endpoints["dns"][i])
assert endpoints["vip"] == sdk_hosts.vip_host(FOLDERED_NAME, "broker",
9092)
def test_verify_https_ports(node_type, port, hdfs_service):
"""
Verify that HTTPS port is open name, journal and data node types.
"""
host = sdk_hosts.autoip_host(config.SERVICE_NAME,
"{}-0-node".format(node_type), port)
@retrying.retry(wait_fixed=1000,
stop_max_delay=config.DEFAULT_HDFS_TIMEOUT * 1000,
retry_on_result=lambda res: not res)
def fn():
exit_status, output = sdk_cmd.master_ssh(_curl_https_get_code(host))
return exit_status and output == '200'
assert fn()
def test_verify_https_ports(node_type, port, hdfs_service_tls):
"""
Verify that HTTPS port is open name, journal and data node types.
"""
host = sdk_hosts.autoip_host(
config.SERVICE_NAME, "{}-0-node".format(node_type), port)
@retrying.retry(
wait_fixed=1000,
stop_max_delay=config.DEFAULT_HDFS_TIMEOUT*1000,
retry_on_result=lambda res: not res)
def fn():
exit_status, output = sdk_cmd.master_ssh(_curl_https_get_code(host))
return exit_status and output == '200'
assert fn()
def test_endpoints_address():
def fun():
ret = sdk_cmd.svc_cli(
config.PACKAGE_NAME, FOLDERED_SERVICE_NAME,
'endpoints {}'.format(config.DEFAULT_TASK_NAME), json=True)
if len(ret['address']) == config.DEFAULT_BROKER_COUNT:
return ret
return False
endpoints = shakedown.wait_for(fun)
# NOTE: do NOT closed-to-extension assert len(endpoints) == _something_
assert len(endpoints['address']) == config.DEFAULT_BROKER_COUNT
assert len(endpoints['dns']) == config.DEFAULT_BROKER_COUNT
for i in range(len(endpoints['dns'])):
assert sdk_hosts.autoip_host(
FOLDERED_SERVICE_NAME, 'kafka-{}-broker'.format(i)) in endpoints['dns'][i]
assert endpoints['vip'] == sdk_hosts.vip_host(
FOLDERED_SERVICE_NAME, 'broker', 9092)
def test_endpoints_address():
def fun():
ret = service_cli('endpoints {}'.format(DEFAULT_TASK_NAME),
service_name=FOLDERED_SERVICE_NAME)
if len(ret['address']) == DEFAULT_BROKER_COUNT:
return ret
return False
endpoints = shakedown.wait_for(fun)
# NOTE: do NOT closed-to-extension assert len(endpoints) == _something_
assert len(endpoints['address']) == DEFAULT_BROKER_COUNT
assert len(endpoints['dns']) == DEFAULT_BROKER_COUNT
for i in range(len(endpoints['dns'])):
assert hosts.autoip_host(
FOLDERED_SERVICE_NAME,
'kafka-{}-broker'.format(i)) in endpoints['dns'][i]
assert endpoints['vips'][0] == hosts.vip_host(FOLDERED_SERVICE_NAME,
'broker', 9092)
def test_verify_https_ports(hdfs_client, node_type, port):
"""
Verify that HTTPS port is open name, journal and data node types.
"""
task_id = "{}-0-node".format(node_type)
host = sdk_hosts.autoip_host(config.SERVICE_NAME, task_id, port)
ca_bundle = transport_encryption.fetch_dcos_ca_bundle(hdfs_client["id"])
ok, stdout, stderr = config.run_client_command("curl -v --cacert {} https://{}".format(ca_bundle, host))
assert ok
assert "SSL connection using TLS1.2 / ECDHE_RSA_AES_128_GCM_SHA256" in stderr
assert "server certificate verification OK" in stderr
assert "common name: {}.{} (matched)".format(task_id, config.SERVICE_NAME) in stderr
# In the Kerberos case we expect a 401 error
assert "401 Authentication required" in stdout
def test_verify_https_ports(hdfs_client, node_type, port):
"""
Verify that HTTPS port is open name, journal and data node types.
"""
task_id = "{}-0-node".format(node_type)
host = sdk_hosts.autoip_host(config.SERVICE_NAME, task_id, port)
ca_bundle = transport_encryption.fetch_dcos_ca_bundle(hdfs_client["id"])
ok, stdout, stderr = config.run_client_command(
"curl -v --cacert {} https://{}".format(ca_bundle, host))
assert ok
assert "server certificate verification OK" in stderr
assert "common name: {}.{} (matched)".format(task_id,
config.SERVICE_NAME) in stderr
# In the Kerberos case we expect a 401 error
assert "401 Authentication required" in stdout
def test_endpoints_address():
foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME)
@retrying.retry(
wait_fixed=1000,
stop_max_delay=120*1000,
retry_on_result=lambda res: not res)
def wait():
ret = sdk_cmd.svc_cli(
config.PACKAGE_NAME, foldered_name,
'endpoints {}'.format(config.DEFAULT_TASK_NAME), json=True)
if len(ret['address']) == config.DEFAULT_BROKER_COUNT:
return ret
return False
endpoints = wait()
# NOTE: do NOT closed-to-extension assert len(endpoints) == _something_
assert len(endpoints['address']) == config.DEFAULT_BROKER_COUNT
assert len(endpoints['dns']) == config.DEFAULT_BROKER_COUNT
for i in range(len(endpoints['dns'])):
assert sdk_hosts.autoip_host(foldered_name, 'kafka-{}-broker'.format(i)) in endpoints['dns'][i]
assert endpoints['vip'] == sdk_hosts.vip_host(foldered_name, 'broker', 9092)
def test_endpoints_address():
foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME)
def fun():
ret = sdk_cmd.svc_cli(config.PACKAGE_NAME,
foldered_name,
'endpoints {}'.format(config.DEFAULT_TASK_NAME),
json=True)
if len(ret['address']) == config.DEFAULT_BROKER_COUNT:
return ret
return False
endpoints = shakedown.wait_for(fun)
# NOTE: do NOT closed-to-extension assert len(endpoints) == _something_
assert len(endpoints['address']) == config.DEFAULT_BROKER_COUNT
assert len(endpoints['dns']) == config.DEFAULT_BROKER_COUNT
for i in range(len(endpoints['dns'])):
assert sdk_hosts.autoip_host(
foldered_name, 'kafka-{}-broker'.format(i)) in endpoints['dns'][i]
assert endpoints['vip'] == sdk_hosts.vip_host(foldered_name, 'broker',
9092)
def _curl_query(service_name,
method,
endpoint,
json_data=None,
role="master",
https=False,
return_json=True):
protocol = "https" if https else "http"
host = sdk_hosts.autoip_host(service_name, "{}-0-node".format(role),
_master_zero_http_port(service_name))
curl_cmd = "/opt/mesosphere/bin/curl -sS -u elastic:changeme -X{} '{}://{}/{}'".format(
method, protocol, host, endpoint)
if json_data:
curl_cmd += " -H 'Content-type: application/json' -d '{}'".format(
json.dumps(json_data))
task_name = "master-0-node"
exit_code, stdout, stderr = sdk_cmd.service_task_exec(
service_name, task_name, curl_cmd)
def build_errmsg(msg):
return "{}\nCommand:\n{}\nstdout:\n{}\nstderr:\n{}".format(
msg, curl_cmd, stdout, stderr)
if exit_code:
log.warning(
build_errmsg(
"Failed to run command on {}, retrying or giving up.".format(
task_name)))
return None
if not return_json:
return stdout
try:
return json.loads(stdout)
except Exception:
log.warning(
build_errmsg(
"Failed to parse stdout as JSON, retrying or giving up."))
return None
def test_verify_https_ports(hdfs_client, node_type, port):
"""
Verify that HTTPS port is open name, journal and data node types.
"""
task_id = "{}-0-node".format(node_type)
host = sdk_hosts.autoip_host(
config.SERVICE_NAME, task_id, port)
cmd = ["curl", "-v",
"--cacert", hdfs_client["dcos_ca_bundle"],
"https://{host}".format(host=host), ]
rc, stdout, stderr = sdk_cmd.marathon_task_exec(hdfs_client["id"], " ".join(cmd))
assert not rc
assert "SSL connection using TLS1.2 / ECDHE_RSA_AES_128_GCM_SHA256" in stderr
assert "server certificate verification OK" in stderr
assert "common name: {}.{} (matched)".format(task_id, config.SERVICE_NAME) in stderr
# In the Kerberos case we expect a 401 error
assert "401 Authentication required" in stdout
def get_service_principals(service_name: str, realm: str, custom_domain: str = None) -> list:
"""
Sets up the appropriate principals needed for a kerberized deployment of HDFS.
:return: A list of said principals
"""
primaries = ["kafka", ]
tasks = [
"kafka-0-broker",
"kafka-1-broker",
"kafka-2-broker",
]
if custom_domain:
instances = map(lambda task: sdk_hosts.custom_host(service_name, task, custom_domain), tasks)
else:
instances = map(lambda task: sdk_hosts.autoip_host(service_name, task), tasks)
principals = kerberos.generate_principal_list(primaries, instances, realm)
principals.extend(kerberos.generate_principal_list(USERS, [None, ], realm))
return principals
def _curl_query(service_name, method, endpoint, json_data=None, role="master", https=False, return_json=True):
protocol = 'https' if https else 'http'
host = sdk_hosts.autoip_host(service_name, "{}-0-node".format(role), _master_zero_http_port(service_name))
curl_cmd = "/opt/mesosphere/bin/curl -sS -u elastic:changeme -X{} '{}://{}/{}'".format(method, protocol, host, endpoint)
if json_data:
curl_cmd += " -H 'Content-type: application/json' -d '{}'".format(json.dumps(json_data))
task_name = "master-0-node"
exit_code, stdout, stderr = sdk_cmd.service_task_exec(service_name, task_name, curl_cmd)
def build_errmsg(msg):
return "{}\nCommand:\n{}\nstdout:\n{}\nstderr:\n{}".format(msg, curl_cmd, stdout, stderr)
if exit_code:
log.warning(build_errmsg("Failed to run command on {}, retrying or giving up.".format(task_name)))
return None
if not return_json:
return stdout
try:
return json.loads(stdout)
except:
log.warning(build_errmsg("Failed to parse stdout as JSON, retrying or giving up."))
return None
def get_foldered_node_address() -> str:
return sdk_hosts.autoip_host(get_foldered_service_name(), "node-0-server")
def test_overlay_network():
"""Verify that the current deploy plan matches the expected plan from the spec."""
deployment_plan = plan.wait_for_completed_deployment(PACKAGE_NAME)
utils.out("deployment_plan: " + str(deployment_plan))
# test that the deployment plan is correct
assert (len(deployment_plan['phases']) == 5)
assert (deployment_plan['phases'][0]['name'] == 'hello-overlay-vip-deploy')
assert (deployment_plan['phases'][1]['name'] == 'hello-overlay-deploy')
assert (deployment_plan['phases'][2]['name'] == 'hello-host-vip-deploy')
assert (deployment_plan['phases'][3]['name'] == 'hello-host-deploy')
assert (deployment_plan["phases"][4]["name"] == "getter-deploy")
assert (len(deployment_plan['phases'][0]['steps']) == 1)
assert (len(deployment_plan["phases"][1]["steps"]) == 1)
assert (len(deployment_plan["phases"][2]["steps"]) == 1)
assert (len(deployment_plan["phases"][3]["steps"]) == 1)
assert (len(deployment_plan["phases"][4]["steps"]) == 4)
# test that the tasks are all up, which tests the overlay DNS
framework_tasks = [
task
for task in shakedown.get_service_tasks(PACKAGE_NAME, completed=False)
]
framework_task_names = [t["name"] for t in framework_tasks]
expected_tasks = [
'getter-0-get-host', 'getter-0-get-overlay',
'getter-0-get-overlay-vip', 'getter-0-get-host-vip',
'hello-host-vip-0-server', 'hello-overlay-vip-0-server',
'hello-host-0-server', 'hello-overlay-0-server'
]
for expected_task in expected_tasks:
assert (expected_task
in framework_task_names), "Missing {expected}".format(
expected=expected_task)
for task in framework_tasks:
name = task["name"]
if "getter" in name: # don't check the "getter" tasks because they don't use ports
continue
resources = task["resources"]
if "host" in name:
assert "ports" in resources.keys(
), "Task {} should have port resources".format(name)
if "overlay" in name:
assert "ports" not in resources.keys(
), "Task {} should NOT have port resources".format(name)
networks.check_task_network("hello-overlay-0-server")
networks.check_task_network("hello-overlay-vip-0-server")
networks.check_task_network("hello-host-0-server",
expected_network_name=None)
networks.check_task_network("hello-host-vip-0-server",
expected_network_name=None)
endpoints_result, _, rc = shakedown.run_dcos_command(
"{pkg} endpoints".format(pkg=PACKAGE_NAME))
endpoints_result = json.loads(endpoints_result)
assert rc == 0, "Getting endpoints failed"
assert len(endpoints_result
) == 2, "Wrong number of endpoints got {} should be 2".format(
len(endpoints_result))
overlay_endpoints_result, _, rc = shakedown.run_dcos_command(
"{pkg} endpoints overlay-vip".format(pkg=PACKAGE_NAME))
assert rc == 0, "Getting overlay endpoints failed"
overlay_endpoints_result = json.loads(overlay_endpoints_result)
assert "address" in overlay_endpoints_result.keys(), "overlay endpoints missing 'address'"\
"{}".format(overlay_endpoints_result)
assert len(overlay_endpoints_result["address"]) == 1
assert overlay_endpoints_result["address"][0].startswith("9")
overlay_port = overlay_endpoints_result["address"][0].split(":")[-1]
assert overlay_port == "4044"
assert "dns" in overlay_endpoints_result.keys()
assert len(overlay_endpoints_result["dns"]) == 1
assert overlay_endpoints_result["dns"][0] == hosts.autoip_host(
PACKAGE_NAME, "hello-overlay-vip-0-server", 4044)
host_endpoints_result, _, rc = shakedown.run_dcos_command(
"{pkg} endpoints host-vip".format(pkg=PACKAGE_NAME))
assert rc == 0, "Getting host endpoints failed"
host_endpoints_result = json.loads(host_endpoints_result)
assert "address" in host_endpoints_result.keys(), "overlay endpoints missing 'address'"\
"{}".format(host_endpoints_result)
assert len(host_endpoints_result["address"]) == 1
assert host_endpoints_result["address"][0].startswith("10")
host_port = host_endpoints_result["address"][0].split(":")[-1]
assert host_port == "4044"
assert "dns" in host_endpoints_result.keys()
assert len(host_endpoints_result["dns"]) == 1
assert host_endpoints_result["dns"][0] == hosts.autoip_host(
PACKAGE_NAME, "hello-host-vip-0-server", 4044)
def _curl_query(
service_name: str,
method: str,
endpoint: str,
json_body: Optional[Dict[str, Any]] = None,
task: str = "master-0-node",
https: bool = False,
return_json: bool = True,
http_user: Optional[str] = DEFAULT_ELASTICSEARCH_USER,
http_password: Optional[str] = DEFAULT_ELASTICSEARCH_PASSWORD,
) -> Optional[Union[str, Dict[str, Any]]]:
protocol = "https" if https else "http"
if http_password:
if not http_user:
http_user = DEFAULT_ELASTICSEARCH_USER
log.info("Using default basic HTTP user: '******'", http_user)
credentials = "-u {}:{}".format(http_user, http_password)
else:
if http_user:
raise Exception(
"HTTP authentication won't work with just a user. Needs both user AND password"
)
credentials = ""
host = sdk_hosts.autoip_host(service_name, task,
_master_zero_http_port(service_name))
curl_cmd = "/opt/mesosphere/bin/curl -sS {} -X{} '{}://{}/{}'".format(
credentials, method, protocol, host, endpoint)
if json_body:
json_body_value = (json_body if isinstance(json_body, str) else
json.dumps(json_body))
curl_cmd += " -H 'Content-type: application/json' -d '{}'".format(
json_body_value)
task_name = "master-0-node"
exit_code, stdout, stderr = sdk_cmd.service_task_exec(
service_name, task_name, curl_cmd)
def build_errmsg(msg: str) -> str:
return "{}\nCommand:\n{}\nstdout:\n{}\nstderr:\n{}".format(
msg, curl_cmd, stdout, stderr)
if exit_code:
log.warning(
build_errmsg(
"Failed to run command on {}, retrying or giving up.".format(
task_name)))
return None
if not return_json:
return stdout
try:
result = json.loads(stdout)
assert isinstance(result, dict)
return result
except Exception:
log.warning(
build_errmsg(
"Failed to parse stdout as JSON, retrying or giving up."))
return None
def test_cassandra_migration():
backup_service_name = os.getenv('CASSANDRA_BACKUP_CLUSTER_NAME')
restore_service_name = os.getenv('CASSANDRA_RESTORE_CLUSTER_NAME')
backup_node_address = os.getenv('BACKUP_NODE_ADDRESS', config.DEFAULT_NODE_ADDRESS)
backup_node_port = os.getenv('BACKUP_NODE_PORT', config.DEFAULT_NODE_PORT)
backup_write_data_job = config.get_write_data_job(backup_node_address, backup_node_port)
backup_verify_data_job = config.get_verify_data_job(backup_node_address, backup_node_port)
backup_delete_data_job = config.get_delete_data_job(backup_node_address, backup_node_port)
backup_verify_deletion_job = config.get_verify_deletion_job(backup_node_address, backup_node_port)
plan_parameters = {
'S3_BUCKET_NAME': os.getenv(
'AWS_BUCKET_NAME', 'infinity-framework-test'
),
'AWS_ACCESS_KEY_ID': os.getenv('AWS_ACCESS_KEY_ID'),
'AWS_SECRET_ACCESS_KEY': os.getenv('AWS_SECRET_ACCESS_KEY'),
'AWS_REGION': os.getenv('AWS_REGION', 'us-west-2'),
'SNAPSHOT_NAME': str(uuid.uuid1()),
'CASSANDRA_KEYSPACES': '"testspace1 testspace2"',
}
backup_install_job_context = sdk_jobs.InstallJobContext(
[backup_write_data_job, backup_verify_data_job,
backup_delete_data_job, backup_verify_deletion_job])
backup_run_job_context = sdk_jobs.RunJobContext(
before_jobs=[backup_write_data_job, backup_verify_data_job],
after_jobs=[backup_delete_data_job, backup_verify_deletion_job])
# Install and run the write/delete data jobs against backup cluster,
# running dcos-cassandra-service
with backup_install_job_context, backup_run_job_context:
# Back this cluster up to S3
backup_parameters = {
'backup_name': plan_parameters['SNAPSHOT_NAME'],
's3_access_key': plan_parameters['AWS_ACCESS_KEY_ID'],
's3_secret_key': plan_parameters['AWS_SECRET_ACCESS_KEY'],
'external_location': 's3://{}'.format(plan_parameters['S3_BUCKET_NAME']),
}
sdk_cmd.service_request('PUT', backup_service_name, '/v1/backup/start', json=backup_parameters)
sdk_plan.wait_for_completed_deployment(backup_service_name)
# Restore data to second instance:
restore_node_address = os.getenv(
'RESTORE_NODE_ADDRESS', sdk_hosts.autoip_host('sdk-cassandra', 'node-0-server'))
restore_node_port = os.getenv('RESTORE_NODE_PORT', '9052')
restore_write_data_job = config.get_write_data_job(restore_node_address, restore_node_port)
restore_verify_data_job = config.get_verify_data_job(restore_node_address, restore_node_port)
restore_delete_data_job = config.get_delete_data_job(restore_node_address, restore_node_port)
restore_verify_deletion_job = config.get_verify_deletion_job(restore_node_address, restore_node_port)
restore_install_job_context = sdk_jobs.InstallJobContext(
[restore_write_data_job, restore_verify_data_job,
restore_delete_data_job, restore_verify_deletion_job]
)
restore_run_job_context = sdk_jobs.RunJobContext(
after_jobs=[restore_verify_data_job, restore_delete_data_job, restore_verify_deletion_job]
)
with restore_install_job_context, restore_run_job_context:
sdk_plan.start_plan(
restore_service_name, 'restore-s3', parameters=plan_parameters
)
sdk_plan.wait_for_completed_plan(restore_service_name, 'restore-s3')
def _curl_api(service_name, method, role="master"):
host = "http://" + sdk_hosts.autoip_host(service_name, "{}-0-node".format(role), _master_zero_http_port(service_name))
return ("curl -X{} -s -u elastic:changeme '" + host).format(method)
def test_overlay_network():
"""Verify that the current deploy plan matches the expected plan from the spec."""
deployment_plan = sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
log.info(sdk_plan.plan_string("deploy", deployment_plan))
# test that the tasks are all up, which tests the overlay DNS
framework_tasks = sdk_tasks.get_service_tasks(config.SERVICE_NAME)
expected_running_tasks = [
"overlay-vip-0-server",
"overlay-0-server",
"host-vip-0-server",
"host-0-server"
]
assert set(expected_running_tasks) == set([t.name for t in framework_tasks])
for task in framework_tasks:
name = task.name
if name.startswith("host-"):
assert "ports" in task.resources.keys(), "Task {} should have port resources".format(
name
)
sdk_networks.check_task_network(name, expected_network_name=None)
elif name.startswith("overlay-"):
assert (
"ports" not in task.resources.keys()
), "Task {} should NOT have port resources".format(
name
)
sdk_networks.check_task_network(name)
else:
assert False, "Unknown task {}".format(name)
endpoints_result = sdk_networks.get_endpoint_names(config.PACKAGE_NAME, config.SERVICE_NAME)
assert len(endpoints_result) == 2, "Expected 2 endpoints, got: {}".format(endpoints_result)
overlay_endpoints_result = sdk_networks.get_endpoint(
config.PACKAGE_NAME, config.SERVICE_NAME, "overlay-vip"
)
assert "address" in overlay_endpoints_result.keys(), (
"overlay endpoints missing 'address': {}".format(overlay_endpoints_result)
)
assert len(overlay_endpoints_result["address"]) == 1
assert overlay_endpoints_result["address"][0].startswith("9")
overlay_port = overlay_endpoints_result["address"][0].split(":")[-1]
assert overlay_port == "4044"
assert "dns" in overlay_endpoints_result.keys()
assert len(overlay_endpoints_result["dns"]) == 1
assert overlay_endpoints_result["dns"][0] == sdk_hosts.autoip_host(
config.SERVICE_NAME, "overlay-vip-0-server", 4044
)
host_endpoints_result = sdk_networks.get_endpoint(
config.PACKAGE_NAME, config.SERVICE_NAME, "host-vip"
)
assert "address" in host_endpoints_result.keys(), (
"overlay endpoints missing 'address'" "{}".format(host_endpoints_result)
)
assert len(host_endpoints_result["address"]) == 1
assert host_endpoints_result["address"][0].startswith("10")
host_port = host_endpoints_result["address"][0].split(":")[-1]
assert host_port == "4044"
assert "dns" in host_endpoints_result.keys()
assert len(host_endpoints_result["dns"]) == 1
assert host_endpoints_result["dns"][0] == sdk_hosts.autoip_host(
config.SERVICE_NAME, "host-vip-0-server", 4044
)
def test_overlay_network():
"""Verify that the current deploy plan matches the expected plan from the spec."""
deployment_plan = sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
log.info("deployment_plan: " + str(deployment_plan))
# test that the deployment plan is correct
assert(len(deployment_plan['phases']) == 5)
assert(deployment_plan['phases'][0]['name'] == 'hello-overlay-deploy')
assert(deployment_plan['phases'][1]['name'] == 'hello-overlay-vip-deploy')
assert(deployment_plan['phases'][2]['name'] == 'hello-host-vip-deploy')
assert(deployment_plan['phases'][3]['name'] == 'hello-host-deploy')
assert(deployment_plan["phases"][4]["name"] == "getter-deploy")
assert(len(deployment_plan['phases'][0]['steps']) == 1)
assert(len(deployment_plan["phases"][1]["steps"]) == 1)
assert(len(deployment_plan["phases"][2]["steps"]) == 1)
assert(len(deployment_plan["phases"][3]["steps"]) == 1)
assert(len(deployment_plan["phases"][4]["steps"]) == 1)
# Due to DNS resolution flakiness, some of the deployed tasks can fail. If so,
# we wait for them to redeploy, but if they don't fail we still want to proceed.
try:
sdk_plan.wait_for_in_progress_recovery(config.SERVICE_NAME, timeout_seconds=60)
sdk_plan.wait_for_completed_recovery(config.SERVICE_NAME, timeout_seconds=60)
except TimeoutExpired:
pass
# test that the tasks are all up, which tests the overlay DNS
framework_tasks = [task for task in shakedown.get_service_tasks(config.SERVICE_NAME, completed=False)]
framework_task_names = [t["name"] for t in framework_tasks]
for expected_task in EXPECTED_TASKS:
assert(expected_task in framework_task_names), "Missing {expected}".format(expected=expected_task)
for task in framework_tasks:
name = task["name"]
if "getter" in name: # don't check the "getter" tasks because they don't use ports
continue
resources = task["resources"]
if "host" in name:
assert "ports" in resources.keys(), "Task {} should have port resources".format(name)
if "overlay" in name:
assert "ports" not in resources.keys(), "Task {} should NOT have port resources".format(name)
sdk_networks.check_task_network("hello-overlay-0-server")
sdk_networks.check_task_network("hello-overlay-vip-0-server")
sdk_networks.check_task_network("hello-host-0-server", expected_network_name=None)
sdk_networks.check_task_network("hello-host-vip-0-server", expected_network_name=None)
endpoints_result = sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, 'endpoints', json=True)
assert len(endpoints_result) == 2, "Wrong number of endpoints got {} should be 2".format(len(endpoints_result))
overlay_endpoints_result = sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, 'endpoints overlay-vip', json=True)
assert "address" in overlay_endpoints_result.keys(), "overlay endpoints missing 'address'"\
"{}".format(overlay_endpoints_result)
assert len(overlay_endpoints_result["address"]) == 1
assert overlay_endpoints_result["address"][0].startswith("9")
overlay_port = overlay_endpoints_result["address"][0].split(":")[-1]
assert overlay_port == "4044"
assert "dns" in overlay_endpoints_result.keys()
assert len(overlay_endpoints_result["dns"]) == 1
assert overlay_endpoints_result["dns"][0] == sdk_hosts.autoip_host(config.SERVICE_NAME, "hello-overlay-vip-0-server", 4044)
host_endpoints_result = sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, 'endpoints host-vip', json=True)
assert "address" in host_endpoints_result.keys(), "overlay endpoints missing 'address'"\
"{}".format(host_endpoints_result)
assert len(host_endpoints_result["address"]) == 1
assert host_endpoints_result["address"][0].startswith("10")
host_port = host_endpoints_result["address"][0].split(":")[-1]
assert host_port == "4044"
assert "dns" in host_endpoints_result.keys()
assert len(host_endpoints_result["dns"]) == 1
assert host_endpoints_result["dns"][0] == sdk_hosts.autoip_host(config.SERVICE_NAME, "hello-host-vip-0-server", 4044)
import sdk_hosts
import sdk_jobs
import sdk_plan
import sdk_utils
PACKAGE_NAME = 'beta-cassandra'
SERVICE_NAME = 'cassandra'
FOLDERED_SERVICE_NAME = sdk_utils.get_foldered_name(SERVICE_NAME)
DEFAULT_TASK_COUNT = 3
DEFAULT_CASSANDRA_TIMEOUT = 600
DEFAULT_NODE_ADDRESS = os.getenv('CASSANDRA_NODE_ADDRESS', sdk_hosts.autoip_host(SERVICE_NAME, 'node-0-server'))
DEFAULT_NODE_PORT = os.getenv('CASSANDRA_NODE_PORT', '9042')
def _get_test_job(name, cmd, restart_policy='ON_FAILURE'):
return {
'description': '{} with restart policy {}'.format(name, restart_policy),
'id': 'test.cassandra.' + name,
'run': {
'cmd': cmd,
'docker': { 'image': 'cassandra:3.0.13' },
'cpus': 1,
'mem': 512,
'user': '******',
'restart': { 'policy': restart_policy }
}
def get_host(self):
return sdk_hosts.autoip_host(service_name="marathon", task_name=self.app_definition["id"])
log = logging.getLogger(__name__)
PACKAGE_NAME = "cassandra"
CASSANDRA_DOCKER_IMAGE = "cassandra:3.11.3"
SERVICE_NAME = os.environ.get("SOAK_SERVICE_NAME") or "cassandra"
DEFAULT_TASK_COUNT = 3
DEFAULT_CASSANDRA_TIMEOUT = 600
SECRET_VALUE = "password"
# Soak artifact scripts may override the service name to test
DEFAULT_NODE_ADDRESS = os.getenv(
"CASSANDRA_NODE_ADDRESS", sdk_hosts.autoip_host(SERVICE_NAME, "node-0-server")
)
DEFAULT_NODE_PORT = os.getenv("CASSANDRA_NODE_PORT", "9042")
def get_foldered_service_name() -> str:
return sdk_utils.get_foldered_name(SERVICE_NAME)
def get_foldered_node_address() -> str:
return sdk_hosts.autoip_host(get_foldered_service_name(), "node-0-server")
def _get_cqlsh_tls_rc_config(
node_address: str,
node_port: str,