def check_cache_refresh_fails_409conflict():
try:
cmd.run_cli('hello-world state refresh_cache')
except Exception as e:
if "failed: 409 Conflict" in e.args[0]:
return True
return False
def _remove_job_by_name(job_name):
try:
# --stop-current-job-runs ensures that fail-looping jobs (with restart.policy=ON_FAILURE) are consistently removed.
sdk_cmd.run_cli('job remove {} --stop-current-job-runs'.format(job_name), print_output=False)
except:
log.info('Failed to remove any existing job named {} (this is likely as expected): {}'.format(
job_name, traceback.format_exc()))
def add_universe_repos():
stub_urls = {}
log.info('Adding universe repos')
# prepare needed universe repositories
stub_universe_urls = os.environ.get('STUB_UNIVERSE_URL')
if not stub_universe_urls:
return stub_urls
log.info('Adding stub URLs: {}'.format(stub_universe_urls))
for url in stub_universe_urls.split():
print('url: {}'.format(url))
package_name = 'testpkg-'
package_name += ''.join(random.choice(string.ascii_lowercase +
string.digits) for _ in range(8))
stub_urls[package_name] = url
# clean up any duplicate repositories
current_universes = sdk_cmd.run_cli('package repo list --json')
for repo in json.loads(current_universes)['repositories']:
if repo['uri'] in stub_urls.values():
log.info('Removing duplicate stub URL: {}'.format(repo['uri']))
sdk_cmd.run_cli('package repo remove {}'.format(repo['name']))
# add the needed universe repositories
for name, url in stub_urls.items():
log.info('Adding stub URL: {}'.format(url))
sdk_cmd.run_cli('package repo add --index=0 {} {}'.format(name, url))
log.info('Finished adding universe repos')
return stub_urls
def run_job(job_dict, timeout_seconds=600, raise_on_failure=True):
job_name = job_dict['id']
sdk_cmd.run_cli('job run {}'.format(job_name))
def wait_for_run_id():
runs = json.loads(sdk_cmd.run_cli('job show runs {} --json'.format(job_name)))
if len(runs) > 0:
return runs[0]['id']
return ''
run_id = shakedown.wait_for(wait_for_run_id, noisy=True, timeout_seconds=timeout_seconds, ignore_exceptions=False)
def fun():
# catch errors from CLI: ensure that the only error raised is our own:
try:
runs = json.loads(sdk_cmd.run_cli(
'job history --show-failures --json {}'.format(job_name), print_output=False))
except:
log.info(traceback.format_exc())
return False
successful_ids = [r['id'] for r in runs['history']['successfulFinishedRuns']]
failed_ids = [r['id'] for r in runs['history']['failedFinishedRuns']]
log.info('Job {} run history (waiting for successful {}): successful={} failed={}'.format(
job_name, run_id, successful_ids, failed_ids))
# note: if a job has restart.policy=ON_FAILURE, it won't show up in failed_ids if it fails
if raise_on_failure and run_id in failed_ids:
raise Exception('Job {} with id {} has failed, exiting early'.format(job_name, run_id))
return run_id in successful_ids
shakedown.wait_for(fun, noisy=True, timeout_seconds=timeout_seconds, ignore_exceptions=False)
return run_id
def _dump_diagnostics_bundle(item: pytest.Item) -> None:
"""Creates and downloads a DC/OS diagnostics bundle, and saves it to the artifact path for this test."""
rc, _, _ = sdk_cmd.run_cli("node diagnostics create all")
if rc:
log.error("Diagnostics bundle creation failed.")
return
@retrying.retry(
wait_fixed=5000,
stop_max_delay=10 * 60 * 1000,
retry_on_result=lambda result: result is None,
)
def wait_for_bundle_file() -> Optional[str]:
rc, stdout, stderr = sdk_cmd.run_cli("node diagnostics --status --json")
if rc:
return None
# e.g. { "some-ip": { stuff we want } }
status = next(iter(json.loads(stdout).values()))
if status["job_progress_percentage"] != 100:
return None
# e.g. "/var/lib/dcos/dcos-diagnostics/diag-bundles/bundle-2018-01-11-1515698691.zip"
return str(os.path.basename(status["last_bundle_dir"]))
bundle_filename = str(wait_for_bundle_file())
if bundle_filename:
sdk_cmd.run_cli(
"node diagnostics download {} --location={}".format(
bundle_filename, _setup_artifact_path(item, bundle_filename)
)
)
else:
log.error("Diagnostics bundle didnt finish in time, giving up.")
def add_stub_universe_urls(stub_universe_urls: list) -> dict:
stub_urls = {}
if not stub_universe_urls:
return stub_urls
log.info('Adding stub URLs: {}'.format(stub_universe_urls))
for idx, url in enumerate(stub_universe_urls):
log.info('URL {}: {}'.format(idx, repr(url)))
package_name = 'testpkg-'
package_name += ''.join(random.choice(string.ascii_lowercase +
string.digits) for _ in range(8))
stub_urls[package_name] = url
# clean up any duplicate repositories
current_universes = sdk_cmd.run_cli('package repo list --json')
for repo in json.loads(current_universes)['repositories']:
if repo['uri'] in stub_urls.values():
log.info('Removing duplicate stub URL: {}'.format(repo['uri']))
sdk_cmd.run_cli('package repo remove {}'.format(repo['name']))
# add the needed universe repositories
for name, url in stub_urls.items():
log.info('Adding stub repo {} URL: {}'.format(name, url))
rc, stdout, stderr = sdk_cmd.run_raw_cli('package repo add --index=0 {} {}'.format(name, url))
if rc != 0 or stderr:
raise Exception(
'Failed to add stub repo {} ({}): stdout=[{}], stderr=[{}]'.format(
name, url, stdout, stderr))
log.info('Finished adding universe repos')
return stub_urls
def soak_upgrade_downgrade(
package_name,
service_name,
running_task_count,
additional_options={},
timeout_seconds=25*60,
wait_for_deployment=True):
sdk_cmd.run_cli("package install --cli {} --yes".format(package_name))
version = 'stub-universe'
print('Upgrading to test version: {} {}'.format(package_name, version))
_upgrade_or_downgrade(
package_name,
version,
service_name,
running_task_count,
additional_options,
timeout_seconds,
wait_for_deployment)
# Default Universe is at --index=0
version = _get_pkg_version(package_name)
print('Downgrading to Universe version: {} {}'.format(package_name, version))
_upgrade_or_downgrade(
package_name,
version,
service_name,
running_task_count,
additional_options,
timeout_seconds,
wait_for_deployment)
def get_dcos_credentials() -> Tuple[str, dict]:
dcosurl = sdk_cmd.run_cli('config show core.dcos_url', print_output=False)
token = sdk_cmd.run_cli('config show core.dcos_acs_token', print_output=False)
headers = {
'Content-Type': 'application/json',
'Authorization': 'token={}'.format(token.strip()),
}
return dcosurl.strip(), headers
def create_secret(secret_value: str, secret_path: str) -> None:
install_enterprise_cli()
delete_secret(secret=secret_path)
sdk_cmd.run_cli(
'security secrets create --value="{account}" "{secret}"'.format(
account=secret_value, secret=secret_path
)
)
def setup_service_account(
service_name: str,
service_account_secret: Optional[str] = None,
) -> Dict[str, Any]:
"""
Setup the service account for TLS. If the account or secret of the specified
name already exists, these are deleted.
"""
if sdk_utils.is_open_dcos():
log.error("The setup of a service account requires DC/OS EE. service_name=%s", service_name)
raise Exception("The setup of a service account requires DC/OS EE")
secret = service_name if service_account_secret is None else service_account_secret
service_account = "{}-service-account".format(service_name.replace("/", ""))
service_account_info = sdk_security.setup_security(
service_name,
linux_user="******",
service_account=service_account,
service_account_secret=secret,
)
log.info("Adding permissions required for TLS.")
if sdk_utils.dcos_version_less_than("1.11"):
sdk_cmd.run_cli("security org groups add_user superusers {sa}".format(sa=service_account))
else:
acls = [
{"rid": "dcos:secrets:default:/{}/*".format(service_name.strip("/")), "action": "full"},
{
"rid": "dcos:secrets:list:default:/{}".format(service_name.strip("/")),
"action": "read",
},
{"rid": "dcos:adminrouter:ops:ca:rw", "action": "full"},
{"rid": "dcos:adminrouter:ops:ca:ro", "action": "full"},
]
for acl in acls:
cmd_list = [
"security",
"org",
"users",
"grant",
"--description",
'"Allow provisioning TLS certificates"',
service_account,
acl["rid"],
acl["action"],
]
sdk_cmd.run_cli(" ".join(cmd_list))
return service_account_info
def teardown(args: dict):
log.info("Tearing down KDC")
sdk_cmd.run_cli(" ".join(["marathon", "app", "remove", "kdc"]))
sdk_security.install_enterprise_cli()
if args.binary_secret:
sdk_security.delete_secret(args.secret_name)
else:
sdk_security.delete_secret("__dcos_base64__{}".format(args.secret_name))
log.info("KDC cluster successfully torn down")
def service_account():
"""
Creates service account and yields the name.
"""
name = config.SERVICE_NAME
sdk_security.create_service_account(
service_account_name=name, service_account_secret=name)
# TODO(mh): Fine grained permissions needs to be addressed in DCOS-16475
sdk_cmd.run_cli(
"security org groups add_user superusers {name}".format(name=name))
yield name
sdk_security.delete_service_account(
service_account_name=name, service_account_secret=name)
def install_job(job_dict, tmp_dir=None):
job_name = job_dict['id']
if not tmp_dir:
tmp_dir = tempfile.mkdtemp(prefix='sdk-test')
out_filename = os.path.join(tmp_dir, '{}.json'.format(job_name))
job_str = json.dumps(job_dict)
log.info('Writing job file for {} to: {}\n{}'.format(job_name, out_filename, job_str))
with open(out_filename, 'w') as f:
f.write(job_str)
_remove_job_by_name(job_name)
sdk_cmd.run_cli('job add {}'.format(out_filename))
def configure_package(configure_security):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
# Create service account
sdk_security.create_service_account(
service_account_name=config.SERVICE_NAME, service_account_secret=config.SERVICE_NAME
)
sdk_cmd.run_cli(
"security org groups add_user superusers {name}".format(name=config.SERVICE_NAME)
)
sdk_install.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
6,
additional_options={
"service": {
"yaml": "tls",
"service_account": config.SERVICE_NAME,
"service_account_secret": config.SERVICE_NAME,
# Legacy values
"principal": config.SERVICE_NAME,
"secret_name": config.SERVICE_NAME,
},
"tls": {"discovery_task_prefix": DISCOVERY_TASK_PREFIX},
},
)
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
yield # let the test session execute
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
sdk_security.delete_service_account(
service_account_name=config.SERVICE_NAME, service_account_secret=config.SERVICE_NAME
)
# Make sure that all the TLS artifacts were removed from the secrets store.
_, output, _ = sdk_cmd.run_cli("security secrets list {name}".format(name=config.SERVICE_NAME))
artifact_suffixes = [
"certificate",
"private-key",
"root-ca-certificate",
"keystore",
"truststore",
]
for suffix in artifact_suffixes:
assert suffix not in output
def test_state_properties_get():
# 'suppressed' could be missing if the scheduler recently started, loop for a bit just in case:
def check_for_nonempty_properties():
stdout = cmd.run_cli('hello-world state properties')
return len(json.loads(stdout)) > 0
spin.time_wait_noisy(lambda: check_for_nonempty_properties(), timeout_seconds=30.)
stdout = cmd.run_cli('hello-world state properties')
jsonobj = json.loads(stdout)
assert len(jsonobj) == 1
assert jsonobj[0] == "suppressed"
stdout = cmd.run_cli('hello-world state property suppressed')
assert stdout == "true\n"
def create_service_account(service_account_name: str, service_account_secret: str) -> None:
log.info('Creating service account for account={account} secret={secret}'.format(
account=service_account_name,
secret=service_account_secret))
log.info('Install cli necessary for security')
sdk_cmd.run_cli('package install dcos-enterprise-cli --yes')
log.info('Remove any existing service account and/or secret')
delete_service_account(service_account_name, service_account_secret)
log.info('Create keypair')
sdk_cmd.run_cli('security org service-accounts keypair private-key.pem public-key.pem')
log.info('Create service account')
sdk_cmd.run_cli(
'security org service-accounts create -p public-key.pem -d "My service account" "{account}"'.format(
account=service_account_name))
log.info('Create secret')
sdk_cmd.run_cli(
'security secrets create-sa-secret --strict private-key.pem "{account}" "{secret}"'.format(
account=service_account_name, secret=service_account_secret))
log.info('Service account created for account={account} secret={secret}'.format(
account=service_account_name,
secret=service_account_secret))
def configure_package(configure_security):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
sdk_cmd.run_cli("package install --cli dcos-enterprise-cli --yes")
try_delete_secrets("{}/".format(config.SERVICE_NAME))
try_delete_secrets("{}/somePath/".format(config.SERVICE_NAME))
try_delete_secrets()
yield # let the test session execute
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
try_delete_secrets("{}/".format(config.SERVICE_NAME))
try_delete_secrets("{}/somePath/".format(config.SERVICE_NAME))
try_delete_secrets()
def delete_service_account(service_account_name: str, service_account_secret: str) -> None:
"""
Deletes service account with private key that belongs to the service account.
"""
# ignore any failures:
sdk_cmd.run_cli("security org service-accounts delete {name}".format(name=service_account_name))
# Files generated by service-accounts keypair command should get removed
for keypair_file in ["private-key.pem", "public-key.pem"]:
try:
os.unlink(keypair_file)
except OSError:
pass
delete_secret(secret=service_account_secret)
def _bundle_directory_name(self) -> str:
_, cluster_name, _ = sdk_cmd.run_cli("config show cluster.name", print_output=False)
return "{}_{}_{}".format(
cluster_name,
sdk_utils.get_deslashed_service_name(self.service_name),
directory_date_string(),
)
def add_dcos_files_to_registry(tmpdir_factory: TempdirFactory) -> None:
# Use DCOS_FILES_PATH if its set to a valid path OR use pytest's tmpdir.
dcos_files_path = os.environ.get("DCOS_FILES_PATH", "")
valid_path_set = os.path.isdir(dcos_files_path)
if valid_path_set and not os.access(dcos_files_path, os.W_OK):
log.warning("{} is not writable.".format(dcos_files_path))
valid_path_set = False
if not valid_path_set:
dcos_files_path = str(tmpdir_factory.mktemp(sdk_utils.random_string()))
stub_universe_urls = sdk_repository.get_repos()
log.info(
"Using {} to build .dcos files (if not exists) from {}".format(
dcos_files_path, stub_universe_urls
)
)
dcos_files_list = build_dcos_files_from_stubs(
stub_universe_urls, dcos_files_path, tmpdir_factory
)
log.info("Bundled .dcos files : {}".format(dcos_files_list))
@retrying.retry(stop_max_delay=5 * 60 * 1000, wait_fixed=5 * 1000)
def wait_for_added_registry(name: str, version: str) -> None:
code, stdout, stderr = sdk_cmd.run_cli(
"registry describe --package-name={} --package-version={} --json".format(name, version),
print_output=False,
)
assert code == 0 and json.loads(stdout).get("status") == "Added"
for file_path, name, version in dcos_files_list:
rc, out, err = sdk_cmd.run_cli("registry add --dcos-file={} --json".format(file_path))
assert rc == 0
assert len(json.loads(out)["packages"]) > 0, "No packages were added"
wait_for_added_registry(name, version)
def grant_perms_for_registry_account(service_uid: str) -> None:
# Grant only required permissions to registry
perms = "dcos:adminrouter:ops:ca:rw"
rc, _, _ = sdk_cmd.run_cli(
" ".join(["security", "org", "users", "grant", service_uid, perms, "full"])
)
assert rc == 0, "Required perms [{}] could not be obtained for {}".format(perms, service_uid)
def build_dcos_file_from_universe_definition(
package: Dict, dcos_files_path: str, tmpdir_factory: TempdirFactory,
) -> Tuple[str, str, str]:
"""
Build the .dcos file if its not already present in the given directory.
Returns a Tuple containing (path of .dcos file, name, and version)
"""
# TODO Ideally we should `migrate` and then `build`.
name = package["name"]
version = package["version"]
target = os.path.join(dcos_files_path, "{}-{}.dcos".format(name, version))
if os.path.isfile(target):
log.info("Skipping build, using cached file : {}".format(target))
else:
del package["releaseVersion"]
del package["selected"]
package_json_file = tmpdir_factory.mktemp(sdk_utils.random_string()).join(
sdk_utils.random_string()
)
package_json_file.write(json.dumps(package))
rc, _, _ = sdk_cmd.run_cli(
" ".join(
[
"registry",
"build",
"--build-definition-file={}".format(str(package_json_file)),
"--output-directory={}".format(dcos_files_path),
"--json",
]
)
)
assert rc == 0
assert os.path.isfile(target), "No valid .dcos file is built"
return target, name, version
def test_pods_restart_graceful_shutdown():
options = {
"world": {
"kill_grace_period": 30
}
}
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
sdk_install.install(config.PACKAGE_NAME, config.SERVICE_NAME, config.DEFAULT_TASK_COUNT,
additional_options=options)
world_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'world-0')
jsonobj = sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, 'pod restart world-0', json=True)
assert len(jsonobj) == 2
assert jsonobj['pod'] == 'world-0'
assert len(jsonobj['tasks']) == 1
assert jsonobj['tasks'][0] == 'world-0-server'
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'world-0', world_ids)
config.check_running()
# ensure the SIGTERM was sent via the "all clean" message in the world
# service's signal trap/handler, BUT not the shell command, indicated
# by "echo".
stdout = sdk_cmd.run_cli(
"task log --completed --lines=1000 {}".format(world_ids[0]))
clean_msg = None
for s in stdout.split('\n'):
if s.find('echo') < 0 and s.find('all clean') >= 0:
clean_msg = s
assert clean_msg is not None
def attached_dcos_cluster() -> (int, Any):
rc, stdout, stderr = sdk_cmd.run_cli("cluster list --attached --json", print_output=False)
try:
attached_clusters = json.loads(stdout)
except json.JSONDecodeError as e:
return (1, "Error decoding JSON while getting attached DC/OS cluster: {}".format(e))
if rc == 0:
if len(attached_clusters) == 0:
return (1, "No cluster is attached")
if len(attached_clusters) > 1:
return (
1,
"More than one attached clusters. This is an invalid DC/OS CLI state\n{}".format(
stdout
),
)
else:
if "No cluster is attached" in stderr:
return (rc, stderr)
else:
return (False, "Unexpected error\nstdout: '{}'\nstderr: '{}'".format(stdout, stderr))
# Having more than one attached cluster is an invalid DC/OS CLI state that is handled above.
# In normal conditions `attached_clusters` is an unary array.
attached_cluster = attached_clusters[0]
return (rc, attached_cluster)
def test_pods_restart_graceful_shutdown():
install_options_helper(30)
world_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, "world-0")
rc, stdout, _ = sdk_cmd.svc_cli(
config.PACKAGE_NAME, config.SERVICE_NAME, "pod restart world-0"
)
assert rc == 0, "Pod restart failed"
jsonobj = json.loads(stdout)
assert len(jsonobj) == 2
assert jsonobj["pod"] == "world-0"
assert len(jsonobj["tasks"]) == 1
assert jsonobj["tasks"][0] == "world-0-server"
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, "world-0", world_ids)
check_healthy()
# ensure the SIGTERM was sent via the "all clean" message in the world
# service's signal trap/handler, BUT not the shell command, indicated
# by "echo".
_, stdout, _ = sdk_cmd.run_cli("task log --completed --lines=1000 {}".format(world_ids[0]))
clean_msg = None
for s in stdout.split("\n"):
if s.find("echo") < 0 and s.find("all clean") >= 0:
clean_msg = s
assert clean_msg is not None
def decommission_agent(agent_id: str) -> None:
assert sdk_utils.dcos_version_at_least("1.11"),\
"node decommission is supported in DC/OS 1.11 and above only"
rc, _, _ = sdk_cmd.run_cli(
"node decommission {}".format(agent_id)
)
assert rc == 0
def create_tls_artifacts(cn: str, marathon_task: str) -> str:
pub_path = "{}_pub.crt".format(cn)
priv_path = "{}_priv.key".format(cn)
log.info("Generating certificate. cn={}, task={}".format(cn, marathon_task))
output = sdk_cmd.marathon_task_exec(
marathon_task,
'openssl req -nodes -newkey rsa:2048 -keyout {} -out request.csr '
'-subj "/C=US/ST=CA/L=SF/O=Mesosphere/OU=Mesosphere/CN={}"'.format(priv_path, cn))
assert output[0] is 0
rc, raw_csr, _ = sdk_cmd.marathon_task_exec(marathon_task, 'cat request.csr')
assert rc is 0
request = {
"certificate_request": raw_csr
}
token = sdk_cmd.run_cli("config show core.dcos_acs_token")
output = sdk_cmd.marathon_task_exec(
marathon_task,
"curl --insecure -L -X POST "
"-H 'Authorization: token={}' "
"leader.mesos/ca/api/v2/sign "
"-d '{}'".format(token, json.dumps(request)))
assert output[0] is 0
# Write the public cert to the client
certificate = json.loads(output[1])["result"]["certificate"]
output = sdk_cmd.marathon_task_exec(marathon_task, "bash -c \"echo '{}' > {}\"".format(certificate, pub_path))
assert output[0] is 0
_create_keystore_truststore(cn, marathon_task)
return "CN={},OU=Mesosphere,O=Mesosphere,L=SF,ST=CA,C=US".format(cn)
def get_task_host(task_name):
_, out, _ = sdk_cmd.run_cli("task {} --json".format(task_name))
tasks_json = json.loads(out)
matching_tasks = list(filter(lambda t: t["name"] == task_name, tasks_json))
assert len(matching_tasks) == 1, "Duplicate tasks found with same name : [{}]".format(tasks_json)
task_info = matching_tasks.pop()
host = None
for label in task_info["labels"]:
if label["key"] == "offer_hostname":
host = label["value"]
break
if host is None:
raise Exception("offer_hostname label is not present!: {}".format(task_info))
# Validation: Check that label matches summary returned by CLI
for task in sdk_tasks.get_summary():
if task.name == task_name:
if task.host == host:
# OK!
return host
else:
# CLI's hostname doesn't match the TaskInfo labels. Bug!
raise Exception(
"offer_hostname label [{}] doesn't match CLI output [{}]\nTask:\n{}".format(
host, task.host, task_info
)
)
# Unable to find desired task in CLI!
raise Exception("Unable to find task named {} in CLI".format(task_name))
def _get_universe_url():
repositories = json.loads(sdk_cmd.run_cli('package repo list --json'))['repositories']
for repo in repositories:
if repo['name'] == 'Universe':
log.info("Found Universe URL: {}".format(repo['uri']))
return repo['uri']
assert False, "Unable to find 'Universe' in list of repos: {}".format(repositories)
def get_task_host(task_name):
out = sdk_cmd.run_cli('task {} --json'.format(task_name))
task_info = json.loads(out)[0]
host = None
for label in task_info['labels']:
if label['key'] == 'offer_hostname':
host = label['value']
break
if host is None:
raise Exception("offer_hostname label is not present!: {}".format(task_info))
# Validation: Check that label matches summary returned by CLI
for task in sdk_tasks.get_summary():
if task.name == task_name:
if task.host == host:
# OK!
return host
else:
# CLI's hostname doesn't match the TaskInfo labels. Bug!
raise Exception("offer_hostname label {} doesn't match CLI output!\nTask:\n{}".format(task_info))
# Unable to find desired task in CLI!
raise Exception("Unable to find task named {} in CLI".format(task_name))
def _retried_uninstall_package_and_wait(package_name, service_name):
if sdk_marathon.app_exists(service_name):
log.info("Uninstalling package {} with service name {}".format(
package_name, service_name))
sdk_cmd.run_cli("package uninstall {} --app-id={} --yes".format(
package_name, service_name),
check=True)
# Wait on the app no longer being listed in Marathon, at which point it is uninstalled.
# At the same time, log the deploy plan state as we wait for the app to finish uninstalling.
@retrying.retry(
stop_max_delay=TIMEOUT_SECONDS * 1000,
wait_fixed=5000,
retry_on_result=lambda result: not result,
)
def wait_for_removal_log_deploy_plan():
if not sdk_marathon.app_exists(service_name):
return True
# App still exists, print the deploy plan. Best effort: It is expected for the scheduler
# to become unavailable once uninstall completes.
try:
log.info(
sdk_plan.plan_string(
"deploy",
sdk_plan.get_plan_once(service_name, "deploy")))
except Exception:
pass # best effort attempt at logging plan content
return False
log.info("Waiting for {} to be removed".format(service_name))
wait_for_removal_log_deploy_plan()
else:
log.info(
'Skipping uninstall of package {}/service {}: App named "{}" doesn\'t exist'
.format(package_name, service_name, service_name))
def fun():
stdout = cmd.run_cli(
'task exec {} /bin/bash -c "JAVA_HOME=$(ls -d jre*/) apache-cassandra-*/bin/nodetool -p 7199 status"'
.format(task_id))
up_ips = []
for line in stdout.split('\n'):
words = list(filter(None, line.split()))
if len(words) < 2:
continue
if not 'UN' == words[0]:
continue
up_ips.append(words[1])
utils.out('UN nodes (want {} entries without {}): {}'.format(
DEFAULT_TASK_COUNT, pod_host, up_ips))
return len(up_ips) == DEFAULT_TASK_COUNT and not pod_host in up_ips
def test_node_replace_replaces_node():
tasks = cmd.run_cli('task')
node_ip = [t for t in tasks.split('\n')
if t.startswith('node-2-server')].pop().split()[1]
# Update the placement constraints so the new node doesn't end up on the
# same host
config = marathon.get_config(PACKAGE_NAME)
config['env']['PLACEMENT_CONSTRAINT'] = 'hostname:UNLIKE:{}'.format(
node_ip)
marathon.update_app(PACKAGE_NAME, config)
plan.wait_for_completed_deployment(PACKAGE_NAME)
# start replace and wait for it to finish
cmd.run_cli('cassandra pods replace node-2')
plan.wait_for_completed_recovery(PACKAGE_NAME)
# Install replace verification job with correct node IP templated
# (the job checks for that IP's absence in the peers list and also verifies
# that the expected number of peers is present, meaning that the node was
# replaced from Cassandra's perspective)
with JobContext([VERIFY_REPLACE_JOB], NODE_IP=node_ip):
spin.time_wait_noisy(lambda: try_job(VERIFY_REPLACE_JOB))
def get_marathon_app(service_name: str) -> (int, Any):
rc, stdout, stderr = sdk_cmd.run_cli(
"marathon app show {}".format(service_name), print_output=False)
if rc != 0:
if "does not exist" in stderr:
return (rc, "Service {} does not exist".format(service_name))
else:
return (rc, "Unexpected error\nstdout: '{}'\nstderr: '{}'".format(
stdout, stderr))
try:
return (rc, json.loads(stdout))
except json.JSONDecodeError as e:
return (1, "Error decoding JSON: {}".format(e))
def test_pod_replace():
world_ids = sdk_tasks.get_task_ids(PACKAGE_NAME, 'world-0')
# get current agent id (TODO: uncomment if/when agent is guaranteed to change in a replace operation):
#stdout = sdk_cmd.run_cli('hello-world pod info world-0')
#old_agent = json.loads(stdout)[0]['info']['slaveId']['value']
jsonobj = json.loads(sdk_cmd.run_cli('hello-world pod replace world-0'))
assert len(jsonobj) == 2
assert jsonobj['pod'] == 'world-0'
assert len(jsonobj['tasks']) == 1
assert jsonobj['tasks'][0] == 'world-0-server'
sdk_tasks.check_tasks_updated(PACKAGE_NAME, 'world-0', world_ids)
check_running()
def _get_kdc_task(task_name: str) -> dict:
"""
:return (dict): The task object of the KDC app with desired properties to be retrieved by other methods.
"""
log.info("Getting KDC task")
raw_tasks = sdk_cmd.run_cli("task --json", print_output=False)
if raw_tasks:
tasks = json.loads(raw_tasks)
for task in tasks:
if task["name"] == task_name:
return task
raise RuntimeError(
"Expecting marathon KDC task but no such task found. Running tasks: {tasks}"
.format(tasks=raw_tasks))
def __create_and_upload_secret(self):
"""
This method base64 encodes the keytab file and creates a secret with this encoded content so the
tasks can fetch it.
"""
log.info("Creating and uploading the keytab file to the secret store")
try:
base64_encode_cmd = "base64 -w 0 {source} > {destination}".format(
source=os.path.join(self.temp_working_dir.name,
self.keytab_file_name),
destination=os.path.join(self.temp_working_dir.name,
self.base64_encoded_keytab_file_name))
run(base64_encode_cmd, shell=True)
except Exception as e:
raise Exception(
"Failed to base64-encode the keytab file: {}".format(repr(e)))
self.keytab_secret_path = "{}_keytab".format(DCOS_BASE64_PREFIX)
# TODO: check if a keytab secret of same name already exists
create_secret_cmd = "security secrets create {keytab_secret_path} --value-file {encoded_keytab_path}".format(
keytab_secret_path=self.keytab_secret_path,
encoded_keytab_path=os.path.join(
self.temp_working_dir.name,
self.base64_encoded_keytab_file_name))
try:
sdk_cmd.run_cli(create_secret_cmd)
except RuntimeError as e:
raise RuntimeError(
"Failed to create secret for the base64-encoded keytab file: {}"
.format(repr(e)))
log.info(
"Successfully uploaded a base64-encoded keytab file to the secret store"
)
def _get_kdc_task_inner(task_name: str) -> dict:
log.info("Getting KDC task")
_, raw_tasks, _ = sdk_cmd.run_cli("task --json", print_output=False)
if raw_tasks:
tasks = json.loads(raw_tasks)
for task in tasks:
assert isinstance(task, dict)
if task["name"] == task_name:
return task
raise RuntimeError(
"Expecting marathon KDC task but no such task found. Running tasks: {tasks}".format(
tasks=raw_tasks
)
)
def test_secrets_basic():
# 1) create Secrets
# 2) install examples/secrets.yml
# 3) if secret file is not created, tasks will fail
# 4) wait till deployment finishes
# 5) do replace operation
# 6) ensure all tasks are running
# 7) delete Secrets
install.uninstall(PACKAGE_NAME)
create_secrets("{}/".format(PACKAGE_NAME))
install.install(PACKAGE_NAME,
NUM_HELLO + NUM_WORLD,
additional_options=secret_options)
# default is serial strategy, hello deploys first
# launch will fail if secrets are not available or not accessible
plan.wait_for_completed_deployment(PACKAGE_NAME)
hello_tasks_0 = tasks.get_task_ids(PACKAGE_NAME, "hello-0")
world_tasks_0 = tasks.get_task_ids(PACKAGE_NAME, "word-0")
# ensure that secrets work after replace
cmd.run_cli('hello-world pods replace hello-0')
cmd.run_cli('hello-world pods replace world-0')
tasks.check_tasks_updated(PACKAGE_NAME, "hello-0", hello_tasks_0)
tasks.check_tasks_updated(PACKAGE_NAME, 'world-0', world_tasks_0)
# tasks will fail if secret files are not created by mesos module
tasks.check_running(PACKAGE_NAME, NUM_HELLO + NUM_WORLD)
# clean up and delete secrets
delete_secrets("{}/".format(PACKAGE_NAME))
def test_shutdown_host_test():
service_ip = shakedown.get_service_ips(PACKAGE_NAME).pop()
sdk_utils.out('marathon ip = {}'.format(service_ip))
node_ip = 0
for pod_id in range(0, DEFAULT_TASK_COUNT):
node_ip = get_pod_host(pod_id)
if node_ip != service_ip:
break
if node_ip is None:
assert Fail, 'could not find a node to shutdown'
old_agent = get_pod_agent(pod_id)
sdk_utils.out('pod id = {}, node_ip = {}, agent = {}'.format(
pod_id, node_ip, old_agent))
task_ids = tasks.get_task_ids(PACKAGE_NAME, 'node-{}'.format(pod_id))
# instead of partition/reconnect, we shutdown host permanently
status, stdout = shakedown.run_command_on_agent(node_ip,
'sudo shutdown -h +1')
sdk_utils.out('shutdown agent {}: [{}] {}'.format(node_ip, status, stdout))
assert status is True
time.sleep(100)
cmd.run_cli('cassandra pods replace node-{}'.format(pod_id))
tasks.check_tasks_updated(PACKAGE_NAME, 'node', task_ids)
#double check all tasks are running
tasks.check_running(PACKAGE_NAME, DEFAULT_TASK_COUNT)
new_agent = get_pod_agent(pod_id)
assert old_agent != new_agent
def check_broker(id: int):
rc, stdout, _ = sdk_cmd.run_cli(
"task log kafka-{}-broker --lines 1000".format(id),
print_output=False)
if rc or not stdout:
raise Exception("No task logs for kafka-{}-broker".format(id))
last_log_index = stdout.rfind(broker_log_line[id])
success_index = stdout.rfind(
"zookeeper state changed (SyncConnected) (org.I0Itec.zkclient.ZkClient)"
)
assert last_log_index > -1 and last_log_index < success_index, "{}:{} STDOUT: {}".format(
last_log_index, success_index, stdout)
def upgrade_downgrade(package_name, running_task_count):
install.uninstall(package_name)
test_version = get_pkg_version(package_name)
print('Found test version: {}'.format(test_version))
repositories = json.loads(
cmd.run_cli('package repo list --json'))['repositories']
print("Repositories: " + str(repositories))
universe_url = "fail"
for repo in repositories:
if repo['name'] == 'Universe':
universe_url = repo['uri']
break
assert "fail" != universe_url
print("Universe URL: " + universe_url)
# Move the Universe repo to the top of the repo list
shakedown.remove_package_repo('Universe')
add_repo('Universe', universe_url, test_version, 0, package_name)
universe_version = get_pkg_version(package_name)
print('Found Universe version: {}'.format(universe_version))
print('Installing Universe version')
install.install(package_name, running_task_count, check_suppression=False)
# Move the Universe repo to the bottom of the repo list
shakedown.remove_package_repo('Universe')
add_last_repo('Universe', universe_url, universe_version, package_name)
print('Upgrading to test version')
upgrade_or_downgrade(package_name, running_task_count)
# Move the Universe repo to the top of the repo list
shakedown.remove_package_repo('Universe')
add_repo('Universe', universe_url, test_version, 0, package_name)
print('Downgrading to master version')
upgrade_or_downgrade(package_name, running_task_count)
# Move the Universe repo to the bottom of the repo list
shakedown.remove_package_repo('Universe')
add_last_repo('Universe', universe_url, universe_version, package_name)
print('Upgrading to test version')
upgrade_or_downgrade(package_name, running_task_count)
def test_pods_list():
stdout = cmd.run_cli('hello-world pods list')
jsonobj = json.loads(stdout)
assert len(jsonobj) == configured_task_count()
# expect: X instances of 'hello-#' followed by Y instances of 'world-#',
# in alphanumerical order
first_world = -1
for i in range(len(jsonobj)):
entry = jsonobj[i]
if first_world < 0:
if entry.startswith('world-'):
first_world = i
if first_world == -1:
assert jsonobj[i] == 'hello-{}'.format(i)
else:
assert jsonobj[i] == 'world-{}'.format(i - first_world)
def _get_host_name(host_id: str) -> str:
"""
Fetches the host name for the host running the KDC app.
:param host_id (str): The ID of the host, used to look up the appropriate node.
:return (str): Name of the host running the KDC app.
"""
log.info("Getting hostname")
raw_nodes = sdk_cmd.run_cli("node --json")
if raw_nodes:
nodes = json.loads(raw_nodes)
for node in nodes:
if "id" in node and node["id"] == host_id:
log.info("Host name is %s", node["hostname"])
return node["hostname"]
raise RuntimeError("Failed to get name of host running the KDC app: {nodes}")
def test_state_refresh_disable_cache():
'''Disables caching via a scheduler envvar'''
check_running(FOLDERED_SERVICE_NAME)
task_ids = tasks.get_task_ids(FOLDERED_SERVICE_NAME, '')
# caching enabled by default:
stdout = cmd.run_cli('hello-world --name={} state refresh_cache'.format(
FOLDERED_SERVICE_NAME))
assert "Received cmd: refresh" in stdout
config = marathon.get_config(FOLDERED_SERVICE_NAME)
config['env']['DISABLE_STATE_CACHE'] = 'any-text-here'
marathon.update_app(FOLDERED_SERVICE_NAME, config)
tasks.check_tasks_not_updated(FOLDERED_SERVICE_NAME, '', task_ids)
check_running(FOLDERED_SERVICE_NAME)
# caching disabled, refresh_cache should fail with a 409 error (eventually, once scheduler is up):
def check_cache_refresh_fails_409conflict():
try:
cmd.run_cli('hello-world --name={} state refresh_cache'.format(
FOLDERED_SERVICE_NAME))
except Exception as e:
if "failed: 409 Conflict" in e.args[0]:
return True
return False
shakedown.wait_for(lambda: check_cache_refresh_fails_409conflict(),
timeout_seconds=120.)
config = marathon.get_config(FOLDERED_SERVICE_NAME)
del config['env']['DISABLE_STATE_CACHE']
marathon.update_app(FOLDERED_SERVICE_NAME, config)
tasks.check_tasks_not_updated(FOLDERED_SERVICE_NAME, '', task_ids)
check_running(FOLDERED_SERVICE_NAME)
shakedown.deployment_wait(
) # ensure marathon thinks the deployment is complete too
# caching reenabled, refresh_cache should succeed (eventually, once scheduler is up):
def check_cache_refresh():
return cmd.run_cli('hello-world --name={} state refresh_cache'.format(
FOLDERED_SERVICE_NAME))
stdout = shakedown.wait_for(lambda: check_cache_refresh(),
timeout_seconds=120.)
assert "Received cmd: refresh" in stdout
def install_enterprise_cli(force=False):
""" Install the enterprise CLI if required """
if not force:
_, stdout, _ = sdk_cmd.run_cli("security --version", print_output=False)
if stdout:
log.info("DC/OS enterprise version %s CLI already installed", stdout.strip())
@retrying.retry(
stop_max_attempt_number=3,
wait_fixed=2000,
retry_on_exception=lambda e: isinstance(e, Exception),
)
def _install_impl():
sdk_cmd.run_cli("package install --yes --cli dcos-enterprise-cli", check=True)
_install_impl()
def current_cluster_name() -> (bool, str):
rc, stdout, stderr = sdk_cmd.run_cli("config show cluster.name",
print_output=False)
if rc != 0:
if "Property 'cluster.name' doesn't exist" in stderr:
return (
False,
"No cluster is set up. Please run `dcos cluster setup`\nstdout: '{}'\nstderr: '{}'"
.format(stdout, stderr),
)
else:
return (False,
"Unexpected error\nstdout: '{}'\nstderr: '{}'".format(
stdout, stderr))
return (True, stdout)
def get_task_files_for_id(task_id: str) -> dict:
try:
ls_lines = sdk_cmd.run_cli('task ls --long --all {}'.format(task_id)).split('\n')
ret = {}
for line in ls_lines:
match = task_ls_pattern.match(line)
if not match:
log.warning('Unable to parse line: {}'.format(line))
continue
# match.group(1): "4096 ", match.group(2): "Jul 21 22:07", match.group(3): "jre1.8.0_144 "
filename = match.group(3).strip()
# build timestamp for use in output filename: 'Jul 21 22:07' => '0721_2207'
timestamp = time.strftime('%m%d_%H%M', time.strptime(match.group(2), '%b %d %H:%M'))
ret[filename] = timestamp
return ret
except:
log.exception('Failed to get list of files for task: {}'.format(task_id))
return {}
def fun():
# catch errors from CLI: ensure that the only error raised is our own:
try:
runs = json.loads(sdk_cmd.run_cli(
'job history --show-failures --json {}'.format(job_name), print_output=False))
except:
sdk_utils.out(traceback.format_exc())
return False
successful_ids = [r['id'] for r in runs['history']['successfulFinishedRuns']]
failed_ids = [r['id'] for r in runs['history']['failedFinishedRuns']]
sdk_utils.out('Job {} run history (waiting for successful {}): successful={} failed={}'.format(
job_name, run_id, successful_ids, failed_ids))
# note: if a job has restart.policy=ON_FAILURE, it won't show up in failed_ids if it fails
if raise_on_failure and run_id in failed_ids:
raise Exception('Job {} with id {} has failed, exiting early'.format(job_name, run_id))
return run_id in successful_ids
def get_summary(with_completed=False):
'''Returns a summary of task information as returned by the DC/OS CLI.
This may be used instead of invoking 'dcos task [--all]' directly.
Returns a list of Task objects.
'''
# Note: We COULD use --json, but there appears to be some fancy handling done in the CLI for the
# non-json version, particularly around the running user. Just grab the non-"--json" version of things.
task_lines = sdk_cmd.run_cli('task --all' if with_completed else 'task', print_output=False).split('\n')
output = []
for task_line in task_lines[1:]: # First line is the header line
task = Task.parse(task_line)
if task is not None:
output.append(task)
log.info('Task summary (with_completed={}):\n- {}'.format(
with_completed, '\n- '.join([str(e) for e in output])))
return output
def run_janitor(service_name, role, service_account, znode):
if role is None:
role = sdk_utils.get_deslashed_service_name(service_name) + '-role'
if service_account is None:
service_account = service_name + '-principal'
if znode is None:
znode = sdk_utils.get_zk_path(service_name)
auth_token = sdk_cmd.run_cli('config show core.dcos_acs_token',
print_output=False).strip()
cmd_list = [
"docker", "run", "mesosphere/janitor", "/janitor.py", "-r", role, "-p",
service_account, "-z", znode, "--auth_token={}".format(auth_token)
]
cmd = " ".join(cmd_list)
sdk_cmd.master_ssh(cmd)
def test_state_refresh_disable_cache():
'''Disables caching via a scheduler envvar'''
check_running()
task_ids = tasks.get_task_ids(PACKAGE_NAME, '')
# caching enabled by default:
stdout = cmd.run_cli('hello-world state refresh_cache')
assert "Received cmd: refresh" in stdout
config = marathon.get_config(PACKAGE_NAME)
cpus = float(config['env']['HELLO_CPUS'])
config['env']['DISABLE_STATE_CACHE'] = 'any-text-here'
cmd.request('put', marathon.api_url('apps/' + PACKAGE_NAME), json=config)
tasks.check_tasks_not_updated(PACKAGE_NAME, '', task_ids)
check_running()
# caching disabled, refresh_cache should fail with a 409 error (eventually, once scheduler is up):
def check_cache_refresh_fails_409conflict():
try:
cmd.run_cli('hello-world state refresh_cache')
except Exception as e:
if "failed: 409 Conflict" in e.args[0]:
return True
return False
spin.time_wait_noisy(lambda: check_cache_refresh_fails_409conflict(),
timeout_seconds=120.)
config = marathon.get_config(PACKAGE_NAME)
cpus = float(config['env']['HELLO_CPUS'])
del config['env']['DISABLE_STATE_CACHE']
cmd.request('put', marathon.api_url('apps/' + PACKAGE_NAME), json=config)
tasks.check_tasks_not_updated(PACKAGE_NAME, '', task_ids)
check_running()
# caching reenabled, refresh_cache should succeed (eventually, once scheduler is up):
def check_cache_refresh():
return cmd.run_cli('hello-world state refresh_cache')
stdout = spin.time_wait_return(lambda: check_cache_refresh(),
timeout_seconds=120.)
assert "Received cmd: refresh" in stdout
def create_tls_artifacts(cn: str, task: str) -> str:
pub_path = "{}_pub.crt".format(cn)
priv_path = "{}_priv.key".format(cn)
log.info("Generating certificate. cn={}, task={}".format(cn, task))
output = sdk_tasks.task_exec(
task,
'openssl req -nodes -newkey rsa:2048 -keyout {} -out request.csr \
-subj "/C=US/ST=CA/L=SF/O=Mesosphere/OU=Mesosphere/CN={}"'.format(
priv_path, cn))
log.info(output)
assert output[0] is 0
raw_csr = sdk_tasks.task_exec(task, 'cat request.csr')
assert raw_csr[0] is 0
request = {
"certificate_request":
raw_csr[1] # The actual content is second in the array
}
token = sdk_cmd.run_cli("config show core.dcos_acs_token")
cmd = "curl -X POST \
-H 'Authorization: token={}' \
leader.mesos/ca/api/v2/sign \
-d '{}'".format(token, json.dumps(request))
output = sdk_tasks.task_exec(
task, "curl -X POST \
-H 'Authorization: token={}' \
leader.mesos/ca/api/v2/sign \
-d '{}'".format(token, json.dumps(request)))
log.info(output)
assert output[0] is 0
# Write the public cert to the client
certificate = json.loads(output[1])["result"]["certificate"]
output = sdk_tasks.task_exec(
task, "bash -c \"echo '{}' > {}\"".format(certificate, pub_path))
log.info(output)
assert output[0] is 0
create_keystore_truststore(cn, task)
return "CN={},OU=Mesosphere,O=Mesosphere,L=SF,ST=CA,C=US".format(cn)
def install_app_from_file(app_name: str, app_def_path: str) -> (bool, str):
"""
Installs a marathon app using the path to an app definition.
Args:
app_def_path: Path to app definition
Returns:
(bool, str) tuple: Boolean indicates success of install attempt. String indicates
error message if install attempt failed.
"""
output = sdk_cmd.run_cli("{cmd} {file_path}".format(
cmd="marathon app add ", file_path=app_def_path))
if "Created deployment" not in output:
return 1, output
log.info("Waiting for app to be running...")
shakedown.wait_for_task("marathon", app_name)
return 0, ""
def _master_zero_http_port(service_name):
dns = json.loads(
sdk_cmd.run_cli('{} --name={} endpoints master'.format(
PACKAGE_NAME, service_name),
print_output=False))['dns']
# array will initially look something like this in CCM, with some 9300 ports and some lower ones [
# "master-0-node.elastic.[...]:9300",
# "master-0-node.elastic.[...]:1025",
# "master-1-node.elastic.[...]:9300",
# "master-1-node.elastic.[...]:1025",
# "master-2-node.elastic.[...]:9300",
# "master-2-node.elastic.[...]:1025"
# ]
# sort will bubble up "master-0-node.elastic.[...]:1025", the HTTP server host:port
dns.sort()
port = dns[0].split(':')[-1]
sdk_utils.out("Extracted {} as port for {}".format(port, dns[0]))
return port
def add_stub_universe_urls(stub_universe_urls: list) -> dict:
stub_urls = {}
if not stub_universe_urls:
return stub_urls
# clean up any duplicate repositories
_, current_universes, _ = sdk_cmd.run_cli("package repo list --json")
for repo in json.loads(current_universes)["repositories"]:
if repo["uri"] in stub_universe_urls:
log.info("Removing duplicate stub URL: {}".format(repo["uri"]))
assert remove_repo(repo["name"])
# add the needed universe repositories
log.info("Adding stub URLs: {}".format(stub_universe_urls))
for url in stub_universe_urls:
assert add_repo("testpkg-{}".format(sdk_utils.random_string()), url, 0)
return stub_urls
def run_job(job_dict, timeout_seconds=600, raise_on_failure=True):
job_name = job_dict['id']
# start job run, get run ID to be polled against:
run_id = json.loads(sdk_cmd.run_cli(
'job run {} --json'.format(job_name)))['id']
# wait for run to succeed, throw if run fails:
@retrying.retry(wait_fixed=1000,
stop_max_delay=timeout_seconds * 1000,
retry_on_result=lambda res: not res,
retry_on_exception=lambda ex: False)
def wait():
# catch errors from CLI: ensure that the only error raised is our own:
try:
successful_runs = json.loads(
sdk_cmd.run_cli('job history --json {}'.format(job_name),
print_output=False))
failed_runs = json.loads(
sdk_cmd.run_cli(
'job history --failures --json {}'.format(job_name),
print_output=False))
except:
log.info(traceback.format_exc())
return False
successful_ids = [r['id'] for r in successful_runs]
failed_ids = [r['id'] for r in failed_runs]
log.info(
'Job {} run history (waiting for successful {}): successful={} failed={}'
.format(job_name, run_id, successful_ids, failed_ids))
# note: if a job has restart.policy=ON_FAILURE, it won't show up in failed_ids if it fails
if raise_on_failure and run_id in failed_ids:
raise Exception(
'Job {} with id {} has failed, exiting early'.format(
job_name, run_id))
return run_id in successful_ids
wait()
return run_id
def hello_world_service(service_account):
sdk_install.install(
config.PACKAGE_NAME,
1,
service_name=service_account,
additional_options={
"service": {
"spec_file": "examples/tls.yml",
"service_account": service_account,
"service_account_secret": service_account,
# Legacy values
"principal": service_account,
"secret_name": service_account,
},
"tls": {
"discovery_task_prefix": DISCOVERY_TASK_PREFIX,
},
}
)
sdk_plan.wait_for_completed_deployment(config.PACKAGE_NAME)
# Wait for service health check to pass
shakedown.service_healthy(config.PACKAGE_NAME)
# TODO(mh): Add proper wait for health check
time.sleep(15)
yield service_account
sdk_install.uninstall(config.PACKAGE_NAME)
# Make sure that all the TLS artifacts were removed from the secrets store.
output = sdk_cmd.run_cli('security secrets list {name}'.format(
name=config.PACKAGE_NAME))
artifact_suffixes = [
'certificate', 'private-key', 'root-ca-certificate',
'keystore', 'truststore'
]
for suffix in artifact_suffixes:
assert suffix not in output
def test_upgrade_downgrade():
test_repo_name, test_repo_url = get_test_repo_info()
test_version = get_pkg_version()
print('Found test version: {}'.format(test_version))
repositories = json.loads(
cmd.run_cli('package repo list --json'))['repositories']
print("Repositories: " + str(repositories))
universe_url = "fail"
for repo in repositories:
if repo['name'] == 'Universe':
universe_url = repo['uri']
break
assert "fail" != universe_url
print("Universe URL: " + universe_url)
shakedown.remove_package_repo('Universe')
add_repo('Universe', universe_url, test_version, 0)
universe_version = get_pkg_version()
print('Found Universe version: {}'.format(universe_version))
print('Installing Universe version')
install.install(PACKAGE_NAME, DEFAULT_TASK_COUNT)
shakedown.remove_package_repo('Universe')
add_last_repo('Universe', universe_url, universe_version)
print('Upgrading to test version')
marathon.destroy_app(PACKAGE_NAME)
install.install(PACKAGE_NAME, DEFAULT_TASK_COUNT)
shakedown.remove_package_repo('Universe')
add_repo('Universe', universe_url, test_version, 0)
print('Downgrading to master version')
marathon.destroy_app(PACKAGE_NAME)
install.install(PACKAGE_NAME, DEFAULT_TASK_COUNT)
shakedown.remove_package_repo('Universe')
add_last_repo('Universe', universe_url, universe_version)
def wait_for_toxic_sidecar():
"""
Since the sidecar task fails too quickly, we check for the contents of
the file generated in hello-container-path/toxic-output instead
Note that we only check the output of hello-0.
In DC/OS prior to version 1.10, task exec does not run the command in the MESOS_SANDBOX directory and this
causes the check of the file contents to fail. Here we simply rely on the existence of the file.
"""
if sdk_utils.dcos_version_less_than("1.10"):
cmd = "task ls hello-0-server hello-container-path/toxic-output"
expected_output = ""
else:
cmd = "task exec hello-0-server cat hello-container-path/toxic-output"
expected_output = "I'm addicted to you / Don't you know that you're toxic?"
output = sdk_cmd.run_cli(cmd).strip()
logging.info("Checking for toxic output returned: %s", output)
return output == expected_output
