def _retried_run_janitor(service_name):
auth_token = sdk_cmd.run_cli('config show core.dcos_acs_token', print_output=False).strip()
cmd_list = ["docker", "run", "mesosphere/janitor", "/janitor.py",
"-r", sdk_utils.get_role(service_name),
"-p", service_name + '-principal',
"-z", sdk_utils.get_zk_path(service_name),
"--auth_token={}".format(auth_token)]
sdk_cmd.master_ssh(" ".join(cmd_list))
def _retried_run_janitor(service_name):
cmd_list = [
"docker",
"run",
"mesosphere/janitor",
"/janitor.py",
"-r",
sdk_utils.get_role(service_name),
"-p",
service_name + "-principal",
"-z",
sdk_utils.get_zk_path(service_name),
"--auth_token={}".format(sdk_utils.dcos_token()),
]
rc, _, _ = sdk_cmd.master_ssh(" ".join(cmd_list))
assert rc == 0, "Janitor command failed"
def _verify_completed_uninstall(service_name):
state_summary = sdk_cmd.cluster_request('GET', '/mesos/state-summary').json()
# There should be no orphaned resources in the state summary (DCOS-30314)
orphaned_resources = 0
ignored_orphaned_resources = 0
service_role = sdk_utils.get_role(service_name)
for agent in state_summary['slaves']:
# resources should be grouped by role. check for any resources in our expected role:
matching_reserved_resources = agent['reserved_resources'].get(service_role)
if matching_reserved_resources:
if agent['hostname'] in _dead_agent_hosts:
# The test told us ahead of time to expect orphaned resources on this host.
log.info('Ignoring orphaned resources on agent {}/{}: {}'.format(
agent['id'], agent['hostname'], matching_reserved_resources))
ignored_orphaned_resources += len(matching_reserved_resources)
else:
log.error('Orphaned resources on agent {}/{}: {}'.format(
agent['id'], agent['hostname'], matching_reserved_resources))
orphaned_resources += len(matching_reserved_resources)
if orphaned_resources:
log.error('{} orphaned resources (plus {} ignored) after uninstall of {}'.format(
orphaned_resources, ignored_orphaned_resources, service_name))
log.error(state_summary)
raise Exception('Found {} orphaned resources (plus {} ignored) after uninstall of {}'.format(
orphaned_resources, ignored_orphaned_resources, service_name))
elif ignored_orphaned_resources:
log.info('Ignoring {} orphaned resources after uninstall of {}'.format(
ignored_orphaned_resources, service_name))
log.info(state_summary)
else:
log.info('No orphaned resources for role {} were found'.format(service_role))
# There should be no framework entry for this service in the state summary (DCOS-29474)
orphaned_frameworks = [fwk for fwk in state_summary['frameworks'] if fwk['name'] == service_name]
if orphaned_frameworks:
log.error('{} orphaned frameworks named {} after uninstall of {}: {}'.format(
len(orphaned_frameworks), service_name, service_name, orphaned_frameworks))
log.error(state_summary)
raise Exception('Found {} orphaned frameworks named {} after uninstall of {}: {}'.format(
len(orphaned_frameworks), service_name, service_name, orphaned_frameworks))
log.info('No orphaned frameworks for service {} were found'.format(service_name))
def _verify_completed_uninstall(service_name: str) -> None:
state_summary = sdk_cmd.cluster_request("GET",
"/mesos/state-summary").json()
# There should be no orphaned resources in the state summary (DCOS-30314)
orphaned_resources = 0
ignored_orphaned_resources = 0
service_role = sdk_utils.get_role(service_name)
for agent in state_summary["slaves"]:
# resources should be grouped by role. check for any resources in our expected role:
matching_reserved_resources = agent["reserved_resources"].get(
service_role)
if matching_reserved_resources:
global _dead_agent_hosts
if agent["hostname"] in _dead_agent_hosts:
# The test told us ahead of time to expect orphaned resources on this host.
log.info(
"Ignoring orphaned resources on agent {}/{}: {}".format(
agent["id"], agent["hostname"],
matching_reserved_resources))
ignored_orphaned_resources += len(matching_reserved_resources)
else:
log.error("Orphaned resources on agent {}/{}: {}".format(
agent["id"], agent["hostname"],
matching_reserved_resources))
orphaned_resources += len(matching_reserved_resources)
if orphaned_resources:
log.error(
"{} orphaned resources (plus {} ignored) after uninstall of {}".
format(orphaned_resources, ignored_orphaned_resources,
service_name))
log.error(state_summary)
raise Exception(
"Found {} orphaned resources (plus {} ignored) after uninstall of {}"
.format(orphaned_resources, ignored_orphaned_resources,
service_name))
elif ignored_orphaned_resources:
log.info("Ignoring {} orphaned resources after uninstall of {}".format(
ignored_orphaned_resources, service_name))
log.info(state_summary)
else:
log.info("No orphaned resources for role {} were found".format(
service_role))
# There should be no framework entry for this service in the state summary (DCOS-29474)
orphaned_frameworks = [
fwk for fwk in state_summary["frameworks"]
if fwk["name"] == service_name
]
if orphaned_frameworks:
log.error(
"{} orphaned frameworks named {} after uninstall of {}: {}".format(
len(orphaned_frameworks), service_name, service_name,
orphaned_frameworks))
log.error(state_summary)
raise Exception(
"Found {} orphaned frameworks named {} after uninstall of {}: {}".
format(len(orphaned_frameworks), service_name, service_name,
orphaned_frameworks))
log.info("No orphaned frameworks for service {} were found".format(
service_name))