def get_verbose_status_of_marathon_app(marathon_client, app, service, instance,
cluster, soa_dir):
"""Takes a given marathon app object and returns the verbose details
about the tasks, times, hosts, etc"""
output = []
create_datetime = datetime_from_utc_to_local(
isodate.parse_datetime(app.version))
output.append(" Marathon app ID: %s" % PaastaColors.bold(app.id))
output.append(
" App created: %s (%s)" %
(str(create_datetime), humanize.naturaltime(create_datetime)))
autoscaling_info = get_autoscaling_info(marathon_client, service, instance,
cluster, soa_dir)
if autoscaling_info:
output.append(" Autoscaling Info:")
headers = [
field.replace("_", " ").capitalize()
for field in ServiceAutoscalingInfo._fields
]
table = [headers, autoscaling_info]
output.append('\n'.join(
[" %s" % line for line in format_table(table)]))
output.append(" Tasks:")
rows = [("Mesos Task ID", "Host deployed to", "Deployed at what localtime",
"Health")]
for task in app.tasks:
local_deployed_datetime = datetime_from_utc_to_local(task.staged_at)
if task.host is not None:
hostname = "%s:%s" % (task.host.split(".")[0], task.ports[0])
else:
hostname = "Unknown"
if not task.health_check_results:
health_check_status = PaastaColors.grey("N/A")
elif marathon_tools.is_task_healthy(task):
health_check_status = PaastaColors.green("Healthy")
else:
health_check_status = PaastaColors.red("Unhealthy")
rows.append((
get_short_task_id(task.id),
hostname,
'%s (%s)' % (
local_deployed_datetime.strftime("%Y-%m-%dT%H:%M"),
humanize.naturaltime(local_deployed_datetime),
),
health_check_status,
))
output.append('\n'.join([" %s" % line
for line in format_table(rows)]))
if len(app.tasks) == 0:
output.append(" No tasks associated with this marathon app")
return app.tasks, "\n".join(output)
def status_marathon_job_verbose(
service: str,
instance: str,
clients: marathon_tools.MarathonClients,
cluster: str,
soa_dir: str,
job_config: marathon_tools.MarathonServiceConfig,
dashboards: Dict[marathon_tools.MarathonClient, str],
) -> Tuple[List[MarathonTask], str]:
"""Returns detailed information about a marathon apps for a service
and instance. Does not make assumptions about what the *exact*
appid is, but instead does a fuzzy match on any marathon apps
that match the given service.instance"""
all_tasks: List[MarathonTask] = []
all_output: List[str] = []
# For verbose mode, we want to see *any* matching app. As it may
# not be the one that we think should be deployed. For example
# during a bounce we want to see the old and new ones.
marathon_apps_with_clients = marathon_tools.get_marathon_apps_with_clients(
clients=clients.get_all_clients_for_service(job_config),
embed_tasks=True,
)
autoscaling_info = get_autoscaling_info(clients, job_config)
if autoscaling_info:
all_output.append(" Autoscaling Info:")
headers = [
field.replace("_", " ").capitalize()
for field in ServiceAutoscalingInfo._fields
]
table = [headers, autoscaling_info]
all_output.append('\n'.join(
[" %s" % line for line in format_table(table)]))
for app, client in marathon_tools.get_matching_apps_with_clients(
service, instance, marathon_apps_with_clients):
tasks, output = get_verbose_status_of_marathon_app(
marathon_client=client,
app=app,
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
dashboards=dashboards,
)
all_tasks.extend(tasks)
all_output.append(output)
return all_tasks, "\n".join(all_output)
def status_marathon_job(
service: str,
instance: str,
cluster: str,
soa_dir: str,
dashboards: Dict[marathon_tools.MarathonClient, str],
normal_instance_count: int,
clients: marathon_tools.MarathonClients,
job_config: marathon_tools.MarathonServiceConfig,
desired_app_id: str,
verbose: int,
) -> Tuple[List[MarathonTask], str]:
marathon_apps_with_clients = marathon_tools.get_marathon_apps_with_clients(
clients=clients.get_all_clients_for_service(job_config),
embed_tasks=True,
service_name=service,
)
all_tasks = []
all_output = [
""
] # One entry that will be replaced with status_marathon_job_human output later.
running_instances = 0
if verbose > 0:
autoscaling_info = get_autoscaling_info(marathon_apps_with_clients, job_config)
if autoscaling_info:
all_output.append(" Autoscaling Info:")
headers = [
field.replace("_", " ").capitalize()
for field in ServiceAutoscalingInfo._fields
]
table = [headers, humanize_autoscaling_info(autoscaling_info)]
all_output.append(
"\n".join([" %s" % line for line in format_table(table)])
)
deploy_status_for_desired_app = "Waiting for bounce"
matching_apps_with_clients = marathon_tools.get_matching_apps_with_clients(
service, instance, marathon_apps_with_clients
)
for app, client in matching_apps_with_clients:
all_tasks.extend(app.tasks)
(
deploy_status_for_current_app,
running_instances_for_current_app,
out,
) = status_marathon_app(
marathon_client=client,
app=app,
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
dashboards=dashboards,
verbose=verbose,
)
if app.id.lstrip("/") == desired_app_id.lstrip("/"):
deploy_status_for_desired_app = marathon_tools.MarathonDeployStatus.tostring(
deploy_status_for_current_app
)
running_instances += running_instances_for_current_app
all_output.append(out)
all_output[0] = status_marathon_job_human(
service=service,
instance=instance,
deploy_status=deploy_status_for_desired_app,
desired_app_id=desired_app_id,
app_count=len(matching_apps_with_clients),
running_instances=running_instances,
normal_instance_count=normal_instance_count,
)
return all_tasks, "\n".join(all_output)
def marathon_job_status(
service: str,
instance: str,
job_config: marathon_tools.MarathonServiceConfig,
marathon_apps_with_clients: List[Tuple[MarathonApp, MarathonClient]],
verbose: int,
) -> MutableMapping[str, Any]:
job_status_fields: MutableMapping[str, Any] = {
"app_statuses": [],
"app_count":
len(marathon_apps_with_clients),
"desired_state":
job_config.get_desired_state(),
"bounce_method":
job_config.get_bounce_method(),
"expected_instance_count":
job_config.get_instances(),
"active_shas":
list(get_active_shas_for_marathon_apps(marathon_apps_with_clients)),
}
try:
desired_app_id = job_config.format_marathon_app_dict()["id"]
except NoDockerImageError:
error_msg = "Docker image is not in deployments.json."
job_status_fields["error_message"] = error_msg
return job_status_fields
job_status_fields["desired_app_id"] = desired_app_id
deploy_status_for_desired_app = None
dashboard_links = get_marathon_dashboard_links(
settings.marathon_clients, settings.system_paasta_config)
tasks_running = 0
for app, marathon_client in marathon_apps_with_clients:
deploy_status = marathon_tools.get_marathon_app_deploy_status(
marathon_client, app)
app_status = marathon_app_status(
app,
marathon_client,
dashboard_links.get(marathon_client) if dashboard_links else None,
deploy_status,
list_tasks=verbose > 0,
)
job_status_fields["app_statuses"].append(app_status)
if app.id.lstrip("/") == desired_app_id.lstrip("/"):
deploy_status_for_desired_app = marathon_tools.MarathonDeployStatus.tostring(
deploy_status)
tasks_running += app.tasks_running
job_status_fields["deploy_status"] = (deploy_status_for_desired_app
or "Waiting for bounce")
job_status_fields["running_instance_count"] = tasks_running
if verbose > 0:
autoscaling_info = get_autoscaling_info(marathon_apps_with_clients,
job_config)
if autoscaling_info is not None:
autoscaling_info_dict = autoscaling_info._asdict()
for field in ("current_utilization", "target_instances"):
if autoscaling_info_dict[field] is None:
del autoscaling_info_dict[field]
job_status_fields["autoscaling_info"] = autoscaling_info_dict
return job_status_fields
