def marathon_job_status(mstatus, client, job_config):
try:
app_id = job_config.format_marathon_app_dict()['id']
except NoDockerImageError:
error_msg = "Docker image is not in deployments.json."
mstatus['error_message'] = error_msg
return
mstatus['app_id'] = app_id
mstatus['slaves'] = list({task.slave['hostname'] for task in get_running_tasks_from_active_frameworks(app_id)})
mstatus['expected_instance_count'] = job_config.get_instances()
deploy_status = marathon_tools.get_marathon_app_deploy_status(client, app_id)
mstatus['deploy_status'] = marathon_tools.MarathonDeployStatus.tostring(deploy_status)
# by comparing running count with expected count, callers can figure
# out if the instance is in Healthy, Warning or Critical state.
if deploy_status == marathon_tools.MarathonDeployStatus.NotRunning:
mstatus['running_instance_count'] = 0
else:
mstatus['running_instance_count'] = client.get_app(app_id).tasks_running
if deploy_status == marathon_tools.MarathonDeployStatus.Delayed:
_, backoff_seconds = marathon_tools.get_app_queue_status(client, app_id)
mstatus['backoff_seconds'] = backoff_seconds
def marathon_job_status(mstatus, client, job_config, verbose):
try:
app_id = job_config.format_marathon_app_dict()['id']
except NoDockerImageError:
error_msg = "Docker image is not in deployments.json."
mstatus['error_message'] = error_msg
return
mstatus['app_id'] = app_id
if verbose is True:
mstatus['slaves'] = list({
task.slave['hostname']
for task in get_running_tasks_from_frameworks(app_id)
})
mstatus['expected_instance_count'] = job_config.get_instances()
deploy_status = marathon_tools.get_marathon_app_deploy_status(
client, app_id)
mstatus['deploy_status'] = marathon_tools.MarathonDeployStatus.tostring(
deploy_status)
# by comparing running count with expected count, callers can figure
# out if the instance is in Healthy, Warning or Critical state.
if deploy_status == marathon_tools.MarathonDeployStatus.NotRunning:
mstatus['running_instance_count'] = 0
else:
mstatus['running_instance_count'] = client.get_app(
app_id).tasks_running
if deploy_status == marathon_tools.MarathonDeployStatus.Delayed:
_, backoff_seconds = marathon_tools.get_app_queue_status(
client, app_id)
mstatus['backoff_seconds'] = backoff_seconds
def status_marathon_job(service, instance, app_id, normal_instance_count,
client):
status = marathon_tools.get_marathon_app_deploy_status(client, app_id)
app_queue = marathon_tools.get_app_queue(client, app_id)
unused_offers_summary = marathon_tools.summarize_unused_offers(app_queue)
if status == marathon_tools.MarathonDeployStatus.Delayed:
_, backoff_seconds = marathon_tools.get_app_queue_status_from_queue(
app_queue)
deploy_status_human = marathon_app_deploy_status_human(
status, backoff_seconds)
else:
deploy_status_human = marathon_app_deploy_status_human(status)
if status == marathon_tools.MarathonDeployStatus.NotRunning:
running_instances = 0
else:
running_instances = client.get_app(app_id).tasks_running
return status_marathon_job_human(
service,
instance,
deploy_status_human,
app_id,
running_instances,
normal_instance_count,
unused_offers_summary,
)
def status_marathon_job(service, instance, app_id, normal_instance_count, client):
status = marathon_tools.get_marathon_app_deploy_status(client, app_id)
if status == marathon_tools.MarathonDeployStatus.Delayed:
_, backoff_seconds = marathon_tools.get_app_queue_status(client, app_id)
deploy_status_human = marathon_app_deploy_status_human(status, backoff_seconds)
else:
deploy_status_human = marathon_app_deploy_status_human(status)
if status == marathon_tools.MarathonDeployStatus.NotRunning:
running_instances = 0
else:
running_instances = client.get_app(app_id).tasks_running
return status_marathon_job_human(service, instance, deploy_status_human, app_id,
running_instances, normal_instance_count)
def marathon_job_status(
mstatus: MutableMapping[str, Any],
client,
job_config,
verbose: bool,
) -> None:
try:
app_id = job_config.format_marathon_app_dict()['id']
except NoDockerImageError:
error_msg = "Docker image is not in deployments.json."
mstatus['error_message'] = error_msg
return
mstatus['app_id'] = app_id
if verbose is True:
mstatus['slaves'] = list(
{
a_sync.block(task.slave)['hostname']
for task in a_sync.block(get_running_tasks_from_frameworks,
app_id)
}, )
mstatus['expected_instance_count'] = job_config.get_instances()
try:
app = client.get_app(app_id)
except marathon.exceptions.NotFoundError:
mstatus[
'deploy_status'] = marathon_tools.MarathonDeployStatus.tostring(
marathon_tools.MarathonDeployStatus.NotRunning, )
mstatus['running_instance_count'] = 0
else:
deploy_status = marathon_tools.get_marathon_app_deploy_status(
client, app)
mstatus[
'deploy_status'] = marathon_tools.MarathonDeployStatus.tostring(
deploy_status)
# by comparing running count with expected count, callers can figure
# out if the instance is in Healthy, Warning or Critical state.
mstatus['running_instance_count'] = app.tasks_running
if deploy_status == marathon_tools.MarathonDeployStatus.Delayed:
_, backoff_seconds = marathon_tools.get_app_queue_status(
client, app_id)
mstatus['backoff_seconds'] = backoff_seconds
def marathon_job_status(mstatus, client, job_config):
try:
app_id = job_config.format_marathon_app_dict()['id']
except NoDockerImageError:
error_msg = "Docker image is not in deployments.json."
mstatus['error_message'] = error_msg
return
if marathon_tools.is_app_id_running(app_id, client):
app = client.get_app(app_id)
deploy_status, _ = get_marathon_app_deploy_status(app, app_id, client)
mstatus['deploy_status'] = MarathonDeployStatus.tostring(deploy_status)
# by comparing running count with expected count, callers can figure
# out if the instance is in Healthy, Warning or Critical state.
mstatus['running_instance_count'] = app.tasks_running
mstatus['expected_instance_count'] = job_config.get_instances()
else:
mstatus['deploy_status'] = 'Not Running'
def status_marathon_app(
marathon_client: marathon_tools.MarathonClient,
app: marathon_tools.MarathonApp,
service: str,
instance: str,
cluster: str,
soa_dir: str,
dashboards: Dict[marathon_tools.MarathonClient, str],
verbose: int,
) -> Tuple[int, int, str]:
"""Takes a given marathon app object and returns the details about start, times, hosts, etc"""
output = []
create_datetime = datetime_from_utc_to_local(isodate.parse_datetime(app.version))
output.append(get_marathon_dashboard(marathon_client, dashboards, app.id))
output.append(
" "
+ " ".join(
[
f"{app.tasks_running} running,",
f"{app.tasks_healthy} healthy,",
f"{app.tasks_staged} staged",
f"out of {app.instances}",
]
)
)
output.append(
" App created: {} ({})".format(
str(create_datetime), humanize.naturaltime(create_datetime)
)
)
deploy_status = marathon_tools.get_marathon_app_deploy_status(marathon_client, app)
app_queue = marathon_tools.get_app_queue(marathon_client, app.id)
unused_offers_summary = marathon_tools.summarize_unused_offers(app_queue)
if deploy_status == marathon_tools.MarathonDeployStatus.Delayed:
_, backoff_seconds = marathon_tools.get_app_queue_status_from_queue(app_queue)
deploy_status_human = marathon_app_deploy_status_human(
deploy_status, backoff_seconds
)
else:
deploy_status_human = marathon_app_deploy_status_human(deploy_status)
output.append(f" Status: {deploy_status_human}")
if unused_offers_summary is not None and len(unused_offers_summary) > 0:
output.append(" Possibly stalled for:")
output.append(
" ".join([f"{k}: {n} times" for k, n in unused_offers_summary.items()])
)
if verbose > 0:
output.append(" Tasks:")
rows = [
(
"Mesos Task ID",
"Host deployed to",
"Deployed at what localtime",
"Health",
)
]
for task in app.tasks:
local_deployed_datetime = datetime_from_utc_to_local(task.staged_at)
if task.host is not None:
hostname = "{}:{}".format(task.host.split(".")[0], task.ports[0])
else:
hostname = "Unknown"
if not task.health_check_results:
health_check_status = PaastaColors.grey("N/A")
elif marathon_tools.is_task_healthy(task):
health_check_status = PaastaColors.green("Healthy")
else:
health_check_status = PaastaColors.red("Unhealthy")
rows.append(
(
get_short_task_id(task.id),
hostname,
"{} ({})".format(
local_deployed_datetime.strftime("%Y-%m-%dT%H:%M"),
humanize.naturaltime(local_deployed_datetime),
),
health_check_status,
)
)
output.append("\n".join([" %s" % line for line in format_table(rows)]))
if len(app.tasks) == 0:
output.append(" No tasks associated with this marathon app")
return deploy_status, app.tasks_running, "\n".join(output)
def marathon_job_status(
service: str,
instance: str,
job_config: marathon_tools.MarathonServiceConfig,
marathon_apps_with_clients: List[Tuple[MarathonApp, MarathonClient]],
verbose: int,
) -> MutableMapping[str, Any]:
job_status_fields: MutableMapping[str, Any] = {
"app_statuses": [],
"app_count":
len(marathon_apps_with_clients),
"desired_state":
job_config.get_desired_state(),
"bounce_method":
job_config.get_bounce_method(),
"expected_instance_count":
job_config.get_instances(),
"active_shas":
list(get_active_shas_for_marathon_apps(marathon_apps_with_clients)),
}
try:
desired_app_id = job_config.format_marathon_app_dict()["id"]
except NoDockerImageError:
error_msg = "Docker image is not in deployments.json."
job_status_fields["error_message"] = error_msg
return job_status_fields
job_status_fields["desired_app_id"] = desired_app_id
deploy_status_for_desired_app = None
dashboard_links = get_marathon_dashboard_links(
settings.marathon_clients, settings.system_paasta_config)
tasks_running = 0
for app, marathon_client in marathon_apps_with_clients:
deploy_status = marathon_tools.get_marathon_app_deploy_status(
marathon_client, app)
app_status = marathon_app_status(
app,
marathon_client,
dashboard_links.get(marathon_client) if dashboard_links else None,
deploy_status,
list_tasks=verbose > 0,
)
job_status_fields["app_statuses"].append(app_status)
if app.id.lstrip("/") == desired_app_id.lstrip("/"):
deploy_status_for_desired_app = marathon_tools.MarathonDeployStatus.tostring(
deploy_status)
tasks_running += app.tasks_running
job_status_fields["deploy_status"] = (deploy_status_for_desired_app
or "Waiting for bounce")
job_status_fields["running_instance_count"] = tasks_running
if verbose > 0:
autoscaling_info = get_autoscaling_info(marathon_apps_with_clients,
job_config)
if autoscaling_info is not None:
autoscaling_info_dict = autoscaling_info._asdict()
for field in ("current_utilization", "target_instances"):
if autoscaling_info_dict[field] is None:
del autoscaling_info_dict[field]
job_status_fields["autoscaling_info"] = autoscaling_info_dict
return job_status_fields
