def marathon_job_status(mstatus, client, job_config): try: app_id = job_config.format_marathon_app_dict()['id'] except NoDockerImageError: error_msg = "Docker image is not in deployments.json." mstatus['error_message'] = error_msg return mstatus['app_id'] = app_id mstatus['slaves'] = list({task.slave['hostname'] for task in get_running_tasks_from_active_frameworks(app_id)}) mstatus['expected_instance_count'] = job_config.get_instances() deploy_status = marathon_tools.get_marathon_app_deploy_status(client, app_id) mstatus['deploy_status'] = marathon_tools.MarathonDeployStatus.tostring(deploy_status) # by comparing running count with expected count, callers can figure # out if the instance is in Healthy, Warning or Critical state. if deploy_status == marathon_tools.MarathonDeployStatus.NotRunning: mstatus['running_instance_count'] = 0 else: mstatus['running_instance_count'] = client.get_app(app_id).tasks_running if deploy_status == marathon_tools.MarathonDeployStatus.Delayed: _, backoff_seconds = marathon_tools.get_app_queue_status(client, app_id) mstatus['backoff_seconds'] = backoff_seconds
def marathon_job_status(mstatus, client, job_config, verbose): try: app_id = job_config.format_marathon_app_dict()['id'] except NoDockerImageError: error_msg = "Docker image is not in deployments.json." mstatus['error_message'] = error_msg return mstatus['app_id'] = app_id if verbose is True: mstatus['slaves'] = list({ task.slave['hostname'] for task in get_running_tasks_from_frameworks(app_id) }) mstatus['expected_instance_count'] = job_config.get_instances() deploy_status = marathon_tools.get_marathon_app_deploy_status( client, app_id) mstatus['deploy_status'] = marathon_tools.MarathonDeployStatus.tostring( deploy_status) # by comparing running count with expected count, callers can figure # out if the instance is in Healthy, Warning or Critical state. if deploy_status == marathon_tools.MarathonDeployStatus.NotRunning: mstatus['running_instance_count'] = 0 else: mstatus['running_instance_count'] = client.get_app( app_id).tasks_running if deploy_status == marathon_tools.MarathonDeployStatus.Delayed: _, backoff_seconds = marathon_tools.get_app_queue_status( client, app_id) mstatus['backoff_seconds'] = backoff_seconds
def status_marathon_job(service, instance, app_id, normal_instance_count, client): status = marathon_tools.get_marathon_app_deploy_status(client, app_id) app_queue = marathon_tools.get_app_queue(client, app_id) unused_offers_summary = marathon_tools.summarize_unused_offers(app_queue) if status == marathon_tools.MarathonDeployStatus.Delayed: _, backoff_seconds = marathon_tools.get_app_queue_status_from_queue( app_queue) deploy_status_human = marathon_app_deploy_status_human( status, backoff_seconds) else: deploy_status_human = marathon_app_deploy_status_human(status) if status == marathon_tools.MarathonDeployStatus.NotRunning: running_instances = 0 else: running_instances = client.get_app(app_id).tasks_running return status_marathon_job_human( service, instance, deploy_status_human, app_id, running_instances, normal_instance_count, unused_offers_summary, )
def status_marathon_job(service, instance, app_id, normal_instance_count, client): status = marathon_tools.get_marathon_app_deploy_status(client, app_id) if status == marathon_tools.MarathonDeployStatus.Delayed: _, backoff_seconds = marathon_tools.get_app_queue_status(client, app_id) deploy_status_human = marathon_app_deploy_status_human(status, backoff_seconds) else: deploy_status_human = marathon_app_deploy_status_human(status) if status == marathon_tools.MarathonDeployStatus.NotRunning: running_instances = 0 else: running_instances = client.get_app(app_id).tasks_running return status_marathon_job_human(service, instance, deploy_status_human, app_id, running_instances, normal_instance_count)
def marathon_job_status( mstatus: MutableMapping[str, Any], client, job_config, verbose: bool, ) -> None: try: app_id = job_config.format_marathon_app_dict()['id'] except NoDockerImageError: error_msg = "Docker image is not in deployments.json." mstatus['error_message'] = error_msg return mstatus['app_id'] = app_id if verbose is True: mstatus['slaves'] = list( { a_sync.block(task.slave)['hostname'] for task in a_sync.block(get_running_tasks_from_frameworks, app_id) }, ) mstatus['expected_instance_count'] = job_config.get_instances() try: app = client.get_app(app_id) except marathon.exceptions.NotFoundError: mstatus[ 'deploy_status'] = marathon_tools.MarathonDeployStatus.tostring( marathon_tools.MarathonDeployStatus.NotRunning, ) mstatus['running_instance_count'] = 0 else: deploy_status = marathon_tools.get_marathon_app_deploy_status( client, app) mstatus[ 'deploy_status'] = marathon_tools.MarathonDeployStatus.tostring( deploy_status) # by comparing running count with expected count, callers can figure # out if the instance is in Healthy, Warning or Critical state. mstatus['running_instance_count'] = app.tasks_running if deploy_status == marathon_tools.MarathonDeployStatus.Delayed: _, backoff_seconds = marathon_tools.get_app_queue_status( client, app_id) mstatus['backoff_seconds'] = backoff_seconds
def marathon_job_status(mstatus, client, job_config): try: app_id = job_config.format_marathon_app_dict()['id'] except NoDockerImageError: error_msg = "Docker image is not in deployments.json." mstatus['error_message'] = error_msg return if marathon_tools.is_app_id_running(app_id, client): app = client.get_app(app_id) deploy_status, _ = get_marathon_app_deploy_status(app, app_id, client) mstatus['deploy_status'] = MarathonDeployStatus.tostring(deploy_status) # by comparing running count with expected count, callers can figure # out if the instance is in Healthy, Warning or Critical state. mstatus['running_instance_count'] = app.tasks_running mstatus['expected_instance_count'] = job_config.get_instances() else: mstatus['deploy_status'] = 'Not Running'
def status_marathon_app( marathon_client: marathon_tools.MarathonClient, app: marathon_tools.MarathonApp, service: str, instance: str, cluster: str, soa_dir: str, dashboards: Dict[marathon_tools.MarathonClient, str], verbose: int, ) -> Tuple[int, int, str]: """Takes a given marathon app object and returns the details about start, times, hosts, etc""" output = [] create_datetime = datetime_from_utc_to_local(isodate.parse_datetime(app.version)) output.append(get_marathon_dashboard(marathon_client, dashboards, app.id)) output.append( " " + " ".join( [ f"{app.tasks_running} running,", f"{app.tasks_healthy} healthy,", f"{app.tasks_staged} staged", f"out of {app.instances}", ] ) ) output.append( " App created: {} ({})".format( str(create_datetime), humanize.naturaltime(create_datetime) ) ) deploy_status = marathon_tools.get_marathon_app_deploy_status(marathon_client, app) app_queue = marathon_tools.get_app_queue(marathon_client, app.id) unused_offers_summary = marathon_tools.summarize_unused_offers(app_queue) if deploy_status == marathon_tools.MarathonDeployStatus.Delayed: _, backoff_seconds = marathon_tools.get_app_queue_status_from_queue(app_queue) deploy_status_human = marathon_app_deploy_status_human( deploy_status, backoff_seconds ) else: deploy_status_human = marathon_app_deploy_status_human(deploy_status) output.append(f" Status: {deploy_status_human}") if unused_offers_summary is not None and len(unused_offers_summary) > 0: output.append(" Possibly stalled for:") output.append( " ".join([f"{k}: {n} times" for k, n in unused_offers_summary.items()]) ) if verbose > 0: output.append(" Tasks:") rows = [ ( "Mesos Task ID", "Host deployed to", "Deployed at what localtime", "Health", ) ] for task in app.tasks: local_deployed_datetime = datetime_from_utc_to_local(task.staged_at) if task.host is not None: hostname = "{}:{}".format(task.host.split(".")[0], task.ports[0]) else: hostname = "Unknown" if not task.health_check_results: health_check_status = PaastaColors.grey("N/A") elif marathon_tools.is_task_healthy(task): health_check_status = PaastaColors.green("Healthy") else: health_check_status = PaastaColors.red("Unhealthy") rows.append( ( get_short_task_id(task.id), hostname, "{} ({})".format( local_deployed_datetime.strftime("%Y-%m-%dT%H:%M"), humanize.naturaltime(local_deployed_datetime), ), health_check_status, ) ) output.append("\n".join([" %s" % line for line in format_table(rows)])) if len(app.tasks) == 0: output.append(" No tasks associated with this marathon app") return deploy_status, app.tasks_running, "\n".join(output)
def marathon_job_status( service: str, instance: str, job_config: marathon_tools.MarathonServiceConfig, marathon_apps_with_clients: List[Tuple[MarathonApp, MarathonClient]], verbose: int, ) -> MutableMapping[str, Any]: job_status_fields: MutableMapping[str, Any] = { "app_statuses": [], "app_count": len(marathon_apps_with_clients), "desired_state": job_config.get_desired_state(), "bounce_method": job_config.get_bounce_method(), "expected_instance_count": job_config.get_instances(), "active_shas": list(get_active_shas_for_marathon_apps(marathon_apps_with_clients)), } try: desired_app_id = job_config.format_marathon_app_dict()["id"] except NoDockerImageError: error_msg = "Docker image is not in deployments.json." job_status_fields["error_message"] = error_msg return job_status_fields job_status_fields["desired_app_id"] = desired_app_id deploy_status_for_desired_app = None dashboard_links = get_marathon_dashboard_links( settings.marathon_clients, settings.system_paasta_config) tasks_running = 0 for app, marathon_client in marathon_apps_with_clients: deploy_status = marathon_tools.get_marathon_app_deploy_status( marathon_client, app) app_status = marathon_app_status( app, marathon_client, dashboard_links.get(marathon_client) if dashboard_links else None, deploy_status, list_tasks=verbose > 0, ) job_status_fields["app_statuses"].append(app_status) if app.id.lstrip("/") == desired_app_id.lstrip("/"): deploy_status_for_desired_app = marathon_tools.MarathonDeployStatus.tostring( deploy_status) tasks_running += app.tasks_running job_status_fields["deploy_status"] = (deploy_status_for_desired_app or "Waiting for bounce") job_status_fields["running_instance_count"] = tasks_running if verbose > 0: autoscaling_info = get_autoscaling_info(marathon_apps_with_clients, job_config) if autoscaling_info is not None: autoscaling_info_dict = autoscaling_info._asdict() for field in ("current_utilization", "target_instances"): if autoscaling_info_dict[field] is None: del autoscaling_info_dict[field] job_status_fields["autoscaling_info"] = autoscaling_info_dict return job_status_fields