def paasta_autoscale(args):
    log.setLevel(logging.DEBUG)
    service = figure_out_service_name(args)
    api = client.get_paasta_api_client(cluster=args.cluster, http_res=True)
    if not api:
        paasta_print(
            "Could not connect to paasta api. Maybe you misspelled the cluster?"
        )
        return 1

    if args.set is None:
        log.debug("Getting the current autoscaler count...")
        res, http = api.autoscaler.get_autoscaler_count(
            service=service, instance=args.instance).result()
    else:
        log.debug(f"Setting desired instances to {args.set}.")
        body = {"desired_instances": int(args.set)}
        res, http = api.autoscaler.update_autoscaler_count(
            service=service, instance=args.instance, json_body=body).result()

        _log_audit(
            action="manual-scale",
            action_details=body,
            service=service,
            instance=args.instance,
            cluster=args.cluster,
        )

    log.debug(f"Res: {res} Http: {http}")
    print(res["desired_instances"])
    return 0
Пример #2
0
def paasta_metastatus_on_api_endpoint(
    cluster: str,
    system_paasta_config: SystemPaastaConfig,
    groupings: Sequence[str],
    verbose: int,
    autoscaling_info: bool = False,
    use_mesos_cache: bool = False,
) -> Tuple[int, str]:
    client = get_paasta_api_client(cluster, system_paasta_config)
    if not client:
        paasta_print('Cannot get a paasta-api client')
        exit(1)

    try:
        cmd_args, _ = get_paasta_metastatus_cmd_args(
            groupings=groupings,
            verbose=verbose,
            autoscaling_info=autoscaling_info,
            use_mesos_cache=use_mesos_cache,
        )
        res = client.metastatus.metastatus(
            cmd_args=[str(arg) for arg in cmd_args]).result()
        output, exit_code = res.output, res.exit_code
    except HTTPError as exc:
        output, exit_code = exc.response.text, exc.status_code

    return exit_code, output
Пример #3
0
def paasta_autoscale(args):
    log.setLevel(logging.DEBUG)
    service = figure_out_service_name(args)
    api = client.get_paasta_api_client(cluster=args.cluster, http_res=True)
    if not api:
        paasta_print(
            'Could not connect to paasta api. Maybe you misspelled the cluster?'
        )
        return 1

    if args.set is None:
        log.debug("Getting the current autoscaler count...")
        res, http = api.autoscaler.get_autoscaler_count(
            service=service, instance=args.instance).result()
    else:
        log.debug("Setting desired instances to {}.".format(args.set))
        body = {'desired_instances': int(args.set)}
        res, http = api.autoscaler.update_autoscaler_count(
            service=service,
            instance=args.instance,
            json_body=body,
        ).result()

    log.debug("Res: {} Http: {}".format(res, http))
    print(res["desired_instances"])
    return 0
Пример #4
0
def paasta_status_on_api_endpoint(
    cluster: str,
    service: str,
    instance: str,
    output: List[str],
    system_paasta_config: SystemPaastaConfig,
    verbose: int,
) -> int:
    client = get_paasta_api_client(cluster, system_paasta_config)
    if not client:
        paasta_print('Cannot get a paasta-api client')
        exit(1)

    try:
        status = client.service.status_instance(service=service,
                                                instance=instance).result()
    except HTTPError as exc:
        paasta_print(exc.response.text)
        return exc.status_code

    output.append('    instance: %s' % PaastaColors.blue(instance))
    output.append('    Git sha:    %s (desired)' % status.git_sha)

    if status.marathon is not None:
        return print_marathon_status(service, instance, output,
                                     status.marathon)
    elif status.kubernetes is not None:
        return print_kubernetes_status(service, instance, output,
                                       status.kubernetes)
    else:
        paasta_print(
            "Not implemented: Looks like %s is not a Marathon or Kubernetes instance"
            % instance)
        return 0
Пример #5
0
def test_get_paasta_api_client(system_paasta_config):
    with mock.patch("paasta_tools.api.client.load_system_paasta_config",
                    autospec=True) as mock_load_system_paasta_config:
        mock_load_system_paasta_config.return_value = system_paasta_config

        client = get_paasta_api_client()
        assert client
Пример #6
0
def get_service_autoscale_pause_time(cluster):
    api = client.get_paasta_api_client(cluster=cluster, http_res=True)
    if not api:
        paasta_print(
            'Could not connect to paasta api. Maybe you misspelled the cluster?'
        )
        return 1
    pause_time, http = api.service_autoscaler.get_service_autoscaler_pause(
    ).result()
    if http.status_code == 500:
        paasta_print('Could not connect to zookeeper server')
        return 2

    pause_time = float(pause_time)
    if pause_time < time.time():
        paasta_print('Service autoscaler is not paused')
    else:
        local_tz = get_localzone()
        paused_readable = local_tz.localize(
            datetime.fromtimestamp(pause_time)).strftime(
                '%F %H:%M:%S %Z')  # NOQA
        paasta_print(
            'Service autoscaler is paused until {}'.format(paused_readable))

    return 0
Пример #7
0
def paasta_status_on_api_endpoint(cluster, service, instance,
                                  system_paasta_config, verbose):
    client = get_paasta_api_client(cluster, system_paasta_config)
    if not client:
        paasta_print('Cannot get a paasta-api client')
        exit(1)

    try:
        status = client.service.status_instance(service=service,
                                                instance=instance).result()
    except HTTPError as exc:
        paasta_print(exc.response.text)
        return exc.status_code

    paasta_print('instance: %s' % PaastaColors.blue(instance))
    paasta_print('Git sha:    %s (desired)' % status.git_sha)

    marathon_status = status.marathon
    if marathon_status is None:
        paasta_print(
            "Not implemented: Looks like %s is not a Marathon instance" %
            instance)
        return 0
    elif marathon_status.error_message:
        paasta_print(marathon_status.error_message)
        return 1

    bouncing_status = bouncing_status_human(
        marathon_status.app_count,
        marathon_status.bounce_method,
    )
    desired_state = desired_state_human(
        marathon_status.desired_state,
        marathon_status.expected_instance_count,
    )
    paasta_print("State:      %s - Desired state: %s" %
                 (bouncing_status, desired_state))

    status = MarathonDeployStatus.fromstring(marathon_status.deploy_status)
    if status != MarathonDeployStatus.NotRunning:
        if status == MarathonDeployStatus.Delayed:
            deploy_status = marathon_app_deploy_status_human(
                status, marathon_status.backoff_seconds)
        else:
            deploy_status = marathon_app_deploy_status_human(status)
    else:
        deploy_status = 'NotRunning'

    paasta_print(
        status_marathon_job_human(
            service=service,
            instance=instance,
            deploy_status=deploy_status,
            desired_app_id=marathon_status.app_id,
            app_count=marathon_status.app_count,
            running_instances=marathon_status.running_instance_count,
            normal_instance_count=marathon_status.expected_instance_count,
        ), )
    return 0
Пример #8
0
def get_status_for_instance(cluster, service, instance):
    api = client.get_paasta_api_client(cluster=cluster)
    if not api:
        sys.exit(1)
    status = api.service.status_instance(service=service, instance=instance).result()
    if not status.marathon:
        log.error("Not a marathon service, exiting")
        sys.exit(1)
    return status
Пример #9
0
def get_status_for_instance(cluster, service, instance):
    api = client.get_paasta_api_client(cluster=cluster)
    if not api:
        sys.exit(1)
    status = api.service.status_instance(service=service, instance=instance).result()
    if not status.marathon:
        log.error("Not a marathon service, exiting")
        sys.exit(1)
    return status
Пример #10
0
def get_task_from_instance(cluster,
                           service,
                           instance,
                           slave_hostname=None,
                           task_id=None,
                           verbose=True):
    api = client.get_paasta_api_client(cluster=cluster)
    if not api:
        log.error(f"Could not get API client for cluster {cluster}")
        raise PaastaTaskNotFound
    if task_id:
        log.warning("Specifying a task_id, so ignoring hostname if specified")
        task = api.service.task_instance(
            service=service,
            instance=instance,
            verbose=True,
            task_id=task_id,
        ).result()
        return task
    try:
        if task_id:
            log.warning(
                "Specifying a task_id, so ignoring hostname if specified")
            task = api.service.task_instance(
                service=service,
                instance=instance,
                verbose=True,
                task_id=task_id,
            ).result()
            return task
        tasks = api.service.tasks_instance(
            service=service,
            instance=instance,
            verbose=True,
            slave_hostname=slave_hostname,
        ).result()
    except HTTPNotFound:
        log.error(
            "Cannot find instance {}, for service {}, in cluster {}".format(
                instance,
                service,
                cluster,
            ))
        raise PaastaTaskNotFound
    except HTTPError as e:
        log.error("Problem with API call to find task details")
        log.error(e.response.text)
        raise PaastaTaskNotFound
    if not tasks:
        log.error(
            "Cannot find any tasks on host: {} or with task_id: {}".format(
                slave_hostname,
                task_id,
            ))
        raise PaastaTaskNotFound
    return tasks[0]
Пример #11
0
def paasta_status_on_api_endpoint(
    cluster: str,
    service: str,
    instance: str,
    output: List[str],
    system_paasta_config: SystemPaastaConfig,
    verbose: int,
) -> int:
    output.append("    instance: %s" % PaastaColors.blue(instance))
    client = get_paasta_api_client(cluster, system_paasta_config)
    if not client:
        paasta_print("Cannot get a paasta-api client")
        exit(1)
    try:
        status = client.service.status_instance(
            service=service, instance=instance, verbose=verbose
        ).result()
    except HTTPError as exc:
        output.append(PaastaColors.red(exc.response.text))
        return exc.status_code
    except (BravadoConnectionError, BravadoTimeoutError) as exc:
        output.append(
            PaastaColors.red(f"Could not connect to API: {exc.__class__.__name__}")
        )
        return 1
    except Exception:
        tb = sys.exc_info()[2]
        output.append(PaastaColors.red(f"Exception when talking to the API:"))
        output.extend(line.strip() for line in traceback.format_tb(tb))
        return 1

    if status.git_sha != "":
        output.append("    Git sha:    %s (desired)" % status.git_sha)

    if status.marathon is not None:
        return print_marathon_status(service, instance, output, status.marathon)
    elif status.kubernetes is not None:
        return print_kubernetes_status(service, instance, output, status.kubernetes)
    elif status.tron is not None:
        return print_tron_status(service, instance, output, status.tron, verbose)
    elif status.adhoc is not None:
        return print_adhoc_status(
            cluster, service, instance, output, status.adhoc, verbose
        )
    elif status.flink is not None:
        return print_flink_status(
            cluster, service, instance, output, status.flink, verbose
        )
    else:
        paasta_print(
            "Not implemented: Looks like %s is not a Marathon or Kubernetes instance"
            % instance
        )
        return 0
Пример #12
0
def test_get_paasta_api_client():
    with mock.patch('paasta_tools.api.client.load_system_paasta_config',
                    autospec=True) as mock_load_system_paasta_config:
        mock_load_system_paasta_config.return_value = SystemPaastaConfig({
            'api_endpoints': {
                'fake_cluster': "http://fake_cluster:5054"
            },
            'cluster': 'fake_cluster'
        }, 'fake_directory')

        client = get_paasta_api_client()
        assert client
def delete_service_autoscale_pause_time(cluster):
    api = client.get_paasta_api_client(cluster=cluster, http_res=True)
    if not api:
        print("Could not connect to paasta api. Maybe you misspelled the cluster?")
        return 1
    res, http = api.service_autoscaler.delete_service_autoscaler_pause().result()
    if http.status_code == 500:
        print("Could not connect to zookeeper server")
        return 2

    print("Service autoscaler is unpaused")
    return 0
Пример #14
0
def update_service_autoscale_pause_time(cluster, mins):
    api = client.get_paasta_api_client(cluster=cluster, http_res=True)
    if not api:
        paasta_print('Could not connect to paasta api. Maybe you misspelled the cluster?')
        return 1
    body = {'minutes': mins}
    res, http = api.service_autoscaler.update_service_autoscaler_pause(json_body=body).result()
    if http.status_code == 500:
        paasta_print('Could not connect to zookeeper server')
        return 2

    paasta_print(f'Service autoscaler is paused for {mins}')
    return 0
Пример #15
0
def test_get_paasta_api_client():
    with mock.patch('paasta_tools.api.client.load_system_paasta_config',
                    autospec=True) as mock_load_system_paasta_config:
        mock_load_system_paasta_config.return_value = SystemPaastaConfig(
            {
                'api_endpoints': {
                    'fake_cluster': "http://fake_cluster:5054"
                },
                'cluster': 'fake_cluster'
            }, 'fake_directory')

        client = get_paasta_api_client()
        assert client
Пример #16
0
def paasta_status_on_api_endpoint(cluster, service, instance, system_paasta_config, verbose):
    client = get_paasta_api_client(cluster, system_paasta_config)
    if not client:
        paasta_print('Cannot get a paasta-api client')
        exit(1)

    try:
        status = client.service.status_instance(service=service, instance=instance).result()
    except HTTPError as exc:
        paasta_print(exc.response.text)
        return exc.status_code

    paasta_print('instance: %s' % PaastaColors.blue(instance))
    paasta_print('Git sha:    %s (desired)' % status.git_sha)

    marathon_status = status.marathon
    if marathon_status is None:
        paasta_print("Not implemented: Looks like %s is not a Marathon instance" % instance)
        return 0
    elif marathon_status.error_message:
        paasta_print(marathon_status.error_message)
        return 1

    bouncing_status = bouncing_status_human(marathon_status.app_count,
                                            marathon_status.bounce_method)
    desired_state = desired_state_human(marathon_status.desired_state,
                                        marathon_status.expected_instance_count)
    paasta_print("State:      %s - Desired state: %s" % (bouncing_status, desired_state))

    status = MarathonDeployStatus.fromstring(marathon_status.deploy_status)
    if status != MarathonDeployStatus.NotRunning:
        if status == MarathonDeployStatus.Delayed:
            deploy_status = marathon_app_deploy_status_human(status, marathon_status.backoff_seconds)
        else:
            deploy_status = marathon_app_deploy_status_human(status)
    else:
        deploy_status = 'NotRunning'

    paasta_print(
        status_marathon_job_human(
            service,
            instance,
            deploy_status,
            marathon_status.app_id,
            marathon_status.running_instance_count,
            marathon_status.expected_instance_count,
        )
    )
    return 0
Пример #17
0
def paasta_autoscale(args):
    log.setLevel(logging.DEBUG)
    service = figure_out_service_name(args)
    api = client.get_paasta_api_client(cluster=args.cluster, http_res=True)
    if not api:
        paasta_print(
            "Could not connect to paasta api. Maybe you misspelled the cluster?"
        )
        return 1
    try:
        if args.set is None:
            log.debug("Getting the current autoscaler count...")
            res, http = api.autoscaler.get_autoscaler_count(
                service=service, instance=args.instance
            ).result()
        else:
            log.debug(f"Setting desired instances to {args.set}.")
            body = {"desired_instances": int(args.set)}
            res, http = api.autoscaler.update_autoscaler_count(
                service=service, instance=args.instance, json_body=body
            ).result()

            _log_audit(
                action="manual-scale",
                action_details=body,
                service=service,
                instance=args.instance,
                cluster=args.cluster,
            )
    except HTTPNotFound:
        paasta_print(
            PaastaColors.red(
                f"ERROR: '{args.instance}' is not configured to autoscale, "
                f"so paasta autoscale could not scale it up on demand. "
                f"If you want to be able to boost this service, please configure autoscaling for the service "
                f"in its config file by setting min and max instances. Example: \n"
                f"{args.instance}:\n"
                f"     min_instances: 5\n"
                f"     max_instances: 50"
            )
        )
        return 0

    log.debug(f"Res: {res} Http: {http}")
    print(res["desired_instances"])
    return 0
Пример #18
0
def paasta_status_on_api_endpoint(
    cluster: str,
    service: str,
    instance: str,
    output: List[str],
    system_paasta_config: SystemPaastaConfig,
    verbose: int,
) -> int:
    client = get_paasta_api_client(cluster, system_paasta_config)
    if not client:
        paasta_print("Cannot get a paasta-api client")
        exit(1)
    try:
        status = client.service.status_instance(
            service=service, instance=instance, verbose=verbose
        ).result()
    except HTTPError as exc:
        paasta_print(exc.response.text)
        return exc.status_code

    output.append("    instance: %s" % PaastaColors.blue(instance))
    if status.git_sha != "":
        output.append("    Git sha:    %s (desired)" % status.git_sha)

    if status.marathon is not None:
        return print_marathon_status(service, instance, output, status.marathon)
    elif status.kubernetes is not None:
        return print_kubernetes_status(service, instance, output, status.kubernetes)
    elif status.tron is not None:
        return print_tron_status(service, instance, output, status.tron, verbose)
    elif status.adhoc is not None:
        return print_adhoc_status(
            cluster, service, instance, output, status.adhoc, verbose
        )
    elif status.flink is not None:
        return print_flink_status(
            cluster, service, instance, output, status.flink.get("status"), verbose
        )
    elif status.chronos is not None:
        return print_chronos_status(output, status.chronos.output)
    else:
        paasta_print(
            "Not implemented: Looks like %s is not a Marathon or Kubernetes instance"
            % instance
        )
        return 0
Пример #19
0
def list_deploy_queue(args, ) -> int:
    cluster = args.cluster
    all_clusters = list_clusters(soa_dir=args.soa_dir)
    if cluster not in all_clusters:
        paasta_print(
            f"{cluster} does not appear to be a valid cluster. Run `paasta "
            "list-clusters` to see available options.")
        return 1

    system_paasta_config = load_system_paasta_config()
    client = get_paasta_api_client(cluster,
                                   system_paasta_config,
                                   http_res=True)
    if not client:
        paasta_print("Cannot get a paasta API client")
        return 1

    try:
        deploy_queues, raw_response = client.deploy_queue.deploy_queue(
        ).result()
    except HTTPError as exc:
        paasta_print(PaastaColors.red(exc.response.text))
        return exc.status_code
    except (BravadoConnectionError, BravadoTimeoutError) as exc:
        paasta_print(
            PaastaColors.red(
                f"Could not connect to API: {exc.__class__.__name__}"))
        return 1
    except Exception:
        tb = sys.exc_info()[2]
        paasta_print(PaastaColors.red(f"Exception when talking to the API:"))
        paasta_print("".join(traceback.format_tb(tb)))
        return 1

    if args.json:
        paasta_print(raw_response.text)
    else:
        formatted_deploy_queues = format_deploy_queues(deploy_queues, cluster)
        paasta_print(formatted_deploy_queues)

    return 0
Пример #20
0
def paasta_status_on_api_endpoint(cluster, service, instance, system_paasta_config, verbose):
    client = get_paasta_api_client(cluster, system_paasta_config)
    if not client:
        print 'Cannot get a paasta-api client'
        exit(1)

    try:
        status = client.service.status_instance(service=service, instance=instance).result()
    except HTTPError as exc:
        print exc.response.text
        return

    print 'instance: %s' % PaastaColors.blue(instance)
    print 'Git sha:    %s (desired)' % status.git_sha

    marathon_status = status.marathon
    if marathon_status.error_message:
        print marathon_status.error_message
        return

    bouncing_status = bouncing_status_human(marathon_status.app_count,
                                            marathon_status.bounce_method)
    desired_state = desired_state_human(marathon_status.desired_state,
                                        marathon_status.expected_instance_count)
    print "State:      %s - Desired state: %s" % (bouncing_status, desired_state)

    status = MarathonDeployStatus.fromstring(marathon_status.deploy_status)
    if status != MarathonDeployStatus.NotRunning:
        if status == MarathonDeployStatus.Delayed:
            deploy_status = marathon_app_deploy_status_human(status, marathon_status.backoff_seconds)
        else:
            deploy_status = marathon_app_deploy_status_human(status)
    else:
        deploy_status = 'NotRunning'

    print status_marathon_job_human(service, instance, deploy_status,
                                    marathon_status.app_id,
                                    marathon_status.running_instance_count,
                                    marathon_status.expected_instance_count)
Пример #21
0
def get_task_from_instance(cluster, service, instance, slave_hostname=None, task_id=None, verbose=True):
    api = client.get_paasta_api_client(cluster=cluster)
    if not api:
        log.error("Could not get API client for cluster {0}".format(cluster))
        raise PaastaTaskNotFound
    if task_id:
        log.warning("Specifying a task_id, so ignoring hostname if specified")
        task = api.service.task_instance(service=service,
                                         instance=instance,
                                         verbose=True,
                                         task_id=task_id).result()
        return task
    try:
        if task_id:
            log.warning("Specifying a task_id, so ignoring hostname if specified")
            task = api.service.task_instance(service=service,
                                             instance=instance,
                                             verbose=True,
                                             task_id=task_id).result()
            return task
        tasks = api.service.tasks_instance(service=service,
                                           instance=instance,
                                           verbose=True,
                                           slave_hostname=slave_hostname).result()
    except HTTPNotFound:
        log.error("Cannot find instance {0}, for service {1}, in cluster {2}".format(instance,
                                                                                     service,
                                                                                     cluster))
        raise PaastaTaskNotFound
    except HTTPError as e:
        log.error("Problem with API call to find task details")
        log.error(e.response.text)
        raise PaastaTaskNotFound
    if not tasks:
        log.error("Cannot find any tasks on host: {0} or with task_id: {1}".format(slave_hostname,
                                                                                   task_id))
        raise PaastaTaskNotFound
    return tasks[0]
Пример #22
0
def are_instances_deployed(cluster, service, instances, git_sha):
    api = client.get_paasta_api_client(cluster=cluster)
    if not api:
        # Assume not deployed if we can't reach API
        return False
    statuses = []
    for instance in instances:
        try:
            statuses.append(
                api.service.status_instance(service=service,
                                            instance=instance).result())
        except HTTPError as e:
            if e.response.status_code == 404:
                log.warning(
                    "Can't get status for instance {0}, service {1} in cluster {2}. "
                    "This is normally because it is a new service that hasn't been "
                    "deployed by PaaSTA yet".format(instance, service,
                                                    cluster))
                statuses.append(None)
    results = []
    for status in statuses:
        if not status:
            results.append(False)
        # if it's a chronos service etc then skip waiting for it to deploy
        elif not status.marathon:
            results.append(True)
        elif status.marathon.expected_instance_count == 0 or status.marathon.desired_state == 'stop':
            results.append(True)
        else:
            results.append(
                git_sha.startswith(status.git_sha)
                and status.marathon.app_count == 1
                and status.marathon.deploy_status == 'Running'
                and status.marathon.expected_instance_count
                == status.marathon.running_instance_count)
    return all(results)
Пример #23
0
def instances_deployed(cluster, service, instances, git_sha):
    api = client.get_paasta_api_client(cluster=cluster)
    if not api:
        log.warning(
            "Couldn't reach the PaaSTA api for {}! Assuming it is not deployed there yet."
            .format(cluster))
        return False
    statuses = []
    for instance in instances:
        log.info("Inspecting the deployment status of {}.{} on {}".format(
            service, instance, cluster))
        try:
            status = api.service.status_instance(service=service,
                                                 instance=instance).result()
            statuses.append(status)
        except HTTPError as e:
            if e.response.status_code == 404:
                log.warning(
                    "Can't get status for instance {0}, service {1} in cluster {2}. "
                    "This is normally because it is a new service that hasn't been "
                    "deployed by PaaSTA yet".format(instance, service,
                                                    cluster))
            else:
                log.warning(
                    "Error getting service status from PaaSTA API: {0}: {1}".
                    format(e.response.status_code, e.response.text))
            statuses.append(None)
        except ConnectionError as e:
            log.warning(
                "Error getting service status from PaaSTA API for {0}: {1}".
                format(cluster, e))
            statuses.append(None)
    results = []
    for status in statuses:
        if not status:
            log.info(
                "No status for an unknown instance in {}. Not deployed yet.".
                format(cluster))
            results.append(False)
        elif not status.marathon:
            log.info("{}.{} in {} is not a Marathon job. Marked as deployed.".
                     format(service, status.instance, cluster))
            results.append(True)
        elif status.marathon.expected_instance_count == 0 or status.marathon.desired_state == 'stop':
            log.info(
                "{}.{} in {} is marked as stopped. Marked as deployed.".format(
                    service, status.instance, cluster))
            results.append(True)
        else:
            if status.marathon.app_count != 1:
                log.info("{}.{} on {} is still bouncing, {} versions running".
                         format(service, status.instance, cluster,
                                status.marathon.app_count))
                results.append(False)
                continue
            if not git_sha.startswith(status.git_sha):
                log.info(
                    "{}.{} on {} doesn't have the right sha yet: {}".format(
                        service, status.instance, cluster, status.git_sha))
                results.append(False)
                continue
            if status.marathon.deploy_status != 'Running':
                log.info("{}.{} on {} isn't running yet: {}".format(
                    service, status.instance, cluster,
                    status.marathon.deploy_status))
                results.append(False)
                continue
            if status.marathon.expected_instance_count != status.marathon.running_instance_count:
                log.info(
                    "{}.{} on {} isn't scaled up yet, has {} out of {}".format(
                        service, status.instance, cluster,
                        status.marathon.running_instance_count,
                        status.marathon.expected_instance_count))
                results.append(False)
                continue
            log.info(
                "{}.{} on {} looks 100% deployed at {} instances on {}".format(
                    service, status.instance, cluster,
                    status.marathon.running_instance_count, status.git_sha))
            results.append(True)

    return sum(results)
Пример #24
0
def _run_instance_worker(cluster_data, instances_out, green_light):
    """Get instances from the instances_in queue and check them one by one.

    If an instance isn't deployed, add it to the instances_out queue
    to re-check it later.

    :param cluster_data: an instance of ClusterData.
    :param instances_out: See the docstring for instances_deployed().
    :param green_light: See the docstring for _query_clusters().
    """

    api = client.get_paasta_api_client(cluster=cluster_data.cluster)
    if not api:
        log.warning("Couldn't reach the PaaSTA api for {}! Assuming it is not "
                    "deployed there yet.".format(cluster_data.cluster))
        while not cluster_data.instances_queue.empty():
            try:
                instance_config = cluster_data.instances_queue.get(block=False)
            except Empty:
                return
            cluster_data.instances_queue.task_done()
            instances_out.put(instance_config)

    while not cluster_data.instances_queue.empty() and green_light.is_set():
        try:
            instance_config = cluster_data.instances_queue.get(block=False)
        except Empty:
            return

        instance = instance_config.get_instance()

        log.debug("Inspecting the deployment status of {}.{} on {}".format(
            cluster_data.service, instance, cluster_data.cluster))
        try:
            status = None
            status = api.service.status_instance(
                service=cluster_data.service,
                instance=instance,
            ).result()
        except HTTPError as e:
            if e.response.status_code == 404:
                log.warning(
                    "Can't get status for instance {}, service {} in "
                    "cluster {}. This is normally because it is a new "
                    "service that hasn't been deployed by PaaSTA yet".format(
                        instance,
                        cluster_data.service,
                        cluster_data.cluster,
                    ))
            else:
                log.warning(
                    "Error getting service status from PaaSTA API for {}: {}"
                    "{}".format(
                        cluster_data.cluster,
                        e.response.status_code,
                        e.response.text,
                    ))
        except ConnectionError as e:
            log.warning("Error getting service status from PaaSTA API for {}:"
                        "{}".format(cluster_data.cluster, e))

        if not status:
            log.debug("No status for {}.{}, in {}. Not deployed yet.".format(
                cluster_data.service,
                instance,
                cluster_data.cluster,
            ))
            cluster_data.instances_queue.task_done()
            instances_out.put(instance_config)
        elif not status.marathon:
            log.debug("{}.{} in {} is not a Marathon job. Marked as deployed.".
                      format(
                          cluster_data.service,
                          instance,
                          cluster_data.cluster,
                      ))
        elif (status.marathon.expected_instance_count == 0
              or status.marathon.desired_state == 'stop'):
            log.debug(
                "{}.{} in {} is marked as stopped. Marked as deployed.".format(
                    cluster_data.service,
                    status.instance,
                    cluster_data.cluster,
                ))
        else:
            if status.marathon.app_count != 1:
                paasta_print("  {}.{} on {} is still bouncing, {} versions "
                             "running".format(
                                 cluster_data.service,
                                 status.instance,
                                 cluster_data.cluster,
                                 status.marathon.app_count,
                             ))
                cluster_data.instances_queue.task_done()
                instances_out.put(instance_config)
                continue
            if not cluster_data.git_sha.startswith(status.git_sha):
                paasta_print(
                    "  {}.{} on {} doesn't have the right sha yet: {}".format(
                        cluster_data.service,
                        instance,
                        cluster_data.cluster,
                        status.git_sha,
                    ))
                cluster_data.instances_queue.task_done()
                instances_out.put(instance_config)
                continue
            if status.marathon.deploy_status not in [
                    'Running', 'Deploying', 'Waiting'
            ]:
                paasta_print("  {}.{} on {} isn't running yet: {}".format(
                    cluster_data.service,
                    instance,
                    cluster_data.cluster,
                    status.marathon.deploy_status,
                ))
                cluster_data.instances_queue.task_done()
                instances_out.put(instance_config)
                continue

            # The bounce margin factor defines what proportion of instances we need to be "safe",
            # so consider it scaled up "enough" if we have that proportion of instances ready.
            required_instance_count = int(
                math.ceil(
                    instance_config.get_bounce_margin_factor() *
                    status.marathon.expected_instance_count, ))
            if required_instance_count > status.marathon.running_instance_count:
                paasta_print(
                    "  {}.{} on {} isn't scaled up yet, "
                    "has {} out of {} required instances (out of a total of {})"
                    .format(
                        cluster_data.service,
                        instance,
                        cluster_data.cluster,
                        status.marathon.running_instance_count,
                        required_instance_count,
                        status.marathon.expected_instance_count,
                    ))
                cluster_data.instances_queue.task_done()
                instances_out.put(instance_config)
                continue
            paasta_print("Complete: {}.{} on {} looks 100% deployed at {} "
                         "instances on {}".format(
                             cluster_data.service,
                             instance,
                             cluster_data.cluster,
                             status.marathon.running_instance_count,
                             status.git_sha,
                         ))
            cluster_data.instances_queue.task_done()
Пример #25
0
def paasta_start_or_stop(args, desired_state):
    """Requests a change of state to start or stop given branches of a service."""
    soa_dir = args.soa_dir

    pargs = apply_args_filters(args)
    if len(pargs) == 0:
        return 1

    affected_services = {
        s
        for service_list in pargs.values() for s in service_list.keys()
    }
    if len(affected_services) > 1:
        print(
            PaastaColors.red(
                "Warning: trying to start/stop/restart multiple services:"))

        for cluster, services_instances in pargs.items():
            print("Cluster %s:" % cluster)
            for service, instances in services_instances.items():
                print("    Service %s:" % service)
                print("        Instances %s" % ",".join(instances.keys()))

        if sys.stdin.isatty():
            confirm = choice.Binary("Are you sure you want to continue?",
                                    False).ask()
        else:
            confirm = False
        if not confirm:
            print()
            print("exiting")
            return 1

    invalid_deploy_groups = []
    marathon_message_printed = False
    affected_flinks = []

    if args.clusters is None or args.instances is None:
        if confirm_to_continue(pargs.items(), desired_state) is False:
            print()
            print("exiting")
            return 1

    for cluster, services_instances in pargs.items():
        for service, instances in services_instances.items():
            for instance in instances.keys():
                service_config = get_instance_config(
                    service=service,
                    cluster=cluster,
                    instance=instance,
                    soa_dir=soa_dir,
                    load_deployments=False,
                )
                if isinstance(service_config, FlinkDeploymentConfig):
                    affected_flinks.append(service_config)
                    continue

                try:
                    remote_refs = get_remote_refs(service, soa_dir)
                except remote_git.LSRemoteException as e:
                    msg = (
                        "Error talking to the git server: %s\n"
                        "This PaaSTA command requires access to the git server to operate.\n"
                        "The git server may be down or not reachable from here.\n"
                        "Try again from somewhere where the git server can be reached, "
                        "like your developer environment.") % str(e)
                    print(msg)
                    return 1

                deploy_group = service_config.get_deploy_group()
                (deploy_tag,
                 _) = get_latest_deployment_tag(remote_refs, deploy_group)

                if deploy_tag not in remote_refs:
                    invalid_deploy_groups.append(deploy_group)
                else:
                    force_bounce = utils.format_timestamp(
                        datetime.datetime.utcnow())
                    if (isinstance(service_config, MarathonServiceConfig)
                            and not marathon_message_printed):
                        print_marathon_message(desired_state)
                        marathon_message_printed = True

                    issue_state_change_for_service(
                        service_config=service_config,
                        force_bounce=force_bounce,
                        desired_state=desired_state,
                    )

    return_val = 0

    # TODO: Refactor to discover if set_state is available for given
    #       instance_type in API
    if affected_flinks:
        print_flink_message(desired_state)
        csi = defaultdict(lambda: defaultdict(list))
        for service_config in affected_flinks:
            csi[service_config.cluster][service_config.service].append(
                service_config.instance)

        system_paasta_config = load_system_paasta_config()
        for cluster, services_instances in csi.items():
            client = get_paasta_api_client(cluster, system_paasta_config)
            if not client:
                print("Cannot get a paasta-api client")
                exit(1)

            for service, instances in services_instances.items():
                for instance in instances:
                    try:
                        client.service.instance_set_state(
                            service=service,
                            instance=instance,
                            desired_state=desired_state,
                        ).result()
                    except HTTPError as exc:
                        print(exc.response.text)
                        return exc.status_code

                return_val = 0

    if invalid_deploy_groups:
        print(f"No deploy tags found for {', '.join(invalid_deploy_groups)}.")
        print(f"Has {service} been deployed there yet?")
        return_val = 1

    return return_val
Пример #26
0
def instances_deployed(cluster, service, instances, git_sha):
    api = client.get_paasta_api_client(cluster=cluster)
    if not api:
        log.warning("Couldn't reach the PaaSTA api for {}! Assuming it is not deployed there yet.".format(cluster))
        return False
    statuses = []
    for instance in instances:
        log.debug("Inspecting the deployment status of {}.{} on {}".format(service, instance, cluster))
        try:
            status = api.service.status_instance(service=service, instance=instance).result()
            statuses.append(status)
        except HTTPError as e:
            if e.response.status_code == 404:
                log.warning("Can't get status for instance {0}, service {1} in cluster {2}. "
                            "This is normally because it is a new service that hasn't been "
                            "deployed by PaaSTA yet".format(instance, service, cluster))
            else:
                log.warning("Error getting service status from PaaSTA API: {0}: {1}".format(e.response.status_code,
                                                                                            e.response.text))
            statuses.append(None)
        except ConnectionError as e:
            log.warning("Error getting service status from PaaSTA API for {0}: {1}".format(cluster, e))
            statuses.append(None)
    results = []
    for status in statuses:
        if not status:
            log.debug("No status for an unknown instance in {}. Not deployed yet.".format(cluster))
            results.append(False)
        elif not status.marathon:
            log.debug("{}.{} in {} is not a Marathon job. Marked as deployed.".format(
                service, status.instance, cluster))
            results.append(True)
        elif status.marathon.expected_instance_count == 0 or status.marathon.desired_state == 'stop':
            log.debug("{}.{} in {} is marked as stopped. Marked as deployed.".format(
                service, status.instance, cluster))
            results.append(True)
        else:
            if status.marathon.app_count != 1:
                paasta_print("  {}.{} on {} is still bouncing, {} versions running".format(
                    service, status.instance, cluster, status.marathon.app_count))
                results.append(False)
                continue
            if not git_sha.startswith(status.git_sha):
                paasta_print("  {}.{} on {} doesn't have the right sha yet: {}".format(
                    service, status.instance, cluster, status.git_sha))
                results.append(False)
                continue
            if status.marathon.deploy_status != 'Running':
                paasta_print("  {}.{} on {} isn't running yet: {}".format(
                    service, status.instance, cluster, status.marathon.deploy_status))
                results.append(False)
                continue
            if status.marathon.expected_instance_count != status.marathon.running_instance_count:
                paasta_print("  {}.{} on {} isn't scaled up yet, has {} out of {}".format(
                    service, status.instance, cluster, status.marathon.running_instance_count,
                    status.marathon.expected_instance_count))
                results.append(False)
                continue
            paasta_print("Complete: {}.{} on {} looks 100% deployed at {} instances on {}".format(
                service, status.instance, cluster, status.marathon.running_instance_count, status.git_sha))
            results.append(True)

    return sum(results)
Пример #27
0
def run_capacity_check():
    options = parse_capacity_check_options()
    system_paasta_config = load_system_paasta_config()
    cluster = options.cluster if options.cluster is not None else system_paasta_config.get_cluster(
    )
    value_to_check = options.type

    client = get_paasta_api_client(cluster=cluster)
    if client is None:
        paasta_print('UNKNOWN Failed to load paasta api client')
        sys.exit(3)

    overrides = read_overrides(options.overrides)

    attributes = options.attributes.split(',')

    try:
        resource_use = client.resources.resources(
            groupings=attributes).result()
    except HTTPError as e:
        paasta_print("UNKNOWN recieved exception from paasta api:\n\t%s" % e)
        sys.exit(3)

    default_check = {
        'warn': {
            'cpus': options.warn,
            'mem': options.warn,
            'disk': options.warn,
        },
        'crit': {
            'cpus': options.crit,
            'mem': options.crit,
            'disk': options.crit,
        },
    }

    failures = defaultdict(list)
    for usage_value in resource_use:
        check = get_check_from_overrides(overrides, default_check,
                                         usage_value['groupings'])
        usage_percent = calc_percent_usage(usage_value, value_to_check)
        for c in ['crit', 'warn']:
            if usage_percent > check[c][value_to_check]:
                failures[c].append({
                    'attrs': [{
                        'attr': a,
                        'value': v
                    } for a, v in usage_value['groupings'].items()],
                    'maximum':
                    check[c][value_to_check],
                    'current':
                    usage_percent,
                })
                break

    return_value = [0]
    if len(failures['crit']) > 0:
        result = error_message(failures['crit'], 'CRITICAL', cluster,
                               value_to_check)
        paasta_print(result)
        return_value.append(2)
    if len(failures['warn']) > 0:
        result = error_message(failures['warn'], 'WARNING', cluster,
                               value_to_check)
        paasta_print(result)
        return_value.append(1)

    if max(return_value) == 0:
        paasta_print("OK cluster %s is below critical capacity in %s" %
                     (cluster, value_to_check))

    sys.exit(max(return_value))
Пример #28
0
def _run_instance_worker(cluster_data, instances_out, green_light):
    """Get instances from the instances_in queue and check them one by one.

    If an instance isn't deployed, add it to the instances_out queue
    to re-check it later.

    :param cluster_data: an instance of ClusterData.
    :param instances_out: See the docstring for instances_deployed().
    :param green_light: See the docstring for _query_clusters().
    """

    api = client.get_paasta_api_client(cluster=cluster_data.cluster)
    if not api:
        log.warning("Couldn't reach the PaaSTA api for {}! Assuming it is not "
                    "deployed there yet.".format(cluster_data.cluster))
        while not cluster_data.instances_queue.empty():
            instances_out.put(cluster_data.instances_queue.get())
            cluster_data.instances_queue.task_done()

    while not cluster_data.instances_queue.empty() and green_light.is_set():
        instance = cluster_data.instances_queue.get()
        log.debug("Inspecting the deployment status of {}.{} on {}".format(
            cluster_data.service, instance, cluster_data.cluster))
        try:
            status = None
            status = api.service.status_instance(service=cluster_data.service,
                                                 instance=instance).result()
        except HTTPError as e:
            if e.response.status_code == 404:
                log.warning(
                    "Can't get status for instance {0}, service {1} in "
                    "cluster {2}. This is normally because it is a new "
                    "service that hasn't been deployed by PaaSTA yet".format(
                        instance, cluster_data.service, cluster_data.cluster))
            else:
                log.warning(
                    "Error getting service status from PaaSTA API: {0}:"
                    "{1}".format(e.response.status_code, e.response.text))
        except ConnectionError as e:
            log.warning("Error getting service status from PaaSTA API for {0}:"
                        "{1}".format(cluster_data.cluster, e))

        if not status:
            log.debug("No status for {}.{}, in {}. Not deployed yet.".format(
                cluster_data.service, instance, cluster_data.cluster))
            cluster_data.instances_queue.task_done()
            instances_out.put(instance)
        elif not status.marathon:
            log.debug("{}.{} in {} is not a Marathon job. Marked as deployed.".
                      format(cluster_data.service, instance,
                             cluster_data.cluster))
        elif (status.marathon.expected_instance_count == 0
              or status.marathon.desired_state == 'stop'):
            log.debug(
                "{}.{} in {} is marked as stopped. Marked as deployed.".format(
                    cluster_data.service, status.instance,
                    cluster_data.cluster))
        else:
            if status.marathon.app_count != 1:
                paasta_print("  {}.{} on {} is still bouncing, {} versions "
                             "running".format(cluster_data.service,
                                              status.instance,
                                              cluster_data.cluster,
                                              status.marathon.app_count))
                cluster_data.instances_queue.task_done()
                instances_out.put(instance)
                continue
            if not cluster_data.git_sha.startswith(status.git_sha):
                paasta_print(
                    "  {}.{} on {} doesn't have the right sha yet: {}".format(
                        cluster_data.service, instance, cluster_data.cluster,
                        status.git_sha))
                cluster_data.instances_queue.task_done()
                instances_out.put(instance)
                continue
            if status.marathon.deploy_status != 'Running':
                paasta_print("  {}.{} on {} isn't running yet: {}".format(
                    cluster_data.service, instance, cluster_data.cluster,
                    status.marathon.deploy_status))
                cluster_data.instances_queue.task_done()
                instances_out.put(instance)
                continue
            if (status.marathon.expected_instance_count !=
                    status.marathon.running_instance_count):
                paasta_print("  {}.{} on {} isn't scaled up yet, "
                             "has {} out of {}".format(
                                 cluster_data.service, instance,
                                 cluster_data.cluster,
                                 status.marathon.running_instance_count,
                                 status.marathon.expected_instance_count))
                cluster_data.instances_queue.task_done()
                instances_out.put(instance)
                continue
            paasta_print("Complete: {}.{} on {} looks 100% deployed at {} "
                         "instances on {}".format(
                             cluster_data.service, instance,
                             cluster_data.cluster,
                             status.marathon.running_instance_count,
                             status.git_sha))
            cluster_data.instances_queue.task_done()
Пример #29
0
def run_capacity_check():
    options = parse_capacity_check_options()
    system_paasta_config = load_system_paasta_config()
    cluster = (
        options.cluster
        if options.cluster is not None
        else system_paasta_config.get_cluster()
    )
    value_to_check = options.type

    client = get_paasta_api_client(cluster=cluster)
    if client is None:
        print("UNKNOWN Failed to load paasta api client")
        sys.exit(3)

    overrides = read_overrides(options.overrides)

    attributes = options.attributes.split(",")

    try:
        resource_use = client.resources.resources(groupings=attributes).result()
    except HTTPError as e:
        print("UNKNOWN received exception from paasta api:\n\t%s" % e)
        sys.exit(3)

    default_check = {
        "warn": {"cpus": options.warn, "mem": options.warn, "disk": options.warn},
        "crit": {"cpus": options.crit, "mem": options.crit, "disk": options.crit},
    }

    failures = defaultdict(list)
    for usage_value in resource_use:
        check = get_check_from_overrides(
            overrides, default_check, usage_value["groupings"]
        )
        usage_percent = calc_percent_usage(usage_value, value_to_check)
        for c in ["crit", "warn"]:
            if usage_percent > check[c][value_to_check]:
                failures[c].append(
                    {
                        "attrs": [
                            {"attr": a, "value": v}
                            for a, v in usage_value["groupings"].items()
                        ],
                        "maximum": check[c][value_to_check],
                        "current": usage_percent,
                    }
                )
                break

    return_value = [0]
    if len(failures["crit"]) > 0:
        result = error_message(failures["crit"], "CRITICAL", cluster, value_to_check)
        print(result)
        return_value.append(2)
    if len(failures["warn"]) > 0:
        result = error_message(failures["warn"], "WARNING", cluster, value_to_check)
        print(result)
        return_value.append(1)

    if max(return_value) == 0:
        print(f"OK cluster {cluster} is below critical capacity in {value_to_check}")

    sys.exit(max(return_value))