Exemple #1
0
def paasta_status(args):
    """Print the status of a Yelp service running on PaaSTA.
    :param args: argparse.Namespace obj created from sys.args by cli"""
    soa_dir = args.soa_dir
    service = figure_out_service_name(args, soa_dir)
    actual_deployments = get_actual_deployments(service, soa_dir)
    system_paasta_config = load_system_paasta_config()

    if args.clusters is not None:
        cluster_whitelist = args.clusters.split(",")
    else:
        cluster_whitelist = []
    if args.instances is not None:
        instance_whitelist = args.instances.split(",")
    else:
        instance_whitelist = []

    if actual_deployments:
        deploy_pipeline = list(get_planned_deployments(service, soa_dir))
        try:
            report_status(
                service=service,
                deploy_pipeline=deploy_pipeline,
                actual_deployments=actual_deployments,
                cluster_whitelist=cluster_whitelist,
                instance_whitelist=instance_whitelist,
                system_paasta_config=system_paasta_config,
                verbose=args.verbose,
            )
        except CalledProcessError as e:
            print PaastaColors.grey(PaastaColors.bold(e.cmd + " exited with non-zero return code."))
            print PaastaColors.grey(e.output)
            return e.returncode
    else:
        print missing_deployments_message(service)
Exemple #2
0
def paasta_rollback(args):
    """Call mark_for_deployment with rollback parameters
    :param args: contains all the arguments passed onto the script: service,
    deploy groups and sha. These arguments will be verified and passed onto
    mark_for_deployment.
    """
    service = figure_out_service_name(args)
    git_url = get_git_url(service)
    commit = args.commit
    given_deploy_groups = [deploy_group for deploy_group in args.deploy_groups.split(",") if deploy_group]

    service_deploy_groups = set(config.get_deploy_group() for config in get_instance_config_for_service(
        soa_dir=DEFAULT_SOA_DIR,
        service=service,
    ))
    deploy_groups, invalid = validate_given_deploy_groups(service_deploy_groups, given_deploy_groups)
    if len(invalid) > 0:
        print PaastaColors.yellow("These deploy groups are not valid and will be skipped: %s.\n" % (",").join(invalid))

    if len(deploy_groups) == 0:
        print PaastaColors.red("ERROR: No valid deploy groups specified for %s.\n" % (service))
        returncode = 1

    for deploy_group in deploy_groups:
        returncode = mark_for_deployment(
            git_url=git_url,
            service=service,
            deploy_group=deploy_group,
            commit=commit,
        )

    sys.exit(returncode)
def paasta_autoscale(args):
    log.setLevel(logging.DEBUG)
    service = figure_out_service_name(args)
    api = client.get_paasta_api_client(cluster=args.cluster, http_res=True)
    if not api:
        paasta_print(
            "Could not connect to paasta api. Maybe you misspelled the cluster?"
        )
        return 1

    if args.set is None:
        log.debug("Getting the current autoscaler count...")
        res, http = api.autoscaler.get_autoscaler_count(
            service=service, instance=args.instance).result()
    else:
        log.debug(f"Setting desired instances to {args.set}.")
        body = {"desired_instances": int(args.set)}
        res, http = api.autoscaler.update_autoscaler_count(
            service=service, instance=args.instance, json_body=body).result()

        _log_audit(
            action="manual-scale",
            action_details=body,
            service=service,
            instance=args.instance,
            cluster=args.cluster,
        )

    log.debug(f"Res: {res} Http: {http}")
    print(res["desired_instances"])
    return 0
def paasta_emergency_start(args):
    """Performs an emergency start on a given service instance on a given cluster

    Warning: This command is not magic and cannot actually get a service to start if it couldn't
    run before. This includes configurations that prevent the service from running,
    such as 'instances: 0' (for Marathon apps).

    All it does for Marathon apps is ask Marathon to resume normal operation by scaling up to
    the instance count defined in the service's config.
    All it does for Chronos jobs is send the latest version of the job config to Chronos and run it immediately.
    """
    system_paasta_config = load_system_paasta_config()
    service = figure_out_service_name(args, soa_dir=args.soa_dir)
    print "Performing an emergency start on %s..." % compose_job_id(service, args.instance)
    output = execute_paasta_serviceinit_on_remote_master(
        subcommand="start",
        cluster=args.cluster,
        service=service,
        instances=args.instance,
        system_paasta_config=system_paasta_config,
    )
    print "%s" % "\n".join(paasta_emergency_start.__doc__.splitlines()[-8:])
    print "Output: %s" % PaastaColors.grey(output)
    print "Run this command to see the status:"
    print "paasta status --service %s --clusters %s" % (service, args.cluster)
Exemple #5
0
def paasta_logs(args):
    """Print the logs for as Paasta service.
    :param args: argparse.Namespace obj created from sys.args by cli"""
    soa_dir = args.soa_dir
    service = figure_out_service_name(args, soa_dir)

    if args.clusters is None:
        clusters = list_clusters(service, soa_dir=soa_dir)
    else:
        clusters = args.clusters.split(",")

    if args.components is not None:
        components = args.components.split(",")
    else:
        components = DEFAULT_COMPONENTS

    if args.verbose:
        log.setLevel(logging.DEBUG)
        levels = [DEFAULT_LOGLEVEL, 'debug']
    else:
        log.setLevel(logging.WARNING)
        levels = [DEFAULT_LOGLEVEL]

    log.info("Going to get logs for %s on clusters %s" % (service, clusters))

    log_reader = get_log_reader()

    if args.tail:
        log_reader.tail_logs(service, levels, components, clusters, raw_mode=args.raw_mode)
    else:
        print "Non-tailing actions are not yet supported"
def paasta_emergency_stop(args):
    """Performs an emergency stop on a given service instance on a given cluster

    Warning: This command does not permanently stop the service. The next time the service is updated
    (config change, deploy, bounce, etc.), those settings will override the emergency stop.

    If you want this stop to be permanant, adjust the relevant config file to reflect that.
    For example, this can be done for Marathon apps by setting 'instances: 0', or
    for Chronos jobs by setting 'disabled: True'. Alternatively, remove the config yaml entirely.
    """
    system_paasta_config = load_system_paasta_config()
    service = figure_out_service_name(args, soa_dir=args.soa_dir)
    print "Performing an emergency stop on %s..." % compose_job_id(service, args.instance)
    output = execute_paasta_serviceinit_on_remote_master(
        subcommand="stop",
        cluster=args.cluster,
        service=service,
        instances=args.instance,
        system_paasta_config=system_paasta_config,
        app_id=args.appid,
    )
    print "Output: %s" % output
    print "%s" % "\n".join(paasta_emergency_stop.__doc__.splitlines()[-7:])
    print "To start this service again asap, run:"
    print "paasta emergency-start --service %s --instance %s --cluster %s" % (service, args.instance, args.cluster)
Exemple #7
0
def paasta_emergency_start(args):
    """Performs an emergency start on a given service instance on a given cluster

    Warning: This command is not magic and cannot actually get a service to start if it couldn't
    run before. This includes configurations that prevent the service from running,
    such as 'instances: 0' (for Marathon apps).

    All it does for Marathon apps is ask Marathon to resume normal operation by scaling up to
    the instance count defined in the service's config.
    All it does for Chronos jobs is send the latest version of the job config to Chronos and run it immediately.
    """
    system_paasta_config = load_system_paasta_config()
    service = figure_out_service_name(args, soa_dir=args.soa_dir)
    print "Performing an emergency start on %s..." % compose_job_id(
        service, args.instance)
    output = execute_paasta_serviceinit_on_remote_master(
        subcommand='start',
        cluster=args.cluster,
        service=service,
        instances=args.instance,
        system_paasta_config=system_paasta_config)
    print "%s" % "\n".join(paasta_emergency_start.__doc__.splitlines()[-8:])
    print "Output: %s" % PaastaColors.grey(output)
    print "Run this command to see the status:"
    print "paasta status --service %s --clusters %s" % (service, args.cluster)
Exemple #8
0
def paasta_status(args):
    """Print the status of a Yelp service running on PaaSTA.
    :param args: argparse.Namespace obj created from sys.args by cli"""
    service = figure_out_service_name(args)
    actual_deployments = get_actual_deployments(service)
    if args.clusters is not None:
        cluster_whitelist = args.clusters.split(",")
    else:
        cluster_whitelist = []
    if args.instances is not None:
        instance_whitelist = args.instances.split(",")
    else:
        instance_whitelist = []

    if actual_deployments:
        deploy_pipeline = list(get_planned_deployments(service))
        report_status(
            service=service,
            deploy_pipeline=deploy_pipeline,
            actual_deployments=actual_deployments,
            cluster_whitelist=cluster_whitelist,
            instance_whitelist=instance_whitelist,
            verbose=args.verbose,
        )
    else:
        print missing_deployments_message(service)
Exemple #9
0
def paasta_logs(args):
    """Print the logs for as Paasta service.
    :param args: argparse.Namespace obj created from sys.args by cli"""
    if scribereader is None:
        sys.exit(
            "Unfortunately, `paasta logs` is unavailable without Scribe."
            " We're working to support alternative logging backends in PaaSTA:"
            " follow https://github.com/Yelp/paasta/issues/64 for updates."
        )
    service = figure_out_service_name(args)

    if args.clusters is None:
        clusters = list_clusters(service)
    else:
        clusters = args.clusters.split(",")

    if args.components is not None:
        components = args.components.split(",")
    else:
        components = DEFAULT_COMPONENTS

    if args.debug:
        log.setLevel(logging.DEBUG)
        levels = [DEFAULT_LOGLEVEL, 'debug']
    else:
        log.setLevel(logging.WARNING)
        levels = [DEFAULT_LOGLEVEL]

    log.info("Going to get logs for %s on clusters %s" % (service, clusters))
    if args.tail:
        tail_paasta_logs(service, levels, components, clusters, raw_mode=args.raw_mode)
    else:
        print "Non-tailing actions are not yet supported"
Exemple #10
0
def paasta_emergency_stop(args):
    """Performs an emergency stop on a given service instance on a given cluster

    Warning: This command does not permanently stop the service. The next time the service is updated
    (config change, deploy, bounce, etc.), those settings will override the emergency stop.

    If you want this stop to be permanant, adjust the relevant config file to reflect that.
    For example, this can be done for Marathon apps by setting 'instances: 0', or
    for Chronos jobs by setting 'disabled: True'. Alternatively, remove the config yaml entirely.
    """
    system_paasta_config = load_system_paasta_config()
    service = figure_out_service_name(args, soa_dir=args.soa_dir)
    print "Performing an emergency stop on %s..." % compose_job_id(
        service, args.instance)
    output = execute_paasta_serviceinit_on_remote_master(
        subcommand='stop',
        cluster=args.cluster,
        service=args.service,
        instances=args.instance,
        system_paasta_config=system_paasta_config,
        app_id=args.appid)
    print "Output: %s" % output
    print "%s" % "\n".join(paasta_emergency_stop.__doc__.splitlines()[-7:])
    print "To start this service again asap, run:"
    print "paasta emergency-start --service %s --instance %s --cluster %s" % (
        service, args.instance, args.cluster)
Exemple #11
0
def paasta_start_or_stop(args, desired_state):
    """Requests a change of state to start or stop given branches of a service."""
    instance = args.instance
    cluster = args.cluster
    soa_dir = args.soa_dir
    service = figure_out_service_name(args=args, soa_dir=soa_dir)

    service_config = get_instance_config(
        service=service,
        cluster=cluster,
        instance=instance,
        soa_dir=soa_dir,
        load_deployments=False,
    )

    remote_refs = remote_git.list_remote_refs(utils.get_git_url(service))

    if 'refs/heads/paasta-%s' % service_config.get_deploy_group() not in remote_refs:
        print "No branches found for %s in %s." % \
            (service_config.get_deploy_group(), remote_refs)
        print "Has it been deployed there yet?"
        sys.exit(1)

    force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
    issue_state_change_for_service(
        service_config=service_config,
        force_bounce=force_bounce,
        desired_state=desired_state,
    )
Exemple #12
0
def paasta_emergency_start(args):
    """Performs an emergency start on a given service instance on a given cluster

    All it does for Chronos jobs is send the latest version of the job config to Chronos and run it immediately.
    """
    system_paasta_config = load_system_paasta_config()
    service = figure_out_service_name(args, soa_dir=args.soa_dir)
    paasta_print("Performing an emergency start on %s..." % compose_job_id(service, args.instance))
    return_code, output = execute_paasta_serviceinit_on_remote_master(
        subcommand='start',
        cluster=args.cluster,
        service=service,
        instances=args.instance,
        system_paasta_config=system_paasta_config,
    )
    _log_audit(
        action='emergency-start',
        service=service,
        cluster=args.cluster,
        instance=args.instance,
    )

    paasta_print("%s" % "\n".join(paasta_emergency_start.__doc__.splitlines()[-8:]))
    paasta_print("Output: %s" % PaastaColors.grey(output))
    paasta_print("Run this command to see the status:")
    paasta_print(f"paasta status --service {service} --clusters {args.cluster}")

    return return_code
Exemple #13
0
def paasta_emergency_stop(args):
    """Performs an emergency stop on a given service instance on a given cluster
    """
    system_paasta_config = load_system_paasta_config()
    service = figure_out_service_name(args, soa_dir=args.soa_dir)
    paasta_print("Performing an emergency stop on %s..." %
                 compose_job_id(service, args.instance))
    return_code, output = execute_paasta_serviceinit_on_remote_master(
        subcommand="stop",
        cluster=args.cluster,
        service=service,
        instances=args.instance,
        system_paasta_config=system_paasta_config,
    )
    _log_audit(
        action="emergency-stop",
        service=service,
        cluster=args.cluster,
        instance=args.instance,
    )

    paasta_print("Output: %s" % output)
    paasta_print("%s" %
                 "\n".join(paasta_emergency_stop.__doc__.splitlines()[-7:]))
    paasta_print("To start this service again asap, run:")
    paasta_print(
        f"paasta emergency-start --service {service} --instance {args.instance} --cluster {args.cluster}"
    )

    return return_code
Exemple #14
0
def paasta_logs(args):
    """Print the logs for as Paasta service.
    :param args: argparse.Namespace obj created from sys.args by cli"""
    service = figure_out_service_name(args)

    if args.clusters is None:
        clusters = list_clusters(service)
    else:
        clusters = args.clusters.split(",")

    if args.components is not None:
        components = args.components.split(",")
    else:
        components = DEFAULT_COMPONENTS

    if args.verbose:
        log.setLevel(logging.DEBUG)
        levels = [DEFAULT_LOGLEVEL, 'debug']
    else:
        log.setLevel(logging.WARNING)
        levels = [DEFAULT_LOGLEVEL]

    log.info("Going to get logs for %s on clusters %s" % (service, clusters))

    log_reader = get_log_reader()

    if args.tail:
        log_reader.tail_logs(service,
                             levels,
                             components,
                             clusters,
                             raw_mode=args.raw_mode)
    else:
        print "Non-tailing actions are not yet supported"
Exemple #15
0
def paasta_autoscale(args):
    log.setLevel(logging.DEBUG)
    service = figure_out_service_name(args)
    api = client.get_paasta_api_client(cluster=args.cluster, http_res=True)
    if not api:
        paasta_print(
            'Could not connect to paasta api. Maybe you misspelled the cluster?'
        )
        return 1

    if args.set is None:
        log.debug("Getting the current autoscaler count...")
        res, http = api.autoscaler.get_autoscaler_count(
            service=service, instance=args.instance).result()
    else:
        log.debug("Setting desired instances to {}.".format(args.set))
        body = {'desired_instances': int(args.set)}
        res, http = api.autoscaler.update_autoscaler_count(
            service=service,
            instance=args.instance,
            json_body=body,
        ).result()

    log.debug("Res: {} Http: {}".format(res, http))
    print(res["desired_instances"])
    return 0
Exemple #16
0
def paasta_emergency_restart(args):
    """Performs an emergency restart on a given service instance on a given cluster

    Warning: This command is only intended to be used in an emergency.
    It should not be needed in normal circumstances.
    """
    service = figure_out_service_name(args, args.soa_dir)
    system_paasta_config = load_system_paasta_config()
    paasta_print("Performing an emergency restart on %s...\n" %
                 compose_job_id(service, args.instance))
    return_code, output = execute_paasta_serviceinit_on_remote_master(
        subcommand='restart',
        cluster=args.cluster,
        service=args.service,
        instances=args.instance,
        system_paasta_config=system_paasta_config,
    )
    paasta_print("Output: %s" % output)
    paasta_print("%s" %
                 "\n".join(paasta_emergency_restart.__doc__.splitlines()[-7:]))
    paasta_print("Run this to see the status:")
    paasta_print(
        f"paasta status --service {service} --clusters {args.cluster}")

    return return_code
Exemple #17
0
def paasta_status(args):
    """Print the status of a Yelp service running on PaaSTA.
    :param args: argparse.Namespace obj created from sys.args by cli"""
    soa_dir = args.soa_dir
    service = figure_out_service_name(args, soa_dir)
    actual_deployments = get_actual_deployments(service, soa_dir)
    if 'USE_API_ENDPOINT' in os.environ:
        use_api_endpoint = strtobool(os.environ.get('USE_API_ENDPOINT'))
    else:
        use_api_endpoint = False

    pargs = paasta_args_mixer(args, service)
    if pargs is None:
        return 1

    if actual_deployments:
        deploy_pipeline = list(get_planned_deployments(service, soa_dir))
        return_code = report_status(
            service=service,
            deploy_pipeline=deploy_pipeline,
            actual_deployments=actual_deployments,
            cluster_whitelist=pargs.cluster_whitelist,
            instance_whitelist=pargs.instance_whitelist,
            system_paasta_config=load_system_paasta_config(),
            verbose=args.verbose,
            use_api_endpoint=use_api_endpoint)
        return return_code
    else:
        paasta_print(missing_deployments_message(service))
        return 1
Exemple #18
0
def paasta_status(args):
    """Print the status of a Yelp service running on PaaSTA.
    :param args: argparse.Namespace obj created from sys.args by cli"""
    soa_dir = args.soa_dir
    service = figure_out_service_name(args, soa_dir)
    actual_deployments = get_actual_deployments(service, soa_dir)
    system_paasta_config = load_system_paasta_config()

    if args.clusters is not None:
        cluster_whitelist = args.clusters.split(",")
    else:
        cluster_whitelist = []
    if args.instances is not None:
        instance_whitelist = args.instances.split(",")
    else:
        instance_whitelist = []

    if actual_deployments:
        deploy_pipeline = list(get_planned_deployments(service, soa_dir))
        report_status(
            service=service,
            deploy_pipeline=deploy_pipeline,
            actual_deployments=actual_deployments,
            cluster_whitelist=cluster_whitelist,
            instance_whitelist=instance_whitelist,
            system_paasta_config=system_paasta_config,
            verbose=args.verbose,
        )
    else:
        print missing_deployments_message(service)
Exemple #19
0
def paasta_start_or_stop(args, desired_state):
    """Requests a change of state to start or stop given branches of a service."""
    instance = args.instance
    cluster = args.cluster
    soa_dir = args.soa_dir
    service = figure_out_service_name(args=args, soa_dir=soa_dir)

    service_config = get_instance_config(
        service=service,
        cluster=cluster,
        instance=instance,
        soa_dir=soa_dir,
        load_deployments=False,
    )

    remote_refs = remote_git.list_remote_refs(utils.get_git_url(service))

    if 'refs/heads/paasta-%s' % service_config.get_deploy_group(
    ) not in remote_refs:
        print "No branches found for %s in %s." % \
            (service_config.get_deploy_group(), remote_refs)
        print "Has it been deployed there yet?"
        sys.exit(1)

    force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
    issue_state_change_for_service(
        service_config=service_config,
        force_bounce=force_bounce,
        desired_state=desired_state,
    )
Exemple #20
0
def apply_args_filters(
    args, ) -> Mapping[str, Mapping[str, Mapping[str, Type[InstanceConfig]]]]:
    """
    Take an args object and returns the dict of cluster:service:instances
    Currently, will filter by clusters, instances, services, and deploy_groups
    If no instances are found, will print a message and try to find matching instances
    for each service

    :param args: args object containing attributes to filter by
    :returns: Dict of dicts, in format {cluster_name: {service_name: {instance1, instance2}}}
    """
    clusters_services_instances: DefaultDict[str, DefaultDict[str, Dict[
        str, Type[InstanceConfig]]]] = defaultdict(lambda: defaultdict(dict))

    if args.service is None and args.owner is None:
        args.service = figure_out_service_name(args, soa_dir=args.soa_dir)

    filters = get_filters(args)

    all_services = list_services(soa_dir=args.soa_dir)

    if args.service and args.service not in all_services:
        paasta_print(
            PaastaColors.red(f'The service "{args.service}" does not exist.'))
        suggestions = difflib.get_close_matches(args.service,
                                                all_services,
                                                n=5,
                                                cutoff=0.5)
        if suggestions:
            paasta_print(PaastaColors.red(f'Did you mean any of these?'))
            for suggestion in suggestions:
                paasta_print(PaastaColors.red(f'  {suggestion}'))
        return clusters_services_instances

    i_count = 0
    for service in all_services:
        if args.service and service != args.service:
            continue

        for instance_conf in get_instance_configs_for_service(
                service, soa_dir=args.soa_dir):
            if all([f(instance_conf) for f in filters]):
                cluster_service = clusters_services_instances[
                    instance_conf.get_cluster()][service]
                cluster_service[
                    instance_conf.get_instance()] = instance_conf.__class__
                i_count += 1

    if i_count == 0 and args.service and args.instances:
        if args.clusters:
            clusters = args.clusters.split(',')
        else:
            clusters = list_clusters()
        for service in args.service.split(','):
            verify_instances(args.instances, service, clusters)

    return clusters_services_instances
Exemple #21
0
def extract_args(args):
    try:
        system_paasta_config = load_system_paasta_config()
    except PaastaNotConfiguredError:
        paasta_print(
            PaastaColors.yellow(
                "Warning: Couldn't load config files from '/etc/paasta'. This indicates"
                "PaaSTA is not configured locally on this host, and remote-run may not behave"
                "the same way it would behave on a server configured for PaaSTA.",
            ),
            sep='\n',
        )
        system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta')

    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
    cluster = args.cluster or system_paasta_config.get_local_run_config().get('default_cluster', None)

    if not cluster:
        paasta_print(
            PaastaColors.red(
                "PaaSTA on this machine has not been configured with a default cluster."
                "Please pass one using '-c'.",
            ),
            sep='\n',
            file=sys.stderr,
        )
        sys.exit(1)

    soa_dir = args.yelpsoa_config_root
    instance = args.instance
    if instance is None:
        instance_type = 'adhoc'
        instance = 'remote'
    else:
        instance_type = validate_service_instance(
            service, instance, cluster, soa_dir,
        )
        if instance_type != 'adhoc':
            paasta_print(
                PaastaColors.red(
                    (
                        "Please use instance declared in adhoc.yaml for use "
                        "with remote-run, {} is declared as {}"
                    ).format(instance, instance_type),
                ),
            )
            sys.exit(1)

    return (
        system_paasta_config,
        service,
        cluster,
        soa_dir,
        instance,
        instance_type,
    )
Exemple #22
0
def paasta_rollback(args):
    """Call mark_for_deployment with rollback parameters
    :param args: contains all the arguments passed onto the script: service,
    deploy groups and sha. These arguments will be verified and passed onto
    mark_for_deployment.
    """
    soa_dir = args.soa_dir
    service = figure_out_service_name(args, soa_dir)

    git_url = get_git_url(service, soa_dir)
    given_deploy_groups = {
        deploy_group
        for deploy_group in args.deploy_groups.split(",") if deploy_group
    }

    service_deploy_groups = {
        config.get_deploy_group()
        for config in get_instance_config_for_service(
            service=service,
            soa_dir=soa_dir,
        )
    }
    deploy_groups, invalid = validate_given_deploy_groups(
        service_deploy_groups, given_deploy_groups)

    if len(invalid) > 0:
        print PaastaColors.yellow(
            "These deploy groups are not valid and will be skipped: %s.\n" %
            (",").join(invalid))

    if len(deploy_groups) == 0:
        print PaastaColors.red(
            "ERROR: No valid deploy groups specified for %s.\n" % (service))
        return 1

    commit = args.commit
    if not commit:
        list_previous_commits(service, deploy_groups,
                              bool(given_deploy_groups), soa_dir)
        return 1

    returncode = 0

    for deploy_group in deploy_groups:
        returncode = max(
            mark_for_deployment(
                git_url=git_url,
                service=service,
                deploy_group=deploy_group,
                commit=commit,
            ),
            returncode,
        )

    return returncode
Exemple #23
0
def paasta_start_or_stop(args, desired_state):
    """Requests a change of state to start or stop given branches of a service."""
    instance = args.instance
    clusters = args.clusters
    soa_dir = args.soa_dir
    service = figure_out_service_name(args=args, soa_dir=soa_dir)

    if args.clusters is not None:
        clusters = args.clusters.split(",")
    else:
        clusters = list_clusters(service)

    try:
        remote_refs = remote_git.list_remote_refs(utils.get_git_url(service, soa_dir))
    except remote_git.LSRemoteException as e:
        msg = (
            "Error talking to the git server: %s\n"
            "This PaaSTA command requires access to the git server to operate.\n"
            "The git server may be down or not reachable from here.\n"
            "Try again from somewhere where the git server can be reached, "
            "like your developer environment."
        ) % str(e)
        print msg
        return 1

    invalid_deploy_groups = []
    for cluster in clusters:
        service_config = get_instance_config(
            service=service,
            cluster=cluster,
            instance=instance,
            soa_dir=soa_dir,
            load_deployments=False,
        )
        deploy_group = service_config.get_deploy_group()
        (deploy_tag, _) = get_latest_deployment_tag(remote_refs, deploy_group)

        if deploy_tag not in remote_refs:
            invalid_deploy_groups.append(deploy_group)
        else:
            force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
            issue_state_change_for_service(
                service_config=service_config,
                force_bounce=force_bounce,
                desired_state=desired_state,
            )

    return_val = 0
    if invalid_deploy_groups:
        print "No branches found for %s in %s." % \
            (", ".join(invalid_deploy_groups), remote_refs)
        print "Has %s been deployed there yet?" % service
        return_val = 1

    return return_val
def paasta_start_or_stop(args, desired_state):
    """Requests a change of state to start or stop given branches of a service."""
    instance = args.instance
    clusters = args.clusters
    soa_dir = args.soa_dir
    service = figure_out_service_name(args=args, soa_dir=soa_dir)

    if args.clusters is not None:
        clusters = args.clusters.split(",")
    else:
        clusters = list_clusters(service)

    try:
        remote_refs = remote_git.list_remote_refs(
            utils.get_git_url(service, soa_dir))
    except remote_git.LSRemoteException as e:
        msg = (
            "Error talking to the git server: %s\n"
            "This PaaSTA command requires access to the git server to operate.\n"
            "The git server may be down or not reachable from here.\n"
            "Try again from somewhere where the git server can be reached, "
            "like your developer environment.") % str(e)
        print msg
        return 1

    invalid_deploy_groups = []
    for cluster in clusters:
        service_config = get_instance_config(
            service=service,
            cluster=cluster,
            instance=instance,
            soa_dir=soa_dir,
            load_deployments=False,
        )
        deploy_group = service_config.get_deploy_group()
        (deploy_tag, _) = get_latest_deployment_tag(remote_refs, deploy_group)

        if deploy_tag not in remote_refs:
            invalid_deploy_groups.append(deploy_group)
        else:
            force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
            issue_state_change_for_service(
                service_config=service_config,
                force_bounce=force_bounce,
                desired_state=desired_state,
            )

    return_val = 0
    if invalid_deploy_groups:
        print "No branches found for %s in %s." % \
            (", ".join(invalid_deploy_groups), remote_refs)
        print "Has %s been deployed there yet?" % service
        return_val = 1

    return return_val
Exemple #25
0
def paasta_autoscale(args):
    log.setLevel(logging.DEBUG)
    service = figure_out_service_name(args)
    api = client.get_paasta_oapi_client(cluster=args.cluster, http_res=True)
    if not api:
        print(
            "Could not connect to paasta api. Maybe you misspelled the cluster?"
        )
        return 1

    try:
        if args.set is None:
            log.debug("Getting the current autoscaler count...")
            res, status, _ = api.autoscaler.get_autoscaler_count(
                service=service,
                instance=args.instance,
                _return_http_data_only=False)
        else:
            log.debug(f"Setting desired instances to {args.set}.")
            msg = paastamodels.AutoscalerCountMsg(
                desired_instances=int(args.set))
            res, status, _ = api.autoscaler.update_autoscaler_count(
                service=service,
                instance=args.instance,
                autoscaler_count_msg=msg,
                _return_http_data_only=False,
            )

            _log_audit(
                action="manual-scale",
                action_details=str(msg),
                service=service,
                instance=args.instance,
                cluster=args.cluster,
            )
    except api.api_error as exc:
        status = exc.status

    if not 200 <= status <= 299:
        print(
            PaastaColors.red(
                f"ERROR: '{args.instance}' is not configured to autoscale, "
                f"so paasta autoscale could not scale it up on demand. "
                f"If you want to be able to boost this service, please configure autoscaling for the service "
                f"in its config file by setting min and max instances. Example: \n"
                f"{args.instance}:\n"
                f"     min_instances: 5\n"
                f"     max_instances: 50"))
        return 0

    log.debug(f"Res: {res} Http: {status}")
    print(res.desired_instances)
    return 0
Exemple #26
0
def paasta_emergency_restart(args):
    """Performs an emergency restart on a given service instance on a given cluster

    Warning: This command is only intended to be used in an emergency.
    It should not be needed in normal circumstances.
    """
    service = figure_out_service_name(args)
    print "Performing an emergency restart on %s...\n" % compose_job_id(service, args.instance)
    execute_paasta_serviceinit_on_remote_master('restart', args.cluster, service, args.instance)
    print "%s" % "\n".join(paasta_emergency_restart.__doc__.splitlines()[-7:])
    print "Run this to see the status:"
    print "paasta status --service %s --clusters %s" % (service, args.cluster)
Exemple #27
0
def extract_args(args):
    system_paasta_config = get_system_paasta_config()
    soa_dir = args.yelpsoa_config_root
    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)

    cluster = args.cluster or \
        system_paasta_config.get_remote_run_config().get('default_cluster', None)
    if not cluster:
        paasta_print(
            PaastaColors.red(
                "PaaSTA on this machine has not been configured with a default cluster."
                "Please pass one using '-c'.", ))
        emit_counter_metric('paasta.remote_run.' + args.action + '.failed',
                            service, 'UNKNOWN')
        sys.exit(1)

    instance = args.instance
    if instance is None:
        instance_type = 'adhoc'
        instance = 'remote'
    else:
        try:
            instance_type = validate_service_instance(
                service,
                instance,
                cluster,
                soa_dir,
            )
        except NoConfigurationForServiceError as e:
            paasta_print(e)
            emit_counter_metric('paasta.remote_run.' + args.action + '.failed',
                                service, instance)
            sys.exit(1)

        if instance_type != 'adhoc':
            paasta_print(
                PaastaColors.red(
                    "Please use instance declared in adhoc.yaml for use "
                    f"with remote-run, {instance} is declared as {instance_type}",
                ))
            emit_counter_metric('paasta.remote_run.' + args.action + '.failed',
                                service, instance)
            sys.exit(1)

    return (
        system_paasta_config,
        service,
        cluster,
        soa_dir,
        instance,
        instance_type,
    )
Exemple #28
0
def paasta_rollback(args):
    """Call mark_for_deployment with rollback parameters
    :param args: contains all the arguments passed onto the script: service,
    deploy groups and sha. These arguments will be verified and passed onto
    mark_for_deployment.
    """
    soa_dir = args.soa_dir
    service = figure_out_service_name(args, soa_dir)

    git_url = get_git_url(service, soa_dir)
    given_deploy_groups = {deploy_group for deploy_group in args.deploy_groups.split(",") if deploy_group}

    all_deploy_groups = list_deploy_groups(service=service, soa_dir=soa_dir)
    deploy_groups, invalid = validate_given_deploy_groups(all_deploy_groups, given_deploy_groups)

    if len(invalid) > 0:
        paasta_print(
            PaastaColors.yellow(
                "These deploy groups are not valid and will be skipped: %s.\n" % (",").join(invalid),
            ),
        )

    if len(deploy_groups) == 0:
        paasta_print(PaastaColors.red("ERROR: No valid deploy groups specified for %s.\n" % (service)))
        return 1

    git_shas = get_git_shas_for_service(service, deploy_groups, soa_dir)
    commit = args.commit
    if not commit:
        paasta_print("Please specify a commit to mark for rollback (-k, --commit).")
        list_previous_commits(service, deploy_groups, bool(given_deploy_groups), git_shas)
        return 1
    elif commit not in git_shas and not args.force:
        paasta_print(PaastaColors.red("This Git SHA has never been deployed before."))
        paasta_print("Please double check it or use --force to skip this verification.\n")
        list_previous_commits(service, deploy_groups, bool(given_deploy_groups), git_shas)
        return 1

    returncode = 0

    for deploy_group in deploy_groups:
        returncode = max(
            mark_for_deployment(
                git_url=git_url,
                service=service,
                deploy_group=deploy_group,
                commit=commit,
            ),
            returncode,
        )

    return returncode
Exemple #29
0
def paasta_emergency_restart(args):
    """Performs an emergency restart on a given service instance on a given cluster

    Warning: This command is only intended to be used in an emergency.
    It should not be needed in normal circumstances.
    """
    service = figure_out_service_name(args)
    print "Performing an emergency restart on %s...\n" % compose_job_id(
        service, args.instance)
    execute_paasta_serviceinit_on_remote_master('restart', args.cluster,
                                                service, args.instance)
    print "%s" % "\n".join(paasta_emergency_restart.__doc__.splitlines()[-7:])
    print "Run this to see the status:"
    print "paasta status --service %s --clusters %s" % (service, args.cluster)
Exemple #30
0
def paasta_emergency_scale(args):
    """Performs an emergency scale on a given service instance on a given cluster

    Warning: This command does not permanently scale the service. The next time the service is updated
    (config change, deploy, bounce, etc.), those settings will override the emergency scale.

    If you want this scale to be permanant, adjust the relevant config file to reflect that.
    For example, this can be done for Marathon apps by setting 'instances: n'
    """
    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
    print "Performing an emergency scale on %s..." % compose_job_id(service, args.instance)
    output = execute_paasta_serviceinit_on_remote_master('scale', args.cluster, service, args.instance,
                                                         app_id=args.appid, delta=args.delta)
    print "Output: %s" % output
    print "%s" % "\n".join(paasta_emergency_scale.__doc__.splitlines()[-7:])
def paasta_emergency_scale(args):
    """Performs an emergency scale on a given service instance on a given cluster

    Warning: This command does not permanently scale the service. The next time the service is updated
    (config change, deploy, bounce, etc.), those settings will override the emergency scale.

    If you want this scale to be permanant, adjust the relevant config file to reflect that.
    For example, this can be done for Marathon apps by setting 'instances: n'
    """
    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
    print "Performing an emergency scale on %s..." % compose_job_id(service, args.instance)
    output = execute_paasta_serviceinit_on_remote_master('scale', args.cluster, service, args.instance,
                                                         app_id=args.appid, delta=args.delta)
    print "Output: %s" % output
    print "%s" % "\n".join(paasta_emergency_scale.__doc__.splitlines()[-7:])
Exemple #32
0
def paasta_local_run(args):
    if args.action == 'build' and not makefile_responds_to('cook-image'):
        sys.stderr.write(
            "A local Makefile with a 'cook-image' target is required for --build\n"
        )
        sys.stderr.write(
            "If you meant to pull the docker image from the registry, explicitly pass --pull\n"
        )
        return 1

    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
    cluster = guess_cluster(service=service, args=args)
    instance = guess_instance(service=service, cluster=cluster, args=args)
    docker_client = get_docker_client()

    if args.action == 'build':
        default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
        tag = os.environ.get('DOCKER_TAG', default_tag)
        os.environ['DOCKER_TAG'] = tag
        pull_image = False
        cook_return = paasta_cook_image(args=None,
                                        service=service,
                                        soa_dir=args.yelpsoa_config_root)
        if cook_return != 0:
            return cook_return
    elif args.action == 'dry_run':
        pull_image = False
        tag = None
    else:
        pull_image = True
        tag = None

    try:
        configure_and_run_docker_container(
            docker_client=docker_client,
            docker_hash=tag,
            service=service,
            instance=instance,
            cluster=cluster,
            args=args,
            pull_image=pull_image,
            dry_run=args.action == 'dry_run',
        )
    except errors.APIError as e:
        sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
        return 1
Exemple #33
0
def paasta_check(args):
    """Analyze the service in the PWD to determine if it is paasta ready
    :param args: argparse.Namespace obj created from sys.args by cli"""
    soa_dir = args.yelpsoa_config_root
    service = figure_out_service_name(args, soa_dir)
    service_path = os.path.join(soa_dir, service)

    service_dir_check(service, soa_dir)
    deploy_check(service_path)
    deploy_has_security_check(service, soa_dir)
    git_repo_check(service, soa_dir)
    docker_check()
    makefile_check()
    deployments_check(service, soa_dir)
    sensu_check(service, service_path, soa_dir)
    smartstack_check(service, service_path, soa_dir)
    paasta_validate_soa_configs(service, service_path)
Exemple #34
0
def paasta_autoscale(args):
    log.setLevel(logging.DEBUG)
    service = figure_out_service_name(args)
    api = client.get_paasta_api_client(cluster=args.cluster, http_res=True)
    if not api:
        paasta_print(
            "Could not connect to paasta api. Maybe you misspelled the cluster?"
        )
        return 1
    try:
        if args.set is None:
            log.debug("Getting the current autoscaler count...")
            res, http = api.autoscaler.get_autoscaler_count(
                service=service, instance=args.instance
            ).result()
        else:
            log.debug(f"Setting desired instances to {args.set}.")
            body = {"desired_instances": int(args.set)}
            res, http = api.autoscaler.update_autoscaler_count(
                service=service, instance=args.instance, json_body=body
            ).result()

            _log_audit(
                action="manual-scale",
                action_details=body,
                service=service,
                instance=args.instance,
                cluster=args.cluster,
            )
    except HTTPNotFound:
        paasta_print(
            PaastaColors.red(
                f"ERROR: '{args.instance}' is not configured to autoscale, "
                f"so paasta autoscale could not scale it up on demand. "
                f"If you want to be able to boost this service, please configure autoscaling for the service "
                f"in its config file by setting min and max instances. Example: \n"
                f"{args.instance}:\n"
                f"     min_instances: 5\n"
                f"     max_instances: 50"
            )
        )
        return 0

    log.debug(f"Res: {res} Http: {http}")
    print(res["desired_instances"])
    return 0
Exemple #35
0
def paasta_check(args):
    """Analyze the service in the PWD to determine if it is paasta ready
    :param args: argparse.Namespace obj created from sys.args by cli"""
    service = figure_out_service_name(args)
    service_path = os.path.join('/nail/etc/services', service)

    service_dir_check(service)
    deploy_check(service_path)
    deploy_has_security_check(service)
    deploy_has_performance_check(service)
    pipeline_check(service)
    git_repo_check(service)
    docker_check()
    makefile_check()
    yaml_check(service_path)
    deployments_check(service_path)
    sensu_check(service, service_path)
    smartstack_check(service, service_path)
Exemple #36
0
def paasta_local_run(args):
    if args.pull or args.dry_run:
        build = False
    elif args.build:
        build = True
    else:
        build = local_makefile_present()

    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
    cluster = guess_cluster(service=service, args=args)
    instance = guess_instance(service=service, cluster=cluster, args=args)
    docker_client = get_docker_client()

    if build:
        default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
        tag = os.environ.get('DOCKER_TAG', default_tag)
        os.environ['DOCKER_TAG'] = tag
        pull_image = False
        cook_return = paasta_cook_image(args=None,
                                        service=service,
                                        soa_dir=args.yelpsoa_config_root)
        if cook_return != 0:
            return cook_return
    elif args.dry_run:
        pull_image = False
        tag = None
    else:
        pull_image = True
        tag = None

    try:
        configure_and_run_docker_container(
            docker_client=docker_client,
            docker_hash=tag,
            service=service,
            instance=instance,
            cluster=cluster,
            args=args,
            pull_image=pull_image,
            dry_run=args.dry_run,
        )
    except errors.APIError as e:
        sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
        return 1
Exemple #37
0
def paasta_rollback(args):
    """Call mark_for_deployment with rollback parameters
    :param args: contains all the arguments passed onto the script: service,
    deploy groups and sha. These arguments will be verified and passed onto
    mark_for_deployment.
    """
    soa_dir = args.soa_dir
    service = figure_out_service_name(args, soa_dir)

    git_url = get_git_url(service, soa_dir)
    given_deploy_groups = {deploy_group for deploy_group in args.deploy_groups.split(",") if deploy_group}

    service_deploy_groups = {config.get_deploy_group() for config in get_instance_config_for_service(
        service=service,
        soa_dir=soa_dir,
    )}
    deploy_groups, invalid = validate_given_deploy_groups(service_deploy_groups, given_deploy_groups)

    if len(invalid) > 0:
        print PaastaColors.yellow("These deploy groups are not valid and will be skipped: %s.\n" % (",").join(invalid))

    if len(deploy_groups) == 0:
        print PaastaColors.red("ERROR: No valid deploy groups specified for %s.\n" % (service))
        return 1

    commit = args.commit
    if not commit:
        list_previous_commits(service, deploy_groups, bool(given_deploy_groups), soa_dir)
        return 1

    returncode = 0

    for deploy_group in deploy_groups:
        returncode = max(
            mark_for_deployment(
                git_url=git_url,
                service=service,
                deploy_group=deploy_group,
                commit=commit,
            ),
            returncode,
        )

    return returncode
def paasta_start_or_stop(args, desired_state):
    """Requests a change of state to start or stop given branches of a service."""
    service = figure_out_service_name(args)
    instance = args.instance
    cluster = args.cluster

    branch = "paasta-%s.%s" % (cluster, instance)
    all_branches = list(get_branches(service))

    if branch not in all_branches:
        print "No branches found for %s in %s." % \
            (branch, all_branches)
        print "Has it been deployed there yet?"
        sys.exit(1)

    force_bounce = format_timestamp(datetime.datetime.utcnow())
    branches = [branch]
    issue_state_change_for_branches(service, instance, cluster, branches, force_bounce,
                                    desired_state)
Exemple #39
0
def paasta_check(args):
    """Analyze the service in the PWD to determine if it is paasta ready
    :param args: argparse.Namespace obj created from sys.args by cli"""
    soa_dir = args.yelpsoa_config_root
    service = figure_out_service_name(args, soa_dir)
    service_path = os.path.join(soa_dir, service)

    service_dir_check(service, soa_dir)
    deploy_check(service_path)
    deploy_has_security_check(service, soa_dir)
    deploy_has_performance_check(service, soa_dir)
    git_repo_check(service)
    docker_check()
    makefile_check()
    yaml_check(service_path)
    deployments_check(service, soa_dir)
    sensu_check(service, service_path, soa_dir)
    smartstack_check(service, service_path, soa_dir)
    paasta_validate_soa_configs(service_path)
Exemple #40
0
def paasta_emergency_stop(args):
    """Performs an emergency stop on a given service instance on a given cluster
    """
    system_paasta_config = load_system_paasta_config()
    service = figure_out_service_name(args, soa_dir=args.soa_dir)
    print "Performing an emergency stop on %s..." % compose_job_id(service, args.instance)
    return_code, output = execute_paasta_serviceinit_on_remote_master(
        subcommand='stop',
        cluster=args.cluster,
        service=service,
        instances=args.instance,
        system_paasta_config=system_paasta_config,
    )
    print "Output: %s" % output
    print "%s" % "\n".join(paasta_emergency_stop.__doc__.splitlines()[-7:])
    print "To start this service again asap, run:"
    print "paasta emergency-start --service %s --instance %s --cluster %s" % (service, args.instance, args.cluster)

    return return_code
Exemple #41
0
def paasta_emergency_stop(args):
    """Performs an emergency stop on a given service instance on a given cluster
    """
    system_paasta_config = load_system_paasta_config()
    service = figure_out_service_name(args, soa_dir=args.soa_dir)
    print "Performing an emergency stop on %s..." % compose_job_id(
        service, args.instance)
    output = execute_paasta_serviceinit_on_remote_master(
        subcommand='stop',
        cluster=args.cluster,
        service=service,
        instances=args.instance,
        system_paasta_config=system_paasta_config,
    )
    print "Output: %s" % output
    print "%s" % "\n".join(paasta_emergency_stop.__doc__.splitlines()[-7:])
    print "To start this service again asap, run:"
    print "paasta emergency-start --service %s --instance %s --cluster %s" % (
        service, args.instance, args.cluster)
Exemple #42
0
def paasta_start_or_stop(args, desired_state):
    """Requests a change of state to start or stop given branches of a service."""
    service = figure_out_service_name(args)
    instance = args.instance
    cluster = args.cluster

    branch = "paasta-%s.%s" % (cluster, instance)
    all_branches = list(get_branches(service))

    if branch not in all_branches:
        print "No branches found for %s in %s." % \
            (branch, all_branches)
        print "Has it been deployed there yet?"
        sys.exit(1)

    force_bounce = format_timestamp(datetime.datetime.utcnow())
    branches = [branch]
    issue_state_change_for_branches(service, instance, cluster, branches,
                                    force_bounce, desired_state)
Exemple #43
0
def paasta_local_run(args):
    if args.pull or args.dry_run:
        build = False
    elif args.build:
        build = True
    else:
        build = local_makefile_present()

    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
    cluster = guess_cluster(service=service, args=args)
    instance = guess_instance(service=service, cluster=cluster, args=args)
    docker_client = get_docker_client()

    if build:
        default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
        tag = os.environ.get('DOCKER_TAG', default_tag)
        os.environ['DOCKER_TAG'] = tag
        pull_image = False
        cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
        if cook_return != 0:
            return cook_return
    elif args.dry_run:
        pull_image = False
        tag = None
    else:
        pull_image = True
        tag = None

    try:
        configure_and_run_docker_container(
            docker_client=docker_client,
            docker_hash=tag,
            service=service,
            instance=instance,
            cluster=cluster,
            args=args,
            pull_image=pull_image,
            dry_run=args.dry_run,
        )
    except errors.APIError as e:
        sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
        return 1
Exemple #44
0
def paasta_rollback(args):
    """Call mark_for_deployment with rollback parameters
    :param args: contains all the arguments passed onto the script: service,
    cluster, instance and sha. These arguments will be verified and passed onto
    mark_for_deployment.
    """
    service = figure_out_service_name(args)
    cluster = args.cluster
    git_url = get_git_url(service)
    commit = args.commit
    given_instances = args.instances.split(",")

    if cluster in list_clusters(service):
        service_instances = list_all_instances_for_service(service)
        instances, invalid = validate_given_instances(service_instances,
                                                      given_instances)

        if len(invalid) > 0:
            print PaastaColors.yellow(
                "These instances are not valid and will be skipped: %s.\n" %
                (",").join(invalid))

        if len(instances) is 0:
            print PaastaColors.red(
                "ERROR: No valid instances specified for %s.\n" % (service))
            returncode = 1

        for instance in instances:
            returncode = mark_for_deployment(
                git_url=git_url,
                cluster=cluster,
                instance=instance,
                service=service,
                commit=commit,
            )
    else:
        print PaastaColors.red(
            "ERROR: The service %s is not deployed into cluster %s.\n" %
            (service, cluster))
        returncode = 1

    sys.exit(returncode)
Exemple #45
0
def paasta_emergency_start(args):
    """Performs an emergency start on a given service instance on a given cluster

    All it does for Chronos jobs is send the latest version of the job config to Chronos and run it immediately.
    """
    system_paasta_config = load_system_paasta_config()
    service = figure_out_service_name(args, soa_dir=args.soa_dir)
    paasta_print("Performing an emergency start on %s..." % compose_job_id(service, args.instance))
    return_code, output = execute_paasta_serviceinit_on_remote_master(
        subcommand='start',
        cluster=args.cluster,
        service=service,
        instances=args.instance,
        system_paasta_config=system_paasta_config
    )
    paasta_print("%s" % "\n".join(paasta_emergency_start.__doc__.splitlines()[-8:]))
    paasta_print("Output: %s" % PaastaColors.grey(output))
    paasta_print("Run this command to see the status:")
    paasta_print("paasta status --service %s --clusters %s" % (service, args.cluster))

    return return_code
Exemple #46
0
def paasta_rollback(args):
    """Call mark_for_deployment with rollback parameters
    :param args: contains all the arguments passed onto the script: service,
    deploy groups and sha. These arguments will be verified and passed onto
    mark_for_deployment.
    """
    service = figure_out_service_name(args)
    git_url = get_git_url(service)
    commit = args.commit
    given_deploy_groups = [
        deploy_group for deploy_group in args.deploy_groups.split(",")
        if deploy_group
    ]

    service_deploy_groups = set(config.get_deploy_group()
                                for config in get_instance_config_for_service(
                                    soa_dir=DEFAULT_SOA_DIR,
                                    service=service,
                                ))
    deploy_groups, invalid = validate_given_deploy_groups(
        service_deploy_groups, given_deploy_groups)
    if len(invalid) > 0:
        print PaastaColors.yellow(
            "These deploy groups are not valid and will be skipped: %s.\n" %
            (",").join(invalid))

    if len(deploy_groups) == 0:
        print PaastaColors.red(
            "ERROR: No valid deploy groups specified for %s.\n" % (service))
        returncode = 1

    for deploy_group in deploy_groups:
        returncode = mark_for_deployment(
            git_url=git_url,
            service=service,
            deploy_group=deploy_group,
            commit=commit,
        )

    sys.exit(returncode)
Exemple #47
0
def paasta_local_run(args):
    if args.action == 'build' and not makefile_responds_to('cook-image'):
        sys.stderr.write("A local Makefile with a 'cook-image' target is required for --build\n")
        sys.stderr.write("If you meant to pull the docker image from the registry, explicitly pass --pull\n")
        return 1

    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
    cluster = guess_cluster(service=service, args=args)
    instance = guess_instance(service=service, cluster=cluster, args=args)
    docker_client = get_docker_client()

    if args.action == 'build':
        default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
        tag = os.environ.get('DOCKER_TAG', default_tag)
        os.environ['DOCKER_TAG'] = tag
        pull_image = False
        cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
        if cook_return != 0:
            return cook_return
    elif args.action == 'dry_run':
        pull_image = False
        tag = None
    else:
        pull_image = True
        tag = None

    try:
        configure_and_run_docker_container(
            docker_client=docker_client,
            docker_hash=tag,
            service=service,
            instance=instance,
            cluster=cluster,
            args=args,
            pull_image=pull_image,
            dry_run=args.action == 'dry_run',
        )
    except errors.APIError as e:
        sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
        return 1
Exemple #48
0
def paasta_start_or_stop(args, desired_state):
    """Requests a change of state to start or stop given branches of a service."""
    instance = args.instance
    cluster = args.cluster
    soa_dir = args.soa_dir
    service = figure_out_service_name(args=args, soa_dir=soa_dir)

    service_config = get_instance_config(
        service=service,
        cluster=cluster,
        instance=instance,
        soa_dir=soa_dir,
        load_deployments=False,
    )

    try:
        remote_refs = remote_git.list_remote_refs(utils.get_git_url(service))
    except remote_git.LSRemoteException as e:
        msg = (
            "Error talking to the git server: %s\n"
            "This PaaSTA command requires access to the git server to operate.\n"
            "The git server may be down or not reachable from here.\n"
            "Try again from somewhere where the git server can be reached, "
            "like your developer environment."
        ) % str(e)
        print msg
        return 1

    if 'refs/heads/paasta-%s' % service_config.get_deploy_group() not in remote_refs:
        print "No branches found for %s in %s." % \
            (service_config.get_deploy_group(), remote_refs)
        print "Has it been deployed there yet?"
        return 1

    force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
    issue_state_change_for_service(
        service_config=service_config,
        force_bounce=force_bounce,
        desired_state=desired_state,
    )
Exemple #49
0
def paasta_status(args):
    """Print the status of a Yelp service running on PaaSTA.
    :param args: argparse.Namespace obj created from sys.args by cli"""
    soa_dir = args.soa_dir
    service = figure_out_service_name(args, soa_dir)
    actual_deployments = get_actual_deployments(service, soa_dir)
    system_paasta_config = load_system_paasta_config()
    if 'USE_API_ENDPOINT' in os.environ:
        use_api_endpoint = strtobool(os.environ.get('USE_API_ENDPOINT'))
    else:
        use_api_endpoint = False

    if args.clusters is not None:
        cluster_whitelist = args.clusters.split(",")
    else:
        cluster_whitelist = []
    if args.instances is not None:
        instance_whitelist = args.instances.split(",")
    else:
        instance_whitelist = []

    if actual_deployments:
        deploy_pipeline = list(get_planned_deployments(service, soa_dir))
        try:
            report_status(service=service,
                          deploy_pipeline=deploy_pipeline,
                          actual_deployments=actual_deployments,
                          cluster_whitelist=cluster_whitelist,
                          instance_whitelist=instance_whitelist,
                          system_paasta_config=system_paasta_config,
                          verbose=args.verbose,
                          use_api_endpoint=use_api_endpoint)
        except CalledProcessError as e:
            print PaastaColors.grey(
                PaastaColors.bold(e.cmd +
                                  " exited with non-zero return code."))
            print PaastaColors.grey(e.output)
            return e.returncode
    else:
        print missing_deployments_message(service)
Exemple #50
0
def paasta_emergency_restart(args):
    """Performs an emergency restart on a given service instance on a given cluster

    Warning: This command is only intended to be used in an emergency.
    It should not be needed in normal circumstances.
    """
    service = figure_out_service_name(args, args.soa_dir)
    system_paasta_config = load_system_paasta_config()
    paasta_print("Performing an emergency restart on %s...\n" % compose_job_id(service, args.instance))
    return_code, output = execute_paasta_serviceinit_on_remote_master(
        subcommand='restart',
        cluster=args.cluster,
        service=args.service,
        instances=args.instance,
        system_paasta_config=system_paasta_config
    )
    paasta_print("Output: %s" % output)
    paasta_print("%s" % "\n".join(paasta_emergency_restart.__doc__.splitlines()[-7:]))
    paasta_print("Run this to see the status:")
    paasta_print("paasta status --service %s --clusters %s" % (service, args.cluster))

    return return_code
Exemple #51
0
def apply_args_filters(args):
    """
    Take an args object and returns the dict of cluster:service:instances
    Currently, will filter by clusters, instances, services, and deploy_groups
    If no instances are found, will print a message and try to find matching instances
    for each service

    :param args: args object containing attributes to filter by
    :returns: Dict of dicts, in format {cluster_name: {service_name: {instance1, instance2}}}
    """
    clusters_services_instances = defaultdict(lambda: defaultdict(set))

    if args.service is None and args.owner is None:
        args.service = figure_out_service_name(args, soa_dir=args.soa_dir)

    filters = get_filters(args)

    i_count = 0
    for service in list_services(soa_dir=args.soa_dir):
        if args.service and service != args.service:
            continue

        for instance_conf in get_instance_configs_for_service(
                service, soa_dir=args.soa_dir):
            if all([f(instance_conf) for f in filters]):
                clusters_services_instances[
                    instance_conf.get_cluster()][service].add(
                        instance_conf.get_instance())
                i_count += 1

    if i_count == 0 and args.service and args.instances:
        if args.clusters:
            clusters = args.clusters.split(',')
        else:
            clusters = list_clusters()
        for service in args.service.split(','):
            verify_instances(args.instances, service, clusters)

    return clusters_services_instances
Exemple #52
0
def paasta_local_run(args):
    if args.pull:
        build = False
    elif args.build:
        build = True
    else:
        build = local_makefile_present()

    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
    cluster = guess_cluster(service=service, args=args)
    instance = guess_instance(service=service, cluster=cluster, args=args)
    base_docker_url = get_docker_host()
    docker_client = Client(base_url=base_docker_url)

    if build:
        default_tag = "paasta-local-run-%s-%s" % (service, get_username())
        tag = os.environ.get("DOCKER_TAG", default_tag)
        os.environ["DOCKER_TAG"] = tag
        pull_image = False
        paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
    else:
        pull_image = True
        tag = None

    try:
        configure_and_run_docker_container(
            docker_client=docker_client,
            docker_hash=tag,
            service=service,
            instance=instance,
            cluster=cluster,
            args=args,
            pull_image=pull_image,
        )
    except errors.APIError as e:
        sys.stderr.write("Can't run Docker container. Error: %s\n" % str(e))
        sys.exit(1)
Exemple #53
0
def paasta_status(args):
    """Print the status of a Yelp service running on PaaSTA.
    :param args: argparse.Namespace obj created from sys.args by cli"""
    soa_dir = args.soa_dir
    service = figure_out_service_name(args, soa_dir)
    actual_deployments = get_actual_deployments(service, soa_dir)
    system_paasta_config = load_system_paasta_config()
    if 'USE_API_ENDPOINT' in os.environ:
        use_api_endpoint = strtobool(os.environ.get('USE_API_ENDPOINT'))
    else:
        use_api_endpoint = False

    if args.clusters is not None:
        cluster_whitelist = args.clusters.split(",")
    else:
        cluster_whitelist = []
    if args.instances is not None:
        instance_whitelist = args.instances.split(",")
    else:
        instance_whitelist = []

    if actual_deployments:
        deploy_pipeline = list(get_planned_deployments(service, soa_dir))
        return_code = report_status(
            service=service,
            deploy_pipeline=deploy_pipeline,
            actual_deployments=actual_deployments,
            cluster_whitelist=cluster_whitelist,
            instance_whitelist=instance_whitelist,
            system_paasta_config=system_paasta_config,
            verbose=args.verbose,
            use_api_endpoint=use_api_endpoint
        )
        return return_code
    else:
        paasta_print(missing_deployments_message(service))
        return 1
Exemple #54
0
def paasta_rollback(args):
    """Call mark_for_deployment with rollback parameters
    :param args: contains all the arguments passed onto the script: service,
    cluster, instance and sha. These arguments will be verified and passed onto
    mark_for_deployment.
    """
    service = figure_out_service_name(args)
    cluster = args.cluster
    git_url = get_git_url(service)
    commit = args.commit
    given_instances = args.instances.split(",")

    if cluster in list_clusters(service):
        service_instances = list_all_instances_for_service(service)
        instances, invalid = validate_given_instances(service_instances, given_instances)

        if len(invalid) > 0:
            print PaastaColors.yellow("These instances are not valid and will be skipped: %s.\n" % (",").join(invalid))

        if len(instances) is 0:
            print PaastaColors.red("ERROR: No valid instances specified for %s.\n" % (service))
            returncode = 1

        for instance in instances:
            returncode = mark_for_deployment(
                git_url=git_url,
                cluster=cluster,
                instance=instance,
                service=service,
                commit=commit,
            )
    else:
        print PaastaColors.red("ERROR: The service %s is not deployed into cluster %s.\n" % (service, cluster))
        returncode = 1

    sys.exit(returncode)
def paasta_start_or_stop(args, desired_state):
    """Requests a change of state to start or stop given branches of a service."""
    soa_dir = args.soa_dir
    service = figure_out_service_name(args=args, soa_dir=soa_dir)

    if args.clusters is not None:
        clusters = args.clusters.split(",")
    else:
        clusters = list_clusters(service)

    if args.instances is not None:
        instances = args.instances.split(",")
    else:
        instances = None

    try:
        remote_refs = remote_git.list_remote_refs(utils.get_git_url(service, soa_dir))
    except remote_git.LSRemoteException as e:
        msg = (
            "Error talking to the git server: %s\n"
            "This PaaSTA command requires access to the git server to operate.\n"
            "The git server may be down or not reachable from here.\n"
            "Try again from somewhere where the git server can be reached, "
            "like your developer environment."
        ) % str(e)
        print msg
        return 1

    invalid_deploy_groups = []
    marathon_message_printed, chronos_message_printed = False, False
    for cluster in clusters:
        # If they haven't specified what instances to act on, do it for all of them.
        # If they have specified what instances, only iterate over them if they're
        # actually within this cluster.
        if instances is None:
            cluster_instances = list_all_instances_for_service(service, clusters=[cluster], soa_dir=soa_dir)
        else:
            all_cluster_instances = list_all_instances_for_service(service, clusters=[cluster], soa_dir=soa_dir)
            cluster_instances = all_cluster_instances.intersection(set(instances))

        for instance in cluster_instances:
            service_config = get_instance_config(
                service=service,
                cluster=cluster,
                instance=instance,
                soa_dir=soa_dir,
                load_deployments=False,
            )
            deploy_group = service_config.get_deploy_group()
            (deploy_tag, _) = get_latest_deployment_tag(remote_refs, deploy_group)

            if deploy_tag not in remote_refs:
                invalid_deploy_groups.append(deploy_group)
            else:
                force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
                if isinstance(service_config, MarathonServiceConfig) and not marathon_message_printed:
                    print_marathon_message(desired_state)
                    marathon_message_printed = True
                elif isinstance(service_config, ChronosJobConfig) and not chronos_message_printed:
                    print_chronos_message(desired_state)
                    chronos_message_printed = True

                issue_state_change_for_service(
                    service_config=service_config,
                    force_bounce=force_bounce,
                    desired_state=desired_state,
                )

    return_val = 0
    if invalid_deploy_groups:
        print "No branches found for %s in %s." % \
            (", ".join(invalid_deploy_groups), remote_refs)
        print "Has %s been deployed there yet?" % service
        return_val = 1

    return return_val
Exemple #56
0
def paasta_local_run(args):
    if args.action == 'build' and not makefile_responds_to('cook-image'):
        sys.stderr.write("A local Makefile with a 'cook-image' target is required for --build\n")
        sys.stderr.write("If you meant to pull the docker image from the registry, explicitly pass --pull\n")
        return 1

    try:
        system_paasta_config = load_system_paasta_config()
    except PaastaNotConfiguredError:
        sys.stdout.write(PaastaColors.yellow(
            "Warning: Couldn't load config files from '/etc/paasta'. This indicates\n"
            "PaaSTA is not configured locally on this host, and local-run may not behave\n"
            "the same way it would behave on a server configured for PaaSTA.\n"
        ))
        system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta')

    local_run_config = system_paasta_config.get_local_run_config()

    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
    if args.cluster:
        cluster = args.cluster
    else:
        try:
            cluster = local_run_config['default_cluster']
        except KeyError:
            sys.stderr.write(PaastaColors.red(
                "PaaSTA on this machine has not been configured with a default cluster.\n"
                "Please pass one to local-run using '-c'.\n"))
            return 1
    instance = args.instance
    docker_client = get_docker_client()

    if args.action == 'build':
        default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
        tag = os.environ.get('DOCKER_TAG', default_tag)
        os.environ['DOCKER_TAG'] = tag
        pull_image = False
        cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
        if cook_return != 0:
            return cook_return
    elif args.action == 'dry_run':
        pull_image = False
        tag = None
    else:
        pull_image = True
        tag = None

    try:
        configure_and_run_docker_container(
            docker_client=docker_client,
            docker_hash=tag,
            service=service,
            instance=instance,
            cluster=cluster,
            args=args,
            pull_image=pull_image,
            system_paasta_config=system_paasta_config,
            dry_run=args.action == 'dry_run',
        )
    except errors.APIError as e:
        sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
        return 1
Exemple #57
0
def paasta_logs(args):
    """Print the logs for as Paasta service.
    :param args: argparse.Namespace obj created from sys.args by cli"""
    soa_dir = args.soa_dir
    service = figure_out_service_name(args, soa_dir)

    if args.clusters is None:
        clusters = list_clusters(service, soa_dir=soa_dir)
    else:
        clusters = args.clusters.split(",")

    if args.instances is None:
        instances = None
    else:
        instances = args.instances.split(",")

    if args.components is not None:
        components = args.components.split(",")
    else:
        components = DEFAULT_COMPONENTS
    components = set(components)

    if "app_output" in components:
        components.remove("app_output")
        components.add("stdout")
        components.add("stderr")

    if args.verbose:
        log.setLevel(logging.DEBUG)
    else:
        log.setLevel(logging.WARNING)

    levels = [DEFAULT_LOGLEVEL, "debug"]

    log.info("Going to get logs for %s on clusters %s" % (service, clusters))

    log_reader = get_log_reader()

    if not validate_filtering_args(args, log_reader):
        return 1

    # They haven't specified what kind of filtering they want, decide for them
    if args.line_count is None and args.time_from is None and not args.tail:
        return pick_default_log_mode(args, log_reader, service, levels, components, clusters, instances)

    if args.tail:
        paasta_print(PaastaColors.cyan("Tailing logs"), file=sys.stderr)
        log_reader.tail_logs(
            service=service,
            levels=levels,
            components=components,
            clusters=clusters,
            instances=instances,
            raw_mode=args.raw_mode,
        )
        return 0

    # If the logger doesn't support offsetting the number of lines by a particular line number
    # there is no point in distinguishing between a positive/negative number of lines since it
    # can only get the last N lines
    if not log_reader.SUPPORTS_LINE_OFFSET and args.line_count is not None:
        args.line_count = abs(args.line_count)

    # Handle line based filtering
    if args.line_count is not None and args.line_offset is None:
        log_reader.print_last_n_logs(
            service=service,
            line_count=args.line_count,
            levels=levels,
            components=components,
            clusters=clusters,
            instances=instances,
            raw_mode=args.raw_mode,
        )
        return 0
    elif args.line_count is not None and args.line_offset is not None:
        log_reader.print_logs_by_offset(
            service=service,
            line_count=args.line_count,
            line_offset=args.line_offset,
            levels=levels,
            components=components,
            cluters=clusters,
            instances=instances,
            raw_mode=args.raw_mode,
        )
        return 0

    # Handle time based filtering
    try:
        start_time, end_time = generate_start_end_time(args.time_from, args.time_to)
    except ValueError as e:
        paasta_print(PaastaColors.red(e.message), file=sys.stderr)
        return 1

    log_reader.print_logs_by_time(
        service=service,
        start_time=start_time,
        end_time=end_time,
        levels=levels,
        components=components,
        clusters=clusters,
        instances=instances,
        raw_mode=args.raw_mode,
    )
Exemple #58
0
def paasta_info(args):
    """Prints general information about a service"""
    soa_dir = args.soa_dir
    service = figure_out_service_name(args, soa_dir=soa_dir)
    print get_service_info(service, soa_dir)
Exemple #59
0
def paasta_rerun(args):
    """Reruns a Chronos job.
    :param args: argparse.Namespace obj created from sys.args by cli"""
    soa_dir = args.soa_dir
    service = figure_out_service_name(args, soa_dir)  # exit with an error if the service doesn't exist
    if args.execution_date:
        execution_date = args.execution_date
    else:
        execution_date = None

    all_clusters = list_clusters(soa_dir=soa_dir)
    actual_deployments = get_actual_deployments(service, soa_dir)  # cluster.instance: sha
    if actual_deployments:
        deploy_pipeline = list(get_planned_deployments(service, soa_dir))  # cluster.instance
        deployed_clusters = list_deployed_clusters(deploy_pipeline, actual_deployments)
        deployed_cluster_instance = _get_cluster_instance(actual_deployments.keys())

    if args.clusters is not None:
        clusters = args.clusters.split(",")
    else:
        clusters = deployed_clusters

    for cluster in clusters:
        print "cluster: %s" % cluster

        if cluster not in all_clusters:
            print "  Warning: \"%s\" does not look like a valid cluster." % cluster
            continue
        if cluster not in deployed_clusters:
            print "  Warning: service \"%s\" has not been deployed to \"%s\" yet." % (service, cluster)
            continue
        if not deployed_cluster_instance[cluster].get(args.instance, False):
            print ("  Warning: instance \"%s\" is either invalid "
                   "or has not been deployed to \"%s\" yet." % (args.instance, cluster))
            continue

        try:
            chronos_job_config = chronos_tools.load_chronos_job_config(
                service, args.instance, cluster, load_deployments=False, soa_dir=soa_dir)
            if chronos_tools.uses_time_variables(chronos_job_config) and execution_date is None:
                print ("  Warning: \"%s\" uses time variables interpolation, "
                       "please supply a `--execution_date` argument." % args.instance)
                continue
        except chronos_tools.UnknownChronosJobError as e:
            print "  Warning: %s" % e.message
            continue
        if execution_date is None:
            execution_date = _get_default_execution_date()

        rc, output = execute_chronos_rerun_on_remote_master(
            service=service,
            instancename=args.instance,
            cluster=cluster,
            verbose=args.verbose,
            execution_date=execution_date.strftime(chronos_tools.EXECUTION_DATE_FORMAT)
        )
        if rc == 0:
            print PaastaColors.green('  successfully created job')
        else:
            print PaastaColors.red('  error')
            print output
def paasta_start_or_stop(args, desired_state):
    """Requests a change of state to start or stop given branches of a service."""
    soa_dir = args.soa_dir
    service = figure_out_service_name(args=args, soa_dir=soa_dir)
    instances = args.instances.split(",") if args.instances else None

    # assert that each of the clusters that the user specifies are 'valid'
    # for the instance list provided; that is, assert that at least one of the instances
    # provided in the -i argument is deployed there.
    # if there are no instances defined in the args, then assert that the service
    # is deployed to that cluster.
    # If args.clusters is falsey, then default to *all* clusters that a service is deployed to,
    # and we figure out which ones are needed for each service later.
    if instances:
        instance_clusters = [list_clusters(service, soa_dir, instance) for instance in args.instances]
        valid_clusters = sorted(list(set([cluster for cluster_list in instance_clusters for cluster in cluster_list])))
    else:
        valid_clusters = list_clusters(service, soa_dir)

    if args.clusters:
        clusters = args.clusters.split(",")
        invalid_clusters = [cluster for cluster in clusters if cluster not in valid_clusters]
        if invalid_clusters:
            print ("Invalid cluster name(s) specified: %s." "Valid options: %s") % (
                " ".join(invalid_clusters),
                " ".join(valid_clusters),
            )
            return 1
    else:
        clusters = valid_clusters

    try:
        remote_refs = remote_git.list_remote_refs(utils.get_git_url(service, soa_dir))
    except remote_git.LSRemoteException as e:
        msg = (
            "Error talking to the git server: %s\n"
            "This PaaSTA command requires access to the git server to operate.\n"
            "The git server may be down or not reachable from here.\n"
            "Try again from somewhere where the git server can be reached, "
            "like your developer environment."
        ) % str(e)
        print msg
        return 1

    invalid_deploy_groups = []
    marathon_message_printed, chronos_message_printed = False, False
    for cluster in clusters:
        # If they haven't specified what instances to act on, do it for all of them.
        # If they have specified what instances, only iterate over them if they're
        # actually within this cluster.
        if instances is None:
            cluster_instances = list_all_instances_for_service(service, clusters=[cluster], soa_dir=soa_dir)
            print ("no instances specified; restarting all instances for service")
        else:
            all_cluster_instances = list_all_instances_for_service(service, clusters=[cluster], soa_dir=soa_dir)
            cluster_instances = all_cluster_instances.intersection(set(instances))

        for instance in cluster_instances:
            service_config = get_instance_config(
                service=service, cluster=cluster, instance=instance, soa_dir=soa_dir, load_deployments=False
            )
            deploy_group = service_config.get_deploy_group()
            (deploy_tag, _) = get_latest_deployment_tag(remote_refs, deploy_group)

            if deploy_tag not in remote_refs:
                invalid_deploy_groups.append(deploy_group)
            else:
                force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
                if isinstance(service_config, MarathonServiceConfig) and not marathon_message_printed:
                    print_marathon_message(desired_state)
                    marathon_message_printed = True
                elif isinstance(service_config, ChronosJobConfig) and not chronos_message_printed:
                    print_chronos_message(desired_state)
                    chronos_message_printed = True

                issue_state_change_for_service(
                    service_config=service_config, force_bounce=force_bounce, desired_state=desired_state
                )

    return_val = 0
    if invalid_deploy_groups:
        print "No branches found for %s in %s." % (", ".join(invalid_deploy_groups), remote_refs)
        print "Has %s been deployed there yet?" % service
        return_val = 1

    return return_val