コード例 #1
0
ファイル: smartstack_tools.py プロジェクト: 00mjk/paasta
    def get_allowed_locations_and_hosts(
        self, instance_config: LongRunningServiceConfig
    ) -> Dict[str, Sequence[DiscoveredHost]]:
        """Returns a dict of locations and lists of corresponding mesos slaves
        where deployment of the instance is allowed.

        :param instance_config: An instance of MarathonServiceConfig
        :returns: A dict {"uswest1-prod": [DiscoveredHost(), DiscoveredHost(), ...]}
        """
        discover_location_type = marathon_tools.load_service_namespace_config(
            service=instance_config.service,
            namespace=instance_config.get_nerve_namespace(),
            soa_dir=instance_config.soa_dir,
        ).get_discover()
        attribute_to_slaves = mesos_tools.get_mesos_slaves_grouped_by_attribute(
            slaves=self._mesos_slaves, attribute=discover_location_type
        )
        ret: Dict[str, Sequence[DiscoveredHost]] = {}
        for attr, slaves in attribute_to_slaves.items():
            ret[attr] = [
                DiscoveredHost(
                    hostname=slave["hostname"], pool=slave["attributes"]["pool"]
                )
                for slave in slaves
            ]
        return ret
コード例 #2
0
    def _get_allowed_locations_and_hosts(
        self, instance_config: InstanceConfig
    ) -> Dict[str, Sequence[SmartstackHost]]:
        """Returns a dict of locations and lists of corresponding mesos slaves
        where deployment of the instance is allowed.

        :param instance_config: An instance of MarathonServiceConfig
        :returns: A dict {"uswest1-prod": [SmartstackHost(), SmartstackHost(), ...]}
        """
        monitoring_blacklist = instance_config.get_monitoring_blacklist(
            system_deploy_blacklist=self._system_paasta_config.
            get_deploy_blacklist(), )
        filtered_slaves = mesos_tools.filter_mesos_slaves_by_blacklist(
            slaves=self._mesos_slaves,
            blacklist=monitoring_blacklist,
            whitelist=None,
        )
        discover_location_type = marathon_tools.load_service_namespace_config(
            service=instance_config.service,
            namespace=instance_config.instance,
            soa_dir=instance_config.soa_dir,
        ).get_discover()
        attribute_to_slaves = mesos_tools.get_mesos_slaves_grouped_by_attribute(
            slaves=filtered_slaves,
            attribute=discover_location_type,
        )
        ret: Dict[str, Sequence[SmartstackHost]] = {}
        for attr, slaves in attribute_to_slaves.items():
            ret[attr] = [
                SmartstackHost(hostname=slave['hostname'],
                               pool=slave['attributes']['pool'])
                for slave in slaves
            ]
        return ret
コード例 #3
0
ファイル: info.py プロジェクト: kkellyy/paasta
def get_deployments_strings(service: str, soa_dir: str) -> List[str]:
    output = []
    try:
        deployments = get_actual_deployments(service, soa_dir)
    except NoDeploymentsAvailable:
        deployments = {}
    if deployments == {}:
        output.append(' - N/A: Not deployed to any PaaSTA Clusters')
    else:
        service_config = load_service_namespace_config(
            service=service,
            namespace='main',
            soa_dir=soa_dir,
        )
        service_mode = service_config.get_mode()
        for cluster in deployments_to_clusters(deployments):
            if service_mode == "tcp":
                service_port = service_config.get('proxy_port')
                link = PaastaColors.cyan('%s://paasta-%s.yelp:%d/' %
                                         (service_mode, cluster, service_port))
            elif service_mode == "http" or service_mode == "https":
                link = PaastaColors.cyan('%s://%s.paasta-%s.yelp/' %
                                         (service_mode, service, cluster))
            else:
                link = "N/A"
            output.append(' - %s (%s)' % (cluster, link))
    return output
コード例 #4
0
ファイル: smartstack_tools.py プロジェクト: white105/paasta
    def _get_allowed_locations_and_hostnames(
            self, instance_config) -> Dict[str, list]:
        """Returns a dict of locations and lists of corresponding mesos slaves
        where deployment of the instance is allowed.

        :param instance_config: An instance of MarathonServiceConfig
        :returns: A dict {"uswest1-prod": ['hostname1', 'hostname2], ...}.
        """
        monitoring_blacklist = instance_config.get_monitoring_blacklist(
            system_deploy_blacklist=self._system_paasta_config.
            get_deploy_blacklist(), )
        filtered_slaves = mesos_tools.filter_mesos_slaves_by_blacklist(
            slaves=self._mesos_slaves,
            blacklist=monitoring_blacklist,
            whitelist=None,
        )
        discover_location_type = marathon_tools.load_service_namespace_config(
            service=instance_config.service,
            namespace=instance_config.instance,
            soa_dir=instance_config.soa_dir,
        ).get_discover()
        slaves_grouped_by_attribute = mesos_tools.get_mesos_slaves_grouped_by_attribute(
            slaves=filtered_slaves,
            attribute=discover_location_type,
        )
        return {
            attr: [s['hostname'] for s in slaves]
            for attr, slaves in slaves_grouped_by_attribute.items()
        }
コード例 #5
0
def load_smartstack_info_for_service(service, namespace, soa_dir, blacklist):
    """Retrives number of available backends for given services

    :param service_instances: A list of tuples of (service, instance)
    :param namespaces: list of Smartstack namespaces
    :param blacklist: A list of blacklisted location tuples in the form (location, value)
    :returns: a dictionary of the form

    ::

        {
          'location_type': {
              'unique_location_name': {
                  'service.instance': <# ofavailable backends>
              },
              'other_unique_location_name': ...
          }
        }

    """
    service_namespace_config = marathon_tools.load_service_namespace_config(service, namespace,
                                                                            soa_dir=soa_dir)
    discover_location_type = service_namespace_config.get_discover()
    return get_smartstack_replication_for_attribute(
        attribute=discover_location_type,
        service=service,
        namespace=namespace,
        blacklist=blacklist)
コード例 #6
0
def get_deployments_strings(service: str, soa_dir: str) -> List[str]:
    output = []
    try:
        deployments = get_actual_deployments(service, soa_dir)
    except NoDeploymentsAvailable:
        deployments = {}
    if deployments == {}:
        output.append(" - N/A: Not deployed to any PaaSTA Clusters")
    else:
        service_config = load_service_namespace_config(service=service,
                                                       namespace="main",
                                                       soa_dir=soa_dir)
        service_mode = service_config.get_mode()
        for cluster in deployments_to_clusters(deployments):
            if service_mode == "tcp":
                service_port = service_config.get("proxy_port")
                link = PaastaColors.cyan("%s://paasta-%s.yelp:%d/" %
                                         (service_mode, cluster, service_port))
            elif service_mode == "http" or service_mode == "https":
                link = PaastaColors.cyan(
                    f"{service_mode}://{service}.paasta-{cluster}.yelp/")
            else:
                link = "N/A"
            output.append(f" - {cluster} ({link})")
    return output
コード例 #7
0
def load_smartstack_info_for_service(service, namespace, soa_dir, blacklist):
    """Retrives number of available backends for given services

    :param service_instances: A list of tuples of (service, instance)
    :param namespaces: list of Smartstack namespaces
    :param blacklist: A list of blacklisted location tuples in the form (location, value)
    :returns: a dictionary of the form

    ::

        {
          'location_type': {
              'unique_location_name': {
                  'service.instance': <# ofavailable backends>
              },
              'other_unique_location_name': ...
          }
        }

    """
    service_namespace_config = marathon_tools.load_service_namespace_config(service, namespace, soa_dir=soa_dir)
    discover_location_type = service_namespace_config.get_discover()
    return get_smartstack_replication_for_attribute(
        attribute=discover_location_type, service=service, namespace=namespace, blacklist=blacklist
    )
def kill_marathon_app(full_appid, cluster, client, soa_dir):
    service, instance, _, __ = (s.replace("--", "_")
                                for s in decompose_job_id(full_appid))
    service_instance_config = marathon_tools.load_marathon_service_config(
        service=service, instance=instance, cluster=cluster, soa_dir=soa_dir)
    complete_config = service_instance_config.format_marathon_app_dict()
    registrations = service_instance_config.get_registrations()
    service_namespace_config = marathon_tools.load_service_namespace_config(
        service=service, namespace=registrations[0])
    drain_method = drain_lib.get_drain_method(
        service_instance_config.get_drain_method(service_namespace_config),
        service=service,
        instance=instance,
        registrations=registrations,
        drain_method_params=service_instance_config.get_drain_method_params(
            service_namespace_config),
    )

    bounce_func = bounce_lib.get_bounce_method_func("down")

    while marathon_tools.is_app_id_running(app_id=full_appid, client=client):
        app_to_kill = client.get_app(full_appid)
        (
            old_app_live_happy_tasks,
            old_app_live_unhappy_tasks,
            old_app_draining_tasks,
            old_app_at_risk_tasks,
        ) = get_tasks_by_state(
            other_apps=[app_to_kill],
            drain_method=drain_method,
            service=service,
            nerve_ns=registrations[0],
            bounce_health_params=service_instance_config.
            get_bounce_health_params(service_namespace_config),
        )
        do_bounce(
            bounce_func=bounce_func,
            drain_method=drain_method,
            config=complete_config,
            new_app_running="",
            happy_new_tasks=[],
            old_app_live_happy_tasks=old_app_live_happy_tasks,
            old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
            old_app_draining_tasks=old_app_draining_tasks,
            old_app_at_risk_tasks=old_app_at_risk_tasks,
            serviceinstance=f"{service}.{instance}",
            bounce_method="down",
            service=service,
            cluster=cluster,
            instance=instance,
            marathon_jobid=full_appid,
            client=client,
            soa_dir=soa_dir,
        )

        paasta_print("Sleeping for 10 seconds to give the tasks time to drain")
        time.sleep(10)

    paasta_print(f"Successfully killed {full_appid}")
コード例 #9
0
def setup_service(
    service: str,
    instance: str,
    clients: marathon_tools.MarathonClients,
    job_config: marathon_tools.MarathonServiceConfig,
    marathon_apps_with_clients: Sequence[Tuple[MarathonApp, MarathonClient]],
    soa_dir: str,
) -> Tuple[int, str, Optional[float]]:
    """Setup the service instance given and attempt to deploy it, if possible.
    Doesn't do anything if the service is already in Marathon and hasn't changed.
    If it's not, attempt to find old instances of the service and bounce them.

    :param service: The service name to setup
    :param instance: The instance of the service to setup
    :param clients: A MarathonClients object
    :param job_config: The service instance's configuration dict
    :returns: A tuple of (status, output, bounce_in_seconds) to be used with send_sensu_event"""

    log.info("Setting up instance %s for service %s", instance, service)
    try:
        marathon_app_dict = job_config.format_marathon_app_dict()
    except NoDockerImageError:
        error_msg = (
            "Docker image for {0}.{1} not in deployments.json. Exiting. Has Jenkins deployed it?\n"
        ).format(
            service,
            instance,
        )
        log.error(error_msg)
        return (1, error_msg, None)

    full_id = marathon_app_dict['id']
    service_namespace_config = marathon_tools.load_service_namespace_config(
        service=service,
        namespace=job_config.get_nerve_namespace(),
        soa_dir=soa_dir,
    )

    log.info("Desired Marathon instance id: %s", full_id)
    return deploy_service(
        service=service,
        instance=instance,
        marathon_jobid=full_id,
        config=marathon_app_dict,
        clients=clients,
        marathon_apps_with_clients=marathon_apps_with_clients,
        bounce_method=job_config.get_bounce_method(),
        drain_method_name=job_config.get_drain_method(
            service_namespace_config),
        drain_method_params=job_config.get_drain_method_params(
            service_namespace_config),
        nerve_ns=job_config.get_nerve_namespace(),
        registrations=job_config.get_registrations(),
        bounce_health_params=job_config.get_bounce_health_params(
            service_namespace_config),
        soa_dir=soa_dir,
        job_config=job_config,
        bounce_margin_factor=job_config.get_bounce_margin_factor(),
    )
コード例 #10
0
def kill_marathon_app(full_appid, cluster, client, soa_dir):
    service, instance, _, __ = (s.replace('--', '_') for s in decompose_job_id(full_appid))
    service_instance_config = marathon_tools.load_marathon_service_config(
        service=service,
        instance=instance,
        cluster=cluster,
        soa_dir=soa_dir,
    )
    complete_config = service_instance_config.format_marathon_app_dict()
    nerve_ns = service_instance_config.get_nerve_namespace()
    service_namespace_config = marathon_tools.load_service_namespace_config(service=service, namespace=nerve_ns)
    drain_method = drain_lib.get_drain_method(
        service_instance_config.get_drain_method(service_namespace_config),
        service=service,
        instance=instance,
        nerve_ns=nerve_ns,
        drain_method_params=service_instance_config.get_drain_method_params(service_namespace_config),
    )

    bounce_func = bounce_lib.get_bounce_method_func('down')

    while marathon_tools.is_app_id_running(app_id=full_appid, client=client):
        app_to_kill = client.get_app(full_appid)
        (
            old_app_live_happy_tasks,
            old_app_live_unhappy_tasks,
            old_app_draining_tasks,
            old_app_at_risk_tasks,
        ) = get_tasks_by_state(
            other_apps=[app_to_kill],
            drain_method=drain_method,
            service=service,
            nerve_ns=nerve_ns,
            bounce_health_params=service_instance_config.get_bounce_health_params(service_namespace_config),
        )
        do_bounce(
            bounce_func=bounce_func,
            drain_method=drain_method,
            config=complete_config,
            new_app_running='',
            happy_new_tasks=[],
            old_app_live_happy_tasks=old_app_live_happy_tasks,
            old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
            old_app_draining_tasks=old_app_draining_tasks,
            old_app_at_risk_tasks=old_app_at_risk_tasks,
            serviceinstance="{}.{}".format(service, instance),
            bounce_method='down',
            service=service,
            cluster=cluster,
            instance=instance,
            marathon_jobid=full_appid,
            client=client,
            soa_dir=soa_dir,
        )

        paasta_print("Sleeping for 10 seconds to give the tasks time to drain")
        time.sleep(10)

    paasta_print("Sucessfully killed {}".format(full_appid))
コード例 #11
0
ファイル: updown_service.py プロジェクト: Yelp/nerve-tools
def _get_timeout_s(service_name, timeout):
    if timeout is not None:
        return timeout

    srv_name, namespace = service_name.split('.')
    namespace_configuration = load_service_namespace_config(srv_name, namespace)
    timeout_s = namespace_configuration.get('updown_timeout_s', DEFAULT_TIMEOUT_S)
    return timeout_s
コード例 #12
0
ファイル: bounce_lib.py プロジェクト: Yelp/paasta
def get_happy_tasks(app, service, nerve_ns, system_paasta_config, min_task_uptime=None, check_haproxy=False):
    """Given a MarathonApp object, return the subset of tasks which are considered healthy.
    With the default options, this returns tasks where at least one of the defined Marathon healthchecks passes.
    For it to do anything interesting, set min_task_uptime or check_haproxy.

    :param app: A MarathonApp object.
    :param service: The name of the service.
    :param nerve_ns: The nerve namespace
    :param min_task_uptime: Minimum number of seconds that a task must be running before we consider it healthy. Useful
                            if tasks take a while to start up.
    :param check_haproxy: Whether to check the local haproxy to make sure this task has been registered and discovered.
    """
    tasks = app.tasks
    happy = []
    now = datetime.datetime.utcnow()

    if check_haproxy:
        tasks_in_smartstack = []
        service_namespace = compose_job_id(service, nerve_ns)

        service_namespace_config = marathon_tools.load_service_namespace_config(
            service=service, namespace=nerve_ns)
        discover_location_type = service_namespace_config.get_discover()
        unique_values = mesos_tools.get_mesos_slaves_grouped_by_attribute(
            slaves=mesos_tools.get_slaves(),
            attribute=discover_location_type
        )

        for value, hosts in unique_values.iteritems():
            synapse_hostname = hosts[0]['hostname']
            tasks_in_smartstack.extend(get_registered_marathon_tasks(
                synapse_hostname,
                system_paasta_config.get_synapse_port(),
                system_paasta_config.get_synapse_haproxy_url_format(),
                service_namespace,
                tasks,
            ))
        tasks = tasks_in_smartstack

    for task in tasks:
        if task.started_at is None:
            # Can't be healthy if it hasn't started
            continue

        if min_task_uptime is not None:
            if (now - task.started_at).total_seconds() < min_task_uptime:
                continue

        # if there are healthchecks defined for the app but none have executed yet, then task is unhappy
        if len(app.health_checks) > 0 and len(task.health_check_results) == 0:
            continue

        # if there are health check results, check if at least one healthcheck is passing
        if not marathon_tools.is_task_healthy(task, require_all=False, default_healthy=True):
            continue
        happy.append(task)

    return happy
コード例 #13
0
def _should_manage_service(service_name):
    srv_name, namespace = service_name.split('.')
    marathon_config = load_service_namespace_config(srv_name, namespace)
    classic_config = read_service_configuration(srv_name)

    should_manage = marathon_config.get('proxy_port') is not None
    blacklisted = classic_config.get('no_updown_service')

    return (should_manage and not blacklisted)
コード例 #14
0
ファイル: updown_service.py プロジェクト: Yelp/nerve-tools
def _should_manage_service(service_name):
    srv_name, namespace = service_name.split('.')
    marathon_config = load_service_namespace_config(srv_name, namespace)
    classic_config = read_service_configuration(srv_name)

    should_manage = marathon_config.get('proxy_port') is not None
    blacklisted = classic_config.get('no_updown_service')

    return (should_manage and not blacklisted)
コード例 #15
0
def _should_manage_service(service_name):
    srv_name, namespace = service_name.split('.')
    marathon_config = load_service_namespace_config(srv_name, namespace)
    classic_config = read_service_configuration(srv_name)

    # None is a valid value of proxy_port indicating a discovery only service
    should_manage = marathon_config.get('proxy_port', -1) != -1
    blacklisted = classic_config.get('no_updown_service')

    return (should_manage and not blacklisted)
コード例 #16
0
def _get_timeout_s(service_name, timeout):
    if timeout is not None:
        return timeout

    srv_name, namespace = service_name.split('.')
    namespace_configuration = load_service_namespace_config(
        srv_name, namespace)
    timeout_s = namespace_configuration.get('updown_timeout_s',
                                            DEFAULT_TIMEOUT_S)
    return timeout_s
コード例 #17
0
def status_smartstack_backends(service, instance, job_config, cluster, tasks,
                               expected_count, soa_dir, verbose, synapse_port,
                               synapse_haproxy_url_format):
    """Returns detailed information about smartstack backends for a service
    and instance.
    return: A newline separated string of the smarststack backend status
    """
    output = []
    nerve_ns = marathon_tools.read_namespace_for_service_instance(
        service, instance, cluster)
    service_instance = compose_job_id(service, nerve_ns)

    service_namespace_config = marathon_tools.load_service_namespace_config(
        service, instance, soa_dir=soa_dir)
    discover_location_type = service_namespace_config.get_discover()
    monitoring_blacklist = job_config.get_monitoring_blacklist()

    filtered_slaves = get_all_slaves_for_blacklist_whitelist(
        blacklist=monitoring_blacklist, whitelist=[])

    grouped_slaves = get_mesos_slaves_grouped_by_attribute(
        slaves=filtered_slaves,
        attribute=discover_location_type,
    )

    # rebuild the dict, replacing the slave object
    # with just their hostname
    grouped_slave_hostname = {
        attribute_value: [slave['hostname'] for slave in slaves]
        for attribute_value, slaves in grouped_slaves.items()
    }

    if len(grouped_slave_hostname) == 0:
        output.append("Smartstack: ERROR - %s is NOT in smartstack at all!" %
                      service_instance)
    else:
        output.append("Smartstack:")
        if verbose:
            output.append("  Haproxy Service Name: %s" % service_instance)
            output.append("  Backends:")

        output.extend(
            pretty_print_smartstack_backends_for_locations(
                service_instance=service_instance,
                tasks=tasks,
                locations=grouped_slave_hostname,
                expected_count=expected_count,
                verbose=verbose,
                synapse_port=synapse_port,
                synapse_haproxy_url_format=synapse_haproxy_url_format,
            ))
    return "\n".join(output)
コード例 #18
0
def marathon_instance_status(
    instance_status: Mapping[str, Any],
    service: str,
    instance: str,
    verbose: int,
    include_smartstack: bool,
    include_mesos: bool,
) -> Mapping[str, Any]:
    mstatus: Dict[str, Any] = {}

    job_config = marathon_tools.load_marathon_service_config(
        service, instance, settings.cluster, soa_dir=settings.soa_dir
    )
    marathon_apps_with_clients = marathon_tools.get_marathon_apps_with_clients(
        clients=settings.marathon_clients.get_all_clients_for_service(job_config),
        embed_tasks=True,
        service_name=service,
    )
    matching_apps_with_clients = marathon_tools.get_matching_apps_with_clients(
        service, instance, marathon_apps_with_clients
    )

    mstatus.update(
        marathon_job_status(
            service, instance, job_config, matching_apps_with_clients, verbose
        )
    )

    if include_smartstack:
        service_namespace_config = marathon_tools.load_service_namespace_config(
            service=service,
            namespace=job_config.get_nerve_namespace(),
            soa_dir=settings.soa_dir,
        )
        if "proxy_port" in service_namespace_config:
            tasks = [
                task for app, _ in matching_apps_with_clients for task in app.tasks
            ]

            mstatus["smartstack"] = marathon_smartstack_status(
                service,
                instance,
                job_config,
                service_namespace_config,
                tasks,
                should_return_individual_backends=verbose > 0,
            )

    if include_mesos:
        mstatus["mesos"] = marathon_mesos_status(service, instance, verbose)

    return mstatus
コード例 #19
0
ファイル: marathon_serviceinit.py プロジェクト: Yelp/paasta
def status_smartstack_backends(service, instance, job_config, cluster, tasks, expected_count, soa_dir, verbose,
                               synapse_port, synapse_haproxy_url_format):
    """Returns detailed information about smartstack backends for a service
    and instance.
    return: A newline separated string of the smarststack backend status
    """
    output = []
    service_instance = marathon_tools.read_registration_for_service_instance(
        service, instance, cluster
    )

    service_namespace_config = marathon_tools.load_service_namespace_config(
        service=service, namespace=instance, soa_dir=soa_dir)
    discover_location_type = service_namespace_config.get_discover()
    monitoring_blacklist = job_config.get_monitoring_blacklist()

    filtered_slaves = get_all_slaves_for_blacklist_whitelist(
        blacklist=monitoring_blacklist,
        whitelist=[]
    )

    grouped_slaves = get_mesos_slaves_grouped_by_attribute(
        slaves=filtered_slaves,
        attribute=discover_location_type,
    )

    # rebuild the dict, replacing the slave object
    # with just their hostname
    grouped_slave_hostname = {
        attribute_value: [slave['hostname'] for slave in slaves]
        for attribute_value, slaves in grouped_slaves.items()
    }

    if len(grouped_slave_hostname) == 0:
        output.append("Smartstack: ERROR - %s is NOT in smartstack at all!" % service_instance)
    else:
        output.append("Smartstack:")
        if verbose:
            output.append("  Haproxy Service Name: %s" % service_instance)
            output.append("  Backends:")

        output.extend(pretty_print_smartstack_backends_for_locations(
            service_instance=service_instance,
            tasks=tasks,
            locations=grouped_slave_hostname,
            expected_count=expected_count,
            verbose=verbose,
            synapse_port=synapse_port,
            synapse_haproxy_url_format=synapse_haproxy_url_format,
        ))
    return "\n".join(output)
コード例 #20
0
ファイル: setup_marathon_job.py プロジェクト: sbcoba/paasta
def setup_service(service, instance, client, marathon_config,
                  service_marathon_config, soa_dir):
    """Setup the service instance given and attempt to deploy it, if possible.
    Doesn't do anything if the service is already in Marathon and hasn't changed.
    If it's not, attempt to find old instances of the service and bounce them.

    :param service: The service name to setup
    :param instance: The instance of the service to setup
    :param client: A MarathonClient object
    :param marathon_config: The marathon configuration dict
    :param service_marathon_config: The service instance's configuration dict
    :returns: A tuple of (status, output) to be used with send_sensu_event"""

    log.info("Setting up instance %s for service %s", instance, service)
    try:
        complete_config = marathon_tools.create_complete_config(
            service, instance, marathon_config)
    except NoDockerImageError:
        error_msg = (
            "Docker image for {0}.{1} not in deployments.json. Exiting. Has Jenkins deployed it?\n"
        ).format(
            service,
            instance,
        )
        log.error(error_msg)
        return (1, error_msg)

    full_id = complete_config['id']
    service_namespace_config = marathon_tools.load_service_namespace_config(
        service, instance)

    log.info("Desired Marathon instance id: %s", full_id)
    return deploy_service(
        service=service,
        instance=instance,
        marathon_jobid=full_id,
        config=complete_config,
        client=client,
        bounce_method=service_marathon_config.get_bounce_method(),
        drain_method_name=service_marathon_config.get_drain_method(
            service_namespace_config),
        drain_method_params=service_marathon_config.get_drain_method_params(
            service_namespace_config),
        nerve_ns=service_marathon_config.get_nerve_namespace(),
        bounce_health_params=service_marathon_config.get_bounce_health_params(
            service_namespace_config),
        soa_dir=soa_dir,
    )
コード例 #21
0
def status_smartstack_backends(service, instance, job_config, cluster, tasks,
                               expected_count, soa_dir, verbose, synapse_port,
                               synapse_haproxy_url_format):
    """Returns detailed information about smartstack backends for a service
    and instance.
    return: A newline separated string of the smarststack backend status
    """
    output = []
    nerve_ns = marathon_tools.read_namespace_for_service_instance(
        service, instance, cluster)
    service_instance = compose_job_id(service, nerve_ns)

    if instance != nerve_ns:
        ns_string = PaastaColors.bold(nerve_ns)
        output.append(
            "Smartstack: N/A - %s is announced in the %s namespace." %
            (instance, ns_string))
        # If verbose mode is specified, then continue to show backends anyway, otherwise stop early
        if not verbose:
            return "\n".join(output)

    service_namespace_config = marathon_tools.load_service_namespace_config(
        service, instance, soa_dir=soa_dir)
    discover_location_type = service_namespace_config.get_discover()
    monitoring_blacklist = job_config.get_monitoring_blacklist()
    unique_attributes = get_mesos_slaves_grouped_by_attribute(
        attribute=discover_location_type, blacklist=monitoring_blacklist)
    if len(unique_attributes) == 0:
        output.append("Smartstack: ERROR - %s is NOT in smartstack at all!" %
                      service_instance)
    else:
        output.append("Smartstack:")
        if verbose:
            output.append("  Haproxy Service Name: %s" % service_instance)
            output.append("  Backends:")

        output.extend(
            pretty_print_smartstack_backends_for_locations(
                service_instance,
                tasks,
                unique_attributes,
                expected_count,
                verbose,
                synapse_port,
                synapse_haproxy_url_format,
            ))
    return "\n".join(output)
コード例 #22
0
ファイル: setup_marathon_job.py プロジェクト: Yelp/paasta
def setup_service(service, instance, client, service_marathon_config, marathon_apps, soa_dir):
    """Setup the service instance given and attempt to deploy it, if possible.
    Doesn't do anything if the service is already in Marathon and hasn't changed.
    If it's not, attempt to find old instances of the service and bounce them.

    :param service: The service name to setup
    :param instance: The instance of the service to setup
    :param client: A MarathonClient object
    :param service_marathon_config: The service instance's configuration dict
    :returns: A tuple of (status, output) to be used with send_sensu_event"""

    log.info("Setting up instance %s for service %s", instance, service)
    try:
        marathon_app_dict = service_marathon_config.format_marathon_app_dict()
    except NoDockerImageError:
        error_msg = (
            "Docker image for {0}.{1} not in deployments.json. Exiting. Has Jenkins deployed it?\n"
        ).format(
            service,
            instance,
        )
        log.error(error_msg)
        return (1, error_msg)

    full_id = marathon_app_dict['id']
    service_namespace_config = marathon_tools.load_service_namespace_config(
        service=service, namespace=instance, soa_dir=soa_dir)

    log.info("Desired Marathon instance id: %s", full_id)
    return deploy_service(
        service=service,
        instance=instance,
        marathon_jobid=full_id,
        config=marathon_app_dict,
        client=client,
        marathon_apps=marathon_apps,
        bounce_method=service_marathon_config.get_bounce_method(),
        drain_method_name=service_marathon_config.get_drain_method(service_namespace_config),
        drain_method_params=service_marathon_config.get_drain_method_params(service_namespace_config),
        nerve_ns=service_marathon_config.get_nerve_namespace(),
        bounce_health_params=service_marathon_config.get_bounce_health_params(service_namespace_config),
        soa_dir=soa_dir,
        bounce_margin_factor=service_marathon_config.get_bounce_margin_factor(),
    )
コード例 #23
0
ファイル: info.py プロジェクト: oktopuz/paasta
def get_deployments_strings(service, soa_dir):
    output = []
    try:
        deployments = get_actual_deployments(service, soa_dir)
    except NoDeploymentsAvailable:
        deployments = {}
    if deployments == {}:
        output.append(' - N/A: Not deployed to any PaaSTA Clusters')
    else:
        service_config = load_service_namespace_config(service, 'main', soa_dir)
        service_mode = service_config.get_mode()
        for cluster in deployments_to_clusters(deployments):
            if service_mode == "tcp":
                service_port = service_config.get('proxy_port')
                link = PaastaColors.cyan('%s://paasta-%s.yelp:%d/' % (service_mode, cluster, service_port))
            elif service_mode == "http":
                link = PaastaColors.cyan('%s://%s.paasta-%s.yelp/' % (service_mode, service, cluster))
            else:
                link = "N/A"
            output.append(' - %s (%s)' % (cluster, link))
    return output
コード例 #24
0
def status_smartstack_backends(service, instance, job_config, cluster, tasks, expected_count, soa_dir, verbose,
                               synapse_port, synapse_haproxy_url_format):
    """Returns detailed information about smartstack backends for a service
    and instance.
    return: A newline separated string of the smarststack backend status
    """
    output = []
    nerve_ns = marathon_tools.read_namespace_for_service_instance(service, instance, cluster)
    service_instance = compose_job_id(service, nerve_ns)

    if instance != nerve_ns:
        ns_string = PaastaColors.bold(nerve_ns)
        output.append("Smartstack: N/A - %s is announced in the %s namespace." % (instance, ns_string))
        # If verbose mode is specified, then continue to show backends anyway, otherwise stop early
        if not verbose:
            return "\n".join(output)

    service_namespace_config = marathon_tools.load_service_namespace_config(service, instance, soa_dir=soa_dir)
    discover_location_type = service_namespace_config.get_discover()
    monitoring_blacklist = job_config.get_monitoring_blacklist()
    unique_attributes = get_mesos_slaves_grouped_by_attribute(
        attribute=discover_location_type, blacklist=monitoring_blacklist)
    if len(unique_attributes) == 0:
        output.append("Smartstack: ERROR - %s is NOT in smartstack at all!" % service_instance)
    else:
        output.append("Smartstack:")
        if verbose:
            output.append("  Haproxy Service Name: %s" % service_instance)
            output.append("  Backends:")

        output.extend(pretty_print_smartstack_backends_for_locations(
            service_instance,
            tasks,
            unique_attributes,
            expected_count,
            verbose,
            synapse_port,
            synapse_haproxy_url_format,
        ))
    return "\n".join(output)
コード例 #25
0
def load_smartstack_info_for_service(
    service: str,
    namespace: str,
    blacklist: DeployBlacklist,
    system_paasta_config: SystemPaastaConfig,
    soa_dir: str = DEFAULT_SOA_DIR,
) -> Dict[str, Dict[str, int]]:
    """Retrieves number of available backends for given services

    :param service_instances: A list of tuples of (service, instance)
    :param namespaces: list of Smartstack namespaces
    :param blacklist: A list of blacklisted location tuples in the form (location, value)
    :param system_paasta_config: A SystemPaastaConfig object representing the system configuration.
    :returns: a dictionary of the form

    ::

        {
          'location_type': {
              'unique_location_name': {
                  'service.instance': <# ofavailable backends>
              },
              'other_unique_location_name': ...
          }
        }

    """
    service_namespace_config = marathon_tools.load_service_namespace_config(
        service=service,
        namespace=namespace,
        soa_dir=soa_dir,
    )
    discover_location_type = service_namespace_config.get_discover()
    return get_smartstack_replication_for_attribute(
        attribute=discover_location_type,
        service=service,
        namespace=namespace,
        blacklist=blacklist,
        system_paasta_config=system_paasta_config,
    )
コード例 #26
0
def perform_command(
    command: str,
    service: str,
    instance: str,
    cluster: str,
    verbose: int,
    soa_dir: str,
    clients: marathon_tools.MarathonClients,
    job_config: marathon_tools.MarathonServiceConfig,
    app_id: str = None,
) -> int:
    """Performs a start/stop/restart/status on an instance
    :param command: String of start, stop, restart, status
    :param service: service name
    :param instance: instance name, like "main" or "canary"
    :param cluster: cluster name
    :param verbose: int verbosity level
    :param client: MarathonClient or CachingMarathonClient
    :returns: A unix-style return code
    """
    system_config = load_system_paasta_config()

    if not app_id:
        try:
            app_id = job_config.format_marathon_app_dict()["id"]
        except NoDockerImageError:
            job_id = compose_job_id(service, instance)
            paasta_print(
                "Docker image for %s not in deployments.json. Exiting. Has Jenkins deployed it?"
                % job_id
            )
            return 1

    normal_instance_count = job_config.get_instances()

    current_client = clients.get_current_client_for_service(job_config)

    if command == "restart":
        restart_marathon_job(service, instance, app_id, current_client, cluster)
    elif command == "status":
        paasta_print(
            status_desired_state(service, instance, current_client, job_config)
        )
        dashboards = get_marathon_dashboard_links(clients, system_config)
        tasks, out = status_marathon_job(
            service=service,
            instance=instance,
            cluster=cluster,
            soa_dir=soa_dir,
            dashboards=dashboards,
            normal_instance_count=normal_instance_count,
            clients=clients,
            job_config=job_config,
            desired_app_id=app_id,
            verbose=verbose,
        )
        paasta_print(out)
        service_namespace_config = marathon_tools.load_service_namespace_config(
            service=service, namespace=job_config.get_nerve_namespace(), soa_dir=soa_dir
        )

        paasta_print(
            status_mesos_tasks(service, instance, normal_instance_count, verbose)
        )

        proxy_port = service_namespace_config.get("proxy_port")
        if proxy_port is not None:
            normal_smartstack_count = marathon_tools.get_expected_instance_count_for_namespace(
                service, instance, cluster
            )
            paasta_print(
                status_smartstack_backends(
                    service=service,
                    instance=instance,
                    cluster=cluster,
                    job_config=job_config,
                    service_namespace_config=service_namespace_config,
                    tasks=tasks,
                    expected_count=normal_smartstack_count,
                    soa_dir=soa_dir,
                    verbose=verbose > 0,
                    synapse_port=system_config.get_synapse_port(),
                    synapse_haproxy_url_format=system_config.get_synapse_haproxy_url_format(),
                    system_deploy_blacklist=system_config.get_deploy_blacklist(),
                    system_deploy_whitelist=system_config.get_deploy_whitelist(),
                )
            )
    else:
        # The command parser shouldn't have let us get this far...
        raise NotImplementedError("Command %s is not implemented!" % command)
    return 0
コード例 #27
0
ファイル: graceful_app_drain.py プロジェクト: sbcoba/paasta
def main():
    args = parse_args()
    full_appid = args.appname.lstrip('/')
    soa_dir = args.soa_dir
    marathon_config = marathon_tools.load_marathon_config()
    client = marathon_tools.get_marathon_client(
        url=marathon_config.get_url(),
        user=marathon_config.get_username(),
        passwd=marathon_config.get_password(),
    )

    if not marathon_tools.is_app_id_running(app_id=full_appid, client=client):
        print("Couldn't find an app named {0}".format(full_appid))
        sys.exit(1)

    service, instance, _, __ = (s.replace('--', '_')
                                for s in decompose_job_id(full_appid))
    complete_config = marathon_tools.create_complete_config(
        service, instance, marathon_config)
    cluster = load_system_paasta_config().get_cluster()
    service_instance_config = marathon_tools.load_marathon_service_config(
        service=service,
        instance=instance,
        cluster=cluster,
        soa_dir=soa_dir,
    )
    nerve_ns = service_instance_config.get_nerve_namespace()
    service_namespace_config = marathon_tools.load_service_namespace_config(
        service=service, namespace=nerve_ns)
    drain_method = drain_lib.get_drain_method(
        service_instance_config.get_drain_method(service_namespace_config),
        service=service,
        instance=instance,
        nerve_ns=nerve_ns,
        drain_method_params=service_instance_config.get_drain_method_params(
            service_namespace_config),
    )

    bounce_func = bounce_lib.get_bounce_method_func('down')

    while marathon_tools.is_app_id_running(app_id=full_appid, client=client):
        app_to_kill = client.get_app(full_appid)
        old_app_live_tasks, old_app_draining_tasks = get_old_live_draining_tasks(
            [app_to_kill], drain_method)
        do_bounce(
            bounce_func=bounce_func,
            drain_method=drain_method,
            config=complete_config,
            new_app_running='',
            happy_new_tasks=[],
            old_app_live_tasks=old_app_live_tasks,
            old_app_draining_tasks=old_app_draining_tasks,
            serviceinstance="{0}.{1}".format(service, instance),
            bounce_method='down',
            service=service,
            cluster=cluster,
            instance=instance,
            marathon_jobid=full_appid,
            client=client,
            soa_dir=soa_dir,
        )

        print "Sleeping for 10 seconds to give the tasks time to drain"
        time.sleep(10)

    print("Sucessfully killed {0}".format(full_appid))
コード例 #28
0
def main():
    args = parse_args()
    full_appid = args.appname.lstrip('/')
    soa_dir = args.soa_dir
    marathon_config = marathon_tools.load_marathon_config()
    client = marathon_tools.get_marathon_client(
        url=marathon_config.get_url(),
        user=marathon_config.get_username(),
        passwd=marathon_config.get_password(),
    )

    if not marathon_tools.is_app_id_running(app_id=full_appid, client=client):
        print("Couldn't find an app named {0}".format(full_appid))
        sys.exit(1)

    service, instance, _, __ = (s.replace('--', '_') for s in decompose_job_id(full_appid))
    complete_config = marathon_tools.create_complete_config(service, instance, marathon_config)
    cluster = load_system_paasta_config().get_cluster()
    service_instance_config = marathon_tools.load_marathon_service_config(
        service=service,
        instance=instance,
        cluster=cluster,
        soa_dir=soa_dir,
    )
    nerve_ns = service_instance_config.get_nerve_namespace()
    service_namespace_config = marathon_tools.load_service_namespace_config(service=service, namespace=nerve_ns)
    drain_method = drain_lib.get_drain_method(
        service_instance_config.get_drain_method(service_namespace_config),
        service=service,
        instance=instance,
        nerve_ns=nerve_ns,
        drain_method_params=service_instance_config.get_drain_method_params(service_namespace_config),
    )

    bounce_func = bounce_lib.get_bounce_method_func('down')

    while marathon_tools.is_app_id_running(app_id=full_appid, client=client):
        app_to_kill = client.get_app(full_appid)
        old_app_live_tasks, old_app_draining_tasks = get_old_live_draining_tasks([app_to_kill], drain_method)
        do_bounce(
            bounce_func=bounce_func,
            drain_method=drain_method,
            config=complete_config,
            new_app_running='',
            happy_new_tasks=[],
            old_app_live_tasks=old_app_live_tasks,
            old_app_draining_tasks=old_app_draining_tasks,
            serviceinstance="{0}.{1}".format(service, instance),
            bounce_method='down',
            service=service,
            cluster=cluster,
            instance=instance,
            marathon_jobid=full_appid,
            client=client,
            soa_dir=soa_dir,
        )

        print "Sleeping for 10 seconds to give the tasks time to drain"
        time.sleep(10)

    print("Sucessfully killed {0}".format(full_appid))
コード例 #29
0
ファイル: bounce_lib.py プロジェクト: edric-shen/paasta
def get_happy_tasks(app,
                    service,
                    nerve_ns,
                    system_paasta_config,
                    min_task_uptime=None,
                    check_haproxy=False):
    """Given a MarathonApp object, return the subset of tasks which are considered healthy.
    With the default options, this returns tasks where at least one of the defined Marathon healthchecks passes.
    For it to do anything interesting, set min_task_uptime or check_haproxy.

    :param app: A MarathonApp object.
    :param service: The name of the service.
    :param nerve_ns: The nerve namespace
    :param min_task_uptime: Minimum number of seconds that a task must be running before we consider it healthy. Useful
                            if tasks take a while to start up.
    :param check_haproxy: Whether to check the local haproxy to make sure this task has been registered and discovered.
    """
    tasks = app.tasks
    happy = []
    now = datetime.datetime.utcnow()

    if check_haproxy:
        tasks_in_smartstack = []
        service_namespace = compose_job_id(service, nerve_ns)

        service_namespace_config = marathon_tools.load_service_namespace_config(
            service=service, namespace=nerve_ns)
        discover_location_type = service_namespace_config.get_discover()
        unique_values = mesos_tools.get_mesos_slaves_grouped_by_attribute(
            slaves=mesos_tools.get_slaves(), attribute=discover_location_type)

        for value, hosts in unique_values.iteritems():
            synapse_hostname = hosts[0]['hostname']
            tasks_in_smartstack.extend(
                get_registered_marathon_tasks(
                    synapse_hostname,
                    system_paasta_config.get_synapse_port(),
                    system_paasta_config.get_synapse_haproxy_url_format(),
                    service_namespace,
                    tasks,
                ))
        tasks = tasks_in_smartstack

    for task in tasks:
        if task.started_at is None:
            # Can't be healthy if it hasn't started
            continue

        if min_task_uptime is not None:
            if (now - task.started_at).total_seconds() < min_task_uptime:
                continue

        # if there are healthchecks defined for the app but none have executed yet, then task is unhappy
        if len(app.health_checks) > 0 and len(task.health_check_results) == 0:
            continue

        # if there are health check results, check if at least one healthcheck is passing
        if not marathon_tools.is_task_healthy(
                task, require_all=False, default_healthy=True):
            continue
        happy.append(task)

    return happy