def check_service_replication(
instance_config: MarathonServiceConfig,
all_tasks_or_pods: Sequence[MarathonTask],
replication_checker: MesosSmartstackEnvoyReplicationChecker,
) -> Optional[bool]:
"""Checks a service's replication levels based on how the service's replication
should be monitored. (smartstack/envoy or mesos)
:param instance_config: an instance of MarathonServiceConfig
:param replication_checker: an instance of MesosSmartstackEnvoyReplicationChecker
"""
expected_count = instance_config.get_instances()
log.info("Expecting %d total tasks for %s" %
(expected_count, instance_config.job_id))
proxy_port = get_proxy_port_for_instance(instance_config)
registrations = instance_config.get_registrations()
# if the primary registration does not match the service_instance name then
# the best we can do is check marathon for replication (for now).
if proxy_port is not None and registrations[0] == instance_config.job_id:
is_well_replicated = monitoring_tools.check_replication_for_instance(
instance_config=instance_config,
expected_count=expected_count,
replication_checker=replication_checker,
)
return is_well_replicated
else:
check_healthy_marathon_tasks_for_service_instance(
instance_config=instance_config,
expected_count=expected_count,
all_tasks=all_tasks_or_pods,
)
return None
def setup_service(
service: str,
instance: str,
clients: marathon_tools.MarathonClients,
job_config: marathon_tools.MarathonServiceConfig,
marathon_apps_with_clients: Sequence[Tuple[MarathonApp, MarathonClient]],
soa_dir: str,
) -> Tuple[int, str, Optional[float]]:
"""Setup the service instance given and attempt to deploy it, if possible.
Doesn't do anything if the service is already in Marathon and hasn't changed.
If it's not, attempt to find old instances of the service and bounce them.
:param service: The service name to setup
:param instance: The instance of the service to setup
:param clients: A MarathonClients object
:param job_config: The service instance's configuration dict
:returns: A tuple of (status, output, bounce_in_seconds) to be used with send_sensu_event"""
log.info("Setting up instance %s for service %s", instance, service)
try:
marathon_app_dict = job_config.format_marathon_app_dict()
except NoDockerImageError:
error_msg = (
"Docker image for {0}.{1} not in deployments.json. Exiting. Has Jenkins deployed it?\n"
).format(
service,
instance,
)
log.error(error_msg)
return (1, error_msg, None)
full_id = marathon_app_dict['id']
service_namespace_config = marathon_tools.load_service_namespace_config(
service=service,
namespace=job_config.get_nerve_namespace(),
soa_dir=soa_dir,
)
log.info("Desired Marathon instance id: %s", full_id)
return deploy_service(
service=service,
instance=instance,
marathon_jobid=full_id,
config=marathon_app_dict,
clients=clients,
marathon_apps_with_clients=marathon_apps_with_clients,
bounce_method=job_config.get_bounce_method(),
drain_method_name=job_config.get_drain_method(
service_namespace_config),
drain_method_params=job_config.get_drain_method_params(
service_namespace_config),
nerve_ns=job_config.get_nerve_namespace(),
registrations=job_config.get_registrations(),
bounce_health_params=job_config.get_bounce_health_params(
service_namespace_config),
soa_dir=soa_dir,
job_config=job_config,
bounce_margin_factor=job_config.get_bounce_margin_factor(),
)
def marathon_smartstack_status(
service: str,
instance: str,
job_config: marathon_tools.MarathonServiceConfig,
service_namespace_config: ServiceNamespaceConfig,
tasks: Sequence[MarathonTask],
should_return_individual_backends: bool = False,
) -> Mapping[str, Any]:
registration = job_config.get_registrations()[0]
discover_location_type = service_namespace_config.get_discover()
monitoring_blacklist = job_config.get_monitoring_blacklist(
system_deploy_blacklist=settings.system_paasta_config.
get_deploy_blacklist())
filtered_slaves = get_all_slaves_for_blacklist_whitelist(
blacklist=monitoring_blacklist, whitelist=None)
grouped_slaves = get_mesos_slaves_grouped_by_attribute(
slaves=filtered_slaves, attribute=discover_location_type)
# rebuild the dict, replacing the slave object with just their hostname
slave_hostname_by_location = {
attribute_value: [slave["hostname"] for slave in slaves]
for attribute_value, slaves in grouped_slaves.items()
}
expected_smartstack_count = marathon_tools.get_expected_instance_count_for_namespace(
service, instance, settings.cluster)
expected_count_per_location = int(expected_smartstack_count /
len(slave_hostname_by_location))
smartstack_status: MutableMapping[str, Any] = {
"registration": registration,
"expected_backends_per_location": expected_count_per_location,
"locations": [],
}
for location, hosts in slave_hostname_by_location.items():
synapse_host = hosts[0]
sorted_backends = sorted(
get_backends(
registration,
synapse_host=synapse_host,
synapse_port=settings.system_paasta_config.get_synapse_port(),
synapse_haproxy_url_format=settings.system_paasta_config.
get_synapse_haproxy_url_format(),
),
key=lambda backend: backend["status"],
reverse=True, # put 'UP' backends above 'MAINT' backends
)
matched_backends_and_tasks = match_backends_and_tasks(
sorted_backends, tasks)
location_dict = build_smartstack_location_dict(
location, matched_backends_and_tasks,
should_return_individual_backends)
smartstack_status["locations"].append(location_dict)
return smartstack_status
def marathon_service_mesh_status(
service: str,
service_mesh: pik.ServiceMesh,
instance: str,
job_config: marathon_tools.MarathonServiceConfig,
service_namespace_config: ServiceNamespaceConfig,
tasks: Sequence[MarathonTask],
should_return_individual_backends: bool = False,
) -> Mapping[str, Any]:
registration = job_config.get_registrations()[0]
discover_location_type = service_namespace_config.get_discover()
grouped_slaves = get_mesos_slaves_grouped_by_attribute(
slaves=get_slaves(), attribute=discover_location_type)
# rebuild the dict, replacing the slave object with just their hostname
slave_hostname_by_location = {
attribute_value: [slave["hostname"] for slave in slaves]
for attribute_value, slaves in grouped_slaves.items()
}
expected_instance_count = marathon_tools.get_expected_instance_count_for_namespace(
service, instance, settings.cluster)
expected_count_per_location = int(expected_instance_count /
len(slave_hostname_by_location))
service_mesh_status: MutableMapping[str, Any] = {
"registration": registration,
"expected_backends_per_location": expected_count_per_location,
"locations": [],
}
for location, hosts in slave_hostname_by_location.items():
if service_mesh == pik.ServiceMesh.SMARTSTACK:
service_mesh_status["locations"].append(
_build_smartstack_location_dict_for_backends(
synapse_host=hosts[0],
registration=registration,
tasks=tasks,
location=location,
should_return_individual_backends=
should_return_individual_backends,
))
elif service_mesh == pik.ServiceMesh.ENVOY:
service_mesh_status["locations"].append(
_build_envoy_location_dict_for_backends(
envoy_host=hosts[0],
registration=registration,
tasks=tasks,
location=location,
should_return_individual_backends=
should_return_individual_backends,
))
return service_mesh_status
