def read_chronos_jobs_for_service(service, cluster, soa_dir=DEFAULT_SOA_DIR):
chronos_conf_file = 'chronos-%s' % cluster
log.info("Reading Chronos configuration file: %s/%s/chronos-%s.yaml" %
(soa_dir, service, cluster))
return service_configuration_lib.read_extra_service_information(
service, chronos_conf_file, soa_dir=soa_dir)
def load_adhoc_job_config(service, instance, cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR):
general_config = service_configuration_lib.read_service_configuration(
service,
soa_dir=soa_dir
)
adhoc_conf_file = "adhoc-%s" % cluster
log.info("Reading adhoc configuration file: %s.yaml", adhoc_conf_file)
instance_configs = service_configuration_lib.read_extra_service_information(
service_name=service,
extra_info=adhoc_conf_file,
soa_dir=soa_dir
)
if instance not in instance_configs:
raise NoConfigurationForServiceError(
"%s not found in config file %s/%s/%s.yaml." % (instance, soa_dir, service, adhoc_conf_file)
)
general_config = deep_merge_dictionaries(overrides=instance_configs[instance], defaults=general_config)
branch_dict = {}
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
branch = general_config.get('branch', get_paasta_branch(cluster, instance))
deploy_group = general_config.get('deploy_group', branch)
branch_dict = deployments_json.get_branch_dict_v2(service, branch, deploy_group)
return AdhocJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
)
def setup_kube_crd(
kube_client: KubeClient,
cluster: str,
services: Sequence[str],
soa_dir: str = DEFAULT_SOA_DIR,
) -> bool:
existing_crds = kube_client.apiextensions.list_custom_resource_definition(
label_selector=paasta_prefixed("service"))
desired_crds = []
for service in services:
crd_config = service_configuration_lib.read_extra_service_information(
service, f"crd-{cluster}", soa_dir=soa_dir)
if not crd_config:
log.info("nothing to deploy")
continue
metadata = crd_config.get("metadata", {})
if "labels" not in metadata:
metadata["labels"] = {}
metadata["labels"]["yelp.com/paasta_service"] = service
metadata["labels"][paasta_prefixed("service")] = service
desired_crd = V1beta1CustomResourceDefinition(
api_version=crd_config.get("apiVersion"),
kind=crd_config.get("kind"),
metadata=metadata,
spec=crd_config.get("spec"),
)
desired_crds.append(desired_crd)
return update_crds(
kube_client=kube_client,
desired_crds=desired_crds,
existing_crds=existing_crds,
)
def get_service_instance_list(name, cluster=None, instance_type=None, soa_dir=DEFAULT_SOA_DIR):
"""Enumerate the instances defined for a service as a list of tuples.
:param name: The service name
:param cluster: The cluster to read the configuration for
:param instance_type: The type of instances to examine: 'marathon', 'chronos', or None (default) for both
:param soa_dir: The SOA config directory to read from
:returns: A list of tuples of (name, instance) for each instance defined for the service name
"""
if not cluster:
cluster = load_system_paasta_config().get_cluster()
if instance_type == 'marathon' or instance_type == 'chronos':
instance_types = [instance_type]
else:
instance_types = ['marathon', 'chronos']
instance_list = []
for srv_instance_type in instance_types:
conf_file = "%s-%s" % (srv_instance_type, cluster)
log.info("Enumerating all instances for config file: %s/*/%s.yaml" % (soa_dir, conf_file))
instances = service_configuration_lib.read_extra_service_information(
name,
conf_file,
soa_dir=soa_dir
)
for instance in instances:
instance_list.append((name, instance))
log.debug("Enumerated the following instances: %s", instance_list)
return instance_list
def load_tron_service_config(service,
tron_cluster,
load_deployments=True,
soa_dir=DEFAULT_SOA_DIR):
"""Load all configured jobs for a service, and any additional config values."""
config = service_configuration_lib.read_extra_service_information(
service, 'tron-' + tron_cluster, soa_dir)
if not config:
tron_conf_path = os.path.join(
os.path.abspath(soa_dir),
'tron',
tron_cluster,
service + '.yaml',
)
config = service_configuration_lib._read_yaml_file(tron_conf_path)
if not config:
raise NoConfigurationForServiceError(
'No Tron configuration found for service %s' % service)
extra_config = {
key: value
for key, value in config.items() if key != 'jobs'
}
job_configs = [
TronJobConfig(
config_dict=job,
load_deployments=load_deployments,
soa_dir=soa_dir,
) for job in config.get('jobs') or []
]
return job_configs, extra_config
def get_existing_configs(self, service: str,
extra_info: str) -> Dict[str, Any]:
return read_extra_service_information(
service,
f"{AUTO_SOACONFIG_SUBDIR}/{extra_info}",
soa_dir=self.working_dir,
)
def load_tron_service_config_no_cache(
service,
cluster,
load_deployments=True,
soa_dir=DEFAULT_SOA_DIR,
for_validation=False,
):
"""Load all configured jobs for a service, and any additional config values."""
config = read_extra_service_information(
service_name=service, extra_info=f"tron-{cluster}", soa_dir=soa_dir
)
jobs = filter_templates_from_config(config)
job_configs = [
TronJobConfig(
name=name,
service=service,
cluster=cluster,
config_dict=job,
load_deployments=load_deployments,
soa_dir=soa_dir,
for_validation=for_validation,
)
for name, job in jobs.items()
]
return job_configs
def get_service_instance_list(name,
cluster=None,
instance_type=None,
soa_dir=DEFAULT_SOA_DIR):
"""Enumerate the instances defined for a service as a list of tuples.
:param name: The service name
:param cluster: The cluster to read the configuration for
:param instance_type: The type of instances to examine: 'marathon', 'chronos', or None (default) for both
:param soa_dir: The SOA config directory to read from
:returns: A list of tuples of (name, instance) for each instance defined for the service name
"""
if not cluster:
cluster = load_system_paasta_config().get_cluster()
if instance_type == 'marathon' or instance_type == 'chronos':
instance_types = [instance_type]
else:
instance_types = ['marathon', 'chronos']
instance_list = []
for srv_instance_type in instance_types:
conf_file = "%s-%s" % (srv_instance_type, cluster)
log.info("Enumerating all instances for config file: %s/*/%s.yaml" %
(soa_dir, conf_file))
instances = service_configuration_lib.read_extra_service_information(
name, conf_file, soa_dir=soa_dir)
for instance in instances:
instance_list.append((name, instance))
log.debug("Enumerated the following instances: %s", instance_list)
return instance_list
def load_adhoc_job_config(service, instance, cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR):
general_config = service_configuration_lib.read_service_configuration(
service,
soa_dir=soa_dir
)
adhoc_conf_file = "adhoc-%s" % cluster
log.info("Reading adhoc configuration file: %s.yaml", adhoc_conf_file)
instance_configs = service_configuration_lib.read_extra_service_information(
service_name=service,
extra_info=adhoc_conf_file,
soa_dir=soa_dir
)
if instance not in instance_configs:
raise NoConfigurationForServiceError(
"%s not found in config file %s/%s/%s.yaml." % (instance, soa_dir, service, adhoc_conf_file)
)
general_config = deep_merge_dictionaries(overrides=instance_configs[instance], defaults=general_config)
branch_dict = {}
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
branch = general_config.get('branch', get_paasta_branch(cluster, instance))
deploy_group = general_config.get('deploy_group', branch)
branch_dict = deployments_json.get_branch_dict_v2(service, branch, deploy_group)
return AdhocJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
)
def read_chronos_jobs_for_service(service, cluster, soa_dir=DEFAULT_SOA_DIR):
chronos_conf_file = 'chronos-%s' % cluster
return service_configuration_lib.read_extra_service_information(
service,
chronos_conf_file,
soa_dir=soa_dir,
)
def read_chronos_jobs_for_service(service, cluster, soa_dir=DEFAULT_SOA_DIR):
chronos_conf_file = 'chronos-%s' % cluster
log.info("Reading Chronos configuration file: %s/%s/chronos-%s.yaml" % (soa_dir, service, cluster))
return service_configuration_lib.read_extra_service_information(
service,
chronos_conf_file,
soa_dir=soa_dir
)
def read_paasta_native_jobs_for_service(service, cluster, soa_dir=DEFAULT_SOA_DIR):
paasta_native_conf_file = 'paasta_native-%s' % cluster
log.info("Reading paasta_native configuration file: %s/%s/paasta_native-%s.yaml" % (soa_dir, service, cluster))
return service_configuration_lib.read_extra_service_information(
service,
paasta_native_conf_file,
soa_dir=soa_dir
)
def read_namespace_for_service_instance(name, instance, cluster=None, soa_dir=DEFAULT_SOA_DIR):
"""Retreive a service instance's nerve namespace from its configuration file.
If one is not defined in the config file, returns instance instead."""
if not cluster:
cluster = load_system_paasta_config().get_cluster()
srv_info = service_configuration_lib.read_extra_service_information(name, "marathon-%s" % cluster, soa_dir)[
instance
]
return srv_info["nerve_ns"] if "nerve_ns" in srv_info else instance
def _refresh_framework_config(self, cluster: str,
instance_type_class: Type[InstanceConfig_T]):
conf_name = self._framework_config_filename(cluster,
instance_type_class)
log.info("Reading configuration file: %s.yaml", conf_name)
instances = read_extra_service_information(service_name=self._service,
extra_info=conf_name,
soa_dir=self._soa_dir)
self._framework_configs[(cluster, instance_type_class)] = instances
def load_marathon_service_config_no_cache(service,
instance,
cluster,
load_deployments=True,
soa_dir=DEFAULT_SOA_DIR):
"""Read a service instance's configuration for marathon.
If a branch isn't specified for a config, the 'branch' key defaults to
paasta-${cluster}.${instance}.
:param name: The service name
:param instance: The instance of the service to retrieve
:param cluster: The cluster to read the configuration for
:param load_deployments: A boolean indicating if the corresponding deployments.json for this service
should also be loaded
:param soa_dir: The SOA configuration directory to read from
:returns: A dictionary of whatever was in the config for the service instance"""
log.info("Reading service configuration files from dir %s/ in %s" %
(service, soa_dir))
log.info("Reading general configuration file: service.yaml")
general_config = service_configuration_lib.read_service_configuration(
service,
soa_dir=soa_dir,
)
marathon_conf_file = "marathon-%s" % cluster
log.info("Reading marathon configuration file: %s.yaml",
marathon_conf_file)
instance_configs = service_configuration_lib.read_extra_service_information(
service,
marathon_conf_file,
soa_dir=soa_dir,
)
if instance not in instance_configs:
raise NoConfigurationForServiceError(
"%s not found in config file %s/%s/%s.yaml." %
(instance, soa_dir, service, marathon_conf_file))
general_config = deep_merge_dictionaries(
overrides=instance_configs[instance], defaults=general_config)
branch_dict = {}
if load_deployments:
deployments_json = load_deployments_json(service, soa_dir=soa_dir)
branch = general_config.get('branch',
get_paasta_branch(cluster, instance))
branch_dict = deployments_json.get_branch_dict(service, branch)
return MarathonServiceConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
def get_existing_configs(self,
service: str,
extra_info: str,
sub_dir: Optional[str] = None) -> Dict[str, Any]:
path = f"{sub_dir}/{extra_info}" if sub_dir else extra_info
return read_extra_service_information(
service,
path,
soa_dir=self.working_dir,
)
def read_paasta_native_jobs_for_service(service,
cluster,
soa_dir=DEFAULT_SOA_DIR):
paasta_native_conf_file = 'paasta_native-%s' % cluster
log.info(
"Reading paasta_native configuration file: %s/%s/paasta_native-%s.yaml"
% (soa_dir, service, cluster))
return service_configuration_lib.read_extra_service_information(
service, paasta_native_conf_file, soa_dir=soa_dir)
def test_read_extra_service_information(self, info_patch, abs_patch, join_patch):
expected = {'what': 'info'}
actual = service_configuration_lib.read_extra_service_information(
'noname',
'noinfo', soa_dir='whatsadir',
)
abs_patch.assert_called_once_with('whatsadir')
join_patch.assert_called_once_with('real_soa_dir', 'noname', 'noinfo.yaml')
info_patch.assert_called_once_with('together_forever', deepcopy=True)
assert expected == actual
def read_namespace_for_service_instance(name,
instance,
cluster=None,
soa_dir=DEFAULT_SOA_DIR):
"""Retreive a service instance's nerve namespace from its configuration file.
If one is not defined in the config file, returns instance instead."""
if not cluster:
cluster = load_system_paasta_config().get_cluster()
srv_info = service_configuration_lib.read_extra_service_information(
name, "marathon-%s" % cluster, soa_dir)[instance]
return srv_info['nerve_ns'] if 'nerve_ns' in srv_info else instance
def load_all_configs(cluster: str, file_prefix: str,
soa_dir: str) -> Mapping[str, Mapping[str, Any]]:
config_dicts = {}
for service in os.listdir(soa_dir):
config_dicts[
service] = service_configuration_lib.read_extra_service_information(
service,
f"{file_prefix}-{cluster}",
soa_dir=soa_dir,
)
return config_dicts
def load_tron_yaml(service: str, cluster: str, soa_dir: str) -> Dict[str, Any]:
tronfig_folder = get_tronfig_folder(soa_dir=soa_dir, cluster=cluster)
config = service_configuration_lib.read_extra_service_information(
service_name=service,
extra_info=f'tron-{cluster}',
soa_dir=soa_dir,
)
if not config:
config = service_configuration_lib._read_yaml_file(os.path.join(tronfig_folder, f"{service}.yaml"))
if not config:
raise NoConfigurationForServiceError('No Tron configuration found for service %s' % service)
return config
def load_adhoc_job_config(service,
instance,
cluster,
load_deployments=True,
soa_dir=DEFAULT_SOA_DIR):
general_config = service_configuration_lib.read_service_configuration(
service,
soa_dir=soa_dir,
)
adhoc_conf_file = "adhoc-%s" % cluster
instance_configs = service_configuration_lib.read_extra_service_information(
service_name=service,
extra_info=adhoc_conf_file,
soa_dir=soa_dir,
)
if instance not in instance_configs:
raise NoConfigurationForServiceError(
"%s not found in config file %s/%s/%s.yaml." %
(instance, soa_dir, service, adhoc_conf_file), )
general_config = deep_merge_dictionaries(
overrides=instance_configs[instance], defaults=general_config)
branch_dict = None
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
temp_instance_config = AdhocJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=None,
soa_dir=soa_dir,
)
branch = temp_instance_config.get_branch()
deploy_group = temp_instance_config.get_deploy_group()
branch_dict = deployments_json.get_branch_dict(service, branch,
deploy_group)
return AdhocJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
def load_marathon_service_config(service, instance, cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR):
"""Read a service instance's configuration for marathon.
If a branch isn't specified for a config, the 'branch' key defaults to
paasta-${cluster}.${instance}.
:param name: The service name
:param instance: The instance of the service to retrieve
:param cluster: The cluster to read the configuration for
:param load_deployments: A boolean indicating if the corresponding deployments.json for this service
should also be loaded
:param soa_dir: The SOA configuration directory to read from
:returns: A dictionary of whatever was in the config for the service instance"""
log.info("Reading service configuration files from dir %s/ in %s" % (service, soa_dir))
log.info("Reading general configuration file: service.yaml")
general_config = service_configuration_lib.read_service_configuration(
service,
soa_dir=soa_dir
)
marathon_conf_file = "marathon-%s" % cluster
log.info("Reading marathon configuration file: %s.yaml", marathon_conf_file)
instance_configs = service_configuration_lib.read_extra_service_information(
service,
marathon_conf_file,
soa_dir=soa_dir
)
if instance not in instance_configs:
raise NoConfigurationForServiceError(
"%s not found in config file %s/%s/%s.yaml." % (instance, soa_dir, service, marathon_conf_file)
)
general_config = deep_merge_dictionaries(overrides=instance_configs[instance], defaults=general_config)
branch_dict = {}
if load_deployments:
deployments_json = load_deployments_json(service, soa_dir=soa_dir)
branch = general_config.get('branch', get_paasta_branch(cluster, instance))
branch_dict = deployments_json.get_branch_dict(service, branch)
return MarathonServiceConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
)
def cleanup_kube_crd(
kube_client: KubeClient,
cluster: str,
soa_dir: str = DEFAULT_SOA_DIR,
dry_run: bool = False,
) -> bool:
existing_crds = kube_client.apiextensions.list_custom_resource_definition(
label_selector="yelp.com/paasta_service", )
success = True
for crd in existing_crds.items:
service = crd.metadata.labels['yelp.com/paasta_service']
if not service:
log.error(
f"CRD {crd.metadata.name} has empty paasta_service label", )
continue
crd_config = service_configuration_lib.read_extra_service_information(
service,
f'crd-{cluster}',
soa_dir=soa_dir,
)
if crd_config:
log.debug(
f"CRD {crd.metadata.name} declaration found in {service}")
continue
log.info(f"CRD {crd.metadata.name} not found in {service} service")
if dry_run:
log.info("not deleting in dry-run mode")
continue
try:
kube_client.apiextensions.delete_custom_resource_definition(
name=crd.metadata.name,
body=V1DeleteOptions(),
)
log.info(f"deleted {crd.metadata.name} for {cluster}:{service}")
except ApiException as exc:
log.error(
f"error deploying crd for {cluster}:{service}, "
f"status: {exc.status}, reason: {exc.reason}", )
log.debug(exc.body)
success = False
return success
def read_service_config(service,
instance,
instance_type,
cluster,
soa_dir=DEFAULT_SOA_DIR):
conf_file = '%s-%s' % (instance_type, cluster)
full_path = '%s/%s/%s.yaml' % (soa_dir, service, conf_file)
paasta_print("Reading paasta-remote configuration file: %s" % full_path)
config = service_configuration_lib.read_extra_service_information(
service, conf_file, soa_dir=soa_dir)
if instance not in config:
raise UnknownNativeServiceError(
'No job named "%s" in config file %s: \n%s' %
(instance, full_path, open(full_path).read()))
return config
def read_service_config(service,
instance,
instance_type,
cluster,
soa_dir=DEFAULT_SOA_DIR):
conf_file = f"{instance_type}-{cluster}"
full_path = f"{soa_dir}/{service}/{conf_file}.yaml"
paasta_print("Reading paasta-remote configuration file: %s" % full_path)
config = service_configuration_lib.read_extra_service_information(
service, conf_file, soa_dir=soa_dir)
if instance not in config:
raise UnknownNativeServiceError(
'No job named "{}" in config file {}: \n{}'.format(
instance, full_path,
open(full_path).read()))
return config
def load_kubernetes_service_config_no_cache(
service: str,
instance: str,
cluster: str,
load_deployments: bool = True,
soa_dir: str = DEFAULT_SOA_DIR,
) -> "KubernetesDeploymentConfig":
"""Read a service instance's configuration for kubernetes.
If a branch isn't specified for a config, the 'branch' key defaults to
paasta-${cluster}.${instance}.
:param name: The service name
:param instance: The instance of the service to retrieve
:param cluster: The cluster to read the configuration for
:param load_deployments: A boolean indicating if the corresponding deployments.json for this service
should also be loaded
:param soa_dir: The SOA configuration directory to read from
:returns: A dictionary of whatever was in the config for the service instance"""
general_config = service_configuration_lib.read_service_configuration(
service,
soa_dir=soa_dir,
)
kubernetes_conf_file = "kubernetes-%s" % cluster
instance_configs = service_configuration_lib.read_extra_service_information(
service,
kubernetes_conf_file,
soa_dir=soa_dir,
)
if instance.startswith('_'):
raise InvalidJobNameError(
f"Unable to load kubernetes job config for {service}.{instance} as instance name starts with '_'",
)
if instance not in instance_configs:
raise NoConfigurationForServiceError(
f"{instance} not found in config file {soa_dir}/{service}/{kubernetes_conf_file}.yaml.",
)
general_config = deep_merge_dictionaries(
overrides=instance_configs[instance], defaults=general_config)
branch_dict: Optional[BranchDictV2] = None
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
temp_instance_config = KubernetesDeploymentConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=None,
soa_dir=soa_dir,
)
branch = temp_instance_config.get_branch()
deploy_group = temp_instance_config.get_deploy_group()
branch_dict = deployments_json.get_branch_dict(service, branch,
deploy_group)
return KubernetesDeploymentConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
def load_service_namespace_config(
service: str,
namespace: str,
soa_dir: str = DEFAULT_SOA_DIR) -> ServiceNamespaceConfig:
"""Attempt to read the configuration for a service's namespace in a more strict fashion.
Retrieves the following keys:
- proxy_port: the proxy port defined for the given namespace
- healthcheck_mode: the mode for the healthcheck (http or tcp)
- healthcheck_port: An alternate port to use for health checking
- healthcheck_uri: URI target for healthchecking
- healthcheck_timeout_s: healthcheck timeout in seconds
- healthcheck_body_expect: an expected string in healthcheck response body
- updown_timeout_s: updown_service timeout in seconds
- timeout_connect_ms: proxy frontend timeout in milliseconds
- timeout_server_ms: proxy server backend timeout in milliseconds
- timeout_client_ms: proxy server client timeout in milliseconds
- retries: the number of retries on a proxy backend
- mode: the mode the service is run in (http or tcp)
- routes: a list of tuples of (source, destination)
- discover: the scope at which to discover services e.g. 'habitat'
- advertise: a list of scopes to advertise services at e.g. ['habitat', 'region']
- extra_advertise: a list of tuples of (source, destination)
e.g. [('region:dc6-prod', 'region:useast1-prod')]
- extra_healthcheck_headers: a dict of HTTP headers that must
be supplied when health checking. E.g. { 'Host': 'example.com' }
:param service: The service name
:param namespace: The namespace to read
:param soa_dir: The SOA config directory to read from
:returns: A dict of the above keys, if they were defined
"""
smartstack_config = service_configuration_lib.read_extra_service_information(
service_name=service,
extra_info="smartstack",
soa_dir=soa_dir,
deepcopy=False,
)
namespace_config_from_file = smartstack_config.get(namespace, {})
service_namespace_config = ServiceNamespaceConfig()
# We can't really use .get, as we don't want the key to be in the returned
# dict at all if it doesn't exist in the config file.
# We also can't just copy the whole dict, as we only care about some keys
# and there's other things that appear in the smartstack section in
# several cases.
key_whitelist = {
"healthcheck_mode",
"healthcheck_uri",
"healthcheck_port",
"healthcheck_timeout_s",
"healthcheck_body_expect",
"updown_timeout_s",
"proxy_port",
"timeout_connect_ms",
"timeout_server_ms",
"timeout_client_ms",
"retries",
"mode",
"discover",
"advertise",
"extra_healthcheck_headers",
}
for key, value in namespace_config_from_file.items():
if key in key_whitelist:
service_namespace_config[key] = value
# Other code in paasta_tools checks 'mode' after the config file
# is loaded, so this ensures that it is set to the appropriate default
# if not otherwise specified, even if appropriate default is None.
service_namespace_config["mode"] = service_namespace_config.get_mode()
if "routes" in namespace_config_from_file:
service_namespace_config["routes"] = [
(route["source"], dest)
for route in namespace_config_from_file["routes"]
for dest in route["destinations"]
]
if "extra_advertise" in namespace_config_from_file:
service_namespace_config["extra_advertise"] = [
(src, dst) for src in namespace_config_from_file["extra_advertise"]
for dst in namespace_config_from_file["extra_advertise"][src]
]
return service_namespace_config
def load_performance_check_config(service, soa_dir):
return read_extra_service_information(
service_name=service,
extra_info='performance-check',
soa_dir=soa_dir,
)
def load_performance_check_config(service, soa_dir):
return read_extra_service_information(
service_name=service,
extra_info='performance-check',
soa_dir=soa_dir,
)
def setup_kube_crd(
kube_client: KubeClient,
cluster: str,
services: Sequence[str],
soa_dir: str = DEFAULT_SOA_DIR,
) -> bool:
existing_crds = kube_client.apiextensions.list_custom_resource_definition(
label_selector="paasta.yelp.com/service")
success = True
for service in services:
crd_config = service_configuration_lib.read_extra_service_information(
service, f"crd-{cluster}", soa_dir=soa_dir)
if not crd_config:
log.info("nothing to deploy")
continue
metadata = crd_config.get("metadata", {})
if "labels" not in metadata:
metadata["labels"] = {}
metadata["labels"]["yelp.com/paasta_service"] = service
metadata["labels"]["paasta.yelp.com/service"] = service
desired_crd = V1beta1CustomResourceDefinition(
api_version=crd_config.get("apiVersion"),
kind=crd_config.get("kind"),
metadata=metadata,
spec=crd_config.get("spec"),
)
existing_crd = None
for crd in existing_crds.items:
if crd.metadata.name == desired_crd.metadata["name"]:
existing_crd = crd
break
try:
if existing_crd:
desired_crd.metadata[
"resourceVersion"] = existing_crd.metadata.resource_version
kube_client.apiextensions.replace_custom_resource_definition(
name=desired_crd.metadata["name"], body=desired_crd)
else:
try:
kube_client.apiextensions.create_custom_resource_definition(
body=desired_crd)
except ValueError as err:
# TODO: kubernetes server will sometimes reply with conditions:null,
# figure out how to deal with this correctly, for more details:
# https://github.com/kubernetes/kubernetes/pull/64996
if "`conditions`, must not be `None`" in str(err):
pass
else:
raise err
log.info(
f"deployed {desired_crd.metadata['name']} for {cluster}:{service}"
)
except ApiException as exc:
log.error(f"error deploying crd for {cluster}:{service}, "
f"status: {exc.status}, reason: {exc.reason}")
log.debug(exc.body)
success = False
return success
