def load_chronos_job_config(service, instance, cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR):
general_config = service_configuration_lib.read_service_configuration(
service,
soa_dir=soa_dir,
)
service_chronos_jobs = read_chronos_jobs_for_service(service, cluster, soa_dir=soa_dir)
if instance not in service_chronos_jobs:
raise NoConfigurationForServiceError('No job named "%s" in config file chronos-%s.yaml' % (instance, cluster))
branch_dict = {}
if load_deployments:
deployments_json = load_deployments_json(service, soa_dir=soa_dir)
branch = get_paasta_branch(cluster=cluster, instance=instance)
branch_dict = deployments_json.get_branch_dict(service, branch)
general_config = deep_merge_dictionaries(overrides=service_chronos_jobs[instance], defaults=general_config)
return ChronosJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
def get_autoscaling_params(self):
default_params = {
'metrics_provider': 'mesos_cpu',
'decision_policy': 'pid',
'setpoint': 0.8,
}
return deep_merge_dictionaries(overrides=self.config_dict.get('autoscaling', {}), defaults=default_params)
def load_adhoc_job_config(service, instance, cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR):
general_config = service_configuration_lib.read_service_configuration(
service,
soa_dir=soa_dir
)
adhoc_conf_file = "adhoc-%s" % cluster
log.info("Reading adhoc configuration file: %s.yaml", adhoc_conf_file)
instance_configs = service_configuration_lib.read_extra_service_information(
service_name=service,
extra_info=adhoc_conf_file,
soa_dir=soa_dir
)
if instance not in instance_configs:
raise NoConfigurationForServiceError(
"%s not found in config file %s/%s/%s.yaml." % (instance, soa_dir, service, adhoc_conf_file)
)
general_config = deep_merge_dictionaries(overrides=instance_configs[instance], defaults=general_config)
branch_dict = {}
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
branch = general_config.get('branch', get_paasta_branch(cluster, instance))
deploy_group = general_config.get('deploy_group', branch)
branch_dict = deployments_json.get_branch_dict_v2(service, branch, deploy_group)
return AdhocJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
)
def load_adhoc_job_config(service, instance, cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR):
general_config = service_configuration_lib.read_service_configuration(
service,
soa_dir=soa_dir
)
adhoc_conf_file = "adhoc-%s" % cluster
log.info("Reading adhoc configuration file: %s.yaml", adhoc_conf_file)
instance_configs = service_configuration_lib.read_extra_service_information(
service_name=service,
extra_info=adhoc_conf_file,
soa_dir=soa_dir
)
if instance not in instance_configs:
raise NoConfigurationForServiceError(
"%s not found in config file %s/%s/%s.yaml." % (instance, soa_dir, service, adhoc_conf_file)
)
general_config = deep_merge_dictionaries(overrides=instance_configs[instance], defaults=general_config)
branch_dict = {}
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
branch = general_config.get('branch', get_paasta_branch(cluster, instance))
deploy_group = general_config.get('deploy_group', branch)
branch_dict = deployments_json.get_branch_dict_v2(service, branch, deploy_group)
return AdhocJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
)
def test_deep_merge_dictionaries():
source = {
"common_key": "value",
"common_dict": {"subkey1": 1, "subkey2": 2, "subkey3": 3},
"just_in_source": "value",
"just_in_source_dict": {"key": "value"},
"overwriting_key": "value",
"overwriting_dict": {"test": "value"},
}
destination = {
"common_key": "overwritten_value",
"common_dict": {"subkey1": "overwritten_value", "subkey4": 4, "subkey5": 5},
"just_in_dest": "value",
"just_in_dest_dict": {"key": "value"},
"overwriting_key": {"overwritten-key", "overwritten-value"},
"overwriting_dict": "overwritten-value",
}
expected = {
"common_key": "value",
"common_dict": {"subkey1": 1, "subkey2": 2, "subkey3": 3, "subkey4": 4, "subkey5": 5},
"just_in_source": "value",
"just_in_source_dict": {"key": "value"},
"just_in_dest": "value",
"just_in_dest_dict": {"key": "value"},
"overwriting_key": "value",
"overwriting_dict": {"test": "value"},
}
assert utils.deep_merge_dictionaries(source, destination) == expected
def get_autoscaling_params(self):
default_params = {
'metrics_provider': 'mesos_cpu',
'decision_policy': 'pid',
'setpoint': 0.8,
}
return deep_merge_dictionaries(overrides=self.config_dict.get('autoscaling', {}), defaults=default_params)
def _get_merged_config(
self,
config: utils.InstanceConfigDict) -> utils.InstanceConfigDict:
if self._general_config is None:
self._general_config = read_service_configuration(
service_name=self._service, soa_dir=self._soa_dir)
return deep_merge_dictionaries(overrides=config,
defaults=self._general_config)
def load_marathon_service_config_no_cache(service,
instance,
cluster,
load_deployments=True,
soa_dir=DEFAULT_SOA_DIR):
"""Read a service instance's configuration for marathon.
If a branch isn't specified for a config, the 'branch' key defaults to
paasta-${cluster}.${instance}.
:param name: The service name
:param instance: The instance of the service to retrieve
:param cluster: The cluster to read the configuration for
:param load_deployments: A boolean indicating if the corresponding deployments.json for this service
should also be loaded
:param soa_dir: The SOA configuration directory to read from
:returns: A dictionary of whatever was in the config for the service instance"""
log.info("Reading service configuration files from dir %s/ in %s" %
(service, soa_dir))
log.info("Reading general configuration file: service.yaml")
general_config = service_configuration_lib.read_service_configuration(
service,
soa_dir=soa_dir,
)
marathon_conf_file = "marathon-%s" % cluster
log.info("Reading marathon configuration file: %s.yaml",
marathon_conf_file)
instance_configs = service_configuration_lib.read_extra_service_information(
service,
marathon_conf_file,
soa_dir=soa_dir,
)
if instance not in instance_configs:
raise NoConfigurationForServiceError(
"%s not found in config file %s/%s/%s.yaml." %
(instance, soa_dir, service, marathon_conf_file))
general_config = deep_merge_dictionaries(
overrides=instance_configs[instance], defaults=general_config)
branch_dict = {}
if load_deployments:
deployments_json = load_deployments_json(service, soa_dir=soa_dir)
branch = general_config.get('branch',
get_paasta_branch(cluster, instance))
branch_dict = deployments_json.get_branch_dict(service, branch)
return MarathonServiceConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
def get_autoscaling_params(self) -> AutoscalingParamsDict:
default_params: AutoscalingParamsDict = {
"metrics_provider": "mesos_cpu",
"decision_policy": "proportional",
"setpoint": 0.8,
}
return deep_merge_dictionaries(
overrides=self.config_dict.get("autoscaling", AutoscalingParamsDict({})),
defaults=default_params,
)
def _get_merged_config(self, config):
if self._general_config is None:
self._general_config = read_service_configuration(
service_name=self._service,
soa_dir=self._soa_dir,
)
return deep_merge_dictionaries(
overrides=config,
defaults=self._general_config,
)
def load_monkrelaycluster_instance_config(
service: str,
instance: str,
cluster: str,
load_deployments: bool = True,
soa_dir: str = DEFAULT_SOA_DIR,
) -> MonkRelayClusterDeploymentConfig:
"""Read a service instance's configuration for MonkRelayCluster.
If a branch isn't specified for a config, the 'branch' key defaults to
paasta-${cluster}.${instance}.
:param service: The service name
:param instance: The instance of the service to retrieve
:param cluster: The cluster to read the configuration for
:param load_deployments: A boolean indicating if the corresponding deployments.json for this service
should also be loaded
:param soa_dir: The SOA configuration directory to read from
:returns: A dictionary of whatever was in the config for the service instance"""
general_config = service_configuration_lib.read_service_configuration(
service, soa_dir=soa_dir)
instance_config = load_service_instance_config(service,
instance,
"monkrelays",
cluster,
soa_dir=soa_dir)
general_config = deep_merge_dictionaries(overrides=instance_config,
defaults=general_config)
branch_dict: Optional[BranchDictV2] = None
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
temp_instance_config = MonkRelayClusterDeploymentConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=None,
soa_dir=soa_dir,
)
branch = temp_instance_config.get_branch()
deploy_group = temp_instance_config.get_deploy_group()
branch_dict = deployments_json.get_branch_dict(service, branch,
deploy_group)
return MonkRelayClusterDeploymentConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
def load_chronos_job_config(
service: str,
instance: str,
cluster: str,
load_deployments: bool = True,
soa_dir: str = DEFAULT_SOA_DIR,
) -> 'ChronosJobConfig':
general_config = service_configuration_lib.read_service_configuration(
service,
soa_dir=soa_dir,
)
if instance.startswith('_'):
raise InvalidJobNameError(
"Unable to load chronos job config for %s.%s as instance name starts with '_'"
% (service, instance), )
service_chronos_jobs = read_chronos_jobs_for_service(service,
cluster,
soa_dir=soa_dir)
if instance not in service_chronos_jobs:
raise NoConfigurationForServiceError(
'No job named "%s" in config file chronos-%s.yaml' %
(instance, cluster))
branch_dict = None
general_config = deep_merge_dictionaries(
overrides=service_chronos_jobs[instance], defaults=general_config)
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
temp_instance_config = ChronosJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=None,
soa_dir=soa_dir,
)
branch = temp_instance_config.get_branch()
deploy_group = temp_instance_config.get_deploy_group()
branch_dict = deployments_json.get_branch_dict(service, branch,
deploy_group)
return ChronosJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
def load_adhoc_job_config(service,
instance,
cluster,
load_deployments=True,
soa_dir=DEFAULT_SOA_DIR):
general_config = service_configuration_lib.read_service_configuration(
service,
soa_dir=soa_dir,
)
adhoc_conf_file = "adhoc-%s" % cluster
instance_configs = service_configuration_lib.read_extra_service_information(
service_name=service,
extra_info=adhoc_conf_file,
soa_dir=soa_dir,
)
if instance not in instance_configs:
raise NoConfigurationForServiceError(
"%s not found in config file %s/%s/%s.yaml." %
(instance, soa_dir, service, adhoc_conf_file), )
general_config = deep_merge_dictionaries(
overrides=instance_configs[instance], defaults=general_config)
branch_dict = None
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
temp_instance_config = AdhocJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=None,
soa_dir=soa_dir,
)
branch = temp_instance_config.get_branch()
deploy_group = temp_instance_config.get_deploy_group()
branch_dict = deployments_json.get_branch_dict(service, branch,
deploy_group)
return AdhocJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
def load_marathon_service_config(service, instance, cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR):
"""Read a service instance's configuration for marathon.
If a branch isn't specified for a config, the 'branch' key defaults to
paasta-${cluster}.${instance}.
:param name: The service name
:param instance: The instance of the service to retrieve
:param cluster: The cluster to read the configuration for
:param load_deployments: A boolean indicating if the corresponding deployments.json for this service
should also be loaded
:param soa_dir: The SOA configuration directory to read from
:returns: A dictionary of whatever was in the config for the service instance"""
log.info("Reading service configuration files from dir %s/ in %s" % (service, soa_dir))
log.info("Reading general configuration file: service.yaml")
general_config = service_configuration_lib.read_service_configuration(
service,
soa_dir=soa_dir
)
marathon_conf_file = "marathon-%s" % cluster
log.info("Reading marathon configuration file: %s.yaml", marathon_conf_file)
instance_configs = service_configuration_lib.read_extra_service_information(
service,
marathon_conf_file,
soa_dir=soa_dir
)
if instance not in instance_configs:
raise NoConfigurationForServiceError(
"%s not found in config file %s/%s/%s.yaml." % (instance, soa_dir, service, marathon_conf_file)
)
general_config = deep_merge_dictionaries(overrides=instance_configs[instance], defaults=general_config)
branch_dict = {}
if load_deployments:
deployments_json = load_deployments_json(service, soa_dir=soa_dir)
branch = general_config.get('branch', get_paasta_branch(cluster, instance))
branch_dict = deployments_json.get_branch_dict(service, branch)
return MarathonServiceConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
)
def test_deep_merge_dictionaries():
overrides = {
'common_key': 'value',
'common_dict': {
'subkey1': 1,
'subkey2': 2,
'subkey3': 3,
},
'just_in_overrides': 'value',
'just_in_overrides_dict': {'key': 'value'},
'overwriting_key': 'value',
'overwriting_dict': {'test': 'value'},
}
defaults = {
'common_key': 'overwritten_value',
'common_dict': {
'subkey1': 'overwritten_value',
'subkey4': 4,
'subkey5': 5,
},
'just_in_defaults': 'value',
'just_in_defaults_dict': {'key': 'value'},
'overwriting_key': {'overwritten-key', 'overwritten-value'},
'overwriting_dict': 'overwritten-value',
}
expected = {
'common_key': 'value',
'common_dict': {
'subkey1': 1,
'subkey2': 2,
'subkey3': 3,
'subkey4': 4,
'subkey5': 5,
},
'just_in_overrides': 'value',
'just_in_overrides_dict': {'key': 'value'},
'just_in_defaults': 'value',
'just_in_defaults_dict': {'key': 'value'},
'overwriting_key': 'value',
'overwriting_dict': {'test': 'value'},
}
assert utils.deep_merge_dictionaries(overrides, defaults) == expected
def test_deep_merge_dictionaries():
overrides = {
'common_key': 'value',
'common_dict': {
'subkey1': 1,
'subkey2': 2,
'subkey3': 3,
},
'just_in_overrides': 'value',
'just_in_overrides_dict': {'key': 'value'},
'overwriting_key': 'value',
'overwriting_dict': {'test': 'value'},
}
defaults = {
'common_key': 'overwritten_value',
'common_dict': {
'subkey1': 'overwritten_value',
'subkey4': 4,
'subkey5': 5,
},
'just_in_defaults': 'value',
'just_in_defaults_dict': {'key': 'value'},
'overwriting_key': {'overwritten-key', 'overwritten-value'},
'overwriting_dict': 'overwritten-value',
}
expected = {
'common_key': 'value',
'common_dict': {
'subkey1': 1,
'subkey2': 2,
'subkey3': 3,
'subkey4': 4,
'subkey5': 5,
},
'just_in_overrides': 'value',
'just_in_overrides_dict': {'key': 'value'},
'just_in_defaults': 'value',
'just_in_defaults_dict': {'key': 'value'},
'overwriting_key': 'value',
'overwriting_dict': {'test': 'value'},
}
assert utils.deep_merge_dictionaries(overrides, defaults) == expected
def load_vitess_instance_config(
service: str,
instance: str,
cluster: str,
load_deployments: bool = True,
soa_dir: str = DEFAULT_SOA_DIR,
) -> VitessDeploymentConfig:
general_config = service_configuration_lib.read_service_configuration(
service, soa_dir=soa_dir)
instance_config = load_service_instance_config(service,
instance,
" vitesscluster",
cluster,
soa_dir=soa_dir)
general_config = deep_merge_dictionaries(overrides=instance_config,
defaults=general_config)
branch_dict: Optional[BranchDictV2] = None
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
temp_instance_config = VitessDeploymentConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=None,
soa_dir=soa_dir,
)
branch = temp_instance_config.get_branch()
deploy_group = temp_instance_config.get_deploy_group()
branch_dict = deployments_json.get_branch_dict(service, branch,
deploy_group)
return VitessDeploymentConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
def load_adhoc_job_config(
service, instance, cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR
):
general_config = service_configuration_lib.read_service_configuration(
service, soa_dir=soa_dir
)
instance_config = load_service_instance_config(
service=service,
instance=instance,
instance_type="adhoc",
cluster=cluster,
soa_dir=soa_dir,
)
general_config = deep_merge_dictionaries(
overrides=instance_config, defaults=general_config
)
branch_dict = None
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
temp_instance_config = AdhocJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=None,
soa_dir=soa_dir,
)
branch = temp_instance_config.get_branch()
deploy_group = temp_instance_config.get_deploy_group()
branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group)
return AdhocJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
def load_kubernetes_service_config_no_cache(
service: str,
instance: str,
cluster: str,
load_deployments: bool = True,
soa_dir: str = DEFAULT_SOA_DIR,
) -> "KubernetesDeploymentConfig":
"""Read a service instance's configuration for kubernetes.
If a branch isn't specified for a config, the 'branch' key defaults to
paasta-${cluster}.${instance}.
:param name: The service name
:param instance: The instance of the service to retrieve
:param cluster: The cluster to read the configuration for
:param load_deployments: A boolean indicating if the corresponding deployments.json for this service
should also be loaded
:param soa_dir: The SOA configuration directory to read from
:returns: A dictionary of whatever was in the config for the service instance"""
general_config = service_configuration_lib.read_service_configuration(
service,
soa_dir=soa_dir,
)
kubernetes_conf_file = "kubernetes-%s" % cluster
instance_configs = service_configuration_lib.read_extra_service_information(
service,
kubernetes_conf_file,
soa_dir=soa_dir,
)
if instance.startswith('_'):
raise InvalidJobNameError(
f"Unable to load kubernetes job config for {service}.{instance} as instance name starts with '_'",
)
if instance not in instance_configs:
raise NoConfigurationForServiceError(
f"{instance} not found in config file {soa_dir}/{service}/{kubernetes_conf_file}.yaml.",
)
general_config = deep_merge_dictionaries(
overrides=instance_configs[instance], defaults=general_config)
branch_dict: Optional[BranchDictV2] = None
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
temp_instance_config = KubernetesDeploymentConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=None,
soa_dir=soa_dir,
)
branch = temp_instance_config.get_branch()
deploy_group = temp_instance_config.get_deploy_group()
branch_dict = deployments_json.get_branch_dict(service, branch,
deploy_group)
return KubernetesDeploymentConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
def get_autoscaling_params(self):
default_params = {
'method': 'default',
}
return deep_merge_dictionaries(source=self.config_dict.get('autoscaling', {}), destination=default_params)
def get_autoscaling_params(self):
default_params = {"metrics_provider": "mesos_cpu", "decision_policy": "pid", "setpoint": 0.8}
return deep_merge_dictionaries(overrides=self.config_dict.get("autoscaling", {}), defaults=default_params)
