def load_chronos_job_config(service, instance, cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR): general_config = service_configuration_lib.read_service_configuration( service, soa_dir=soa_dir, ) service_chronos_jobs = read_chronos_jobs_for_service(service, cluster, soa_dir=soa_dir) if instance not in service_chronos_jobs: raise NoConfigurationForServiceError('No job named "%s" in config file chronos-%s.yaml' % (instance, cluster)) branch_dict = {} if load_deployments: deployments_json = load_deployments_json(service, soa_dir=soa_dir) branch = get_paasta_branch(cluster=cluster, instance=instance) branch_dict = deployments_json.get_branch_dict(service, branch) general_config = deep_merge_dictionaries(overrides=service_chronos_jobs[instance], defaults=general_config) return ChronosJobConfig( service=service, cluster=cluster, instance=instance, config_dict=general_config, branch_dict=branch_dict, soa_dir=soa_dir, )
def get_autoscaling_params(self): default_params = { 'metrics_provider': 'mesos_cpu', 'decision_policy': 'pid', 'setpoint': 0.8, } return deep_merge_dictionaries(overrides=self.config_dict.get('autoscaling', {}), defaults=default_params)
def load_adhoc_job_config(service, instance, cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR): general_config = service_configuration_lib.read_service_configuration( service, soa_dir=soa_dir ) adhoc_conf_file = "adhoc-%s" % cluster log.info("Reading adhoc configuration file: %s.yaml", adhoc_conf_file) instance_configs = service_configuration_lib.read_extra_service_information( service_name=service, extra_info=adhoc_conf_file, soa_dir=soa_dir ) if instance not in instance_configs: raise NoConfigurationForServiceError( "%s not found in config file %s/%s/%s.yaml." % (instance, soa_dir, service, adhoc_conf_file) ) general_config = deep_merge_dictionaries(overrides=instance_configs[instance], defaults=general_config) branch_dict = {} if load_deployments: deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir) branch = general_config.get('branch', get_paasta_branch(cluster, instance)) deploy_group = general_config.get('deploy_group', branch) branch_dict = deployments_json.get_branch_dict_v2(service, branch, deploy_group) return AdhocJobConfig( service=service, cluster=cluster, instance=instance, config_dict=general_config, branch_dict=branch_dict, )
def test_deep_merge_dictionaries(): source = { "common_key": "value", "common_dict": {"subkey1": 1, "subkey2": 2, "subkey3": 3}, "just_in_source": "value", "just_in_source_dict": {"key": "value"}, "overwriting_key": "value", "overwriting_dict": {"test": "value"}, } destination = { "common_key": "overwritten_value", "common_dict": {"subkey1": "overwritten_value", "subkey4": 4, "subkey5": 5}, "just_in_dest": "value", "just_in_dest_dict": {"key": "value"}, "overwriting_key": {"overwritten-key", "overwritten-value"}, "overwriting_dict": "overwritten-value", } expected = { "common_key": "value", "common_dict": {"subkey1": 1, "subkey2": 2, "subkey3": 3, "subkey4": 4, "subkey5": 5}, "just_in_source": "value", "just_in_source_dict": {"key": "value"}, "just_in_dest": "value", "just_in_dest_dict": {"key": "value"}, "overwriting_key": "value", "overwriting_dict": {"test": "value"}, } assert utils.deep_merge_dictionaries(source, destination) == expected
def _get_merged_config( self, config: utils.InstanceConfigDict) -> utils.InstanceConfigDict: if self._general_config is None: self._general_config = read_service_configuration( service_name=self._service, soa_dir=self._soa_dir) return deep_merge_dictionaries(overrides=config, defaults=self._general_config)
def load_marathon_service_config_no_cache(service, instance, cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR): """Read a service instance's configuration for marathon. If a branch isn't specified for a config, the 'branch' key defaults to paasta-${cluster}.${instance}. :param name: The service name :param instance: The instance of the service to retrieve :param cluster: The cluster to read the configuration for :param load_deployments: A boolean indicating if the corresponding deployments.json for this service should also be loaded :param soa_dir: The SOA configuration directory to read from :returns: A dictionary of whatever was in the config for the service instance""" log.info("Reading service configuration files from dir %s/ in %s" % (service, soa_dir)) log.info("Reading general configuration file: service.yaml") general_config = service_configuration_lib.read_service_configuration( service, soa_dir=soa_dir, ) marathon_conf_file = "marathon-%s" % cluster log.info("Reading marathon configuration file: %s.yaml", marathon_conf_file) instance_configs = service_configuration_lib.read_extra_service_information( service, marathon_conf_file, soa_dir=soa_dir, ) if instance not in instance_configs: raise NoConfigurationForServiceError( "%s not found in config file %s/%s/%s.yaml." % (instance, soa_dir, service, marathon_conf_file)) general_config = deep_merge_dictionaries( overrides=instance_configs[instance], defaults=general_config) branch_dict = {} if load_deployments: deployments_json = load_deployments_json(service, soa_dir=soa_dir) branch = general_config.get('branch', get_paasta_branch(cluster, instance)) branch_dict = deployments_json.get_branch_dict(service, branch) return MarathonServiceConfig( service=service, cluster=cluster, instance=instance, config_dict=general_config, branch_dict=branch_dict, soa_dir=soa_dir, )
def get_autoscaling_params(self) -> AutoscalingParamsDict: default_params: AutoscalingParamsDict = { "metrics_provider": "mesos_cpu", "decision_policy": "proportional", "setpoint": 0.8, } return deep_merge_dictionaries( overrides=self.config_dict.get("autoscaling", AutoscalingParamsDict({})), defaults=default_params, )
def _get_merged_config(self, config): if self._general_config is None: self._general_config = read_service_configuration( service_name=self._service, soa_dir=self._soa_dir, ) return deep_merge_dictionaries( overrides=config, defaults=self._general_config, )
def load_monkrelaycluster_instance_config( service: str, instance: str, cluster: str, load_deployments: bool = True, soa_dir: str = DEFAULT_SOA_DIR, ) -> MonkRelayClusterDeploymentConfig: """Read a service instance's configuration for MonkRelayCluster. If a branch isn't specified for a config, the 'branch' key defaults to paasta-${cluster}.${instance}. :param service: The service name :param instance: The instance of the service to retrieve :param cluster: The cluster to read the configuration for :param load_deployments: A boolean indicating if the corresponding deployments.json for this service should also be loaded :param soa_dir: The SOA configuration directory to read from :returns: A dictionary of whatever was in the config for the service instance""" general_config = service_configuration_lib.read_service_configuration( service, soa_dir=soa_dir) instance_config = load_service_instance_config(service, instance, "monkrelays", cluster, soa_dir=soa_dir) general_config = deep_merge_dictionaries(overrides=instance_config, defaults=general_config) branch_dict: Optional[BranchDictV2] = None if load_deployments: deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir) temp_instance_config = MonkRelayClusterDeploymentConfig( service=service, cluster=cluster, instance=instance, config_dict=general_config, branch_dict=None, soa_dir=soa_dir, ) branch = temp_instance_config.get_branch() deploy_group = temp_instance_config.get_deploy_group() branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group) return MonkRelayClusterDeploymentConfig( service=service, cluster=cluster, instance=instance, config_dict=general_config, branch_dict=branch_dict, soa_dir=soa_dir, )
def load_chronos_job_config( service: str, instance: str, cluster: str, load_deployments: bool = True, soa_dir: str = DEFAULT_SOA_DIR, ) -> 'ChronosJobConfig': general_config = service_configuration_lib.read_service_configuration( service, soa_dir=soa_dir, ) if instance.startswith('_'): raise InvalidJobNameError( "Unable to load chronos job config for %s.%s as instance name starts with '_'" % (service, instance), ) service_chronos_jobs = read_chronos_jobs_for_service(service, cluster, soa_dir=soa_dir) if instance not in service_chronos_jobs: raise NoConfigurationForServiceError( 'No job named "%s" in config file chronos-%s.yaml' % (instance, cluster)) branch_dict = None general_config = deep_merge_dictionaries( overrides=service_chronos_jobs[instance], defaults=general_config) if load_deployments: deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir) temp_instance_config = ChronosJobConfig( service=service, cluster=cluster, instance=instance, config_dict=general_config, branch_dict=None, soa_dir=soa_dir, ) branch = temp_instance_config.get_branch() deploy_group = temp_instance_config.get_deploy_group() branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group) return ChronosJobConfig( service=service, cluster=cluster, instance=instance, config_dict=general_config, branch_dict=branch_dict, soa_dir=soa_dir, )
def load_adhoc_job_config(service, instance, cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR): general_config = service_configuration_lib.read_service_configuration( service, soa_dir=soa_dir, ) adhoc_conf_file = "adhoc-%s" % cluster instance_configs = service_configuration_lib.read_extra_service_information( service_name=service, extra_info=adhoc_conf_file, soa_dir=soa_dir, ) if instance not in instance_configs: raise NoConfigurationForServiceError( "%s not found in config file %s/%s/%s.yaml." % (instance, soa_dir, service, adhoc_conf_file), ) general_config = deep_merge_dictionaries( overrides=instance_configs[instance], defaults=general_config) branch_dict = None if load_deployments: deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir) temp_instance_config = AdhocJobConfig( service=service, cluster=cluster, instance=instance, config_dict=general_config, branch_dict=None, soa_dir=soa_dir, ) branch = temp_instance_config.get_branch() deploy_group = temp_instance_config.get_deploy_group() branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group) return AdhocJobConfig( service=service, cluster=cluster, instance=instance, config_dict=general_config, branch_dict=branch_dict, soa_dir=soa_dir, )
def load_marathon_service_config(service, instance, cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR): """Read a service instance's configuration for marathon. If a branch isn't specified for a config, the 'branch' key defaults to paasta-${cluster}.${instance}. :param name: The service name :param instance: The instance of the service to retrieve :param cluster: The cluster to read the configuration for :param load_deployments: A boolean indicating if the corresponding deployments.json for this service should also be loaded :param soa_dir: The SOA configuration directory to read from :returns: A dictionary of whatever was in the config for the service instance""" log.info("Reading service configuration files from dir %s/ in %s" % (service, soa_dir)) log.info("Reading general configuration file: service.yaml") general_config = service_configuration_lib.read_service_configuration( service, soa_dir=soa_dir ) marathon_conf_file = "marathon-%s" % cluster log.info("Reading marathon configuration file: %s.yaml", marathon_conf_file) instance_configs = service_configuration_lib.read_extra_service_information( service, marathon_conf_file, soa_dir=soa_dir ) if instance not in instance_configs: raise NoConfigurationForServiceError( "%s not found in config file %s/%s/%s.yaml." % (instance, soa_dir, service, marathon_conf_file) ) general_config = deep_merge_dictionaries(overrides=instance_configs[instance], defaults=general_config) branch_dict = {} if load_deployments: deployments_json = load_deployments_json(service, soa_dir=soa_dir) branch = general_config.get('branch', get_paasta_branch(cluster, instance)) branch_dict = deployments_json.get_branch_dict(service, branch) return MarathonServiceConfig( service=service, cluster=cluster, instance=instance, config_dict=general_config, branch_dict=branch_dict, )
def test_deep_merge_dictionaries(): overrides = { 'common_key': 'value', 'common_dict': { 'subkey1': 1, 'subkey2': 2, 'subkey3': 3, }, 'just_in_overrides': 'value', 'just_in_overrides_dict': {'key': 'value'}, 'overwriting_key': 'value', 'overwriting_dict': {'test': 'value'}, } defaults = { 'common_key': 'overwritten_value', 'common_dict': { 'subkey1': 'overwritten_value', 'subkey4': 4, 'subkey5': 5, }, 'just_in_defaults': 'value', 'just_in_defaults_dict': {'key': 'value'}, 'overwriting_key': {'overwritten-key', 'overwritten-value'}, 'overwriting_dict': 'overwritten-value', } expected = { 'common_key': 'value', 'common_dict': { 'subkey1': 1, 'subkey2': 2, 'subkey3': 3, 'subkey4': 4, 'subkey5': 5, }, 'just_in_overrides': 'value', 'just_in_overrides_dict': {'key': 'value'}, 'just_in_defaults': 'value', 'just_in_defaults_dict': {'key': 'value'}, 'overwriting_key': 'value', 'overwriting_dict': {'test': 'value'}, } assert utils.deep_merge_dictionaries(overrides, defaults) == expected
def load_vitess_instance_config( service: str, instance: str, cluster: str, load_deployments: bool = True, soa_dir: str = DEFAULT_SOA_DIR, ) -> VitessDeploymentConfig: general_config = service_configuration_lib.read_service_configuration( service, soa_dir=soa_dir) instance_config = load_service_instance_config(service, instance, " vitesscluster", cluster, soa_dir=soa_dir) general_config = deep_merge_dictionaries(overrides=instance_config, defaults=general_config) branch_dict: Optional[BranchDictV2] = None if load_deployments: deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir) temp_instance_config = VitessDeploymentConfig( service=service, cluster=cluster, instance=instance, config_dict=general_config, branch_dict=None, soa_dir=soa_dir, ) branch = temp_instance_config.get_branch() deploy_group = temp_instance_config.get_deploy_group() branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group) return VitessDeploymentConfig( service=service, cluster=cluster, instance=instance, config_dict=general_config, branch_dict=branch_dict, soa_dir=soa_dir, )
def load_adhoc_job_config( service, instance, cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR ): general_config = service_configuration_lib.read_service_configuration( service, soa_dir=soa_dir ) instance_config = load_service_instance_config( service=service, instance=instance, instance_type="adhoc", cluster=cluster, soa_dir=soa_dir, ) general_config = deep_merge_dictionaries( overrides=instance_config, defaults=general_config ) branch_dict = None if load_deployments: deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir) temp_instance_config = AdhocJobConfig( service=service, cluster=cluster, instance=instance, config_dict=general_config, branch_dict=None, soa_dir=soa_dir, ) branch = temp_instance_config.get_branch() deploy_group = temp_instance_config.get_deploy_group() branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group) return AdhocJobConfig( service=service, cluster=cluster, instance=instance, config_dict=general_config, branch_dict=branch_dict, soa_dir=soa_dir, )
def load_kubernetes_service_config_no_cache( service: str, instance: str, cluster: str, load_deployments: bool = True, soa_dir: str = DEFAULT_SOA_DIR, ) -> "KubernetesDeploymentConfig": """Read a service instance's configuration for kubernetes. If a branch isn't specified for a config, the 'branch' key defaults to paasta-${cluster}.${instance}. :param name: The service name :param instance: The instance of the service to retrieve :param cluster: The cluster to read the configuration for :param load_deployments: A boolean indicating if the corresponding deployments.json for this service should also be loaded :param soa_dir: The SOA configuration directory to read from :returns: A dictionary of whatever was in the config for the service instance""" general_config = service_configuration_lib.read_service_configuration( service, soa_dir=soa_dir, ) kubernetes_conf_file = "kubernetes-%s" % cluster instance_configs = service_configuration_lib.read_extra_service_information( service, kubernetes_conf_file, soa_dir=soa_dir, ) if instance.startswith('_'): raise InvalidJobNameError( f"Unable to load kubernetes job config for {service}.{instance} as instance name starts with '_'", ) if instance not in instance_configs: raise NoConfigurationForServiceError( f"{instance} not found in config file {soa_dir}/{service}/{kubernetes_conf_file}.yaml.", ) general_config = deep_merge_dictionaries( overrides=instance_configs[instance], defaults=general_config) branch_dict: Optional[BranchDictV2] = None if load_deployments: deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir) temp_instance_config = KubernetesDeploymentConfig( service=service, cluster=cluster, instance=instance, config_dict=general_config, branch_dict=None, soa_dir=soa_dir, ) branch = temp_instance_config.get_branch() deploy_group = temp_instance_config.get_deploy_group() branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group) return KubernetesDeploymentConfig( service=service, cluster=cluster, instance=instance, config_dict=general_config, branch_dict=branch_dict, soa_dir=soa_dir, )
def get_autoscaling_params(self): default_params = { 'method': 'default', } return deep_merge_dictionaries(source=self.config_dict.get('autoscaling', {}), destination=default_params)
def get_autoscaling_params(self): default_params = {"metrics_provider": "mesos_cpu", "decision_policy": "pid", "setpoint": 0.8} return deep_merge_dictionaries(overrides=self.config_dict.get("autoscaling", {}), defaults=default_params)