def __init__( self, cluster: str, pool: str, scheduler: str, app: str, config_namespace: str, metrics_client: ClustermanMetricsBotoClient, signal_namespace: str, ) -> None: """ Create an encapsulation of the Unix sockets via which we communicate with signals :param cluster: the name of the cluster this signal is for :param pool: the name of the pool this signal is for :param app: the name of the application this signal is for :param config_namespace: the staticconf namespace we can find the signal config in :param metrics_client: the metrics client to use to populate signal metrics :param signal_namespace: the namespace in the signals repo to find the signal class (if this is None, we default to the app name) """ reader = staticconf.NamespaceReaders(config_namespace) try: signal_name = reader.read_string('autoscale_signal.name') except ConfigurationError as e: raise NoSignalConfiguredException from e super().__init__(signal_name, cluster, pool, scheduler, app, config_namespace) self.required_metrics: list = reader.read_list('autoscale_signal.required_metrics', default=[]) self.metrics_client: ClustermanMetricsBotoClient = metrics_client self.signal_namespace = signal_namespace self._signal_conn: socket.socket = self._connect_to_signal_process()
def __init__( self, cluster: str, pool: str, scheduler: str, fetch_state: bool = True, ) -> None: self.cluster = cluster self.pool = pool self.scheduler = scheduler self.cluster_connector = ClusterConnector.load(self.cluster, self.pool, self.scheduler) self.pool_config = staticconf.NamespaceReaders( POOL_NAMESPACE.format(pool=self.pool, scheduler=self.scheduler)) self.draining_enabled = self.pool_config.read_bool('draining_enabled', default=False) self.draining_client: Optional[DrainingClient] = DrainingClient( cluster) if self.draining_enabled else None self.min_capacity = self.pool_config.read_int( 'scaling_limits.min_capacity') self.max_capacity = self.pool_config.read_int( 'scaling_limits.max_capacity') self.max_tasks_to_kill = read_int_or_inf( self.pool_config, 'scaling_limits.max_tasks_to_kill') self.max_weight_to_add = self.pool_config.read_int( 'scaling_limits.max_weight_to_add') self.max_weight_to_remove = self.pool_config.read_int( 'scaling_limits.max_weight_to_remove') if fetch_state: self.reload_state()
def get_autoscaling_config(config_namespace: str) -> AutoscalingConfig: """ Load autoscaling configuration values from the provided config_namespace, falling back to the values stored in the default namespace if none are specified. :param config_namespace: namespace to read from before falling back to the default namespace :returns: AutoscalingConfig object with loaded config values """ default_excluded_resources = staticconf.read_list( 'autoscaling.excluded_resources', default=[]) default_setpoint = staticconf.read_float('autoscaling.setpoint') default_target_capacity_margin = staticconf.read_float( 'autoscaling.target_capacity_margin') reader = staticconf.NamespaceReaders(config_namespace) return AutoscalingConfig( excluded_resources=reader.read_list( 'autoscaling.excluded_resources', default=default_excluded_resources), setpoint=reader.read_float('autoscaling.setpoint', default=default_setpoint), target_capacity_margin=reader.read_float( 'autoscaling.target_capacity_margin', default=default_target_capacity_margin, ), )
def process_warning_queue(self) -> None: host_to_process = self.get_warned_host() if host_to_process: logger.info( f'Processing spot warning for {host_to_process.hostname}') spot_fleet_resource_groups = [] for pool in get_pool_name_list( self.cluster, 'mesos'): # draining only supported for Mesos clusters pool_config = staticconf.NamespaceReaders( POOL_NAMESPACE.format(pool=pool, scheduler='mesos')) for resource_group_conf in pool_config.read_list( 'resource_groups'): spot_fleet_resource_groups.extend( list( SpotFleetResourceGroup.load( cluster=self.cluster, pool=pool, config=list(resource_group_conf.values())[0], ).keys())) # we should definitely ignore termination warnings that aren't from this # cluster or maybe not even paasta instances... if host_to_process.group_id in spot_fleet_resource_groups: logger.info( f'Sending spot warned host to drain: {host_to_process.hostname}' ) self.submit_host_for_draining(host_to_process) else: logger.info( f'Ignoring spot warned host because not in our SFRs: {host_to_process.hostname}' ) self.delete_warning_messages([host_to_process])
def provision_user(user_id, key_length=None, key_valid_duration=None, hash_algorithm=None): """Provision a totp user for the given user_id :param user_id: int :param key_length: int in SUPPORTED_KEY_LENGTHS """ config = staticconf.NamespaceReaders('adjure') key_length = key_length or config.read('auth.key_length', default=6) key_valid_duration = key_valid_duration or config.read( 'auth.key_valid_duration', default=30) hash_algorithm = hash_algorithm or config.read('auth.hash_algorithm', default='SHA256') validate_key_length(key_length) validate_hash_algorithm(hash_algorithm) if load_user(user_id): raise UserCreationException( 'User id {} already provisioned.'.format(user_id)) auth_user = AuthUser(user_id=user_id, secret=os.urandom(SECRET_KEY_BYTES), key_length=key_length, key_valid_duration=key_valid_duration, hash_algorithm=hash_algorithm) session.add(auth_user) session.commit() generate_recovery_codes_for_user(auth_user) return auth_user
def __init__( self, cluster: str, pool: str, scheduler: str, app: str, config_namespace: str, metrics_client: ClustermanMetricsBotoClient, signal_namespace: str, ) -> None: """ Create an encapsulation of the Unix sockets via which we communicate with signals :param cluster: the name of the cluster this signal is for :param pool: the name of the pool this signal is for :param app: the name of the application this signal is for :param config_namespace: the staticconf namespace we can find the signal config in :param metrics_client: the metrics client to use to populate signal metrics :param signal_namespace: the namespace in the signals repo to find the signal class (if this is None, we default to the app name) """ reader = staticconf.NamespaceReaders(config_namespace) try: self.name: str = reader.read_string('autoscale_signal.name') except ConfigurationError: raise NoSignalConfiguredException( f'No signal was configured in {config_namespace}') self.cluster: str = cluster self.pool: str = pool self.scheduler: str = scheduler self.app: str = app self.period_minutes: int = reader.read_int( 'autoscale_signal.period_minutes') if self.period_minutes <= 0: raise SignalValidationError( f'Length of signal period must be positive, got {self.period_minutes}' ) self.parameters: Dict = { key: value for param_dict in reader.read_list('autoscale_signal.parameters', default=[]) for (key, value) in param_dict.items() } # Even if cluster and pool were set in parameters, we override them here # as we want to preserve a single source of truth self.parameters.update(dict( cluster=self.cluster, pool=self.pool, )) self.required_metrics: list = reader.read_list( 'autoscale_signal.required_metrics', default=[]) self.metrics_client: ClustermanMetricsBotoClient = metrics_client self.signal_namespace = signal_namespace self._signal_conn: socket.socket = self._connect_to_signal_process()
def test_unknown_rg_type(self, mock_logger, mock_pool_manager): with staticconf.testing.MockConfiguration( {'resource_groups': [{'fake_rg_type': 'bar'}]}, namespace='bar.mesos_config', ): mock_pool_manager.pool_config = staticconf.NamespaceReaders('bar.mesos_config') mock_pool_manager._reload_resource_groups() assert not mock_pool_manager.resource_groups assert 'Unknown resource group' in mock_logger.error.call_args[0][0]
def test_malformed_config(self, mock_logger, mock_pool_manager): with staticconf.testing.MockConfiguration( {'resource_groups': ['asdf']}, namespace='bar.mesos_config', ): mock_pool_manager.pool_config = staticconf.NamespaceReaders('bar.mesos_config') mock_pool_manager._reload_resource_groups() assert not mock_pool_manager.resource_groups assert 'Malformed config' in mock_logger.error.call_args[0][0]
def __init__(self, args): self.validate_args(args) self.hosts_dict = {} self.config_watchers = [] self.attribute_key = args.get("attribute_key", "source_host") for hostfile in args["hostfiles"]: self.config_watchers.append(build_configuration( hostfile, __name__)) for config_watcher in self.config_watchers: config_watcher.config_loader() self.config = staticconf.NamespaceReaders(__name__)
def __init__(self, args: dict): super().__init__(args) self.hosts_dict = {} self.config_watchers = [] self.attribute_key = args.get('attribute_key', 'source_host') for hostfile in args['hostfiles']: self.config_watchers.append( build_configuration(hostfile, __name__), ) for config_watcher in self.config_watchers: config_watcher.config_loader() self.config = staticconf.NamespaceReaders(__name__)
def __init__(self, backup_name: str) -> None: """ A BackupStore object controls all the reading and writing of data from a particular backup location (local, S3, ssh, etc) This is an abstract class that needs to be subclassed with a _save and _load function that determines how to actually read and write data from/to the store. The remaining methods for this class are common across different types of stores, and are what establish many of the "safety" guarantees of backuppy. :param backup_name: the name of the backup this store corresponds to in the configuration file """ self.backup_name = backup_name self.config = staticconf.NamespaceReaders(backup_name) self._manifest = None
def authorize_user(user_id, code_to_verify): """Authorize the user given a code entry. :param user_id: int :param code_to_verify: ASCII encoded bytes """ config = staticconf.NamespaceReaders('adjure') sliding_windows = config.read_int('sliding_windows', default=1) user = load_user(user_id) if not user: raise ValidationException('{} is not a known user.'.format(user_id)) return totp_verify( user.secret, user.key_length, user.hash_algorithm, user.key_valid_duration, code_to_verify, current_time(), sliding_windows, )
def __init__(self, cluster: str, pool: str) -> None: self.cluster = cluster self.pool = pool self.pool_config = staticconf.NamespaceReaders( POOL_NAMESPACE.format(pool=self.pool, scheduler=self.SCHEDULER))
# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import staticconf from clusterman_metrics.util.constants import CONFIG_NAMESPACE config_reader = staticconf.NamespaceReaders(CONFIG_NAMESPACE) def estimate_cost_per_hour( cluster, pool, cpus=0, mem=0, ): cpu_cost = cpus * _get_resource_cost('cpus', cluster, pool) mem_cost = mem * _get_resource_cost('mem', cluster, pool) return max(cpu_cost, mem_cost) def _get_resource_cost(resource, cluster, pool): default_cost = config_reader.read_float(
# -*- coding: utf-8 -*- import staticconf CONFIG_NS = 'moneybot' config = staticconf.NamespaceReaders(CONFIG_NS) def load_config(path): staticconf.YamlConfiguration(path, namespace=CONFIG_NS)
def register_app_config(config_path): staticconf.YamlConfiguration(config_path, namespace='adjure') return staticconf.NamespaceReaders('adjure')
# specific language governing permissions and limitations # under the License. from __future__ import absolute_import from __future__ import unicode_literals import logging import os import staticconf from bravado.client import SwaggerClient from cached_property import cached_property from kafka_utils.util.config import ClusterConfig namespace = 'data_pipeline' data_pipeline_conf = staticconf.NamespaceReaders(namespace) class Config(object): """Contains configuration data for the clientlib. Configuration can be adjusted using staticconf. Example:: When using the clientlib in a service, the minimum recommended configuration is:: module_config: ... - namespace: smartstack_services