def __init__(
self,
cluster: str,
pool: str,
scheduler: str,
app: str,
config_namespace: str,
metrics_client: ClustermanMetricsBotoClient,
signal_namespace: str,
) -> None:
""" Create an encapsulation of the Unix sockets via which we communicate with signals
:param cluster: the name of the cluster this signal is for
:param pool: the name of the pool this signal is for
:param app: the name of the application this signal is for
:param config_namespace: the staticconf namespace we can find the signal config in
:param metrics_client: the metrics client to use to populate signal metrics
:param signal_namespace: the namespace in the signals repo to find the signal class
(if this is None, we default to the app name)
"""
reader = staticconf.NamespaceReaders(config_namespace)
try:
signal_name = reader.read_string('autoscale_signal.name')
except ConfigurationError as e:
raise NoSignalConfiguredException from e
super().__init__(signal_name, cluster, pool, scheduler, app, config_namespace)
self.required_metrics: list = reader.read_list('autoscale_signal.required_metrics', default=[])
self.metrics_client: ClustermanMetricsBotoClient = metrics_client
self.signal_namespace = signal_namespace
self._signal_conn: socket.socket = self._connect_to_signal_process()
def __init__(
self,
cluster: str,
pool: str,
scheduler: str,
fetch_state: bool = True,
) -> None:
self.cluster = cluster
self.pool = pool
self.scheduler = scheduler
self.cluster_connector = ClusterConnector.load(self.cluster, self.pool,
self.scheduler)
self.pool_config = staticconf.NamespaceReaders(
POOL_NAMESPACE.format(pool=self.pool, scheduler=self.scheduler))
self.draining_enabled = self.pool_config.read_bool('draining_enabled',
default=False)
self.draining_client: Optional[DrainingClient] = DrainingClient(
cluster) if self.draining_enabled else None
self.min_capacity = self.pool_config.read_int(
'scaling_limits.min_capacity')
self.max_capacity = self.pool_config.read_int(
'scaling_limits.max_capacity')
self.max_tasks_to_kill = read_int_or_inf(
self.pool_config, 'scaling_limits.max_tasks_to_kill')
self.max_weight_to_add = self.pool_config.read_int(
'scaling_limits.max_weight_to_add')
self.max_weight_to_remove = self.pool_config.read_int(
'scaling_limits.max_weight_to_remove')
if fetch_state:
self.reload_state()
def get_autoscaling_config(config_namespace: str) -> AutoscalingConfig:
""" Load autoscaling configuration values from the provided config_namespace, falling back to the
values stored in the default namespace if none are specified.
:param config_namespace: namespace to read from before falling back to the default namespace
:returns: AutoscalingConfig object with loaded config values
"""
default_excluded_resources = staticconf.read_list(
'autoscaling.excluded_resources', default=[])
default_setpoint = staticconf.read_float('autoscaling.setpoint')
default_target_capacity_margin = staticconf.read_float(
'autoscaling.target_capacity_margin')
reader = staticconf.NamespaceReaders(config_namespace)
return AutoscalingConfig(
excluded_resources=reader.read_list(
'autoscaling.excluded_resources',
default=default_excluded_resources),
setpoint=reader.read_float('autoscaling.setpoint',
default=default_setpoint),
target_capacity_margin=reader.read_float(
'autoscaling.target_capacity_margin',
default=default_target_capacity_margin,
),
)
def process_warning_queue(self) -> None:
host_to_process = self.get_warned_host()
if host_to_process:
logger.info(
f'Processing spot warning for {host_to_process.hostname}')
spot_fleet_resource_groups = []
for pool in get_pool_name_list(
self.cluster,
'mesos'): # draining only supported for Mesos clusters
pool_config = staticconf.NamespaceReaders(
POOL_NAMESPACE.format(pool=pool, scheduler='mesos'))
for resource_group_conf in pool_config.read_list(
'resource_groups'):
spot_fleet_resource_groups.extend(
list(
SpotFleetResourceGroup.load(
cluster=self.cluster,
pool=pool,
config=list(resource_group_conf.values())[0],
).keys()))
# we should definitely ignore termination warnings that aren't from this
# cluster or maybe not even paasta instances...
if host_to_process.group_id in spot_fleet_resource_groups:
logger.info(
f'Sending spot warned host to drain: {host_to_process.hostname}'
)
self.submit_host_for_draining(host_to_process)
else:
logger.info(
f'Ignoring spot warned host because not in our SFRs: {host_to_process.hostname}'
)
self.delete_warning_messages([host_to_process])
def provision_user(user_id,
key_length=None,
key_valid_duration=None,
hash_algorithm=None):
"""Provision a totp user for the given user_id
:param user_id: int
:param key_length: int in SUPPORTED_KEY_LENGTHS
"""
config = staticconf.NamespaceReaders('adjure')
key_length = key_length or config.read('auth.key_length', default=6)
key_valid_duration = key_valid_duration or config.read(
'auth.key_valid_duration', default=30)
hash_algorithm = hash_algorithm or config.read('auth.hash_algorithm',
default='SHA256')
validate_key_length(key_length)
validate_hash_algorithm(hash_algorithm)
if load_user(user_id):
raise UserCreationException(
'User id {} already provisioned.'.format(user_id))
auth_user = AuthUser(user_id=user_id,
secret=os.urandom(SECRET_KEY_BYTES),
key_length=key_length,
key_valid_duration=key_valid_duration,
hash_algorithm=hash_algorithm)
session.add(auth_user)
session.commit()
generate_recovery_codes_for_user(auth_user)
return auth_user
def __init__(
self,
cluster: str,
pool: str,
scheduler: str,
app: str,
config_namespace: str,
metrics_client: ClustermanMetricsBotoClient,
signal_namespace: str,
) -> None:
""" Create an encapsulation of the Unix sockets via which we communicate with signals
:param cluster: the name of the cluster this signal is for
:param pool: the name of the pool this signal is for
:param app: the name of the application this signal is for
:param config_namespace: the staticconf namespace we can find the signal config in
:param metrics_client: the metrics client to use to populate signal metrics
:param signal_namespace: the namespace in the signals repo to find the signal class
(if this is None, we default to the app name)
"""
reader = staticconf.NamespaceReaders(config_namespace)
try:
self.name: str = reader.read_string('autoscale_signal.name')
except ConfigurationError:
raise NoSignalConfiguredException(
f'No signal was configured in {config_namespace}')
self.cluster: str = cluster
self.pool: str = pool
self.scheduler: str = scheduler
self.app: str = app
self.period_minutes: int = reader.read_int(
'autoscale_signal.period_minutes')
if self.period_minutes <= 0:
raise SignalValidationError(
f'Length of signal period must be positive, got {self.period_minutes}'
)
self.parameters: Dict = {
key: value
for param_dict in reader.read_list('autoscale_signal.parameters',
default=[])
for (key, value) in param_dict.items()
}
# Even if cluster and pool were set in parameters, we override them here
# as we want to preserve a single source of truth
self.parameters.update(dict(
cluster=self.cluster,
pool=self.pool,
))
self.required_metrics: list = reader.read_list(
'autoscale_signal.required_metrics', default=[])
self.metrics_client: ClustermanMetricsBotoClient = metrics_client
self.signal_namespace = signal_namespace
self._signal_conn: socket.socket = self._connect_to_signal_process()
def test_unknown_rg_type(self, mock_logger, mock_pool_manager):
with staticconf.testing.MockConfiguration(
{'resource_groups': [{'fake_rg_type': 'bar'}]},
namespace='bar.mesos_config',
):
mock_pool_manager.pool_config = staticconf.NamespaceReaders('bar.mesos_config')
mock_pool_manager._reload_resource_groups()
assert not mock_pool_manager.resource_groups
assert 'Unknown resource group' in mock_logger.error.call_args[0][0]
def test_malformed_config(self, mock_logger, mock_pool_manager):
with staticconf.testing.MockConfiguration(
{'resource_groups': ['asdf']},
namespace='bar.mesos_config',
):
mock_pool_manager.pool_config = staticconf.NamespaceReaders('bar.mesos_config')
mock_pool_manager._reload_resource_groups()
assert not mock_pool_manager.resource_groups
assert 'Malformed config' in mock_logger.error.call_args[0][0]
def __init__(self, args):
self.validate_args(args)
self.hosts_dict = {}
self.config_watchers = []
self.attribute_key = args.get("attribute_key", "source_host")
for hostfile in args["hostfiles"]:
self.config_watchers.append(build_configuration(
hostfile, __name__))
for config_watcher in self.config_watchers:
config_watcher.config_loader()
self.config = staticconf.NamespaceReaders(__name__)
def __init__(self, args: dict):
super().__init__(args)
self.hosts_dict = {}
self.config_watchers = []
self.attribute_key = args.get('attribute_key', 'source_host')
for hostfile in args['hostfiles']:
self.config_watchers.append(
build_configuration(hostfile, __name__), )
for config_watcher in self.config_watchers:
config_watcher.config_loader()
self.config = staticconf.NamespaceReaders(__name__)
def __init__(self, backup_name: str) -> None:
""" A BackupStore object controls all the reading and writing of data from a particular
backup location (local, S3, ssh, etc)
This is an abstract class that needs to be subclassed with a _save and _load function
that determines how to actually read and write data from/to the store. The remaining
methods for this class are common across different types of stores, and are what establish
many of the "safety" guarantees of backuppy.
:param backup_name: the name of the backup this store corresponds to in the
configuration file
"""
self.backup_name = backup_name
self.config = staticconf.NamespaceReaders(backup_name)
self._manifest = None
def authorize_user(user_id, code_to_verify):
"""Authorize the user given a code entry.
:param user_id: int
:param code_to_verify: ASCII encoded bytes
"""
config = staticconf.NamespaceReaders('adjure')
sliding_windows = config.read_int('sliding_windows', default=1)
user = load_user(user_id)
if not user:
raise ValidationException('{} is not a known user.'.format(user_id))
return totp_verify(
user.secret,
user.key_length,
user.hash_algorithm,
user.key_valid_duration,
code_to_verify,
current_time(),
sliding_windows,
)
def __init__(self, cluster: str, pool: str) -> None:
self.cluster = cluster
self.pool = pool
self.pool_config = staticconf.NamespaceReaders(
POOL_NAMESPACE.format(pool=self.pool, scheduler=self.SCHEDULER))
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import staticconf
from clusterman_metrics.util.constants import CONFIG_NAMESPACE
config_reader = staticconf.NamespaceReaders(CONFIG_NAMESPACE)
def estimate_cost_per_hour(
cluster,
pool,
cpus=0,
mem=0,
):
cpu_cost = cpus * _get_resource_cost('cpus', cluster, pool)
mem_cost = mem * _get_resource_cost('mem', cluster, pool)
return max(cpu_cost, mem_cost)
def _get_resource_cost(resource, cluster, pool):
default_cost = config_reader.read_float(
# -*- coding: utf-8 -*-
import staticconf
CONFIG_NS = 'moneybot'
config = staticconf.NamespaceReaders(CONFIG_NS)
def load_config(path):
staticconf.YamlConfiguration(path, namespace=CONFIG_NS)
def register_app_config(config_path):
staticconf.YamlConfiguration(config_path, namespace='adjure')
return staticconf.NamespaceReaders('adjure')
# specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
from __future__ import unicode_literals
import logging
import os
import staticconf
from bravado.client import SwaggerClient
from cached_property import cached_property
from kafka_utils.util.config import ClusterConfig
namespace = 'data_pipeline'
data_pipeline_conf = staticconf.NamespaceReaders(namespace)
class Config(object):
"""Contains configuration data for the clientlib.
Configuration can be adjusted using staticconf.
Example::
When using the clientlib in a service, the minimum recommended
configuration is::
module_config:
...
- namespace: smartstack_services
