def test_config_strict_mode_errors():
config.register(Item, strict_mode=True)
test_config_path = testing.relative_module_path(__file__, 'test_config.yaml')
with pytest.raises(ConfigLoadError, match='missing type declaration'):
config.load_config(test_config_path)
def test_config_unsafe_object_creation():
from ruamel import yaml
import calendar
test_config_path = testing.relative_module_path(__file__, 'test_config_unsafe.yaml')
# Unsafe default loader allows any python object initialization
data = yaml.load(open(test_config_path), Loader=yaml.Loader)
assert 'time' in data
assert isinstance(data['time'], calendar.Calendar)
# With use safe version only to allow construct previously registered objects
with pytest.raises(config.ConfigLoadError, match='could not determine a constructor'):
config.load_config(test_config_path)
def main(config):
# Initialize logging subsystem.
log_levels = {}
log_levels.setdefault(DEFAULT_MODULE, 'info')
logger.configure_loggers_from_dict(log_levels)
log.warning('This software is pre-production and should not be deployed to production servers!')
log.debug('started PID=%r', os.getpid())
log.info('Starting wca-scheduler.')
# Initialize all necessary objects.
register_algorithms()
register_dataproviders()
try:
configuration = load_config(config)
except ConfigLoadError as e:
log.error('Error: Cannot load config file! : %s', e)
exit(1)
# Configure loggers using configuration file.
if 'loggers' in configuration:
log_levels_config = configuration['loggers']
if not isinstance(log_levels, dict):
log.error('Loggers configuration error: log levels are mapping from logger name to'
'log level!')
exit(1)
log_levels = dict(log_levels, **log_levels_config)
logger.configure_loggers_from_dict(log_levels)
server = Server(configuration)
return server.app
def allocate(
self,
platform: Platform,
tasks_data: TasksData,
) -> (TasksAllocations, List[Anomaly], List[Metric]):
rules = []
if self.rules:
rules.extend(self.rules)
if self.config:
if not os.path.exists(self.config):
log.warning(
'StaticAllocator: cannot find config file %r - ignoring!',
self.config)
else:
rules.extend(load_config(self.config))
if len(rules) == 0:
log.warning('StaticAllocator: no rules were provided!')
return {}, [], []
log.log(TRACE, 'StaticAllocator: handling allocations for %i tasks. ',
len(tasks_data))
for task, data in tasks_data.items():
log.log(
TRACE, '%s', ' '.join('%s=%s' % (k, v)
for k, v in sorted(data.labels.items())))
tasks_allocations = _build_allocations_from_rules(tasks_data, rules)
log.debug('StaticAllocator: final tasks allocations: \n %s',
pprint.pformat(tasks_allocations))
return tasks_allocations, [], []
def test_config_with_env_class():
test_config_path = testing.relative_module_path(__file__,
'test_config_env.yaml')
data = config.load_config(test_config_path)
user = data['from_env']
assert user == os.environ["USER"]
assert user.strip() != ''
assert data['unset_env'] == ''
def __post_init__(self):
init_logging('info', 'Tester')
self.testcases = load_config(self.config)['tests']
self.testcases_count = len(self.testcases)
self.current_iteration = 0
self.processes: Dict[str, subprocess.Popen] = {}
self.tasks: List[Task] = []
# To keep between consecutive get_tasks calls.
self.metrics = []
self.checks = []
def main():
# Parse file argument
parser = argparse.ArgumentParser()
parser.add_argument('-c',
'--config',
required=True,
help='Configuration file')
args = parser.parse_args()
# Initialize logging subsystem.
log_levels = {}
log_levels.setdefault(DEFAULT_MODULE, 'info')
logger.configure_loggers_from_dict(log_levels)
log.warning(
'This software is pre-production and should not be deployed to production servers!'
)
log.debug('started PID=%r', os.getpid())
log.info('Starting wca-admission-controller.')
register_components()
try:
configuration = load_config(args.config)
except ConfigLoadError as e:
log.error('Cannot load config file! : %s', e)
exit(1)
if 'loggers' in configuration:
log_levels_config = configuration['loggers']
if not isinstance(log_levels, dict):
log.error(
'Loggers configuration error: log levels are mapping from logger name to'
'log level!')
exit(1)
log_levels = dict(log_levels, **log_levels_config)
logger.configure_loggers_from_dict(log_levels)
annotating_service = AnnotatingService(configuration)
annotating_service.app.run(
host="0.0.0.0",
ssl_context=(
'./ssl/server-cert.pem', # nosec
'./ssl/server-key.pem')) # nosec
def test_dataclass():
test_config_path = testing.relative_module_path(__file__,
'test_dataclasses.yaml')
data = config.load_config(test_config_path)
dc1 = data['dc1']
assert dc1.x == 5
assert dc1.y is None
assert dc1.z == [1, 2, 3]
assert dc1.d == 'asd'
assert dc1.e == '/abc/def'
assert dc1.foo == 'foobaz'
dc2 = data['dc2']
assert dc2.x == 1
assert dc2.y == 'newble'
assert dc2.z == [3, 4, 5]
assert dc2.d == 4
assert dc2.foo == FooEnum.BAR
def test_config_with_simple_classes():
# Another method for registering items (other than using decorator).
config.register(Item, strict_mode=False)
test_config_path = testing.relative_module_path(__file__, 'test_config.yaml')
data = config.load_config(test_config_path)
foo_with_defaults = data['foo_with_defaults']
assert foo_with_defaults.f == 1
empty_boo = data['empty_boo']
assert empty_boo.foo is None
foo = data['foo']
boo = data['boo']
assert foo.s == 'some_string'
assert foo.f == 2.5
assert boo.foo is foo
assert len(boo.items) == 2
assert isinstance(boo.items[0], Item)
def allocate(
self,
platform: Platform,
tasks_measurements: TasksMeasurements,
tasks_resources: TasksResources,
tasks_labels: TasksLabels,
tasks_allocations: TasksAllocations,
) -> (TasksAllocations, List[Anomaly], List[Metric]):
rules = []
if self.rules:
rules.extend(self.rules)
if self.config:
if not os.path.exists(self.config):
log.warning('StaticAllocator: cannot find config file %r - ignoring!', self.config)
else:
rules.extend(load_config(self.config))
if len(rules) == 0:
log.warning('StaticAllocator: no rules were provided!')
return {}, [], []
# Merge all tasks ids.
all_tasks_ids = (set(tasks_labels.keys()) | set(tasks_resources.keys()) |
set(tasks_allocations.keys()))
log.log(TRACE,
'StaticAllocator: handling allocations for %i tasks. ', len(all_tasks_ids))
for task_id, labels in tasks_labels.items():
log.log(TRACE, '%s', ' '.join('%s=%s' % (k, v) for k, v in sorted(labels.items())))
tasks_allocations = _build_allocations_from_rules(all_tasks_ids, tasks_labels, rules)
log.debug('StaticAllocator: final tasks allocations: \n %s',
pprint.pformat(tasks_allocations))
return tasks_allocations, [], []
def test_dataclass_validation():
test_config_path = testing.relative_module_path(
__file__, 'test_dataclasses_validation.yaml')
data = config.load_config(test_config_path)
assert data['dc1'].rr == [[2, 3], []]
def test_dataclass_exceed_maximum():
test_config_path = testing.relative_module_path(
__file__, 'test_dataclasses_exceed_maximum.yaml')
with pytest.raises(config.ConfigLoadError,
match="field 'x'.*Maximum value is 5. Got 123."):
config.load_config(test_config_path)
def test_dataclass_invalid_field():
test_config_path = testing.relative_module_path(
__file__, 'test_dataclasses_invalid_field.yaml')
with pytest.raises(config.ConfigLoadError, match="constructor signature"):
config.load_config(test_config_path)
def test_invalid_dataclass():
test_config_path = testing.relative_module_path(
__file__, 'test_dataclasses_invalid.yaml')
with pytest.raises(config.ConfigLoadError,
match="field 'x'.*Invalid type"):
config.load_config(test_config_path)
def test_dataclass_validation_invalid_dict():
test_config_path = testing.relative_module_path(
__file__, 'test_dataclasses_validation_invalid_dict.yaml')
with pytest.raises(ConfigLoadError, match='wrong_value'):
config.load_config(test_config_path)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-c',
'--config',
help="Configuration",
default=None,
required=True)
parser.add_argument(
'-l',
'--log-level',
help='Log level for modules (by default for wca) in [module:]level form,'
'where level can be one of: CRITICAL,ERROR,WARNING,INFO,DEBUG,TRACE'
'Example -l debug -l example:debug. Defaults to wca:INFO.'
'Can be overridden at runtime with config.yaml "loggers" section.',
default=[],
action='append',
dest='levels',
)
parser.add_argument('-r',
'--register',
action='append',
dest='components',
help="Register additional components in config",
default=[])
parser.add_argument('-v',
'--version',
action='version',
version=platforms.get_wca_version(),
help="Show version")
parser.add_argument('-0',
'--root',
help="Allow WCA process to be run using root account",
dest='is_root_allowed',
action='store_true')
args = parser.parse_args()
# Do not allow to run WCA with root privileges unless user indicates that it is intended.
uid = os.geteuid()
if uid == 0 and not args.is_root_allowed:
log.fatal(
"Do not run WCA with root privileges. Consult documentation "
"to understand what capabilities are required. If root account "
"has to be used then set --root/-0 argument to override.")
exit(2)
# Initialize logging subsystem from command line options.
log_levels = logger.parse_loggers_from_list(args.levels)
log_levels_copy_with_default = dict(**log_levels)
log_levels_copy_with_default.setdefault(logger.DEFAULT_MODULE, 'info')
logger.configure_loggers_from_dict(log_levels_copy_with_default)
log.warning(
'This software is pre-production and should not be deployed to production servers.'
)
log.debug('started PID=%r', os.getpid())
log.info('Version wca: %s', platforms.get_wca_version())
# Register internal & external components.
components.register_components(extra_components=args.components)
valid_config_file(args.config)
# Initialize all necessary objects.
try:
configuration = config.load_config(args.config)
except config.ConfigLoadError as e:
log.error('Error: Cannot load config file! : %s', e)
if log.getEffectiveLevel() <= logging.DEBUG:
log.exception('Detailed exception:')
exit(1)
for key in configuration:
if key != 'loggers' and key != 'runner':
log.error('Error: Unknown fields in configuration '
'file! Possible are: \'loggers\', '
'\'runner\'')
exit(1)
assure_type(configuration, dict)
assert 'runner' in configuration, 'Improper config - missing runner instance!'
# Configure loggers using configuration file.
if 'loggers' in configuration:
log_levels_config = configuration['loggers']
if not isinstance(log_levels, dict):
log.error(
'Loggers configuration error: log levels are mapping from logger name to'
'log level!')
exit(1)
# Merge config from cmd line and config file.
# Overwrite config file values with values provided from command line.
log_levels = dict(log_levels_config, **log_levels)
logger.configure_loggers_from_dict(log_levels)
# Dump loggers configurations to debug issues with loggers.
if os.environ.get('WCA_DUMP_LOGGERS') == 'True':
print(
'------------------------------------ Logging tree ---------------------'
)
import logging_tree
logging_tree.printout()
print(
'------------------------------------ Logging tree END------------------'
)
# Extract main loop component.
runner = configuration['runner']
assure_type(runner, Runner)
# Prepare and run the "main loop".
exit_code = runner.run()
exit(exit_code)
def test_invalid_dataclass_union():
test_config_path = testing.relative_module_path(
__file__, 'test_dataclasses_invalid_union.yaml')
with pytest.raises(config.ConfigLoadError,
match="field 'd'.*improper type from union"):
config.load_config(test_config_path)
