class ValidateConfig(Validator):
"""Given a parsed config file (should be only basic literals and
containers), return an immutable, fully populated series of namedtuples and
FrozenDicts with all defaults filled in, all valid values, and no unused
values. Throws a ConfigError if any part of the input dict is invalid.
"""
config_class = TronConfig
defaults = {
'action_runner': {},
'output_stream_dir': None,
'command_context': {},
'ssh_options': ValidateSSHOptions.defaults,
'notification_options': None,
'time_zone': None,
'state_persistence': DEFAULT_STATE_PERSISTENCE,
'nodes': {'localhost': DEFAULT_NODE},
'node_pools': {},
'jobs': (),
'services': (),
}
node_pools = build_dict_name_validator(valid_node_pool, allow_empty=True)
nodes = build_dict_name_validator(valid_node, allow_empty=True)
validators = {
'action_runner': ValidateActionRunner(),
'output_stream_dir': valid_output_stream_dir,
'command_context': valid_command_context,
'ssh_options': valid_ssh_options,
'notification_options': valid_notification_options,
'time_zone': valid_time_zone,
'state_persistence': valid_state_persistence,
'nodes': nodes,
'node_pools': node_pools,
}
optional = False
def validate_node_pool_nodes(self, config):
"""Validate that each node in a node_pool is in fact a node, and not
another pool.
"""
all_node_names = set(config['nodes'])
for node_pool in config['node_pools'].itervalues():
invalid_names = set(node_pool.nodes) - all_node_names
if invalid_names:
msg = "NodePool %s contains other NodePools: " % node_pool.name
raise ConfigError(msg + ",".join(invalid_names))
def post_validation(self, config, _):
"""Validate a non-named config."""
node_names = config_utils.unique_names(
'Node and NodePool names must be unique %s',
config['nodes'], config.get('node_pools', []))
if config.get('node_pools'):
self.validate_node_pool_nodes(config)
config_context = ConfigContext('config', node_names,
config.get('command_context'), MASTER_NAMESPACE)
validate_jobs_and_services(config, config_context)
def validate_jobs_and_services(config, config_context):
"""Validate jobs and services."""
valid_jobs = build_dict_name_validator(valid_job, allow_empty=True)
valid_services = build_dict_name_validator(valid_service, allow_empty=True)
validation = [('jobs', valid_jobs), ('services', valid_services)]
for config_name, valid in validation:
child_context = config_context.build_child_context(config_name)
config[config_name] = valid(config.get(config_name, []), child_context)
fmt_string = 'Job and Service names must be unique %s'
config_utils.unique_names(fmt_string, config['jobs'], config['services'])
def validate_jobs_and_services(config, config_context):
"""Validate jobs and services."""
valid_jobs = build_dict_name_validator(valid_job, allow_empty=True)
valid_services = build_dict_name_validator(valid_service, allow_empty=True)
validation = [('jobs', valid_jobs), ('services', valid_services)]
for config_name, valid in validation:
child_context = config_context.build_child_context(config_name)
config[config_name] = valid(config.get(config_name, []), child_context)
fmt_string = 'Job and Service names must be unique %s'
config_utils.unique_names(fmt_string, config['jobs'], config['services'])
class ValidateJob(Validator):
"""Validate jobs."""
config_class = ConfigJob
defaults = {
'run_limit': 50,
'all_nodes': False,
'cleanup_action': None,
'enabled': True,
'queueing': True,
'allow_overlap': False,
'max_runtime': None,
}
validators = {
'name': valid_name_identifier,
'schedule': valid_schedule,
'run_limit': valid_int,
'all_nodes': valid_bool,
'actions': build_dict_name_validator(valid_action),
'cleanup_action': valid_cleanup_action,
'node': valid_node_name,
'queueing': valid_bool,
'enabled': valid_bool,
'allow_overlap': valid_bool,
'max_runtime': config_utils.valid_time_delta,
}
def cast(self, in_dict, config_context):
in_dict['namespace'] = config_context.namespace
return in_dict
# TODO: extract common code to a util function
def _validate_dependencies(self,
job,
actions,
base_action,
current_action=None,
stack=None):
"""Check for circular or misspelled dependencies."""
stack = stack or []
current_action = current_action or base_action
stack.append(current_action.name)
for dep in current_action.requires:
if dep == base_action.name and len(stack) > 0:
msg = 'Circular dependency in job.%s: %s'
raise ConfigError(msg % (job['name'], ' -> '.join(stack)))
if dep not in actions:
raise ConfigError('Action jobs.%s.%s has a dependency "%s"'
' that is not in the same job!' %
(job['name'], current_action.name, dep))
self._validate_dependencies(job, actions, base_action,
actions[dep], stack)
stack.pop()
def post_validation(self, job, config_context):
"""Validate actions for the job."""
for action in job['actions'].itervalues():
self._validate_dependencies(job, job['actions'], action)
def validate_jobs_and_services(config, config_context):
"""Validate jobs and services."""
if 'command_context' in config:
sorted_context = [(key,config['command_context'][key]) for key in \
sorted(config['command_context'], key=len, reverse=True)]
for job in config['jobs']:
tron_cmd = 'enable' if ('enabled' not in job or job['enabled']) \
else 'disable'
#namespace_job = '%s.%s' % (config_context.namespace,
# job['name'].replace(" ", "_"))
#disable_pr = subprocess.Popen(['tronctl', tron_cmd,
# namespace_job], stdout=subprocess.PIPE,
# stderr=subprocess.PIPE)
#disable_pr.communicate()
if tron_cmd == 'disable':
print '%s %sd' % (job['name'], tron_cmd)
if 'report' in job:
if job['report'] is True:
job['email'] = 'report-%s' % job['email']
del job['report']
if 'node' not in job:
job['node'] = 'be-master'
if 'actions' not in job:
job['actions'] = [{}]
job['actions'][0]['command'] = job['command']
job['actions'][0]['name'] = job['email'] \
if 'email' in job else 'none'
for var, repl_var in sorted_context:
old_cmd = job['actions'][0]['command']
if var in old_cmd:
job['actions'][0]['command'] = old_cmd.replace(
var, repl_var)
continue
valid_jobs = build_dict_name_validator(valid_job, allow_empty=True)
valid_services = build_dict_name_validator(valid_service, allow_empty=True)
validation = [('jobs', valid_jobs), ('services', valid_services)]
for config_name, valid in validation:
child_context = config_context.build_child_context(config_name)
config[config_name] = valid(config.get(config_name, []), child_context)
fmt_string = 'Job and Service names must be unique %s'
config_utils.unique_names(fmt_string, config['jobs'], config['services'])
def validate_jobs_and_services(config, config_context):
"""Validate jobs and services."""
if 'command_context' in config:
sorted_context = [(key,config['command_context'][key]) for key in \
sorted(config['command_context'], key=len, reverse=True)]
for job in config['jobs']:
tron_cmd = 'enable' if ('enabled' not in job or job['enabled']) \
else 'disable'
#namespace_job = '%s.%s' % (config_context.namespace,
# job['name'].replace(" ", "_"))
#disable_pr = subprocess.Popen(['tronctl', tron_cmd,
# namespace_job], stdout=subprocess.PIPE,
# stderr=subprocess.PIPE)
#disable_pr.communicate()
if tron_cmd == 'disable':
print '%s %sd' % (job['name'], tron_cmd)
if 'report' in job:
if job['report'] is True:
job['email'] = 'report-%s' % job['email']
del job['report']
if 'node' not in job:
job['node'] = 'be-master'
if 'actions' not in job:
job['actions'] = [{}]
job['actions'][0]['command'] = job['command']
job['actions'][0]['name'] = job['email'] \
if 'email' in job else 'none'
for var, repl_var in sorted_context:
old_cmd = job['actions'][0]['command']
if var in old_cmd:
job['actions'][0]['command'] = old_cmd.replace(var, repl_var)
continue
valid_jobs = build_dict_name_validator(valid_job, allow_empty=True)
valid_services = build_dict_name_validator(valid_service, allow_empty=True)
validation = [('jobs', valid_jobs), ('services', valid_services)]
for config_name, valid in validation:
child_context = config_context.build_child_context(config_name)
config[config_name] = valid(config.get(config_name, []), child_context)
fmt_string = 'Job and Service names must be unique %s'
config_utils.unique_names(fmt_string, config['jobs'], config['services'])
