def apply_default_apps(self, default_apps):
# These annoying names are to prevent conflicts with fields in the default app definitions
PLACEMENT_KEY = 'default_placement'
CONDITION_KEY = 'default_condition'
for section in default_apps:
if section not in self._data:
self._data[section] = []
insert_idx = 0
for app in default_apps[section]:
placement = app.get(PLACEMENT_KEY, 'before')
if PLACEMENT_KEY in app:
del app[PLACEMENT_KEY]
if CONDITION_KEY in app:
condition = app[CONDITION_KEY]
del app[CONDITION_KEY]
template = Template()
tmp_vars = dict(VARS=dict(self._vars))
# Continue to next item if we have a condition and it evaluated to False
if not template.evaluate_condition(condition, tmp_vars):
continue
if placement in ('before', 'pre'):
self._data[section].insert(insert_idx, app)
insert_idx += 1
elif placement in ('after', 'post'):
self._data[section].append(app)
else:
raise DeployConfigError(
'invalid default app placement: %s' % placement)
def __init__(self, env=None):
self._display = Display()
self._config = self._defaults
# Used for replacing vars in include paths
tmp_vars = dict()
if env is not None:
tmp_vars['env'] = env
self._template = Template(default_vars=tmp_vars)
def __init__(self, varset, output_dir, config_version):
self._vars = varset
self._output_dir = output_dir
self._display = Display()
self._template = Template()
self._site_config = SiteConfig()
self._config_version = config_version
self.build_config()
def main():
global DISPLAY, SITE_CONFIG
parser = argparse.ArgumentParser()
parser.add_argument(
'-c', '--config',
help='Path to config file',
)
parser.add_argument(
'-o', '--output-dir',
help="Directory to output generated docs to (defaults to '../docs')",
default='../docs'
)
args = parser.parse_args()
DISPLAY = Display()
# Verbosity of 3 is needed for displaying tracebacks
DISPLAY.set_verbosity(3)
SITE_CONFIG = SiteConfig()
if args.config:
try:
SITE_CONFIG.load(args.config)
except ConfigError as e:
DISPLAY.display('Failed to load site config: %s' % str(e))
sys.exit(1)
# Needed for instantiating output plugin classes
varset = Vars()
# Disable recursive variable lookups to avoid rendering templates
# in field defaults from the site config
tmpl = Template(recursive=False)
output_plugins = load_output_plugins(varset)
for plugin in output_plugins:
tmp_fields = dict()
for section in plugin._fields:
tmp_fields[section] = get_plugin_fields(plugin._fields[section])
tmp_vars = dict(
plugin=plugin,
fields=tmp_fields,
)
output_file = os.path.join(os.path.dirname(__file__), args.output_dir, 'plugin_%s.md' % plugin.NAME)
DISPLAY.display('Writing file %s' % output_file)
with open(output_file, 'w') as f:
f.write(tmpl.render_template(PLUGIN_DOC_TEMPLATE, tmp_vars))
class TestTemplate(unittest.TestCase):
def setUp(self):
self._template = Template()
def test_template_plain(self):
tpl = '''
Plain text
More text
'''
output = self._template.render_template(inspect.cleandoc(tpl), {})
self.assertEqual(output, 'Plain text\nMore text')
def test_template_var_simple(self):
tpl = '''
foo {{ bar }} baz
'''
my_vars = {'bar': 'whatever'}
output = self._template.render_template(inspect.cleandoc(tpl), my_vars)
self.assertEqual(output, 'foo whatever baz')
def test_template_if_statement(self):
tpl = '''
foo
{% if bar is defined %}
bar
{% endif %}
baz
'''
output = self._template.render_template(inspect.cleandoc(tpl), {})
self.assertEqual(output, 'foo\n\nbaz')
def test_template_filter_to_json(self):
tpl = '''
{{ foo | to_json }}
'''
my_vars = {'foo': {'bar': ['item 1', 'item 2']}}
output = self._template.render_template(inspect.cleandoc(tpl), my_vars)
self.assertEqual(output, '{"bar": ["item 1", "item 2"]}')
def load_vars(varset,
deploy_dir,
env='BAD_VALUE_NO_MATCH',
only_site_config=False):
# Used for replacing vars in file patterns
template = Template()
tmp_vars = dict(env=env)
# Load vars from site config
for key, value in list(SITE_CONFIG.default_vars.items()):
if not isinstance(value, str):
SITE_CONFIG.default_vars[key] = str(value)
DISPLAY.warn(
"implicitly converted non-string value for var '%s' from site config to string"
% key)
varset.update(SITE_CONFIG.default_vars)
if not only_site_config:
vars_path = os.path.join(deploy_dir, SITE_CONFIG.vars_dir)
# Load env vars
if SITE_CONFIG.use_env_vars:
DISPLAY.v('Loading vars from environment')
varset.update(os.environ)
# Load "defaults" vars
load_vars_files(
varset, vars_path,
template.render_template(SITE_CONFIG.defaults_vars_file_patterns,
tmp_vars))
# Load env-specific vars
load_vars_files(
varset, vars_path,
template.render_template(SITE_CONFIG.env_vars_file_patterns,
tmp_vars))
def main():
global DISPLAY, SITE_CONFIG
parser = argparse.ArgumentParser()
parser.add_argument(
'path',
help='Path to service dir',
)
parser.add_argument(
'-v',
'--verbose',
action='count',
help='Increase verbosity level',
default=0,
)
parser.add_argument(
'-c',
'--config',
help='Path to config file',
)
parser.add_argument(
'-e',
'--env',
help="Environment to generate deploy configs for",
default='local',
)
parser.add_argument(
'-o',
'--output-dir',
help=
"Directory to output generated deploy configs to (defaults to '.')",
default='.')
parser.add_argument(
'--dump-vars',
help="Output all site config vars in shell format",
action='store_true',
default=False,
)
args = parser.parse_args()
DISPLAY = Display()
DISPLAY.set_verbosity(args.verbose)
DISPLAY.vv('Running with args:')
DISPLAY.vv()
for arg in dir(args):
if arg.startswith('_'):
continue
DISPLAY.vv('%s: %s' % (arg, getattr(args, arg)))
DISPLAY.vv()
SITE_CONFIG = SiteConfig(env=args.env)
if args.config is None:
# Possible config locations
config_paths = [
os.path.join(os.environ.get('HOME', None),
'.deploy-config-generator-site.yml'),
]
for path in config_paths:
if os.path.exists(path):
args.config = path
break
if args.config:
try:
SITE_CONFIG.load(args.config)
except ConfigError as e:
DISPLAY.display('Failed to load site config: %s' % str(e))
show_traceback(DISPLAY.get_verbosity())
sys.exit(1)
DISPLAY.vvvv('Site config:')
DISPLAY.vvvv()
DISPLAY.vvvv(yaml_dump(SITE_CONFIG.get_config()))
DISPLAY.vvvv()
varset = Vars()
varset['env'] = args.env
deploy_dir = None
if not args.dump_vars:
deploy_dir = find_deploy_dir(args.path)
try:
load_vars(varset,
deploy_dir,
args.env,
only_site_config=(True if args.dump_vars else False))
except Exception as e:
DISPLAY.display('Error loading vars: %s' % str(e))
show_traceback(DISPLAY.get_verbosity())
sys.exit(1)
if args.dump_vars:
template = Template()
templated_vars = template.render_template(dict(varset),
dict(VARS=dict(varset)))
for key in sorted(templated_vars.keys()):
print('%s=%s' % (key, shlex.quote(templated_vars[key])))
sys.exit(0)
DISPLAY.vvvv()
DISPLAY.vvvv('Vars:')
DISPLAY.vvvv()
DISPLAY.vvvv(yaml_dump(dict(varset), indent=2))
try:
deploy_config = DeployConfig(
os.path.join(deploy_dir, SITE_CONFIG.deploy_config_file), varset)
deploy_config.set_config(
varset.replace_vars(deploy_config.get_config()))
except DeployConfigError as e:
DISPLAY.display('Error loading deploy config: %s' % str(e))
show_traceback(DISPLAY.get_verbosity())
sys.exit(1)
except VarsReplacementError as e:
DISPLAY.display(
'Error loading deploy config: variable replacement error: %s' %
str(e))
show_traceback(DISPLAY.get_verbosity())
sys.exit(1)
DISPLAY.vvvv('Deploy config:')
DISPLAY.vvvv()
DISPLAY.vvvv(yaml_dump(deploy_config.get_config(), indent=2))
deploy_config_version = deploy_config.get_version(
) or SITE_CONFIG.default_config_version
output_plugins = load_output_plugins(varset, args.output_dir,
deploy_config_version)
DISPLAY.vvv('Available output plugins:')
DISPLAY.vvv()
valid_sections = []
for plugin in output_plugins:
DISPLAY.vvv('- %s (%s)' %
(plugin.NAME, plugin.DESCR or 'No description'))
valid_sections += plugin._fields.keys()
DISPLAY.vvv()
try:
deploy_config.apply_default_apps(SITE_CONFIG.default_apps)
deploy_config.validate_sections(valid_sections)
except DeployConfigError as e:
DISPLAY.display('Error validating deploy config: %s' % str(e))
sys.exit(1)
# TODO: do validation before running each plugin so that it doesn't complain about
# values populated by wrapper plugins. This is not straightforward, since it breaks
# the existing code that reports on unknown fields
for section in deploy_config.get_config():
for plugin in output_plugins:
plugin.set_section(section)
for app_idx, app in enumerate(deploy_config.get_config()[section]):
app_validate_fields(app, app_idx, output_plugins)
try:
for plugin in output_plugins:
DISPLAY.vvv('Generating output using %s plugin' % plugin.NAME)
plugin.generate(deploy_config.get_config())
except DeployConfigGenerationError as e:
DISPLAY.display('Failed to generate deploy config: %s' % str(e))
sys.exit(1)
class SiteConfig(object, metaclass=Singleton):
_path = None
_config = None
_display = None
_defaults = {
'default_output':
None,
# Directory within service dir where deploy config is located
'deploy_dir':
'deploy',
# Name of deploy config file
'deploy_config_file':
'config.yml',
# Directory within deploy dir to look for vars files
'vars_dir':
'var',
# Patterns for finding vars files
'defaults_vars_file_patterns': [
'defaults.var',
'defaults.yml',
'defaults.yaml',
'defaults.json',
],
'env_vars_file_patterns': [
'{{ env }}.var',
'env_{{ env }}.var',
'{{ env }}.yml',
'{{ env }}.yaml',
'{{ env }}.json',
'env_{{ env }}.yml',
'env_{{ env }}.yaml',
'env_{{ env }}.json',
],
# Whether to use vars from environment
'use_env_vars':
True,
# Deploy config version to assume if none is provided (defaults to latest)
'default_config_version':
'1',
# Additional plugins dirs
'plugin_dirs': [],
# Plugin-specific options
'plugins': {},
# Default apps
'default_apps': {},
# Default vars
'default_vars': {},
}
def __init__(self, env=None):
self._display = Display()
self._config = self._defaults
# Used for replacing vars in include paths
tmp_vars = dict()
if env is not None:
tmp_vars['env'] = env
self._template = Template(default_vars=tmp_vars)
def __getattr__(self, key):
'''Allows object-like access to keys in the _config dict'''
if key in self._config:
if False and isinstance(self._config[key], dict):
return objdict(self._config[key])
return self._config[key]
else:
raise AttributeError('No such attribute/key: %s' % key)
def __getitem__(self, key):
return self.__getattr__(key)
def __setattr__(self, key, value):
'''Allow setting of internal attributes'''
if key.startswith('_'):
super(SiteConfig, self).__setattr__(key, value)
else:
raise AttributeError('Config object is not directly writeable')
def __setitem__(self, key, value):
self.__setattr__(key, value)
def __contains__(self, key):
'''Allows using 'in' keyword'''
if key in self._config:
return True
return False
def get_config(self):
return self._config
def load_file(self, path):
path = os.path.realpath(path)
with open(path) as f:
data = yaml_load(f)
# Handle empty site config file
# This is mostly here to allow doing '-c /dev/null' in the integration
# tests to prevent them from picking up the user site config
if data is None:
data = {}
if not isinstance(data, dict):
raise ConfigError(
'config file %s should be formatted as YAML dict' % path)
if 'include' in data:
include_paths = self._template.render_template(data['include'])
if not isinstance(include_paths, list):
include_paths = [include_paths]
for include_path in include_paths:
if not include_path.startswith('/'):
# Normalize include path based on location of parent file
include_path = os.path.join(os.path.dirname(path),
include_path)
self._display.v('Loading site config from included file %s' %
include_path)
include_data = self.load_file(include_path)
data = dict_merge(data, include_data)
del data['include']
return data
def load(self, path):
try:
self._display.v('Loading site config from %s' % path)
self._path = os.path.realpath(path)
data = self.load_file(self._path)
# Special case for plugin dirs
if 'plugin_dirs' in data:
if not isinstance(data['plugin_dirs'], list):
data['plugin_dirs'] = [data['plugin_dirs']]
for idx, entry in enumerate(data['plugin_dirs']):
if not entry.startswith('/'):
# Normalize path based on location of site config
data['plugin_dirs'][idx] = os.path.join(
os.path.dirname(self._path), entry)
# Special case for default apps
if 'default_apps' in data:
if not isinstance(data['default_apps'], dict):
raise ConfigError(
'"default_apps" key expects a dict, got: %s' %
type(data['default_apps']))
for section, v in data['default_apps'].items():
if not isinstance(v, list):
raise ConfigError(
'"default_apps" key expects a dict with section names and a list of default apps'
)
self._config.update(data)
except Exception as e:
raise ConfigError(str(e))
def setUp(self):
self._template = Template()
class OutputPluginBase(object):
'''
Base class for output plugins
'''
_vars = None
_output_dir = None
_display = None
_section = None
_plugin_config = None
_fields = None
_config_version = None
COMMON_DEFAULT_CONFIG = dict(enabled=True, )
PRIORITY = 1
def __init__(self, varset, output_dir, config_version):
self._vars = varset
self._output_dir = output_dir
self._display = Display()
self._template = Template()
self._site_config = SiteConfig()
self._config_version = config_version
self.build_config()
# Comparison functions for sorting plugins
# Sort first by priority and then by name (for consistency)
def __lt__(self, other):
return (self.PRIORITY < other.PRIORITY or
(self.PRIORITY == other.PRIORITY and self.NAME < other.NAME))
def __gt__(self, other):
return (self.PRIORITY > other.PRIORITY or
(self.PRIORITY == other.PRIORITY and self.NAME > other.NAME))
def __le__(self, other):
return (self.PRIORITY <= other.PRIORITY or
(self.PRIORITY == other.PRIORITY and self.NAME <= other.NAME))
def __ge__(self, other):
return (self.PRIORITY >= other.PRIORITY or
(self.PRIORITY == other.PRIORITY and self.NAME >= other.NAME))
def __eq__(self, other):
return (self.PRIORITY == other.PRIORITY or
(self.PRIORITY == other.PRIORITY and self.NAME == other.NAME))
def __ne__(self, other):
return (self.PRIORITY != other.PRIORITY or
(self.PRIORITY == other.PRIORITY and self.NAME != other.NAME))
def build_config(self):
'''
Build the plugin config
'''
self._plugin_config = self.COMMON_DEFAULT_CONFIG.copy()
self._plugin_config.update(self.DEFAULT_CONFIG)
# Helper var to tidy up the code
self._fields = copy.deepcopy(self._plugin_config['fields'])
# Convert field definitions into PluginField objects
for section in self._fields:
section_fields = self._fields[section]
for k, v in section_fields.items():
section_fields[k] = PluginField(k, v, self._config_version,
self._template)
self.build_config_site()
def build_config_site(self):
'''
Merge in plugin config values from site config
This will also do a deep merge of deeply nested field definitions
'''
if self.NAME in self._site_config.plugins:
for k, v in self._site_config['plugins'][self.NAME].items():
if k == 'fields':
for section in v:
for field_name, field in v[section].items():
# Create section if it doesn't exist
if section not in self._fields:
self._fields[section] = {}
# Update existing field config or create new
if field_name in self._fields[section]:
self._fields[section][
field_name].update_config(field)
else:
self._fields[section][
field_name] = PluginField(
field_name, field,
self._config_version, self._template)
else:
if k in self._plugin_config:
if isinstance(v, dict):
self._plugin_config[k] = v.copy()
elif isinstance(v, list):
self._plugin_config[k] = v[:]
else:
self._plugin_config[k] = v
else:
raise ConfigError('unrecognized config option: %s' % k)
def set_section(self, section):
'''
Sets the active section of the deploy config
This is used to figure out which set of fields to process
'''
self._section = section
def has_field(self, field):
'''
Check if a field exists in the current section for this plugin
'''
if self._section in self._fields and field in self._fields[
self._section]:
if self._fields[
self._section][field].is_valid_for_config_version():
return True
return False
def get_required_fields(self):
'''
Return a list of fields in the current section with required=True
'''
ret = []
if self._section in self._fields:
for k, v in self._fields[self._section].items():
if v.required and v.default is None and v.is_valid_for_config_version(
):
ret.append(k)
return ret
def is_field_locked(self, field):
'''
Check if a field has been marked as 'locked' (cannot be overridden by user)
'''
if self._section in self._fields and field in self._fields[
self._section]:
if self._fields[self._section][field].locked:
return True
return False
def is_needed(self, app):
'''
Determine whether this plugin is needed based on the provided deploy config
'''
# We aren't needed if we're marked as disabled (enabled: False)
if self._plugin_config.get('enabled', True) is False:
return False
# We aren't needed if we have no fields for the current section
if self._section not in self._fields:
return False
# We are needed if we're the configured default plugin
if self._site_config.default_output == self.NAME:
return True
# Check if any of our required top-level fields are provided
for field in self.get_required_fields():
if field in app:
return True
# If nothing above matched, then we're probably not needed
return False
def merge_with_field_defaults(self, app):
'''
Merge user-provided values with configured field defaults
'''
ret = {}
# Apply defaults/transforms
for field, value in self._fields[self._section].items():
ret[field] = value.apply_default(app.get(field, None))
ret[field] = value.apply_transform(ret.get(field, None))
return ret
def validate_fields(self, app):
'''
Validate the provided app config against plugin field definitions
'''
# Check that all required top-level fields are provided
req_fields = self.get_required_fields()
for field in req_fields:
if field not in app:
raise DeployConfigError("required field '%s' not defined" %
field)
# Check field/subfield types, required, and if field is locked
unmatched = []
for field, value in app.items():
if self.has_field(field):
if self.is_field_locked(field):
raise DeployConfigError(
"the field '%s' has been locked by the plugin config and cannot be overridden"
% field)
field_unmatched = self._fields[self._section][field].validate(
value)
unmatched.extend(field_unmatched)
else:
unmatched.append(field)
return unmatched
def build_app_vars(self, index, app, path=''):
# Build vars for template
app_vars = {
'PLUGIN_NAME': self.NAME,
'APP_INDEX': index,
# App config
'APP': self.merge_with_field_defaults(app),
# Parsed vars
'VARS': dict(self._vars),
}
return app_vars
def pre_process(self, config):
pass
def generate(self, config):
'''
Write out the generated config to disk
'''
try:
self.pre_process(config)
for section in config:
if section in self._fields:
self.set_section(section)
for idx, app in enumerate(config[section]):
# We want a 1-based index for the output files
index = idx + 1
if self.is_needed(app):
# Build vars for template
app_vars = self.build_app_vars(index, app)
# Check conditionals
for field, value in self._fields[
self._section].items():
app_vars['APP'][
field] = value.check_conditionals(
app_vars['APP'].get(field, None),
app_vars)
# Generate output
output = self.generate_output(app_vars)
path_suffix = None
if isinstance(output, (tuple, list)):
output, path_suffix = output
if output is None:
continue
path = os.path.join(
self._output_dir, '%s-%03d%s%s' %
(self.NAME, index,
('-%s' % path_suffix if path_suffix else ''),
self.FILE_EXT))
self._display.v('Writing output file %s' % path)
with open(path, 'w') as f:
f.write(output)
except Exception as e:
show_traceback(self._display.get_verbosity())
raise DeployConfigGenerationError(str(e))
def generate_output(self, app_vars):
'''
Generate output content
By default, this renders the Jinja template defined in the 'TEMPLATE'
class var. However, it can be overridden by an output plugin to provide
a custom method for generating the output.
'''
output = self._template.render_template(
inspect.cleandoc(self.TEMPLATE), app_vars)
return output