def load_inner(data, help_path):
for name in os.listdir(help_path):
path = os.path.join(help_path, name)
if os.path.isfile(path):
if path.endswith('.yml'):
with open(path, 'r') as file:
file_data = yaml.safe_load(file.read())
deep_merge(data, file_data)
else:
load_inner(data, path)
def load_directory(base_path):
if settings.APP_DIR in base_path:
module = 'core'
module_path = settings.APP_DIR
else:
module = base_path.replace(settings.MODULE_BASE_PATH + '/',
'').split('/')[1]
module_path = os.path.join(self.manager.module_dir, module)
module_info = Collection(
module=module, path=self._get_module_lib_dir(module_path))
for name in os.listdir(base_path):
file_path = os.path.join(base_path, name)
if os.path.isdir(file_path):
load_directory(file_path)
elif name[0] != '_' and re.match(r'^[^\.]+\.(yml|yaml)$',
name, re.IGNORECASE):
logger.debug(
"Loading specification from file: {}".format(
file_path))
spec_data = load_yaml(file_path)
if spec_data:
for key, info in spec_data.items():
if key[0] != '_':
self.module_map.setdefault(key, {})
if key == 'roles':
for name, description in info.items():
self.module_map[key][
name] = module_info
else:
for name, spec in info.items():
app_name = spec.get('app', name)
self.module_map[key][
app_name] = module_info
if key in ('data', 'data_base',
'data_mixins'):
module_name = model_index.get_module_name(
key, app_name)
model_class = model_index.get_model_name(
name, spec)
dynamic_class = model_index.get_dynamic_class_name(
model_class)
self.model_class_path[
model_class] = module_name
self.model_class_path[
dynamic_class] = module_name
deep_merge(self._spec, spec_data)
def load_template_type(self, template_path, type_name):
template_type_path = os.path.join(template_path, type_name)
for package_name in os.listdir(template_type_path):
template_package_path = os.path.join(template_type_path, package_name)
template_package_home = os.path.join(self.template_dir, type_name, package_name)
create_dir(template_package_home)
for file_info in get_files(template_package_path):
if len(file_info) == 2 and file_info[1] == 'index.yml':
package_module_config = load_yaml(os.path.join(*file_info))
template_home_index = os.path.join(template_package_home, 'index.yml')
package_home_config = load_yaml(template_home_index)
if package_home_config is None:
package_home_config = package_module_config
else:
package_home_config = deep_merge(package_home_config, package_module_config)
save_yaml(template_home_index, package_home_config)
else:
if file_info[1:-1]:
create_dir(os.path.join(*[ template_package_home, *file_info[1:-1] ]))
if file_info[1:]:
copyfile(
os.path.join(*file_info),
os.path.join(*[ template_package_home, *file_info[1:] ])
)
def get_schema(self, config):
schema = {}
for profile in self.parents:
profile_schema = profile.get_schema(config)
profile_schema['config'] = self.interpolate_config(
profile_schema.get('config', {}))
self.merge_schema(schema, profile_schema)
profile_schema['config'] = self.interpolate_config(
profile_schema.get('config_store', {}))
self.merge_schema(schema, profile_schema)
self.data['config'] = self.interpolate_config(
deep_merge(self.data.get('config', {}), config))
self.merge_schema(schema, self.data)
self.data['config'] = self.interpolate_config(
self.data.get('config_store', {}))
self.merge_schema(schema, self.data)
for component in [
'run', 'pre_run', 'post_run', 'destroy', 'pre_destroy',
'post_destroy', 'profile'
]:
if component in schema:
for name, component_config in schema[component].items():
if 'module' not in component_config:
component_config['module'] = self.module.instance.name
return schema
def destroy(self, name, config):
host = self.pop_value('host', config)
profile = self.pop_value('profile', config)
if not profile:
self.command.error(
"Profile {} requires 'profile' field".format(name))
module = self.pop_value('module', config)
operations = self.pop_values('operations', config)
components = self.pop_values('components', config)
if self.profile.components and components:
components = list(set(self.profile.components) & set(components))
elif self.profile.components:
components = self.profile.components
if not operations or 'destroy' in operations:
self.pop_value('once', config)
try:
self.exec('destroy',
environment_host=host,
module_name=module,
profile_name=profile,
profile_config_fields=deep_merge(
copy.deepcopy(self.profile.data['config']),
config),
profile_components=components)
except (ConnectTimeout, ConnectionError):
self.command.warning(
"Remote host does not exist for: {}".format(name))
self.command.delete_state(self.state_name(module, profile))
def load_inner(data, help_path):
for name in os.listdir(help_path):
path = os.path.join(help_path, name)
if os.path.isfile(path):
if path.endswith('.yml'):
file_data = load_yaml(path)
data = deep_merge(data, file_data)
else:
load_inner(data, path)
def save_module_config(self, module_name, config):
module_path = os.path.join(self.manager.module_dir, module_name)
zimagi_config_path = os.path.join(module_path, '.zimagi.yml')
loaded_config = {}
if os.path.isfile(zimagi_config_path):
loaded_config = load_yaml(zimagi_config_path)
if not isinstance(loaded_config, dict):
loaded_config = {}
config = deep_merge(loaded_config, config)
save_yaml(zimagi_config_path, config)
def destroy(self, name, config, display_only=False):
config = copy.deepcopy(config)
host = self.pop_value('_host', config)
profile = self.pop_value('_profile', config)
if not profile:
self.command.error(
"Profile {} requires '_profile' field".format(name))
queue = self.pop_value(
'_queue',
config) if '_queue' in config else settings.QUEUE_COMMANDS
module = self.pop_value('_module', config)
operations = self.pop_values('_operations', config)
wait_keys = self.pop_values('_wait_keys', config)
log_key = None
components = self.pop_values('_components', config)
if self.profile.components and components:
components = list(set(self.profile.components) & set(components))
elif self.profile.components:
components = self.profile.components
if not operations or 'destroy' in operations:
self.pop_value('_once', config)
options = {
"environment_host":
host,
"module_name":
module,
"profile_name":
profile,
"profile_config_fields":
deep_merge(copy.deepcopy(self.profile.data['config']), config),
"profile_components":
components,
"display_only":
display_only,
"_wait_keys":
wait_keys
}
if settings.QUEUE_COMMANDS:
options['push_queue'] = queue if not display_only else False
try:
log_key = self.exec('destroy', **options)
except (ConnectTimeout, ConnectionError):
self.command.warning(
"Remote host does not exist for: {}".format(name))
self.command.delete_state(self.state_name(module, profile))
return log_key if queue else None
def run(self, name, config, display_only=False):
host = self.pop_value('host', config)
profile = self.pop_value('profile', config)
if not profile:
self.command.error(
"Profile {} requires 'profile' field".format(name))
module = self.pop_value('module', config)
state_name = self.state_name(module, profile)
operations = self.pop_values('operations', config)
components = self.pop_values('components', config)
if self.profile.components and components:
components = list(set(self.profile.components) & set(components))
elif self.profile.components:
components = self.profile.components
if display_only or not operations or 'run' in operations:
once = self.pop_value('once', config)
if display_only or not once or not self.command.get_state(
state_name):
run_options = {
"environment_host":
host,
"module_name":
module,
"profile_name":
profile,
"profile_config_fields":
deep_merge(copy.deepcopy(self.profile.data['config']),
config),
"profile_components":
components,
"display_only":
display_only,
"plan":
self.test
}
try:
self.exec('run', **run_options)
except (ConnectTimeout, ConnectionError) as e:
if display_only:
run_options.pop('environment_host', None)
self.command.warning(
"Displaying local profile for: {}\n".format(name))
self.exec('run', **run_options)
else:
raise e
self.command.set_state(state_name, True)
def _get_module_config(self, path):
if path not in self.module_index:
self.module_index[path] = {}
for config_file in ('zimagi.yml', '.zimagi.yml'):
zimagi_config = os.path.join(path, config_file)
if os.path.isfile(zimagi_config):
self.module_index[path] = deep_merge(
self.module_index[path], load_yaml(zimagi_config))
if not self.module_index.get(path, None):
self.module_index[path] = {'lib': '.'}
return self.module_index[path]
def get_task(self, task_name, show_options=True):
instance = self.check_instance('module get task')
module_config = self.module_config()
tasks = {}
module_config.setdefault('tasks', 'tasks')
if module_config['tasks']:
module_path = self.module_path(instance.name)
tasks_path = os.path.join(module_path, module_config['tasks'])
tasks = self.import_tasks(tasks_path)
if task_name == 'list' or task_name not in tasks:
if show_options:
self.command.info("Available tasks in this module:\n")
for name in sorted(tasks.keys()):
task = self.command.get_provider('task',
tasks[name]['provider'],
self, tasks[name])
fields = deep_merge(task.get_fields(),
{'provider': '<required>'})
self.command.info(" * {}\n".format(
self.command.header_color(name)))
self.command.notice(
yaml.dump(deep_merge(fields, tasks[name])))
if task_name == 'list':
self.command.error('')
else:
self.command.error(
"Task {} not found in module {} zimagi.yml".format(
task_name, self.instance.name))
config = tasks[task_name]
provider = config.pop('provider', 'command')
return self.command.get_provider('task', provider, self, config)
def run(self, name, config, display_only=False):
config = copy.deepcopy(config)
host = self.pop_value('_host', config)
profile = self.pop_value('_profile', config)
if not profile:
self.command.error(
"Profile {} requires '_profile' field".format(name))
queue = self.pop_value(
'_queue',
config) if '_queue' in config else settings.QUEUE_COMMANDS
module = self.pop_value('_module', config)
state_name = self.state_name(module, profile)
operations = self.pop_values('_operations', config)
wait_keys = self.pop_values('_wait_keys', config)
log_key = None
components = self.pop_values('_components', config)
if self.profile.components and components:
components = list(set(self.profile.components) & set(components))
elif self.profile.components:
components = self.profile.components
if display_only or not operations or 'run' in operations:
once = self.pop_value('_once', config)
if display_only or not once or not self.command.get_state(
state_name):
options = {
"environment_host":
host,
"module_name":
module,
"profile_name":
profile,
"profile_config_fields":
deep_merge(copy.deepcopy(self.profile.data['config']),
config),
"profile_components":
components,
"display_only":
display_only,
"test":
self.test,
'_wait_keys':
wait_keys
}
if settings.QUEUE_COMMANDS:
options[
'push_queue'] = queue if not display_only else False
try:
log_key = self.exec('run', **options)
except (ConnectTimeout, ConnectionError) as e:
if display_only:
options.pop('environment_host', None)
options.pop('push_queue', None)
self.command.warning(
"Displaying local profile for: {}\n".format(name))
log_key = self.exec('run', **options)
else:
raise e
self.command.set_state(state_name, True)
return log_key if queue else None
def _store_template_map(self, module, index, template_fields,
display_only):
self.notice('Template variables:')
self.table([['Variable', 'Value', 'Help']] +
[[key, value, index.variables[key].get('help', 'NA')]
for key, value in template_fields.items()], 'variables')
self.info('')
for path, info in index.map.items():
target = None
if isinstance(info, str):
target = info
info = {}
elif info.get('when', True) and 'target' in info:
target = info['target']
if target:
path_components = os.path.split(
self.manager.get_module_path(module, target))
target_path = os.path.join(*path_components)
if info.get('template', True):
file_content = self._render_package_template(
index.name, path, template_fields)
else:
file_content = load_file(
self.manager.get_template_path(index.name, path))
if info.get('location', None) and path.endswith('.yml'):
file_data = normalize_value(load_yaml(target_path),
strip_quotes=True,
parse_json=True)
if not file_data:
file_data = {}
location = info['location'].split('.')
embed_data = normalize_value(oyaml.safe_load(file_content),
strip_quotes=True,
parse_json=True)
merge_data = {}
iter_data = merge_data
for index, key in enumerate(location):
if (index + 1) == len(location):
iter_data[key] = embed_data
else:
iter_data[key] = {}
iter_data = iter_data[key]
file_content = oyaml.dump(deep_merge(
file_data, merge_data))
self.data('Path', path, 'path')
self.data('Target', target, 'target')
if info.get('location', None):
self.data('location', info['location'], 'location')
self.notice('-' * self.display_width)
if info.get('template', True):
self.info(file_content)
self.info('')
if not display_only:
create_dir(path_components[0])
save_file(target_path, file_content)