def _collect_fields(self, facade):
fields = {facade.pk: True, facade.key(): True, self.field_field: True}
def add_fields(data):
if data:
for key, values in data.items():
for value in ensure_list(values):
if isinstance(value, str):
match = re.search(r'\@\{?([a-zA-Z0-9\_\-]+)\}?',
value.strip())
if match:
for field in match.groups():
fields[field] = True
add_fields(self.field_filters)
if self.field_parents:
for data_name, record in self.field_parents.items():
add_fields(record)
for name, info in self.field_params.items():
if isinstance(info, str):
fields[info] = True
else:
add_fields(info.get('filters', {}))
for field in ensure_list(info.get('order', [])):
fields[re.sub(r'^[~-]', '', field)] = True
if self.field_extra_fields:
for name in ensure_list(self.field_extra_fields):
fields[name] = True
return list(fields.keys())
def get_instances(self,
facade,
names=[],
objects=[],
groups=[],
fields={}):
search_items = []
results = {}
if not names and not groups and not objects and not fields:
search_items = facade.all()
else:
search_items.extend(data.ensure_list(names))
search_items.extend(data.ensure_list(objects))
for group in data.ensure_list(groups):
search_items.extend(facade.keys(groups__name=group))
def init_instance(object):
if isinstance(object, str):
cached = self._get_cache_instance(facade, object)
if not cached:
instance = facade.retrieve(object)
else:
instance = cached
else:
instance = object
cached = self._get_cache_instance(facade,
getattr(instance, facade.pk))
if instance:
id = getattr(instance, facade.pk)
if not cached:
if instance.initialize(self):
self._set_cache_instance(facade, id, instance)
else:
instance = None
else:
instance = cached
if instance:
if fields:
for field, values in fields.items():
values = data.normalize_value(values)
value = getattr(instance, field, None)
if isinstance(values,
str) and not value and re.match(
r'^(none|null)$', values,
re.IGNORECASE):
results[id] = instance
elif value and value in data.ensure_list(values):
results[id] = instance
else:
results[id] = instance
else:
self.error("{} instance {} does not exist".format(
facade.name.title(), object))
self.run_list(search_items, init_instance)
return results.values()
def has_permission(self, request, view):
if request.method not in ('GET', 'OPTIONS', 'HEAD'):
raise exceptions.MethodNotAllowed(request.method)
if not getattr(view, 'queryset', None):
# Schema view should be accessible
return True
model_name = view.queryset.model._meta.data_name
roles = settings.MANAGER.get_spec('data.{}.roles'.format(model_name))
groups = roles.get('edit', [])
if roles.get('view', None):
groups.extend(ensure_list(roles['view']))
if not groups:
raise exceptions.AuthenticationFailed(
'Not authorized to view this data set')
elif 'public' in groups:
return True
if not request.user:
raise exceptions.AuthenticationFailed(
'Authentication credentials were not provided')
return request.user.env_groups.filter(name__in=groups).exists()
def process_instance(name):
instance_config = copy.deepcopy(data[component.name][name])
name = self.interpolate_config_value(name)
if self.include_instance(name, instance_config):
if isinstance(instance_config, dict):
if '_foreach' in instance_config:
expansion[priority] = True
if priority not in expansion and \
name not in processed and \
check_include(instance_config):
instance_config = self.interpolate_config_value(
instance_config)
if isinstance(instance_config, dict):
requirements[name] = instance_config.pop(
'_requires', [])
if requirements[name]:
instance_config[
'_wait_keys'] = get_wait_keys(name)
if settings.DEBUG_COMMAND_PROFILES:
self.command.info(
yaml.dump({name: instance_config},
Dumper=noalias_dumper))
log_keys = component_method(name, instance_config)
processed[name] = ensure_list(
log_keys) if log_keys else []
def load_parents(self):
self.parents = []
self.set_config(self.get_config())
if 'parents' in self.data:
parents = self.data.pop('parents')
for parent in reversed(ensure_list(parents)):
module = self.module.instance
if isinstance(parent, str):
profile_name = self.interpolate_config_value(parent)
else:
profile_name = self.interpolate_config_value(
parent['profile'])
if 'module' in parent:
module_name = self.interpolate_config_value(
parent['module'])
if module_name != 'self':
module = self.get_module(module_name)
self.parents.insert(0,
module.provider.get_profile(profile_name))
for profile in reversed(self.parents):
profile.load_parents()
def parse_fields(self, facade, name, optional = False, help_callback = None, callback_args = None, callback_options = None, exclude_fields = None):
if not callback_args:
callback_args = []
if not callback_options:
callback_options = {}
if exclude_fields:
exclude_fields = data.ensure_list(exclude_fields)
callback_options['exclude_fields'] = exclude_fields
if name not in self.option_map:
if facade:
help_text = "\n".join(self.field_help(facade, exclude_fields))
else:
help_text = "\nfields as key value pairs\n"
if help_callback and callable(help_callback):
help_text += "\n".join(help_callback(*callback_args, **callback_options))
self.add_schema_field(name,
args.parse_key_values(self.parser, name, help_text,
value_label = 'field=VALUE',
optional = optional
),
optional
)
self.option_map[name] = True
def init_instance(object):
if isinstance(object, str):
cached = self._get_cache_instance(facade, object)
if not cached:
instance = facade.retrieve(object)
else:
instance = cached
else:
instance = object
cached = self._get_cache_instance(facade, getattr(instance, facade.pk))
if instance:
id = getattr(instance, facade.pk)
if not cached:
if instance.initialize(self):
self._set_cache_instance(facade, id, instance)
else:
instance = None
else:
instance = cached
if instance:
if fields:
for field, values in fields.items():
values = data.normalize_value(values)
value = getattr(instance, field, None)
if isinstance(values, str) and not value and re.match(r'^(none|null)$', values, re.IGNORECASE):
results[id] = instance
elif value and value in data.ensure_list(values):
results[id] = instance
else:
results[id] = instance
else:
self.error("{} instance {} does not exist".format(facade.name.title(), object))
def kube_exec(self, filesystem, executable, options = None, env = None):
args = self.options.get('args', [])
if options is None:
options = []
if options:
command = [ executable ] + ensure_list(options) + args
else:
command = [ executable ] + args
config_path = os.path.join(self.manager.data_dir, '.kube', 'config')
command_env = {
"KUBECONFIG": filesystem.link(config_path, '.kube')
}
if env and isinstance(env, dict):
command_env = { **command_env, **env }
success = self.sh(
command,
env = command_env,
cwd = self.manager.module_dir,
display = True
)
if not success:
self.error("Command {} failed: {}".format(executable, " ".join(command)))
def remove_related(self, instance, relation, facade, names):
queryset = query.get_queryset(instance, relation)
instance_name = type(instance).__name__.lower()
key = getattr(instance, instance.facade.key())
keep_index = instance.facade.keep_relations().get(relation, {})
keep = data.ensure_list(keep_index.get(key, []))
if queryset:
for name in names:
if name not in keep:
sub_instance = facade.retrieve(name)
if sub_instance:
try:
with facade.thread_lock:
queryset.remove(sub_instance)
except Exception as e:
self.command.error("{} remove failed: {}".format(
facade.name.title(), str(e)))
self.command.success(
"Successfully removed {} {} from {} {}".format(
sub_instance.facade.name, name,
instance.facade.name, key))
else:
self.command.warning("{} {} does not exist".format(
facade.name.title(), name))
else:
self.command.error(
"{} {} removal from {} is restricted".format(
facade.name.title(), name, key))
else:
self.command.error("There is no relation {} on {} class".format(
relation, instance_name))
def exec(self, list_value, limit = None):
values = ensure_list(list_value)
random.shuffle(values)
if limit:
return values[:min(int(limit), len(values))]
return values
def get_merge_values(merge_filters):
values = []
for merge_field in data.ensure_list(dataframe_merge_fields):
values.append(
re.sub(r'[^a-z0-9]+', '',
str(merge_filters[merge_field]).lower()))
return values
def execute(self, results, params):
if 'script' in self.config:
script_path = self.get_path(self.config['script'])
else:
self.command.error("Remote script task provider must have a 'script' property specified that links to an executable file")
if not os.path.exists(script_path):
self.command.error("Remote script task provider file {} does not exist".format(script_path))
script_base, script_ext = os.path.splitext(script_path)
temp_path = "/tmp/{}{}".format(self.generate_name(24), script_ext)
env = self._env_vars(params)
sudo = self.config.get('sudo', False)
lock = self.config.get('lock', False)
options = self._merge_options(self.config.get('options', {}), params, lock)
args = ensure_list(self.config.get('args', []))
def exec_server(server):
ssh = server.provider.ssh(env = env)
ssh.upload(script_path, temp_path, mode = 755)
try:
self._ssh_exec(server, temp_path,
self._interpolate(args, options),
sudo = sudo,
env = env,
ssh = ssh
)
finally:
ssh.sudo('rm -f', temp_path)
self.command.run_list(
self._ssh_servers(params),
exec_server
)
def delete(self, key, **filters):
if key not in data.ensure_list(self.keep(key)):
filters[self.key()] = key
return self.clear(**filters)
else:
raise RestrictedError("Removal of {} {} is restricted".format(
self.model.__name__.lower(), key))
def accessor(self):
if 'default_callback' in method_info:
default_callback = getattr(self, method_info['default_callback'],
None)
if default_callback is None:
raise CallbackNotExistsError(
"Command parameter default callback {} does not exist".
format(default_callback))
default_value = default_callback()
else:
default_value = method_info.get('default', None)
value = self.options.get(method_base_name, default_value)
if value is not None and method_info['parser'] == 'variables':
value = ensure_list(value)
if 'postprocessor' in method_info:
postprocessor = getattr(self, method_info['postprocessor'], None)
if postprocessor is None:
raise CallbackNotExistsError(
"Command parameter postprocessor {} does not exist".format(
postprocessor))
if value is not None:
value = postprocessor(value)
return value
def _order_data(self, spec):
dependencies = {}
priorities = {}
priority_map = {}
if isinstance(spec, dict):
for name, config in spec.items():
if config is not None and isinstance(config, dict):
requires = ensure_list(config.get('requires', []))
dependencies[name] = requires
for name, requires in dependencies.items():
priorities[name] = 0
for index in range(0, len(dependencies.keys())):
for name in list(dependencies.keys()):
for require in dependencies[name]:
priorities[name] = max(priorities[name],
priorities[require] + 1)
for name, priority in priorities.items():
if priority not in priority_map:
priority_map[priority] = []
priority_map[priority].append(name)
return priority_map
def relation_fields(self):
scope = []
if getattr(self.meta, 'relation', None):
for field in data.ensure_list(self.meta.relation):
if field not in self.scope_fields:
scope.append(field)
return scope
def __init__(self):
self.default_modules = ensure_list(settings.DEFAULT_MODULES)
self.remote_module_names = {}
self.ordered_modules = None
self.module_index = {}
self.module_dependencies = {}
super().__init__()
def get_merged_dataframe(field_info):
merge_fields = data.ensure_list(dataframe_merge_fields)
merge_filter_index = {}
dataframe = None
for merge_filters in list(facade.values(*merge_fields, **filters)):
merge_values = get_merge_values(merge_filters)
merge_filter_id = data.get_identifier(merge_values)
if merge_filter_id not in merge_filter_index:
sub_dataframe = get_dataframe(field_info, {
**filters,
**merge_filters
})
value_prefix = "_".join(merge_values)
sub_dataframe.columns = [
"{}_{}".format(value_prefix, column)
for column in sub_dataframe.columns
]
if dataframe is None:
dataframe = sub_dataframe
else:
dataframe = dataframe.merge(sub_dataframe,
how="outer",
left_index=True,
right_index=True)
merge_filter_index[merge_filter_id] = True
return dataframe
def execute(self, results, params):
script_path = self.get_path(self.field_script)
if not os.path.exists(script_path):
self.command.error(
"Remote script task provider file {} does not exist".format(
script_path))
script_base, script_ext = os.path.splitext(script_path)
temp_path = "/tmp/{}{}".format(self.generate_name(24), script_ext)
env = self._env_vars(params)
options = self._merge_options(self.field_options, params,
self.field_lock)
args = ensure_list(self.field_args, [])
ssh = self._get_ssh(env)
ssh.upload(script_path, temp_path, mode=755)
try:
self._ssh_exec(temp_path,
self._interpolate(args, options),
sudo=self.field_sudo,
ssh=ssh)
finally:
ssh.exec('rm -f', temp_path)
def initialize_terraform(self, instance, created):
instance.config['rule_type'] = 'cidr'
instance.config['source_firewall_id'] = None
if instance.config['source_firewall']:
instance.config['rule_type'] = 'link'
instance.config['self_only'] = False
instance.cidrs = []
tries = 60
while True:
firewall = self.command._firewall.retrieve(instance.config['source_firewall'])
if firewall:
firewall.initialize(self.command)
instance.config['source_firewall_id'] = firewall.provider.get_firewall_id()
break
time.sleep(2)
tries -= 2
if not tries:
self.command.error("Source firewall {} could not be retrieved".format(instance.config['source_firewall']))
elif instance.config['self_only']:
instance.config['rule_type'] = 'link'
instance.cidrs = []
elif instance.cidrs:
instance.cidrs = [str(self.address.parse_cidr(x.strip())) for x in ensure_list(instance.cidrs)]
elif not instance.config['self_only'] and not instance.config['source_firewall']:
instance.cidrs = ['0.0.0.0/0']
def exec(self, *elements):
values = []
for element in elements:
values.extend(ensure_list(element))
return values
def send_notification(self, recipient, subject, body):
if settings.EMAIL_HOST and settings.EMAIL_HOST_USER:
try:
html_body = body.replace("\n", '<br/>')
html_body = html_body.replace(" ", ' ')
html_body = '<font face="Courier New, Courier, monospace">{}</font>'.format(html_body)
send_mail(
subject,
body,
settings.EMAIL_HOST_USER,
ensure_list(recipient),
html_message = html_body
)
logger.info("Notification message '{}' sent to: {}".format(subject, recipient))
except SMTPConnectError as e:
logger.error("Notification delivery failed: {}".format(e))
raise self.retry(exc = e)
except SMTPServerDisconnected as e:
logger.error("Notification service disconnected: {}".format(e))
raise self.retry(exc = e)
except Exception as e:
logger.error("Notification error: {}".format(e))
raise e
def exec_methods(instance, methods):
for method, params in methods.items():
method = getattr(instance, method)
if not params:
method()
elif isinstance(params, dict):
method(**params)
else:
method(*ensure_list(params))
def get_wait_keys(_name):
wait_keys = []
if _name in requirements and requirements[_name]:
for _child_name in flatten(ensure_list(requirements[_name])):
if processed[_child_name]:
wait_keys.extend(processed[_child_name])
wait_keys.extend(get_wait_keys(_child_name))
return list(set(wait_keys))
def set_order(self, order):
if order:
self.order = [
re.sub(r'^~', '-', x) for x in data.ensure_list(order)
]
else:
self.order = None
return self
def __call__(self, parser, namespace, values, option_string=None):
arg_values = []
if not values:
values = []
for value in ensure_list(values):
arg_values.append(self.type(value))
setattr(namespace, self.dest, arg_values)
def init_parents(self):
if 'base' not in self.spec:
self.parents = [self.base_model]
else:
self.parents = [self.get_model(self.spec['base'], BaseModel)]
if 'mixins' in self.spec:
for mixin in ensure_list(self.spec['mixins']):
self.parents.append(self.get_model(mixin, ModelMixin))
def GetCommand(parents,
base_name,
facade_name,
view_roles=None,
name_field=None):
_parents = ensure_list(parents)
_facade_name = get_facade(facade_name)
_name_field = get_joined_value(name_field, base_name, 'name')
def __get_priority(self):
return 10
def __groups_allowed(self):
from settings.roles import Roles
return [Roles.admin] + ensure_list(view_roles)
def __get_epilog(self):
facade = getattr(self, _facade_name)
variable = "{}_display_fields".format(facade.name)
fields = [x.name for x in reversed(facade.meta.get_fields())]
return 'field display config: {}\n\n> {} fields: {}'.format(
self.header_color(variable), facade.name,
self.notice_color(", ".join(fields)))
def __parse(self):
facade = getattr(self, _facade_name)
if not name_field:
getattr(self, "parse_{}".format(_name_field))()
else:
self.parse_scope(facade)
self.parse_dependency(facade)
parse_field_names(self)
def __exec(self):
facade = getattr(self, _facade_name)
instance = getattr(self, base_name)
self.table(self.render_display(facade,
getattr(instance, facade.key()),
allowed_fields=get_field_names(self)),
row_labels=True)
def __str__(self):
return "Get <{}>".format(base_name)
methods = {
'get_priority': __get_priority,
'get_epilog': __get_epilog,
'parse': __parse,
'exec': __exec,
'__str__': __str__
}
if view_roles:
methods['groups_allowed'] = __groups_allowed
return type('GetCommand', tuple(_parents), methods)
def parse_requirements(self):
requirements = []
for path, config in self.index.get_ordered_modules().items():
if 'requirements' in config:
for requirement_path in ensure_list(config['requirements']):
requirement_path = os.path.join(path, requirement_path)
file_contents = load_file(requirement_path)
if file_contents:
requirements.extend([ req for req in file_contents.split("\n") if req and req[0].strip() != '#' ])
return requirements
def add_fields(data):
if data:
for key, values in data.items():
for value in ensure_list(values):
if isinstance(value, str):
match = re.search(r'\@\{?([a-zA-Z0-9\_\-]+)\}?',
value.strip())
if match:
for field in match.groups():
fields[field] = True
def destroy(self, name, children):
if children:
if isinstance(children, dict):
for child, grandchildren in children.items():
self.destroy(child, grandchildren)
elif isinstance(children, (list, tuple, str)):
for child in ensure_list(children):
self.destroy(child, None)
return self.exec('group remove', group_name=name, force=True)
def include_instance(self, name, config):
if isinstance(config, dict):
when = config.pop('when', None)
when_not = config.pop('when_not', None)
when_in = config.pop('when_in', None)
when_not_in = config.pop('when_not_in', None)
when_type = config.pop('when_type', 'AND').upper()
if when is not None:
result = True if when_type == 'AND' else False
for variable in ensure_list(when):
value = format_value(
'bool', self.command.options.interpolate(variable))
if when_type == 'AND':
if not value:
return False
else:
if value:
result = True
return result
if when_not is not None:
result = True if when_type == 'AND' else False
for variable in ensure_list(when_not):
value = format_value(
'bool', self.command.options.interpolate(variable))
if when_type == 'AND':
if value:
return False
else:
if not value:
result = True
return result
if when_in is not None:
value = self.command.options.interpolate(when_in)
return name in ensure_list(value)
if when_not_in is not None:
value = self.command.options.interpolate(when_not_in)
return name not in ensure_list(value)
return True
def initialize_terraform(self, instance, created):
super().initialize_terraform(instance, created)
instance.healthy_status = ensure_list(instance.healthy_status)
