def _collect_fields(self, facade): fields = {facade.pk: True, facade.key(): True, self.field_field: True} def add_fields(data): if data: for key, values in data.items(): for value in ensure_list(values): if isinstance(value, str): match = re.search(r'\@\{?([a-zA-Z0-9\_\-]+)\}?', value.strip()) if match: for field in match.groups(): fields[field] = True add_fields(self.field_filters) if self.field_parents: for data_name, record in self.field_parents.items(): add_fields(record) for name, info in self.field_params.items(): if isinstance(info, str): fields[info] = True else: add_fields(info.get('filters', {})) for field in ensure_list(info.get('order', [])): fields[re.sub(r'^[~-]', '', field)] = True if self.field_extra_fields: for name in ensure_list(self.field_extra_fields): fields[name] = True return list(fields.keys())
def get_instances(self, facade, names=[], objects=[], groups=[], fields={}): search_items = [] results = {} if not names and not groups and not objects and not fields: search_items = facade.all() else: search_items.extend(data.ensure_list(names)) search_items.extend(data.ensure_list(objects)) for group in data.ensure_list(groups): search_items.extend(facade.keys(groups__name=group)) def init_instance(object): if isinstance(object, str): cached = self._get_cache_instance(facade, object) if not cached: instance = facade.retrieve(object) else: instance = cached else: instance = object cached = self._get_cache_instance(facade, getattr(instance, facade.pk)) if instance: id = getattr(instance, facade.pk) if not cached: if instance.initialize(self): self._set_cache_instance(facade, id, instance) else: instance = None else: instance = cached if instance: if fields: for field, values in fields.items(): values = data.normalize_value(values) value = getattr(instance, field, None) if isinstance(values, str) and not value and re.match( r'^(none|null)$', values, re.IGNORECASE): results[id] = instance elif value and value in data.ensure_list(values): results[id] = instance else: results[id] = instance else: self.error("{} instance {} does not exist".format( facade.name.title(), object)) self.run_list(search_items, init_instance) return results.values()
def has_permission(self, request, view): if request.method not in ('GET', 'OPTIONS', 'HEAD'): raise exceptions.MethodNotAllowed(request.method) if not getattr(view, 'queryset', None): # Schema view should be accessible return True model_name = view.queryset.model._meta.data_name roles = settings.MANAGER.get_spec('data.{}.roles'.format(model_name)) groups = roles.get('edit', []) if roles.get('view', None): groups.extend(ensure_list(roles['view'])) if not groups: raise exceptions.AuthenticationFailed( 'Not authorized to view this data set') elif 'public' in groups: return True if not request.user: raise exceptions.AuthenticationFailed( 'Authentication credentials were not provided') return request.user.env_groups.filter(name__in=groups).exists()
def process_instance(name): instance_config = copy.deepcopy(data[component.name][name]) name = self.interpolate_config_value(name) if self.include_instance(name, instance_config): if isinstance(instance_config, dict): if '_foreach' in instance_config: expansion[priority] = True if priority not in expansion and \ name not in processed and \ check_include(instance_config): instance_config = self.interpolate_config_value( instance_config) if isinstance(instance_config, dict): requirements[name] = instance_config.pop( '_requires', []) if requirements[name]: instance_config[ '_wait_keys'] = get_wait_keys(name) if settings.DEBUG_COMMAND_PROFILES: self.command.info( yaml.dump({name: instance_config}, Dumper=noalias_dumper)) log_keys = component_method(name, instance_config) processed[name] = ensure_list( log_keys) if log_keys else []
def load_parents(self): self.parents = [] self.set_config(self.get_config()) if 'parents' in self.data: parents = self.data.pop('parents') for parent in reversed(ensure_list(parents)): module = self.module.instance if isinstance(parent, str): profile_name = self.interpolate_config_value(parent) else: profile_name = self.interpolate_config_value( parent['profile']) if 'module' in parent: module_name = self.interpolate_config_value( parent['module']) if module_name != 'self': module = self.get_module(module_name) self.parents.insert(0, module.provider.get_profile(profile_name)) for profile in reversed(self.parents): profile.load_parents()
def parse_fields(self, facade, name, optional = False, help_callback = None, callback_args = None, callback_options = None, exclude_fields = None): if not callback_args: callback_args = [] if not callback_options: callback_options = {} if exclude_fields: exclude_fields = data.ensure_list(exclude_fields) callback_options['exclude_fields'] = exclude_fields if name not in self.option_map: if facade: help_text = "\n".join(self.field_help(facade, exclude_fields)) else: help_text = "\nfields as key value pairs\n" if help_callback and callable(help_callback): help_text += "\n".join(help_callback(*callback_args, **callback_options)) self.add_schema_field(name, args.parse_key_values(self.parser, name, help_text, value_label = 'field=VALUE', optional = optional ), optional ) self.option_map[name] = True
def init_instance(object): if isinstance(object, str): cached = self._get_cache_instance(facade, object) if not cached: instance = facade.retrieve(object) else: instance = cached else: instance = object cached = self._get_cache_instance(facade, getattr(instance, facade.pk)) if instance: id = getattr(instance, facade.pk) if not cached: if instance.initialize(self): self._set_cache_instance(facade, id, instance) else: instance = None else: instance = cached if instance: if fields: for field, values in fields.items(): values = data.normalize_value(values) value = getattr(instance, field, None) if isinstance(values, str) and not value and re.match(r'^(none|null)$', values, re.IGNORECASE): results[id] = instance elif value and value in data.ensure_list(values): results[id] = instance else: results[id] = instance else: self.error("{} instance {} does not exist".format(facade.name.title(), object))
def kube_exec(self, filesystem, executable, options = None, env = None): args = self.options.get('args', []) if options is None: options = [] if options: command = [ executable ] + ensure_list(options) + args else: command = [ executable ] + args config_path = os.path.join(self.manager.data_dir, '.kube', 'config') command_env = { "KUBECONFIG": filesystem.link(config_path, '.kube') } if env and isinstance(env, dict): command_env = { **command_env, **env } success = self.sh( command, env = command_env, cwd = self.manager.module_dir, display = True ) if not success: self.error("Command {} failed: {}".format(executable, " ".join(command)))
def remove_related(self, instance, relation, facade, names): queryset = query.get_queryset(instance, relation) instance_name = type(instance).__name__.lower() key = getattr(instance, instance.facade.key()) keep_index = instance.facade.keep_relations().get(relation, {}) keep = data.ensure_list(keep_index.get(key, [])) if queryset: for name in names: if name not in keep: sub_instance = facade.retrieve(name) if sub_instance: try: with facade.thread_lock: queryset.remove(sub_instance) except Exception as e: self.command.error("{} remove failed: {}".format( facade.name.title(), str(e))) self.command.success( "Successfully removed {} {} from {} {}".format( sub_instance.facade.name, name, instance.facade.name, key)) else: self.command.warning("{} {} does not exist".format( facade.name.title(), name)) else: self.command.error( "{} {} removal from {} is restricted".format( facade.name.title(), name, key)) else: self.command.error("There is no relation {} on {} class".format( relation, instance_name))
def exec(self, list_value, limit = None): values = ensure_list(list_value) random.shuffle(values) if limit: return values[:min(int(limit), len(values))] return values
def get_merge_values(merge_filters): values = [] for merge_field in data.ensure_list(dataframe_merge_fields): values.append( re.sub(r'[^a-z0-9]+', '', str(merge_filters[merge_field]).lower())) return values
def execute(self, results, params): if 'script' in self.config: script_path = self.get_path(self.config['script']) else: self.command.error("Remote script task provider must have a 'script' property specified that links to an executable file") if not os.path.exists(script_path): self.command.error("Remote script task provider file {} does not exist".format(script_path)) script_base, script_ext = os.path.splitext(script_path) temp_path = "/tmp/{}{}".format(self.generate_name(24), script_ext) env = self._env_vars(params) sudo = self.config.get('sudo', False) lock = self.config.get('lock', False) options = self._merge_options(self.config.get('options', {}), params, lock) args = ensure_list(self.config.get('args', [])) def exec_server(server): ssh = server.provider.ssh(env = env) ssh.upload(script_path, temp_path, mode = 755) try: self._ssh_exec(server, temp_path, self._interpolate(args, options), sudo = sudo, env = env, ssh = ssh ) finally: ssh.sudo('rm -f', temp_path) self.command.run_list( self._ssh_servers(params), exec_server )
def delete(self, key, **filters): if key not in data.ensure_list(self.keep(key)): filters[self.key()] = key return self.clear(**filters) else: raise RestrictedError("Removal of {} {} is restricted".format( self.model.__name__.lower(), key))
def accessor(self): if 'default_callback' in method_info: default_callback = getattr(self, method_info['default_callback'], None) if default_callback is None: raise CallbackNotExistsError( "Command parameter default callback {} does not exist". format(default_callback)) default_value = default_callback() else: default_value = method_info.get('default', None) value = self.options.get(method_base_name, default_value) if value is not None and method_info['parser'] == 'variables': value = ensure_list(value) if 'postprocessor' in method_info: postprocessor = getattr(self, method_info['postprocessor'], None) if postprocessor is None: raise CallbackNotExistsError( "Command parameter postprocessor {} does not exist".format( postprocessor)) if value is not None: value = postprocessor(value) return value
def _order_data(self, spec): dependencies = {} priorities = {} priority_map = {} if isinstance(spec, dict): for name, config in spec.items(): if config is not None and isinstance(config, dict): requires = ensure_list(config.get('requires', [])) dependencies[name] = requires for name, requires in dependencies.items(): priorities[name] = 0 for index in range(0, len(dependencies.keys())): for name in list(dependencies.keys()): for require in dependencies[name]: priorities[name] = max(priorities[name], priorities[require] + 1) for name, priority in priorities.items(): if priority not in priority_map: priority_map[priority] = [] priority_map[priority].append(name) return priority_map
def relation_fields(self): scope = [] if getattr(self.meta, 'relation', None): for field in data.ensure_list(self.meta.relation): if field not in self.scope_fields: scope.append(field) return scope
def __init__(self): self.default_modules = ensure_list(settings.DEFAULT_MODULES) self.remote_module_names = {} self.ordered_modules = None self.module_index = {} self.module_dependencies = {} super().__init__()
def get_merged_dataframe(field_info): merge_fields = data.ensure_list(dataframe_merge_fields) merge_filter_index = {} dataframe = None for merge_filters in list(facade.values(*merge_fields, **filters)): merge_values = get_merge_values(merge_filters) merge_filter_id = data.get_identifier(merge_values) if merge_filter_id not in merge_filter_index: sub_dataframe = get_dataframe(field_info, { **filters, **merge_filters }) value_prefix = "_".join(merge_values) sub_dataframe.columns = [ "{}_{}".format(value_prefix, column) for column in sub_dataframe.columns ] if dataframe is None: dataframe = sub_dataframe else: dataframe = dataframe.merge(sub_dataframe, how="outer", left_index=True, right_index=True) merge_filter_index[merge_filter_id] = True return dataframe
def execute(self, results, params): script_path = self.get_path(self.field_script) if not os.path.exists(script_path): self.command.error( "Remote script task provider file {} does not exist".format( script_path)) script_base, script_ext = os.path.splitext(script_path) temp_path = "/tmp/{}{}".format(self.generate_name(24), script_ext) env = self._env_vars(params) options = self._merge_options(self.field_options, params, self.field_lock) args = ensure_list(self.field_args, []) ssh = self._get_ssh(env) ssh.upload(script_path, temp_path, mode=755) try: self._ssh_exec(temp_path, self._interpolate(args, options), sudo=self.field_sudo, ssh=ssh) finally: ssh.exec('rm -f', temp_path)
def initialize_terraform(self, instance, created): instance.config['rule_type'] = 'cidr' instance.config['source_firewall_id'] = None if instance.config['source_firewall']: instance.config['rule_type'] = 'link' instance.config['self_only'] = False instance.cidrs = [] tries = 60 while True: firewall = self.command._firewall.retrieve(instance.config['source_firewall']) if firewall: firewall.initialize(self.command) instance.config['source_firewall_id'] = firewall.provider.get_firewall_id() break time.sleep(2) tries -= 2 if not tries: self.command.error("Source firewall {} could not be retrieved".format(instance.config['source_firewall'])) elif instance.config['self_only']: instance.config['rule_type'] = 'link' instance.cidrs = [] elif instance.cidrs: instance.cidrs = [str(self.address.parse_cidr(x.strip())) for x in ensure_list(instance.cidrs)] elif not instance.config['self_only'] and not instance.config['source_firewall']: instance.cidrs = ['0.0.0.0/0']
def exec(self, *elements): values = [] for element in elements: values.extend(ensure_list(element)) return values
def send_notification(self, recipient, subject, body): if settings.EMAIL_HOST and settings.EMAIL_HOST_USER: try: html_body = body.replace("\n", '<br/>') html_body = html_body.replace(" ", ' ') html_body = '<font face="Courier New, Courier, monospace">{}</font>'.format(html_body) send_mail( subject, body, settings.EMAIL_HOST_USER, ensure_list(recipient), html_message = html_body ) logger.info("Notification message '{}' sent to: {}".format(subject, recipient)) except SMTPConnectError as e: logger.error("Notification delivery failed: {}".format(e)) raise self.retry(exc = e) except SMTPServerDisconnected as e: logger.error("Notification service disconnected: {}".format(e)) raise self.retry(exc = e) except Exception as e: logger.error("Notification error: {}".format(e)) raise e
def exec_methods(instance, methods): for method, params in methods.items(): method = getattr(instance, method) if not params: method() elif isinstance(params, dict): method(**params) else: method(*ensure_list(params))
def get_wait_keys(_name): wait_keys = [] if _name in requirements and requirements[_name]: for _child_name in flatten(ensure_list(requirements[_name])): if processed[_child_name]: wait_keys.extend(processed[_child_name]) wait_keys.extend(get_wait_keys(_child_name)) return list(set(wait_keys))
def set_order(self, order): if order: self.order = [ re.sub(r'^~', '-', x) for x in data.ensure_list(order) ] else: self.order = None return self
def __call__(self, parser, namespace, values, option_string=None): arg_values = [] if not values: values = [] for value in ensure_list(values): arg_values.append(self.type(value)) setattr(namespace, self.dest, arg_values)
def init_parents(self): if 'base' not in self.spec: self.parents = [self.base_model] else: self.parents = [self.get_model(self.spec['base'], BaseModel)] if 'mixins' in self.spec: for mixin in ensure_list(self.spec['mixins']): self.parents.append(self.get_model(mixin, ModelMixin))
def GetCommand(parents, base_name, facade_name, view_roles=None, name_field=None): _parents = ensure_list(parents) _facade_name = get_facade(facade_name) _name_field = get_joined_value(name_field, base_name, 'name') def __get_priority(self): return 10 def __groups_allowed(self): from settings.roles import Roles return [Roles.admin] + ensure_list(view_roles) def __get_epilog(self): facade = getattr(self, _facade_name) variable = "{}_display_fields".format(facade.name) fields = [x.name for x in reversed(facade.meta.get_fields())] return 'field display config: {}\n\n> {} fields: {}'.format( self.header_color(variable), facade.name, self.notice_color(", ".join(fields))) def __parse(self): facade = getattr(self, _facade_name) if not name_field: getattr(self, "parse_{}".format(_name_field))() else: self.parse_scope(facade) self.parse_dependency(facade) parse_field_names(self) def __exec(self): facade = getattr(self, _facade_name) instance = getattr(self, base_name) self.table(self.render_display(facade, getattr(instance, facade.key()), allowed_fields=get_field_names(self)), row_labels=True) def __str__(self): return "Get <{}>".format(base_name) methods = { 'get_priority': __get_priority, 'get_epilog': __get_epilog, 'parse': __parse, 'exec': __exec, '__str__': __str__ } if view_roles: methods['groups_allowed'] = __groups_allowed return type('GetCommand', tuple(_parents), methods)
def parse_requirements(self): requirements = [] for path, config in self.index.get_ordered_modules().items(): if 'requirements' in config: for requirement_path in ensure_list(config['requirements']): requirement_path = os.path.join(path, requirement_path) file_contents = load_file(requirement_path) if file_contents: requirements.extend([ req for req in file_contents.split("\n") if req and req[0].strip() != '#' ]) return requirements
def add_fields(data): if data: for key, values in data.items(): for value in ensure_list(values): if isinstance(value, str): match = re.search(r'\@\{?([a-zA-Z0-9\_\-]+)\}?', value.strip()) if match: for field in match.groups(): fields[field] = True
def destroy(self, name, children): if children: if isinstance(children, dict): for child, grandchildren in children.items(): self.destroy(child, grandchildren) elif isinstance(children, (list, tuple, str)): for child in ensure_list(children): self.destroy(child, None) return self.exec('group remove', group_name=name, force=True)
def include_instance(self, name, config): if isinstance(config, dict): when = config.pop('when', None) when_not = config.pop('when_not', None) when_in = config.pop('when_in', None) when_not_in = config.pop('when_not_in', None) when_type = config.pop('when_type', 'AND').upper() if when is not None: result = True if when_type == 'AND' else False for variable in ensure_list(when): value = format_value( 'bool', self.command.options.interpolate(variable)) if when_type == 'AND': if not value: return False else: if value: result = True return result if when_not is not None: result = True if when_type == 'AND' else False for variable in ensure_list(when_not): value = format_value( 'bool', self.command.options.interpolate(variable)) if when_type == 'AND': if value: return False else: if not value: result = True return result if when_in is not None: value = self.command.options.interpolate(when_in) return name in ensure_list(value) if when_not_in is not None: value = self.command.options.interpolate(when_not_in) return name not in ensure_list(value) return True
def initialize_terraform(self, instance, created): super().initialize_terraform(instance, created) instance.healthy_status = ensure_list(instance.healthy_status)