def parse(self, value, config): if not isinstance(value, str) or not value.startswith('?>'): return value if config.conditional_suppress: config.conditional_suppress = re.compile(config.conditional_suppress) conditional_match = re.search(self.conditional_pattern, value) if conditional_match: test_element = normalize_value(self.command.options.interpolate(conditional_match.group(1).strip(), **config.export())) true_value = normalize_value(self.command.options.interpolate(conditional_match.group(2).strip(), **config.export())) false_value = normalize_value(self.command.options.interpolate(conditional_match.group(3).strip(), **config.export())) if config.conditional_suppress \ and ((isinstance(test_element, str) and config.conditional_suppress.search(test_element)) \ or (isinstance(true_value, str) and config.conditional_suppress.search(true_value)) \ or (isinstance(false_value, str) and config.conditional_suppress.search(false_value))): value = "?> {} ? {} | {}".format(test_element, true_value, false_value) else: try: test = eval(test_element) if isinstance(test_element, str) else test_element except NameError: test = test_element if test: try: value = eval(true_value) if isinstance(true_value, str) else true_value except NameError: value = true_value else: try: value = eval(false_value) if isinstance(false_value, str) else false_value except NameError: value = false_value return value
def init_instance(object): if isinstance(object, str): cached = self._get_cache_instance(facade, object) if not cached: instance = facade.retrieve(object) else: instance = cached else: instance = object cached = self._get_cache_instance(facade, getattr(instance, facade.pk)) if instance: id = getattr(instance, facade.pk) if not cached: if instance.initialize(self): self._set_cache_instance(facade, id, instance) else: instance = None else: instance = cached if instance: if fields: for field, values in fields.items(): values = data.normalize_value(values) value = getattr(instance, field, None) if isinstance(values, str) and not value and re.match(r'^(none|null)$', values, re.IGNORECASE): results[id] = instance elif value and value in data.ensure_list(values): results[id] = instance else: results[id] = instance else: self.error("{} instance {} does not exist".format(facade.name.title(), object))
def _load_package(self, package_name, template_fields, display_only): template_fields = self._prepare_template_fields( self._load_package_index(package_name, template_fields), normalize_value(template_fields, strip_quotes=True, parse_json=True), display_only) return self._load_package_index(package_name, template_fields), template_fields
def load(cls, path, default=None): if not default: default = {} data = default if os.path.exists(path): data = {} for statement in load_file(path).split("\n"): statement = statement.strip() if statement and statement[0] != '#': (variable, value) = statement.split("=") data[variable] = normalize_value(value) return data
def interpolate(data, variables): if isinstance(data, dict): generated = {} for key, value in data.items(): key = interpolate(key, variables) if key is not None: generated[key] = interpolate(value, variables) data = generated elif isinstance(data, (list, tuple)): generated = [] for value in data: value = interpolate(value, variables) if value is not None: generated.append(value) data = generated elif isinstance(data, str): parser = Template(data) data = normalize_value(parser.substitute(**variables).strip()) data = None if not data else data return data
def exec(self): if not self.check_channel_permission(): self.error( "You do not have permission to access the {} channel".format( self.communication_channel)) connection = self.manager.task_connection() if connection: data = { 'user': self.active_user.name, 'time': Time().now_string, 'message': normalize_value(self.communication_message, parse_json=True) } connection.publish( channel_communication_key(self.communication_channel), dump_json(data, indent=2)) self.success("Message sent to channel {}: {}".format( self.communication_channel, self.communication_message))
def exec(self): if not self.check_channel_permission(): self.error( "You do not have permission to access the {} channel".format( self.communication_channel)) connection = self.manager.task_connection() if connection: subscription = connection.pubsub(ignore_subscribe_messages=True) try: subscription.subscribe( channel_communication_key(self.communication_channel)) start_time = time.time() current_time = start_time self.data('Listening for messages on channel', self.communication_channel) self.info('') while not self.disconnected: message = subscription.get_message() if message: if message['type'] == 'message': self.data( Time().now_string, normalize_value(message['data'], parse_json=True), 'message') start_time = time.time() self.sleep(0.25) current_time = time.time() if self.communication_timeout and ( (current_time - start_time) > self.communication_timeout): self.error( "Listener timed out without any messages after {} seconds" .format(self.communication_timeout)) finally: subscription.close()
def _prepare_template_fields(self, index, template_fields, display_only): processed_fields = OrderedDict() for field, info in index.variables.items(): if info.get('required', False) and field not in template_fields: if display_only: template_fields[field] = "<{{{}}}>".format(field) else: raise TemplateException( "Field {} is required for template {}".format( field, index.name)) if field in template_fields: processed_fields[field] = template_fields[field] elif info.get('default', None) is not None: processed_fields[field] = normalize_value(info['default'], strip_quotes=True, parse_json=True) else: processed_fields[field] = None return processed_fields
def parse_annotations(): annotations = {} for index, field in enumerate(fields): field_components = field.split(processor_separator) processor = field_components[1] if len( field_components) > 1 else None field_components = field_components[0].split(':') if len(field_components) > 1: try: field_name = field_components[0] aggregator = field_components[1] expression = field_components[2] aggregator_options = {} if len(field_components) == 4: for assignment in field_components[3].split(';'): name, value = assignment.split('=') aggregator_options[ name] = data.normalize_value(value, True) annotations[field_name] = [ aggregator, expression, aggregator_options ] fields[index] = "{}{}{}".format( field_name, processor_separator, processor) if processor else field_name aggregates.append(field_name) except Exception as e: self.error( "When passing aggregators as fields to get_data_set format must be field_name:GROUP_FUNC:expression[:option=value[,...]]" ) return annotations
def collect_fields(): expanded_fields = OrderedDict() for field in fields: field_components = field.split(processor_separator) if len(field_components) > 1: field = field_components[0] processor_components = field_components[1].split(':') processor_info = { 'processor': processor_components[0], 'options': {} } if len(processor_components) > 1: processor_info['field'] = processor_components[1] if processor_info['field'] not in expanded_fields: expanded_fields[processor_info['field']] = {} removals.append(processor_info['field']) if len(processor_components) == 3: processor_options = {} for assignment in processor_components[2].split( ';'): name, value = assignment.split('=') processor_options[name] = data.normalize_value( value, True) processor_info['options'] = processor_options else: processor_info['field'] = field expanded_fields[field] = processor_info else: expanded_fields[field] = {} return expanded_fields
def load(self, data): super().load(data) self.message = normalize_value(self.message, strip_quotes=True, parse_json=False)
def start_service(self, name, image, ports=None, entrypoint=None, command=None, environment={}, volumes={}, memory='250m', wait=30, **options): if not self.client: return data = self.get_service(name, create=False) if data and self._service_container(data['id']): return data['id'] self.print("{} {}".format( self.notice_color('Launching Zimagi service'), self.key_color(name))) options = normalize_value(options) container_name = self._normalize_name(name) network = self._get_network("{}-{}".format(self.app_name, self.env.name)) dns_map = {} for spec_name, spec in self.get_spec('services').items(): dns_map[self._normalize_name(spec_name)] = spec_name volume_info = {} for local_path, remote_config in volumes.items(): if local_path[0] != '/': local_path = self._normalize_name(local_path) self._get_volume(local_path) volume_info[local_path] = remote_config options.pop('requires', None) if options.get('runtime', '') == 'standard': options.pop('runtime') service = self._service_container(container_name) if service: service.remove(force=True) service = self.client.containers.run(image, entrypoint=entrypoint, command=command, name=container_name, hostname=name, links=dns_map, detach=True, mem_limit=memory, network=network.name, ports=ports, volumes=volume_info, environment=environment, **options) success, service = self._check_service(name, service) self._save_service(name, service.id, { 'image': image, 'volumes': volumes, 'success': success }) if not success: self._service_error(name, service) return service.id
def load(self, data): super().load(data) self.data = normalize_value(self.data, strip_quotes=True, parse_json=True)
def _store_template_map(self, module, index, template_fields, display_only): self.notice('Template variables:') self.table([['Variable', 'Value', 'Help']] + [[key, value, index.variables[key].get('help', 'NA')] for key, value in template_fields.items()], 'variables') self.info('') for path, info in index.map.items(): target = None if isinstance(info, str): target = info info = {} elif info.get('when', True) and 'target' in info: target = info['target'] if target: path_components = os.path.split( self.manager.get_module_path(module, target)) target_path = os.path.join(*path_components) if info.get('template', True): file_content = self._render_package_template( index.name, path, template_fields) else: file_content = load_file( self.manager.get_template_path(index.name, path)) if info.get('location', None) and path.endswith('.yml'): file_data = normalize_value(load_yaml(target_path), strip_quotes=True, parse_json=True) if not file_data: file_data = {} location = info['location'].split('.') embed_data = normalize_value(oyaml.safe_load(file_content), strip_quotes=True, parse_json=True) merge_data = {} iter_data = merge_data for index, key in enumerate(location): if (index + 1) == len(location): iter_data[key] = embed_data else: iter_data[key] = {} iter_data = iter_data[key] file_content = oyaml.dump(deep_merge( file_data, merge_data)) self.data('Path', path, 'path') self.data('Target', target, 'target') if info.get('location', None): self.data('location', info['location'], 'location') self.notice('-' * self.display_width) if info.get('template', True): self.info(file_content) self.info('') if not display_only: create_dir(path_components[0]) save_file(target_path, file_content)
def interpolate_config_value(self, value, **options): options['config_overrides'] = self.config.export() return normalize_value( self.command.options.interpolate(value, **options))
def search_instances(self, facade, queries=None, joiner='AND', error_on_empty=True): if not queries: queries = [] valid_fields = facade.query_fields queries = data.ensure_list(queries) joiner = joiner.upper() results = {} def perform_query(filters, excludes, fields): instances = facade.query(**filters).exclude(**excludes) if len(instances) > 0: for instance in self.get_instances(facade, objects=list(instances), fields=fields): results[getattr(instance, facade.pk)] = instance if queries: filters = {} excludes = {} extra = {} for query in queries: matches = re.search( r'^([\~\-])?([^\s\=]+)\s*(?:(\=|[^\s]*))\s*(.*)', query) if matches: negate = True if matches.group(1) else False field = matches.group(2).strip() field_list = re.split(r'\.|__', field) lookup = matches.group(3) if not lookup and len(field_list) > 1: lookup = field_list.pop() value = re.sub(r'^[\'\"]|[\'\"]$', '', matches.group(4).strip()) if not lookup and not value: value = field lookup = '=' field_list[0] = facade.key() base_field = field_list[0] field_path = "__".join(field_list) if lookup != '=': field_path = "{}__{}".format(field_path, lookup) value = data.normalize_value(value, strip_quotes=False, parse_json=True) if joiner.upper() == 'OR': filters = {} excludes = {} extra = {} if base_field in valid_fields: if negate: excludes[field_path] = value else: filters[field_path] = value else: extra[field_path] = value if joiner == 'OR': perform_query(filters, excludes, extra) else: self.error( "Search filter must be of the format: field[.subfield][.lookup] [=] value" .format(query)) if joiner == 'AND': perform_query(filters, excludes, extra) else: for instance in self.get_instances(facade): results[getattr(instance, facade.pk)] = instance if error_on_empty and not results: if queries: self.warning("No {} instances were found: {}".format( facade.name, ", ".join(queries))) else: self.warning("No {} instances were found".format(facade.name)) return results.values()
def exec_function(self, value, config): function_match = re.search(self.function_pattern, value) exec_function = True if config.function_suppress: config.function_suppress = re.compile(config.function_suppress) if function_match: function_variable = function_match.group(1) function_name = function_match.group(2) function_parameters = [] function_options = {} if function_match.group(3): for parameter in re.split(r'\s*\,\s*', function_match.group(3)): parameter = parameter.strip() option_components = parameter.split('=') if len(option_components) == 2: option_name = option_components[0].strip() value = option_components[1].strip().lstrip("\'\"").rstrip("\'\"") if config.function_suppress and config.function_suppress.match(value): exec_function = False else: value = self.command.options.interpolate(value, **config.export()) function_options[option_name] = normalize_value(value, strip_quotes = False, parse_json = True) else: parameter = parameter.lstrip("\'\"").rstrip("\'\"") if config.function_suppress and config.function_suppress.match(parameter): exec_function = False else: parameter = self.command.options.interpolate(parameter, **config.export()) function_parameters.append(normalize_value(parameter, strip_quotes = False, parse_json = True)) if exec_function: function = self.command.get_provider('function', function_name) result = function.exec(*function_parameters, **function_options) if function_variable: self.command.options.get_parser('config').set(function_variable, result) return result else: if function_options: parameter_str = '' if function_parameters: parameter_str = "{}, ".format(", ".join(function_parameters)) option_str = [] for name, value in function_options.items(): option_str.append("{} = {}".format(name, value)) option_str = ", ".join(option_str) return "#{}({}{})".format(function_name, parameter_str, option_str) return "#{}({})".format(function_name, ", ".join(function_parameters)) # Not found, assume desired return value
def exec(self, data, default): value = normalize_value(data) return default if value is None else value
def parse_reference(self, value): ref_match = re.search(self.reference_pattern, value) reference_variable = ref_match.group(1) operations = ref_match.group(2) if operations: operations = operations.strip() else: operations = '' facade = self.command.facade(ref_match.group(3), False) scopes = ref_match.group(4) scope_filters = {} if scopes: for scope_filter in scopes.replace(' ', '').split(';'): scope_field, scope_value = scope_filter.split('=') scope_value = scope_value.replace(' ', '') if ',' in scope_value: scope_value = scope_value.split(',') scope_filters[scope_field] = normalize_value(scope_value) facade.set_scope(scope_filters) names = ref_match.group(5) if names: search_names = names.replace(' ', '').split(',') names = [] for name in search_names: if name == '*': names.extend(set(facade.keys())) if name[-1] == '*': keys = facade.keys( **{"{}__startswith".format(facade.key()): name[:-1]}) names.extend(set(keys)) elif name[0] == '*': keys = facade.keys( **{"{}__endswith".format(facade.key()): name[1:]}) names.extend(set(keys)) else: names.append(name) fields = re.split(r'\s*,\s*', ref_match.group(6)) keys = ref_match.group(7) if keys and len(fields) == 1: keys = keys.replace(' ', '').split('][') def _get_instance_values(): values = [] filters = {} if names: filters["{}__in".format(facade.key())] = names query = facade.values(*fields, **filters) if '!' in operations: query = query.distinct() for data in list(query): if keys: for field in fields: instance_value = data.get(field, None) if instance_value is not None: if isinstance(instance_value, (dict, list, tuple)): def _get_value(_data, key_list): if isinstance( _data, (dict, list, tuple)) and len(key_list): base_key = normalize_index( key_list.pop(0)) try: return _get_value( _data[base_key], key_list) except Exception: return None return _data data[field] = _get_value(instance_value, keys) if len(fields) == 1: data = data[fields[0]] values.append(data) return values instance_values = _get_instance_values() if '!' in operations and len(fields) == 1: instance_values = list(set(instance_values)) if names and len(names) == 1: if len(instance_values) == 1: instance_values = instance_values[0] elif not instance_values: instance_values = None if reference_variable: self.command.options.get_parser('config').set( reference_variable, instance_values) return instance_values
def exec(self, data): return normalize_value(data)