def parse(self, value, config):
if not isinstance(value, str) or not value.startswith('?>'):
return value
if config.conditional_suppress:
config.conditional_suppress = re.compile(config.conditional_suppress)
conditional_match = re.search(self.conditional_pattern, value)
if conditional_match:
test_element = normalize_value(self.command.options.interpolate(conditional_match.group(1).strip(), **config.export()))
true_value = normalize_value(self.command.options.interpolate(conditional_match.group(2).strip(), **config.export()))
false_value = normalize_value(self.command.options.interpolate(conditional_match.group(3).strip(), **config.export()))
if config.conditional_suppress \
and ((isinstance(test_element, str) and config.conditional_suppress.search(test_element)) \
or (isinstance(true_value, str) and config.conditional_suppress.search(true_value)) \
or (isinstance(false_value, str) and config.conditional_suppress.search(false_value))):
value = "?> {} ? {} | {}".format(test_element, true_value, false_value)
else:
try:
test = eval(test_element) if isinstance(test_element, str) else test_element
except NameError:
test = test_element
if test:
try:
value = eval(true_value) if isinstance(true_value, str) else true_value
except NameError:
value = true_value
else:
try:
value = eval(false_value) if isinstance(false_value, str) else false_value
except NameError:
value = false_value
return value
def init_instance(object):
if isinstance(object, str):
cached = self._get_cache_instance(facade, object)
if not cached:
instance = facade.retrieve(object)
else:
instance = cached
else:
instance = object
cached = self._get_cache_instance(facade, getattr(instance, facade.pk))
if instance:
id = getattr(instance, facade.pk)
if not cached:
if instance.initialize(self):
self._set_cache_instance(facade, id, instance)
else:
instance = None
else:
instance = cached
if instance:
if fields:
for field, values in fields.items():
values = data.normalize_value(values)
value = getattr(instance, field, None)
if isinstance(values, str) and not value and re.match(r'^(none|null)$', values, re.IGNORECASE):
results[id] = instance
elif value and value in data.ensure_list(values):
results[id] = instance
else:
results[id] = instance
else:
self.error("{} instance {} does not exist".format(facade.name.title(), object))
def _load_package(self, package_name, template_fields, display_only):
template_fields = self._prepare_template_fields(
self._load_package_index(package_name, template_fields),
normalize_value(template_fields,
strip_quotes=True,
parse_json=True), display_only)
return self._load_package_index(package_name,
template_fields), template_fields
def load(cls, path, default=None):
if not default:
default = {}
data = default
if os.path.exists(path):
data = {}
for statement in load_file(path).split("\n"):
statement = statement.strip()
if statement and statement[0] != '#':
(variable, value) = statement.split("=")
data[variable] = normalize_value(value)
return data
def interpolate(data, variables):
if isinstance(data, dict):
generated = {}
for key, value in data.items():
key = interpolate(key, variables)
if key is not None:
generated[key] = interpolate(value, variables)
data = generated
elif isinstance(data, (list, tuple)):
generated = []
for value in data:
value = interpolate(value, variables)
if value is not None:
generated.append(value)
data = generated
elif isinstance(data, str):
parser = Template(data)
data = normalize_value(parser.substitute(**variables).strip())
data = None if not data else data
return data
def exec(self):
if not self.check_channel_permission():
self.error(
"You do not have permission to access the {} channel".format(
self.communication_channel))
connection = self.manager.task_connection()
if connection:
data = {
'user':
self.active_user.name,
'time':
Time().now_string,
'message':
normalize_value(self.communication_message, parse_json=True)
}
connection.publish(
channel_communication_key(self.communication_channel),
dump_json(data, indent=2))
self.success("Message sent to channel {}: {}".format(
self.communication_channel, self.communication_message))
def exec(self):
if not self.check_channel_permission():
self.error(
"You do not have permission to access the {} channel".format(
self.communication_channel))
connection = self.manager.task_connection()
if connection:
subscription = connection.pubsub(ignore_subscribe_messages=True)
try:
subscription.subscribe(
channel_communication_key(self.communication_channel))
start_time = time.time()
current_time = start_time
self.data('Listening for messages on channel',
self.communication_channel)
self.info('')
while not self.disconnected:
message = subscription.get_message()
if message:
if message['type'] == 'message':
self.data(
Time().now_string,
normalize_value(message['data'],
parse_json=True), 'message')
start_time = time.time()
self.sleep(0.25)
current_time = time.time()
if self.communication_timeout and (
(current_time - start_time) >
self.communication_timeout):
self.error(
"Listener timed out without any messages after {} seconds"
.format(self.communication_timeout))
finally:
subscription.close()
def _prepare_template_fields(self, index, template_fields, display_only):
processed_fields = OrderedDict()
for field, info in index.variables.items():
if info.get('required', False) and field not in template_fields:
if display_only:
template_fields[field] = "<{{{}}}>".format(field)
else:
raise TemplateException(
"Field {} is required for template {}".format(
field, index.name))
if field in template_fields:
processed_fields[field] = template_fields[field]
elif info.get('default', None) is not None:
processed_fields[field] = normalize_value(info['default'],
strip_quotes=True,
parse_json=True)
else:
processed_fields[field] = None
return processed_fields
def parse_annotations():
annotations = {}
for index, field in enumerate(fields):
field_components = field.split(processor_separator)
processor = field_components[1] if len(
field_components) > 1 else None
field_components = field_components[0].split(':')
if len(field_components) > 1:
try:
field_name = field_components[0]
aggregator = field_components[1]
expression = field_components[2]
aggregator_options = {}
if len(field_components) == 4:
for assignment in field_components[3].split(';'):
name, value = assignment.split('=')
aggregator_options[
name] = data.normalize_value(value, True)
annotations[field_name] = [
aggregator, expression, aggregator_options
]
fields[index] = "{}{}{}".format(
field_name, processor_separator,
processor) if processor else field_name
aggregates.append(field_name)
except Exception as e:
self.error(
"When passing aggregators as fields to get_data_set format must be field_name:GROUP_FUNC:expression[:option=value[,...]]"
)
return annotations
def collect_fields():
expanded_fields = OrderedDict()
for field in fields:
field_components = field.split(processor_separator)
if len(field_components) > 1:
field = field_components[0]
processor_components = field_components[1].split(':')
processor_info = {
'processor': processor_components[0],
'options': {}
}
if len(processor_components) > 1:
processor_info['field'] = processor_components[1]
if processor_info['field'] not in expanded_fields:
expanded_fields[processor_info['field']] = {}
removals.append(processor_info['field'])
if len(processor_components) == 3:
processor_options = {}
for assignment in processor_components[2].split(
';'):
name, value = assignment.split('=')
processor_options[name] = data.normalize_value(
value, True)
processor_info['options'] = processor_options
else:
processor_info['field'] = field
expanded_fields[field] = processor_info
else:
expanded_fields[field] = {}
return expanded_fields
def load(self, data):
super().load(data)
self.message = normalize_value(self.message,
strip_quotes=True,
parse_json=False)
def start_service(self,
name,
image,
ports=None,
entrypoint=None,
command=None,
environment={},
volumes={},
memory='250m',
wait=30,
**options):
if not self.client:
return
data = self.get_service(name, create=False)
if data and self._service_container(data['id']):
return data['id']
self.print("{} {}".format(
self.notice_color('Launching Zimagi service'),
self.key_color(name)))
options = normalize_value(options)
container_name = self._normalize_name(name)
network = self._get_network("{}-{}".format(self.app_name,
self.env.name))
dns_map = {}
for spec_name, spec in self.get_spec('services').items():
dns_map[self._normalize_name(spec_name)] = spec_name
volume_info = {}
for local_path, remote_config in volumes.items():
if local_path[0] != '/':
local_path = self._normalize_name(local_path)
self._get_volume(local_path)
volume_info[local_path] = remote_config
options.pop('requires', None)
if options.get('runtime', '') == 'standard':
options.pop('runtime')
service = self._service_container(container_name)
if service:
service.remove(force=True)
service = self.client.containers.run(image,
entrypoint=entrypoint,
command=command,
name=container_name,
hostname=name,
links=dns_map,
detach=True,
mem_limit=memory,
network=network.name,
ports=ports,
volumes=volume_info,
environment=environment,
**options)
success, service = self._check_service(name, service)
self._save_service(name, service.id, {
'image': image,
'volumes': volumes,
'success': success
})
if not success:
self._service_error(name, service)
return service.id
def load(self, data):
super().load(data)
self.data = normalize_value(self.data,
strip_quotes=True,
parse_json=True)
def _store_template_map(self, module, index, template_fields,
display_only):
self.notice('Template variables:')
self.table([['Variable', 'Value', 'Help']] +
[[key, value, index.variables[key].get('help', 'NA')]
for key, value in template_fields.items()], 'variables')
self.info('')
for path, info in index.map.items():
target = None
if isinstance(info, str):
target = info
info = {}
elif info.get('when', True) and 'target' in info:
target = info['target']
if target:
path_components = os.path.split(
self.manager.get_module_path(module, target))
target_path = os.path.join(*path_components)
if info.get('template', True):
file_content = self._render_package_template(
index.name, path, template_fields)
else:
file_content = load_file(
self.manager.get_template_path(index.name, path))
if info.get('location', None) and path.endswith('.yml'):
file_data = normalize_value(load_yaml(target_path),
strip_quotes=True,
parse_json=True)
if not file_data:
file_data = {}
location = info['location'].split('.')
embed_data = normalize_value(oyaml.safe_load(file_content),
strip_quotes=True,
parse_json=True)
merge_data = {}
iter_data = merge_data
for index, key in enumerate(location):
if (index + 1) == len(location):
iter_data[key] = embed_data
else:
iter_data[key] = {}
iter_data = iter_data[key]
file_content = oyaml.dump(deep_merge(
file_data, merge_data))
self.data('Path', path, 'path')
self.data('Target', target, 'target')
if info.get('location', None):
self.data('location', info['location'], 'location')
self.notice('-' * self.display_width)
if info.get('template', True):
self.info(file_content)
self.info('')
if not display_only:
create_dir(path_components[0])
save_file(target_path, file_content)
def interpolate_config_value(self, value, **options):
options['config_overrides'] = self.config.export()
return normalize_value(
self.command.options.interpolate(value, **options))
def search_instances(self,
facade,
queries=None,
joiner='AND',
error_on_empty=True):
if not queries:
queries = []
valid_fields = facade.query_fields
queries = data.ensure_list(queries)
joiner = joiner.upper()
results = {}
def perform_query(filters, excludes, fields):
instances = facade.query(**filters).exclude(**excludes)
if len(instances) > 0:
for instance in self.get_instances(facade,
objects=list(instances),
fields=fields):
results[getattr(instance, facade.pk)] = instance
if queries:
filters = {}
excludes = {}
extra = {}
for query in queries:
matches = re.search(
r'^([\~\-])?([^\s\=]+)\s*(?:(\=|[^\s]*))\s*(.*)', query)
if matches:
negate = True if matches.group(1) else False
field = matches.group(2).strip()
field_list = re.split(r'\.|__', field)
lookup = matches.group(3)
if not lookup and len(field_list) > 1:
lookup = field_list.pop()
value = re.sub(r'^[\'\"]|[\'\"]$', '',
matches.group(4).strip())
if not lookup and not value:
value = field
lookup = '='
field_list[0] = facade.key()
base_field = field_list[0]
field_path = "__".join(field_list)
if lookup != '=':
field_path = "{}__{}".format(field_path, lookup)
value = data.normalize_value(value,
strip_quotes=False,
parse_json=True)
if joiner.upper() == 'OR':
filters = {}
excludes = {}
extra = {}
if base_field in valid_fields:
if negate:
excludes[field_path] = value
else:
filters[field_path] = value
else:
extra[field_path] = value
if joiner == 'OR':
perform_query(filters, excludes, extra)
else:
self.error(
"Search filter must be of the format: field[.subfield][.lookup] [=] value"
.format(query))
if joiner == 'AND':
perform_query(filters, excludes, extra)
else:
for instance in self.get_instances(facade):
results[getattr(instance, facade.pk)] = instance
if error_on_empty and not results:
if queries:
self.warning("No {} instances were found: {}".format(
facade.name, ", ".join(queries)))
else:
self.warning("No {} instances were found".format(facade.name))
return results.values()
def exec_function(self, value, config):
function_match = re.search(self.function_pattern, value)
exec_function = True
if config.function_suppress:
config.function_suppress = re.compile(config.function_suppress)
if function_match:
function_variable = function_match.group(1)
function_name = function_match.group(2)
function_parameters = []
function_options = {}
if function_match.group(3):
for parameter in re.split(r'\s*\,\s*', function_match.group(3)):
parameter = parameter.strip()
option_components = parameter.split('=')
if len(option_components) == 2:
option_name = option_components[0].strip()
value = option_components[1].strip().lstrip("\'\"").rstrip("\'\"")
if config.function_suppress and config.function_suppress.match(value):
exec_function = False
else:
value = self.command.options.interpolate(value, **config.export())
function_options[option_name] = normalize_value(value, strip_quotes = False, parse_json = True)
else:
parameter = parameter.lstrip("\'\"").rstrip("\'\"")
if config.function_suppress and config.function_suppress.match(parameter):
exec_function = False
else:
parameter = self.command.options.interpolate(parameter, **config.export())
function_parameters.append(normalize_value(parameter, strip_quotes = False, parse_json = True))
if exec_function:
function = self.command.get_provider('function', function_name)
result = function.exec(*function_parameters, **function_options)
if function_variable:
self.command.options.get_parser('config').set(function_variable, result)
return result
else:
if function_options:
parameter_str = ''
if function_parameters:
parameter_str = "{}, ".format(", ".join(function_parameters))
option_str = []
for name, value in function_options.items():
option_str.append("{} = {}".format(name, value))
option_str = ", ".join(option_str)
return "#{}({}{})".format(function_name, parameter_str, option_str)
return "#{}({})".format(function_name, ", ".join(function_parameters))
# Not found, assume desired
return value
def exec(self, data, default):
value = normalize_value(data)
return default if value is None else value
def parse_reference(self, value):
ref_match = re.search(self.reference_pattern, value)
reference_variable = ref_match.group(1)
operations = ref_match.group(2)
if operations:
operations = operations.strip()
else:
operations = ''
facade = self.command.facade(ref_match.group(3), False)
scopes = ref_match.group(4)
scope_filters = {}
if scopes:
for scope_filter in scopes.replace(' ', '').split(';'):
scope_field, scope_value = scope_filter.split('=')
scope_value = scope_value.replace(' ', '')
if ',' in scope_value:
scope_value = scope_value.split(',')
scope_filters[scope_field] = normalize_value(scope_value)
facade.set_scope(scope_filters)
names = ref_match.group(5)
if names:
search_names = names.replace(' ', '').split(',')
names = []
for name in search_names:
if name == '*':
names.extend(set(facade.keys()))
if name[-1] == '*':
keys = facade.keys(
**{"{}__startswith".format(facade.key()): name[:-1]})
names.extend(set(keys))
elif name[0] == '*':
keys = facade.keys(
**{"{}__endswith".format(facade.key()): name[1:]})
names.extend(set(keys))
else:
names.append(name)
fields = re.split(r'\s*,\s*', ref_match.group(6))
keys = ref_match.group(7)
if keys and len(fields) == 1:
keys = keys.replace(' ', '').split('][')
def _get_instance_values():
values = []
filters = {}
if names:
filters["{}__in".format(facade.key())] = names
query = facade.values(*fields, **filters)
if '!' in operations:
query = query.distinct()
for data in list(query):
if keys:
for field in fields:
instance_value = data.get(field, None)
if instance_value is not None:
if isinstance(instance_value, (dict, list, tuple)):
def _get_value(_data, key_list):
if isinstance(
_data,
(dict, list, tuple)) and len(key_list):
base_key = normalize_index(
key_list.pop(0))
try:
return _get_value(
_data[base_key], key_list)
except Exception:
return None
return _data
data[field] = _get_value(instance_value, keys)
if len(fields) == 1:
data = data[fields[0]]
values.append(data)
return values
instance_values = _get_instance_values()
if '!' in operations and len(fields) == 1:
instance_values = list(set(instance_values))
if names and len(names) == 1:
if len(instance_values) == 1:
instance_values = instance_values[0]
elif not instance_values:
instance_values = None
if reference_variable:
self.command.options.get_parser('config').set(
reference_variable, instance_values)
return instance_values
def exec(self, data):
return normalize_value(data)