class Provider(BaseProvider('validator', 'number')):
def validate(self, value, record):
try:
value = number(value)
except (ValueError, TypeError):
self.warning("Value {} is not a number".format(value))
return False
if not self.field_nan and math.isnan(value):
self.warning("Value can not be NaN")
return False
if self.field_min is not None:
if value < self.field_min:
self.warning("Value {} is below minimum allowed: {}".format(
value, self.field_min))
return False
if self.field_max is not None:
if value > self.field_max:
self.warning("Value {} is above maximum allowed: {}".format(
value, self.field_max))
return False
return True
class Provider(BaseProvider('formatter', 'name_joiner')):
def format(self, value):
date = self.format_value(value[1],
self.field_date_formatter_provider,
format=self.field_date_format)
return self.field_join.join(
[str(int(value[0])), date.strftime("%Y-%m-%d")])
class Provider(BaseProvider('parser', 'state')):
variable_pattern = r'^\$\{?([a-zA-Z][\_\-a-zA-Z0-9]+)(?:\[([^\]]+)\])?\}?$'
variable_value_pattern = r'(?<!\$)\$\>?\{?([a-zA-Z][\_\-a-zA-Z0-9]+(?:\[[^\]]+\])?)\}?'
runtime_variables = {}
def __init__(self, type, name, command, config):
super().__init__(type, name, command, config)
self.variables = {}
def initialize(self, reset=False):
if reset or not self.variables:
self.variables = {}
for state in self.command.get_instances(self.command._state):
self.variables[state.name] = state.value
def parse(self, value):
if not isinstance(value, str):
return value
if re.search(self.variable_pattern, value):
value = self.parse_variable(value)
else:
for ref_match in re.finditer(self.variable_value_pattern, value):
variable_value = self.parse_variable("${}".format(
ref_match.group(1)))
if isinstance(variable_value, (list, tuple)):
variable_value = ",".join(variable_value)
elif isinstance(variable_value, dict):
variable_value = json.dumps(variable_value)
if variable_value:
value = value.replace(ref_match.group(0),
str(variable_value)).strip()
return value
def parse_variable(self, value):
state_match = re.search(self.variable_pattern, value)
if state_match:
variables = {**self.runtime_variables, **self.variables}
new_value = state_match.group(1)
key = state_match.group(2)
if new_value in variables:
data = variables[new_value]
if isinstance(data, dict) and key:
try:
return data[key]
except KeyError:
return value
elif isinstance(data, (list, tuple)) and key:
try:
return data[int(key)]
except KeyError:
return value
else:
return data
# Not found, assume desired
return value
class Provider(BaseProvider('parser', 'token')):
token_pattern = r'^\%([\%\!])([a-zA-Z][\_\-a-zA-Z0-9]+)(?:\:(\d+))?$'
def parse(self, value):
if not isinstance(value, str):
return value
ref_match = re.search(self.token_pattern, value)
if ref_match:
operation = ref_match.group(1)
length = ref_match.group(3)
if length:
length = int(length)
else:
length = 20
state_name = "token-{}-{}".format(ref_match.group(2), length)
if operation == '%':
value = self.command.get_state(state_name)
else:
value = None
if not value:
value = create_token(length)
self.command.set_state(state_name, value)
return value
class Provider(BaseProvider('task', 'remote_script')):
def execute(self, results, params):
script_path = self.get_path(self.field_script)
if not os.path.exists(script_path):
self.command.error(
"Remote script task provider file {} does not exist".format(
script_path))
script_base, script_ext = os.path.splitext(script_path)
temp_path = "/tmp/{}{}".format(self.generate_name(24), script_ext)
env = self._env_vars(params)
options = self._merge_options(self.field_options, params,
self.field_lock)
args = ensure_list(self.field_args, [])
ssh = self._get_ssh(env)
ssh.upload(script_path, temp_path, mode=755)
try:
self._ssh_exec(temp_path,
self._interpolate(args, options),
sudo=self.field_sudo,
ssh=ssh)
finally:
ssh.exec('rm -f', temp_path)
class Provider(BaseProvider('validator', 'exists')):
def validate(self, value, record):
if value is None:
self.warning("Value can not be nothing to check for existence")
return False
facade = self.command.facade(self.field_data, False)
filters = {}
scope_text = ''
if self.field_scope:
scope = {}
for field in self.field_scope:
scope[field] = record[field]
facade.set_scope(scope)
scope_text = "within scope {}".format(scope)
field = self.field_field if self.field_field else facade.key()
filters[field] = value
if not facade.keys(**filters):
self.warning("Model {} {}: {} does not exist {}".format(self.field_data, field, value, scope_text))
return False
return True
class Provider(BaseProvider('encryption', 'aes256')):
@classmethod
def generate_key(self):
return hashlib.sha256(super().generate_key().encode()).hexdigest()[::2]
def initialize(self, options):
options['key'] = hashlib.sha256(
options['key'].encode()).hexdigest()[::2]
def encrypt_text(self, plain_text):
iv = Random.new().read(AES.block_size)
iv_int = int(binascii.hexlify(iv), AES.block_size)
ctr = Counter.new(AES.block_size * 8, initial_value=iv_int)
cipher = AES.new(self.field_key.encode(), AES.MODE_CTR, counter=ctr)
return base64.b64encode(iv + cipher.encrypt(plain_text.encode()))
def decrypt_text(self, cipher_text):
cipher_text = base64.b64decode(cipher_text)
iv = cipher_text[:AES.block_size]
cipher_text = cipher_text[AES.block_size:]
iv_int = int(iv.hex(), AES.block_size)
ctr = Counter.new(AES.block_size * 8, initial_value=iv_int)
cipher = AES.new(self.field_key.encode(), AES.MODE_CTR, counter=ctr)
return cipher.decrypt(cipher_text)
class Provider(BaseProvider('task', 'remote_command')):
def execute(self, results, params):
env = self._env_vars(params)
options = self._merge_options(self.field_options, params,
self.field_lock)
command = self._interpolate(self.field_command, options)
self._ssh_exec(command, env=env, sudo=self.field_sudo)
class Provider(BaseProvider('function', 'random_keys')):
def exec(self, dict_value, limit=None):
keys = list(dict_value.keys())
random.shuffle(keys)
if limit:
return keys[:min(int(limit), len(keys))]
return keys
class SubnetProvider(BaseProvider('network.subnet', 'aws')):
def initialize_terraform(self, instance, created):
if instance.config['zone'] is None and instance.config[
'zone_suffix'] is not None:
instance.config['zone'] = "{}{}".format(
instance.network.config['region'],
instance.config['zone_suffix'])
super().initialize_terraform(instance, created)
class Provider(BaseProvider('function', 'join')):
def exec(self, *elements):
values = []
for element in elements:
values.extend(ensure_list(element))
return values
class Provider(BaseProvider('function', 'random_values')):
def exec(self, list_value, limit = None):
values = ensure_list(list_value)
random.shuffle(values)
if limit:
return values[:min(int(limit), len(values))]
return values
class Provider(BaseProvider('source', 'csv_file')):
def load(self):
return self.load_csv_data_from_file(
self.field_file,
self.import_columns,
archive_file=self.field_archive_file,
separator=self.field_separator,
data_type=self.field_data_type,
header=self.field_header)
class Provider(BaseProvider('encryption', 'aes256_user')):
def initialize(self, options):
if options.get('user', None):
user_facade = self.manager.index.get_facade_index()['user']
user = user_facade.retrieve(options['user'])
if not user:
raise EncryptionError("User {} does not exist".format(options['user']))
options['key'] = user.encryption_key
class Provider(BaseProvider('field_processor', 'combined_text')):
def exec(self, dataset, field_data, append = None, separator = ","):
if append:
for field in append.split(','):
field_data = field_data.str.cat(
dataset[field].astype(str),
sep = separator
)
return field_data
class Provider(BaseProvider('function', 'keys')):
def exec(self, data, prefix = None):
keys = list(data.keys())
if prefix is None:
return keys
for index, key in enumerate(keys):
keys[index] = "{}{}".format(prefix, key)
return keys
class Provider(BaseProvider('formatter', 'date_time')):
def format(self, value, record):
if isinstance(value, float):
value = int(value)
try:
value = datetime.datetime.strptime(str(value), self.field_format)
except ValueError as e:
self.error("Value {} is not a valid date time according to pattern: {}".format(value, self.field_format))
return value
class Provider(BaseProvider('formatter', 'number')):
def format(self, value, record):
try:
if isinstance(value, str):
value = float(value)
except Exception:
return None
if value is None or math.isnan(value):
return None
return number(value)
class Provider(BaseProvider('validator', 'date_time')):
def validate(self, value):
if isinstance(value, float):
value = int(value)
try:
datetime.datetime.strptime(str(value), self.field_format)
except ValueError as e:
self.warning("Value {} is not a valid date time according to pattern: {}".format(value, self.field_format))
return False
return True
class Provider(BaseProvider('worker', 'docker')):
def start_worker(self):
self.command.manager.stop_service('worker', remove = True)
service_spec = self.command.manager.get_service_spec('worker')
service_spec['environment']['ZIMAGI_WORKER_TYPE'] = self.field_worker_type
self.command.manager.start_service('worker', **service_spec)
def check_worker(self):
return self.command.manager.get_service('worker', restart = False, create = False)
class Provider(BaseProvider('function', 'value')):
def exec(self, data, keys, default=None):
if isinstance(keys, str):
keys = key.split('.')
last_index = len(keys) - 1
for index, key in enumerate(keys):
if index == last_index:
data = data.get(key, default)
else:
data = data.get(key, {})
return data
class Provider(BaseProvider('task', 'upload')):
def execute(self, results, params):
file_path = self.get_path(self.field_file)
if not os.path.exists(file_path):
self.command.error("Upload task provider file {} does not exist".format(file_path))
ssh = self._get_ssh()
ssh.upload(file_path, self.field_remote_path,
mode = self.field_mode,
owner = self.field_owner,
group = self.field_group
)
class Provider(BaseProvider('server', 'ec2')):
def initialize_terraform(self, instance, created):
if 'key_name' not in instance.config:
key_name, private_key = self.create_aws_ec2_keypair(instance.subnet.network)
instance.config['key_name'] = key_name
instance.private_key = private_key
super().initialize_terraform(instance, created)
def finalize_terraform(self, instance):
super().finalize_terraform(instance)
self.delete_aws_ec2_keypair(instance.subnet.network, instance.config['key_name'])
class Provider(BaseProvider('function', 'prefix')):
def exec(self, values, prefix=None):
if isinstance(values, dict):
values = list(values.keys())
else:
values = ensure_list(values)
if prefix is None:
return values
for index, value in enumerate(values):
values[index] = "{}{}".format(prefix, value)
return values
class Provider(BaseProvider('function', 'flatten')):
def exec(self, *elements):
values = []
def flatten(value):
if isinstance(value, (list, tuple)):
for item in value:
flatten(item)
else:
values.append(value)
for element in elements:
flatten(element)
return values
class Provider(BaseProvider('task', 'command')):
def execute(self, results, params):
env = self._env_vars(params)
stdin = params.pop('input', self.field_input)
cwd = params.pop('cwd', self.field_cwd)
display = params.pop('display', self.field_display)
options = self._merge_options(self.field_options, params,
self.field_lock)
command = self._interpolate(self.field_command, options)
self.command.sh(shlex.split(command[0]),
input=stdin,
display=display,
env=env,
cwd=cwd,
sudo=self.field_sudo)
class Provider(BaseProvider('parser', 'conditional_value')):
conditional_pattern = r'^\?\>([^\?]+)\?([^\|]+)\|(.+)$'
def parse(self, value):
if not isinstance(value, str):
return value
conditional_match = re.search(self.conditional_pattern, value)
if conditional_match:
if eval(conditional_match.group(1).strip()):
value = eval(conditional_match.group(2).strip())
else:
value = eval(conditional_match.group(3).strip())
return value
class Provider(BaseProvider('validator', 'string')):
def validate(self, value):
if not isinstance(value, str):
self.warning("Value {} is not a string".format(value))
return False
if not self.field_empty and len(value) == 0:
self.warning("Empty strings not allowed")
return False
if self.field_pattern:
pattern = re.compile(self.field_pattern)
if not pattern.match(value):
self.warning("Value {} does not match pattern: {}".format(
value, self.field_pattern))
return False
return True
class Provider(BaseProvider('function', 'filter')):
def exec(self, data, **filters):
filtered_data = {}
for key, value in data.items():
add_key = True
for filter_param, filter_value in filters.items():
if isinstance(value, dict) and filter_param in value:
if value[filter_param] != filter_value:
add_key = False
elif not isinstance(filter_value,
bool) or filter_value == True:
add_key = False
if add_key:
filtered_data[key] = value
return filtered_data
class Provider(BaseProvider('parser', 'function')):
function_pattern = r'^\#([a-zA-Z][\_\-a-zA-Z0-9]+)\((.*?)\)\;'
function_value_pattern = r'(?<!\#)\#\>?([a-zA-Z][\_\-a-zA-Z0-9]+\(.*?\)\;)'
def parse(self, value):
if not isinstance(value, str):
return value
standalone_function = re.search(self.function_pattern, value)
if standalone_function and len(
standalone_function.group(0)) == len(value):
value = self.exec_function(value)
else:
for ref_match in re.finditer(self.function_value_pattern, value):
function_value = self.exec_function("#{}".format(
ref_match.group(1)))
if isinstance(function_value, (list, tuple)):
function_value = ",".join(function_value)
elif isinstance(function_value, dict):
function_value = json.dumps(function_value)
if function_value:
value = value.replace(ref_match.group(0),
str(function_value)).strip()
return value
def exec_function(self, value):
function_match = re.search(self.function_pattern, value)
if function_match:
function_name = function_match.group(1)
function_parameters = re.split(r'\s*\|\|\s*',
function_match.group(2))
for index, parameter in enumerate(function_parameters):
parameter = parameter.lstrip("\'\"").rstrip("\'\"")
function_parameters[index] = self.command.options.interpolate(
parameter)
function = self.command.get_provider('function', function_name)
return function.exec(*function_parameters)
# Not found, assume desired
return value