class Provider(BaseProvider('validator', 'number')): def validate(self, value, record): try: value = number(value) except (ValueError, TypeError): self.warning("Value {} is not a number".format(value)) return False if not self.field_nan and math.isnan(value): self.warning("Value can not be NaN") return False if self.field_min is not None: if value < self.field_min: self.warning("Value {} is below minimum allowed: {}".format( value, self.field_min)) return False if self.field_max is not None: if value > self.field_max: self.warning("Value {} is above maximum allowed: {}".format( value, self.field_max)) return False return True
class Provider(BaseProvider('formatter', 'name_joiner')): def format(self, value): date = self.format_value(value[1], self.field_date_formatter_provider, format=self.field_date_format) return self.field_join.join( [str(int(value[0])), date.strftime("%Y-%m-%d")])
class Provider(BaseProvider('parser', 'state')): variable_pattern = r'^\$\{?([a-zA-Z][\_\-a-zA-Z0-9]+)(?:\[([^\]]+)\])?\}?$' variable_value_pattern = r'(?<!\$)\$\>?\{?([a-zA-Z][\_\-a-zA-Z0-9]+(?:\[[^\]]+\])?)\}?' runtime_variables = {} def __init__(self, type, name, command, config): super().__init__(type, name, command, config) self.variables = {} def initialize(self, reset=False): if reset or not self.variables: self.variables = {} for state in self.command.get_instances(self.command._state): self.variables[state.name] = state.value def parse(self, value): if not isinstance(value, str): return value if re.search(self.variable_pattern, value): value = self.parse_variable(value) else: for ref_match in re.finditer(self.variable_value_pattern, value): variable_value = self.parse_variable("${}".format( ref_match.group(1))) if isinstance(variable_value, (list, tuple)): variable_value = ",".join(variable_value) elif isinstance(variable_value, dict): variable_value = json.dumps(variable_value) if variable_value: value = value.replace(ref_match.group(0), str(variable_value)).strip() return value def parse_variable(self, value): state_match = re.search(self.variable_pattern, value) if state_match: variables = {**self.runtime_variables, **self.variables} new_value = state_match.group(1) key = state_match.group(2) if new_value in variables: data = variables[new_value] if isinstance(data, dict) and key: try: return data[key] except KeyError: return value elif isinstance(data, (list, tuple)) and key: try: return data[int(key)] except KeyError: return value else: return data # Not found, assume desired return value
class Provider(BaseProvider('parser', 'token')): token_pattern = r'^\%([\%\!])([a-zA-Z][\_\-a-zA-Z0-9]+)(?:\:(\d+))?$' def parse(self, value): if not isinstance(value, str): return value ref_match = re.search(self.token_pattern, value) if ref_match: operation = ref_match.group(1) length = ref_match.group(3) if length: length = int(length) else: length = 20 state_name = "token-{}-{}".format(ref_match.group(2), length) if operation == '%': value = self.command.get_state(state_name) else: value = None if not value: value = create_token(length) self.command.set_state(state_name, value) return value
class Provider(BaseProvider('task', 'remote_script')): def execute(self, results, params): script_path = self.get_path(self.field_script) if not os.path.exists(script_path): self.command.error( "Remote script task provider file {} does not exist".format( script_path)) script_base, script_ext = os.path.splitext(script_path) temp_path = "/tmp/{}{}".format(self.generate_name(24), script_ext) env = self._env_vars(params) options = self._merge_options(self.field_options, params, self.field_lock) args = ensure_list(self.field_args, []) ssh = self._get_ssh(env) ssh.upload(script_path, temp_path, mode=755) try: self._ssh_exec(temp_path, self._interpolate(args, options), sudo=self.field_sudo, ssh=ssh) finally: ssh.exec('rm -f', temp_path)
class Provider(BaseProvider('validator', 'exists')): def validate(self, value, record): if value is None: self.warning("Value can not be nothing to check for existence") return False facade = self.command.facade(self.field_data, False) filters = {} scope_text = '' if self.field_scope: scope = {} for field in self.field_scope: scope[field] = record[field] facade.set_scope(scope) scope_text = "within scope {}".format(scope) field = self.field_field if self.field_field else facade.key() filters[field] = value if not facade.keys(**filters): self.warning("Model {} {}: {} does not exist {}".format(self.field_data, field, value, scope_text)) return False return True
class Provider(BaseProvider('encryption', 'aes256')): @classmethod def generate_key(self): return hashlib.sha256(super().generate_key().encode()).hexdigest()[::2] def initialize(self, options): options['key'] = hashlib.sha256( options['key'].encode()).hexdigest()[::2] def encrypt_text(self, plain_text): iv = Random.new().read(AES.block_size) iv_int = int(binascii.hexlify(iv), AES.block_size) ctr = Counter.new(AES.block_size * 8, initial_value=iv_int) cipher = AES.new(self.field_key.encode(), AES.MODE_CTR, counter=ctr) return base64.b64encode(iv + cipher.encrypt(plain_text.encode())) def decrypt_text(self, cipher_text): cipher_text = base64.b64decode(cipher_text) iv = cipher_text[:AES.block_size] cipher_text = cipher_text[AES.block_size:] iv_int = int(iv.hex(), AES.block_size) ctr = Counter.new(AES.block_size * 8, initial_value=iv_int) cipher = AES.new(self.field_key.encode(), AES.MODE_CTR, counter=ctr) return cipher.decrypt(cipher_text)
class Provider(BaseProvider('task', 'remote_command')): def execute(self, results, params): env = self._env_vars(params) options = self._merge_options(self.field_options, params, self.field_lock) command = self._interpolate(self.field_command, options) self._ssh_exec(command, env=env, sudo=self.field_sudo)
class Provider(BaseProvider('function', 'random_keys')): def exec(self, dict_value, limit=None): keys = list(dict_value.keys()) random.shuffle(keys) if limit: return keys[:min(int(limit), len(keys))] return keys
class SubnetProvider(BaseProvider('network.subnet', 'aws')): def initialize_terraform(self, instance, created): if instance.config['zone'] is None and instance.config[ 'zone_suffix'] is not None: instance.config['zone'] = "{}{}".format( instance.network.config['region'], instance.config['zone_suffix']) super().initialize_terraform(instance, created)
class Provider(BaseProvider('function', 'join')): def exec(self, *elements): values = [] for element in elements: values.extend(ensure_list(element)) return values
class Provider(BaseProvider('function', 'random_values')): def exec(self, list_value, limit = None): values = ensure_list(list_value) random.shuffle(values) if limit: return values[:min(int(limit), len(values))] return values
class Provider(BaseProvider('source', 'csv_file')): def load(self): return self.load_csv_data_from_file( self.field_file, self.import_columns, archive_file=self.field_archive_file, separator=self.field_separator, data_type=self.field_data_type, header=self.field_header)
class Provider(BaseProvider('encryption', 'aes256_user')): def initialize(self, options): if options.get('user', None): user_facade = self.manager.index.get_facade_index()['user'] user = user_facade.retrieve(options['user']) if not user: raise EncryptionError("User {} does not exist".format(options['user'])) options['key'] = user.encryption_key
class Provider(BaseProvider('field_processor', 'combined_text')): def exec(self, dataset, field_data, append = None, separator = ","): if append: for field in append.split(','): field_data = field_data.str.cat( dataset[field].astype(str), sep = separator ) return field_data
class Provider(BaseProvider('function', 'keys')): def exec(self, data, prefix = None): keys = list(data.keys()) if prefix is None: return keys for index, key in enumerate(keys): keys[index] = "{}{}".format(prefix, key) return keys
class Provider(BaseProvider('formatter', 'date_time')): def format(self, value, record): if isinstance(value, float): value = int(value) try: value = datetime.datetime.strptime(str(value), self.field_format) except ValueError as e: self.error("Value {} is not a valid date time according to pattern: {}".format(value, self.field_format)) return value
class Provider(BaseProvider('formatter', 'number')): def format(self, value, record): try: if isinstance(value, str): value = float(value) except Exception: return None if value is None or math.isnan(value): return None return number(value)
class Provider(BaseProvider('validator', 'date_time')): def validate(self, value): if isinstance(value, float): value = int(value) try: datetime.datetime.strptime(str(value), self.field_format) except ValueError as e: self.warning("Value {} is not a valid date time according to pattern: {}".format(value, self.field_format)) return False return True
class Provider(BaseProvider('worker', 'docker')): def start_worker(self): self.command.manager.stop_service('worker', remove = True) service_spec = self.command.manager.get_service_spec('worker') service_spec['environment']['ZIMAGI_WORKER_TYPE'] = self.field_worker_type self.command.manager.start_service('worker', **service_spec) def check_worker(self): return self.command.manager.get_service('worker', restart = False, create = False)
class Provider(BaseProvider('function', 'value')): def exec(self, data, keys, default=None): if isinstance(keys, str): keys = key.split('.') last_index = len(keys) - 1 for index, key in enumerate(keys): if index == last_index: data = data.get(key, default) else: data = data.get(key, {}) return data
class Provider(BaseProvider('task', 'upload')): def execute(self, results, params): file_path = self.get_path(self.field_file) if not os.path.exists(file_path): self.command.error("Upload task provider file {} does not exist".format(file_path)) ssh = self._get_ssh() ssh.upload(file_path, self.field_remote_path, mode = self.field_mode, owner = self.field_owner, group = self.field_group )
class Provider(BaseProvider('server', 'ec2')): def initialize_terraform(self, instance, created): if 'key_name' not in instance.config: key_name, private_key = self.create_aws_ec2_keypair(instance.subnet.network) instance.config['key_name'] = key_name instance.private_key = private_key super().initialize_terraform(instance, created) def finalize_terraform(self, instance): super().finalize_terraform(instance) self.delete_aws_ec2_keypair(instance.subnet.network, instance.config['key_name'])
class Provider(BaseProvider('function', 'prefix')): def exec(self, values, prefix=None): if isinstance(values, dict): values = list(values.keys()) else: values = ensure_list(values) if prefix is None: return values for index, value in enumerate(values): values[index] = "{}{}".format(prefix, value) return values
class Provider(BaseProvider('function', 'flatten')): def exec(self, *elements): values = [] def flatten(value): if isinstance(value, (list, tuple)): for item in value: flatten(item) else: values.append(value) for element in elements: flatten(element) return values
class Provider(BaseProvider('task', 'command')): def execute(self, results, params): env = self._env_vars(params) stdin = params.pop('input', self.field_input) cwd = params.pop('cwd', self.field_cwd) display = params.pop('display', self.field_display) options = self._merge_options(self.field_options, params, self.field_lock) command = self._interpolate(self.field_command, options) self.command.sh(shlex.split(command[0]), input=stdin, display=display, env=env, cwd=cwd, sudo=self.field_sudo)
class Provider(BaseProvider('parser', 'conditional_value')): conditional_pattern = r'^\?\>([^\?]+)\?([^\|]+)\|(.+)$' def parse(self, value): if not isinstance(value, str): return value conditional_match = re.search(self.conditional_pattern, value) if conditional_match: if eval(conditional_match.group(1).strip()): value = eval(conditional_match.group(2).strip()) else: value = eval(conditional_match.group(3).strip()) return value
class Provider(BaseProvider('validator', 'string')): def validate(self, value): if not isinstance(value, str): self.warning("Value {} is not a string".format(value)) return False if not self.field_empty and len(value) == 0: self.warning("Empty strings not allowed") return False if self.field_pattern: pattern = re.compile(self.field_pattern) if not pattern.match(value): self.warning("Value {} does not match pattern: {}".format( value, self.field_pattern)) return False return True
class Provider(BaseProvider('function', 'filter')): def exec(self, data, **filters): filtered_data = {} for key, value in data.items(): add_key = True for filter_param, filter_value in filters.items(): if isinstance(value, dict) and filter_param in value: if value[filter_param] != filter_value: add_key = False elif not isinstance(filter_value, bool) or filter_value == True: add_key = False if add_key: filtered_data[key] = value return filtered_data
class Provider(BaseProvider('parser', 'function')): function_pattern = r'^\#([a-zA-Z][\_\-a-zA-Z0-9]+)\((.*?)\)\;' function_value_pattern = r'(?<!\#)\#\>?([a-zA-Z][\_\-a-zA-Z0-9]+\(.*?\)\;)' def parse(self, value): if not isinstance(value, str): return value standalone_function = re.search(self.function_pattern, value) if standalone_function and len( standalone_function.group(0)) == len(value): value = self.exec_function(value) else: for ref_match in re.finditer(self.function_value_pattern, value): function_value = self.exec_function("#{}".format( ref_match.group(1))) if isinstance(function_value, (list, tuple)): function_value = ",".join(function_value) elif isinstance(function_value, dict): function_value = json.dumps(function_value) if function_value: value = value.replace(ref_match.group(0), str(function_value)).strip() return value def exec_function(self, value): function_match = re.search(self.function_pattern, value) if function_match: function_name = function_match.group(1) function_parameters = re.split(r'\s*\|\|\s*', function_match.group(2)) for index, parameter in enumerate(function_parameters): parameter = parameter.lstrip("\'\"").rstrip("\'\"") function_parameters[index] = self.command.options.interpolate( parameter) function = self.command.get_provider('function', function_name) return function.exec(*function_parameters) # Not found, assume desired return value