def __form_body(self, variables) -> str or dict: if self.method == 'get': return False, None body = self.body if body is None: body = read_file(fill_template_str(self.file, variables)) if isinstance(body, dict): # dump body to json to be able fill templates in body = json.dumps(body) isjson = 'tojson' in body return isjson, fill_template(body, variables, isjson=isjson)
def _prepare_include_vars(test, global_variables): if test.include: include_vars = try_get_object( fill_template_str(test.include.variables, global_variables)) else: include_vars = {} override_keys = report_override(test.variables, include_vars) if override_keys: debug('Include variables override these variables: ' + str(override_keys)) return include_vars
def populate(self, variables, conf=None, schema=None, data: dict = None, use_json=False, **kwargs): """ :Input: Populate database with prepared scripts (DDL or CSV with data). :populate: - populate a database with the data and/or run DDL to create schema. :conf: postgres configuration. Can be a single line string or object. **Required**. - dbname: name of the database to connect to - user: database user - host: database host - password: user's password - port: database port :schema: path to the schema file. *Optional* :data: dictionary with keys = tables and values - paths to csv files with data. *Optional* :use_json: try to recognize json strings and convert them to json. *Optional*, default is false. :F.e.: populate postgres :: variables: pg_schema: schema.sql pg_data: foo: foo.csv bar: bar.csv steps: - prepare: populate: postgres: conf: {{ pg_conf }} schema: {{ pg_schema }} data: {{ pg_data }} use_json: true """ resources = variables['RESOURCES_DIR'] if schema is not None: with open(os.path.join(resources, schema)) as fd: ddl_sql = fill_template_str(fd.read(), variables) self.__execute(conf, ddl_sql) if data is not None and data: for table_name, path_to_csv in data.items(): self.__populate_csv(conf, table_name, os.path.join(resources, path_to_csv), variables, use_json)
def action(self, includes: dict, variables: dict) -> dict: filled_vars = dict([(k, fill_template_str(v, variables)) for (k, v) in self.variables.items()]) out = fill_template_str(self.include, variables) test, tag = get_tag(out, self.tag) if test not in includes: error('No include registered for name ' + test) raise Exception('No include registered for name ' + test) include = includes[test] variables = merge_two_dicts(include.variables, merge_two_dicts(variables, filled_vars)) include.variables = variables try: variables = include.run(tag=tag, raise_stop=True) except StopException as e: raise e except Exception as e: if not self.ignore_errors: raise Exception('Step run ' + test + ' failed: ' + str(e)) return variables
def simple_input(self, variables): """ Use this method to get simple input as python object, with all templates filled in :param variables: :return: python object """ json_args = fill_template_str(json.dumps(self.data), variables) return try_get_objects(json_args)
def __run_actions(self, includes, variables: dict) -> dict: output = variables for action in self.do_action: try: output = action.action(includes, output) except Exception as e: if action.ignore_errors: debug('{} got {} but we ignore it'.format( fill_template_str(action.name, variables), e)) break raise e return output
def _form_request(self, url, variables: dict) -> dict: headers = dict([(fill_template_str(k, variables), fill_template_str(v, variables)) for k, v in self.headers.items()]) rq = dict(verify=self.verify, headers=headers, files=self.__form_files(variables)) isjson, body = self.__form_body(variables) debug('http ' + str(self.method) + ' ' + str(url) + ', ' + str(headers) + ', ' + str(body)) content_type = self.__get_content_type(headers) if isjson or isinstance(body, dict): # contains tojson or dict supplied if isinstance(body, dict) and content_type == 'application/json': # json body formed manually via python dict rq['json'] = body else: # json string or form-data dict rq['data'] = body else: # raw body (or body is None) rq['data'] = body rq['timeout'] = self.timeout return rq
def action(self, includes: dict, variables: dict) -> any: # if virtual host is not specified default it to / config = try_get_objects(fill_template_str(self.config, variables)) if config.get('virtualhost') is None: config['virtualhost'] = '' disconnect_timeout = int(config.get('disconnect_timeout', 10)) # 10 sec for connection closed exception connection_parameters = self._get_connection_parameters(config) if self.method == 'publish': message = self.form_body(self.message, self.file, variables) return variables, self.publish(connection_parameters, fill_template_str(self.exchange, variables), fill_template_str(self.routing_key, variables), fill_template(self.headers, variables), message, disconnect_timeout) elif self.method == 'consume': return variables, self.consume(connection_parameters, fill_template_str(self.queue, variables), disconnect_timeout) else: raise AttributeError('unknown method: ' + self.method)
def action(self, includes: dict, variables: dict) -> tuple: if self.source_file: # read from file resources = variables['RESOURCES_DIR'] out = fill_template_str( read_file(os.path.join(resources, self.source_file)), variables) else: out = fill_template(self.source, variables) if self.dst is None: info(out) else: dst = fill_template(self.dst, variables) with open(join(self.path, dst), 'w') as f: f.write(str(out)) return variables, out
def __form_body(self, variables) -> tuple: if self.method == 'get': return False, None body = self.body if body is None and self.file is not None: resources = variables['RESOURCES_DIR'] body = file_utils.read_file( fill_template_str(os.path.join(resources, self.file), variables)) if isinstance(body, dict) or isinstance( body, list): # dump body to json to be able fill templates in body = json.dumps(body) if body is None: return False, None isjson = 'tojson' in body return isjson, fill_template(body, variables, isjson=isjson)
def prepare_variables(self, test, global_variables: dict): """ Create variables for test :param global_variables: :param test: test with test variables """ # system env + inventory # (test template is filled in based on system, inventory & cmd vars) test.variables = try_get_object( fill_template_str(test.variables, merge_two_dicts(global_variables, self._cmd_env))) # test local global_variables.update(test.variables) # include vars override test local global_variables.update( self._prepare_include_vars(test, global_variables)) # cmd_env override everything global_variables.update(self._cmd_env) test.variables = global_variables
def __init__(self, path: str, system_environment=None, inventory_vars: dict = None, cmd_env: Optional[dict] = None, resources: Optional[dict] = None) -> None: system_vars = system_environment or {} if system_vars: debug('Use system variables: ' + str(list(system_vars.keys()))) self._variables = system_vars else: self._variables = {} inventory = try_get_object( fill_template_str(inventory_vars, self._variables)) # fill env vars self._cmd_env = cmd_env or {} self._variables.update(inventory) self._variables['CURRENT_DIR'] = path self._variables['RESOURCES_DIR'] = resources or os.path.join( path, 'resources')
def action(self, includes: dict, variables: dict) -> dict: output = variables if self.type == 'while': operator = Operator.find_operator(self.if_clause) while operator.operation(output): output = self.__run_actions(includes, output) if self.max_cycle is not None: if self.max_cycle == 0: break self.max_cycle = self.max_cycle - 1 return output elif self.type == 'foreach': loop_var = try_get_objects( fill_template_str(json.dumps(self.in_var), variables)) if not isinstance(loop_var, collections.Iterable): raise ValueError(str(loop_var) + ' is not iterable') for entry in loop_var: output['ITEM'] = entry output = self.__run_actions(includes, output) return output
def run_tests(self) -> bool: try: self._compose.up() if self.system_vars: debug('Use system variables: ' + str(list(self.system_vars.keys()))) variables = self.system_vars else: variables = {} if self.inventory is not None: inv_vars = read_source_file(self.inventory) inv_vars['INVENTORY'] = get_filename(self.inventory) variables = try_get_object( fill_template_str(inv_vars, variables)) # fill env vars variables['CURRENT_DIR'] = self.path variables[ 'RESOURCES_DIR'] = self.resources or self.path + '/resources' test_files = get_files(self.tests_path) results = [] for file in test_files: self.all_includes = [] try: variables['TEST_NAME'] = file test = self.prepare_test(file, variables) logger.log_storage.test_start(file) test.run() results.append(True) info('Test ' + file + ' passed.') logger.log_storage.test_end(file, True) except Exception as e: warning('Test ' + file + ' failed: ' + str(e)) results.append(False) logger.log_storage.test_end(file, False, str(e)) return all(results) finally: logger.log_storage.write_report(self.path) self._compose.down()
def __read_n_fill_csv(cls, csv_path, variables): with open(csv_path) as csv_file: csv_content = fill_template_str(csv_file.read(), variables).replace('\n\n', '\n') return StringIO(csv_content)
def get_action_name(action_type: str, action: Step, variables: dict): if action.name is not None: return fill_template_str(action.name, variables) return action_type
def __form_body(self, variables): data = self.data if data is None: data = read_file(fill_template_str(self.file, variables)) return fill_template_str(data, variables)