def run_tests(self) -> bool: try: self._compose.up() variables = {} if self.inventory is not None: variables = read_source_file(self.inventory) variables['INVENTORY'] = get_filename(self.inventory) variables = try_get_object(fill_template_str( variables, {})) # fill env vars variables['CURRENT_DIR'] = self.path test_files = get_files(self.tests_path) results = [] for file in test_files: self.all_includes = [] try: variables['TEST_NAME'] = file test = self.prepare_test(file, variables) test.run() results.append(True) info('Test ' + file + ' passed.') except Exception as e: warning('Test ' + file + ' failed: ' + str(e)) results.append(False) return all(results) finally: self._compose.down()
def _run_test(self, test: Test, global_variables: dict, output: str = 'full', test_type='test') -> bool: try: self.var_holder.prepare_variables(test, global_variables) logger.log_storage.test_start(test.file, test_type=test_type) test.check_ignored() with OptionalOutput(output == 'limited'): test.run() info(test_type.capitalize() + ' ' + cut_path(self.tests_path, test.file) + logger.green(' passed.')) logger.log_storage.test_end(test.file, True, test_type=test_type) return True except SkipException: info(test_type.capitalize() + ' ' + cut_path(self.tests_path, test.file) + logger.yellow(' skipped.')) logger.log_storage.test_end(test.file, True, end_comment='Skipped', test_type=test_type) return True except Exception as e: warning(test_type.capitalize() + ' ' + cut_path(self.tests_path, test.file) + logger.red(' failed: ') + str(e)) debug(traceback.format_exc()) logger.log_storage.test_end(test.file, False, str(e), test_type=test_type) return False
def fill_connections(inventory, conf, dialect, fernet_key): """ Populate airflow connections based on catcher's inventory file :param inventory: path to inventory file :param conf: db configuration :param dialect: dialect :param fernet_key: is used in passwords encryption :return: """ inv_dict = file_utils.read_source_file(inventory) engine = db_utils.get_engine(conf, dialect) with engine.connect() as connection: for name, value in inv_dict.items(): try: if isinstance(value, dict) and 'type' in value: if _check_conn_id_exists(name, connection): debug('{} already exists'.format(name)) continue query, params = _prepare_connection(value, fernet_key) from sqlalchemy.sql import text connection.execute(text(query), name=name, **params) debug('{} added'.format(name)) else: debug('{} ignored. No type specified'.format(name)) except Exception as e: warning('Can\'t add {}:{} - {}'.format(name, value, e))
def run_cmd_simple(cmd: str, variables: dict, env=None, args: List[str] = None, libraries=None) -> Union[dict, str]: """ Run cmd with variables written in environment. :param args: cmd arguments :param cmd: to run :param variables: variables :param env: custom environment :param libraries: additional libraries used for source compilation :return: output in json (if can be parsed) or plaintext """ env = _prepare_env(variables, env=env) cmd, cwd = _prepare_cmd(cmd, args, variables, libraries=libraries) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env, cwd=cwd) if p.wait() == 0: out = p.stdout.read().decode() debug(out) return _parse_output(out) else: out = p.stdout.read().decode() warning(out) raise Exception('Execution failed.')
def _wait_for_running(url, dag_id, execution_date, timeout, run_id, dialect=None, db_conf=None): while True: try: state = airflow_client.get_run_status(url, dag_id, execution_date) except OldAirflowVersionException: warning( "Your airflow version does not support rest API method for DAG status. Call backend db directly" ) state = airflow_db_client.get_dag_run_by_run_ud( run_id=run_id, conf=db_conf, dialect=dialect)["state"] debug(state) if state.lower() != 'running': return state.lower() if timeout > 0: sleep(1) timeout -= 1 else: raise Exception('Dag {} still running'.format(dag_id))
def __env_to_variables(environment: list) -> dict: variables = {} for env in environment: if '=' not in env: warning('Skip not kv env param ' + env) else: [k, v] = env.split('=') variables[k] = v return variables
def __form_files(self, variables) -> Optional[list]: if self.files is not None: if isinstance(self.files, dict): return [self.__prepare_file(self.files, variables)] elif isinstance(self.files, list): return [self.__prepare_file(f, variables) for f in self.files] else: warning('Don\'t know how to prepare ' + type(self.files)) return None
def publish(connection_parameters, exchange, routing_key, headers, message, disconnect_timeout): import pika from pika import exceptions properties = pika.BasicProperties(headers=headers) try: connection_parameters.blocked_connection_timeout = disconnect_timeout with pika.BlockingConnection(connection_parameters) as connection: channel = connection.channel() channel.basic_publish(exchange=exchange, routing_key=routing_key, properties=properties, body=message) except exceptions.ConnectionClosed: warning('Failed to gracefully close rabbit connection.')
def __fill_dialect(url: str, driver: str): if '://' not in url: # simple case - user:password@host:port/database return driver + '://' + url else: if '+' in url or url.startswith('postgresql'): return url # dialect specified - mysql+pymysql://user:password@host:port/database else: # no dialect - need to set up dialect from driver driver_used = url.split(':')[0] found = [k for k in default_ports.keys() if k.startswith(driver_used)] if not found: warning('Can\'t find dialect for driver ' + driver_used + '. Will try default ' + driver) found = [driver] return found[0] + ':' + ':'.join(url.split(':')[1:])
def add_package_to_globals(package: str, glob=None, warn_missing_package=True) -> dict: if glob is None: glob = globals() try: mod = importlib.import_module(package) glob[package] = mod except ImportError as e: if warn_missing_package: warning(str(e)) else: debug(str(e)) return glob
def prepare_modules(module_paths: list, available: dict) -> dict: """ Scan all paths for external modules and form key-value dict. :param module_paths: list of external modules (either python packages or third-party scripts) :param available: dict of all registered python modules (can contain python modules from module_paths) :return: dict of external modules, where keys are filenames (same as stepnames) and values are the paths """ indexed = {} for path in module_paths: if not os.path.exists(path) and path not in available: err = 'No such path: ' + path error(err) else: for f in os.listdir(path): mod_path = join(path, f) if f in indexed: warning('Override ' + indexed[f] + ' with ' + mod_path) indexed[f] = mod_path return indexed
def _run_dag(self, oper, inventory): dag_id = oper['dag_id'] config = oper['config'] db_conf = config['db_conf'] backend = config.get('backend', 'postgresql') url = config['url'] self._prepare_dag_run(dag_id, config, inventory) dag_config = oper.get('dag_config', {}) run_id = airflow_client.trigger_dag(url, dag_id, dag_config) sync = oper.get('sync', False) if not sync: return run_id else: wait_timeout = oper.get('wait_timeout', 5) try: execution_date = airflow_client.get_dag_run( url, dag_id, run_id)['execution_date'] except OldAirflowVersionException: warning( "Your airflow version does not support rest API method: dag_runs. Call backend db directly" ) execution_date = airflow_db_client.get_execution_date_by_run_ud( run_id=run_id, conf=db_conf, dialect=backend) state = self._wait_for_running(url=url, dag_id=dag_id, execution_date=execution_date, timeout=wait_timeout, run_id=run_id, dialect=backend, db_conf=db_conf) if state != 'success': failed_task = airflow_db_client.get_failed_task( dag_id=dag_id, execution_time=execution_date, conf=db_conf, dialect=backend) raise Exception('Dag {} failed task {} with state {}'.format( dag_id, failed_task, state)) return run_id
def _run_finally(self, test, result: bool): if test and test.final: logger.log_storage.test_start(test.file, test_type='{}_cleanup'.format( test.file)) try: test.run_finally(result) info('Test ' + cut_path(self.tests_path, test.file) + ' [cleanup] ' + logger.green(' passed.')) logger.log_storage.test_end(test.file, True, test_type='{} [cleanup]'.format( test.file)) except Exception as e: warning('Test ' + cut_path(self.tests_path, test.file) + ' [cleanup] ' + logger.red(' failed: ') + str(e)) debug(traceback.format_exc()) logger.log_storage.test_end(test.file, False, test_type='{} [cleanup]'.format( test.file))
def _prepare_dag_run(dag_id, config, inventory): db_conf = config['db_conf'] backend = config.get('backend', 'postgresql') url = config['url'] if not airflow_db_client.check_dag_exists(dag_id, db_conf, backend): errors = airflow_db_client.check_import_errors( dag_id, db_conf, backend) msg = 'No dag {} found.'.format(dag_id) if errors: msg = msg + ' Possible import errors: {}'.format(str(errors)) raise Exception(msg) if inventory is not None and config.get('populate_connections', False): # fill connections from inventory to airflow airflow_db_client.fill_connections(inventory, db_conf, backend, config['fernet_key']) try: airflow_client.unpause_dag(url, dag_id) except OldAirflowVersionException: warning( "Your airflow version does not support rest API method to unpause dag. Call backend db directly" ) airflow_db_client.unpause_dag(dag_id, db_conf, backend)
def run_tests(self, output: str = 'full') -> bool: """ Run the testcase :param output: 'full' - all output possible. 'limited' - only test end report & summary. 'final' - only summary. """ try: [mod.before() for mod in ModulesFactory().modules.values()] results = [] for parse_result in self.parser.read_tests(self.tests_path): if parse_result.should_run: # parse successful variables = self.var_holder.variables # each test has it's own copy of global variables variables[ 'TEST_NAME'] = parse_result.test.file # variables are shared between test and includes with OptionalOutput(output == 'final'): for include in parse_result.run_on_include: # run all includes before the main test. self._run_test(include, variables, output=output, test_type='include') result = self._run_test(parse_result.test, variables, output=output) results.append(result) self._run_finally(parse_result.test, result) else: # parse failed (dependency/parsing problem) warning('Test ' + cut_path(self.tests_path, parse_result.test) + logger.red(' failed: ') + str(parse_result.parse_error)) logger.log_storage.test_parse_fail( parse_result.test, parse_result.parse_error) results.append(False) return all(results) finally: logger.log_storage.write_report(join(self.path, 'reports')) logger.log_storage.print_summary(self.tests_path) [mod.after() for mod in ModulesFactory().modules.values()]
def run_tests(self) -> bool: try: self._compose.up() if self.system_vars: debug('Use system variables: ' + str(list(self.system_vars.keys()))) variables = self.system_vars else: variables = {} if self.inventory is not None: inv_vars = read_source_file(self.inventory) inv_vars['INVENTORY'] = get_filename(self.inventory) variables = try_get_object( fill_template_str(inv_vars, variables)) # fill env vars variables['CURRENT_DIR'] = self.path variables[ 'RESOURCES_DIR'] = self.resources or self.path + '/resources' test_files = get_files(self.tests_path) results = [] for file in test_files: self.all_includes = [] try: variables['TEST_NAME'] = file test = self.prepare_test(file, variables) logger.log_storage.test_start(file) test.run() results.append(True) info('Test ' + file + ' passed.') logger.log_storage.test_end(file, True) except Exception as e: warning('Test ' + file + ' failed: ' + str(e)) results.append(False) logger.log_storage.test_end(file, False, str(e)) return all(results) finally: logger.log_storage.write_report(self.path) self._compose.down()