def read_file(file: str) -> str: if not os.path.exists(file): err = 'No such file: ' + file error(err) raise FileNotFoundError(err) with io.open(file, mode='r', encoding='utf-8') as stream: return stream.read()
def delete_dag(self, dag_id): client = docker.from_env() airflow = client.containers.get('catcher_modules_webserver_1') res = airflow.exec_run(cmd=f'/entrypoint.sh airflow delete_dag -y {dag_id}') if res.exit_code != 0: error("Can't delete dag {}: {}".format(dag_id, res.output.decode())) raise Exception(f"Can't delete dag {dag_id}")
def action(self, includes: dict, variables: dict) -> dict: filled_vars = dict([(k, fill_template_str(v, variables)) for (k, v) in self.variables.items()]) out = fill_template_str(self.include, variables) test, tag = get_tag(out) if test not in includes: error('No include registered for name ' + test) raise Exception('No include registered for name ' + test) include = includes[test] variables = merge_two_dicts(include.variables, merge_two_dicts(variables, filled_vars)) include.variables = try_get_object(fill_template_str(variables, variables)) try: info('Running {}'.format(test)) logger.log_storage.nested_test_in() variables = include.run(tag=tag, raise_stop=True) logger.log_storage.nested_test_out() except SkipException: logger.log_storage.nested_test_out() debug('Include ignored') return variables except StopException as e: logger.log_storage.nested_test_out() raise e except Exception as e: logger.log_storage.nested_test_out() if not self.ignore_errors: raise Exception('Step run ' + test + ' failed: ' + str(e)) return variables
def _read_json_file(file: str) -> dict: with open(file, 'r') as stream: try: return json.load(stream) or {} except yaml.YAMLError as exc: err = 'Wrong YAML format for file ' + file + ' : ' + str(exc) error(err) raise yaml.YAMLError(err)
def add_dag(self, dag_name: str): copyfile(join(self.global_resource_dir, dag_name), join(dirname(test.__file__), '../dags', dag_name)) client = docker.from_env() airflow = client.containers.get('catcher_modules_webserver_1') res = airflow.exec_run(cmd='/entrypoint.sh python -c "from airflow.models import DagBag; d = DagBag();"') if res.exit_code != 0: error("Can't trigger airflow dagbag refresh: {}".format(res.output.decode())) raise Exception("Can't trigger airflow dagbag refresh")
def read_source_file(file: str) -> dict: if not os.path.exists(file): err = 'No such file: ' + file error(err) raise FileNotFoundError(err) if file.lower().endswith('json'): return _read_json_file(file) else: return _read_yaml_file(file)
def get_files(path: str) -> list: if not os.path.exists(path): err = 'No such path: ' + path error(err) raise FileNotFoundError(err) file = [] if os.path.isdir(path): for f in os.listdir(path): path_in_dir = join(path, f) if os.path.isfile(path_in_dir) and (path_in_dir.endswith('yaml') or path_in_dir.endswith('yml')): file.append(path_in_dir) elif os.path.isdir(path_in_dir): file += get_files(path_in_dir) else: file.append(path) return file
def prepare_modules(module_paths: list, available: dict) -> dict: """ Scan all paths for external modules and form key-value dict. :param module_paths: list of external modules (either python packages or third-party scripts) :param available: dict of all registered python modules (can contain python modules from module_paths) :return: dict of external modules, where keys are filenames (same as stepnames) and values are the paths """ indexed = {} for path in module_paths: if not os.path.exists(path) and path not in available: err = 'No such path: ' + path error(err) else: for f in os.listdir(path): mod_path = join(path, f) if f in indexed: warning('Override ' + indexed[f] + ' with ' + mod_path) indexed[f] = mod_path return indexed
def action(self, includes: dict, variables: dict) -> dict: filled_vars = dict([(k, fill_template_str(v, variables)) for (k, v) in self.variables.items()]) out = fill_template_str(self.include, variables) test, tag = get_tag(out, self.tag) if test not in includes: error('No include registered for name ' + test) raise Exception('No include registered for name ' + test) include = includes[test] variables = merge_two_dicts(include.variables, merge_two_dicts(variables, filled_vars)) include.variables = variables try: variables = include.run(tag=tag, raise_stop=True) except StopException as e: raise e except Exception as e: if not self.ignore_errors: raise Exception('Step run ' + test + ' failed: ' + str(e)) return variables