def read_file(file: str) -> str:
if not os.path.exists(file):
err = 'No such file: ' + file
error(err)
raise FileNotFoundError(err)
with io.open(file, mode='r', encoding='utf-8') as stream:
return stream.read()
def delete_dag(self, dag_id):
client = docker.from_env()
airflow = client.containers.get('catcher_modules_webserver_1')
res = airflow.exec_run(cmd=f'/entrypoint.sh airflow delete_dag -y {dag_id}')
if res.exit_code != 0:
error("Can't delete dag {}: {}".format(dag_id, res.output.decode()))
raise Exception(f"Can't delete dag {dag_id}")
def action(self, includes: dict, variables: dict) -> dict:
filled_vars = dict([(k, fill_template_str(v, variables)) for (k, v) in self.variables.items()])
out = fill_template_str(self.include, variables)
test, tag = get_tag(out)
if test not in includes:
error('No include registered for name ' + test)
raise Exception('No include registered for name ' + test)
include = includes[test]
variables = merge_two_dicts(include.variables, merge_two_dicts(variables, filled_vars))
include.variables = try_get_object(fill_template_str(variables, variables))
try:
info('Running {}'.format(test))
logger.log_storage.nested_test_in()
variables = include.run(tag=tag, raise_stop=True)
logger.log_storage.nested_test_out()
except SkipException:
logger.log_storage.nested_test_out()
debug('Include ignored')
return variables
except StopException as e:
logger.log_storage.nested_test_out()
raise e
except Exception as e:
logger.log_storage.nested_test_out()
if not self.ignore_errors:
raise Exception('Step run ' + test + ' failed: ' + str(e))
return variables
def _read_json_file(file: str) -> dict:
with open(file, 'r') as stream:
try:
return json.load(stream) or {}
except yaml.YAMLError as exc:
err = 'Wrong YAML format for file ' + file + ' : ' + str(exc)
error(err)
raise yaml.YAMLError(err)
def add_dag(self, dag_name: str):
copyfile(join(self.global_resource_dir, dag_name), join(dirname(test.__file__), '../dags', dag_name))
client = docker.from_env()
airflow = client.containers.get('catcher_modules_webserver_1')
res = airflow.exec_run(cmd='/entrypoint.sh python -c "from airflow.models import DagBag; d = DagBag();"')
if res.exit_code != 0:
error("Can't trigger airflow dagbag refresh: {}".format(res.output.decode()))
raise Exception("Can't trigger airflow dagbag refresh")
def read_source_file(file: str) -> dict:
if not os.path.exists(file):
err = 'No such file: ' + file
error(err)
raise FileNotFoundError(err)
if file.lower().endswith('json'):
return _read_json_file(file)
else:
return _read_yaml_file(file)
def get_files(path: str) -> list:
if not os.path.exists(path):
err = 'No such path: ' + path
error(err)
raise FileNotFoundError(err)
file = []
if os.path.isdir(path):
for f in os.listdir(path):
path_in_dir = join(path, f)
if os.path.isfile(path_in_dir) and (path_in_dir.endswith('yaml') or
path_in_dir.endswith('yml')):
file.append(path_in_dir)
elif os.path.isdir(path_in_dir):
file += get_files(path_in_dir)
else:
file.append(path)
return file
def prepare_modules(module_paths: list, available: dict) -> dict:
"""
Scan all paths for external modules and form key-value dict.
:param module_paths: list of external modules (either python packages or third-party scripts)
:param available: dict of all registered python modules (can contain python modules from module_paths)
:return: dict of external modules, where keys are filenames (same as stepnames) and values are the paths
"""
indexed = {}
for path in module_paths:
if not os.path.exists(path) and path not in available:
err = 'No such path: ' + path
error(err)
else:
for f in os.listdir(path):
mod_path = join(path, f)
if f in indexed:
warning('Override ' + indexed[f] + ' with ' + mod_path)
indexed[f] = mod_path
return indexed
def action(self, includes: dict, variables: dict) -> dict:
filled_vars = dict([(k, fill_template_str(v, variables))
for (k, v) in self.variables.items()])
out = fill_template_str(self.include, variables)
test, tag = get_tag(out, self.tag)
if test not in includes:
error('No include registered for name ' + test)
raise Exception('No include registered for name ' + test)
include = includes[test]
variables = merge_two_dicts(include.variables,
merge_two_dicts(variables, filled_vars))
include.variables = variables
try:
variables = include.run(tag=tag, raise_stop=True)
except StopException as e:
raise e
except Exception as e:
if not self.ignore_errors:
raise Exception('Step run ' + test + ' failed: ' + str(e))
return variables