def run_cmd_simple(cmd: str,
variables: dict,
env=None,
args: List[str] = None,
libraries=None) -> Union[dict, str]:
"""
Run cmd with variables written in environment.
:param args: cmd arguments
:param cmd: to run
:param variables: variables
:param env: custom environment
:param libraries: additional libraries used for source compilation
:return: output in json (if can be parsed) or plaintext
"""
env = _prepare_env(variables, env=env)
cmd, cwd = _prepare_cmd(cmd, args, variables, libraries=libraries)
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env,
cwd=cwd)
if p.wait() == 0:
out = p.stdout.read().decode()
debug(out)
return _parse_output(out)
else:
out = p.stdout.read().decode()
warning(out)
raise Exception('Execution failed.')
def process_include(self, include_file: str or dict, includes: dict,
variables: dict) -> dict:
include_file = self.path_from_root(include_file)
self.check_circular(include_file)
include = Include(**include_file)
self.all_includes.append(include)
include.test = self.prepare_test(include.file, variables,
include.variables)
if include.alias is not None:
includes[include.alias] = include.test
if include.run_on_include:
try:
logger.log_storage.test_start(include_file['file'],
test_type='include')
debug('Run include: ' + str(include.test))
res = include.test.run()
logger.log_storage.test_end(include.test.path,
True,
res,
test_type='include')
return res
except Exception as e:
logger.log_storage.test_end(include.test.path,
False,
str(e),
test_type='include')
if not include.ignore_errors:
raise Exception('Include ' + include.file + ' failed: ' +
str(e))
return include.test.variables
def _prepare_cmd(file: str,
args: list = None,
variables=None,
libraries=None) -> Tuple[List[str], Optional[str]]:
cmd = None
cwd = None
if file.endswith('.py'): # python executable
cmd = ['python', file]
if file.endswith('.js'): # node js executable
cmd = ['node', file]
# TODO Scala/Groovy compilation?
if file.endswith('.java'): # java source file (need to compile)
classpath = _form_classpath_args(libraries)
cmd = ['java'] + classpath + [
_compile_java(file, variables, libraries=libraries)
]
cwd = variables[
'RESOURCES_DIR'] # compiled java class should be run from resources
if file.endswith('.kt'): # kotlin source file (need to compile)
cmd = [
'java', '-jar',
_compile_kotlin(file, variables, libraries=libraries)
]
cwd = variables[
'RESOURCES_DIR'] # compiled java class should be run from resources
if file.endswith('.jar'): # executable jar
cmd = ['java', '-jar', file]
if cmd is None: # local executable or other command
cmd = [file]
if args is not None:
cmd += args
debug(str(cmd))
return cmd, cwd
def _form_request(self, url, variables: dict) -> dict:
headers = dict([(fill_template_str(k, variables),
fill_template_str(v, variables))
for k, v in self.headers.items()])
rq = dict(verify=self.verify,
headers=headers,
files=self.__form_files(variables))
isjson, body = self.__form_body(variables)
debug('http ' + str(self.method) + ' ' + str(url) + ', ' +
str(headers) + ', ' + str(body))
content_type = self.__get_content_type(headers)
if isinstance(
body,
str): # decode all strings to utf to prevents latin-1 errors
body = body.encode('utf-8')
if isjson or isinstance(body,
dict): # contains tojson or dict supplied
if isinstance(body, dict) and content_type == 'application/json':
# json body formed manually via python dict
rq['json'] = body
else: # json string or form-data dict
rq['data'] = body
else: # raw body (or body is None)
rq['data'] = body
rq['timeout'] = self.timeout
return rq
def trigger_dag(aiflow_url, dag_id, dag_config):
url = posixpath.join(aiflow_url,
'api/experimental/dags/{}/dag_runs'.format(dag_id))
r = request('POST',
url,
json=dag_config,
headers={'content-type': 'application/json'})
if r.status_code in [404, 405]: # old airflow, rest api is not supported
debug('Endpoint not found: ' + r.text)
raise OldAirflowVersionException(
'Can\'t trigger dag {}'.format(dag_id))
if r.status_code != 200:
raise Exception('Can\'t trigger dag: {}'.format(r.json()))
debug(r.json())
message = r.json()['message']
if not message.startswith('Created'):
raise Exception('Dag run was not created: ' + message)
# message text expected:
# 'Created <DagRun init_data_sync @ 2020-01-04 13:22:16+00:00: manual__2020-01-04T13:22:16+00:00,
# When triggered with the execution date dag config
# 'Created <DagRun hello_world @ 2019-08-25T14:15:22+00:00: manual__2019-08-25T14:15:22+00:00, externally triggered: True>'
filtered_run_id = [
message_section for message_section in message.split(' ')
if message_section.startswith('manual__')
]
if not filtered_run_id:
raise Exception('Dag run id was not found: ' + message)
return filtered_run_id[0].strip(',')
def run(self, tag=None, raise_stop=False) -> dict:
for step in self.steps:
[action] = step.keys()
ignore_errors = get_or_default('ignore_errors', step[action],
False)
if tag is not None:
step_tag = get_or_default('tag', step[action], None)
if step_tag != tag:
continue
actions = step_factory.get_actions(self.path, step, self.modules)
for action_object in actions:
# override all variables with cmd variables
variables = merge_two_dicts(self.variables, self.override_vars)
action_name = get_action_name(action, action_object, variables)
try:
self.variables = action_object.action(
self.includes, variables)
info('Step ' + action_name + ' OK')
except StopException as e:
if raise_stop:
raise e
debug('Skip ' + action_name + ' due to ' + str(e))
info('Step ' + action_name + ' OK')
return self.variables # stop current test
except Exception as e:
if ignore_errors:
debug('Step ' + action_name +
' failed, but we ignore it')
continue
info('Step ' + action_name + ' failed: ' + str(e))
raise e
return self.variables
def _wait_for_running(url,
dag_id,
execution_date,
timeout,
run_id,
dialect=None,
db_conf=None):
while True:
try:
state = airflow_client.get_run_status(url, dag_id,
execution_date)
except OldAirflowVersionException:
warning(
"Your airflow version does not support rest API method for DAG status. Call backend db directly"
)
state = airflow_db_client.get_dag_run_by_run_ud(
run_id=run_id, conf=db_conf, dialect=dialect)["state"]
debug(state)
if state.lower() != 'running':
return state.lower()
if timeout > 0:
sleep(1)
timeout -= 1
else:
raise Exception('Dag {} still running'.format(dag_id))
def operation(self, variables) -> bool:
body = self.subject[self.body]
source = fill_template(body['of'], variables)
if isinstance(source, list):
elements = source
elif isinstance(source, dict):
elements = source.items()
else:
debug(str(source) + ' not iterable')
return False
results = []
for element in elements:
oper_body = dict([(k, v) for (k, v) in body.items() if k != 'of'])
[next_operator] = oper_body.keys()
if not isinstance(oper_body[next_operator],
dict): # terminator in short form
if next_operator == 'equals':
oper_body[next_operator] = Equals.to_long_form(
'{{ ITEM }}', oper_body[next_operator])
if next_operator == 'contains':
oper_body[next_operator] = Contains.to_long_form(
'{{ ITEM }}', oper_body[next_operator])
next_operation = Operator.find_operator(oper_body)
variables['ITEM'] = element
results.append(next_operation.operation(variables))
return self.operator(results)
def get_run_status(aiflow_url: str, dag_id: str,
execution_date: Union[str, datetime.datetime]) -> str:
"""
Obtain pipeline status depends on dag_id and execution_date
:param aiflow_url:
:param dag_id: unique id for DAG. Name of the pipeline.
:param execution_date: it is datetime object already, not string.
:return: dag state (success, running, failed)
"""
date_format = "%Y-%m-%dT%H:%M:%S"
if isinstance(execution_date, datetime.datetime):
date_fmt = execution_date.strftime(date_format)
else:
date_fmt = execution_date
url = posixpath.join(
aiflow_url,
'api/experimental/dags/{}/dag_runs/{}'.format(dag_id, date_fmt))
r = request('GET', url)
if r.status_code in [404, 405]: # old airflow, rest api is not supported
debug('Endpoint not found: ' + r.text)
raise OldAirflowVersionException(
'Can\'t get dag run status {}'.format(dag_id))
if r.status_code != 200:
raise Exception('Can\'t get run status for {}:{} {}'.format(
dag_id, date_fmt, r.json()))
return r.json()['state']
def _run_test(self,
test: Test,
global_variables: dict,
output: str = 'full',
test_type='test') -> bool:
try:
self.var_holder.prepare_variables(test, global_variables)
logger.log_storage.test_start(test.file, test_type=test_type)
test.check_ignored()
with OptionalOutput(output == 'limited'):
test.run()
info(test_type.capitalize() + ' ' +
cut_path(self.tests_path, test.file) +
logger.green(' passed.'))
logger.log_storage.test_end(test.file, True, test_type=test_type)
return True
except SkipException:
info(test_type.capitalize() + ' ' +
cut_path(self.tests_path, test.file) +
logger.yellow(' skipped.'))
logger.log_storage.test_end(test.file,
True,
end_comment='Skipped',
test_type=test_type)
return True
except Exception as e:
warning(test_type.capitalize() + ' ' +
cut_path(self.tests_path, test.file) +
logger.red(' failed: ') + str(e))
debug(traceback.format_exc())
logger.log_storage.test_end(test.file,
False,
str(e),
test_type=test_type)
return False
def action(self, includes: dict, variables: dict) -> Union[tuple, dict]:
url = fill_template(self.url, variables)
session = Http.sessions.get(self.session, requests.Session())
r = None
try:
r = session.request(self.method, url,
**self._form_request(url, variables))
if self._should_fail: # fail expected
raise RuntimeError('Request expected to fail, but it doesn\'t')
except requests.exceptions.ConnectionError as e:
debug(str(e))
if self._should_fail: # fail expected
return variables
self.__fix_cookies(url, session)
if self.session is not None: # save session if name is specified
Http.sessions[self.session] = session
if r is None:
raise Exception('No response received')
debug(r.text)
try:
response = r.json()
except ValueError:
response = r.text
if self.__check_code(r.status_code, self.code):
raise RuntimeError('Code mismatch: ' + str(r.status_code) +
' vs ' + str(self.code))
return variables, response
def action(self, includes: dict, variables: dict) -> dict:
filled_vars = dict([(k, fill_template_str(v, variables)) for (k, v) in self.variables.items()])
out = fill_template_str(self.include, variables)
test, tag = get_tag(out)
if test not in includes:
error('No include registered for name ' + test)
raise Exception('No include registered for name ' + test)
include = includes[test]
variables = merge_two_dicts(include.variables, merge_two_dicts(variables, filled_vars))
include.variables = try_get_object(fill_template_str(variables, variables))
try:
info('Running {}'.format(test))
logger.log_storage.nested_test_in()
variables = include.run(tag=tag, raise_stop=True)
logger.log_storage.nested_test_out()
except SkipException:
logger.log_storage.nested_test_out()
debug('Include ignored')
return variables
except StopException as e:
logger.log_storage.nested_test_out()
raise e
except Exception as e:
logger.log_storage.nested_test_out()
if not self.ignore_errors:
raise Exception('Step run ' + test + ' failed: ' + str(e))
return variables
def action(self, variables):
container = self.get_container()
res = container.stop()
if self.delete:
debug('Removing {}'.format(container.name))
container.remove()
return res
def check_circular(parent: str, all_includes: nx.DiGraph,
current_include: dict) -> 'Include':
path = current_include['file']
all_includes.add_edge(parent, path)
if list(nx.simple_cycles(all_includes)):
debug(str(all_includes.edges))
raise Exception('Circular dependencies for ' + path)
return Include(**current_include)
def action(self, includes: dict, variables: dict) -> dict or tuple:
cmd = fill_template(self._cmd, variables)
return_code, stdout, stderr = external_utils.run_cmd(cmd.split(' '),
variables,
fill_template(self._path, variables))
if return_code != int(fill_template(self._return_code, variables)):
debug('Process return code {}.\nStderr is {}\nStdout is {}'.format(return_code, stderr, stdout))
raise Exception(stderr)
return variables, stdout
def operation(self, variables: dict):
body = self.subject[self.body]
source = fill_template(self.determine_source(body), variables)
subject = fill_template(body['the'], variables)
result = subject in source
if self.negative:
result = not result
if not result:
debug(str(subject) + ' is not in ' + str(source))
return result
def __init__(self, **kwargs) -> None:
"""
:param kwargs: resources_dir param must exist for the initial load
"""
super().__init__()
# TODO find all submodules of catcher.modules.Module dynamically!
self._modules = {
'compose': DockerCompose(kwargs['resources_dir']),
'requirements': Requirements(kwargs['resources_dir'])
}
debug('Loaded modules: {}'.format(list(self._modules.keys())))
def unpause_dag(aiflow_url, dag_id):
url = posixpath.join(
aiflow_url, 'api/experimental/dags/{}/paused/false'.format(dag_id))
r = request('GET', url)
if r.status_code in [404, 405]: # old airflow, rest api is not supported
debug('Endpoint not found: ' + r.text)
raise OldAirflowVersionException(
'Can\'t unpause the dag {}'.format(dag_id))
if r.status_code != 200:
debug(r.text)
raise Exception('Can\'t unpause dag: {}'.format(dag_id))
def _prepare_include_vars(test, global_variables):
if test.include:
include_vars = try_get_object(
fill_template_str(test.include.variables, global_variables))
else:
include_vars = {}
override_keys = report_override(test.variables, include_vars)
if override_keys:
debug('Include variables override these variables: ' +
str(override_keys))
return include_vars
def __run_actions(self, includes, variables: dict) -> dict:
output = variables
for action in self.do_action:
try:
output = action.action(includes, output)
except Exception as e:
if action.ignore_errors:
debug('{} got {} but we ignore it'.format(
fill_template_str(action.name, variables), e))
break
raise e
return output
def render(source: str, variables: dict) -> str:
template = Template(source)
holder = FiltersFactory()
for filter_mod, value in holder.filters.items():
template.environment.filters[filter_mod] = value
for fun_mod, value in holder.functions.items():
template.globals[fun_mod] = value
try:
return template.render(variables)
except UndefinedError as e:
debug(e.message)
return source
def _read(self, step):
config = step['conf']
from marketorestpython.client import MarketoClient
mc = MarketoClient(config['munchkin_id'], config['client_id'], config['client_secret'], None, None)
filter_values = self._get_field_as_list(step['filter_value'])
fields = self._get_field_as_list(step['fields'])
res = mc.execute(method='get_multiple_leads_by_filter_type',
filterType=step.get('filter_key', 'email'),
filterValues=[str(v) for v in filter_values],
fields=fields,
batchSize=None)
debug('Found {}'.format(res))
return res
def _write(self, step):
config = step['conf']
from marketorestpython.client import MarketoClient
mc = MarketoClient(config['munchkin_id'], config['client_id'], config['client_secret'], None, None)
leads = self._get_field_as_list(step['leads'])
res = mc.execute(method='create_update_leads',
leads=leads,
action=step['action'],
lookupField=step.get('lookupField', 'email'),
asyncProcessing='false',
partitionName='Default')
debug('Write result = {}'.format(res))
return res
def _put_file(self, s3_client, path, content, retry=True):
from botocore.exceptions import ClientError
bucket, filename = self._parse_path(path)
debug('Put {}/{}'.format(bucket, filename))
try:
res = s3_client.put_object(Bucket=bucket, Key=filename, Body=content)
return self._check_response(res)
except ClientError as e:
if retry and hasattr(e, 'response') and 'Error' in e.response and 'Code' in e.response['Error']:
if e.response['Error']['Code'] == 'NoSuchBucket':
res = s3_client.create_bucket(Bucket=bucket)
self._check_response(res)
return self._put_file(s3_client, path, content, False)
raise e
def _delete(self, step):
config = step['conf']
from marketorestpython.client import MarketoClient
mc = MarketoClient(config['munchkin_id'], config['client_id'], config['client_secret'], None, None)
having = self._get_field_as_list(step['having'])
found = mc.execute(method='get_multiple_leads_by_filter_type',
filterType=step.get('by', 'email'),
filterValues=[str(v) for v in having],
fields=['id'],
batchSize=None)
debug('Found {}'.format(found))
res = mc.execute(method='delete_lead', id=[lead['id'] for lead in found])
debug(res)
return res
def __automap_table(cls, table_name: str, engine):
from sqlalchemy.ext.automap import automap_base
schema = None
if '.' in table_name:
[schema, table_name] = table_name.split('.')
debug('Mapping {}. It can take a while'.format(table_name))
Base = automap_base()
Base.prepare(engine, reflect=True, schema=schema)
try:
return Base.classes[table_name]
except KeyError:
raise Exception('Can\'t map table without primary key.')
def add_package_to_globals(package: str,
glob=None,
warn_missing_package=True) -> dict:
if glob is None:
glob = globals()
try:
mod = importlib.import_module(package)
glob[package] = mod
except ImportError as e:
if warn_missing_package:
warning(str(e))
else:
debug(str(e))
return glob
def action(self, includes: dict, variables: dict) -> dict or tuple:
cmd = fill_template(self._cmd, variables)
process = subprocess.Popen(cmd.split(' '),
cwd=fill_template(self._path, variables),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
stdout, stderr = process.communicate()
if process.returncode != int(
fill_template(self._return_code, variables)):
debug('Process return code {}.\nStderr is {}\nStdout is {}'.format(
process.returncode, stderr, stdout))
raise Exception(stderr)
return variables, stdout
def gather_response(cursor):
try:
response = cursor.fetchall()
if len(
response
) == 1: # for only one value select * from .. where id = 1 -> [('a', 1, 2)]
response = response[0]
if len(
response
) == 1: # for only one value select count(*) from ... -> (2,)
response = response[0]
return response
except Exception as e:
debug('Execution error {}'.format(e))
return None
def operation(self, variables: dict) -> bool:
if isinstance(self.subject, str):
subject = fill_template(self.subject, variables)
source = True
else:
body = self.subject[self.body]
if isinstance(body, str):
body = Equals.to_long_form(body, True)
subject = fill_template(body['the'], variables)
source = fill_template(self.determine_source(body), variables)
result = source == subject
if self.negative:
result = not result
if not result:
debug(str(source) + ' is not equal to ' + str(subject))
return result