def test_invoke_nested_handlers(handler_run):
"""context: test nested handlers invocation"""
# Run a number of EmptyHandlers
context.invoke_handler([
_get_empty_handler_conf(p3=50),
[
_get_empty_handler_conf(),
_get_empty_handler_conf(),
_get_empty_handler_conf()
],
_get_empty_handler_conf(),
])
# Verify handler runs
print(handler_run.call_args_list)
assert handler_run.call_count == 5
assert \
handler_run.call_args_list == [
call({'p1': 10, 'p2': 20, 'p3': 50, 'module': 'etl.tests.dummy_handlers', 'class': 'EmptyHandler'}),
# Here we're going into a sub-pipeline
call({'p1': 10, 'p2': 20, 'p3': 51, 'module': 'etl.tests.dummy_handlers', 'class': 'EmptyHandler'}),
call({'p1': 10, 'p2': 20, 'p3': 52, 'module': 'etl.tests.dummy_handlers', 'class': 'EmptyHandler'}),
call({'p1': 10, 'p2': 20, 'p3': 53, 'module': 'etl.tests.dummy_handlers', 'class': 'EmptyHandler'}),
# Here we come back out again
call({'p1': 10, 'p2': 20, 'p3': 54, 'module': 'etl.tests.dummy_handlers', 'class': 'EmptyHandler'}),
]
def run(self, config):
"""Override the abstract method of the base class."""
# Fetch the config parameters
input_param = config['input_param', 'data']
output_param = config['output_param', 'data']
passthrough_params = config['passthrough_params', None]
chomp = bool(config['chomp', True])
skip_blank_lines = bool(config['skip_blank_lines', True])
# Prepare child params
if passthrough_params is None:
sub_params = {}
else:
sub_params = {k: config[k] for k in passthrough_params}
# Read the input data
line_num = 0
for line in config.lines(input_param):
# Strip the linebreak if needed
if chomp:
line = line.rstrip('\r\n')
# Invoke the handler
if not skip_blank_lines or len(line) > 0:
line_num += 1
logger.log('Processing line #{}'.format(line_num))
sub_params[output_param] = line
context.invoke_handler(config['handler'], sub_params)
def run(self, config):
"""Override the abstract method of the base class."""
# Get attributes from config
base_url = config['base_url']
output_param = config['output_param', 'data']
verify_cert = bool(config['verify_cert', True])
detect_compressed = bool(config['detect_compressed', False])
file_defs = config['file_defs', None]
passthrough_params = config['passthrough_params', None]
username = config['username', None]
password = config['password', '']
encoding = config['encoding', 'utf-8']
# Substitute parameter values
base_url = base_url.format(**config)
if username is not None:
username = username.format(**config)
if password is not None:
password = password.format(**config)
# If there are no file definitions
if file_defs is None:
# Fetch and return the page at base_url
return {
output_param:
self.fetch(base_url, True, verify_cert, detect_compressed,
username, password, encoding)
}
# File definitions present
else:
# Prepare child params
if passthrough_params is None:
sub_params = {}
else:
sub_params = {k: config[k] for k in passthrough_params}
# Iterate through file definitions
for file_def in file_defs:
# Fetch definition details
fd_name = file_def['name']
fd_reqd = bool(file_def['required', True])
handler_conf = file_def['handler']
# Perform config variable substitution
file_name = fd_name.format(**config)
# Download file content
file_content = self.fetch(base_url + file_name, fd_reqd,
verify_cert, detect_compressed,
username, password, encoding)
# Run the handler, if the file exists
if file_content is not None:
sub_params['file_name'] = file_name
sub_params[output_param] = file_content
context.invoke_handler(handler_conf, sub_params)
def test_invoke_and_include_relative(handler_run):
"""context: test config file inclusion by relative path"""
with configure('empty.json', None, False):
context.invoke_handler('test_conf.include.json')
# Verify handler runs
print(handler_run.call_args_list)
handler_run.assert_called_once_with({
'module': 'etl.tests.dummy_handlers',
'class': 'EmptyHandler',
'p1': 100
})
def test_invoke_single_handler(handler_run):
"""context: test single handler invocation"""
context.invoke_handler(_get_empty_handler_conf(), {'p1': 11, 'p3': 30})
# Verify handler has run
handler_run.assert_called_once_with({
'module': 'etl.tests.dummy_handlers',
'class': 'EmptyHandler',
'p1': 10, # Local handler config must override parent config
'p2': 20, # Local config must be passed to handler
'p3': 30 # Parent config must be passed to handler
})
def test_invoke_and_include_absolute(handler_run):
"""context: test config file inclusion by absolute path"""
with configure('empty.json', None, False):
context.invoke_handler(
path.join(path.dirname(__file__), 'test_conf_2.include.json'))
# Verify handler runs
print(handler_run.call_args_list)
handler_run.assert_called_once_with({
'module': 'etl.tests.dummy_handlers',
'class': 'EmptyHandler',
'p1': 200
})
def test_globals(handler_run):
"""context: test globals specification in config file"""
with configure('globals.json', None, False, {}):
# Run a handler
context.invoke_handler(_get_empty_handler_conf())
# Verify the handler has run
handler_run.assert_called_once_with({
'module': 'etl.tests.dummy_handlers',
'class': 'EmptyHandler',
'p1': 10,
'p2': 20,
# Must have been called with these globals
'param_a': 'A',
'param_b': 14
})
def test_invoke_chained_handlers(handler_run):
"""context: test chained handlers invocation"""
# Run a chain of two EmptyHandlers
context.invoke_handler(
[_get_empty_handler_conf(),
_get_empty_handler_conf()])
# Verify handler runs
print(handler_run.call_args_list)
assert handler_run.call_count == 2
assert \
handler_run.call_args_list == [
call({'p1': 10, 'p2': 20, 'module': 'etl.tests.dummy_handlers', 'class': 'EmptyHandler'}),
call({'p1': 10, 'p2': 100, 'module': 'etl.tests.dummy_handlers', 'class': 'EmptyHandler'})
], \
'p2 gets multiplied by 5'
def test_param_block(handler_run):
"""context: test publishing parameter blocks on pipeline"""
# Run a chain of two EmptyHandlers
context.invoke_handler(
[config.Config({
'a': 42,
'b': 43
}), _get_empty_handler_conf()])
# Verify handler has run
handler_run.assert_called_once_with({
'module': 'etl.tests.dummy_handlers',
'class': 'EmptyHandler',
'p1': 10,
'p2': 20,
'a': 42,
'b': 43,
})
def test_global_overrides(handler_run):
"""context: test globals specification overrides"""
with configure('globals.json', None, False, {
'param_b': 'TEST',
'param_c': 700
}):
# Run a handler
context.invoke_handler(_get_empty_handler_conf())
# Verify the handler has run
handler_run.assert_called_once_with({
'module': 'etl.tests.dummy_handlers',
'class': 'EmptyHandler',
'p1': 10,
'p2': 20,
'param_a': 'A', # Unchanged global param
'param_b': 'TEST', # Overridden global param
'param_c': 700 # New global param
})
def test_invoke_misconfiguration():
"""context: test invoking handler with config of a wrong type"""
context.invoke_handler(42)