def _invoke_with(base_conf, extra_conf, mappings, dry_run, mock_context,
mock_inserter):
"""Run the handler with specified config."""
# Create a fake DB connection
mockdb = MagicMock()
mockdb.get_param_placeholder.return_value = '.' # Required to allow building SQL statements
# Create a fake DBArrayInserter
mockins = MagicMock()
# This instance will be returned by mocked DBArrayInserter() constructor
mock_inserter.return_value = mockins
# Set up our fake context
mock_context.dry_run_mode = dry_run
mock_context.dry_run_prefix = ''
mock_context.get_db_connection.return_value = mockdb
# Prepare config
conf = config.Config(
{
'target_database': 'mockdb',
'target_table': 'TBL',
'column_mappings': [config.Config(m) for m in mappings]
}, **base_conf)
conf.update(**extra_conf)
# Instantiate and run the handler
db_uploader.Handler().run(conf)
# Verify our DB was requested
mock_context.get_db_connection.assert_called_with('mockdb')
# Verify commit has [not] been done
assert mockdb.commit.call_count == 0 if dry_run else 1
return mockins
def test_substring_replace_limit():
"""handlers.str_replacer: test substring replace limit"""
result = str_replacer.Handler().run(config.Config(
_config,
rules=[
config.Config({'search': 'e', 'replace': 'E', 'count': 3})
]))
print(result)
assert result['out'] == 'somE tExt\nwhich wE will use\nto test our\nstring replacer'
def test_regex_replace():
"""handlers.str_replacer: test regex replace"""
result = str_replacer.Handler().run(config.Config(
_config,
rules=[
# Let's swap two last letters of each word
config.Config({'search': r'(\w)(\w)\b', 'replace': r'\2\1', 'is_regex': True})
]))
print(result)
assert result['out'] == 'soem tetx\nwhihc ew will ues\not tets oru\nstrign replacre'
def test_substring_replace():
"""handlers.str_replacer: test substring replace"""
result = str_replacer.Handler().run(config.Config(
_config,
rules=[
config.Config({'search': 'e', 'replace': '*'}),
config.Config({'search': 't*', 'replace': 'mi'})
]))
print(result)
assert result['out'] == 'som* mixt\nwhich w* will us*\nto mist our\nstring r*plac*r'
def test_regex_replace_limit():
"""handlers.str_replacer: test regex replace limit"""
result = str_replacer.Handler().run(config.Config(
_config,
rules=[
# Swap two last letters of each word for first 6 words
config.Config({'search': r'(\w)(\w)\b', 'replace': r'\2\1', 'is_regex': True, 'count': 6})
]))
print(result)
assert result['out'] == 'soem tetx\nwhihc ew will ues\nto test our\nstring replacer'
def test_multi_regex_replace():
"""handlers.str_replacer: test multiple regex replace"""
result = str_replacer.Handler().run(config.Config(
_config,
rules=[
# Swap two last letters of each word
config.Config({'search': r'(\w)(\w)\b', 'replace': r'\2\1', 'is_regex': True}),
# Do it twice, and we're back to where we started
config.Config({'search': r'(\w)(\w)\b', 'replace': r'\2\1', 'is_regex': True})
]))
print(result)
assert result['out'] == _config['in']
def _get_mock_process_config(name, handler, stop_on_error) -> config.Config:
"""Create and return mock process configuration."""
return MagicMock(
return_value=config.Config({
'processes': [
config.Config({
'name': name,
'handler': handler,
'stop_on_error': stop_on_error
})
]
}))
def test_criteria_list():
"""handlers.line_filter: test multiple criteria"""
_invoke_with(
{
'criteria': [
config.Config({
'search': r'[agx]\w{2}\d$',
'is_regex': True
}),
config.Config({
'search': 'hi',
'is_regex': False
})
]
}, 'ghi1\n')
def _invoke_with(sql, conf, mock_context):
"""Run the handler with the specified config, in a mocked context. Return a tuple consisting of the handler's result
and the mocked DB cursor object."""
# Create a fake DB connection and a cursor
mock_db = MagicMock()
mock_db.cursor.return_value = mock_cursor = MagicMock()
# Mock iterator result and column headers of the cursor
mock_cursor.__iter__.return_value = [['abc', 'def', 10, True],
['ghi', 'jkl', 42, False]]
mock_cursor.description = [['str1'], ['str2'], ['int'], ['bool']]
# Set up our fake context
mock_context.get_db_connection.return_value = mock_db
# Run the handler
result = sql_query.Handler().run(
config.Config({
'database': 'mockdb',
'sql': sql
}, **conf))
# Verify our DB was requested
mock_context.get_db_connection.assert_called_once_with('mockdb')
# Check the cursor was requested and closed after use
mock_db.cursor.assert_called_once_with()
mock_cursor.close.assert_called_once_with()
return result, mock_cursor
def test_start_line():
"""handlers.line_merger: test start_line"""
result = line_merger.Handler().run(config.Config(_config, start_line=2))
print(result)
assert result['out'] == \
'line_2 line_3\n' \
'line_4\n' \
'line_6 line_7\n'
def _get_empty_handler_conf(**additional_conf):
"""Return configuration for EmptyHandler."""
return config.Config(
{
'module': 'etl.tests.dummy_handlers',
'class': 'EmptyHandler',
'p1': 10,
'p2': 20
}, **additional_conf)
def test_criteria_negation():
"""handlers.line_filter: test criteria negation"""
_invoke_with(
{
'criteria': [
config.Config({
'search': r'[agx]\w{2}\d$',
'is_regex': True,
'negate': False
}),
config.Config({
'search': 'hi',
'is_regex': False,
'negate': True
})
]
}, 'abc0\n'
'xyz2\n')
def test_simple_auth():
"""handlers.http_loader: test simple invocation with authentication"""
with patch('etl.handlers.http_loader.utils.http_fetch') as mocked_fetch:
# Instantiate and run the handler
h = http_loader.Handler()
h.run(config.Config({'base_url': _http_base_url, 'username': '******', 'password': '******'}))
# Check the result
mocked_fetch.assert_called_once_with(_http_base_url, True, True, 'JOHN', 'DOE')
def test_delimiter():
"""handlers.line_merger: test delimiter"""
result = line_merger.Handler().run(config.Config(_config, delimiter=':)'))
print(result)
assert result['out'] == \
'line_1:)line_2\n' \
' line_3:)line_4\n' \
':)line_6 \n' \
'line_7\n'
def test_text_output():
"""handlers.xslt: test plain-text output"""
result = xslt.Handler().run(config.Config(_config, xslt_param='xslt_text'))
print(result)
assert result['out'] == \
'A,B,C,4,text_1\n' \
'D,E,F,5,text_2\n' \
'G,H,I,6,text_3\n' \
'J,K,L,7,text_4\n'
def test_skip_blank_lines():
"""handlers.line_merger: test skip_blank_lines"""
result = line_merger.Handler().run(
config.Config(_config, skip_blank_lines=True))
print(result)
assert result['out'] == \
'line_1line_2\n' \
' line_3line_4\n' \
'line_6 line_7\n'
def test_simple_ignore_invalid_cert():
"""handlers.http_loader: test simple invocation with ignoring invalid SSL certificate"""
with patch('etl.handlers.http_loader.utils.http_fetch') as mocked_fetch:
# Instantiate and run the handler
h = http_loader.Handler()
h.run(config.Config({'base_url': _http_base_url, 'verify_cert': False}))
# Check the result
mocked_fetch.assert_called_once_with(_http_base_url, True, False, None, '')
def test_defaults():
"""handlers.line_merger: test default invocation"""
result = line_merger.Handler().run(config.Config(_config))
print(result)
assert result['out'] == \
'line_1line_2\n' \
' line_3line_4\n' \
'line_6 \n' \
'line_7\n'
def test_update():
"""config: test update"""
c = config.Config({'a': 1, 'b': 'xxx', 'c': False})
c.update({'b': 400, 'c': 17}, d='new', e={'nested': 5})
assert c['a'] == 1
assert c['b'] == 400
assert c['c'] == 17
assert c['d'] == 'new'
assert type(c['e']) is config.Config
assert c['e']['nested'] == 5
def _invoke_simple(conf: dict, out_param: str, fetched_data: bytes, expected_data: str):
"""Invoke the handler in 'simple' mode and check the result."""
with patch('etl.handlers.http_loader.utils.http_fetch', return_value=fetched_data) as mocked_fetch:
# Instantiate and run the handler
h = http_loader.Handler()
result = h.run(config.Config(conf))
# Check the result
assert result[out_param] == expected_data
mocked_fetch.assert_called_once_with(_http_base_url, True, True, None, '')
def test_xml_output_unformatted():
"""handlers.xslt: test unformatted XML output"""
result = xslt.Handler().run(config.Config(_config, xslt_param='xslt_xml'))
print(result)
assert result['out'] == \
'<data>' \
'<item><type>A</type><sort>B</sort><name>C</name><value>4</value><text>text_1</text></item>' \
'<item><type>D</type><sort>E</sort><name>F</name><value>5</value><text>text_2</text></item>' \
'<item><type>G</type><sort>H</sort><name>I</name><value>6</value><text>text_3</text></item>' \
'<item><type>J</type><sort>K</sort><name>L</name><value>7</value><text>text_4</text></item>' \
'</data>'
def test_reverse():
"""handlers.line_sorter: test reverse sort"""
result = line_sorter.Handler().run(config.Config(_config, reverse=True))
print(result)
assert result['out'] == \
'zyx\n' \
'xyz\n' \
'def\n' \
'abc\n' \
'%$#\n' \
'\n'
def test_defaults():
"""handlers.line_sorter: test default sort"""
result = line_sorter.Handler().run(config.Config(_config))
print(result)
assert result['out'] == \
'\n' \
'%$#\n' \
'abc\n' \
'def\n' \
'xyz\n' \
'zyx\n'
def test_single_param_substitution():
"""handlers.line_filter: test single criterion with parameter substitution"""
_invoke_with(
{
'NEEDLE':
'i1',
'criteria':
config.Config({
'search': '{NEEDLE}',
'substitute_params': True
})
}, 'ghi1\n')
def test_existing():
"""handlers.file_reader: test reading existing file"""
# Instantiate and run the handler against this very file
h = file_reader.Handler()
result = h.run(
config.Config({
'this_file': __file__,
'file_name': '{this_file}', # Also test parameter substitution
'output_param': 'the_text'
}))
# Check for this text
assert '=WHATEVER=' in result['the_text']
def _invoke_with(extra_conf,
expected_output,
input_param='data',
output_param='data'):
"""Invoke the handler with the specified config, and validates the result."""
# Instantiate and run the handler
result = line_filter.Handler().run(
config.Config({input_param: INPUT_DATA}, **extra_conf))
# Validate the result
print(result)
assert result[output_param] == expected_output
return result
def test_filedefs_out_param():
"""handlers.http_loader: test file defs with output param mapping"""
mocks = _invoke_filedefs({
'base_url': _http_base_url,
'output_param': 'TEST',
'file_defs': [
config.Config({
'name': 'file_B',
'handler': {}
})
]
})
# Check handler invocation
mocks[1].assert_called_once_with({}, {'file_name': 'file_B', 'TEST': _http_content_str})
def test_rejected_lines():
"""handlers.line_filter: test collecting rejected lines"""
result = _invoke_with(
{
'criteria': config.Config({
'search': r'.*[12]$',
'is_regex': True
}),
'rejected_param': 'TRASH'
}, 'ghi1\n'
'jkl1\n'
'xyz2\n')
# Check the rejected lines
assert result['TRASH'] == 'abc0\ndef0\n\n'
def test_filedefs_default():
"""handlers.http_loader: test file defs with defaults"""
mocks = _invoke_filedefs({
'base_url': _http_base_url,
'file_defs': [
config.Config({
'name': 'file_A',
'handler': {}
})
]
})
# Check http_fetch() invocation
mocks[0].assert_called_once_with(_http_base_url + 'file_A', True, True, None, '')
# Check handler invocation
mocks[1].assert_called_once_with({}, {'file_name': 'file_A', 'data': _http_content_str})
def setup_module():
global _conf
_conf = config.Config({
'key_str': 'value_str',
'key_int': 42,
'key_bool': True,
'key_list': [1, 2, 3, 4],
'key_text': 'several lines\nas textual\nvalue',
'key_dict': {
'nested_int_key': 1,
'nested_dict_key': {
'nested2_key': 1
}
}
})
