def _invoke_with(base_conf, extra_conf, mappings, dry_run, mock_context, mock_inserter): """Run the handler with specified config.""" # Create a fake DB connection mockdb = MagicMock() mockdb.get_param_placeholder.return_value = '.' # Required to allow building SQL statements # Create a fake DBArrayInserter mockins = MagicMock() # This instance will be returned by mocked DBArrayInserter() constructor mock_inserter.return_value = mockins # Set up our fake context mock_context.dry_run_mode = dry_run mock_context.dry_run_prefix = '' mock_context.get_db_connection.return_value = mockdb # Prepare config conf = config.Config( { 'target_database': 'mockdb', 'target_table': 'TBL', 'column_mappings': [config.Config(m) for m in mappings] }, **base_conf) conf.update(**extra_conf) # Instantiate and run the handler db_uploader.Handler().run(conf) # Verify our DB was requested mock_context.get_db_connection.assert_called_with('mockdb') # Verify commit has [not] been done assert mockdb.commit.call_count == 0 if dry_run else 1 return mockins
def test_substring_replace_limit(): """handlers.str_replacer: test substring replace limit""" result = str_replacer.Handler().run(config.Config( _config, rules=[ config.Config({'search': 'e', 'replace': 'E', 'count': 3}) ])) print(result) assert result['out'] == 'somE tExt\nwhich wE will use\nto test our\nstring replacer'
def test_regex_replace(): """handlers.str_replacer: test regex replace""" result = str_replacer.Handler().run(config.Config( _config, rules=[ # Let's swap two last letters of each word config.Config({'search': r'(\w)(\w)\b', 'replace': r'\2\1', 'is_regex': True}) ])) print(result) assert result['out'] == 'soem tetx\nwhihc ew will ues\not tets oru\nstrign replacre'
def test_substring_replace(): """handlers.str_replacer: test substring replace""" result = str_replacer.Handler().run(config.Config( _config, rules=[ config.Config({'search': 'e', 'replace': '*'}), config.Config({'search': 't*', 'replace': 'mi'}) ])) print(result) assert result['out'] == 'som* mixt\nwhich w* will us*\nto mist our\nstring r*plac*r'
def test_regex_replace_limit(): """handlers.str_replacer: test regex replace limit""" result = str_replacer.Handler().run(config.Config( _config, rules=[ # Swap two last letters of each word for first 6 words config.Config({'search': r'(\w)(\w)\b', 'replace': r'\2\1', 'is_regex': True, 'count': 6}) ])) print(result) assert result['out'] == 'soem tetx\nwhihc ew will ues\nto test our\nstring replacer'
def test_multi_regex_replace(): """handlers.str_replacer: test multiple regex replace""" result = str_replacer.Handler().run(config.Config( _config, rules=[ # Swap two last letters of each word config.Config({'search': r'(\w)(\w)\b', 'replace': r'\2\1', 'is_regex': True}), # Do it twice, and we're back to where we started config.Config({'search': r'(\w)(\w)\b', 'replace': r'\2\1', 'is_regex': True}) ])) print(result) assert result['out'] == _config['in']
def _get_mock_process_config(name, handler, stop_on_error) -> config.Config: """Create and return mock process configuration.""" return MagicMock( return_value=config.Config({ 'processes': [ config.Config({ 'name': name, 'handler': handler, 'stop_on_error': stop_on_error }) ] }))
def test_criteria_list(): """handlers.line_filter: test multiple criteria""" _invoke_with( { 'criteria': [ config.Config({ 'search': r'[agx]\w{2}\d$', 'is_regex': True }), config.Config({ 'search': 'hi', 'is_regex': False }) ] }, 'ghi1\n')
def _invoke_with(sql, conf, mock_context): """Run the handler with the specified config, in a mocked context. Return a tuple consisting of the handler's result and the mocked DB cursor object.""" # Create a fake DB connection and a cursor mock_db = MagicMock() mock_db.cursor.return_value = mock_cursor = MagicMock() # Mock iterator result and column headers of the cursor mock_cursor.__iter__.return_value = [['abc', 'def', 10, True], ['ghi', 'jkl', 42, False]] mock_cursor.description = [['str1'], ['str2'], ['int'], ['bool']] # Set up our fake context mock_context.get_db_connection.return_value = mock_db # Run the handler result = sql_query.Handler().run( config.Config({ 'database': 'mockdb', 'sql': sql }, **conf)) # Verify our DB was requested mock_context.get_db_connection.assert_called_once_with('mockdb') # Check the cursor was requested and closed after use mock_db.cursor.assert_called_once_with() mock_cursor.close.assert_called_once_with() return result, mock_cursor
def test_start_line(): """handlers.line_merger: test start_line""" result = line_merger.Handler().run(config.Config(_config, start_line=2)) print(result) assert result['out'] == \ 'line_2 line_3\n' \ 'line_4\n' \ 'line_6 line_7\n'
def _get_empty_handler_conf(**additional_conf): """Return configuration for EmptyHandler.""" return config.Config( { 'module': 'etl.tests.dummy_handlers', 'class': 'EmptyHandler', 'p1': 10, 'p2': 20 }, **additional_conf)
def test_criteria_negation(): """handlers.line_filter: test criteria negation""" _invoke_with( { 'criteria': [ config.Config({ 'search': r'[agx]\w{2}\d$', 'is_regex': True, 'negate': False }), config.Config({ 'search': 'hi', 'is_regex': False, 'negate': True }) ] }, 'abc0\n' 'xyz2\n')
def test_simple_auth(): """handlers.http_loader: test simple invocation with authentication""" with patch('etl.handlers.http_loader.utils.http_fetch') as mocked_fetch: # Instantiate and run the handler h = http_loader.Handler() h.run(config.Config({'base_url': _http_base_url, 'username': '******', 'password': '******'})) # Check the result mocked_fetch.assert_called_once_with(_http_base_url, True, True, 'JOHN', 'DOE')
def test_delimiter(): """handlers.line_merger: test delimiter""" result = line_merger.Handler().run(config.Config(_config, delimiter=':)')) print(result) assert result['out'] == \ 'line_1:)line_2\n' \ ' line_3:)line_4\n' \ ':)line_6 \n' \ 'line_7\n'
def test_text_output(): """handlers.xslt: test plain-text output""" result = xslt.Handler().run(config.Config(_config, xslt_param='xslt_text')) print(result) assert result['out'] == \ 'A,B,C,4,text_1\n' \ 'D,E,F,5,text_2\n' \ 'G,H,I,6,text_3\n' \ 'J,K,L,7,text_4\n'
def test_skip_blank_lines(): """handlers.line_merger: test skip_blank_lines""" result = line_merger.Handler().run( config.Config(_config, skip_blank_lines=True)) print(result) assert result['out'] == \ 'line_1line_2\n' \ ' line_3line_4\n' \ 'line_6 line_7\n'
def test_simple_ignore_invalid_cert(): """handlers.http_loader: test simple invocation with ignoring invalid SSL certificate""" with patch('etl.handlers.http_loader.utils.http_fetch') as mocked_fetch: # Instantiate and run the handler h = http_loader.Handler() h.run(config.Config({'base_url': _http_base_url, 'verify_cert': False})) # Check the result mocked_fetch.assert_called_once_with(_http_base_url, True, False, None, '')
def test_defaults(): """handlers.line_merger: test default invocation""" result = line_merger.Handler().run(config.Config(_config)) print(result) assert result['out'] == \ 'line_1line_2\n' \ ' line_3line_4\n' \ 'line_6 \n' \ 'line_7\n'
def test_update(): """config: test update""" c = config.Config({'a': 1, 'b': 'xxx', 'c': False}) c.update({'b': 400, 'c': 17}, d='new', e={'nested': 5}) assert c['a'] == 1 assert c['b'] == 400 assert c['c'] == 17 assert c['d'] == 'new' assert type(c['e']) is config.Config assert c['e']['nested'] == 5
def _invoke_simple(conf: dict, out_param: str, fetched_data: bytes, expected_data: str): """Invoke the handler in 'simple' mode and check the result.""" with patch('etl.handlers.http_loader.utils.http_fetch', return_value=fetched_data) as mocked_fetch: # Instantiate and run the handler h = http_loader.Handler() result = h.run(config.Config(conf)) # Check the result assert result[out_param] == expected_data mocked_fetch.assert_called_once_with(_http_base_url, True, True, None, '')
def test_xml_output_unformatted(): """handlers.xslt: test unformatted XML output""" result = xslt.Handler().run(config.Config(_config, xslt_param='xslt_xml')) print(result) assert result['out'] == \ '<data>' \ '<item><type>A</type><sort>B</sort><name>C</name><value>4</value><text>text_1</text></item>' \ '<item><type>D</type><sort>E</sort><name>F</name><value>5</value><text>text_2</text></item>' \ '<item><type>G</type><sort>H</sort><name>I</name><value>6</value><text>text_3</text></item>' \ '<item><type>J</type><sort>K</sort><name>L</name><value>7</value><text>text_4</text></item>' \ '</data>'
def test_reverse(): """handlers.line_sorter: test reverse sort""" result = line_sorter.Handler().run(config.Config(_config, reverse=True)) print(result) assert result['out'] == \ 'zyx\n' \ 'xyz\n' \ 'def\n' \ 'abc\n' \ '%$#\n' \ '\n'
def test_defaults(): """handlers.line_sorter: test default sort""" result = line_sorter.Handler().run(config.Config(_config)) print(result) assert result['out'] == \ '\n' \ '%$#\n' \ 'abc\n' \ 'def\n' \ 'xyz\n' \ 'zyx\n'
def test_single_param_substitution(): """handlers.line_filter: test single criterion with parameter substitution""" _invoke_with( { 'NEEDLE': 'i1', 'criteria': config.Config({ 'search': '{NEEDLE}', 'substitute_params': True }) }, 'ghi1\n')
def test_existing(): """handlers.file_reader: test reading existing file""" # Instantiate and run the handler against this very file h = file_reader.Handler() result = h.run( config.Config({ 'this_file': __file__, 'file_name': '{this_file}', # Also test parameter substitution 'output_param': 'the_text' })) # Check for this text assert '=WHATEVER=' in result['the_text']
def _invoke_with(extra_conf, expected_output, input_param='data', output_param='data'): """Invoke the handler with the specified config, and validates the result.""" # Instantiate and run the handler result = line_filter.Handler().run( config.Config({input_param: INPUT_DATA}, **extra_conf)) # Validate the result print(result) assert result[output_param] == expected_output return result
def test_filedefs_out_param(): """handlers.http_loader: test file defs with output param mapping""" mocks = _invoke_filedefs({ 'base_url': _http_base_url, 'output_param': 'TEST', 'file_defs': [ config.Config({ 'name': 'file_B', 'handler': {} }) ] }) # Check handler invocation mocks[1].assert_called_once_with({}, {'file_name': 'file_B', 'TEST': _http_content_str})
def test_rejected_lines(): """handlers.line_filter: test collecting rejected lines""" result = _invoke_with( { 'criteria': config.Config({ 'search': r'.*[12]$', 'is_regex': True }), 'rejected_param': 'TRASH' }, 'ghi1\n' 'jkl1\n' 'xyz2\n') # Check the rejected lines assert result['TRASH'] == 'abc0\ndef0\n\n'
def test_filedefs_default(): """handlers.http_loader: test file defs with defaults""" mocks = _invoke_filedefs({ 'base_url': _http_base_url, 'file_defs': [ config.Config({ 'name': 'file_A', 'handler': {} }) ] }) # Check http_fetch() invocation mocks[0].assert_called_once_with(_http_base_url + 'file_A', True, True, None, '') # Check handler invocation mocks[1].assert_called_once_with({}, {'file_name': 'file_A', 'data': _http_content_str})
def setup_module(): global _conf _conf = config.Config({ 'key_str': 'value_str', 'key_int': 42, 'key_bool': True, 'key_list': [1, 2, 3, 4], 'key_text': 'several lines\nas textual\nvalue', 'key_dict': { 'nested_int_key': 1, 'nested_dict_key': { 'nested2_key': 1 } } })