def test_vertica_import_options_datetimeformat(self): # Vertica doesn't currently allow any configurability on # input datetimeformat. Check again before adding any test cases # here! should_raise = { 'YYYY-MM-DD HH:MI:SS': True, 'YYYY-MM-DD HH24:MI:SS': False, 'MM/DD/YY HH24:MI': True, 'YYYY-MM-DD HH12:MI AM': True, } for datetimeformat in DATETIME_CASES: records_format = DelimitedRecordsFormat(variant='vertica', hints={ 'datetimeformat': datetimeformat, }) unhandled_hints = set(records_format.hints) processing_instructions = ProcessingInstructions( max_failure_rows=123) load_plan = RecordsLoadPlan( processing_instructions=processing_instructions, records_format=records_format) try: vertica_import_options(unhandled_hints, load_plan) except NotImplementedError: if should_raise[datetimeformat]: pass else: self.fail()
def test_quote_all_with_doublequote(self): vertica_format = DelimitedRecordsFormat(variant='csv', hints={ 'quoting': 'all' }) processing_instructions = ProcessingInstructions() load_plan = RecordsLoadPlan(processing_instructions=processing_instructions, records_format=vertica_format) unhandled_hints = set(load_plan.records_format.hints.keys()) with self.assertRaisesRegexp(NotImplementedError, r"Implement hint doublequote=True or try again with " "fail_if_cant_handle_hint=False"): vertica_import_options(unhandled_hints, load_plan)
def test_weird_timeonlyformat(self): vertica_format = DelimitedRecordsFormat(variant='dumb', hints={ 'timeonlyformat': 'something else' }) processing_instructions = ProcessingInstructions() load_plan = RecordsLoadPlan(processing_instructions=processing_instructions, records_format=vertica_format) unhandled_hints = set(load_plan.records_format.hints.keys()) with self.assertRaisesRegexp(NotImplementedError, "Implement hint timeonlyformat='something else' or try again " "with fail_if_cant_handle_hint=False"): vertica_import_options(unhandled_hints, load_plan)
def test_christmas_tree_format_1_permissive(self): vertica_format = DelimitedRecordsFormat(variant='dumb', hints=christmas_tree_format_1_hints) processing_instructions = ProcessingInstructions(fail_if_cant_handle_hint=False) load_plan = RecordsLoadPlan(processing_instructions=processing_instructions, records_format=vertica_format) unhandled_hints = set(load_plan.records_format.hints.keys()) with patch.object(driver_logger, 'warning') as mock_warning: options = vertica_import_options(unhandled_hints, load_plan) expected_options = { 'abort_on_error': True, 'delimiter': '\x01', 'enforcelength': True, 'error_tolerance': False, 'escape_as': '\\', 'load_method': 'AUTO', 'no_commit': False, 'null_as': None, 'record_terminator': '\x02', 'rejectmax': 1, 'skip': 1, 'trailing_nullcols': False, } self.assertDictEqual(options, expected_options) self.assertListEqual(mock_warning.mock_calls, [call("Ignoring hint compression = 'LZO'"), call("Ignoring hint quoting = 'nonnumeric'")]) self.assertEqual(unhandled_hints, set())
def test_vertica_format_permissive(self): vertica_format = DelimitedRecordsFormat(variant='vertica') processing_instructions = ProcessingInstructions(fail_if_row_invalid=False) load_plan = RecordsLoadPlan(processing_instructions=processing_instructions, records_format=vertica_format) unhandled_hints = set(load_plan.records_format.hints.keys()) options = vertica_import_options(unhandled_hints, load_plan) expected_options = { 'abort_on_error': False, 'delimiter': '\x01', 'enclosed_by': None, 'enforcelength': False, 'error_tolerance': True, 'escape_as': None, 'gzip': False, 'load_method': 'AUTO', 'no_commit': False, 'null_as': None, 'record_terminator': '\x02', 'rejectmax': None, 'skip': 0, 'trailing_nullcols': True, } self.assertDictEqual(options, expected_options) self.assertEqual(unhandled_hints, set())
def test_vertica_import_options_max_failure_rows_specified(self): records_format = DelimitedRecordsFormat(variant='vertica') unhandled_hints = set(records_format.hints) processing_instructions = ProcessingInstructions(max_failure_rows=123) load_plan = RecordsLoadPlan( processing_instructions=processing_instructions, records_format=records_format) out = vertica_import_options(unhandled_hints, load_plan) expectations = { 'trailing_nullcols': True, 'rejectmax': 123, 'enforcelength': None, 'error_tolerance': None, 'abort_on_error': None, } self.assertTrue(set(expectations.items()).issubset(set(out.items())))
def test_quote_all_without_doublequote(self): vertica_format = DelimitedRecordsFormat(variant='csv', hints={ 'quoting': 'all', 'doublequote': False, # Vertica doesn't support exporting CSV variant style dates by # default, so let's pick some it can for purposes of this # test: 'dateformat': 'YYYY-MM-DD', 'datetimeformat': 'YYYY-MM-DD HH:MI:SS', 'datetimeformattz': 'YYYY-MM-DD HH:MI:SSOF', }) processing_instructions = ProcessingInstructions() load_plan = RecordsLoadPlan(processing_instructions=processing_instructions, records_format=vertica_format) unhandled_hints = set(load_plan.records_format.hints.keys()) out = vertica_import_options(unhandled_hints, load_plan) self.assertEqual(out['enclosed_by'], '"')