def test_vertica_import_options_datetimeformat(self):
# Vertica doesn't currently allow any configurability on
# input datetimeformat. Check again before adding any test cases
# here!
should_raise = {
'YYYY-MM-DD HH:MI:SS': True,
'YYYY-MM-DD HH24:MI:SS': False,
'MM/DD/YY HH24:MI': True,
'YYYY-MM-DD HH12:MI AM': True,
}
for datetimeformat in DATETIME_CASES:
records_format = DelimitedRecordsFormat(variant='vertica',
hints={
'datetimeformat':
datetimeformat,
})
unhandled_hints = set(records_format.hints)
processing_instructions = ProcessingInstructions(
max_failure_rows=123)
load_plan = RecordsLoadPlan(
processing_instructions=processing_instructions,
records_format=records_format)
try:
vertica_import_options(unhandled_hints, load_plan)
except NotImplementedError:
if should_raise[datetimeformat]:
pass
else:
self.fail()
def test_quote_all_with_doublequote(self):
vertica_format = DelimitedRecordsFormat(variant='csv', hints={
'quoting': 'all'
})
processing_instructions = ProcessingInstructions()
load_plan = RecordsLoadPlan(processing_instructions=processing_instructions,
records_format=vertica_format)
unhandled_hints = set(load_plan.records_format.hints.keys())
with self.assertRaisesRegexp(NotImplementedError,
r"Implement hint doublequote=True or try again with "
"fail_if_cant_handle_hint=False"):
vertica_import_options(unhandled_hints, load_plan)
def test_weird_timeonlyformat(self):
vertica_format = DelimitedRecordsFormat(variant='dumb', hints={
'timeonlyformat': 'something else'
})
processing_instructions = ProcessingInstructions()
load_plan = RecordsLoadPlan(processing_instructions=processing_instructions,
records_format=vertica_format)
unhandled_hints = set(load_plan.records_format.hints.keys())
with self.assertRaisesRegexp(NotImplementedError,
"Implement hint timeonlyformat='something else' or try again "
"with fail_if_cant_handle_hint=False"):
vertica_import_options(unhandled_hints, load_plan)
def test_christmas_tree_format_1_permissive(self):
vertica_format = DelimitedRecordsFormat(variant='dumb', hints=christmas_tree_format_1_hints)
processing_instructions = ProcessingInstructions(fail_if_cant_handle_hint=False)
load_plan = RecordsLoadPlan(processing_instructions=processing_instructions,
records_format=vertica_format)
unhandled_hints = set(load_plan.records_format.hints.keys())
with patch.object(driver_logger, 'warning') as mock_warning:
options = vertica_import_options(unhandled_hints, load_plan)
expected_options = {
'abort_on_error': True,
'delimiter': '\x01',
'enforcelength': True,
'error_tolerance': False,
'escape_as': '\\',
'load_method': 'AUTO',
'no_commit': False,
'null_as': None,
'record_terminator': '\x02',
'rejectmax': 1,
'skip': 1,
'trailing_nullcols': False,
}
self.assertDictEqual(options, expected_options)
self.assertListEqual(mock_warning.mock_calls,
[call("Ignoring hint compression = 'LZO'"),
call("Ignoring hint quoting = 'nonnumeric'")])
self.assertEqual(unhandled_hints, set())
def test_vertica_format_permissive(self):
vertica_format = DelimitedRecordsFormat(variant='vertica')
processing_instructions = ProcessingInstructions(fail_if_row_invalid=False)
load_plan = RecordsLoadPlan(processing_instructions=processing_instructions,
records_format=vertica_format)
unhandled_hints = set(load_plan.records_format.hints.keys())
options = vertica_import_options(unhandled_hints, load_plan)
expected_options = {
'abort_on_error': False,
'delimiter': '\x01',
'enclosed_by': None,
'enforcelength': False,
'error_tolerance': True,
'escape_as': None,
'gzip': False,
'load_method': 'AUTO',
'no_commit': False,
'null_as': None,
'record_terminator': '\x02',
'rejectmax': None,
'skip': 0,
'trailing_nullcols': True,
}
self.assertDictEqual(options, expected_options)
self.assertEqual(unhandled_hints, set())
def test_vertica_import_options_max_failure_rows_specified(self):
records_format = DelimitedRecordsFormat(variant='vertica')
unhandled_hints = set(records_format.hints)
processing_instructions = ProcessingInstructions(max_failure_rows=123)
load_plan = RecordsLoadPlan(
processing_instructions=processing_instructions,
records_format=records_format)
out = vertica_import_options(unhandled_hints, load_plan)
expectations = {
'trailing_nullcols': True,
'rejectmax': 123,
'enforcelength': None,
'error_tolerance': None,
'abort_on_error': None,
}
self.assertTrue(set(expectations.items()).issubset(set(out.items())))
def test_quote_all_without_doublequote(self):
vertica_format = DelimitedRecordsFormat(variant='csv', hints={
'quoting': 'all',
'doublequote': False,
# Vertica doesn't support exporting CSV variant style dates by
# default, so let's pick some it can for purposes of this
# test:
'dateformat': 'YYYY-MM-DD',
'datetimeformat': 'YYYY-MM-DD HH:MI:SS',
'datetimeformattz': 'YYYY-MM-DD HH:MI:SSOF',
})
processing_instructions = ProcessingInstructions()
load_plan = RecordsLoadPlan(processing_instructions=processing_instructions,
records_format=vertica_format)
unhandled_hints = set(load_plan.records_format.hints.keys())
out = vertica_import_options(unhandled_hints, load_plan)
self.assertEqual(out['enclosed_by'], '"')
