def test_redshift_copy_options_datetimeformattz(self): # Redshift's time_format doesn't support separate # configuration for datetimeformat vs datetimeformattz, but # the 'auto' flag seems to work with specific things (see # tests run in records_copy.py). # # Please verify new formats have a test run # and documented in records_copy.py before putting an entry in # here. expectations = { 'YYYY-MM-DD HH:MI:SSOF': 'auto', 'YYYY-MM-DD HH:MI:SS': 'YYYY-MM-DD HH:MI:SS', 'YYYY-MM-DD HH24:MI:SSOF': 'auto', 'MM/DD/YY HH24:MIOF': 'auto', 'MM/DD/YY HH24:MI': 'MM/DD/YY HH24:MI', } for datetimeformattz in DATETIMETZ_CASES: hints = { 'datetimeformattz': datetimeformattz, 'datetimeformat': datetimeformattz.replace('OF', '') } records_format =\ DelimitedRecordsFormat(variant='bluelabs', hints=hints) unhandled_hints = set(records_format.hints.keys()) out = redshift_copy_options(unhandled_hints, records_format, fail_if_cant_handle_hint=True, fail_if_row_invalid=True, max_failure_rows=0) self.assertEquals(out['time_format'], expectations[datetimeformattz])
def test_redshift_copy_options_dateformat(self): # The records spec's date/time formats are based on Redshift's # spec originally, so it's expected that everything here would # be accepted as-is, but please double-check with Redshift's # docs as new test cases are added accept_as_is = { 'YYYY-MM-DD': True, 'MM-DD-YYYY': True, 'DD-MM-YYYY': True, 'MM/DD/YY': True, 'DD/MM/YY': True, 'DD-MM-YY': True, } for dateformat in DATE_CASES: records_format =\ DelimitedRecordsFormat(variant='bluelabs', hints={ 'dateformat': dateformat }) unhandled_hints = set(records_format.hints.keys()) out = redshift_copy_options(unhandled_hints, records_format, fail_if_cant_handle_hint=True, fail_if_row_invalid=True, max_failure_rows=0) if accept_as_is[dateformat]: self.assertIs(out['date_format'], dateformat) else: self.fail('define what to expect here')
def test_redshift_copy_options_avro(self): records_format = AvroRecordsFormat() unhandled_hints = set() out = redshift_copy_options(unhandled_hints, records_format, fail_if_cant_handle_hint=True, fail_if_row_invalid=True, max_failure_rows=0) self.assertIs(out['format'], Format.avro)
def test_redshift_copy_options_timeonlyformat(self): # Redshift didn't used to have a time only format, so Records # Mover doesn't yet support anything here and just treats # these as strings. Nothing should raise. # # https://github.com/bluelabsio/records-mover/issues/141 for timeonlyformat in TIMEONLY_CASES: hints = { 'timeonlyformat': timeonlyformat, } records_format =\ DelimitedRecordsFormat(variant='bluelabs', hints=hints) unhandled_hints = set(records_format.hints.keys()) redshift_copy_options(unhandled_hints, records_format, fail_if_cant_handle_hint=True, fail_if_row_invalid=True, max_failure_rows=0)
def test_redshift_copy_options_datetimeformat(self): # Redshift's time_format doesn't support separate # configuration for datetimeformat vs datetimeformattz, but # the 'auto' flag seems to work with specific things (see # tests run in records_copy.py). # # Please verify new formats have a test run # and documented in records_copy.py before putting an entry in # here. for datetimeformat in DATETIME_CASES: hints = { 'datetimeformattz': f"{datetimeformat}OF", 'datetimeformat': datetimeformat, } records_format =\ DelimitedRecordsFormat(variant='bluelabs', hints=hints) unhandled_hints = set(records_format.hints.keys()) out = redshift_copy_options(unhandled_hints, records_format, fail_if_cant_handle_hint=True, fail_if_row_invalid=True, max_failure_rows=0) self.assertEqual(out['time_format'], 'auto') for datetimeformat in DATETIME_CASES: hints = { 'datetimeformattz': datetimeformat, 'datetimeformat': datetimeformat, } records_format =\ DelimitedRecordsFormat(variant='bluelabs', hints=hints) unhandled_hints = set(records_format.hints.keys()) out = redshift_copy_options(unhandled_hints, records_format, fail_if_cant_handle_hint=True, fail_if_row_invalid=True, max_failure_rows=0) self.assertEqual(out['time_format'], datetimeformat)
def test_redshift_copy_options_encodings(self): tests = { 'UTF16': Encoding.utf16, 'UTF16LE': Encoding.utf16le, 'UTF16BE': Encoding.utf16be } for hint_spelling, redshift_sqlalchemy_spelling in tests.items(): records_format =\ DelimitedRecordsFormat(variant='bluelabs', hints={ 'encoding': hint_spelling }) unhandled_hints = set(records_format.hints.keys()) out = redshift_copy_options(unhandled_hints, records_format, fail_if_cant_handle_hint=True, fail_if_row_invalid=True, max_failure_rows=0) self.assertIs(out['encoding'], redshift_sqlalchemy_spelling)