def test_bigquery_uncompressed(self): records_format = DelimitedRecordsFormat(variant='bigquery', hints={'compression': None}) unhandled_hints = set(records_format.hints) processing_instructions = ProcessingInstructions() load_plan = RecordsLoadPlan(processing_instructions, records_format) date_input_style, copy_options = postgres_copy_from_options( unhandled_hints, load_plan) self.assertEqual(date_input_style, None) self.assertEqual( copy_options, { 'format': 'csv', 'quote': '"', 'encoding': 'UTF8', 'header': True, 'delimiter': ',' })
def test_load_job_config_unsupported_datetimeformattz(self): records_format = DelimitedRecordsFormat( variant='bigquery', hints={'datetimeformattz': 'MM/DD/YY HH:MI:SSOF'}) processing_instructions = ProcessingInstructions( fail_if_dont_understand=True, fail_if_cant_handle_hint=True, fail_if_row_invalid=True) load_plan = RecordsLoadPlan( processing_instructions=processing_instructions, records_format=records_format) unhandled_hints = set(records_format.hints.keys()) with self.assertRaisesRegex( NotImplementedError, r"Implement hint datetimeformattz='MM/DD/YY HH:MI:SSOF' " "or try again with fail_if_cant_handle_hint=False"): load_job_config(unhandled_hints, load_plan)
def test_quote_all_without_doublequote(self): vertica_format = DelimitedRecordsFormat(variant='csv', hints={ 'quoting': 'all', 'doublequote': False, # Vertica doesn't support exporting CSV variant style dates by # default, so let's pick some it can for purposes of this # test: 'dateformat': 'YYYY-MM-DD', 'datetimeformat': 'YYYY-MM-DD HH:MI:SS', 'datetimeformattz': 'YYYY-MM-DD HH:MI:SSOF', }) processing_instructions = ProcessingInstructions() load_plan = RecordsLoadPlan(processing_instructions=processing_instructions, records_format=vertica_format) unhandled_hints = set(load_plan.records_format.hints.keys()) out = vertica_import_options(unhandled_hints, load_plan) self.assertEqual(out['enclosed_by'], '"')
def can_load_this_format(self, source_records_format: BaseRecordsFormat) -> bool: try: processing_instructions = ProcessingInstructions() load_plan = RecordsLoadPlan( records_format=source_records_format, processing_instructions=processing_instructions) if not isinstance(load_plan.records_format, DelimitedRecordsFormat): return False unhandled_hints = set(load_plan.records_format.hints.keys()) processing_instructions = load_plan.processing_instructions mysql_load_options(unhandled_hints, load_plan.records_format, fail_if_cant_handle_hint=True) complain_on_unhandled_hints(fail_if_dont_understand=True, unhandled_hints=unhandled_hints, hints=load_plan.records_format.hints) return True except NotImplementedError: return False
def test_load_job_config_parquet(self): records_format = ParquetRecordsFormat() processing_instructions = ProcessingInstructions( fail_if_dont_understand=True, fail_if_cant_handle_hint=True, fail_if_row_invalid=True) load_plan = RecordsLoadPlan( processing_instructions=processing_instructions, records_format=records_format) unhandled_hints = set() out = load_job_config(unhandled_hints, load_plan) expectations = { 'allowJaggedRows': False, 'autodetect': False, 'createDisposition': 'CREATE_NEVER', 'destinationTableProperties': {}, 'ignoreUnknownValues': True, 'maxBadRecords': 0, 'schemaUpdateOptions': None, 'sourceFormat': 'PARQUET', 'writeDisposition': 'WRITE_APPEND' } self.assertEqual(expectations, out.to_api_repr()['load'])
def test_vertica_format(self): vertica_format = DelimitedRecordsFormat(variant='vertica') load_plan = RecordsLoadPlan(processing_instructions=ProcessingInstructions(), records_format=vertica_format) unhandled_hints = set(load_plan.records_format.hints.keys()) options = vertica_import_options(unhandled_hints, load_plan) expected_options = { 'abort_on_error': True, 'delimiter': '\x01', 'enclosed_by': None, 'enforcelength': True, 'error_tolerance': False, 'escape_as': None, 'gzip': False, 'load_method': 'AUTO', 'no_commit': False, 'null_as': None, 'record_terminator': '\x02', 'rejectmax': 1, 'skip': 0, 'trailing_nullcols': False, } self.assertDictEqual(options, expected_options) self.assertEqual(unhandled_hints, set())