def test_bigquery_uncompressed(self):
records_format = DelimitedRecordsFormat(variant='bigquery',
hints={'compression': None})
unhandled_hints = set(records_format.hints)
processing_instructions = ProcessingInstructions()
load_plan = RecordsLoadPlan(processing_instructions, records_format)
date_input_style, copy_options = postgres_copy_from_options(
unhandled_hints, load_plan)
self.assertEqual(date_input_style, None)
self.assertEqual(
copy_options, {
'format': 'csv',
'quote': '"',
'encoding': 'UTF8',
'header': True,
'delimiter': ','
})
def test_load_job_config_unsupported_datetimeformattz(self):
records_format = DelimitedRecordsFormat(
variant='bigquery',
hints={'datetimeformattz': 'MM/DD/YY HH:MI:SSOF'})
processing_instructions = ProcessingInstructions(
fail_if_dont_understand=True,
fail_if_cant_handle_hint=True,
fail_if_row_invalid=True)
load_plan = RecordsLoadPlan(
processing_instructions=processing_instructions,
records_format=records_format)
unhandled_hints = set(records_format.hints.keys())
with self.assertRaisesRegex(
NotImplementedError,
r"Implement hint datetimeformattz='MM/DD/YY HH:MI:SSOF' "
"or try again with fail_if_cant_handle_hint=False"):
load_job_config(unhandled_hints, load_plan)
def test_quote_all_without_doublequote(self):
vertica_format = DelimitedRecordsFormat(variant='csv', hints={
'quoting': 'all',
'doublequote': False,
# Vertica doesn't support exporting CSV variant style dates by
# default, so let's pick some it can for purposes of this
# test:
'dateformat': 'YYYY-MM-DD',
'datetimeformat': 'YYYY-MM-DD HH:MI:SS',
'datetimeformattz': 'YYYY-MM-DD HH:MI:SSOF',
})
processing_instructions = ProcessingInstructions()
load_plan = RecordsLoadPlan(processing_instructions=processing_instructions,
records_format=vertica_format)
unhandled_hints = set(load_plan.records_format.hints.keys())
out = vertica_import_options(unhandled_hints, load_plan)
self.assertEqual(out['enclosed_by'], '"')
def can_load_this_format(self,
source_records_format: BaseRecordsFormat) -> bool:
try:
processing_instructions = ProcessingInstructions()
load_plan = RecordsLoadPlan(
records_format=source_records_format,
processing_instructions=processing_instructions)
if not isinstance(load_plan.records_format,
DelimitedRecordsFormat):
return False
unhandled_hints = set(load_plan.records_format.hints.keys())
processing_instructions = load_plan.processing_instructions
mysql_load_options(unhandled_hints,
load_plan.records_format,
fail_if_cant_handle_hint=True)
complain_on_unhandled_hints(fail_if_dont_understand=True,
unhandled_hints=unhandled_hints,
hints=load_plan.records_format.hints)
return True
except NotImplementedError:
return False
def test_load_job_config_parquet(self):
records_format = ParquetRecordsFormat()
processing_instructions = ProcessingInstructions(
fail_if_dont_understand=True,
fail_if_cant_handle_hint=True,
fail_if_row_invalid=True)
load_plan = RecordsLoadPlan(
processing_instructions=processing_instructions,
records_format=records_format)
unhandled_hints = set()
out = load_job_config(unhandled_hints, load_plan)
expectations = {
'allowJaggedRows': False,
'autodetect': False,
'createDisposition': 'CREATE_NEVER',
'destinationTableProperties': {},
'ignoreUnknownValues': True,
'maxBadRecords': 0,
'schemaUpdateOptions': None,
'sourceFormat': 'PARQUET',
'writeDisposition': 'WRITE_APPEND'
}
self.assertEqual(expectations, out.to_api_repr()['load'])
def test_vertica_format(self):
vertica_format = DelimitedRecordsFormat(variant='vertica')
load_plan = RecordsLoadPlan(processing_instructions=ProcessingInstructions(),
records_format=vertica_format)
unhandled_hints = set(load_plan.records_format.hints.keys())
options = vertica_import_options(unhandled_hints, load_plan)
expected_options = {
'abort_on_error': True,
'delimiter': '\x01',
'enclosed_by': None,
'enforcelength': True,
'error_tolerance': False,
'escape_as': None,
'gzip': False,
'load_method': 'AUTO',
'no_commit': False,
'null_as': None,
'record_terminator': '\x02',
'rejectmax': 1,
'skip': 0,
'trailing_nullcols': False,
}
self.assertDictEqual(options, expected_options)
self.assertEqual(unhandled_hints, set())
