def test_syncing_of_compressed_gz_file(self, mocked_handle_file):
mocked_handle_file.return_value = 0
config = {}
table_spec = {}
stream = {}
s3_path = "csv_files.gz"
sync.sync_table_file(config, s3_path, table_spec, stream)
mocked_handle_file.assert_called_with(config, s3_path, table_spec,
stream, "gz")
def test_sync_csv_file_called_in_sync_table_file_for_txt_file(self, mock_sync_csv_file, mocked_file_handle):
s3_path = "test\\abc.txt"
table_spec = {'table_name': 'test_table'}
config = {'bucket': 'Test'}
stream = {'stream': 'jsonl_table', 'tap_stream_id': 'jsonl_table', 'schema': {'type': 'object', 'properties': {'name': {'type': ['string', 'null']}, 'id': {'type': ['integer', 'string', 'null']}, 'marks': {'type': 'array', 'items': {'type': ['integer', 'string', 'null']}}, 'students': {'type': 'object', 'properties': {}}, '_sdc_source_bucket': {'type': 'string'}, '_sdc_source_file': {'type': 'string'}, '_sdc_source_lineno': {'type': 'integer'}, '_sdc_extra': {'type': 'array', 'items': {'type': 'string'}}}}, 'metadata': [{'breadcrumb': [], 'metadata': {'selected': True, 'table-key-properties': ['id']}}, {'breadcrumb': ['properties', 'name'], 'metadata': {
'inclusion': 'available'}}, {'breadcrumb': ['properties', 'id'], 'metadata': {'inclusion': 'automatic'}}, {'breadcrumb': ['properties', 'marks'], 'metadata': {'inclusion': 'available'}}, {'breadcrumb': ['properties', 'students'], 'metadata': {'inclusion': 'available'}}, {'breadcrumb': ['properties', '_sdc_source_bucket'], 'metadata': {'inclusion': 'available'}}, {'breadcrumb': ['properties', '_sdc_source_file'], 'metadata': {'inclusion': 'available'}}, {'breadcrumb': ['properties', '_sdc_source_lineno'], 'metadata': {'inclusion': 'available'}}, {'breadcrumb': ['properties', '_sdc_extra'], 'metadata': {'inclusion': 'available'}}]}
sync.sync_table_file(config, s3_path, table_spec, stream)
self.assertEqual(mock_sync_csv_file.call_count, 1)
def test_syncing_of_file_without_extension(self, mocked_logger):
config = {}
table_spec = {}
stream = {}
s3_path = "unittest_compressed_files/sample"
records = sync.sync_table_file(config, s3_path, table_spec, stream)
self.assertEqual(0, records)
mocked_logger.assert_called_with(
'"%s" without extension will not be synced.', s3_path)
def test_syncing_of_unsupported_file(self, mocked_logger):
config = {}
table_spec = {}
stream = {}
s3_path = "csv_files.exe"
extension = "exe"
records = sync.sync_table_file(config, s3_path, table_spec, stream)
mocked_logger.assert_called_with(
'"%s" having the ".%s" extension will not be synced.', s3_path,
extension)
self.assertEqual(0, records)
def test_syncing_of_gz_file_with_csv_extension(self, mock_csv_sample_file,
mocked_logger):
config = {"bucket": "bucket_name"}
table_spec = {"table_name": "GZ_DATA"}
stream = {}
s3_path = "unittest_compressed_files/gz_stored_as_csv.csv"
extension = "csv"
records = sync.sync_table_file(config, s3_path, table_spec, stream)
mocked_logger.assert_called_with(
'Skipping %s file as parsing failed. Verify an extension of the file.',
s3_path)
self.assertEqual(0, records)