def test_raise_error_if_d_and_b_none_project_id_not_none(self): with self.assertRaises(ValueError) as cm: create_loader_quick_setup(project_id='pi', dataset_name=None, bucket_name=None) msg = ('At least one of dataset_name or bucket_name ' 'must be provided if project_id is provided') self.assertEqual(msg, str(cm.exception))
def test_keep_source_in_bucket(self): populate_bucket() gpl = create_loader_quick_setup( bucket_dir_path=constants.bucket_subdir_path) gpl.load(source='bucket', destination='dataframe', data_name='a10') blob_name = ids.build_blob_name_2('a10') self.assertTrue(exist.blob_exists(blob_name))
def test_keep_source_in_local(self): populate_local() gpl = create_loader_quick_setup( dataset_name=None, bucket_dir_path=constants.bucket_dir_path) gpl.load(source='local', destination='bucket', data_name='a10') local_file_path = ids.build_local_file_path_0('a10') self.assertTrue(exist.local_file_exists(local_file_path))
def test_list_blob_uris(self): populate_bucket() gpl00 = create_loader() gpl10 = create_loader(bucket_dir_path=constants.bucket_dir_path) gpl20 = create_loader_quick_setup( bucket_dir_path=constants.bucket_subdir_path, separator='#') blob_names = [ids.build_blob_name_0(f'a{i}') for i in range(7, 12)] blob_uris = [ids.build_bucket_uri(n) for n in blob_names] blob_uris = sorted(blob_uris) self.assertEqual(blob_uris, gpl00.list_blob_uris('a')) blob_names = [ids.build_blob_name_1(f'a{i}') for i in range(10, 13)] blob_uris = [ids.build_bucket_uri(n) for n in blob_names] blob_uris = sorted(blob_uris) self.assertEqual(blob_uris, gpl10.list_blob_uris('a1')) blob_names = [ids.build_blob_name_2(f'a{i}') for i in range(9, 14)] blob_uris = [ids.build_bucket_uri(n) for n in blob_names] blob_uris = sorted(blob_uris) self.assertEqual(blob_uris, gpl20.list_blob_uris('a')) self.assertEqual([], gpl00.list_blob_uris('dir')) self.assertEqual([], gpl10.list_blob_uris('su'))
def test_local_to_bucket(self): expected = pandas.DataFrame(data={'y': ['c', 'a', 'b']}) local_file_path = ids.build_local_file_path_0('b') load.dataframe_to_local(expected, local_file_path) gpl = create_loader_quick_setup( dataset_name=None, bucket_dir_path=constants.bucket_subdir_path) gpl.load(source='local', destination='bucket', data_name='b') blob_name = ids.build_blob_name_2('b') computed = load.bucket_to_dataframe(blob_name, decompress=False) self.assert_pandas_equal(expected, computed)
def test_write_disposition_default_bucket_to_dataset(self): expected = pandas.DataFrame(data={'x': [1]}) blob_name = ids.build_blob_name_2('s10') load.dataframe_to_bucket(expected, blob_name) gpl = create_loader_quick_setup( bucket_dir_path=constants.bucket_subdir_path, local_dir_path=None) for _ in range(2): gpl.load(source='bucket', destination='dataset', data_name='s10') computed = load.dataset_to_dataframe('s10') self.assert_pandas_equal(expected, computed)
def test_dataframe_to_dataset(self): expected = pandas.DataFrame(data={'x': [1, 2, 3], 'y': [1, 2, 4]}) populate() gpl = create_loader_quick_setup() gpl.load(source='dataframe', destination='dataset', dataframe=expected, data_name='a1') computed = load.dataset_to_dataframe('a1') self.assert_pandas_equal(expected, computed)
def test_write_truncate_query_to_dataset(self): expected = pandas.DataFrame(data={'x': [1]}) gpl = create_loader_quick_setup(bucket_name=None, local_dir_path=None) for _ in range(2): gpl.load(source='query', destination='dataset', query='select 1 as x', data_name='s11', write_disposition='WRITE_TRUNCATE') computed = load.dataset_to_dataframe('s11') self.assert_pandas_equal(expected, computed)
def test_config_repeated(self): expected = pandas.DataFrame(data={'x': [3]}) populate() config = LoadConfig(source='query', destination='dataframe', query='select 3 as x') gpl = create_loader_quick_setup( local_dir_path=constants.local_subdir_path) computeds = gpl.multi_load(configs=[config] * 3) for computed in computeds: self.assert_pandas_equal(expected, computed)
def test_local_to_bucket(self): populate_local() with self.assertLogs('google_pandas_load.loader', level='DEBUG') as cm: gpl = create_loader_quick_setup( dataset_name=None, bucket_dir_path=constants.bucket_dir_path) gpl.load(source='local', destination='bucket', data_name='a9') records = cm.records self.assertEqual(2, len(records)) log = formatter.format(records[0]) self.assertEqual( 'google_pandas_load.loader # DEBUG # ' 'Starting local to bucket...', log)
def test_bucket_to_dataset(self): expected = pandas.DataFrame( data={'x': [f'a{i}_bucket' for i in range(7, 12)]}) populate_dataset() populate_bucket() gpl = create_loader_quick_setup(local_dir_path=None) gpl.load( source='bucket', destination='dataset', data_name='a', bq_schema=[bigquery.SchemaField(name='x', field_type='STRING')]) computed = load.dataset_to_dataframe('a') self.assert_pandas_equal(expected, computed)
def test_compress_dataframe_to_local(self): gpl = create_loader_quick_setup( project_id=None, dataset_name=None, bucket_name=None, local_dir_path=constants.local_subdir_path) gpl.load( source='dataframe', destination='local', dataframe=pandas.DataFrame(data={'x': [1]}), data_name='b100') local_file_path = ids.build_local_file_path_1('b100.csv.gz') self.assertTrue(is_gz_file(local_file_path))
def test_post_clear_dataframe_to_dataset(self): populate() blob_name = ids.build_blob_name_2('a10') local_file_path = ids.build_local_file_path_0('a10') self.assertTrue(exist.blob_exists(blob_name)) self.assertTrue(exist.local_file_exists(local_file_path)) gpl = create_loader_quick_setup( bucket_dir_path=constants.bucket_subdir_path) gpl.load(source='dataframe', destination='dataset', dataframe=pandas.DataFrame(data={'x': [1]}), data_name='a10') self.assertFalse(exist.blob_exists(blob_name)) self.assertFalse(exist.local_file_exists(local_file_path))
def test_exist_in_bucket(self): gpl01 = create_loader(local_dir_path=constants.local_subdir_path) gpl11 = create_loader(bucket_dir_path=constants.bucket_dir_path, local_dir_path=constants.local_subdir_path) gpl21 = create_loader_quick_setup( dataset_name=None, bucket_dir_path=constants.bucket_subdir_path, local_dir_path=constants.local_subdir_path) self.assertFalse(gpl01.exist_in_bucket('a1')) self.assertFalse(gpl11.exist_in_bucket('a10')) self.assertFalse(gpl21.exist_in_bucket('a')) populate_bucket() self.assertTrue(gpl01.exist_in_bucket('a1')) self.assertTrue(gpl11.exist_in_bucket('a10')) self.assertTrue(gpl21.exist_in_bucket('a'))
def test_local_to_dataframe(self): populate_local() with self.assertLogs('google_pandas_load.loader', level='DEBUG') as cm: gpl = create_loader_quick_setup( project_id=None, dataset_name=None, bucket_name=None, bucket_dir_path=constants.bucket_subdir_path) gpl.load(source='local', destination='dataframe', data_name='a9') records = cm.records self.assertEqual(2, len(records)) regexp = (r'^google_pandas_load.loader # DEBUG # ' r'Ended local to dataframe \[[0-9]+s\]$') pattern = re.compile(regexp) log = formatter.format(records[1]) self.assertIsNotNone(pattern.search(log))
def test_call_loader_getters(self): gpl00 = create_loader() gpl10 = create_loader(bucket_dir_path=bucket_dir_path) gpl20 = create_loader(bucket_dir_path=bucket_subdir_path) gpl01 = create_loader_quick_setup(project_id=None, dataset_name=None, bucket_name=None, local_dir_path=local_subdir_path) self.assertIsNotNone(gpl00.bq_client) self.assertIsNotNone(gpl00.gs_client) self.assertIsNotNone(gpl00.bucket) self.assertEqual(dataset_id, gpl00.dataset_id) self.assertEqual(dataset_name, gpl00.dataset_name) self.assertEqual(bucket_name, gpl00.bucket_name) self.assertIsNone(gpl00.bucket_dir_path) self.assertEqual(bucket_dir_path, gpl10.bucket_dir_path) self.assertEqual(bucket_subdir_path, gpl20.bucket_dir_path) self.assertEqual(local_dir_path, gpl00.local_dir_path) self.assertEqual(local_subdir_path, gpl01.local_dir_path)
def test_list_blobs(self): populate_bucket() gpl00 = create_loader_quick_setup() gpl10 = create_loader(bucket_dir_path=constants.bucket_dir_path) gpl20 = create_loader(bucket_dir_path=constants.bucket_subdir_path) self.assertEqual( sorted([ids.build_blob_name_0(f'a{i}') for i in range(7, 12)]), [b.name for b in gpl00.list_blobs('a')]) self.assertEqual( sorted([ids.build_blob_name_1(f'a{i}') for i in range(10, 13)]), [b.name for b in gpl10.list_blobs('a1')]) self.assertEqual( sorted([ids.build_blob_name_2(f'a{i}') for i in range(9, 14)]), [b.name for b in gpl20.list_blobs('')]) self.assertEqual([], gpl00.list_blobs('dir')) self.assertEqual([], gpl10.list_blobs('su'))
def test_exist_in_dataset(self): gpl = create_loader_quick_setup( local_dir_path=constants.local_subdir_path) self.assertFalse(gpl.exist_in_dataset('a8')) populate_dataset() self.assertTrue(gpl.exist_in_dataset('a8'))
def test_call_loader_quick_setup_getters(self): gpl = create_loader_quick_setup(bucket_name=None) self.assertEqual(project_id, gpl.project_id)
def test_raise_error_if_project_id_none_bucket_name_not_none(self): with self.assertRaises(ValueError) as cm: create_loader_quick_setup(project_id=None, dataset_name=None) msg = 'project_id must provided if bucket_name is provided' self.assertEqual(msg, str(cm.exception))