def test_raise_error_if_infer_bq_schema_from_no_columns_dataframe(self): with self.assertRaises(ValueError) as cm: LoadConfig.bq_schema_inferred_from_dataframe( dataframe=pandas.DataFrame(data={})) msg = ('A non empty bq_schema cannot be inferred ' 'from a dataframe with no columns') self.assertEqual(str(cm.exception), msg)
def test_xmload(self): df0 = pandas.DataFrame(data={'x': [4]}) config1 = LoadConfig(source='dataframe', destination='query', data_name='e100', dataframe=df0) config2 = LoadConfig(source='query', destination='bq', data_name='e101', query='select 3') config3 = LoadConfig(source='dataframe', destination='query', data_name='e102', dataframe=df0) xmlr = gpl3.xmload(configs=[config1, config2, config3]) self.assertEqual( set(vars(xmlr)), { 'load_results', 'data_names', 'duration', 'durations', 'query_cost', 'query_costs' }) self.assertEqual(xmlr.data_names, ['e100', 'e101', 'e102']) self.assertTrue(xmlr.duration > 0) self.assertEqual(set(vars(xmlr.durations)), set(ATOMIC_FUNCTION_NAMES)) for n in ATOMIC_FUNCTION_NAMES: duration = vars(xmlr.durations)[n] if duration is not None: self.assertTrue(duration >= 0) self.assertEqual(xmlr.query_cost, 0.0) self.assertEqual(xmlr.query_costs, [None, 0.0, None])
def test_heterogeneous_configs(self): expected1 = pandas.DataFrame(data={'x': [3, 10]}) expected2 = pandas.DataFrame(data={'y': [4]}) expected3 = pandas.DataFrame(data={'x': ['b'], 'y': ['a']}) populate() config1 = LoadConfig(source='dataframe', destination='dataset', dataframe=expected1, data_name='a10') config2 = LoadConfig(source='query', destination='dataframe', query='select 4 as y') config3 = LoadConfig(source='query', destination='bucket', query="select 'b' as x, 'a' as y", data_name='a11') gpl = create_loader(bucket_dir_path=constants.bucket_subdir_path) load_results = gpl.multi_load([config1, config2, config3]) self.assertEqual(len(load_results), 3) self.assertTrue(load_results[0] is None) self.assertTrue(load_results[2] is None) computed1 = load.dataset_to_dataframe('a10') self.assert_pandas_equal(expected1, computed1) computed2 = load_results[1] self.assert_pandas_equal(expected2, computed2) blob_name = ids.build_blob_name_2('a11-000000000000.csv.gz') computed3 = load.bucket_to_dataframe(blob_name, decompress=True) self.assert_pandas_equal(expected3, computed3)
def test_raise_error_if_invalid_source_or_destination(self): with self.assertRaises(ValueError) as cm: LoadConfig(source='queryy', destination='dataframe') self.assertEqual(str(cm.exception), "source must be one of 'query' or 'bq' or 'gs' or 'local' or 'dataframe'") with self.assertRaises(ValueError) as cm: LoadConfig(source='query', destination='dataframee') self.assertEqual( str(cm.exception), "destination must be one of 'query' or 'bq' or 'gs' or 'local' or 'dataframe'")
def test_raise_error_if_prefix(self): config1 = LoadConfig(source='dataframe', destination='dataset', dataframe=pandas.DataFrame(data={'x': [3]}), data_name='a') config2 = LoadConfig(source='query', destination='dataframe', query='select 4 as y', data_name='aa') with self.assertRaises(ValueError) as cm: create_loader().multi_load(configs=[config1, config2]) self.assertEqual('a is a prefix of aa', str(cm.exception))
def test_raise_error_if_missing_required_values(self): with self.assertRaises(ValueError) as cm: LoadConfig(source='query', destination='bucket', query='select 3') msg = ("data_name must be provided if source or destination is one of " "'dataset' or 'bucket' or 'local'") self.assertEqual(msg, str(cm.exception)) with self.assertRaises(ValueError) as cm: LoadConfig(source='query', destination='dataframe') msg = "query must be provided if source = 'query'" self.assertEqual(msg, str(cm.exception)) with self.assertRaises(ValueError) as cm: LoadConfig(source='dataframe', destination='local', data_name='a1') msg = "dataframe must be provided if source = 'dataframe'" self.assertEqual(msg, str(cm.exception))
def test_raise_error_if_invalid_destination(self): df = pandas.DataFrame(data={'x': [1]}) with self.assertRaises(ValueError) as cm: LoadConfig(source='dataframe', destination='query', dataframe=df) msg = ("destination must be one of 'dataset' or 'bucket' or 'local' " "or 'dataframe'") self.assertEqual(msg, str(cm.exception))
def test_raise_error_if_configs_is_not_a_list(self): config = LoadConfig(source='bucket', destination='local', data_name='a1') with self.assertRaises(ValueError) as cm: create_loader().multi_load(configs={config}) self.assertEqual('configs must be a list', str(cm.exception))
def test_raise_error_if_data_name_contains_slash(self): with self.assertRaises(ValueError) as cm: LoadConfig(source='query', destination='dataset', query='select 3', data_name='a/b') msg = 'data_name=a/b must not contain a /' self.assertEqual(msg, str(cm.exception))
def test_raise_error_if_data_name_is_empty_string(self): with self.assertRaises(ValueError) as cm: LoadConfig(source='query', destination='dataset', query='select 3', data_name='') msg = 'data_name must not be the empty string' self.assertEqual(msg, str(cm.exception))
def test_raise_error_if_missing_required_values(self): with self.assertRaises(ValueError) as cm: LoadConfig(source='query', destination='dataframe') self.assertEqual(str(cm.exception), "query must be given if source = 'query'") with self.assertRaises(ValueError) as cm: LoadConfig(source='dataframe', destination='local') self.assertEqual(str(cm.exception), "dataframe must be given if source = 'dataframe'") with self.assertRaises(ValueError) as cm: LoadConfig(source='query', destination='gs', query='select 3') self.assertEqual( str(cm.exception), "data_name must be given if source or destination is one of 'bq' or 'gs' or 'local'") with self.assertRaises(ValueError) as cm: LoadConfig(source='gs', destination='bq', data_name='e0') self.assertEqual(str(cm.exception), 'bq_schema is missing')
def test_raise_error_if_invalid_source(self): with self.assertRaises(ValueError) as cm: LoadConfig(source='queryy', destination='dataframe', query='select 3') msg = ("source must be one of 'query' or 'dataset' or " "'bucket' or 'local' or 'dataframe") self.assertEqual(msg, str(cm.exception))
def test_config_repeated(self): df0 = pandas.DataFrame(data={'x': [3]}) populate() config = LoadConfig(source='query', destination='dataframe', query='select 3 as x') dfs = gpl5.mload(configs=[config] * 3) for df in dfs: self.assertTrue(df0.equals(df))
def test_config_repeated(self): expected = pandas.DataFrame(data={'x': [3]}) populate() config = LoadConfig(source='query', destination='dataframe', query='select 3 as x') gpl = create_loader_quick_setup( local_dir_path=constants.local_subdir_path) computeds = gpl.multi_load(configs=[config] * 3) for computed in computeds: self.assert_pandas_equal(expected, computed)
def test_diamond(self): df0 = pandas.DataFrame(data={'x': [3]}) query = 'select 3 as x' populate() df1 = gpl5.xload(source='query', destination='dataframe', query=query).load_result config = LoadConfig(source='query', destination='dataframe', query=query) df2 = gpl5.mload(configs=[config])[0] self.assertTrue(df0.equals(df1)) self.assertTrue(df0.equals(df2))
def test_mload(self): populate() config1 = LoadConfig(source='dataframe', destination='query', data_name='a10', dataframe=pandas.DataFrame(data={'x': [3]})) config2 = LoadConfig(source='query', destination='dataframe', query='select 4 as y') config3 = LoadConfig(source='query', destination='gs', data_name='e0', query='select 4 as y') load_results = gpl5.mload(configs=[config1, config2, config3]) self.assertEqual(len(load_results), 3) self.assertEqual( load_results[0], 'select * from `{}.{}.a10`'.format(project_id, dataset_id)) self.assertTrue(load_results[1].equals( pandas.DataFrame(data={'y': [4]}))) self.assertTrue(load_results[2] is None)
def upload_dfs(gpl, dfs): data_names = build_data_names() cols_to_upload = build_cols_to_upload() configs = [ LoadConfig( source='dataframe', destination='bq', data_name=data_names[kind], dataframe=dfs[kind][cols_to_upload[kind]], timestamp_cols=['ts'], date_cols=['d']) for kind in kinds] gpl.mload(configs)
def test_no_skip_blank_lines(self): df0 = pandas.DataFrame(data={'x': [3, numpy.nan]}) df1 = pandas.DataFrame(data={'x': [numpy.nan, 4]}) df2 = pandas.DataFrame(data={'x': [numpy.nan, 5], 'y': [numpy.nan, 6]}) df3 = pandas.DataFrame(data={'x': [7, numpy.nan], 'y': [8, numpy.nan]}) expecteds = [df0, df1, df2, df3] populate() query0 = 'select 3 as x union all select null as x' query1 = 'select null as x union all select 4 as x' query2 = 'select null as x, null as y union all ' \ 'select 5 as x, 6 as y' query3 = 'select 7 as x, 8 as y union all ' \ 'select null as x, null as y' queries = [query0, query1, query2, query3] configs = [] for query in queries: config = LoadConfig(source='query', destination='dataframe', query=query) configs.append(config) gpl = create_loader() computed = gpl.multi_load(configs) for df, dg in zip(expecteds, computed): self.assert_pandas_equal(df, dg)
def test_raise_error_if_source_is_equal_to_destination(self): with self.assertRaises(ValueError) as cm: LoadConfig(source='query', destination='query') self.assertEqual(str(cm.exception), 'source must be different from destination')
def test_raise_error_if_bq_schema_inferred_from_dataframe_is_given_a_dataframe_with_no_columns(self): with self.assertRaises(ValueError) as cm: LoadConfig.bq_schema_inferred_from_dataframe(dataframe=pandas.DataFrame(data={})) self.assertEqual(str(cm.exception), 'A non empty bq_schema cannot be inferred from a dataframe with no columns')
def test_raise_error_if_source_is_equal_to_destination(self): with self.assertRaises(ValueError) as cm: LoadConfig(source='local', destination='local', data_name='a') msg = 'source must be different from destination' self.assertEqual(msg, str(cm.exception))