def test_get_partitions(self, mock_get_conn): response = [{ 'Partitions': [{ 'Values': ['2015-01-01'] }] }] mock_paginator = mock.Mock() mock_paginator.paginate.return_value = response mock_conn = mock.Mock() mock_conn.get_paginator.return_value = mock_paginator mock_get_conn.return_value = mock_conn hook = AwsGlueCatalogHook(region_name="us-east-1") result = hook.get_partitions('db', 'tbl', expression='foo=bar', page_size=2, max_items=3) self.assertEqual(result, {('2015-01-01',)}) mock_conn.get_paginator.assert_called_once_with('get_partitions') mock_paginator.paginate.assert_called_once_with(DatabaseName='db', TableName='tbl', Expression='foo=bar', PaginationConfig={ 'PageSize': 2, 'MaxItems': 3})
def get_hook(self) -> AwsGlueCatalogHook: """Gets the AwsGlueCatalogHook""" if self.hook: return self.hook self.hook = AwsGlueCatalogHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) return self.hook
def test_check_for_partition(self, mock_get_partitions): mock_get_partitions.return_value = {('2018-01-01', )} hook = AwsGlueCatalogHook(region_name="us-east-1") self.assertTrue(hook.check_for_partition('db', 'tbl', 'expr')) mock_get_partitions.assert_called_once_with('db', 'tbl', 'expr', max_items=1)
def get_hook(self): """ Gets the AwsGlueCatalogHook """ if not hasattr(self, 'hook'): from airflow.providers.amazon.aws.hooks.glue_catalog import AwsGlueCatalogHook self.hook = AwsGlueCatalogHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) return self.hook
def test_get_partitions_empty(self, mock_get_conn): response = set() mock_get_conn.get_paginator.paginate.return_value = response hook = AwsGlueCatalogHook(region_name="us-east-1") self.assertEqual(hook.get_partitions('db', 'tbl'), set())
def test_region(self): hook = AwsGlueCatalogHook(region_name="us-west-2") self.assertEqual(hook.region_name, 'us-west-2')
def test_conn_id(self): hook = AwsGlueCatalogHook(aws_conn_id='my_aws_conn_id', region_name="us-east-1") self.assertEqual(hook.aws_conn_id, 'my_aws_conn_id')
def test_get_conn_returns_a_boto3_connection(self): hook = AwsGlueCatalogHook(region_name="us-east-1") self.assertIsNotNone(hook.get_conn())
def setUp(self): self.client = boto3.client('glue', region_name='us-east-1') self.hook = AwsGlueCatalogHook(region_name="us-east-1")
def test_check_for_partition_false(self, mock_get_partitions): mock_get_partitions.return_value = set() hook = AwsGlueCatalogHook(region_name="us-east-1") self.assertFalse(hook.check_for_partition('db', 'tbl', 'expr'))