def test_extraction_with_database_specified(self) -> None: """ Test DATABASE_KEY in extractor result """ with patch.object(SQLAlchemyExtractor, '_get_connection') as mock_connection: connection = MagicMock() mock_connection.return_value = connection sql_execute = MagicMock() connection.execute = sql_execute sql_execute.return_value = [{ 'schema': 'test_schema', 'table_name': 'test_table', 'last_updated_time': 1000, 'cluster': 'MY_CLUSTER', }] extractor = SnowflakeTableLastUpdatedExtractor() extractor.init(self.conf) actual = extractor.extract() expected = TableLastUpdated(schema='test_schema', table_name='test_table', last_updated_time_epoch=1000, db=self.database_key, cluster='MY_CLUSTER') self.assertEqual(expected.__repr__(), actual.__repr__()) self.assertIsNone(extractor.extract())
def test_extraction_with_single_result(self) -> None: """ Test Extraction with default cluster and database and with one table as result """ with patch.object(SQLAlchemyExtractor, '_get_connection') as mock_connection: connection = MagicMock() mock_connection.return_value = connection sql_execute = MagicMock() connection.execute = sql_execute sql_execute.return_value = [{ 'schema': 'test_schema', 'table_name': 'test_table', 'last_updated_time': 1000, 'cluster': self.conf['extractor.snowflake_table_last_updated.{}'.format( SnowflakeTableLastUpdatedExtractor.CLUSTER_KEY)], }] extractor = SnowflakeTableLastUpdatedExtractor() extractor.init(self.conf) actual = extractor.extract() expected = TableLastUpdated(schema='test_schema', table_name='test_table', last_updated_time_epoch=1000, db='snowflake', cluster='MY_CLUSTER') self.assertEqual(expected.__repr__(), actual.__repr__()) self.assertIsNone(extractor.extract())
def test_sql_statement(self) -> None: """ Test DATABASE_KEY in extractor sql stmt """ with patch.object(SQLAlchemyExtractor, '_get_connection'): extractor = SnowflakeTableLastUpdatedExtractor() extractor.init(self.conf) self.assertFalse(self.database_key in extractor.sql_stmt)
def test_sql_statement(self) -> None: """ Test cluster_key in extractor sql stmt """ with patch.object(SQLAlchemyExtractor, '_get_connection'): extractor = SnowflakeTableLastUpdatedExtractor() extractor.init(self.conf) self.assertTrue(self.cluster_key in extractor.sql_stmt)
def test_sql_statement(self) -> None: """ test where clause in extractor sql statement """ with patch.object(SQLAlchemyExtractor, '_get_connection'): extractor = SnowflakeTableLastUpdatedExtractor() extractor.init(self.conf) self.assertTrue(self.where_clause_suffix in extractor.sql_stmt)
def test_sql_statement(self) -> None: """ Ensure catalog is used as cluster in extract sql stmt """ with patch.object(SQLAlchemyExtractor, '_get_connection'): extractor = SnowflakeTableLastUpdatedExtractor() extractor.init(self.conf) self.assertTrue('table_catalog' in extractor.sql_stmt) self.assertFalse(self.cluster_key in extractor.sql_stmt)
def test_extraction_with_empty_query_result(self) -> None: """ Test Extraction with empty result from query """ with patch.object(SQLAlchemyExtractor, '_get_connection'): extractor = SnowflakeTableLastUpdatedExtractor() extractor.init(self.conf) results = extractor.extract() self.assertIsNone(results)
def test_extraction_with_multiple_result(self) -> None: """ Test Extraction with default cluster and database and with multiple tables as result """ with patch.object(SQLAlchemyExtractor, '_get_connection') as mock_connection: connection = MagicMock() mock_connection.return_value = connection sql_execute = MagicMock() connection.execute = sql_execute default_cluster = self.conf[ 'extractor.snowflake_table_last_updated.{}'.format( SnowflakeTableLastUpdatedExtractor.CLUSTER_KEY)] table = { 'schema': 'test_schema1', 'table_name': 'test_table1', 'last_updated_time': 1000, 'cluster': default_cluster } table1 = { 'schema': 'test_schema1', 'table_name': 'test_table2', 'last_updated_time': 2000, 'cluster': default_cluster } table2 = { 'schema': 'test_schema2', 'table_name': 'test_table3', 'last_updated_time': 3000, 'cluster': default_cluster } sql_execute.return_value = [table, table1, table2] extractor = SnowflakeTableLastUpdatedExtractor() extractor.init(self.conf) expected = TableLastUpdated(schema='test_schema1', table_name='test_table1', last_updated_time_epoch=1000, db='snowflake', cluster='MY_CLUSTER') self.assertEqual(expected.__repr__(), extractor.extract().__repr__()) expected = TableLastUpdated(schema='test_schema1', table_name='test_table2', last_updated_time_epoch=2000, db='snowflake', cluster='MY_CLUSTER') self.assertEqual(expected.__repr__(), extractor.extract().__repr__()) expected = TableLastUpdated(schema='test_schema2', table_name='test_table3', last_updated_time_epoch=3000, db='snowflake', cluster='MY_CLUSTER') self.assertEqual(expected.__repr__(), extractor.extract().__repr__()) self.assertIsNone(extractor.extract())