class TestCreateUDFs(unittest.TestCase):
"""
This class uses the parameterized python package
(https://pypi.org/project/parameterized/) to programmatically
create multiple python test function definitions
(based off `test_create_udf`). It will effectively create a python test
function for each UDF that it encounters as it walks
through the udfs/ directory.
"""
@classmethod
def setUpClass(cls):
cls._client = bigquery.Client(project=os.getenv('BQ_PROJECT_ID'))
@parameterized.expand(utils.get_all_udf_paths())
def test_create_udf(self, udf_path):
try:
with open(udf_path) as udf_file:
udf_sql = udf_file.read()
# Only replace UDF datasets with a test dataset if the
# build was triggered by a pull request or a non-main branch
if (os.getenv('_PR_NUMBER') is not None
or os.getenv('BRANCH_NAME') != 'master'):
udf_sql = utils.replace_with_test_datasets(
self._client.project, udf_sql)
if udf_sql is None:
self.fail(
f'Unable to replace SQL DDL with testing dataset '
f'for UDF: {udf_path}')
udf_creation_result = self._client.query(udf_sql).result()
self.assertIsInstance(udf_creation_result, _EmptyRowIterator)
except GoogleAPICallError as error:
self.fail(error.message)
class CreateUDFSignatures(unittest.TestCase):
"""
This class uses the parameterized python package
(https://pypi.org/project/parameterized/) to programmatically
create multiple python test function definitions
(based off `test_create_udf_signature`). It will effectively create a python
test function for each UDF that it encounters as it walks through
the udfs/ directory.
This class creates all UDFs with NULL bodies in order to prevent errors when
subsequently creating all UDFs concurrently in the TestCreateUDFs class.
Some UDFs may invoke other UDFs within their body and therefore have a
dependency on them which could lead to errors upon creation:
(e.g. one UDF might not have been created yet but is needed by another UDF
currently being created)
"""
@classmethod
def setUpClass(cls):
cls._client = bigquery.Client(project=os.getenv('BQ_PROJECT_ID'))
@parameterized.expand(utils.get_all_udf_paths())
def test_create_udf_signature(self, udf_path):
client = self._client
udf_signature = utils.replace_with_null_body(udf_path)
udf_signature = utils.replace_with_test_datasets(
client.project, udf_signature)
try:
udf_creation_result = client.query(udf_signature).result()
self.assertIsInstance(udf_creation_result, _EmptyRowIterator)
except GoogleAPICallError as error:
self.fail(error.message)
class TestCreateUDFs(unittest.TestCase):
"""
This class uses the parameterized python package (https://pypi.org/project/parameterized/) to programmatically
create multiple python test function definitions (based off `test_create_udf`). It will effectively create a
python test function for each UDF that it encounters as it walks through the udfs/ directory.
"""
@classmethod
def setUpClass(cls):
cls._client = bigquery.Client()
@parameterized.expand(utils.get_all_udf_paths())
def test_create_udf(self, udf_path):
client = self._client
bq_test_dataset = utils.get_target_bq_dataset(udf_path)
job_config = QueryJobConfig()
job_config.default_dataset = (f'{client.project}.{bq_test_dataset}')
try:
udf_sql = utils.replace_with_test_datasets(udf_path,
client.project)
udf_creation_result = client.query(udf_sql,
job_config=job_config).result()
self.assertIsInstance(udf_creation_result, _EmptyRowIterator)
except GoogleAPICallError as e:
self.fail(e.message)
class TestRunUDFs(unittest.TestCase):
"""
This class uses the parameterized python package
(https://pypi.org/project/parameterized/) to programmatically
create multiple python test function definitions
(based off `test_run_udf_and_verify_expected_result`).
It will effectively create a python test function for each UDF that it
encounters as it walks through the udfs/ directory. This class tests
each UDF by running it in BigQuery with the inputs given in the
test_cases.yaml file, and then asserting that the results equal the expected
outputs given in the test_cases.yaml file.
"""
@classmethod
def setUpClass(cls):
cls._client = bigquery.Client(project=os.getenv('BQ_PROJECT_ID'))
@parameterized.expand(utils.get_all_udf_paths())
def test_run_udf_and_verify_expected_result(self, udf_path):
bq_test_dataset = utils.get_test_bq_dataset(udf_path)
if bq_test_dataset is None:
self.fail(f'Unable to retrieve test dataset name for UDF:'
f' {udf_path}')
file_name = Path(udf_path).stem
udf_name = utils.extract_udf_name(udf_path)
self.assertEqual(
udf_name,
file_name,
msg=(f'\nFile name: {udf_path}'
f'\nshould match the function name defined inside: {udf_name}'
))
test_cases = utils.load_test_cases(udf_path, udf_name)
if test_cases is None:
self.skipTest(
f'Test inputs and outputs are not provided for : {udf_path}')
for case in test_cases:
try:
actual_result_rows = self._client.query(
f'SELECT `{bq_test_dataset}.{udf_name}`('
f' {case["input"]} ) AS actual_result_rows').result()
expected_result_rows = self._client.query(
f'SELECT ( {case["expected_output"]} ) '
f'AS expected_result_rows').result()
for expected_result_row, actual_result_row in zip(
expected_result_rows, actual_result_rows):
self.assertSequenceEqual(
expected_result_row.values(),
actual_result_row.values(),
msg=
(f'\nTest failed for: '
f'{self._client.project}.{bq_test_dataset}.{udf_name}'
f'( {case["input"]} )'
f'\nExpected output: {expected_result_row.values()}'
f'\nActual output: {actual_result_row.values()}'))
except GoogleAPICallError as error:
self.fail(error.message)
class CreateUDFSignatures(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls._client = bigquery.Client()
@parameterized.expand(utils.get_all_udf_paths())
def test_create_udf_signature(self, udf_path):
client = self._client
bq_test_dataset = utils.get_target_bq_dataset(udf_path)
job_config = QueryJobConfig()
job_config.default_dataset = (f'{client.project}.{bq_test_dataset}')
udf_signature = utils.extract_udf_signature(udf_path)
udf_sql = utils.replace_with_test_datasets(
project_id=client.project,
udf_sql=f'CREATE OR REPLACE FUNCTION {udf_signature} AS (NULL)')
try:
udf_creation_result = client.query(udf_sql,
job_config=job_config).result()
self.assertIsInstance(udf_creation_result, _EmptyRowIterator)
except GoogleAPICallError as e:
self.fail(e.message)