def cd_step():
"""
CD step to save all beam examples/tests/katas and their outputs on the Google Cloud
"""
root_dir = os.getenv("BEAM_ROOT_DIR")
cd_helper = CDHelper()
examples = find_examples(root_dir)
cd_helper.store_examples(examples)
def test__get_gcs_object_name():
"""
Test getting the path where file will be stored at the bucket
"""
expected_result = "SDK_JAVA/base_folder/file.java"
expected_result_with_extension = "SDK_JAVA/base_folder/file.output"
assert CDHelper()._get_gcs_object_name(SDK_JAVA, "base_folder",
"file") == expected_result
assert CDHelper()._get_gcs_object_name(
SDK_JAVA, "base_folder", "file",
"output") == expected_result_with_extension
def cd_step():
"""
CD step to save all beam examples/tests/katas and their outputs on the GCS
"""
setup_logger()
root_dir = os.getenv("BEAM_ROOT_DIR")
categories_file = os.getenv("BEAM_EXAMPLE_CATEGORIES")
supported_categories = get_supported_categories(categories_file)
cd_helper = CDHelper()
examples = find_examples(root_dir, supported_categories)
cd_helper.store_examples(examples)
def test__get_gcs_object_name():
"""
Test getting the path where file will be stored at the bucket
"""
expected_path = "SDK_JAVA/PRECOMPILED_OBJECT_TYPE_UNIT_TEST/base_folder"
expected_result = "%s/%s" % (expected_path, "file.java")
expected_result_with_extension = "%s/%s" % (expected_path, "file.output")
assert CDHelper()._get_gcs_object_name(SDK_JAVA,
PRECOMPILED_OBJECT_TYPE_UNIT_TEST,
"base_folder",
"file") == expected_result
assert CDHelper()._get_gcs_object_name(
SDK_JAVA, PRECOMPILED_OBJECT_TYPE_UNIT_TEST, "base_folder", "file",
"output") == expected_result_with_extension
def test__write_to_local_fs(delete_temp_folder):
"""
Test writing code of an example, output and meta info to
the filesystem (in temp folder)
Args:
delete_temp_folder: python fixture to clean up temp folder
after method execution
"""
object_meta = {
"name": "name",
"description": "description",
"multifile": False,
"categories": ["category-1", "category-2"],
"pipeline_options": "--option option"
}
example = Example(name="name",
pipeline_id="pipeline_id",
sdk=SDK_JAVA,
filepath="filepath",
code="code_of_example",
output="output_of_example",
status=STATUS_UNSPECIFIED,
tag=Tag(**object_meta),
link="link")
expected_result = {
"SDK_JAVA/name/name.java": "temp/pipeline_id/SDK_JAVA/name/name.java",
"SDK_JAVA/name/name.output":
"temp/pipeline_id/SDK_JAVA/name/name.output",
"SDK_JAVA/name/name.log": "temp/pipeline_id/SDK_JAVA/name/name.log",
"SDK_JAVA/name/meta.info": "temp/pipeline_id/SDK_JAVA/name/meta.info"
}
assert CDHelper()._write_to_local_fs(example) == expected_result
def test__save_to_cloud_storage(mocker):
"""
Test saving examples, outputs and meta to bucket
Args:
mocker: mocker fixture from pytest-mocker
"""
expected_cloud_path = "SDK_JAVA/Example/example.java"
object_meta = {
"name": "name",
"description": "description",
"multifile": False,
"categories": ["category-1", "category-2"],
"pipeline_options": "--option option",
"default_example": True
}
upload_blob_mock = mocker.patch("cd_helper.CDHelper._upload_blob",
return_value=upload_blob)
write_to_os_mock = mocker.patch("cd_helper.CDHelper._write_to_local_fs",
return_value={expected_cloud_path: ""})
example = Example(name="name",
pipeline_id="pipeline_id",
sdk=SDK_JAVA,
filepath="filepath",
code="code_of_example",
output="output_of_example",
status=STATUS_UNSPECIFIED,
tag=Tag(**object_meta),
link="link")
CDHelper()._save_to_cloud_storage([example])
write_to_os_mock.assert_called_with(example)
upload_blob_mock.assert_called_with(
source_file="", destination_blob_name=expected_cloud_path)
def test__write_default_example_path_to_local_fs(delete_temp_folder):
"""
Test writing default example link of sdk to
the filesystem (in temp folder)
Args:
delete_temp_folder: python fixture to clean up temp folder
after method execution
"""
sdk = Sdk.Name(SDK_GO)
default_example_path = "SDK_GO/PRECOMPILED_OBJECT_TYPE_EXAMPLE/WordCount"
expected_result = str(pathlib.Path(sdk, Config.DEFAULT_PRECOMPILED_OBJECT))
cloud_path = CDHelper()._write_default_example_path_to_local_fs(
default_example_path)
assert cloud_path == expected_result
assert os.path.exists(os.path.join("temp", cloud_path))
def test__save_to_cloud_storage(mocker):
"""
Test saving examples, outputs and meta to bucket
Args:
mocker: mocker fixture from pytest-mocker
"""
upload_blob_mock = mocker.patch("cd_helper.CDHelper._upload_blob",
return_value=upload_blob)
write_to_os_mock = mocker.patch("cd_helper.CDHelper._write_to_local_fs",
return_value={"": ""})
example = Example("name", "pipeline_id", SDK_JAVA, "filepath",
"code_of_example", "output_of_example",
STATUS_UNSPECIFIED, None)
CDHelper()._save_to_cloud_storage([example])
write_to_os_mock.assert_called_with(example)
upload_blob_mock.assert_called_with(source_file="",
destination_blob_name="")
def _cd_step(examples: List[Example]): """ CD step to save all beam examples/tests/katas and their outputs on the GCS """ cd_helper = CDHelper() cd_helper.store_examples(examples)
def test_store_examples(mock_run_code, mock_save_to_cloud):
helper = CDHelper()
helper.store_examples([])
mock_run_code.assert_called_once_with([])
mock_save_to_cloud.assert_called_once_with([])
def test__clear_temp_folder():
if not os.path.exists(Config.TEMP_FOLDER):
os.mkdir(Config.TEMP_FOLDER)
CDHelper()._clear_temp_folder()
assert os.path.exists(Config.TEMP_FOLDER) is False