def test__check_file_with_correct_tag(mock_get_tag, mock_validate,
mock_get_example):
tag = ExampleTag({"name": "Name"}, "")
example = Example(name="filename",
sdk=SDK_JAVA,
filepath="/root/filename.java",
code="data",
status=STATUS_UNSPECIFIED,
tag=Tag("Name", "Description", False, [],
'--option option'))
examples = []
mock_get_tag.return_value = tag
mock_validate.return_value = True
mock_get_example.return_value = example
result = _check_file(examples,
"filename.java",
"/root/filename.java", [],
sdk=SDK_JAVA)
assert result is False
assert len(examples) == 1
assert examples[0] == example
mock_get_tag.assert_called_once_with("/root/filename.java")
mock_validate.assert_called_once_with(tag.tag_as_dict, [])
mock_get_example.assert_called_once_with("/root/filename.java",
"filename.java", tag)
def test__write_to_local_fs(delete_temp_folder):
"""
Test writing code of an example, output and meta info to
the filesystem (in temp folder)
Args:
delete_temp_folder: python fixture to clean up temp folder
after method execution
"""
object_meta = {
"name": "name",
"description": "description",
"multifile": False,
"categories": ["category-1", "category-2"],
"pipeline_options": "--option option"
}
example = Example(name="name",
pipeline_id="pipeline_id",
sdk=SDK_JAVA,
filepath="filepath",
code="code_of_example",
output="output_of_example",
status=STATUS_UNSPECIFIED,
tag=Tag(**object_meta),
link="link")
expected_result = {
"SDK_JAVA/name/name.java": "temp/pipeline_id/SDK_JAVA/name/name.java",
"SDK_JAVA/name/name.output":
"temp/pipeline_id/SDK_JAVA/name/name.output",
"SDK_JAVA/name/name.log": "temp/pipeline_id/SDK_JAVA/name/name.log",
"SDK_JAVA/name/meta.info": "temp/pipeline_id/SDK_JAVA/name/meta.info"
}
assert CDHelper()._write_to_local_fs(example) == expected_result
def test__get_example(mock_get_name, mock_get_sdk):
mock_get_name.return_value = "filepath"
mock_get_sdk.return_value = SDK_UNSPECIFIED
result = _get_example(
"/root/filepath.extension",
"filepath.extension",
{
"name": "Name",
"description": "Description",
"multifile": "False",
"categories": [""],
"pipeline_options": "--option option"
})
assert result == Example(
"filepath",
"",
SDK_UNSPECIFIED,
"/root/filepath.extension",
"data",
"",
STATUS_UNSPECIFIED,
Tag("Name", "Description", "False", [""], "--option option"))
mock_get_name.assert_called_once_with("filepath.extension")
mock_get_sdk.assert_called_once_with("filepath.extension")
async def test__update_example_status(mock_grpc_client_run_code,
mock_grpc_client_check_status):
example = Example(name="file",
pipeline_id="pipeline_id",
sdk=SDK_UNSPECIFIED,
filepath="root/file.extension",
code="code",
output="output",
status=STATUS_UNSPECIFIED,
tag={"pipeline_options": "--key value"},
link="link")
mock_grpc_client_run_code.return_value = "pipeline_id"
mock_grpc_client_check_status.side_effect = [
STATUS_VALIDATING, STATUS_FINISHED
]
await _update_example_status(example, GRPCClient())
assert example.pipeline_id == "pipeline_id"
assert example.status == STATUS_FINISHED
mock_grpc_client_run_code.assert_called_once_with(example.code,
example.sdk,
"--key value")
mock_grpc_client_check_status.assert_has_calls([mock.call("pipeline_id")])
def test__check_file_with_correct_tag(
mock_get_tag, mock_validate, mock_get_example):
tag = {"name": "Name"}
example = Example(
"filename",
"",
SDK_UNSPECIFIED,
"/root/filename.java",
"data",
"",
STATUS_UNSPECIFIED,
Tag("Name", "Description", False, [], '--option option'))
examples = []
mock_get_tag.return_value = tag
mock_validate.return_value = True
mock_get_example.return_value = example
result = _check_file(examples, "filename.java", "/root/filename.java", [])
assert result is False
assert len(examples) == 1
assert examples[0] == example
mock_get_tag.assert_called_once_with("/root/filename.java")
mock_validate.assert_called_once_with(tag, [])
mock_get_example.assert_called_once_with(
"/root/filename.java", "filename.java", tag)
def test__save_to_cloud_storage(mocker):
"""
Test saving examples, outputs and meta to bucket
Args:
mocker: mocker fixture from pytest-mocker
"""
expected_cloud_path = "SDK_JAVA/Example/example.java"
object_meta = {
"name": "name",
"description": "description",
"multifile": False,
"categories": ["category-1", "category-2"],
"pipeline_options": "--option option",
"default_example": True
}
upload_blob_mock = mocker.patch("cd_helper.CDHelper._upload_blob",
return_value=upload_blob)
write_to_os_mock = mocker.patch("cd_helper.CDHelper._write_to_local_fs",
return_value={expected_cloud_path: ""})
example = Example(name="name",
pipeline_id="pipeline_id",
sdk=SDK_JAVA,
filepath="filepath",
code="code_of_example",
output="output_of_example",
status=STATUS_UNSPECIFIED,
tag=Tag(**object_meta),
link="link")
CDHelper()._save_to_cloud_storage([example])
write_to_os_mock.assert_called_with(example)
upload_blob_mock.assert_called_with(
source_file="", destination_blob_name=expected_cloud_path)
async def test_get_statuses(mock_update_example_status, mock_grpc_client):
example = Example("file", "pipeline_id", SDK_UNSPECIFIED,
"root/file.extension", "code", "output",
STATUS_UNSPECIFIED, {"name": "Name"})
client = None
mock_grpc_client.return_value = client
await get_statuses([example])
mock_update_example_status.assert_called_once_with(example, client)
async def test_get_statuses(mock_update_example_status, mock_grpc_client):
example = Example(name="file",
pipeline_id="pipeline_id",
sdk=SDK_UNSPECIFIED,
filepath="root/file.extension",
code="code",
output="output",
status=STATUS_UNSPECIFIED,
tag={"name": "Name"})
client = None
mock_grpc_client.return_value = client
await get_statuses([example])
mock_update_example_status.assert_called_once_with(example, client)
async def test__update_example_status(mock_grpc_client_run_code,
mock_grpc_client_check_status):
example = Example("file", "pipeline_id", SDK_UNSPECIFIED,
"root/file.extension", "code", "output",
STATUS_UNSPECIFIED, {"name": "Name"})
mock_grpc_client_run_code.return_value = "pipeline_id"
mock_grpc_client_check_status.side_effect = [
STATUS_VALIDATING, STATUS_FINISHED
]
await _update_example_status(example, GRPCClient())
assert example.pipeline_id == "pipeline_id"
assert example.status == STATUS_FINISHED
mock_grpc_client_run_code.assert_called_once_with(example.code,
example.sdk)
mock_grpc_client_check_status.assert_has_calls([mock.call("pipeline_id")])
def test__save_to_cloud_storage(mocker):
"""
Test saving examples, outputs and meta to bucket
Args:
mocker: mocker fixture from pytest-mocker
"""
upload_blob_mock = mocker.patch("cd_helper.CDHelper._upload_blob",
return_value=upload_blob)
write_to_os_mock = mocker.patch("cd_helper.CDHelper._write_to_local_fs",
return_value={"": ""})
example = Example("name", "pipeline_id", SDK_JAVA, "filepath",
"code_of_example", "output_of_example",
STATUS_UNSPECIFIED, None)
CDHelper()._save_to_cloud_storage([example])
write_to_os_mock.assert_called_with(example)
upload_blob_mock.assert_called_with(source_file="",
destination_blob_name="")
def test__get_example(mock_get_name):
mock_get_name.return_value = "filepath"
result = _get_example(
"/root/filepath.java", "filepath.java", {
"name": "Name",
"description": "Description",
"multifile": "False",
"categories": [""],
"pipeline_options": "--option option"
})
assert result == Example(name="filepath",
sdk=SDK_JAVA,
filepath="/root/filepath.java",
code="data",
status=STATUS_UNSPECIFIED,
tag=Tag("Name", "Description", "False", [""],
"--option option"))
mock_get_name.assert_called_once_with("filepath.java")
def test__get_example(mock_get_name):
mock_get_name.return_value = "filepath"
tag = ExampleTag(
{
"name": "Name",
"description": "Description",
"multifile": "False",
"categories": [""],
"pipeline_options": "--option option"
}, "")
result = _get_example("/root/filepath.java", "filepath.java", tag)
assert result == Example(
name="filepath",
sdk=SDK_JAVA,
filepath="/root/filepath.java",
code="data",
status=STATUS_UNSPECIFIED,
tag=Tag("Name", "Description", "False", [""], "--option option"),
link="https://github.com/apache/beam/blob/master/root/filepath.java")
mock_get_name.assert_called_once_with("filepath.java")
async def test__verify_examples(mock_get_compile_output, mock_get_run_output):
helper = CIHelper()
object_meta = {
"name": "name",
"description": "description",
"multifile": False,
"categories": ["category-1", "category-2"],
"pipeline_options": "--option option",
"default_example": False
}
object_meta_def_ex = copy.copy(object_meta)
object_meta_def_ex["default_example"] = True
pipeline_id = str(uuid.uuid4())
default_example = Example(name="name",
pipeline_id=pipeline_id,
sdk=SDK_JAVA,
filepath="filepath",
code="code_of_example",
output="output_of_example",
status=STATUS_FINISHED,
tag=Tag(**object_meta_def_ex),
link="link")
finished_example = Example(name="name",
pipeline_id=pipeline_id,
sdk=SDK_JAVA,
filepath="filepath",
code="code_of_example",
output="output_of_example",
status=STATUS_FINISHED,
tag=Tag(**object_meta),
link="link")
examples_without_def_ex = [
finished_example,
finished_example,
]
examples_with_several_def_ex = [
default_example,
default_example,
]
examples_without_errors = [
default_example,
finished_example,
]
examples_with_errors = [
Example(name="name",
pipeline_id=pipeline_id,
sdk=SDK_JAVA,
filepath="filepath",
code="code_of_example",
output="output_of_example",
status=STATUS_VALIDATION_ERROR,
tag=Tag(**object_meta_def_ex),
link="link"),
Example(name="name",
pipeline_id=pipeline_id,
sdk=SDK_JAVA,
filepath="filepath",
code="code_of_example",
output="output_of_example",
status=STATUS_ERROR,
tag=Tag(**object_meta),
link="link"),
Example(name="name",
pipeline_id=pipeline_id,
sdk=SDK_JAVA,
filepath="filepath",
code="code_of_example",
output="output_of_example",
status=STATUS_COMPILE_ERROR,
tag=Tag(**object_meta),
link="link"),
Example(name="name",
pipeline_id=pipeline_id,
sdk=SDK_JAVA,
filepath="filepath",
code="code_of_example",
output="output_of_example",
status=STATUS_PREPARATION_ERROR,
tag=Tag(**object_meta),
link="link"),
Example(name="name",
pipeline_id=pipeline_id,
sdk=SDK_JAVA,
filepath="filepath",
code="code_of_example",
output="output_of_example",
status=STATUS_RUN_TIMEOUT,
tag=Tag(**object_meta),
link="link"),
Example(name="name",
pipeline_id=pipeline_id,
sdk=SDK_JAVA,
filepath="filepath",
code="code_of_example",
output="output_of_example",
status=STATUS_VALIDATION_ERROR,
tag=Tag(**object_meta),
link="link"),
Example(name="name",
pipeline_id=pipeline_id,
sdk=SDK_JAVA,
filepath="filepath",
code="code_of_example",
output="output_of_example",
status=STATUS_RUN_ERROR,
tag=Tag(**object_meta),
link="link"),
]
with pytest.raises(VerifyException):
await helper._verify_examples(examples_with_errors)
with pytest.raises(VerifyException):
await helper._verify_examples(examples_without_def_ex)
with pytest.raises(VerifyException):
await helper._verify_examples(examples_with_several_def_ex)
await helper._verify_examples(examples_without_errors)
