def test_env_list_to_dict_function():
env_variables = [
"KEY=value",
None,
"",
" =empty_key",
"=no_key",
"EMPTY_VALUE= ",
"NO_VALUE=",
"KEY2=value2",
"TWO_EQUALS=KEY=value",
"==",
]
env_variables_dict = {"KEY": "value", "KEY2": "value2", "EMPTY_VALUE": " ", "TWO_EQUALS": "KEY=value"}
component_parameters = {
"filename": "elyra/pipeline/tests/resources/archive/test.ipynb",
"env_vars": env_variables,
"runtime_image": "tensorflow/tensorflow:latest",
}
test_operation = GenericOperation(
id="test-id",
type="execution-node",
classifier="execute-notebook-node",
name="test",
component_params=component_parameters,
)
assert test_operation.env_vars_as_dict() == env_variables_dict
def _collect_envs(self, operation: GenericOperation, **kwargs) -> Dict:
"""
Collect the envs stored on the Operation and set the system-defined ELYRA_RUNTIME_ENV
Note: subclasses should call their superclass (this) method first.
:return: dictionary containing environment name/value pairs
"""
envs: Dict = operation.env_vars_as_dict(logger=self.log)
envs["ELYRA_RUNTIME_ENV"] = self.name
# set environment variables for Minio/S3 access, in the following order of precedence:
# 1. use `cos_secret`
# 2. use `cos_username` and `cos_password`
if "cos_secret" in kwargs and kwargs["cos_secret"]:
# ensure the AWS_ACCESS_* envs are NOT set
envs.pop("AWS_ACCESS_KEY_ID", None)
envs.pop("AWS_SECRET_ACCESS_KEY", None)
else:
# set AWS_ACCESS_KEY_ID, if defined
if "cos_username" in kwargs and kwargs["cos_username"]:
envs["AWS_ACCESS_KEY_ID"] = kwargs["cos_username"]
else:
envs.pop("AWS_ACCESS_KEY_ID", None)
# set AWS_SECRET_ACCESS_KEY, if defined
if "cos_password" in kwargs and kwargs["cos_password"]:
envs["AWS_SECRET_ACCESS_KEY"] = kwargs["cos_password"]
else:
envs.pop("AWS_SECRET_ACCESS_KEY", None)
# Convey pipeline logging enablement to operation
envs["ELYRA_ENABLE_PIPELINE_INFO"] = str(self.enable_pipeline_info)
return envs
def test_generate_dependency_archive(processor):
pipelines_test_file = processor.root_dir + "/elyra/tests/pipeline/resources/archive/test.ipynb"
pipeline_dependencies = ["airflow.json"]
correct_filelist = ["test.ipynb", "airflow.json"]
component_parameters = {
"filename": pipelines_test_file,
"dependencies": pipeline_dependencies,
"runtime_image": "tensorflow/tensorflow:latest",
}
test_operation = GenericOperation(
id="123e4567-e89b-12d3-a456-426614174000",
type="execution-node",
classifier="execute-notebook-node",
name="test",
component_params=component_parameters,
)
archive_location = processor._generate_dependency_archive(test_operation)
tar_content = []
with tarfile.open(archive_location, "r:gz") as tar:
for tarinfo in tar:
if tarinfo.isreg():
print(tarinfo.name)
tar_content.append(tarinfo.name)
assert sorted(correct_filelist) == sorted(tar_content)
def test_collect_envs(processor):
pipelines_test_file = "elyra/pipeline/tests/resources/archive/test.ipynb"
# add system-owned envs with bogus values to ensure they get set to system-derived values,
# and include some user-provided edge cases
operation_envs = [
'ELYRA_RUNTIME_ENV="bogus_runtime"',
'ELYRA_ENABLE_PIPELINE_INFO="bogus_pipeline"',
"ELYRA_WRITABLE_CONTAINER_DIR=", # simulate operation reference in pipeline
'AWS_ACCESS_KEY_ID="bogus_key"',
'AWS_SECRET_ACCESS_KEY="bogus_secret"',
"USER_EMPTY_VALUE= ",
"USER_TWO_EQUALS=KEY=value",
"USER_NO_VALUE=",
]
component_parameters = {
"filename": pipelines_test_file,
"env_vars": operation_envs,
"runtime_image": "tensorflow/tensorflow:latest",
}
test_operation = GenericOperation(
id="this-is-a-test-id",
type="execution-node",
classifier="execute-notebook-node",
name="test",
component_params=component_parameters,
)
envs = processor._collect_envs(test_operation,
cos_secret=None,
cos_username="******",
cos_password="******")
assert envs["ELYRA_RUNTIME_ENV"] == "kfp"
assert envs["AWS_ACCESS_KEY_ID"] == "Alice"
assert envs["AWS_SECRET_ACCESS_KEY"] == "secret"
assert envs["ELYRA_ENABLE_PIPELINE_INFO"] == "True"
assert envs["ELYRA_WRITABLE_CONTAINER_DIR"] == "/tmp"
assert envs["USER_EMPTY_VALUE"] == " "
assert envs["USER_TWO_EQUALS"] == "KEY=value"
assert "USER_NO_VALUE" not in envs
# Repeat with non-None secret - ensure user and password envs are not present, but others are
envs = processor._collect_envs(test_operation,
cos_secret="secret",
cos_username="******",
cos_password="******")
assert envs["ELYRA_RUNTIME_ENV"] == "kfp"
assert "AWS_ACCESS_KEY_ID" not in envs
assert "AWS_SECRET_ACCESS_KEY" not in envs
assert envs["ELYRA_ENABLE_PIPELINE_INFO"] == "True"
assert envs["ELYRA_WRITABLE_CONTAINER_DIR"] == "/tmp"
assert envs["USER_EMPTY_VALUE"] == " "
assert envs["USER_TWO_EQUALS"] == "KEY=value"
assert "USER_NO_VALUE" not in envs
def test_fail_create_operation_missing_type():
component_parameters = {
"filename": "elyra/pipeline/tests/resources/archive/test.ipynb",
"runtime_image": "tensorflow/tensorflow:latest",
}
with pytest.raises(TypeError):
GenericOperation(
id="test-id", classifier="execute-notebook-node", name="test", component_params=component_parameters
)
def good_operation():
component_parameters = {
"filename": "elyra/pipeline/tests/resources/archive/test.ipynb",
"runtime_image": "tensorflow/tensorflow:latest",
}
test_operation = GenericOperation(
id="test-id",
type="execution-node",
classifier="execute-notebook-node",
name="test",
component_params=component_parameters,
)
return test_operation
def test_create_operation_correct_naming():
label = "test.ipynb"
filename = "elyra/pipeline/tests/resources/archive/" + label
component_parameters = {"filename": filename, "runtime_image": "tensorflow/tensorflow:latest"}
test_operation = GenericOperation(
id="test-id",
type="execution-node",
classifier="execute-notebook-node",
name=label,
component_params=component_parameters,
)
assert test_operation.name == label.split(".")[0]
def test_fail_creating_operation_with_negative_memory_resources():
component_parameters = {
"filename": "elyra/pipeline/tests/resources/archive/test.ipynb",
"memory": "-1",
"runtime_image": "tensorflow/tensorflow:latest",
}
with pytest.raises(ValueError):
GenericOperation(
id="test-id",
type="execution-node",
classifier="execute-notebook-node",
name="test",
component_params=component_parameters,
)
def valid_operation():
component_parameters = {
"filename": "{{filename}}",
"runtime_image": "{{runtime_image}}",
"env_vars": ["var1=var1", "var2=var2"],
"dependencies": ["a.txt", "b.txt", "c.txt"],
"outputs": ["d.txt", "e.txt", "f.txt"],
}
return GenericOperation(
id="{{uuid}}",
type="execution_node",
classifier="execute-notebook-node",
name="{{label}}",
component_params=component_parameters,
)
def test_fail_validate_max_resource_value_exceeded():
system_max_size = str(sys.maxsize)
component_parameters = {
"filename": "elyra/pipeline/tests/resources/archive/test.ipynb",
"memory": system_max_size,
"runtime_image": "tensorflow/tensorflow:latest",
}
with pytest.raises(ValueError):
GenericOperation(
id="test-id",
type="execution-node",
classifier="execute-notebook-node",
name="test",
component_params=component_parameters,
)
def test_fail_operations_are_equal(good_operation):
parent_operation_ids = ["id-123123-123123-123123", "id-456456-456456-456456"]
component_parameters = {
"filename": "elyra/pipeline/tests/resources/archive/test.ipynb",
"runtime_image": "tensorflow/tensorflow:latest",
}
compare_operation = GenericOperation(
id="test-id",
type="execution-node",
classifier="execute-notebook-node",
name="test",
parent_operation_ids=parent_operation_ids,
component_params=component_parameters,
)
with pytest.raises(AssertionError):
assert compare_operation == good_operation
def test_validate_max_resource_value():
system_max_size = str(sys.maxsize - 1)
component_parameters = {
"filename": "elyra/pipeline/tests/resources/archive/test.ipynb",
"memory": system_max_size,
"runtime_image": "tensorflow/tensorflow:latest",
}
test_operation = GenericOperation(
id="test-id",
type="execution-node",
classifier="execute-notebook-node",
name="test",
component_params=component_parameters,
)
assert test_operation.memory == system_max_size
def test_validate_resource_values_as_none():
component_parameters = {
"filename": "elyra/pipeline/tests/resources/archive/test.ipynb",
"runtime_image": "tensorflow/tensorflow:latest",
}
test_operation = GenericOperation(
id="test-id",
type="execution-node",
classifier="execute-notebook-node",
name="test",
component_params=component_parameters,
)
assert test_operation.cpu is None
assert test_operation.gpu is None
assert test_operation.memory is None
def test_create_operation_with_dependencies():
dependencies = ["elyra/pipline/tests/resources", "elyra/pipline/tests/resources/archive"]
component_parameters = {
"filename": "elyra/pipeline/tests/resources/archive/test.ipynb",
"dependencies": dependencies,
"runtime_image": "tensorflow/tensorflow:latest",
}
test_operation = GenericOperation(
id="test-id",
type="execution-node",
classifier="execute-notebook-node",
name="test",
component_params=component_parameters,
)
assert test_operation.dependencies == dependencies
def test_create_operation_with_parent_operations():
parent_operation_ids = ["id-123123-123123-123123", "id-456456-456456-456456"]
component_parameters = {
"filename": "elyra/pipeline/tests/resources/archive/test.ipynb",
"runtime_image": "tensorflow/tensorflow:latest",
}
test_operation = GenericOperation(
id="test-id",
type="execution-node",
classifier="execute-notebook-node",
name="test",
parent_operation_ids=parent_operation_ids,
component_params=component_parameters,
)
assert test_operation.parent_operation_ids == parent_operation_ids
def test_create_operation_include_subdirectories():
include_subdirectories = True
component_parameters = {
"filename": "elyra/pipeline/tests/resources/archive/test.ipynb",
"include_subdirectories": include_subdirectories,
"runtime_image": "tensorflow/tensorflow:latest",
}
test_operation = GenericOperation(
id="test-id",
type="execution-node",
classifier="execute-notebook-node",
name="test",
component_params=component_parameters,
)
assert test_operation.include_subdirectories == include_subdirectories
def test_create_operation_with_environmental_variables():
env_variables = ['FOO="Bar"', 'BAR="Foo']
component_parameters = {
"filename": "elyra/pipeline/tests/resources/archive/test.ipynb",
"env_vars": env_variables,
"runtime_image": "tensorflow/tensorflow:latest",
}
test_operation = GenericOperation(
id="test-id",
type="execution-node",
classifier="execute-notebook-node",
name="test",
component_params=component_parameters,
)
assert test_operation.env_vars == env_variables
def test_create_operation_with_outputs():
outputs = ["output1.txt", "output2.txt"]
component_parameters = {
"filename": "elyra/pipeline/tests/resources/archive/test.ipynb",
"outputs": outputs,
"runtime_image": "tensorflow/tensorflow:latest",
}
test_operation = GenericOperation(
id="test-id",
type="execution-node",
classifier="execute-notebook-node",
name="test",
component_params=component_parameters,
)
assert test_operation.outputs == outputs
def test_get_dependency_archive_name(processor):
pipelines_test_file = "elyra/pipeline/tests/resources/archive/test.ipynb"
correct_filename = "test-this-is-a-test-id.tar.gz"
component_parameters = {
"filename": pipelines_test_file,
"runtime_image": "tensorflow/tensorflow:latest"
}
test_operation = GenericOperation(
id="this-is-a-test-id",
type="execution-node",
classifier="execute-notebook-node",
name="test",
component_params=component_parameters,
)
filename = processor._get_dependency_archive_name(test_operation)
assert filename == correct_filename
def test_validate_gpu_accepts_zero_as_value():
component_parameters = {
"filename": "elyra/pipeline/tests/resources/archive/test.ipynb",
"cpu": "4",
"gpu": "0",
"memory": "10",
"runtime_image": "tensorflow/tensorflow:latest",
}
test_operation = GenericOperation(
id="test-id",
type="execution-node",
classifier="execute-notebook-node",
name="test",
component_params=component_parameters,
)
assert test_operation.gpu == "0"
def test_fail_generate_dependency_archive(processor):
pipelines_test_file = processor.root_dir + "/elyra/pipeline/tests/resources/archive/test.ipynb"
pipeline_dependencies = ["non_existent_file.json"]
component_parameters = {
"filename": pipelines_test_file,
"dependencies": pipeline_dependencies,
"runtime_image": "tensorflow/tensorflow:latest",
}
test_operation = GenericOperation(
id="123e4567-e89b-12d3-a456-426614174000",
type="execution-node",
classifier="execute-notebook-node",
name="test",
component_params=component_parameters,
)
with pytest.raises(Exception):
processor._generate_dependency_archive(test_operation)
def test_get_dependency_source_dir(processor):
pipelines_test_file = "elyra/pipeline/tests/resources/archive/test.ipynb"
processor.root_dir = "/this/is/an/abs/path/"
correct_filepath = "/this/is/an/abs/path/elyra/pipeline/tests/resources/archive"
component_parameters = {
"filename": pipelines_test_file,
"runtime_image": "tensorflow/tensorflow:latest"
}
test_operation = GenericOperation(
id="123e4567-e89b-12d3-a456-426614174000",
type="execution-node",
classifier="execute-notebook-node",
name="test",
component_params=component_parameters,
)
filepath = processor._get_dependency_source_dir(test_operation)
assert filepath == correct_filepath
def get_operation(self) -> GenericOperation:
self.env_vars = []
if self.fail: # NODE_FILENAME is required, so skip if triggering failure
if "NODE_FILENAME" in os.environ: # remove entry if present
os.environ.pop("NODE_FILENAME")
else:
self.env_vars.append(f"NODE_FILENAME={self.filename}")
if self.inputs:
self.env_vars.append(f"INPUT_FILENAMES={';'.join(self.inputs)}")
if self.outputs:
self.env_vars.append(f"OUTPUT_FILENAMES={';'.join(self.outputs)}")
# Convey the pipeline name
assert self.pipeline_name is not None, "Pipeline name has not been set during construction!"
self.env_vars.append(f"PIPELINE_NAME={self.pipeline_name}")
# Add system-owned here with bogus or no value...
self.env_vars.append("ELYRA_RUNTIME_ENV=bogus_runtime")
component_parameters = {
"filename": self.filename,
"runtime_image": self.image_name or "NA",
"dependencies": self.dependencies,
"env_vars": self.env_vars,
"inputs": self.inputs,
"outputs": self.outputs,
}
return GenericOperation(
self.id,
"execution_node",
self.name,
self.classifier,
parent_operation_ids=self.parent_operations,
component_params=component_parameters,
)
def _collect_envs(operation: GenericOperation, elyra_run_name: str) -> Dict:
envs = os.environ.copy() # Make sure this process's env is "available" in the kernel subprocess
envs.update(operation.env_vars_as_dict())
envs["ELYRA_RUNTIME_ENV"] = "local" # Special case
envs["ELYRA_RUN_NAME"] = elyra_run_name
return envs
