def test_workflow_multiple_roots_forest_valid(self):
env = get_env("/test/support/", "/test/support/validations/")
# DAG has multiple disconnected roots, but is structurally sound.
step_params = {"config_id": "5", "parameters": {"test_param": "test"}}
params = {
"step_1": step_params,
"step_2": step_params,
"step_3": step_params,
"step_4": step_params,
"step_5": step_params
}
validated = self.validate_workflow(env,
'workflow_multiple_roots_forest',
'3', params)
assert (isinstance(validated, ValidWorkflow))
display = """
* step_1
| * step_2
* | step_4
| * step_3
* step_5
"""
assert (clean_string(validated.dag.display()) == clean_string(display))
assert (len(validated.dag.valid_steps) == 5)
def test_local_client(self):
base.set_log_level()
env = self.before()
config = Resources(env).get_config("8")
# DAG has cycle
step_params = {"config_id": "8", "parameters": {"test_param": "test"}}
params = {
"step_1": step_params,
"step_2": step_params,
"step_3": step_params,
"step_4": step_params,
"step_5": step_params,
"step_6": step_params,
}
wf = self.get_workflow(env, "workflow_local_scheduler")
if isinstance(wf, MalformedResource):
raise Exception(f"Workflow not found: {wf.get_message()}")
if isinstance(config, MalformedResource):
raise Exception(f"Config not found: {config.get_message()}")
parameters = WorkflowParameters(parameter_dict=params)
validated = wf.validate(env, config, parameters)
assert (isinstance(validated, ValidWorkflow))
operator_response = validated.run(env, Response())
info = """
Registering workflow dag test_workflow_local_scheduler_ea5b602c-261c-4e06-af21-375ea912b6a5 with local.
Registering DAG in local memory
Registered schedule test_workflow_local_scheduler_ea5b602c-261c-4e06-af21-375ea912b6a5
Triggering schedule: test_workflow_local_scheduler_ea5b602c-261c-4e06-af21-375ea912b6a5
Running dag
* step_1
| * step_2
| | * step_3
| |/
* | step_4
|/
* step_5
* step_6
Running step step_1
Running operator1
Running step step_2
Running operator2
Running step step_3
Running operator3
Running step step_4
Running operator4
Running step step_5
Running operator5
Running step step_6
Running operator6
"""
response = operator_response.response
assert (len(response.errors) == 0)
assert (clean_uuid(clean_string("\n".join(
response.info))) == clean_uuid(clean_string(info)))
def test_workflow_parallel_without_cycles_valid(self):
env = get_env("/test/support/", "/test/support/validations/")
step_params = {"config_id": "5", "parameters": {"test_param": "test"}}
params = {
"step_1": step_params,
"step_2": step_params,
"step_3": step_params,
"step_4": step_params,
"step_5": step_params
}
validated = self.validate_workflow(env,
'workflow_parallel_without_cycles',
'3', params)
assert (isinstance(validated, ValidWorkflow))
# TODO: Improve asciidag rendering of this case
display = """
* step_1
| * step_1
| * step_4
| * step_5
|/
* step_2
* step_3
"""
assert (clean_string(validated.dag.display()) == clean_string(display))
def test_workflow_with_multiple_roots_valid(self):
env = get_env("/test/support/", "/test/support/validations/")
step_params = {"config_id": "5", "parameters": {"test_param": "test"}}
params = {
"step_1": step_params,
"step_2": step_params,
"step_3": step_params,
"step_4": step_params,
"step_5": step_params
}
validated = self.validate_workflow(env, "workflow_multiple_roots", '3',
params)
assert (isinstance(validated, ValidWorkflow))
display = """
* step_1
| * step_4
|/
| * step_5
|/
* step_2
* step_3
"""
assert (clean_string(validated.dag.display()) == clean_string(display))
def test_kubernetes_operator(self):
config = SparkConfig({
'script_type': 'scala-test',
'spark_version': 'test.spark.version',
'main_class': 'test.main.Class',
'docker_image': 'docker/test-docker-image',
'application_file': 'test/jar/file/location/assembly.jar',
'driver_cores': 10,
'driver_memory_mbs': 1024,
'executors': 3,
'executor_memory_mb': 1024,
'executor_cores': 20
})
job = MergeJob(Path("test-input"), Path("test-output"), "parquet")
job.set_id("mason-spark-test_job")
merged = merge_config(config, job)
dumped = hdump(merged)
expects = """
apiVersion: sparkoperator.k8s.io/v1beta2
kind: SparkApplication
metadata:
name: mason-spark-test_job
namespace: default
spec:
arguments:
- --input_path
- test-input
- --output_path
- test-output
- --input_format
- parquet
- --job
- merge
driver:
coreLimit: 1200m
cores: 10
labels:
version: test.spark.version
memory: 1024m
serviceAccount: spark
volumeMounts:
- mountPath: /tmp
name: test-volume
executor:
cores: 20
instances: 3
labels:
version: test.spark.version
memory: 1024m
volumeMounts:
- mountPath: /tmp
name: test-volume
image: docker/test-docker-image
imagePullPolicy: Always
mainApplicationFile: local://test/jar/file/location/assembly.jar
mainClass: test.main.Class
mode: cluster
restartPolicy:
type: Never
sparkVersion: test.spark.version
type: Scala
volumes:
- hostPath:
path: /tmp
type: Directory
name: test-volume
"""
assert clean_string(dumped) == clean_string(expects)
def assert_equal(s1: str, s2: str):
assert (clean_string(s1) == clean_string(s2))
def clean(s: List[str]):
return list(map(lambda i: clean_uuid(clean_string(i)), s))