def test_workflow_multiple_roots_forest_valid(self): env = get_env("/test/support/", "/test/support/validations/") # DAG has multiple disconnected roots, but is structurally sound. step_params = {"config_id": "5", "parameters": {"test_param": "test"}} params = { "step_1": step_params, "step_2": step_params, "step_3": step_params, "step_4": step_params, "step_5": step_params } validated = self.validate_workflow(env, 'workflow_multiple_roots_forest', '3', params) assert (isinstance(validated, ValidWorkflow)) display = """ * step_1 | * step_2 * | step_4 | * step_3 * step_5 """ assert (clean_string(validated.dag.display()) == clean_string(display)) assert (len(validated.dag.valid_steps) == 5)
def test_local_client(self): base.set_log_level() env = self.before() config = Resources(env).get_config("8") # DAG has cycle step_params = {"config_id": "8", "parameters": {"test_param": "test"}} params = { "step_1": step_params, "step_2": step_params, "step_3": step_params, "step_4": step_params, "step_5": step_params, "step_6": step_params, } wf = self.get_workflow(env, "workflow_local_scheduler") if isinstance(wf, MalformedResource): raise Exception(f"Workflow not found: {wf.get_message()}") if isinstance(config, MalformedResource): raise Exception(f"Config not found: {config.get_message()}") parameters = WorkflowParameters(parameter_dict=params) validated = wf.validate(env, config, parameters) assert (isinstance(validated, ValidWorkflow)) operator_response = validated.run(env, Response()) info = """ Registering workflow dag test_workflow_local_scheduler_ea5b602c-261c-4e06-af21-375ea912b6a5 with local. Registering DAG in local memory Registered schedule test_workflow_local_scheduler_ea5b602c-261c-4e06-af21-375ea912b6a5 Triggering schedule: test_workflow_local_scheduler_ea5b602c-261c-4e06-af21-375ea912b6a5 Running dag * step_1 | * step_2 | | * step_3 | |/ * | step_4 |/ * step_5 * step_6 Running step step_1 Running operator1 Running step step_2 Running operator2 Running step step_3 Running operator3 Running step step_4 Running operator4 Running step step_5 Running operator5 Running step step_6 Running operator6 """ response = operator_response.response assert (len(response.errors) == 0) assert (clean_uuid(clean_string("\n".join( response.info))) == clean_uuid(clean_string(info)))
def test_workflow_parallel_without_cycles_valid(self): env = get_env("/test/support/", "/test/support/validations/") step_params = {"config_id": "5", "parameters": {"test_param": "test"}} params = { "step_1": step_params, "step_2": step_params, "step_3": step_params, "step_4": step_params, "step_5": step_params } validated = self.validate_workflow(env, 'workflow_parallel_without_cycles', '3', params) assert (isinstance(validated, ValidWorkflow)) # TODO: Improve asciidag rendering of this case display = """ * step_1 | * step_1 | * step_4 | * step_5 |/ * step_2 * step_3 """ assert (clean_string(validated.dag.display()) == clean_string(display))
def test_workflow_with_multiple_roots_valid(self): env = get_env("/test/support/", "/test/support/validations/") step_params = {"config_id": "5", "parameters": {"test_param": "test"}} params = { "step_1": step_params, "step_2": step_params, "step_3": step_params, "step_4": step_params, "step_5": step_params } validated = self.validate_workflow(env, "workflow_multiple_roots", '3', params) assert (isinstance(validated, ValidWorkflow)) display = """ * step_1 | * step_4 |/ | * step_5 |/ * step_2 * step_3 """ assert (clean_string(validated.dag.display()) == clean_string(display))
def test_kubernetes_operator(self): config = SparkConfig({ 'script_type': 'scala-test', 'spark_version': 'test.spark.version', 'main_class': 'test.main.Class', 'docker_image': 'docker/test-docker-image', 'application_file': 'test/jar/file/location/assembly.jar', 'driver_cores': 10, 'driver_memory_mbs': 1024, 'executors': 3, 'executor_memory_mb': 1024, 'executor_cores': 20 }) job = MergeJob(Path("test-input"), Path("test-output"), "parquet") job.set_id("mason-spark-test_job") merged = merge_config(config, job) dumped = hdump(merged) expects = """ apiVersion: sparkoperator.k8s.io/v1beta2 kind: SparkApplication metadata: name: mason-spark-test_job namespace: default spec: arguments: - --input_path - test-input - --output_path - test-output - --input_format - parquet - --job - merge driver: coreLimit: 1200m cores: 10 labels: version: test.spark.version memory: 1024m serviceAccount: spark volumeMounts: - mountPath: /tmp name: test-volume executor: cores: 20 instances: 3 labels: version: test.spark.version memory: 1024m volumeMounts: - mountPath: /tmp name: test-volume image: docker/test-docker-image imagePullPolicy: Always mainApplicationFile: local://test/jar/file/location/assembly.jar mainClass: test.main.Class mode: cluster restartPolicy: type: Never sparkVersion: test.spark.version type: Scala volumes: - hostPath: path: /tmp type: Directory name: test-volume """ assert clean_string(dumped) == clean_string(expects)
def assert_equal(s1: str, s2: str): assert (clean_string(s1) == clean_string(s2))
def clean(s: List[str]): return list(map(lambda i: clean_uuid(clean_string(i)), s))