def test_error_dag_k8s(): pipeline_name = 'demo_error_pipeline' handle = ExecutionTargetHandle.for_pipeline_module( 'dagster_airflow_tests.test_project.dagster_airflow_demo', pipeline_name) environment_yaml = [ script_relative_path('test_project/env_s3.yaml'), ] environment_dict = load_yaml_from_glob_list(environment_yaml) run_id = str(uuid.uuid4()) execution_date = timezone.utcnow() dag, tasks = make_airflow_dag_kubernetized_for_handle( handle=handle, pipeline_name=pipeline_name, image=IMAGE, namespace='default', environment_dict=environment_dict, ) with pytest.raises(AirflowException) as exc_info: execute_tasks_in_dag(dag, tasks, run_id, execution_date) assert 'Exception: Unusual error' in str(exc_info.value)
def test_error_dag_k8s(dagster_docker_image, cluster_provider): print('--- :airflow: test_kubernetes.test_error_dag_k8s') _check_aws_creds_available() pipeline_name = 'demo_error_pipeline' handle = ReconstructableRepository.for_module('test_pipelines.repo', pipeline_name) environments_path = test_project_environments_path() environment_yaml = [ os.path.join(environments_path, 'env_s3.yaml'), ] environment_dict = load_yaml_from_glob_list(environment_yaml) run_id = make_new_run_id() execution_date = timezone.utcnow() dag, tasks = make_airflow_dag_kubernetized_for_handle( handle=handle, pipeline_name=pipeline_name, image=dagster_docker_image, namespace='default', environment_dict=environment_dict, op_kwargs={ 'config_file': os.environ['KUBECONFIG'], 'env_vars': { 'AWS_ACCESS_KEY_ID': os.environ['AWS_ACCESS_KEY_ID'], 'AWS_SECRET_ACCESS_KEY': os.environ['AWS_SECRET_ACCESS_KEY'], }, }, ) with pytest.raises(AirflowException) as exc_info: execute_tasks_in_dag(dag, tasks, run_id, execution_date) assert 'Exception: Unusual error' in str(exc_info.value)
def test_error_dag_k8s(dagster_docker_image, cluster_provider): _check_aws_creds_available() pipeline_name = "demo_error_pipeline" recon_repo = ReconstructableRepository.for_module( "dagster_test.test_project.test_pipelines.repo", "define_demo_execution_repo") environments_path = test_project_environments_path() environment_yaml = [ os.path.join(environments_path, "env_s3.yaml"), ] run_config = load_yaml_from_glob_list(environment_yaml) run_id = make_new_run_id() execution_date = timezone.utcnow() dag, tasks = make_airflow_dag_kubernetized_for_recon_repo( recon_repo=recon_repo, pipeline_name=pipeline_name, image=dagster_docker_image, namespace="default", run_config=run_config, op_kwargs={ "config_file": os.environ["KUBECONFIG"], "env_vars": { "AWS_ACCESS_KEY_ID": os.environ["AWS_ACCESS_KEY_ID"], "AWS_SECRET_ACCESS_KEY": os.environ["AWS_SECRET_ACCESS_KEY"], }, }, ) with pytest.raises(AirflowException) as exc_info: execute_tasks_in_dag(dag, tasks, run_id, execution_date) assert "Exception: Unusual error" in str(exc_info.value)
def test_error_dag_k8s( dagster_docker_image, environments_path ): # pylint: disable=redefined-outer-name pipeline_name = 'demo_error_pipeline' handle = ExecutionTargetHandle.for_pipeline_module('test_pipelines', pipeline_name) environment_yaml = [ os.path.join(environments_path, 'env_s3.yaml'), ] environment_dict = load_yaml_from_glob_list(environment_yaml) run_id = make_new_run_id() execution_date = timezone.utcnow() dag, tasks = make_airflow_dag_kubernetized_for_handle( handle=handle, pipeline_name=pipeline_name, image=dagster_docker_image, namespace='default', environment_dict=environment_dict, ) with pytest.raises(AirflowException) as exc_info: execute_tasks_in_dag(dag, tasks, run_id, execution_date) assert 'Exception: Unusual error' in str(exc_info.value)
def test_error_dag_python(environments_path): # pylint: disable=redefined-outer-name pipeline_name = 'demo_error_pipeline' handle = ExecutionTargetHandle.for_pipeline_module('test_pipelines', pipeline_name) environment_yaml = [ os.path.join(environments_path, 'env_filesystem.yaml'), ] environment_dict = load_yaml_from_glob_list(environment_yaml) execution_date = timezone.utcnow() dag, tasks = make_airflow_dag_for_handle(handle, pipeline_name, environment_dict) with pytest.raises(AirflowException) as exc_info: execute_tasks_in_dag(dag, tasks, run_id=str(uuid.uuid4()), execution_date=execution_date) assert 'Exception: Unusual error' in str(exc_info.value)
def test_error_dag_python(): # pylint: disable=redefined-outer-name pipeline_name = 'demo_error_pipeline' recon_repo = ReconstructableRepository.for_module('test_pipelines.repo', pipeline_name) environments_path = test_project_environments_path() environment_yaml = [ os.path.join(environments_path, 'env_filesystem.yaml'), ] run_config = load_yaml_from_glob_list(environment_yaml) execution_date = timezone.utcnow() dag, tasks = make_airflow_dag_for_recon_repo(recon_repo, pipeline_name, run_config) with pytest.raises(AirflowException) as exc_info: execute_tasks_in_dag(dag, tasks, run_id=make_new_run_id(), execution_date=execution_date) assert 'Exception: Unusual error' in str(exc_info.value)
def test_airflow_execution_date_tags_containerized(dagster_docker_image, ): # pylint: disable=redefined-outer-name, unused-argument pipeline_name = "demo_airflow_execution_date_pipeline" recon_repo = ReconstructableRepository.for_module( "dagster_test.test_project.test_pipelines.repo", "define_demo_execution_repo") environments_path = test_project_environments_path() environment_yaml = [ os.path.join(environments_path, "env_s3.yaml"), ] run_config = load_yaml_from_glob_list(environment_yaml) execution_date = timezone.utcnow() dag, tasks = make_airflow_dag_containerized_for_recon_repo( recon_repo, pipeline_name, dagster_docker_image, run_config) results = execute_tasks_in_dag(dag, tasks, run_id=make_new_run_id(), execution_date=execution_date) materialized_airflow_execution_date = None for result in results.values(): for event in result: if event.event_type_value == "STEP_MATERIALIZATION": materialization = event.event_specific_data.materialization materialization_entry = materialization.metadata_entries[0] materialized_airflow_execution_date = materialization_entry.entry_data.text assert execution_date.isoformat() == materialized_airflow_execution_date
def test_airflow_execution_date_tags(): pipeline_name = 'demo_airflow_execution_date_pipeline' recon_repo = ReconstructableRepository.for_module( 'dagster_test.test_project.test_pipelines.repo', pipeline_name) environments_path = test_project_environments_path() environment_yaml = [ os.path.join(environments_path, 'env_filesystem.yaml'), ] run_config = load_yaml_from_glob_list(environment_yaml) execution_date = timezone.utcnow() dag, tasks = make_airflow_dag_for_recon_repo(recon_repo, pipeline_name, run_config) results = execute_tasks_in_dag(dag, tasks, run_id=make_new_run_id(), execution_date=execution_date) materialized_airflow_execution_date = None for result in results.values(): for event in result: if event.event_type_value == 'STEP_MATERIALIZATION': materialization = event.event_specific_data.materialization materialization_entry = materialization.metadata_entries[0] materialized_airflow_execution_date = materialization_entry.entry_data.text assert execution_date.isoformat() == materialized_airflow_execution_date
def test_error_dag_containerized(dagster_docker_image): # pylint: disable=redefined-outer-name pipeline_name = 'demo_error_pipeline' handle = ReconstructableRepository.for_module('test_pipelines.repo', pipeline_name) environments_path = test_project_environments_path() environment_yaml = [ os.path.join(environments_path, 'env_s3.yaml'), ] environment_dict = load_yaml_from_glob_list(environment_yaml) run_id = make_new_run_id() execution_date = timezone.utcnow() dag, tasks = make_airflow_dag_containerized_for_handle( handle, pipeline_name, dagster_docker_image, environment_dict) with pytest.raises(AirflowException) as exc_info: execute_tasks_in_dag(dag, tasks, run_id, execution_date) assert 'Exception: Unusual error' in str(exc_info.value)
def test_error_dag_containerized(dagster_docker_image): # pylint: disable=redefined-outer-name pipeline_name = "demo_error_pipeline" recon_repo = ReconstructableRepository.for_module( "dagster_test.test_project.test_pipelines.repo", "define_demo_execution_repo") environments_path = test_project_environments_path() environment_yaml = [ os.path.join(environments_path, "env_s3.yaml"), ] run_config = load_yaml_from_glob_list(environment_yaml) run_id = make_new_run_id() execution_date = timezone.utcnow() dag, tasks = make_airflow_dag_containerized_for_recon_repo( recon_repo, pipeline_name, dagster_docker_image, run_config) with pytest.raises(AirflowException) as exc_info: execute_tasks_in_dag(dag, tasks, run_id, execution_date) assert "Exception: Unusual error" in str(exc_info.value)
def test_airflow_execution_date_tags_k8s(dagster_docker_image, cluster_provider): _check_aws_creds_available() pipeline_name = "demo_airflow_execution_date_pipeline" recon_repo = ReconstructableRepository.for_module( "dagster_test.test_project.test_pipelines.repo", "define_demo_execution_repo") environments_path = test_project_environments_path() environment_yaml = [ os.path.join(environments_path, "env_s3.yaml"), ] run_config = load_yaml_from_glob_list(environment_yaml) execution_date = timezone.utcnow() dag, tasks = make_airflow_dag_kubernetized_for_recon_repo( recon_repo=recon_repo, pipeline_name=pipeline_name, image=dagster_docker_image, namespace="default", run_config=run_config, op_kwargs={ "config_file": os.environ["KUBECONFIG"], "env_vars": { "AWS_ACCESS_KEY_ID": os.environ["AWS_ACCESS_KEY_ID"], "AWS_SECRET_ACCESS_KEY": os.environ["AWS_SECRET_ACCESS_KEY"], }, }, ) results = execute_tasks_in_dag(dag, tasks, run_id=make_new_run_id(), execution_date=execution_date) materialized_airflow_execution_date = None for result in results.values(): for event in result: if event.event_type_value == "STEP_MATERIALIZATION": materialization = event.event_specific_data.materialization materialization_entry = materialization.metadata_entries[0] materialized_airflow_execution_date = materialization_entry.entry_data.text assert execution_date.isoformat() == materialized_airflow_execution_date