def test_for_repo_fn_with_pipeline_name():
handle = ExecutionTargetHandle.for_repo_fn(define_bar_repo)
handle = ExecutionTargetHandle.from_dict(handle.to_dict())
repo = handle.build_repository_definition()
assert repo.name == 'bar'
with pytest.raises(DagsterInvariantViolationError) as exc_info:
handle.build_pipeline_definition()
assert (
str(exc_info.value) ==
'Cannot construct a pipeline from a repository-based '
'ExecutionTargetHandle without a pipeline name. Use with_pipeline_name() to construct a '
'pipeline ExecutionTargetHandle.')
handle_for_pipeline = handle.with_pipeline_name('foo_pipeline')
repo = handle_for_pipeline.build_repository_definition()
assert repo.name == 'bar'
pipe = handle_for_pipeline.build_pipeline_definition()
assert pipe.name == 'foo_pipeline'
handle_double = handle_for_pipeline.with_pipeline_name('foo_pipeline')
pipe = handle_double.build_pipeline_definition()
assert pipe.name == 'foo_pipeline'
def test_repo_entrypoints():
module = importlib.import_module('dagster_examples.intro_tutorial.repos')
expected = LoaderEntrypoint(module,
'dagster_examples.intro_tutorial.repos',
'define_repo')
handle = ExecutionTargetHandle.for_repo_yaml(
file_relative_path(__file__, 'repository.yaml'))
assert handle.entrypoint.module == expected.module
assert handle.entrypoint.module_name == expected.module_name
assert handle.entrypoint.fn_name == expected.fn_name
assert handle.entrypoint.from_handle == handle
handle = ExecutionTargetHandle.from_dict(handle.to_dict())
assert handle.entrypoint.module == expected.module
assert handle.entrypoint.module_name == expected.module_name
assert handle.entrypoint.fn_name == expected.fn_name
module = importlib.import_module('dagster')
expected = LoaderEntrypoint(module, 'dagster', 'define_bar_repo')
handle = ExecutionTargetHandle.for_repo_module(module_name='dagster',
fn_name='define_bar_repo')
assert handle.entrypoint.module == expected.module
assert handle.entrypoint.module_name == expected.module_name
assert handle.entrypoint.fn_name == expected.fn_name
assert handle.entrypoint.from_handle == handle
handle = ExecutionTargetHandle.from_dict(handle.to_dict())
assert handle.entrypoint.module == expected.module
assert handle.entrypoint.module_name == expected.module_name
assert handle.entrypoint.fn_name == expected.fn_name
python_file = file_relative_path(__file__, 'bar_repo.py')
module = import_module_from_path('bar_repo', python_file)
expected = LoaderEntrypoint(module, 'bar_repo', 'define_bar_repo')
handle = ExecutionTargetHandle.for_repo_python_file(
python_file=python_file, fn_name='define_bar_repo')
assert handle.entrypoint.module == expected.module
assert handle.entrypoint.module_name == expected.module_name
assert handle.entrypoint.fn_name == expected.fn_name
assert handle.entrypoint.from_handle == handle
handle = ExecutionTargetHandle.from_dict(handle.to_dict())
assert handle.entrypoint.module == expected.module
assert handle.entrypoint.module_name == expected.module_name
assert handle.entrypoint.fn_name == expected.fn_name
def test_exc_target_handle():
res = ExecutionTargetHandle.for_pipeline_python_file(__file__, 'foo_pipeline')
assert os.path.abspath(res.data.python_file) == os.path.abspath(__file__)
assert res.data.fn_name == 'foo_pipeline'
res = ExecutionTargetHandle.from_dict(res.to_dict())
assert os.path.abspath(res.data.python_file) == os.path.abspath(__file__)
assert res.data.fn_name == 'foo_pipeline'
def test_repo_yaml_file_dynamic_load():
handle = ExecutionTargetHandle.for_repo_yaml(
repository_yaml=file_relative_path(__file__, 'repository_file.yaml'))
handle = ExecutionTargetHandle.from_dict(handle.to_dict())
repository = handle.build_repository_definition()
assert isinstance(repository, RepositoryDefinition)
assert repository.name == 'bar'
assert ExecutionTargetHandle.get_handle(repository) == (handle, None)
def test_repo_yaml_module_dynamic_load():
handle = ExecutionTargetHandle.for_repo_yaml(
repository_yaml=script_relative_path('repository_module.yaml'))
handle = ExecutionTargetHandle.from_dict(handle.to_dict())
repository = handle.build_repository_definition()
assert isinstance(repository, RepositoryDefinition)
assert repository.name == 'hello_cereal_repository'
assert ExecutionTargetHandle.get_handle(repository) == (handle, None)
def test_repo_file_dynamic_load():
handle = ExecutionTargetHandle.for_repo_python_file(
python_file=file_relative_path(__file__, 'test_handle.py'),
fn_name='define_bar_repo')
handle = ExecutionTargetHandle.from_dict(handle.to_dict())
repository = handle.build_repository_definition()
assert isinstance(repository, RepositoryDefinition)
assert repository.name == 'bar'
assert ExecutionTargetHandle.get_handle(repository) == (handle, None)
def test_repo_module_dynamic_load():
handle = ExecutionTargetHandle.for_pipeline_module(
module_name='dagster_examples.intro_tutorial.repos',
fn_name='hello_cereal_pipeline')
handle = ExecutionTargetHandle.from_dict(handle.to_dict())
repository = handle.build_repository_definition()
assert isinstance(repository, RepositoryDefinition)
assert repository.name == EPHEMERAL_NAME
assert ExecutionTargetHandle.get_handle(repository) == (handle, None)
def _execute_plan(_self, instance_ref_dict, handle_dict, run_id, step_keys,
retries_dict):
check.dict_param(instance_ref_dict, 'instance_ref_dict')
check.dict_param(handle_dict, 'handle_dict')
check.str_param(run_id, 'run_id')
check.list_param(step_keys, 'step_keys', of_type=str)
check.dict_param(retries_dict, 'retries_dict')
instance_ref = InstanceRef.from_dict(instance_ref_dict)
instance = DagsterInstance.from_ref(instance_ref)
handle = ExecutionTargetHandle.from_dict(handle_dict)
retries = Retries.from_config(retries_dict)
pipeline_run = instance.get_run_by_id(run_id)
check.invariant(pipeline_run, 'Could not load run {}'.format(run_id))
pipeline_def = handle.build_pipeline_definition().build_sub_pipeline(
pipeline_run.selector.solid_subset)
step_keys_str = ", ".join(step_keys)
execution_plan = create_execution_plan(
pipeline_def,
pipeline_run.environment_dict,
mode=pipeline_run.mode,
step_keys_to_execute=pipeline_run.step_keys_to_execute,
).build_subset_plan(step_keys)
engine_event = instance.report_engine_event(
'Executing steps {} in celery worker'.format(step_keys_str),
pipeline_run,
EngineEventData(
[
EventMetadataEntry.text(step_keys_str, 'step_keys'),
],
marker_end=DELEGATE_MARKER,
),
CeleryEngine,
step_key=execution_plan.step_key_for_single_step_plans(),
)
events = [engine_event]
for step_event in execute_plan_iterator(
execution_plan,
pipeline_run=pipeline_run,
environment_dict=pipeline_run.environment_dict,
instance=instance,
retries=retries,
):
events.append(step_event)
serialized_events = [
serialize_dagster_namedtuple(event) for event in events
]
return serialized_events
def test_repo_file_dynamic_load_from_pipeline():
handle = ExecutionTargetHandle.for_pipeline_python_file(
python_file=file_relative_path(__file__, 'test_handle.py'),
fn_name='foo_pipeline')
handle = ExecutionTargetHandle.from_dict(handle.to_dict())
repository = handle.build_repository_definition()
assert isinstance(repository, RepositoryDefinition)
assert repository.name == EPHEMERAL_NAME
assert repository.get_pipeline('foo_pipeline').name == 'foo_pipeline'
assert ExecutionTargetHandle.get_handle(repository) == (handle, None)
def _execute_query(_self, handle_dict, variables, instance_ref_dict):
instance_ref = InstanceRef.from_dict(instance_ref_dict)
handle = ExecutionTargetHandle.from_dict(handle_dict)
events = execute_execute_plan_mutation(
handle=handle,
variables=variables,
instance_ref=instance_ref,
)
serialized_events = [
serialize_dagster_namedtuple(event) for event in events
]
return serialized_events
def test_build_repository_definition():
handle = ExecutionTargetHandle.for_repo_python_file(
__file__, 'define_foo_pipeline')
handle = ExecutionTargetHandle.from_dict(handle.to_dict())
repo = handle.build_repository_definition()
assert repo.name == EPHEMERAL_NAME
