def get_papermill_parameters(compute_context, inputs, output_log_path):
check.inst_param(compute_context, 'compute_context',
SystemComputeExecutionContext)
check.param_invariant(
isinstance(compute_context.environment_dict, dict),
'compute_context',
'SystemComputeExecutionContext must have valid environment_dict',
)
check.dict_param(inputs, 'inputs', key_type=six.string_types)
run_id = compute_context.run_id
marshal_dir = '/tmp/dagstermill/{run_id}/marshal'.format(run_id=run_id)
mkdir_p(marshal_dir)
(handle, solid_subset) = ExecutionTargetHandle.get_handle(
compute_context.pipeline_def)
if not handle:
raise DagstermillError(
'Can\'t execute a dagstermill solid from a pipeline that wasn\'t instantiated using '
'an ExecutionTargetHandle')
dm_handle_kwargs = handle.data._asdict()
dm_handle_kwargs['pipeline_name'] = compute_context.pipeline_def.name
dm_context_dict = {
'output_log_path': output_log_path,
'marshal_dir': marshal_dir,
'environment_dict': compute_context.environment_dict,
}
dm_solid_handle_kwargs = compute_context.solid_handle._asdict()
parameters = {}
input_def_dict = compute_context.solid_def.input_dict
for input_name, input_value in inputs.items():
assert (
input_name not in RESERVED_INPUT_NAMES
), 'Dagstermill solids cannot have inputs named {input_name}'.format(
input_name=input_name)
dagster_type = input_def_dict[input_name].dagster_type
parameter_value = write_value(
dagster_type, input_value,
os.path.join(marshal_dir, 'input-{}'.format(input_name)))
parameters[input_name] = parameter_value
parameters['__dm_context'] = dm_context_dict
parameters['__dm_handle_kwargs'] = dm_handle_kwargs
parameters['__dm_pipeline_run_dict'] = pack_value(
compute_context.pipeline_run)
parameters['__dm_solid_handle_kwargs'] = dm_solid_handle_kwargs
parameters['__dm_solid_subset'] = solid_subset
parameters['__dm_instance_ref_dict'] = pack_value(
compute_context.instance.get_ref())
return parameters
def test_repo_yaml_file_dynamic_load():
handle = ExecutionTargetHandle.for_repo_yaml(
repository_yaml=script_relative_path('repository_file.yaml'))
repository = handle.build_repository_definition()
assert isinstance(repository, RepositoryDefinition)
assert repository.name == 'bar'
assert ExecutionTargetHandle.get_handle(repository) == (handle, None)
def test_repo_file_dynamic_load():
handle = ExecutionTargetHandle.for_repo_python_file(
python_file=script_relative_path('test_handle.py'), fn_name='define_bar_repo'
)
repository = handle.build_repository_definition()
assert isinstance(repository, RepositoryDefinition)
assert repository.name == 'bar'
assert ExecutionTargetHandle.get_handle(repository) == (handle, None)
def test_repo_module_dynamic_load():
handle = ExecutionTargetHandle.for_pipeline_module(
module_name='dagster_examples.intro_tutorial.repos', fn_name='repo_demo_pipeline'
)
repository = handle.build_repository_definition()
assert isinstance(repository, RepositoryDefinition)
assert repository.name == EPHEMERAL_NAME
assert ExecutionTargetHandle.get_handle(repository) == (handle, None)
def test_repo_yaml_module_dynamic_load():
handle = ExecutionTargetHandle.for_repo_yaml(
repository_yaml=file_relative_path(__file__, 'repository_module.yaml'))
handle = ExecutionTargetHandle.from_dict(handle.to_dict())
repository = handle.build_repository_definition()
assert isinstance(repository, RepositoryDefinition)
assert repository.name == 'hello_cereal_repository'
assert ExecutionTargetHandle.get_handle(repository) == (handle, None)
def test_repo_file_dynamic_load_from_pipeline():
handle = ExecutionTargetHandle.for_pipeline_python_file(
python_file=script_relative_path('test_handle.py'), fn_name='foo_pipeline'
)
repository = handle.build_repository_definition()
assert isinstance(repository, RepositoryDefinition)
assert repository.name == EPHEMERAL_NAME
assert repository.get_pipeline('foo_pipeline').name == 'foo_pipeline'
assert ExecutionTargetHandle.get_handle(repository) == (handle, None)
def test_repo_module_dynamic_load_from_pipeline():
handle = ExecutionTargetHandle.for_pipeline_module(
module_name='dagster_examples.intro_tutorial.repos', fn_name='repo_demo_pipeline'
)
repository = handle.build_repository_definition()
assert isinstance(repository, RepositoryDefinition)
assert repository.name == '<<unnamed>>'
assert repository.get_pipeline('repo_demo_pipeline').name == 'repo_demo_pipeline'
assert ExecutionTargetHandle.get_handle(repository) == handle
