def _execution_plan_or_error(subplan_execution_args, dauphin_pipeline,
evaluate_value_result):
check.inst_param(subplan_execution_args, 'subplan_execution_args',
SubplanExecutionArgs)
check.inst_param(dauphin_pipeline, 'dauphin_pipeline', DauphinPipeline)
check.inst_param(evaluate_value_result, 'evaluate_value_result',
EvaluateValueResult)
execution_plan = create_execution_plan_with_typed_environment(
dauphin_pipeline.get_dagster_pipeline(),
construct_environment_config(evaluate_value_result.value),
subplan_execution_args.execution_metadata,
)
invalid_keys = []
for step_key in subplan_execution_args.step_keys:
if not execution_plan.has_step(step_key):
invalid_keys.append(step_key)
if invalid_keys:
return EitherError(
subplan_execution_args.info.schema.type_named(
'StartSubplanExecutionInvalidStepsError')(
invalid_step_keys=invalid_keys))
else:
return EitherValue(execution_plan)
def test_optional_solid_with_optional_scalar_config():
def _assert_config_none(context, value):
assert context.solid_config is value
pipeline_def = PipelineDefinition(
name='some_pipeline',
solids=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int, is_optional=True),
inputs=[],
outputs=[],
transform_fn=lambda context, _inputs: _assert_config_none(
context, 234),
)
],
)
env_type = pipeline_def.environment_type
assert env_type.fields['solids'].is_optional is True
solids_type = env_type.fields['solids'].config_type
assert solids_type.fields['int_config_solid'].is_optional is True
solids_default_obj = construct_solid_dictionary(
throwing_evaluate_config_value(solids_type, {}))
assert solids_default_obj['int_config_solid'].config is None
env_obj = construct_environment_config(
throwing_evaluate_config_value(pipeline_def.environment_type, {}))
assert env_obj.solids['int_config_solid'].config is None
def test_whole_environment():
pipeline_def = PipelineDefinition(
name='some_pipeline',
context_definitions={
'test':
PipelineContextDefinition(
config_field=Field(Any),
context_fn=lambda *args: ExecutionContext())
},
solids=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
inputs=[],
outputs=[],
transform_fn=lambda *args: None,
),
SolidDefinition(name='no_config_solid',
inputs=[],
outputs=[],
transform_fn=lambda *args: None),
],
)
environment_type = pipeline_def.environment_type
assert environment_type.fields[
'context'].config_type.name == 'SomePipeline.ContextConfig'
solids_type = environment_type.fields['solids'].config_type
assert solids_type.name == 'SomePipeline.SolidsConfigDictionary'
assert (solids_type.fields['int_config_solid'].config_type.name ==
'SomePipeline.SolidConfig.IntConfigSolid')
assert (environment_type.fields['expectations'].config_type.name ==
'SomePipeline.ExpectationsConfig')
env = construct_environment_config(
throwing_evaluate_config_value(
pipeline_def.environment_type,
{
'context': {
'test': {
'config': 1
}
},
'solids': {
'int_config_solid': {
'config': 123
}
}
},
))
assert isinstance(env, EnvironmentConfig)
assert env.context == ContextConfig('test', 1, persistence={'file': {}})
assert env.solids == {'int_config_solid': SolidConfig(123)}
assert env.expectations == ExpectationsConfig(evaluate=True)
def test_default_environment():
pipeline_def = PipelineDefinition(solids=[
SolidDefinition(name='some_solid',
inputs=[],
outputs=[],
transform_fn=lambda *args: None)
])
env_obj = construct_environment_config(
throwing_evaluate_config_value(pipeline_def.environment_type, {}))
assert env_obj.expectations.evaluate is True
def create_plan(pipeline):
config_or_error = _config_or_error_from_pipeline(
info, pipeline, config)
return config_or_error.chain(
lambda evaluate_value_result: info.schema.
type_named('ExecutionPlan')(
pipeline,
create_execution_plan_with_typed_environment(
pipeline.get_dagster_pipeline(),
construct_environment_config(evaluate_value_result.value),
ExecutionMetadata(),
),
))
def test_required_solid_with_required_subfield():
pipeline_def = PipelineDefinition(
name='some_pipeline',
solids=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Dict({'required_field': Field(String)})),
inputs=[],
outputs=[],
transform_fn=lambda *_args: None,
)
],
)
env_type = pipeline_def.environment_type
assert env_type.fields['solids'].is_optional is False
assert env_type.fields['solids'].config_type
solids_type = env_type.fields['solids'].config_type
assert solids_type.fields['int_config_solid'].is_optional is False
int_config_solid_type = solids_type.fields['int_config_solid'].config_type
assert int_config_solid_type.fields['config'].is_optional is False
assert env_type.fields['context'].is_optional
assert env_type.fields['execution'].is_optional
assert env_type.fields['expectations'].is_optional
env_obj = construct_environment_config(
throwing_evaluate_config_value(
pipeline_def.environment_type,
{
'solids': {
'int_config_solid': {
'config': {
'required_field': 'foobar'
}
}
}
},
))
assert env_obj.solids['int_config_solid'].config[
'required_field'] == 'foobar'
with pytest.raises(DagsterEvaluateConfigValueError):
throwing_evaluate_config_value(pipeline_def.environment_type,
{'solids': {}})
with pytest.raises(DagsterEvaluateConfigValueError):
throwing_evaluate_config_value(pipeline_def.environment_type, {})
def test_optional_and_required_context():
pipeline_def = PipelineDefinition(
name='some_pipeline',
solids=[],
context_definitions={
'optional_field_context':
PipelineContextDefinition(
context_fn=lambda *args: None,
config_field=Field(dagster_type=Dict(
fields={'optional_field': Field(String, is_optional=True)
})),
),
'required_field_context':
PipelineContextDefinition(
context_fn=lambda *args: None,
config_field=Field(dagster_type=Dict(
fields={'required_field': Field(String)})),
),
},
)
env_type = pipeline_def.environment_type
assert env_type.fields['solids'].is_optional
assert env_type.fields['context'].is_optional is False
context_type = env_type.fields['context'].config_type
assert context_type.fields['optional_field_context'].is_optional
assert context_type.fields['required_field_context'].is_optional
assert env_type.fields['execution'].is_optional
assert env_type.fields['expectations'].is_optional
env_obj = construct_environment_config(
throwing_evaluate_config_value(
pipeline_def.environment_type,
{
'context': {
'optional_field_context': {
'config': {
'optional_field': 'foobar'
}
}
}
},
))
assert env_obj.context.name == 'optional_field_context'
assert env_obj.context.config == {'optional_field': 'foobar'}
def _start_execution(validated_config_either):
new_run_id = str(uuid.uuid4())
execution_plan = create_execution_plan_with_typed_environment(
pipeline.get_dagster_pipeline(),
construct_environment_config(validated_config_either.value),
ExecutionMetadata(),
)
run = pipeline_run_storage.create_run(new_run_id, selector,
env_config, execution_plan)
pipeline_run_storage.add_run(run)
info.context.execution_manager.execute_pipeline(
info.context.repository_container,
pipeline.get_dagster_pipeline(), run)
return info.schema.type_named('StartPipelineExecutionSuccess')(
run=info.schema.type_named('PipelineRun')(run))
def test_provided_default_config():
pipeline_def = PipelineDefinition(
context_definitions={
'some_context':
PipelineContextDefinition(
config_field=Field(
Dict({
'with_default_int':
Field(Int, is_optional=True, default_value=23434)
})),
context_fn=lambda *args: None,
)
},
solids=[
SolidDefinition(name='some_solid',
inputs=[],
outputs=[],
transform_fn=lambda *args: None)
],
)
env_type = pipeline_def.environment_type
some_context_field = env_type.fields['context'].config_type.fields[
'some_context']
assert some_context_field.is_optional
some_context_config_field = some_context_field.config_type.fields['config']
assert some_context_config_field.is_optional
assert some_context_config_field.default_value == {
'with_default_int': 23434
}
assert some_context_field.default_value == {
'config': {
'with_default_int': 23434
},
'resources': {},
'persistence': {
'file': {}
},
}
value = construct_environment_config(
throwing_evaluate_config_value(pipeline_def.environment_type, {}))
assert value.context.name == 'some_context'
assert env_type.type_attributes.is_system_config
def execute_pipeline_through_queue(
repository_info, pipeline_name, pipeline_solid_subset, config, run_id, message_queue
):
"""
Execute pipeline using message queue as a transport
"""
message_queue.put(ProcessStartedSentinel(os.getpid()))
execution_metadata = ExecutionMetadata(run_id, event_callback=message_queue.put)
from .app import RepositoryContainer
repository_container = RepositoryContainer(repository_info)
if repository_container.repo_error:
message_queue.put(
MultiprocessingError(
serializable_error_info_from_exc_info(repository_container.repo_error)
)
)
return
pipeline = repository_container.repository.get_pipeline(pipeline_name)
pipeline = get_subset_pipeline(pipeline, pipeline_solid_subset)
typed_environment = construct_environment_config(
evaluate_config_value(pipeline.environment_type, config).value
)
try:
result = execute_pipeline_with_metadata(
pipeline,
typed_environment,
execution_metadata=execution_metadata,
throw_on_user_error=False,
)
return result
except: # pylint: disable=W0702
message_queue.put(
MultiprocessingError(serializable_error_info_from_exc_info(sys.exc_info()))
)
finally:
message_queue.put(MultiprocessingDone())
def populate_context(self, run_id, pipeline_def, marshal_dir,
environment_config, output_log_path):
check.dict_param(environment_config, 'environment_config')
check.invariant(pipeline_def.has_solid_def(self.solid_def_name))
self.marshal_dir = marshal_dir
self.populated_by_papermill = True
loggers = None
if output_log_path != 0:
event_logger = construct_json_event_logger(output_log_path)
loggers = [event_logger]
# do not include event_callback in ExecutionMetadata,
# since that'll be taken care of by side-channel established by event_logger
execution_metadata = ExecutionMetadata(run_id, loggers=loggers)
solid = Solid(self.solid_def_name, self.solid_def)
typed_environment = construct_environment_config(environment_config)
with yield_context(pipeline_def, typed_environment,
execution_metadata) as context:
solid_config = None
self.info = TransformExecutionInfo(context, solid_config, solid,
pipeline_def)
return self.info
