def _execution_plan_or_error(subplan_execution_args, dauphin_pipeline, evaluate_value_result): check.inst_param(subplan_execution_args, 'subplan_execution_args', SubplanExecutionArgs) check.inst_param(dauphin_pipeline, 'dauphin_pipeline', DauphinPipeline) check.inst_param(evaluate_value_result, 'evaluate_value_result', EvaluateValueResult) execution_plan = create_execution_plan_with_typed_environment( dauphin_pipeline.get_dagster_pipeline(), construct_environment_config(evaluate_value_result.value), subplan_execution_args.execution_metadata, ) invalid_keys = [] for step_key in subplan_execution_args.step_keys: if not execution_plan.has_step(step_key): invalid_keys.append(step_key) if invalid_keys: return EitherError( subplan_execution_args.info.schema.type_named( 'StartSubplanExecutionInvalidStepsError')( invalid_step_keys=invalid_keys)) else: return EitherValue(execution_plan)
def test_optional_solid_with_optional_scalar_config(): def _assert_config_none(context, value): assert context.solid_config is value pipeline_def = PipelineDefinition( name='some_pipeline', solids=[ SolidDefinition( name='int_config_solid', config_field=Field(Int, is_optional=True), inputs=[], outputs=[], transform_fn=lambda context, _inputs: _assert_config_none( context, 234), ) ], ) env_type = pipeline_def.environment_type assert env_type.fields['solids'].is_optional is True solids_type = env_type.fields['solids'].config_type assert solids_type.fields['int_config_solid'].is_optional is True solids_default_obj = construct_solid_dictionary( throwing_evaluate_config_value(solids_type, {})) assert solids_default_obj['int_config_solid'].config is None env_obj = construct_environment_config( throwing_evaluate_config_value(pipeline_def.environment_type, {})) assert env_obj.solids['int_config_solid'].config is None
def test_whole_environment(): pipeline_def = PipelineDefinition( name='some_pipeline', context_definitions={ 'test': PipelineContextDefinition( config_field=Field(Any), context_fn=lambda *args: ExecutionContext()) }, solids=[ SolidDefinition( name='int_config_solid', config_field=Field(Int), inputs=[], outputs=[], transform_fn=lambda *args: None, ), SolidDefinition(name='no_config_solid', inputs=[], outputs=[], transform_fn=lambda *args: None), ], ) environment_type = pipeline_def.environment_type assert environment_type.fields[ 'context'].config_type.name == 'SomePipeline.ContextConfig' solids_type = environment_type.fields['solids'].config_type assert solids_type.name == 'SomePipeline.SolidsConfigDictionary' assert (solids_type.fields['int_config_solid'].config_type.name == 'SomePipeline.SolidConfig.IntConfigSolid') assert (environment_type.fields['expectations'].config_type.name == 'SomePipeline.ExpectationsConfig') env = construct_environment_config( throwing_evaluate_config_value( pipeline_def.environment_type, { 'context': { 'test': { 'config': 1 } }, 'solids': { 'int_config_solid': { 'config': 123 } } }, )) assert isinstance(env, EnvironmentConfig) assert env.context == ContextConfig('test', 1, persistence={'file': {}}) assert env.solids == {'int_config_solid': SolidConfig(123)} assert env.expectations == ExpectationsConfig(evaluate=True)
def test_default_environment(): pipeline_def = PipelineDefinition(solids=[ SolidDefinition(name='some_solid', inputs=[], outputs=[], transform_fn=lambda *args: None) ]) env_obj = construct_environment_config( throwing_evaluate_config_value(pipeline_def.environment_type, {})) assert env_obj.expectations.evaluate is True
def create_plan(pipeline): config_or_error = _config_or_error_from_pipeline( info, pipeline, config) return config_or_error.chain( lambda evaluate_value_result: info.schema. type_named('ExecutionPlan')( pipeline, create_execution_plan_with_typed_environment( pipeline.get_dagster_pipeline(), construct_environment_config(evaluate_value_result.value), ExecutionMetadata(), ), ))
def test_required_solid_with_required_subfield(): pipeline_def = PipelineDefinition( name='some_pipeline', solids=[ SolidDefinition( name='int_config_solid', config_field=Field(Dict({'required_field': Field(String)})), inputs=[], outputs=[], transform_fn=lambda *_args: None, ) ], ) env_type = pipeline_def.environment_type assert env_type.fields['solids'].is_optional is False assert env_type.fields['solids'].config_type solids_type = env_type.fields['solids'].config_type assert solids_type.fields['int_config_solid'].is_optional is False int_config_solid_type = solids_type.fields['int_config_solid'].config_type assert int_config_solid_type.fields['config'].is_optional is False assert env_type.fields['context'].is_optional assert env_type.fields['execution'].is_optional assert env_type.fields['expectations'].is_optional env_obj = construct_environment_config( throwing_evaluate_config_value( pipeline_def.environment_type, { 'solids': { 'int_config_solid': { 'config': { 'required_field': 'foobar' } } } }, )) assert env_obj.solids['int_config_solid'].config[ 'required_field'] == 'foobar' with pytest.raises(DagsterEvaluateConfigValueError): throwing_evaluate_config_value(pipeline_def.environment_type, {'solids': {}}) with pytest.raises(DagsterEvaluateConfigValueError): throwing_evaluate_config_value(pipeline_def.environment_type, {})
def test_optional_and_required_context(): pipeline_def = PipelineDefinition( name='some_pipeline', solids=[], context_definitions={ 'optional_field_context': PipelineContextDefinition( context_fn=lambda *args: None, config_field=Field(dagster_type=Dict( fields={'optional_field': Field(String, is_optional=True) })), ), 'required_field_context': PipelineContextDefinition( context_fn=lambda *args: None, config_field=Field(dagster_type=Dict( fields={'required_field': Field(String)})), ), }, ) env_type = pipeline_def.environment_type assert env_type.fields['solids'].is_optional assert env_type.fields['context'].is_optional is False context_type = env_type.fields['context'].config_type assert context_type.fields['optional_field_context'].is_optional assert context_type.fields['required_field_context'].is_optional assert env_type.fields['execution'].is_optional assert env_type.fields['expectations'].is_optional env_obj = construct_environment_config( throwing_evaluate_config_value( pipeline_def.environment_type, { 'context': { 'optional_field_context': { 'config': { 'optional_field': 'foobar' } } } }, )) assert env_obj.context.name == 'optional_field_context' assert env_obj.context.config == {'optional_field': 'foobar'}
def _start_execution(validated_config_either): new_run_id = str(uuid.uuid4()) execution_plan = create_execution_plan_with_typed_environment( pipeline.get_dagster_pipeline(), construct_environment_config(validated_config_either.value), ExecutionMetadata(), ) run = pipeline_run_storage.create_run(new_run_id, selector, env_config, execution_plan) pipeline_run_storage.add_run(run) info.context.execution_manager.execute_pipeline( info.context.repository_container, pipeline.get_dagster_pipeline(), run) return info.schema.type_named('StartPipelineExecutionSuccess')( run=info.schema.type_named('PipelineRun')(run))
def test_provided_default_config(): pipeline_def = PipelineDefinition( context_definitions={ 'some_context': PipelineContextDefinition( config_field=Field( Dict({ 'with_default_int': Field(Int, is_optional=True, default_value=23434) })), context_fn=lambda *args: None, ) }, solids=[ SolidDefinition(name='some_solid', inputs=[], outputs=[], transform_fn=lambda *args: None) ], ) env_type = pipeline_def.environment_type some_context_field = env_type.fields['context'].config_type.fields[ 'some_context'] assert some_context_field.is_optional some_context_config_field = some_context_field.config_type.fields['config'] assert some_context_config_field.is_optional assert some_context_config_field.default_value == { 'with_default_int': 23434 } assert some_context_field.default_value == { 'config': { 'with_default_int': 23434 }, 'resources': {}, 'persistence': { 'file': {} }, } value = construct_environment_config( throwing_evaluate_config_value(pipeline_def.environment_type, {})) assert value.context.name == 'some_context' assert env_type.type_attributes.is_system_config
def execute_pipeline_through_queue( repository_info, pipeline_name, pipeline_solid_subset, config, run_id, message_queue ): """ Execute pipeline using message queue as a transport """ message_queue.put(ProcessStartedSentinel(os.getpid())) execution_metadata = ExecutionMetadata(run_id, event_callback=message_queue.put) from .app import RepositoryContainer repository_container = RepositoryContainer(repository_info) if repository_container.repo_error: message_queue.put( MultiprocessingError( serializable_error_info_from_exc_info(repository_container.repo_error) ) ) return pipeline = repository_container.repository.get_pipeline(pipeline_name) pipeline = get_subset_pipeline(pipeline, pipeline_solid_subset) typed_environment = construct_environment_config( evaluate_config_value(pipeline.environment_type, config).value ) try: result = execute_pipeline_with_metadata( pipeline, typed_environment, execution_metadata=execution_metadata, throw_on_user_error=False, ) return result except: # pylint: disable=W0702 message_queue.put( MultiprocessingError(serializable_error_info_from_exc_info(sys.exc_info())) ) finally: message_queue.put(MultiprocessingDone())
def populate_context(self, run_id, pipeline_def, marshal_dir, environment_config, output_log_path): check.dict_param(environment_config, 'environment_config') check.invariant(pipeline_def.has_solid_def(self.solid_def_name)) self.marshal_dir = marshal_dir self.populated_by_papermill = True loggers = None if output_log_path != 0: event_logger = construct_json_event_logger(output_log_path) loggers = [event_logger] # do not include event_callback in ExecutionMetadata, # since that'll be taken care of by side-channel established by event_logger execution_metadata = ExecutionMetadata(run_id, loggers=loggers) solid = Solid(self.solid_def_name, self.solid_def) typed_environment = construct_environment_config(environment_config) with yield_context(pipeline_def, typed_environment, execution_metadata) as context: solid_config = None self.info = TransformExecutionInfo(context, solid_config, solid, pipeline_def) return self.info