def test_opt_two_dim_dict_parm(): assert check.opt_two_dim_dict_param({}, "foo") == {} assert check.opt_two_dim_dict_param({"key": {}}, "foo") assert check.opt_two_dim_dict_param({"key": {"key2": 2}}, "foo") assert check.opt_two_dim_dict_param(None, "foo") == {} with pytest.raises(CheckError): assert check.opt_two_dim_dict_param("str", "foo")
def test_opt_two_dim_dict_parm(): assert check.opt_two_dim_dict_param({}, 'foo') == {} assert check.opt_two_dim_dict_param({'key': {}}, 'foo') assert check.opt_two_dim_dict_param({'key': {'key2': 2}}, 'foo') assert check.opt_two_dim_dict_param(None, 'foo') == {} with pytest.raises(CheckError): assert check.opt_two_dim_dict_param('str', 'foo')
def with_input_marshalling(included_step_keys, marshalled_inputs=None): ''' Create an execution subset that uses the marshalling infrastruture in order to provide values to includes. inputs dictionary is a Dict[str,Dict[str,Any]] mapping step_key => input_name => marshalling_key ''' check.list_param(included_step_keys, 'included_step_keys', of_type=str) marshalled_inputs = check.opt_two_dim_dict_param( marshalled_inputs, 'marshalled_inputs') input_step_factory_fns = defaultdict(dict) def _create_unmarshal_input_factory_fn(marshalling_key): return lambda pipeline_context, step, step_input: create_unmarshal_input_step( pipeline_context, step, step_input, marshalling_key) for step_key, input_marshal_dict in marshalled_inputs.items(): for input_name, key in input_marshal_dict.items(): input_step_factory_fns[step_key][ input_name] = _create_unmarshal_input_factory_fn(key) return ExecutionPlanSubsetInfo(included_step_keys, input_step_factory_fns)
def __new__(cls, included_step_keys, input_step_factory_fns): ''' This should not be called directly, but instead through the static methods on this class. included_step_keys: list of step keys to include in the subset input_step_factory_fns: A 2D dictinoary step_key => input_name => callable callable should have signature of (StepBuilderState, ExecutionStep, StepInput): StepOutputHandle Full type signature is Dict[str, Dict[str, Callable[StepBuilderState, ExecutionStep, StepInput]: StepOutputHandle ] ] ''' return super(ExecutionPlanSubsetInfo, cls).__new__( cls, set( check.list_param(included_step_keys, 'included_step_keys', of_type=str)), check.opt_two_dim_dict_param(input_step_factory_fns, 'input_step_factory_fns', key_type=str), )
def test_opt_two_dim_dict_parm(): assert check.opt_two_dim_dict_param({}, 'foo') == {} assert check.opt_two_dim_dict_param({'key': {}}, 'foo') assert check.opt_two_dim_dict_param({'key': {'key2': 2}}, 'foo') assert check.opt_two_dim_dict_param(None, 'foo') == {}
def __init__(self, solids, name=None, description=None, context_definitions=None, dependencies=None): ''' Args: solids (List[SolidDefinition]): Solids in the pipeline name (str): Name. This is optional, mostly for situations that require ephemeral pipeline definitions for fast scaffolding or testing. description (str): Description of the pipline. context_definitions (Dict[str, PipelineContextDefinition]): See class description. dependencies: (Dict[str, Dict[str, DependencyDefinition]]): See class description. ''' self.name = check.opt_str_param(name, 'name', '<<unnamed>>') self.description = check.opt_str_param(description, 'description') check.list_param(solids, 'solids') if context_definitions is None: context_definitions = default_pipeline_context_definitions() self.context_definitions = check.dict_param( context_definitions, 'context_definitions', key_type=str, value_type=PipelineContextDefinition, ) self.dependencies = check.opt_two_dim_dict_param( dependencies, 'dependencies', key_type=six.string_types + (SolidInstance, ), value_type=DependencyDefinition, ) dependency_structure, pipeline_solid_dict = create_execution_structure( solids, self.dependencies) self._solid_dict = pipeline_solid_dict self.dependency_structure = dependency_structure self.environment_cls = define_environment_cls( EnvironmentClassCreationData( self.name, list(self._solid_dict.values()), context_definitions, dependency_structure, )) self.environment_type = self.environment_cls.inst() self.context_cls = define_context_cls(self) self.context_type = self.context_cls.inst() ( self._config_type_dict_by_name, self._config_type_dict_by_key, ) = construct_config_type_dictionary(solids, self.context_definitions, self.environment_type) self._runtime_type_dict = construct_runtime_type_dictionary(solids)