def validate_map_config( context: ValidationContext, config_value: object) -> EvaluateValueResult[Dict[str, object]]: check.inst_param(context, "context", ValidationContext) check.invariant(context.config_type_snap.kind == ConfigTypeKind.MAP) check.not_none_param(config_value, "config_value") if not isinstance(config_value, dict): return EvaluateValueResult.for_error( create_map_error(context, config_value)) config_value = cast(Dict[object, object], config_value) evaluation_results = [ _validate_config(context.for_map_key(key), key) for key in config_value.keys() ] + [ _validate_config(context.for_map_value(key), config_item) for key, config_item in config_value.items() ] errors = [] for result in evaluation_results: if not result.success: errors += cast(List, result.errors) return EvaluateValueResult(not bool(errors), frozendict(config_value), errors) # type: ignore
def validate_array_config( context: ValidationContext, config_value: object) -> EvaluateValueResult[List[object]]: check.inst_param(context, "context", ValidationContext) check.invariant(context.config_type_snap.kind == ConfigTypeKind.ARRAY) check.not_none_param(config_value, "config_value") if not isinstance(config_value, list): return EvaluateValueResult.for_error( create_array_error(context, config_value)) evaluation_results = [ _validate_config(context.for_array(index), config_item) for index, config_item in enumerate(config_value) ] values = [] errors = [] for result in evaluation_results: if result.success: values.append(result.value) else: errors += cast(List, result.errors) return EvaluateValueResult(not bool(errors), values, errors) # type: ignore
def validate_shape_config( context: ValidationContext, config_value: object ) -> EvaluateValueResult[Dict[str, object]]: check.inst_param(context, "context", ValidationContext) check.invariant(context.config_type_snap.kind == ConfigTypeKind.STRICT_SHAPE) check.not_none_param(config_value, "config_value") return _validate_shape_config(context, config_value, check_for_extra_incoming_fields=True)
def validate_shape_config(context, config_value): check.inst_param(context, 'context', ValidationContext) check.invariant(context.config_type.kind == ConfigTypeKind.STRICT_SHAPE) check.not_none_param(config_value, 'config_value') return _validate_shape_config(context, config_value, check_for_extra_incoming_fields=True)
def validate_permissive_shape_config(context, config_value): check.inst_param(context, "context", ValidationContext) check.invariant( context.config_type_snap.kind == ConfigTypeKind.PERMISSIVE_SHAPE) check.not_none_param(config_value, "config_value") return _validate_shape_config(context, config_value, check_for_extra_incoming_fields=False)
def validate_permissive_shape_config(context, config_value): check.inst_param(context, 'context', TraversalContext) check.invariant( context.config_type.kind == ConfigTypeKind.PERMISSIVE_SHAPE) check.not_none_param(config_value, 'config_value') return _validate_shape_config(context, config_value, check_for_extra_incoming_fields=False)
def validate_selector_config( context: ValidationContext, config_value: object ) -> EvaluateValueResult[Dict[str, object]]: check.inst_param(context, "context", ValidationContext) check.param_invariant(context.config_type_snap.kind == ConfigTypeKind.SELECTOR, "selector_type") check.not_none_param(config_value, "config_value") # Special case the empty dictionary, meaning no values provided for the # value of the selector. # E.g. {'logging': {}} # If there is a single field defined on the selector and if it is optional # it passes validation. (e.g. a single logger "console") if config_value == {}: return _validate_empty_selector_config(context) # type: ignore # Now we ensure that the used-provided config has only a a single entry # and then continue the validation pass if not isinstance(config_value, dict): return EvaluateValueResult.for_error(create_selector_type_error(context, config_value)) if len(config_value) > 1: return EvaluateValueResult.for_error( create_selector_multiple_fields_error(context, config_value) ) field_name, field_value = ensure_single_item(config_value) if not context.config_type_snap.has_field(field_name): return EvaluateValueResult.for_error(create_field_not_defined_error(context, field_name)) field_snap = context.config_type_snap.get_field(field_name) child_evaluate_value_result = _validate_config( context.for_field_snap(field_snap), # This is a very particular special case where we want someone # to be able to select a selector key *without* a value # # e.g. # storage: # filesystem: # # And we want the default values of the child elements of filesystem: # to "fill in" {} if field_value is None and ConfigTypeKind.has_fields( context.config_schema_snapshot.get_config_snap(field_snap.type_key).kind ) else field_value, ) if child_evaluate_value_result.success: return EvaluateValueResult.for_value( # type: ignore frozendict({field_name: child_evaluate_value_result.value}) ) else: return child_evaluate_value_result # type: ignore
def validate_enum_config(context, config_value): check.inst_param(context, "context", ValidationContext) check.invariant(context.config_type_snap.kind == ConfigTypeKind.ENUM) check.not_none_param(config_value, "config_value") if not isinstance(config_value, str): return EvaluateValueResult.for_error(create_enum_type_mismatch_error(context, config_value)) if not context.config_type_snap.has_enum_value(config_value): return EvaluateValueResult.for_error(create_enum_value_missing_error(context, config_value)) return EvaluateValueResult.for_value(config_value)
def validate_enum_config(context, config_value): check.inst_param(context, 'context', ValidationContext) check.invariant(context.config_type.kind == ConfigTypeKind.ENUM) check.not_none_param(config_value, 'config_value') if not isinstance(config_value, six.string_types): return EvaluateValueResult.for_error(create_enum_type_mismatch_error(context, config_value)) if not context.config_type.is_valid_config_enum_value(config_value): return EvaluateValueResult.for_error(create_enum_value_missing_error(context, config_value)) return EvaluateValueResult.for_value(config_value)
def _add_snapshot(self, snapshot_id, snapshot_obj, snapshot_type): check.str_param(snapshot_id, 'snapshot_id') check.not_none_param(snapshot_obj, 'snapshot_obj') check.inst_param(snapshot_type, 'snapshot_type', SnapshotType) with self.connect() as conn: snapshot_insert = SnapshotsTable.insert().values( # pylint: disable=no-value-for-parameter snapshot_id=snapshot_id, snapshot_body=zlib.compress(serialize_dagster_namedtuple(snapshot_obj).encode()), snapshot_type=snapshot_type.value, ) conn.execute(snapshot_insert) return snapshot_id
def _validate_scalar_union_config(context, config_value): check.inst_param(context, 'context', ValidationContext) check.param_invariant(context.config_type.kind == ConfigTypeKind.SCALAR_UNION, 'context') check.not_none_param(config_value, 'config_value') if isinstance(config_value, dict) or isinstance(config_value, list): return _validate_config( context.for_new_config_type(context.config_type.non_scalar_type), config_value ) else: return _validate_config( context.for_new_config_type(context.config_type.scalar_type), config_value )
def _validate_scalar_union_config(context, config_value): check.inst_param(context, "context", ValidationContext) check.param_invariant(context.config_type_snap.kind == ConfigTypeKind.SCALAR_UNION, "context") check.not_none_param(config_value, "config_value") if isinstance(config_value, dict) or isinstance(config_value, list): return _validate_config( context.for_new_config_type_key(context.config_type_snap.non_scalar_type_key), config_value, ) else: return _validate_config( context.for_new_config_type_key(context.config_type_snap.scalar_type_key), config_value, )
def _validate_shape_config(context, config_value, check_for_extra_incoming_fields): check.inst_param(context, "context", ValidationContext) check.not_none_param(config_value, "config_value") check.bool_param(check_for_extra_incoming_fields, "check_for_extra_incoming_fields") if config_value and not isinstance(config_value, dict): return EvaluateValueResult.for_error( create_dict_type_mismatch_error(context, config_value)) field_snaps = context.config_type_snap.fields defined_field_names = {fs.name for fs in field_snaps} incoming_field_names = set(config_value.keys()) errors = [] if check_for_extra_incoming_fields: _append_if_error( errors, _check_for_extra_incoming_fields(context, defined_field_names, incoming_field_names), ) _append_if_error( errors, _compute_missing_fields_error(context, field_snaps, incoming_field_names)) # dict is well-formed. now recursively validate all incoming fields field_errors = [] for field_snap in context.config_type_snap.fields: name = field_snap.name if name in config_value: field_evr = _validate_config(context.for_field_snap(field_snap), config_value[name]) if field_evr.errors: field_errors += field_evr.errors if field_errors: errors += field_errors if errors: return EvaluateValueResult.for_errors(errors) else: return EvaluateValueResult.for_value(frozendict(config_value))
def __init__(self, inner_dagster_type: DagsterType): self._inner_dagster_type = check.inst_param(inner_dagster_type, "inner_dagster_type", DagsterType) self._inner_loader = check.not_none_param(inner_dagster_type.loader, "inner_dagster_type") self._schema_type = ConfigNoneable(self._inner_loader.schema_type)
def __new__( cls, pipeline_def, mode_def, system_storage_def, run_config, environment_config, type_storage_plugin_registry, resources, system_storage_config, ): return super(InitSystemStorageContext, cls).__new__( cls, pipeline_def=check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition), mode_def=check.inst_param(mode_def, 'mode_def', ModeDefinition), system_storage_def=check.inst_param(system_storage_def, 'system_storage_def', SystemStorageDefinition), run_config=check.inst_param(run_config, 'run_config', RunConfig), environment_config=check.inst_param(environment_config, 'environment_config', EnvironmentConfig), type_storage_plugin_registry=check.inst_param( type_storage_plugin_registry, 'type_storage_plugin_registry', TypeStoragePluginRegistry, ), resources=check.not_none_param(resources, 'resources'), system_storage_config=check.dict_param(system_storage_config, system_storage_config, key_type=str), )
def __new__( cls, pipeline_def, mode_def, intermediate_storage_def, pipeline_run, instance, resolved_run_config, type_storage_plugin_registry, resources, intermediate_storage_config, ): return super(InitIntermediateStorageContext, cls).__new__( cls, pipeline_def=check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition), mode_def=check.inst_param(mode_def, "mode_def", ModeDefinition), intermediate_storage_def=check.inst_param( intermediate_storage_def, "intermediate_storage_def", IntermediateStorageDefinition ), pipeline_run=check.inst_param(pipeline_run, "pipeline_run", PipelineRun), instance=check.inst_param(instance, "instance", DagsterInstance), resolved_run_config=check.inst_param( resolved_run_config, "resolved_run_config", ResolvedRunConfig ), type_storage_plugin_registry=check.inst_param( type_storage_plugin_registry, "type_storage_plugin_registry", TypeStoragePluginRegistry, ), resources=check.not_none_param(resources, "resources"), intermediate_storage_config=check.dict_param( intermediate_storage_config, intermediate_storage_config, key_type=str ), )
def _validate_shape_config(context, config_value, check_for_extra_incoming_fields): check.inst_param(context, 'context', ValidationContext) check.not_none_param(config_value, 'config_value') check.bool_param(check_for_extra_incoming_fields, 'check_for_extra_incoming_fields') if config_value and not isinstance(config_value, dict): return EvaluateValueResult.for_error( create_dict_type_mismatch_error(context, config_value)) fields = context.config_type.fields defined_field_names = set(fields.keys()) incoming_field_names = set(config_value.keys()) errors = [] if check_for_extra_incoming_fields: _append_if_error( errors, _check_for_extra_incoming_fields(context, defined_field_names, incoming_field_names), ) _append_if_error( errors, _compute_missing_fields_error(context, fields, incoming_field_names)) # dict is well-formed. now recursively validate all incoming fields field_errors = [] for name, field_def in context.config_type.fields.items(): if name in config_value: field_evr = _validate_config(context.for_field(field_def, name), config_value[name]) if field_evr.errors: field_errors += field_evr.errors if field_errors: errors += field_errors if errors: return EvaluateValueResult.for_errors(errors) else: return EvaluateValueResult.for_value(frozendict(config_value))
def _add_snapshot(self, snapshot_id: str, snapshot_obj, snapshot_type: SnapshotType) -> str: check.str_param(snapshot_id, "snapshot_id") check.not_none_param(snapshot_obj, "snapshot_obj") check.inst_param(snapshot_type, "snapshot_type", SnapshotType) with self.connect() as conn: snapshot_insert = ( SnapshotsTable.insert().values( # pylint: disable=no-value-for-parameter snapshot_id=snapshot_id, snapshot_body=zlib.compress( serialize_dagster_namedtuple(snapshot_obj).encode( "utf-8")), snapshot_type=snapshot_type.value, )) conn.execute(snapshot_insert) return snapshot_id
def test_not_none_param(): assert check.not_none_param(1, 'fine') check.not_none_param(0, 'zero is fine') check.not_none_param('', 'empty str is fine') with pytest.raises(CheckError): check.not_none_param(None, 'none fails')
def test_not_none_param(): assert check.not_none_param(1, "fine") check.not_none_param(0, "zero is fine") check.not_none_param("", "empty str is fine") with pytest.raises(CheckError): check.not_none_param(None, "none fails")
def value(self, key, value): ''' Adds a context value to the Execution for a particular scope, using the python contextmanager abstraction. This allows the user to add scoped metadata just like the framework does (for things such as solid name). Examples: .. code-block:: python with context.value('some_key', 'some_value): context.info('msg with some_key context value') context.info('msg without some_key context value') ''' check.str_param(key, 'key') check.not_none_param(value, 'value') with self.values({key: value}): yield
def validate_list_config(context, config_value): check.inst_param(context, 'context', ValidationContext) check.invariant(context.config_type.kind == ConfigTypeKind.LIST) check.not_none_param(config_value, 'config_value') if not isinstance(config_value, list): return EvaluateValueResult.for_error(create_list_error(context, config_value)) evaluation_results = [ _validate_config(context.for_list(index), config_item) for index, config_item in enumerate(config_value) ] values = [] errors = [] for result in evaluation_results: if result.success: values.append(result.value) else: errors += result.errors return EvaluateValueResult(not bool(errors), values, errors)
def __new__( cls, pipeline_def, mode_def, intermediate_storage_def, pipeline_run, instance, environment_config, type_storage_plugin_registry, resources, intermediate_storage_config, ): return super(InitIntermediateStorageContext, cls).__new__( cls, pipeline_def=check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition), mode_def=check.inst_param(mode_def, 'mode_def', ModeDefinition), intermediate_storage_def=check.inst_param( intermediate_storage_def, 'intermediate_storage_def', IntermediateStorageDefinition), pipeline_run=check.inst_param(pipeline_run, 'pipeline_run', PipelineRun), instance=check.inst_param(instance, 'instance', DagsterInstance), environment_config=check.inst_param(environment_config, 'environment_config', EnvironmentConfig), type_storage_plugin_registry=check.inst_param( type_storage_plugin_registry, 'type_storage_plugin_registry', TypeStoragePluginRegistry, ), resources=check.not_none_param(resources, 'resources'), intermediate_storage_config=check.dict_param( intermediate_storage_config, intermediate_storage_config, key_type=str), )
def __new__( cls, pipeline_def, mode_def, system_storage_def, pipeline_run, instance, environment_config, type_storage_plugin_registry, resources, system_storage_config, ): return super(InitSystemStorageContext, cls).__new__( cls, pipeline_def=check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition), mode_def=check.inst_param(mode_def, "mode_def", ModeDefinition), system_storage_def=check.inst_param(system_storage_def, "system_storage_def", SystemStorageDefinition), pipeline_run=check.inst_param(pipeline_run, "pipeline_run", PipelineRun), instance=check.inst_param(instance, "instance", DagsterInstance), environment_config=check.inst_param(environment_config, "environment_config", EnvironmentConfig), type_storage_plugin_registry=check.inst_param( type_storage_plugin_registry, "type_storage_plugin_registry", TypeStoragePluginRegistry, ), resources=check.not_none_param(resources, "resources"), system_storage_config=check.dict_param(system_storage_config, system_storage_config, key_type=str), )
def ipc_write_unary_response(output_file, obj): check.not_none_param(obj, "obj") with ipc_write_stream(output_file) as stream: stream.send(obj)
def _validate_shape_config( context: ValidationContext, config_value: object, check_for_extra_incoming_fields: bool ) -> EvaluateValueResult[Dict[str, object]]: check.inst_param(context, "context", ValidationContext) check.not_none_param(config_value, "config_value") check.bool_param(check_for_extra_incoming_fields, "check_for_extra_incoming_fields") field_aliases = check.opt_dict_param( cast(Dict[str, str], context.config_type_snap.field_aliases), "field_aliases", key_type=str, value_type=str, ) if not isinstance(config_value, dict): return EvaluateValueResult.for_error( create_dict_type_mismatch_error(context, config_value)) config_value = cast(Dict[str, object], config_value) field_snaps = check.not_none(context.config_type_snap.fields) defined_field_names = {cast(str, fs.name) for fs in field_snaps} defined_field_names = defined_field_names.union(set( field_aliases.values())) incoming_field_names = set(config_value.keys()) errors: List[EvaluationError] = [] if check_for_extra_incoming_fields: _append_if_error( errors, _check_for_extra_incoming_fields( context, defined_field_names, incoming_field_names, ), ) _append_if_error( errors, _compute_missing_fields_error(context, field_snaps, incoming_field_names, field_aliases), ) # dict is well-formed. now recursively validate all incoming fields field_errors = [] field_snaps = check.not_none(context.config_type_snap.fields) for field_snap in field_snaps: name = field_snap.name aliased_name = field_aliases.get(name) if aliased_name is not None and aliased_name in config_value and name in config_value: field_errors.append( create_field_substitution_collision_error( context.for_field_snap(field_snap), name=name, aliased_name=aliased_name)) elif name in config_value: field_evr = _validate_config(context.for_field_snap(field_snap), config_value[name]) if field_evr.errors: field_errors += field_evr.errors elif aliased_name is not None and aliased_name in config_value: field_evr = _validate_config(context.for_field_snap(field_snap), config_value[aliased_name]) if field_evr.errors: field_errors += field_evr.errors if field_errors: errors += field_errors if errors: return EvaluateValueResult.for_errors(errors) else: return EvaluateValueResult.for_value( frozendict(config_value)) # type: ignore
def __getitem__(self, inner_type): check.not_none_param(inner_type, 'inner_type') return OptionalType(inner_type)
def __getitem__(self, inner_type: t.Union[t.Type, DagsterType]) -> OptionalType: inner_type = resolve_dagster_type(check.not_none_param(inner_type, "inner_type")) return OptionalType(inner_type)
def __call__(self, inner_type): check.not_none_param(inner_type, "inner_type") return _List(inner_type)
def __getitem__(self, inner_type): check.not_none_param(inner_type, "inner_type") return _List(resolve_dagster_type(inner_type))