def validate_map_config(
context: ValidationContext,
config_value: object) -> EvaluateValueResult[Dict[str, object]]:
check.inst_param(context, "context", ValidationContext)
check.invariant(context.config_type_snap.kind == ConfigTypeKind.MAP)
check.not_none_param(config_value, "config_value")
if not isinstance(config_value, dict):
return EvaluateValueResult.for_error(
create_map_error(context, config_value))
config_value = cast(Dict[object, object], config_value)
evaluation_results = [
_validate_config(context.for_map_key(key), key)
for key in config_value.keys()
] + [
_validate_config(context.for_map_value(key), config_item)
for key, config_item in config_value.items()
]
errors = []
for result in evaluation_results:
if not result.success:
errors += cast(List, result.errors)
return EvaluateValueResult(not bool(errors), frozendict(config_value),
errors) # type: ignore
def validate_array_config(
context: ValidationContext,
config_value: object) -> EvaluateValueResult[List[object]]:
check.inst_param(context, "context", ValidationContext)
check.invariant(context.config_type_snap.kind == ConfigTypeKind.ARRAY)
check.not_none_param(config_value, "config_value")
if not isinstance(config_value, list):
return EvaluateValueResult.for_error(
create_array_error(context, config_value))
evaluation_results = [
_validate_config(context.for_array(index), config_item)
for index, config_item in enumerate(config_value)
]
values = []
errors = []
for result in evaluation_results:
if result.success:
values.append(result.value)
else:
errors += cast(List, result.errors)
return EvaluateValueResult(not bool(errors), values,
errors) # type: ignore
def validate_shape_config(
context: ValidationContext, config_value: object
) -> EvaluateValueResult[Dict[str, object]]:
check.inst_param(context, "context", ValidationContext)
check.invariant(context.config_type_snap.kind == ConfigTypeKind.STRICT_SHAPE)
check.not_none_param(config_value, "config_value")
return _validate_shape_config(context, config_value, check_for_extra_incoming_fields=True)
def validate_shape_config(context, config_value):
check.inst_param(context, 'context', ValidationContext)
check.invariant(context.config_type.kind == ConfigTypeKind.STRICT_SHAPE)
check.not_none_param(config_value, 'config_value')
return _validate_shape_config(context,
config_value,
check_for_extra_incoming_fields=True)
def validate_permissive_shape_config(context, config_value):
check.inst_param(context, "context", ValidationContext)
check.invariant(
context.config_type_snap.kind == ConfigTypeKind.PERMISSIVE_SHAPE)
check.not_none_param(config_value, "config_value")
return _validate_shape_config(context,
config_value,
check_for_extra_incoming_fields=False)
def validate_permissive_shape_config(context, config_value):
check.inst_param(context, 'context', TraversalContext)
check.invariant(
context.config_type.kind == ConfigTypeKind.PERMISSIVE_SHAPE)
check.not_none_param(config_value, 'config_value')
return _validate_shape_config(context,
config_value,
check_for_extra_incoming_fields=False)
def validate_selector_config(
context: ValidationContext, config_value: object
) -> EvaluateValueResult[Dict[str, object]]:
check.inst_param(context, "context", ValidationContext)
check.param_invariant(context.config_type_snap.kind == ConfigTypeKind.SELECTOR, "selector_type")
check.not_none_param(config_value, "config_value")
# Special case the empty dictionary, meaning no values provided for the
# value of the selector. # E.g. {'logging': {}}
# If there is a single field defined on the selector and if it is optional
# it passes validation. (e.g. a single logger "console")
if config_value == {}:
return _validate_empty_selector_config(context) # type: ignore
# Now we ensure that the used-provided config has only a a single entry
# and then continue the validation pass
if not isinstance(config_value, dict):
return EvaluateValueResult.for_error(create_selector_type_error(context, config_value))
if len(config_value) > 1:
return EvaluateValueResult.for_error(
create_selector_multiple_fields_error(context, config_value)
)
field_name, field_value = ensure_single_item(config_value)
if not context.config_type_snap.has_field(field_name):
return EvaluateValueResult.for_error(create_field_not_defined_error(context, field_name))
field_snap = context.config_type_snap.get_field(field_name)
child_evaluate_value_result = _validate_config(
context.for_field_snap(field_snap),
# This is a very particular special case where we want someone
# to be able to select a selector key *without* a value
#
# e.g.
# storage:
# filesystem:
#
# And we want the default values of the child elements of filesystem:
# to "fill in"
{}
if field_value is None
and ConfigTypeKind.has_fields(
context.config_schema_snapshot.get_config_snap(field_snap.type_key).kind
)
else field_value,
)
if child_evaluate_value_result.success:
return EvaluateValueResult.for_value( # type: ignore
frozendict({field_name: child_evaluate_value_result.value})
)
else:
return child_evaluate_value_result # type: ignore
def validate_enum_config(context, config_value):
check.inst_param(context, "context", ValidationContext)
check.invariant(context.config_type_snap.kind == ConfigTypeKind.ENUM)
check.not_none_param(config_value, "config_value")
if not isinstance(config_value, str):
return EvaluateValueResult.for_error(create_enum_type_mismatch_error(context, config_value))
if not context.config_type_snap.has_enum_value(config_value):
return EvaluateValueResult.for_error(create_enum_value_missing_error(context, config_value))
return EvaluateValueResult.for_value(config_value)
def validate_enum_config(context, config_value):
check.inst_param(context, 'context', ValidationContext)
check.invariant(context.config_type.kind == ConfigTypeKind.ENUM)
check.not_none_param(config_value, 'config_value')
if not isinstance(config_value, six.string_types):
return EvaluateValueResult.for_error(create_enum_type_mismatch_error(context, config_value))
if not context.config_type.is_valid_config_enum_value(config_value):
return EvaluateValueResult.for_error(create_enum_value_missing_error(context, config_value))
return EvaluateValueResult.for_value(config_value)
def _add_snapshot(self, snapshot_id, snapshot_obj, snapshot_type):
check.str_param(snapshot_id, 'snapshot_id')
check.not_none_param(snapshot_obj, 'snapshot_obj')
check.inst_param(snapshot_type, 'snapshot_type', SnapshotType)
with self.connect() as conn:
snapshot_insert = SnapshotsTable.insert().values( # pylint: disable=no-value-for-parameter
snapshot_id=snapshot_id,
snapshot_body=zlib.compress(serialize_dagster_namedtuple(snapshot_obj).encode()),
snapshot_type=snapshot_type.value,
)
conn.execute(snapshot_insert)
return snapshot_id
def _validate_scalar_union_config(context, config_value):
check.inst_param(context, 'context', ValidationContext)
check.param_invariant(context.config_type.kind == ConfigTypeKind.SCALAR_UNION, 'context')
check.not_none_param(config_value, 'config_value')
if isinstance(config_value, dict) or isinstance(config_value, list):
return _validate_config(
context.for_new_config_type(context.config_type.non_scalar_type), config_value
)
else:
return _validate_config(
context.for_new_config_type(context.config_type.scalar_type), config_value
)
def _validate_scalar_union_config(context, config_value):
check.inst_param(context, "context", ValidationContext)
check.param_invariant(context.config_type_snap.kind == ConfigTypeKind.SCALAR_UNION, "context")
check.not_none_param(config_value, "config_value")
if isinstance(config_value, dict) or isinstance(config_value, list):
return _validate_config(
context.for_new_config_type_key(context.config_type_snap.non_scalar_type_key),
config_value,
)
else:
return _validate_config(
context.for_new_config_type_key(context.config_type_snap.scalar_type_key), config_value,
)
def _validate_shape_config(context, config_value,
check_for_extra_incoming_fields):
check.inst_param(context, "context", ValidationContext)
check.not_none_param(config_value, "config_value")
check.bool_param(check_for_extra_incoming_fields,
"check_for_extra_incoming_fields")
if config_value and not isinstance(config_value, dict):
return EvaluateValueResult.for_error(
create_dict_type_mismatch_error(context, config_value))
field_snaps = context.config_type_snap.fields
defined_field_names = {fs.name for fs in field_snaps}
incoming_field_names = set(config_value.keys())
errors = []
if check_for_extra_incoming_fields:
_append_if_error(
errors,
_check_for_extra_incoming_fields(context, defined_field_names,
incoming_field_names),
)
_append_if_error(
errors,
_compute_missing_fields_error(context, field_snaps,
incoming_field_names))
# dict is well-formed. now recursively validate all incoming fields
field_errors = []
for field_snap in context.config_type_snap.fields:
name = field_snap.name
if name in config_value:
field_evr = _validate_config(context.for_field_snap(field_snap),
config_value[name])
if field_evr.errors:
field_errors += field_evr.errors
if field_errors:
errors += field_errors
if errors:
return EvaluateValueResult.for_errors(errors)
else:
return EvaluateValueResult.for_value(frozendict(config_value))
def __init__(self, inner_dagster_type: DagsterType):
self._inner_dagster_type = check.inst_param(inner_dagster_type,
"inner_dagster_type",
DagsterType)
self._inner_loader = check.not_none_param(inner_dagster_type.loader,
"inner_dagster_type")
self._schema_type = ConfigNoneable(self._inner_loader.schema_type)
def __new__(
cls,
pipeline_def,
mode_def,
system_storage_def,
run_config,
environment_config,
type_storage_plugin_registry,
resources,
system_storage_config,
):
return super(InitSystemStorageContext, cls).__new__(
cls,
pipeline_def=check.inst_param(pipeline_def, 'pipeline_def',
PipelineDefinition),
mode_def=check.inst_param(mode_def, 'mode_def', ModeDefinition),
system_storage_def=check.inst_param(system_storage_def,
'system_storage_def',
SystemStorageDefinition),
run_config=check.inst_param(run_config, 'run_config', RunConfig),
environment_config=check.inst_param(environment_config,
'environment_config',
EnvironmentConfig),
type_storage_plugin_registry=check.inst_param(
type_storage_plugin_registry,
'type_storage_plugin_registry',
TypeStoragePluginRegistry,
),
resources=check.not_none_param(resources, 'resources'),
system_storage_config=check.dict_param(system_storage_config,
system_storage_config,
key_type=str),
)
def __new__(
cls,
pipeline_def,
mode_def,
intermediate_storage_def,
pipeline_run,
instance,
resolved_run_config,
type_storage_plugin_registry,
resources,
intermediate_storage_config,
):
return super(InitIntermediateStorageContext, cls).__new__(
cls,
pipeline_def=check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition),
mode_def=check.inst_param(mode_def, "mode_def", ModeDefinition),
intermediate_storage_def=check.inst_param(
intermediate_storage_def, "intermediate_storage_def", IntermediateStorageDefinition
),
pipeline_run=check.inst_param(pipeline_run, "pipeline_run", PipelineRun),
instance=check.inst_param(instance, "instance", DagsterInstance),
resolved_run_config=check.inst_param(
resolved_run_config, "resolved_run_config", ResolvedRunConfig
),
type_storage_plugin_registry=check.inst_param(
type_storage_plugin_registry,
"type_storage_plugin_registry",
TypeStoragePluginRegistry,
),
resources=check.not_none_param(resources, "resources"),
intermediate_storage_config=check.dict_param(
intermediate_storage_config, intermediate_storage_config, key_type=str
),
)
def _validate_shape_config(context, config_value,
check_for_extra_incoming_fields):
check.inst_param(context, 'context', ValidationContext)
check.not_none_param(config_value, 'config_value')
check.bool_param(check_for_extra_incoming_fields,
'check_for_extra_incoming_fields')
if config_value and not isinstance(config_value, dict):
return EvaluateValueResult.for_error(
create_dict_type_mismatch_error(context, config_value))
fields = context.config_type.fields
defined_field_names = set(fields.keys())
incoming_field_names = set(config_value.keys())
errors = []
if check_for_extra_incoming_fields:
_append_if_error(
errors,
_check_for_extra_incoming_fields(context, defined_field_names,
incoming_field_names),
)
_append_if_error(
errors,
_compute_missing_fields_error(context, fields, incoming_field_names))
# dict is well-formed. now recursively validate all incoming fields
field_errors = []
for name, field_def in context.config_type.fields.items():
if name in config_value:
field_evr = _validate_config(context.for_field(field_def, name),
config_value[name])
if field_evr.errors:
field_errors += field_evr.errors
if field_errors:
errors += field_errors
if errors:
return EvaluateValueResult.for_errors(errors)
else:
return EvaluateValueResult.for_value(frozendict(config_value))
def _add_snapshot(self, snapshot_id: str, snapshot_obj,
snapshot_type: SnapshotType) -> str:
check.str_param(snapshot_id, "snapshot_id")
check.not_none_param(snapshot_obj, "snapshot_obj")
check.inst_param(snapshot_type, "snapshot_type", SnapshotType)
with self.connect() as conn:
snapshot_insert = (
SnapshotsTable.insert().values( # pylint: disable=no-value-for-parameter
snapshot_id=snapshot_id,
snapshot_body=zlib.compress(
serialize_dagster_namedtuple(snapshot_obj).encode(
"utf-8")),
snapshot_type=snapshot_type.value,
))
conn.execute(snapshot_insert)
return snapshot_id
def test_not_none_param():
assert check.not_none_param(1, 'fine')
check.not_none_param(0, 'zero is fine')
check.not_none_param('', 'empty str is fine')
with pytest.raises(CheckError):
check.not_none_param(None, 'none fails')
def test_not_none_param():
assert check.not_none_param(1, "fine")
check.not_none_param(0, "zero is fine")
check.not_none_param("", "empty str is fine")
with pytest.raises(CheckError):
check.not_none_param(None, "none fails")
def value(self, key, value):
'''
Adds a context value to the Execution for a particular scope, using the
python contextmanager abstraction. This allows the user to add scoped metadata
just like the framework does (for things such as solid name).
Examples:
.. code-block:: python
with context.value('some_key', 'some_value):
context.info('msg with some_key context value')
context.info('msg without some_key context value')
'''
check.str_param(key, 'key')
check.not_none_param(value, 'value')
with self.values({key: value}):
yield
def validate_list_config(context, config_value):
check.inst_param(context, 'context', ValidationContext)
check.invariant(context.config_type.kind == ConfigTypeKind.LIST)
check.not_none_param(config_value, 'config_value')
if not isinstance(config_value, list):
return EvaluateValueResult.for_error(create_list_error(context, config_value))
evaluation_results = [
_validate_config(context.for_list(index), config_item)
for index, config_item in enumerate(config_value)
]
values = []
errors = []
for result in evaluation_results:
if result.success:
values.append(result.value)
else:
errors += result.errors
return EvaluateValueResult(not bool(errors), values, errors)
def __new__(
cls,
pipeline_def,
mode_def,
intermediate_storage_def,
pipeline_run,
instance,
environment_config,
type_storage_plugin_registry,
resources,
intermediate_storage_config,
):
return super(InitIntermediateStorageContext, cls).__new__(
cls,
pipeline_def=check.inst_param(pipeline_def, 'pipeline_def',
PipelineDefinition),
mode_def=check.inst_param(mode_def, 'mode_def', ModeDefinition),
intermediate_storage_def=check.inst_param(
intermediate_storage_def, 'intermediate_storage_def',
IntermediateStorageDefinition),
pipeline_run=check.inst_param(pipeline_run, 'pipeline_run',
PipelineRun),
instance=check.inst_param(instance, 'instance', DagsterInstance),
environment_config=check.inst_param(environment_config,
'environment_config',
EnvironmentConfig),
type_storage_plugin_registry=check.inst_param(
type_storage_plugin_registry,
'type_storage_plugin_registry',
TypeStoragePluginRegistry,
),
resources=check.not_none_param(resources, 'resources'),
intermediate_storage_config=check.dict_param(
intermediate_storage_config,
intermediate_storage_config,
key_type=str),
)
def __new__(
cls,
pipeline_def,
mode_def,
system_storage_def,
pipeline_run,
instance,
environment_config,
type_storage_plugin_registry,
resources,
system_storage_config,
):
return super(InitSystemStorageContext, cls).__new__(
cls,
pipeline_def=check.inst_param(pipeline_def, "pipeline_def",
PipelineDefinition),
mode_def=check.inst_param(mode_def, "mode_def", ModeDefinition),
system_storage_def=check.inst_param(system_storage_def,
"system_storage_def",
SystemStorageDefinition),
pipeline_run=check.inst_param(pipeline_run, "pipeline_run",
PipelineRun),
instance=check.inst_param(instance, "instance", DagsterInstance),
environment_config=check.inst_param(environment_config,
"environment_config",
EnvironmentConfig),
type_storage_plugin_registry=check.inst_param(
type_storage_plugin_registry,
"type_storage_plugin_registry",
TypeStoragePluginRegistry,
),
resources=check.not_none_param(resources, "resources"),
system_storage_config=check.dict_param(system_storage_config,
system_storage_config,
key_type=str),
)
def ipc_write_unary_response(output_file, obj):
check.not_none_param(obj, "obj")
with ipc_write_stream(output_file) as stream:
stream.send(obj)
def _validate_shape_config(
context: ValidationContext, config_value: object,
check_for_extra_incoming_fields: bool
) -> EvaluateValueResult[Dict[str, object]]:
check.inst_param(context, "context", ValidationContext)
check.not_none_param(config_value, "config_value")
check.bool_param(check_for_extra_incoming_fields,
"check_for_extra_incoming_fields")
field_aliases = check.opt_dict_param(
cast(Dict[str, str], context.config_type_snap.field_aliases),
"field_aliases",
key_type=str,
value_type=str,
)
if not isinstance(config_value, dict):
return EvaluateValueResult.for_error(
create_dict_type_mismatch_error(context, config_value))
config_value = cast(Dict[str, object], config_value)
field_snaps = check.not_none(context.config_type_snap.fields)
defined_field_names = {cast(str, fs.name) for fs in field_snaps}
defined_field_names = defined_field_names.union(set(
field_aliases.values()))
incoming_field_names = set(config_value.keys())
errors: List[EvaluationError] = []
if check_for_extra_incoming_fields:
_append_if_error(
errors,
_check_for_extra_incoming_fields(
context,
defined_field_names,
incoming_field_names,
),
)
_append_if_error(
errors,
_compute_missing_fields_error(context, field_snaps,
incoming_field_names, field_aliases),
)
# dict is well-formed. now recursively validate all incoming fields
field_errors = []
field_snaps = check.not_none(context.config_type_snap.fields)
for field_snap in field_snaps:
name = field_snap.name
aliased_name = field_aliases.get(name)
if aliased_name is not None and aliased_name in config_value and name in config_value:
field_errors.append(
create_field_substitution_collision_error(
context.for_field_snap(field_snap),
name=name,
aliased_name=aliased_name))
elif name in config_value:
field_evr = _validate_config(context.for_field_snap(field_snap),
config_value[name])
if field_evr.errors:
field_errors += field_evr.errors
elif aliased_name is not None and aliased_name in config_value:
field_evr = _validate_config(context.for_field_snap(field_snap),
config_value[aliased_name])
if field_evr.errors:
field_errors += field_evr.errors
if field_errors:
errors += field_errors
if errors:
return EvaluateValueResult.for_errors(errors)
else:
return EvaluateValueResult.for_value(
frozendict(config_value)) # type: ignore
def __getitem__(self, inner_type):
check.not_none_param(inner_type, 'inner_type')
return OptionalType(inner_type)
def __getitem__(self, inner_type: t.Union[t.Type, DagsterType]) -> OptionalType:
inner_type = resolve_dagster_type(check.not_none_param(inner_type, "inner_type"))
return OptionalType(inner_type)
def __call__(self, inner_type):
check.not_none_param(inner_type, "inner_type")
return _List(inner_type)
def __getitem__(self, inner_type):
check.not_none_param(inner_type, "inner_type")
return _List(resolve_dagster_type(inner_type))
