def test_double_dagster_type():
AlwaysSucceedsFoo = DagsterType(name='Foo',
type_check_fn=lambda _, _val: True)
AlwaysFailsFoo = DagsterType(name='Foo',
type_check_fn=lambda _, _val: False)
@lambda_solid
def return_a_thing():
return 1
@lambda_solid(
input_defs=[InputDefinition('succeeds', AlwaysSucceedsFoo)],
output_def=OutputDefinition(AlwaysFailsFoo),
)
def yup(succeeds):
return succeeds
with pytest.raises(DagsterInvalidDefinitionError) as exc_info:
@pipeline
def _should_fail():
yup(return_a_thing())
assert str(exc_info.value) == (
'You have created two dagster types with the same name "Foo". '
'Dagster types have must have unique names.')
def test_contextual_type_check():
def fancy_type_check(context, value):
return TypeCheck(success=context.resources.foo.check(value))
custom = DagsterType(
key="custom", name="custom", type_check_fn=fancy_type_check, required_resource_keys={"foo"}
)
@resource
def foo(_context):
class _Foo:
def check(self, _value):
return True
return _Foo()
@lambda_solid
def return_one():
return 1
@solid(input_defs=[InputDefinition("inp", custom)])
def bar(_context, inp):
return inp
@pipeline(mode_defs=[ModeDefinition(resource_defs={"foo": foo})])
def fancy_pipeline():
bar(return_one())
assert execute_pipeline(fancy_pipeline).success
def test_raise_on_error_true_type_check_raises_exception():
def raise_exception_inner(_context, _):
raise Failure("I am dissapoint")
ThrowExceptionType = DagsterType(name="ThrowExceptionType", type_check_fn=raise_exception_inner)
@solid(output_defs=[OutputDefinition(ThrowExceptionType)])
def foo_solid(_):
return 1
@pipeline
def foo_pipeline():
foo_solid()
with pytest.raises(Failure, match=re.escape("I am dissapoint")):
execute_pipeline(foo_pipeline)
pipeline_result = execute_pipeline(foo_pipeline, raise_on_error=False)
assert not pipeline_result.success
assert [event.event_type_value for event in pipeline_result.step_event_list] == [
DagsterEventType.STEP_START.value,
DagsterEventType.STEP_FAILURE.value,
]
for event in pipeline_result.step_event_list:
if event.event_type_value == DagsterEventType.STEP_FAILURE.value:
assert event.event_specific_data.error.cls_name == "Failure"
def define_custom_dict(name, permitted_key_names):
def type_check_method(_, value):
if not isinstance(value, dict):
return TypeCheck(
False,
description="Value {value} should be of type {type_name}.".format(
value=value, type_name=name
),
)
for key in value:
if not key in permitted_key_names:
return TypeCheck(
False,
description=(
"Key {name} is not a permitted value, values can only be of: " "{name_list}"
).format(name=value.name, name_list=permitted_key_names),
)
return TypeCheck(
True,
metadata_entries=[
EventMetadataEntry.text(label="row_count", text=str(len(value))),
EventMetadataEntry.text(label="series_names", text=", ".join(value.keys())),
],
)
return DagsterType(key=name, name=name, type_check_fn=type_check_method)
def test_raise_on_error_true_type_check_returns_unsuccessful_type_check():
FalsyType = DagsterType(
name="FalsyType",
type_check_fn=lambda _, _val: TypeCheck(
success=False, metadata_entries=[EventMetadataEntry.text("foo", "bar", "baz")]
),
)
@solid(output_defs=[OutputDefinition(FalsyType)])
def foo_solid(_):
return 1
@pipeline
def foo_pipeline():
foo_solid()
with pytest.raises(DagsterTypeCheckDidNotPass) as e:
execute_pipeline(foo_pipeline)
assert e.value.metadata_entries[0].label == "bar"
assert e.value.metadata_entries[0].entry_data.text == "foo"
assert e.value.metadata_entries[0].description == "baz"
assert isinstance(e.value.dagster_type, DagsterType)
pipeline_result = execute_pipeline(foo_pipeline, raise_on_error=False)
assert not pipeline_result.success
assert [event.event_type_value for event in pipeline_result.step_event_list] == [
DagsterEventType.STEP_START.value,
DagsterEventType.STEP_OUTPUT.value,
DagsterEventType.STEP_FAILURE.value,
]
for event in pipeline_result.step_event_list:
if event.event_type_value == DagsterEventType.STEP_FAILURE.value:
assert event.event_specific_data.error.cls_name == "DagsterTypeCheckDidNotPass"
def test_raise_on_error_type_check_returns_false():
FalsyType = DagsterType(name='FalsyType',
type_check_fn=lambda _, _val: False)
@solid(output_defs=[OutputDefinition(FalsyType)])
def foo_solid(_):
return 1
@pipeline
def foo_pipeline():
foo_solid()
with pytest.raises(DagsterTypeCheckDidNotPass):
execute_pipeline(foo_pipeline)
pipeline_result = execute_pipeline(foo_pipeline, raise_on_error=False)
assert not pipeline_result.success
assert [
event.event_type_value for event in pipeline_result.step_event_list
] == [
DagsterEventType.STEP_START.value,
DagsterEventType.STEP_OUTPUT.value,
DagsterEventType.STEP_FAILURE.value,
]
for event in pipeline_result.step_event_list:
if event.event_type_value == DagsterEventType.STEP_FAILURE.value:
assert event.event_specific_data.error.cls_name == 'DagsterTypeCheckDidNotPass'
def test_python_object_type_with_custom_type_check():
def eq_3(_, value):
return isinstance(value, int) and value == 3
Int3 = DagsterType(name="Int3", type_check_fn=eq_3)
assert Int3.unique_name == "Int3"
assert check_dagster_type(Int3, 3).success
assert not check_dagster_type(Int3, 5).success
def test_python_object_type_with_custom_type_check():
def eq_3(value):
return isinstance(value, int) and value == 3
Int3 = DagsterType(name='Int3', type_check_fn=eq_3)
assert Int3.name == 'Int3'
assert_success(Int3, 3)
assert_failure(Int3, 5)
def do_type_check(context: TypeCheckContext, dagster_type: DagsterType, value: Any) -> TypeCheck:
type_check = dagster_type.type_check(context, value)
if not isinstance(type_check, TypeCheck):
return TypeCheck(
success=False,
description=(
"Type checks must return TypeCheck. Type check for type {type_name} returned "
"value of type {return_type} when checking runtime value of type {dagster_type}."
).format(
type_name=dagster_type.display_name,
return_type=type(type_check),
dagster_type=type(value),
),
)
return type_check
def test_metadata_entries():
metadata_entry = MetadataEntry("foo", None, MetadataValue.text("bar"))
# We use `Output` as a stand-in for all events here, they all follow the same pattern of calling
# `normalize_metadata`.
with pytest.warns(DeprecationWarning,
match=re.escape('"metadata_entries" is deprecated')):
Output("foo", "bar", metadata_entries=[metadata_entry])
with pytest.warns(DeprecationWarning,
match=re.escape('"metadata_entries" is deprecated')):
DagsterType(lambda _, __: True,
"foo",
metadata_entries=[metadata_entry])
def test_raise_on_error_true_type_check_returns_successful_type_check():
TruthyExceptionType = DagsterType(
name="TruthyExceptionType",
type_check_fn=lambda _, _val: TypeCheck(
success=True, metadata_entries=[EventMetadataEntry.text("foo", "bar", "baz")]
),
)
@solid(output_defs=[OutputDefinition(TruthyExceptionType)])
def foo_solid(_):
return 1
@pipeline
def foo_pipeline():
foo_solid()
pipeline_result = execute_pipeline(foo_pipeline)
assert pipeline_result.success
for event in pipeline_result.step_event_list:
if event.event_type_value == DagsterEventType.STEP_OUTPUT.value:
assert event.event_specific_data.type_check_data
assert event.event_specific_data.type_check_data.metadata_entries[0].label == "bar"
assert (
event.event_specific_data.type_check_data.metadata_entries[0].entry_data.text
== "foo"
)
assert (
event.event_specific_data.type_check_data.metadata_entries[0].description == "baz"
)
pipeline_result = execute_pipeline(foo_pipeline, raise_on_error=False)
assert pipeline_result.success
assert set(
[
DagsterEventType.STEP_START.value,
DagsterEventType.STEP_OUTPUT.value,
DagsterEventType.STEP_SUCCESS.value,
]
).issubset([event.event_type_value for event in pipeline_result.step_event_list])
def test_raise_on_error_true_type_check_returns_true():
TruthyExceptionType = DagsterType(name="TruthyExceptionType",
type_check_fn=lambda _, _val: True)
@solid(output_defs=[OutputDefinition(TruthyExceptionType)])
def foo_solid(_):
return 1
@pipeline
def foo_pipeline():
foo_solid()
assert execute_pipeline(foo_pipeline).success
pipeline_result = execute_pipeline(foo_pipeline, raise_on_error=False)
assert pipeline_result.success
assert set([
DagsterEventType.STEP_START.value,
DagsterEventType.STEP_OUTPUT.value,
DagsterEventType.STEP_SUCCESS.value,
]).issubset(
[event.event_type_value for event in pipeline_result.step_event_list])
assert step_failure_event.event_specific_data.error.cls_name == "DagsterTypeCheckDidNotPass"
# TODO add more step output use cases
class AlwaysFailsException(Exception):
# Made to make exception explicit so that we aren't accidentally masking other Exceptions
pass
def _always_fails(_, _value):
raise AlwaysFailsException("kdjfkjd")
ThrowsExceptionType = DagsterType(name="ThrowsExceptionType", type_check_fn=_always_fails,)
def _return_bad_value(_, _value):
return "foo"
BadType = DagsterType(name="BadType", type_check_fn=_return_bad_value)
def test_input_type_returns_wrong_thing():
@lambda_solid
def return_one():
return 1
@lambda_solid(input_defs=[InputDefinition("value", BadType)])
assert not type_check_data.success
assert type_check_data.description == 'Value "1" of python type "int" must be a string.'
step_failure_event = solid_result.compute_step_failure_event
assert step_failure_event.event_specific_data.error.cls_name == 'Failure'
# TODO add more step output use cases
def _always_fails(_value):
raise Exception('kdjfkjd')
ThrowsExceptionType = DagsterType(
name='ThrowsExceptionType',
type_check_fn=_always_fails,
)
def _return_bad_value(_value):
return 'kdjfkjd'
BadType = DagsterType(name='BadType', type_check_fn=_return_bad_value)
def test_input_type_returns_wrong_thing():
@lambda_solid
def return_one():
return 1
__all__ = ["fetch_json_data", "generate_dataframe", "save_dataframe_to_csv"]
import requests
import pandas as pd
import dagster
from dagster import solid
from dagster.core.types.dagster_type import Nothing, DagsterType
from dagster.core.definitions import InputDefinition
# Creating a DictOrList type for runtime type checking
DictOrList = DagsterType(
type_check_fn=lambda _, value: type(value) in [dict, list],
name="DictOrList")
@solid(config_schema={"api_endpoint": str, "api_path": str})
def fetch_json_data(context: dagster.SolidExecutionContext):
""" Fetches data from a JSON API
Configuration schema:
- api_endpoint (str): the API base endpoint (e.g. https://api.github.com/)
- api_path (str): the path of the API from which you want to fetch (e.g. /users/gabriel-milan)
"""
# Joins URL parts into a single URL, stripping unnecessary slashes
url_parts: list = [
context.solid_config["api_endpoint"], context.solid_config["api_path"]
]
url: str = "/".join(s.strip("/") for s in url_parts)
self.count = 0
def serialize(self, value, write_file_obj):
if self.count % 2 == 0: # ensure two steps' serdes interleave
time.sleep(1)
write_file_obj.write(bytes(value.encode("utf-8")))
def deserialize(self, read_file_obj):
if self.count % 2 == 1: # ensure two steps' serdes interleave
time.sleep(1)
return read_file_obj.read().decode("utf-8")
ADagsterType = DagsterType(
name="ADagsterType",
type_check_fn=lambda _, value: True,
serialization_strategy=ASerializationStrategy(),
)
yield_something = test_nb_solid(
name="yield_something",
input_defs=[InputDefinition("obj", str)],
output_defs=[OutputDefinition(ADagsterType)],
)
@solid(input_defs=[
InputDefinition("a", ADagsterType),
InputDefinition("b", ADagsterType)
])
def fan_in(a, b):