def test_positive_num():
"""To test custom types, use check_dagster_type, 1st arg is your customized type, 2nd arg is runtime value"""
assert check_dagster_type(PositiveNumber, 2)
type_check = check_dagster_type(PositiveNumber, 0)
assert not type_check.success
assert type_check.description == (
"Numbers cannot be 0 or negative, got 0 for PositiveNumber type")
def test_less_simple_data_frame():
assert check_dagster_type(LessSimpleDataFrame, [{"foo": 1}, {"foo": 2}]).success
type_check = check_dagster_type(LessSimpleDataFrame, [{"foo": 1}, {"bar": 2}])
assert not type_check.success
assert type_check.description == (
"Rows in LessSimpleDataFrame should have the same fields, "
"got ['bar'] for row 2, expected ['foo']"
)
def test_python_object_type_with_custom_type_check():
def eq_3(_, value):
return isinstance(value, int) and value == 3
Int3 = DagsterType(name="Int3", type_check_fn=eq_3)
assert Int3.unique_name == "Int3"
assert check_dagster_type(Int3, 3).success
assert not check_dagster_type(Int3, 5).success
def test_bad_dataframe_type_returns_bad_stuff():
with pytest.raises(DagsterInvariantViolationError):
BadDFBadSummaryStats = create_dagster_pandas_dataframe_type(
"BadDF", event_metadata_fn=lambda _: "ksjdkfsd")
check_dagster_type(BadDFBadSummaryStats, DataFrame({"num": [1]}))
with pytest.raises(DagsterInvariantViolationError):
BadDFBadSummaryStatsListItem = create_dagster_pandas_dataframe_type(
"BadDF", event_metadata_fn=lambda _: ["ksjdkfsd"])
check_dagster_type(BadDFBadSummaryStatsListItem,
DataFrame({"num": [1]}))
def test_bad_dataframe_type_returns_bad_stuff():
with pytest.raises(DagsterInvariantViolationError):
BadDFBadSummaryStats = create_dagster_pandas_dataframe_type(
'BadDF', summary_statistics=lambda _: 'ksjdkfsd')
check_dagster_type(BadDFBadSummaryStats, DataFrame({'num': [1]}))
with pytest.raises(DagsterInvariantViolationError):
BadDFBadSummaryStatsListItem = create_dagster_pandas_dataframe_type(
'BadDF', summary_statistics=lambda _: ['ksjdkfsd'])
check_dagster_type(BadDFBadSummaryStatsListItem,
DataFrame({'num': [1]}))
def test_less_simple_data_frame():
assert check_dagster_type(
LessSimpleDataFrame, [{'foo': 1}, {'foo': 2}]
).success
type_check = check_dagster_type(
LessSimpleDataFrame, [{'foo': 1}, {'bar': 2}]
)
assert not type_check.success
assert type_check.description == (
'Rows in LessSimpleDataFrame should have the same fields, '
'got [\'bar\'] for row 2, expected [\'foo\']'
)
def test_meta_solid():
start_time = time.time()
run_config = {
"solids": {
"get_filing_metadata": {
"inputs": {
"name": "Oracle Corp",
"cik": "0001341439",
"filing": "10-K",
"no_filings": 5
}
}
}
}
result = execute_solid(md.get_filing_metadata, run_config=run_config)
assert isinstance(result, SolidExecutionResult)
assert result.success
assert check_dagster_type(List[Dict[Any, Any]], [{
"foo": "bar"
}, {
"foo": "bar"
}]).success
print("--- %s Seconds ---" % (time.time() - start_time))
def test_create_dagster_pandas_dataframe_type_with_null_event_metadata_fn():
BasicDF = create_dagster_pandas_dataframe_type(
name='BasicDF',
columns=[
PandasColumn.integer_column('pid', non_nullable=True),
PandasColumn.string_column('names'),
],
event_metadata_fn=None,
)
assert isinstance(BasicDF, DagsterType)
basic_type_check = check_dagster_type(BasicDF, DataFrame({'pid': [1], 'names': ['foo']}))
assert basic_type_check.success
def test_create_dagster_pandas_dataframe_type_with_null_event_metadata_fn():
BasicDF = create_dagster_pandas_dataframe_type(
name="BasicDF",
columns=[
PandasColumn.integer_column("pid", non_nullable=True),
PandasColumn.string_column("names"),
],
event_metadata_fn=None,
)
assert isinstance(BasicDF, DagsterType)
basic_type_check = check_dagster_type(BasicDF, DataFrame({"pid": [1], "names": ["foo"]}))
assert basic_type_check.success
def test_simple_example_one_off():
assert check_dagster_type(set_containing_1, {1, 2}).success
def test_simple_example_factory():
set_containing_2 = set_has_element_type_factory(2)
assert check_dagster_type(set_containing_2, {1, 2}).success
def test_nullable_python_object_type():
assert check_dagster_type(Optional[Bar], BarObj()).success
assert check_dagster_type(Optional[Bar], None).success
assert not check_dagster_type(Optional[Bar], "not_a_bar").success
def test_nullable_int_coercion():
assert check_dagster_type(Int, 1).success
assert not check_dagster_type(Int, None).success
assert check_dagster_type(Optional[Int], 1).success
assert check_dagster_type(Optional[Int], None).success
def test_nullable_list_combos_coerciion():
assert not check_dagster_type(List[Int], None).success
assert check_dagster_type(List[Int], []).success
assert check_dagster_type(List[Int], [1]).success
assert not check_dagster_type(List[Int], [None]).success
assert check_dagster_type(Optional[List[Int]], None).success
assert check_dagster_type(Optional[List[Int]], []).success
assert check_dagster_type(Optional[List[Int]], [1]).success
assert not check_dagster_type(Optional[List[Int]], [None]).success
assert not check_dagster_type(List[Optional[Int]], None).success
assert check_dagster_type(List[Optional[Int]], []).success
assert check_dagster_type(List[Optional[Int]], [1]).success
assert check_dagster_type(List[Optional[Int]], [None]).success
assert check_dagster_type(Optional[List[Optional[Int]]], None).success
assert check_dagster_type(Optional[List[Optional[Int]]], []).success
assert check_dagster_type(Optional[List[Optional[Int]]], [1]).success
assert check_dagster_type(Optional[List[Optional[Int]]], [None]).success
# one_off_type_start
from dagster import DagsterType, check_dagster_type
set_containing_1 = DagsterType(
name="set_containing_1",
description=f"A set containing the value 1. May contain any other values.",
type_check_fn=lambda _context, obj: isinstance(obj, set) and 1 in obj,
)
check_dagster_type(set_containing_1, {1, 2}).success # => True
# one_off_type_end
# type_factory_start
def set_has_element_type_factory(x):
return DagsterType(
name=f"set_containing_{x}",
description=f"A set containing the value {x}. May contain any other values.",
type_check_fn=lambda _context, obj: isinstance(obj, set) and x in obj,
)
set_containing_2 = set_has_element_type_factory(2)
check_dagster_type(set_containing_2, {1, 2}).success # => True
# type_factory_end
def test_validate_inv_missing_column(dagster_type, dataframe):
dataframe.drop("a", axis=1, inplace=True)
result = check_dagster_type(dagster_type, dataframe)
assert not result.success
def test_validate_dataframe_ok():
assert check_dagster_type(TripDataFrame, valid_trip_dataframe()).success
def test_validate_ok(dagster_type, dataframe):
result = check_dagster_type(dagster_type, dataframe)
assert isinstance(result, TypeCheck)
assert result.success
def test_validate_inv_bad_value(dagster_type, dataframe):
dataframe.loc[0, "a"] = 11
result = check_dagster_type(dagster_type, dataframe)
assert not result.success
def test_validate_dataframe_missing_column(invalid_dataframe):
assert not check_dagster_type(TripDataFrame, invalid_dataframe).success
def test_validate_dataframe_invalid_categories():
df = valid_weather_dataframe()
df["icon"][0] = "foo"
assert not check_dagster_type(WeatherDataFrame, df).success
def test_validate_training_data_ok():
data = valid_training_set()
assert check_dagster_type(TrainingSet, data).success
def test_validate_training_data_failure(bad_training_data):
assert not check_dagster_type(TrainingSet, bad_training_data).success
def test_dagster_type():
assert check_dagster_type(Dict[Any, Any], {"foo": "bar"}).success
