def test_diamond_task_dag(snapshot):
dag = DAG(
dag_id="diamond_task_dag",
default_args=default_args,
schedule_interval=None,
)
dummy_operator_1 = DummyOperator(
task_id="dummy_operator_1",
dag=dag,
)
dummy_operator_2 = DummyOperator(
task_id="dummy_operator_2",
dag=dag,
)
dummy_operator_3 = DummyOperator(
task_id="dummy_operator_3",
dag=dag,
)
dummy_operator_4 = DummyOperator(
task_id="dummy_operator_4",
dag=dag,
)
dummy_operator_1 >> dummy_operator_2
dummy_operator_1 >> dummy_operator_3
dummy_operator_2 >> dummy_operator_4
dummy_operator_3 >> dummy_operator_4
snapshot.assert_match(
serialize_pp(
PipelineSnapshot.from_pipeline_def(
make_dagster_pipeline_from_airflow_dag(
dag=dag)).dep_structure_snapshot))
def get_pipeline_index(self):
from dagster.core.snap import PipelineSnapshot
from dagster.core.host_representation import PipelineIndex
return PipelineIndex(
PipelineSnapshot.from_pipeline_def(self), self.get_parent_pipeline_snapshot()
)
def test_deserialize_solid_def_snaps_default_field():
@solid(
config_schema={
"foo": Field(str, is_required=False, default_value="hello"),
"bar": Field(str),
})
def noop_solid(_):
pass
@pipeline
def noop_pipeline():
noop_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
solid_def_snap = pipeline_snapshot.get_solid_def_snap("noop_solid")
recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(
solid_def_snap)
assert isinstance(recevied_config_type, Shape)
assert isinstance(recevied_config_type.fields["foo"].config_type, String)
assert isinstance(recevied_config_type.fields["bar"].config_type, String)
assert not recevied_config_type.fields["foo"].is_required
assert recevied_config_type.fields["foo"].default_value == "hello"
_map_has_stable_hashes(
recevied_config_type,
pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key)
def test_deserialize_solid_def_snaps_default_field():
@solid(
config={
'foo': Field(str, is_required=False, default_value='hello'),
'bar': Field(str)
})
def noop_solid(_):
pass
@pipeline
def noop_pipeline():
noop_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
solid_def_snap = pipeline_snapshot.get_solid_def_snap('noop_solid')
recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(
solid_def_snap)
assert isinstance(recevied_config_type, Shape)
assert isinstance(recevied_config_type.fields['foo'].config_type, String)
assert isinstance(recevied_config_type.fields['bar'].config_type, String)
assert not recevied_config_type.fields['foo'].is_required
assert recevied_config_type.fields['foo'].default_value == 'hello'
_map_has_stable_hashes(
recevied_config_type,
pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key)
def get_pipeline_index(self):
if self._cached_pipeline_index is None:
from dagster.core.snap import PipelineIndex, PipelineSnapshot
self._cached_pipeline_index = PipelineIndex(
PipelineSnapshot.from_pipeline_def(self))
return self._cached_pipeline_index
def from_pipeline_def(pipeline_def, solid_subset=None):
if solid_subset:
pipeline_def = pipeline_def.subset_for_execution(solid_subset)
return ExternalPipeline(
PipelineIndex(PipelineSnapshot.from_pipeline_def(pipeline_def)),
external_pipeline_data_from_def(pipeline_def),
solid_subset=solid_subset,
)
def external_pipeline_data_from_def(pipeline_def):
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
return ExternalPipelineData(
name=pipeline_def.name,
pipeline_snapshot=PipelineSnapshot.from_pipeline_def(pipeline_def),
active_presets=sorted(
list(map(external_preset_data_from_def, pipeline_def.preset_defs)),
key=lambda pd: pd.name,
),
)
def test_empty_pipeline_snap_props(snapshot):
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(get_noop_pipeline())
assert pipeline_snapshot.name == 'noop_pipeline'
assert pipeline_snapshot.description is None
assert pipeline_snapshot.tags == {}
assert pipeline_snapshot == serialize_rt(pipeline_snapshot)
snapshot.assert_match(serialize_pp(pipeline_snapshot))
snapshot.assert_match(create_pipeline_snapshot_id(pipeline_snapshot))
def get_pipeline_snapshot_from_cli_args(cli_args):
_cli_load_invariant(cli_args.get('pipeline_name') is not None)
if cli_args.get('image'):
_cli_load_invariant(
is_module_available('docker'),
msg='--image is not supported without dagster[docker] or the Python package docker installed.',
)
active_repo_data = get_active_repository_data_from_image(cli_args.get('image'))
return active_repo_data.get_pipeline_snapshot(cli_args.get('pipeline_name')[0])
else:
pipeline_definition = handle_for_pipeline_cli_args(cli_args).build_pipeline_definition()
return PipelineSnapshot.from_pipeline_def(pipeline_definition)
def get_pipeline_index(self):
from dagster.core.snap import (
PipelineSnapshot,
PipelineSnapshotWithID,
)
from dagster.core.host_representation import PipelineIndex
return PipelineIndex(
PipelineSnapshotWithID.from_snapshot(
PipelineSnapshot.from_pipeline_def(self)),
PipelineSnapshotWithID.from_snapshot(
self.get_parent_pipeline_snapshot())
if self.parent_pipeline_def else None,
)
def test_deserialize_solid_def_snaps_noneable():
@solid(config=Field(Noneable(str)))
def noop_solid(_):
pass
@pipeline
def noop_pipeline():
noop_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
solid_def_snap = pipeline_snapshot.get_solid_def_snap('noop_solid')
recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(solid_def_snap)
assert isinstance(recevied_config_type, Noneable)
assert isinstance(recevied_config_type.inner_type, String)
def test_multi_type_config_array_dict_fields(dict_config_type, snapshot):
@solid(config=Array(dict_config_type({'foo': Field(int), 'bar': Field(str)})))
def fancy_solid(_):
pass
@pipeline
def noop_pipeline():
fancy_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
solid_def_snap = pipeline_snapshot.get_solid_def_snap('fancy_solid')
recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(solid_def_snap)
snapshot.assert_match(serialize_pp(snap_from_config_type(recevied_config_type)))
_array_has_stable_hashes(
recevied_config_type, pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key
)
def test_one_task_dag(snapshot):
dag = DAG(
dag_id="one_task_dag",
default_args=default_args,
schedule_interval=None,
)
dummy_operator = DummyOperator(
task_id="dummy_operator",
dag=dag,
)
snapshot.assert_match(
serialize_pp(
PipelineSnapshot.from_pipeline_def(
make_dagster_pipeline_from_airflow_dag(
dag=dag)).dep_structure_snapshot))
def get_pipeline_snapshot_from_cli_args(cli_args):
_cli_load_invariant(cli_args.get('pipeline_name') is not None)
if cli_args.get('image'):
_cli_load_invariant(
is_module_available('docker'),
msg=
'--image is not supported without dagster[docker] or the Python package docker installed.',
)
external_repo = get_external_repository_from_image(
cli_args.get('image'))
return external_repo.get_external_pipeline(
cli_args.get('pipeline_name')[0]).pipeline_snapshot
else:
pipeline_def = recon_pipeline_for_cli_args(cli_args).get_definition()
return PipelineSnapshot.from_pipeline_def(pipeline_def)
def test_multi_type_config_array_map(snapshot):
@solid(config_schema=Array(Map(str, int)))
def fancy_solid(_):
pass
@pipeline
def noop_pipeline():
fancy_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
solid_def_snap = pipeline_snapshot.get_node_def_snap("fancy_solid")
recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(solid_def_snap)
snapshot.assert_match(serialize_pp(snap_from_config_type(recevied_config_type)))
_array_has_stable_hashes(
recevied_config_type, pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key
)
def test_deserialize_solid_def_snaps_permissive():
@solid(config=Field(Permissive({'foo': Field(str)})))
def noop_solid(_):
pass
@pipeline
def noop_pipeline():
noop_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
solid_def_snap = pipeline_snapshot.get_solid_def_snap('noop_solid')
recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(solid_def_snap)
assert isinstance(recevied_config_type, Permissive)
assert isinstance(recevied_config_type.fields['foo'].config_type, String)
_map_has_stable_hashes(
recevied_config_type, pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key
)
def test_deserialize_solid_def_snaps_array():
@solid(config=Field([str]))
def noop_solid(_):
pass
@pipeline
def noop_pipeline():
noop_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
solid_def_snap = pipeline_snapshot.get_solid_def_snap('noop_solid')
recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(solid_def_snap)
assert isinstance(recevied_config_type, Array)
assert isinstance(recevied_config_type.inner_type, String)
_array_has_stable_hashes(
recevied_config_type, pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key
)
def test_deserialize_solid_def_snaps_selector():
@solid(config_schema=Selector({'foo': Field(str), 'bar': Field(int)}))
def noop_solid(_):
pass
@pipeline
def noop_pipeline():
noop_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
solid_def_snap = pipeline_snapshot.get_solid_def_snap('noop_solid')
recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(solid_def_snap)
assert isinstance(recevied_config_type, Selector)
assert isinstance(recevied_config_type.fields['foo'].config_type, String)
assert isinstance(recevied_config_type.fields['bar'].config_type, Int)
_map_has_stable_hashes(
recevied_config_type, pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key
)
def test_deserialize_solid_def_snaps_map():
@solid(config_schema=Field({str: str}))
def noop_solid(_):
pass
@pipeline
def noop_pipeline():
noop_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
solid_def_snap = pipeline_snapshot.get_node_def_snap("noop_solid")
recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(solid_def_snap)
assert isinstance(recevied_config_type, Map)
assert isinstance(recevied_config_type.key_type, String)
assert isinstance(recevied_config_type.inner_type, String)
_map_has_stable_hashes(
recevied_config_type, pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key
)
def test_multi_type_config_nested_dicts(nested_dict_types, snapshot):
D1, D2, D3 = nested_dict_types
@solid(config=D1({'foo': D2({'bar': D3({'baz': Field(int)})})}))
def fancy_solid(_):
pass
@pipeline
def noop_pipeline():
fancy_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
solid_def_snap = pipeline_snapshot.get_solid_def_snap('fancy_solid')
recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(solid_def_snap)
snapshot.assert_match(serialize_pp(snap_from_config_type(recevied_config_type)))
_map_has_stable_hashes(
recevied_config_type, pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key
)
def test_deserialize_solid_def_snaps_selector():
@solid(config_schema=Selector({"foo": Field(str), "bar": Field(int)}))
def noop_solid(_):
pass
@pipeline
def noop_pipeline():
noop_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
solid_def_snap = pipeline_snapshot.get_node_def_snap("noop_solid")
recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(solid_def_snap)
assert isinstance(recevied_config_type, Selector)
assert isinstance(recevied_config_type.fields["foo"].config_type, String)
assert isinstance(recevied_config_type.fields["bar"].config_type, Int)
_dict_has_stable_hashes(
recevied_config_type, pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key
)
def test_basic_dep_fan_out(snapshot):
@solid
def return_one(_):
return 1
@solid(input_defs=[InputDefinition("value", int)])
def passthrough(_, value):
return value
@pipeline
def single_dep_pipeline():
return_one_result = return_one()
passthrough.alias("passone")(return_one_result)
passthrough.alias("passtwo")(return_one_result)
dep_structure_snapshot = build_dep_structure_snapshot_from_icontains_solids(
single_dep_pipeline.graph
)
index = DependencyStructureIndex(dep_structure_snapshot)
assert index.get_invocation("return_one")
assert index.get_invocation("passone")
assert index.get_invocation("passtwo")
assert index.get_upstream_output("passone", "value") == OutputHandleSnap("return_one", "result")
assert index.get_upstream_output("passtwo", "value") == OutputHandleSnap("return_one", "result")
assert set(index.get_downstream_inputs("return_one", "result")) == set(
[
InputHandle("passthrough", "passone", "value"),
InputHandle("passthrough", "passtwo", "value"),
]
)
assert (
deserialize_json_to_dagster_namedtuple(serialize_dagster_namedtuple(dep_structure_snapshot))
== dep_structure_snapshot
)
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(single_dep_pipeline)
assert pipeline_snapshot == serialize_rt(pipeline_snapshot)
snapshot.assert_match(serialize_pp(pipeline_snapshot))
snapshot.assert_match(create_pipeline_snapshot_id(pipeline_snapshot))
def test_pipeline_snap_all_props(snapshot):
@solid
def noop_solid(_):
pass
@pipeline(description="desc", tags={"key": "value"})
def noop_pipeline():
noop_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
assert pipeline_snapshot.name == "noop_pipeline"
assert pipeline_snapshot.description == "desc"
assert pipeline_snapshot.tags == {"key": "value"}
assert pipeline_snapshot == serialize_rt(pipeline_snapshot)
snapshot.assert_match(serialize_pp(pipeline_snapshot))
snapshot.assert_match(create_pipeline_snapshot_id(pipeline_snapshot))
def test_pipeline_snap_all_props(snapshot):
@solid
def noop_solid(_):
pass
@pipeline(description='desc', tags={'key': 'value'})
def noop_pipeline():
noop_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
assert pipeline_snapshot.name == 'noop_pipeline'
assert pipeline_snapshot.description == 'desc'
assert pipeline_snapshot.tags == {'key': 'value'}
assert pipeline_snapshot == serialize_rt(pipeline_snapshot)
snapshot.assert_match(serialize_pp(pipeline_snapshot))
snapshot.assert_match(create_pipeline_snapshot_id(pipeline_snapshot))
def test_deserialize_solid_def_snaps_map_with_name():
@solid(config_schema=Field(Map(bool, float, key_label_name="title")))
def noop_solid(_):
pass
@pipeline
def noop_pipeline():
noop_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
solid_def_snap = pipeline_snapshot.get_node_def_snap("noop_solid")
recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(solid_def_snap)
assert isinstance(recevied_config_type, Map)
assert isinstance(recevied_config_type.key_type, Bool)
assert isinstance(recevied_config_type.inner_type, Float)
assert recevied_config_type.given_name == "title"
_map_has_stable_hashes(
recevied_config_type, pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key
)
def test_mode_snap(snapshot):
@resource(config_schema={"foo": str})
def a_resource(_):
pass
@resource(description="resource_description")
def no_config_resource(_):
pass
@logger(config_schema={"bar": str})
def a_logger(_):
pass
@logger(description="logger_description")
def no_config_logger(_):
pass
@pipeline(
mode_defs=[
ModeDefinition(
name="a_mode",
description="a_desc",
resource_defs={
"some_resource": a_resource,
"no_config_resource": no_config_resource,
},
logger_defs={"some_logger": a_logger, "no_config_logger": no_config_logger},
)
]
)
def a_pipeline():
pass
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(a_pipeline)
assert len(pipeline_snapshot.mode_def_snaps) == 1
mode_def_snap = pipeline_snapshot.mode_def_snaps[0]
snapshot.assert_match(serialize_dagster_namedtuple(mode_def_snap))
assert mode_def_snap == deserialize_json_to_dagster_namedtuple(
serialize_dagster_namedtuple(mode_def_snap)
)
def test_mode_snap(snapshot):
@resource(config_schema={'foo': str})
def a_resource(_):
pass
@resource(description='resource_description')
def no_config_resource(_):
pass
@logger(config_schema={'bar': str})
def a_logger(_):
pass
@logger(description='logger_description')
def no_config_logger(_):
pass
@pipeline(mode_defs=[
ModeDefinition(
name='a_mode',
description='a_desc',
resource_defs={
'some_resource': a_resource,
'no_config_resource': no_config_resource,
},
logger_defs={
'some_logger': a_logger,
'no_config_logger': no_config_logger
},
)
])
def a_pipeline():
pass
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(a_pipeline)
assert len(pipeline_snapshot.mode_def_snaps) == 1
mode_def_snap = pipeline_snapshot.mode_def_snaps[0]
snapshot.assert_match(serialize_dagster_namedtuple(mode_def_snap))
assert mode_def_snap == deserialize_json_to_dagster_namedtuple(
serialize_dagster_namedtuple(mode_def_snap))
def test_two_invocations_deps_snap(snapshot):
@solid
def noop_solid(_):
pass
@pipeline
def two_solid_pipeline():
noop_solid.alias('one')()
noop_solid.alias('two')()
index = DependencyStructureIndex(
build_dep_structure_snapshot_from_icontains_solids(two_solid_pipeline))
assert index.get_invocation('one')
assert index.get_invocation('two')
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(two_solid_pipeline)
assert pipeline_snapshot == serialize_rt(pipeline_snapshot)
snapshot.assert_match(serialize_pp(pipeline_snapshot))
snapshot.assert_match(create_pipeline_snapshot_id(pipeline_snapshot))
def test_deserialize_solid_def_snaps_enum():
@solid(
config=Field(Enum('CowboyType', [EnumValue('good'), EnumValue('bad'), EnumValue('ugly')]))
)
def noop_solid(_):
pass
@pipeline
def noop_pipeline():
noop_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
solid_def_snap = pipeline_snapshot.get_solid_def_snap('noop_solid')
recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(solid_def_snap)
assert isinstance(recevied_config_type, Enum)
assert recevied_config_type.given_name == 'CowboyType'
assert all(
enum_value.config_value in ('good', 'bad', 'ugly')
for enum_value in recevied_config_type.enum_values
)
def test_deserialize_solid_def_snaps_enum():
@solid(config_schema=Field(
Enum("CowboyType",
[EnumValue("good"),
EnumValue("bad"),
EnumValue("ugly")])))
def noop_solid(_):
pass
@pipeline
def noop_pipeline():
noop_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
solid_def_snap = pipeline_snapshot.get_solid_def_snap("noop_solid")
recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(
solid_def_snap)
assert isinstance(recevied_config_type, Enum)
assert recevied_config_type.given_name == "CowboyType"
assert all(enum_value.config_value in ("good", "bad", "ugly")
for enum_value in recevied_config_type.enum_values)
