def test_complex_type_materialization():
pipeline = multiple_output_pipeline()
with get_temp_file_names(2) as file_tuple:
filename_one, filename_two = file_tuple # pylint: disable=E0632
result = execute_pipeline(
pipeline,
{
'solids': {
'return_one_and_foo': {
'outputs': [
{
'string': {
'json': {
'path': filename_one
}
}
},
{
'number': {
'json': {
'path': filename_two
}
}
},
]
}
}
},
)
assert result.success
for event in result.event_list:
if event.event_type == DagsterEventType.STEP_MATERIALIZATION:
event = event.event_specific_data.materialization
assert len(event.metadata_entries) == 3
assert event.metadata_entries[1] == EventMetadataEntry(
label='system-type-name',
description=None,
entry_data=TextMetadataEntryData(text='String'),
) or event.metadata_entries[1] == EventMetadataEntry(
label='system-type-name',
description=None,
entry_data=TextMetadataEntryData(text='Int'),
)
assert event.metadata_entries[2] == EventMetadataEntry(
label='system-type-description',
description=None,
entry_data=TextMetadataEntryData(text='Any'),
)
def test_basic_type_materialization():
pipeline = single_string_output_pipeline()
with get_temp_file_name() as filename:
result = execute_pipeline(
pipeline,
{
'solids': {
'return_foo': {
'outputs': [{
'result': {
'json': {
'path': filename
}
}
}]
}
}
},
)
assert result.success
for event in result.event_list:
if event.event_type == DagsterEventType.STEP_MATERIALIZATION:
event = event.event_specific_data.materialization
assert len(event.metadata_entries) == 3
assert event.metadata_entries[1] == EventMetadataEntry(
label='system-type-name',
description=None,
entry_data=TextMetadataEntryData(text='String'),
)
assert event.metadata_entries[2] == EventMetadataEntry(
label='system-type-description',
description=None,
entry_data=TextMetadataEntryData(text='Any'),
)
def event_metadata_entries(metadata_entry_datas):
if not metadata_entry_datas:
return
for metadata_entry_data in metadata_entry_datas:
typename = metadata_entry_data['__typename']
label = metadata_entry_data['label']
description = metadata_entry_data.get('description')
if typename == 'EventPathMetadataEntry':
yield EventMetadataEntry.path(label=label,
description=description,
path=metadata_entry_data['path'])
elif typename == 'EventJsonMetadataEntry':
yield EventMetadataEntry.json(
label=label,
description=description,
data=seven.json.loads(metadata_entry_data.get(
'jsonString', '')),
)
elif typename == 'EventMarkdownMetadataEntry':
yield EventMetadataEntry.md(label=label,
description=description,
md_str=metadata_entry_data.get(
'md_str', ''))
elif typename == 'EventTextMetadataEntry':
yield EventMetadataEntry.text(label=label,
description=description,
text=metadata_entry_data['text'])
elif typename == 'EventUrlMetadataEntry':
yield EventMetadataEntry.url(label=label,
description=description,
url=metadata_entry_data['url'])
elif typename == 'EventPythonArtifactMetadataEntry':
yield EventMetadataEntry(
label=label,
description=description,
entry_data=PythonArtifactMetadataEntryData(
metadata_entry_data['module'],
metadata_entry_data['name']),
)
else:
check.not_implemented('TODO for type {}'.format(typename))
def event_metadata_entries(metadata_entry_datas):
if not metadata_entry_datas:
return
for metadata_entry_data in metadata_entry_datas:
typename = metadata_entry_data["__typename"]
label = metadata_entry_data["label"]
description = metadata_entry_data.get("description")
if typename == "EventPathMetadataEntry":
yield EventMetadataEntry.path(label=label,
description=description,
path=metadata_entry_data["path"])
elif typename == "EventJsonMetadataEntry":
yield EventMetadataEntry.json(
label=label,
description=description,
data=seven.json.loads(metadata_entry_data.get(
"jsonString", "")),
)
elif typename == "EventMarkdownMetadataEntry":
yield EventMetadataEntry.md(label=label,
description=description,
md_str=metadata_entry_data.get(
"md_str", ""))
elif typename == "EventTextMetadataEntry":
yield EventMetadataEntry.text(label=label,
description=description,
text=metadata_entry_data["text"])
elif typename == "EventUrlMetadataEntry":
yield EventMetadataEntry.url(label=label,
description=description,
url=metadata_entry_data["url"])
elif typename == "EventPythonArtifactMetadataEntry":
yield EventMetadataEntry(
label=label,
description=description,
entry_data=PythonArtifactMetadataEntryData(
metadata_entry_data["module"],
metadata_entry_data["name"]),
)
else:
check.not_implemented("TODO for type {}".format(typename))
def handle_output(self, context, obj):
super().handle_output(context, obj)
# can pretend this actually came from a library call
yield EventMetadataEntry(label="num rows written to db",
description=None,
entry_data=EventMetadata.int(len(obj)))
