def materialize(_): yield Materialization( label='all_types', description='a materialization with all metadata types', metadata_entries=[ EventMetadataEntry.text('text is cool', 'text'), EventMetadataEntry.url('https://bigty.pe/neato', 'url'), EventMetadataEntry.fspath('/tmp/awesome', 'path'), EventMetadataEntry.json({'is_dope': True}, 'json'), ], ) yield Output(None)
def my_failure_metadata_solid(_): path = "/path/to/files" my_files = get_files(path) if len(my_files) == 0: raise Failure( description="No files to process", metadata_entries=[ EventMetadataEntry.fspath(path, label="filepath"), EventMetadataEntry.url("http://mycoolsite.com/failures", label="dashboard_url"), ], ) return some_calculation(my_files)
def my_metadata_materialization_solid(context, df): do_some_transform(df) persist_to_storage(df) yield AssetMaterialization( asset_key="my_dataset", description="Persisted result to storage", metadata_entries=[ EventMetadataEntry.text("Text-based metadata for this event", label="text_metadata"), EventMetadataEntry.fspath("/path/to/data/on/filesystem"), EventMetadataEntry.url("http://mycoolsite.com/url_for_my_data", label="dashboard_url"), EventMetadataEntry.float(calculate_bytes(df), "size (bytes)"), ], ) yield Output(df)
def many_table_materializations(_context): for table in raw_tables: yield Materialization( label='table_info', metadata_entries=[ EventMetadataEntry.text(text=table, label='table_name'), EventMetadataEntry.fspath(path='/path/to/{}'.format(table), label='table_path'), EventMetadataEntry.json(data={'name': table}, label='table_data'), EventMetadataEntry.url(url='https://bigty.pe/{}'.format(table), label='table_name_big'), ], )
def my_metadata_materialization_solid(context): df = read_df() remote_storage_path = persist_to_storage(df) yield AssetMaterialization( asset_key="my_dataset", description="Persisted result to storage", metadata_entries=[ EventMetadataEntry.text("Text-based metadata for this event", label="text_metadata"), EventMetadataEntry.fspath(remote_storage_path), EventMetadataEntry.url("http://mycoolsite.com/url_for_my_data", label="dashboard_url"), EventMetadataEntry.float(calculate_bytes(df), "size (bytes)"), ], ) yield Output(remote_storage_path)
def materialization_and_expectation(_context): yield Materialization( label='all_types', description='a materialization with all metadata types', metadata_entries=[ EventMetadataEntry.text('text is cool', 'text'), EventMetadataEntry.url('https://bigty.pe/neato', 'url'), EventMetadataEntry.fspath('/tmp/awesome', 'path'), EventMetadataEntry.json({'is_dope': True}, 'json'), ], ) yield ExpectationResult(success=True, label='row_count', description='passed') yield ExpectationResult(True) yield Output(True)
def materialization_and_expectation(_context): yield AssetMaterialization( asset_key="all_types", description="a materialization with all metadata types", metadata_entries=[ EventMetadataEntry.text("text is cool", "text"), EventMetadataEntry.url("https://bigty.pe/neato", "url"), EventMetadataEntry.fspath("/tmp/awesome", "path"), EventMetadataEntry.json({"is_dope": True}, "json"), ], ) yield ExpectationResult(success=True, label="row_count", description="passed") yield ExpectationResult(True) yield Output(True)
def materialize(_): yield Materialization( label='all_types', description='a materialization with all metadata types', metadata_entries=[ EventMetadataEntry.text('text is cool', 'text'), EventMetadataEntry.url('https://bigty.pe/neato', 'url'), EventMetadataEntry.fspath('/tmp/awesome', 'path'), EventMetadataEntry.json({'is_dope': True}, 'json'), EventMetadataEntry.python_artifact(EventMetadataEntry, 'python class'), EventMetadataEntry.python_artifact(file_relative_path, 'python function'), EventMetadataEntry.float(1.2, 'float'), ], ) yield Output(None)
def many_table_materializations(_context): with open(file_relative_path(__file__, MARKDOWN_EXAMPLE), 'r') as f: md_str = f.read() for table in raw_tables: yield Materialization( label='table_info', metadata_entries=[ EventMetadataEntry.text(text=table, label='table_name'), EventMetadataEntry.fspath(path='/path/to/{}'.format(table), label='table_path'), EventMetadataEntry.json(data={'name': table}, label='table_data'), EventMetadataEntry.url( url='https://bigty.pe/{}'.format(table), label='table_name_big' ), EventMetadataEntry.md(md_str=md_str, label='table_blurb'), ], )
def materialize(_): yield AssetMaterialization( asset_key="all_types", description="a materialization with all metadata types", metadata_entries=[ EventMetadataEntry.text("text is cool", "text"), EventMetadataEntry.url("https://bigty.pe/neato", "url"), EventMetadataEntry.fspath("/tmp/awesome", "path"), EventMetadataEntry.json({"is_dope": True}, "json"), EventMetadataEntry.python_artifact(EventMetadataEntry, "python class"), EventMetadataEntry.python_artifact(file_relative_path, "python function"), EventMetadataEntry.float(1.2, "float"), EventMetadataEntry.int(1, "int"), ], ) yield Output(None)
def my_metadata_materialization_solid(context, df): do_some_transform(df) persist_to_storage(df) yield Materialization( label='my_dataset', description='Persisted result to storage', metadata_entries=[ EventMetadataEntry.text('Text-based metadata for this event', label='text_metadata'), EventMetadataEntry.fspath('/path/to/data/on/filesystem'), EventMetadataEntry.url('http://mycoolsite.com/url_for_my_data', label='dashboard_url'), EventMetadataEntry.float(calculate_bytes(df), 'size (bytes)'), ], ) yield Output(df)
def materialize(_): yield AssetMaterialization( asset_key="all_types", description="a materialization with all metadata types", metadata_entries=[ EventMetadataEntry.text("text is cool", "text"), EventMetadataEntry.url("https://bigty.pe/neato", "url"), EventMetadataEntry.fspath("/tmp/awesome", "path"), EventMetadataEntry.json({"is_dope": True}, "json"), EventMetadataEntry.python_artifact(EventMetadataEntry, "python class"), EventMetadataEntry.python_artifact(file_relative_path, "python function"), EventMetadataEntry.float(1.2, "float"), EventMetadataEntry.int(1, "int"), EventMetadataEntry.float(float("nan"), "float NaN"), EventMetadataEntry.int(LONG_INT, "long int"), EventMetadataEntry.pipeline_run("fake_run_id", "pipeline run"), EventMetadataEntry.asset(AssetKey("my_asset"), "my asset"), EventMetadataEntry.table( label="table", records=[ TableRecord(foo=1, bar=2), TableRecord(foo=3, bar=4), ], ), EventMetadataEntry.table_schema( label="table_schema", schema=TableSchema( columns=[ TableColumn( name="foo", type="integer", constraints=TableColumnConstraints( unique=True), ), TableColumn(name="bar", type="string"), ], constraints=TableConstraints(other=["some constraint" ], ), ), ), ], ) yield Output(None)
def many_table_materializations(_context): with open(file_relative_path(__file__, MARKDOWN_EXAMPLE), "r") as f: md_str = f.read() for table in raw_tables: yield AssetMaterialization( asset_key="table_info", metadata_entries=[ EventMetadataEntry.text(text=table, label="table_name"), EventMetadataEntry.fspath(path="/path/to/{}".format(table), label="table_path"), EventMetadataEntry.json(data={"name": table}, label="table_data"), EventMetadataEntry.url( url="https://bigty.pe/{}".format(table), label="table_name_big"), EventMetadataEntry.md(md_str=md_str, label="table_blurb"), ], )
def backcompat_materialize(_): yield Materialization( asset_key="all_types", description="a materialization with all metadata types", metadata_entries=[ EventMetadataEntry.text("text is cool", "text"), EventMetadataEntry.url("https://bigty.pe/neato", "url"), EventMetadataEntry.fspath("/tmp/awesome", "path"), EventMetadataEntry.json({"is_dope": True}, "json"), EventMetadataEntry.python_artifact(EventMetadataEntry, "python class"), EventMetadataEntry.python_artifact(file_relative_path, "python function"), EventMetadataEntry.float(1.2, "float"), EventMetadataEntry.int(1, "int"), EventMetadataEntry.float(float("nan"), "float NaN"), EventMetadataEntry.int(LONG_INT, "long int"), EventMetadataEntry.pipeline_run("fake_run_id", "pipeline run"), EventMetadataEntry.asset(AssetKey("my_asset"), "my asset"), ], ) yield Output(None)
def read_file(context): relative_filename = context.solid_config["filename"] directory = context.solid_config["directory"] filename = os.path.join(directory, relative_filename) try: fstats = os.stat(filename) context.log.info("Found file {}".format(relative_filename)) yield AssetMaterialization( asset_key=AssetKey(["log_file", relative_filename]), metadata_entries=[ EventMetadataEntry.fspath(filename), EventMetadataEntry.json( { "size": fstats.st_size, "ctime": fstats.st_ctime, "mtime": fstats.st_mtime, }, "File stats", ), ], ) yield Output(relative_filename) except FileNotFoundError: context.log.error("No file found: {}".format(relative_filename))
def _t_fn(compute_context, inputs): check.inst_param(compute_context, 'compute_context', ComputeExecutionContext) check.param_invariant( isinstance(compute_context.environment_dict, dict), 'context', 'SystemComputeExecutionContext must have valid environment_dict', ) system_compute_context = compute_context.get_system_context() base_dir = '/tmp/dagstermill/{run_id}/'.format( run_id=compute_context.run_id) output_notebook_dir = os.path.join(base_dir, 'output_notebooks/') mkdir_p(output_notebook_dir) temp_path = os.path.join( output_notebook_dir, '{prefix}-out.ipynb'.format(prefix=str(uuid.uuid4()))) with tempfile.NamedTemporaryFile() as output_log_file: output_log_path = output_log_file.name init_db(output_log_path) nb = load_notebook_node(notebook_path) nb_no_parameters = replace_parameters( system_compute_context, nb, get_papermill_parameters(system_compute_context, inputs, output_log_path), ) intermediate_path = os.path.join( output_notebook_dir, '{prefix}-inter.ipynb'.format(prefix=str(uuid.uuid4()))) write_ipynb(nb_no_parameters, intermediate_path) # Although the type of is_done is threading._Event in py2, not threading.Event, # it is still constructed using the threading.Event() factory is_done = threading.Event() def log_watcher_thread_target(): log_watcher = JsonSqlite3LogWatcher(output_log_path, system_compute_context.log, is_done) log_watcher.watch() log_watcher_thread = threading.Thread( target=log_watcher_thread_target) log_watcher_thread.start() with user_code_error_boundary( DagstermillExecutionError, lambda: 'Error occurred during the execution of Dagstermill solid ' '{solid_name}: {notebook_path}'.format( solid_name=name, notebook_path=notebook_path), ): try: papermill_engines.register('dagstermill', DagstermillNBConvertEngine) papermill.execute_notebook(intermediate_path, temp_path, engine_name='dagstermill', log_output=True) except Exception as exc: yield Materialization( label='output_notebook', description= 'Location of output notebook on the filesystem', metadata_entries=[ EventMetadataEntry.fspath(temp_path) ], ) raise exc finally: is_done.set() log_watcher_thread.join() output_nb = scrapbook.read_notebook(temp_path) system_compute_context.log.debug( 'Notebook execution complete for {name}. Data is {data}'. format(name=name, data=output_nb.scraps)) yield Materialization( label='output_notebook', description='Location of output notebook on the filesystem', metadata_entries=[EventMetadataEntry.fspath(temp_path)], ) for (output_name, output_def ) in system_compute_context.solid_def.output_dict.items(): data_dict = output_nb.scraps.data_dict if output_name in data_dict: value = read_value(output_def.runtime_type, data_dict[output_name]) yield Output(value, output_name) for key, value in output_nb.scraps.items(): print(output_nb.scraps) if key.startswith('event-'): with open(value.data, 'rb') as fd: yield pickle.loads(fd.read())
def _t_fn(compute_context, inputs): check.inst_param(compute_context, "compute_context", SolidExecutionContext) check.param_invariant( isinstance(compute_context.run_config, dict), "context", "SystemComputeExecutionContext must have valid run_config", ) system_compute_context = compute_context.get_system_context() with seven.TemporaryDirectory() as output_notebook_dir: with safe_tempfile_path() as output_log_path: parameterized_notebook_path = os.path.join( output_notebook_dir, "{prefix}-inter.ipynb".format(prefix=str(uuid.uuid4()))) executed_notebook_path = os.path.join( output_notebook_dir, "{prefix}-out.ipynb".format(prefix=str(uuid.uuid4()))) # Scaffold the registration here nb = load_notebook_node(notebook_path) nb_no_parameters = replace_parameters( system_compute_context, nb, get_papermill_parameters(system_compute_context, inputs, output_log_path), ) write_ipynb(nb_no_parameters, parameterized_notebook_path) with user_code_error_boundary( DagstermillExecutionError, lambda: ("Error occurred during the execution of Dagstermill solid " "{solid_name}: {notebook_path}".format( solid_name=name, notebook_path=notebook_path)), ): try: papermill_engines.register("dagstermill", DagstermillNBConvertEngine) papermill.execute_notebook( input_path=parameterized_notebook_path, output_path=executed_notebook_path, engine_name="dagstermill", log_output=True, ) except Exception as exc: # pylint: disable=broad-except try: with open(executed_notebook_path, "rb") as fd: executed_notebook_file_handle = compute_context.resources.file_manager.write( fd, mode="wb", ext="ipynb") executed_notebook_materialization_path = ( executed_notebook_file_handle.path_desc) except Exception as exc_inner: # pylint: disable=broad-except compute_context.log.warning( "Error when attempting to materialize executed notebook using file manager (falling back to local): {exc}" .format(exc=exc_inner)) executed_notebook_materialization_path = executed_notebook_path yield AssetMaterialization( asset_key=(asset_key_prefix + [f"{name}_output_notebook"]), description= "Location of output notebook in file manager", metadata_entries=[ EventMetadataEntry.fspath( executed_notebook_materialization_path, label="executed_notebook_path", ) ], ) raise exc system_compute_context.log.debug( "Notebook execution complete for {name} at {executed_notebook_path}." .format( name=name, executed_notebook_path=executed_notebook_path, )) executed_notebook_file_handle = None try: # use binary mode when when moving the file since certain file_managers such as S3 # may try to hash the contents with open(executed_notebook_path, "rb") as fd: executed_notebook_file_handle = compute_context.resources.file_manager.write( fd, mode="wb", ext="ipynb") executed_notebook_materialization_path = executed_notebook_file_handle.path_desc except Exception as exc: # pylint: disable=broad-except compute_context.log.warning( "Error when attempting to materialize executed notebook using file manager (falling back to local): {exc}" .format(exc=str(exc))) executed_notebook_materialization_path = executed_notebook_path yield AssetMaterialization( asset_key=(asset_key_prefix + [f"{name}_output_notebook"]), description="Location of output notebook in file manager", metadata_entries=[ EventMetadataEntry.fspath( executed_notebook_materialization_path) ], ) if output_notebook is not None: yield Output(executed_notebook_file_handle, output_notebook) # deferred import for perf import scrapbook output_nb = scrapbook.read_notebook(executed_notebook_path) for (output_name, output_def ) in system_compute_context.solid_def.output_dict.items(): data_dict = output_nb.scraps.data_dict if output_name in data_dict: value = read_value(output_def.dagster_type, data_dict[output_name]) yield Output(value, output_name) for key, value in output_nb.scraps.items(): if key.startswith("event-"): with open(value.data, "rb") as fd: yield pickle.loads(fd.read())
def _t_fn(compute_context, inputs): check.inst_param(compute_context, 'compute_context', SolidExecutionContext) check.param_invariant( isinstance(compute_context.environment_dict, dict), 'context', 'SystemComputeExecutionContext must have valid environment_dict', ) system_compute_context = compute_context.get_system_context() with seven.TemporaryDirectory() as output_notebook_dir: with safe_tempfile_path() as output_log_path: parameterized_notebook_path = os.path.join( output_notebook_dir, '{prefix}-inter.ipynb'.format(prefix=str(uuid.uuid4()))) executed_notebook_path = os.path.join( output_notebook_dir, '{prefix}-out.ipynb'.format(prefix=str(uuid.uuid4()))) # Scaffold the registration here nb = load_notebook_node(notebook_path) nb_no_parameters = replace_parameters( system_compute_context, nb, get_papermill_parameters(system_compute_context, inputs, output_log_path), ) write_ipynb(nb_no_parameters, parameterized_notebook_path) with user_code_error_boundary( DagstermillExecutionError, lambda: ('Error occurred during the execution of Dagstermill solid ' '{solid_name}: {notebook_path}'.format( solid_name=name, notebook_path=notebook_path)), ): try: papermill_engines.register('dagstermill', DagstermillNBConvertEngine) papermill.execute_notebook( input_path=parameterized_notebook_path, output_path=executed_notebook_path, engine_name='dagstermill', log_output=True, ) except Exception as exc: # pylint: disable=broad-except try: with open(executed_notebook_path, 'r') as fd: executed_notebook_file_handle = compute_context.file_manager.write( fd, mode='w', ext='ipynb') executed_notebook_materialization_path = ( executed_notebook_file_handle.path_desc) except Exception as exc_inner: # pylint: disable=broad-except compute_context.log.warning( 'Error when attempting to materialize executed notebook using file manager (falling back to local): {exc}' .format(exc=exc_inner)) executed_notebook_materialization_path = executed_notebook_path yield Materialization( label='output_notebook', description= 'Location of output notebook in file manager', metadata_entries=[ EventMetadataEntry.fspath( executed_notebook_materialization_path) ], ) raise exc system_compute_context.log.debug( 'Notebook execution complete for {name} at {executed_notebook_path}.' .format( name=name, executed_notebook_path=executed_notebook_path, )) try: with open(executed_notebook_path, 'r') as fd: executed_notebook_file_handle = compute_context.file_manager.write( fd, mode='w', ext='ipynb') executed_notebook_materialization_path = executed_notebook_file_handle.path_desc except Exception as exc: # pylint: disable=broad-except compute_context.log.warning( 'Error when attempting to materialize executed notebook using file manager (falling back to local): {exc}' .format(exc=str(exc))) executed_notebook_materialization_path = executed_notebook_path yield Materialization( label='output_notebook', description='Location of output notebook in file manager', metadata_entries=[ EventMetadataEntry.fspath( executed_notebook_materialization_path) ], ) if output_notebook is not None: yield Output(executed_notebook_file_handle, output_notebook) # deferred import for perf import scrapbook output_nb = scrapbook.read_notebook(executed_notebook_path) for (output_name, output_def ) in system_compute_context.solid_def.output_dict.items(): data_dict = output_nb.scraps.data_dict if output_name in data_dict: value = read_value(output_def.dagster_type, data_dict[output_name]) yield Output(value, output_name) for key, value in output_nb.scraps.items(): if key.startswith('event-'): with open(value.data, 'rb') as fd: yield pickle.loads(fd.read())
def _t_fn(step_context, inputs): check.inst_param(step_context, "step_context", SolidExecutionContext) check.param_invariant( isinstance(step_context.run_config, dict), "context", "StepExecutionContext must have valid run_config", ) step_execution_context = step_context.get_step_execution_context() with tempfile.TemporaryDirectory() as output_notebook_dir: with safe_tempfile_path() as output_log_path: parameterized_notebook_path = os.path.join( output_notebook_dir, "{prefix}-inter.ipynb".format(prefix=str(uuid.uuid4()))) executed_notebook_path = os.path.join( output_notebook_dir, "{prefix}-out.ipynb".format(prefix=str(uuid.uuid4()))) # Scaffold the registration here nb = load_notebook_node(notebook_path) nb_no_parameters = replace_parameters( step_execution_context, nb, get_papermill_parameters(step_execution_context, inputs, output_log_path), ) write_ipynb(nb_no_parameters, parameterized_notebook_path) try: papermill_engines.register("dagstermill", DagstermillNBConvertEngine) papermill.execute_notebook( input_path=parameterized_notebook_path, output_path=executed_notebook_path, engine_name="dagstermill", log_output=True, ) except Exception as ex: # pylint: disable=broad-except try: with open(executed_notebook_path, "rb") as fd: executed_notebook_file_handle = ( step_context.resources.file_manager.write( fd, mode="wb", ext="ipynb")) executed_notebook_materialization_path = ( executed_notebook_file_handle.path_desc) except Exception: # pylint: disable=broad-except step_context.log.warning( "Error when attempting to materialize executed notebook using file manager (falling back to local): {exc}" .format(exc=str( serializable_error_info_from_exc_info( sys.exc_info())))) executed_notebook_materialization_path = executed_notebook_path yield AssetMaterialization( asset_key=(asset_key_prefix + [f"{name}_output_notebook"]), description= "Location of output notebook in file manager", metadata_entries=[ EventMetadataEntry.fspath( executed_notebook_materialization_path, label="executed_notebook_path", ) ], ) # pylint: disable=no-member if isinstance(ex, PapermillExecutionError) and ( ex.ename == "RetryRequested" or ex.ename == "Failure"): step_execution_context.log.warn( f"Encountered raised {ex.ename} in notebook. Use dagstermill.yield_event " "with RetryRequested or Failure to trigger their behavior." ) raise step_execution_context.log.debug( "Notebook execution complete for {name} at {executed_notebook_path}." .format( name=name, executed_notebook_path=executed_notebook_path, )) executed_notebook_file_handle = None try: # use binary mode when when moving the file since certain file_managers such as S3 # may try to hash the contents with open(executed_notebook_path, "rb") as fd: executed_notebook_file_handle = step_context.resources.file_manager.write( fd, mode="wb", ext="ipynb") executed_notebook_materialization_path = executed_notebook_file_handle.path_desc except Exception: # pylint: disable=broad-except step_context.log.warning( "Error when attempting to materialize executed notebook using file manager (falling back to local): {exc}" .format(exc=str( serializable_error_info_from_exc_info( sys.exc_info())))) executed_notebook_materialization_path = executed_notebook_path yield AssetMaterialization( asset_key=(asset_key_prefix + [f"{name}_output_notebook"]), description="Location of output notebook in file manager", metadata_entries=[ EventMetadataEntry.fspath( executed_notebook_materialization_path) ], ) if output_notebook is not None: yield Output(executed_notebook_file_handle, output_notebook) # deferred import for perf import scrapbook output_nb = scrapbook.read_notebook(executed_notebook_path) for (output_name, output_def ) in step_execution_context.solid_def.output_dict.items(): data_dict = output_nb.scraps.data_dict if output_name in data_dict: value = read_value(output_def.dagster_type, data_dict[output_name]) yield Output(value, output_name) for key, value in output_nb.scraps.items(): if key.startswith("event-"): with open(value.data, "rb") as fd: event = pickle.loads(fd.read()) if isinstance(event, (Failure, RetryRequested)): raise event else: yield event
def _t_fn(compute_context, inputs): check.inst_param(compute_context, 'compute_context', ComputeExecutionContext) check.param_invariant( isinstance(compute_context.environment_dict, dict), 'context', 'SystemComputeExecutionContext must have valid environment_dict', ) system_compute_context = compute_context.get_system_context() base_dir = '/tmp/dagstermill/{run_id}/'.format( run_id=compute_context.run_id) output_notebook_dir = os.path.join(base_dir, 'output_notebooks/') mkdir_p(output_notebook_dir) temp_path = os.path.join( output_notebook_dir, '{prefix}-out.ipynb'.format(prefix=str(uuid.uuid4()))) with safe_tempfile_path() as output_log_path: # Scaffold the registration here nb = load_notebook_node(notebook_path) nb_no_parameters = replace_parameters( system_compute_context, nb, get_papermill_parameters(system_compute_context, inputs, output_log_path), ) intermediate_path = os.path.join( output_notebook_dir, '{prefix}-inter.ipynb'.format(prefix=str(uuid.uuid4()))) write_ipynb(nb_no_parameters, intermediate_path) with user_code_error_boundary( DagstermillExecutionError, lambda: ('Error occurred during the execution of Dagstermill solid ' '{solid_name}: {notebook_path}'.format( solid_name=name, notebook_path=notebook_path)), ): try: papermill_engines.register('dagstermill', DagstermillNBConvertEngine) papermill.execute_notebook(intermediate_path, temp_path, engine_name='dagstermill', log_output=True) except Exception as exc: yield Materialization( label='output_notebook', description= 'Location of output notebook on the filesystem', metadata_entries=[ EventMetadataEntry.fspath(temp_path) ], ) raise exc # deferred import for perf import scrapbook output_nb = scrapbook.read_notebook(temp_path) system_compute_context.log.debug( 'Notebook execution complete for {name}. Data is {data}'. format(name=name, data=output_nb.scraps)) yield Materialization( label='output_notebook', description='Location of output notebook on the filesystem', metadata_entries=[EventMetadataEntry.fspath(temp_path)], ) for (output_name, output_def ) in system_compute_context.solid_def.output_dict.items(): data_dict = output_nb.scraps.data_dict if output_name in data_dict: value = read_value(output_def.dagster_type, data_dict[output_name]) yield Output(value, output_name) for key, value in output_nb.scraps.items(): if key.startswith('event-'): with open(value.data, 'rb') as fd: yield pickle.loads(fd.read())