def create_mem_system_storage_data(init_context): return SystemStorageData( intermediates_manager=InMemoryIntermediatesManager(), file_manager=LocalFileManager.for_instance( init_context.instance, init_context.pipeline_run.run_id ), )
def create_mem_system_storage_data(init_context): return SystemStorageData( run_storage=InMemoryRunStorage(), intermediates_manager=InMemoryIntermediatesManager(), file_manager=LocalFileManager.for_run_id( init_context.run_config.run_id), )
def fs_system_storage(init_context): override_dir = init_context.system_storage_config.get('base_dir') if override_dir: file_manager = LocalFileManager(override_dir) intermediate_store = build_fs_intermediate_store( root_for_run_id=lambda _: override_dir, run_id=init_context.run_config.run_id, type_storage_plugin_registry=init_context. type_storage_plugin_registry, ) else: file_manager = LocalFileManager.for_instance( init_context.instance, init_context.run_config.run_id) intermediate_store = build_fs_intermediate_store( init_context.instance.intermediates_directory, run_id=init_context.run_config.run_id, type_storage_plugin_registry=init_context. type_storage_plugin_registry, ) return SystemStorageData( file_manager=file_manager, intermediates_manager=IntermediateStoreIntermediatesManager( intermediate_store), )
def create_mem_system_storage_data(init_context): return SystemStorageData( intermediates_manager=build_in_mem_intermediates_storage( init_context.pipeline_run.run_id, type_storage_plugin_registry=init_context. type_storage_plugin_registry, ), file_manager=LocalFileManager.for_instance( init_context.instance, init_context.pipeline_run.run_id), )
def fs_system_storage(init_context): base_dir = init_context.system_storage_config.get('base_dir') return SystemStorageData( file_manager=LocalFileManager.for_run_id( init_context.run_config.run_id), run_storage=FilesystemRunStorage(base_dir=base_dir), intermediates_manager=IntermediateStoreIntermediatesManager( FileSystemIntermediateStore( run_id=init_context.run_config.run_id, type_storage_plugin_registry=init_context. type_storage_plugin_registry, base_dir=base_dir, )), )
def fs_system_storage(init_context): '''The default filesystem system storage. Filesystem system storage is available by default on any :py:class:`ModeDefinition` that does not provide custom system storages. To select it, include a fragment such as the following in config: .. code-block:: yaml storage: filesystem: base_dir: '/path/to/dir/' You may omit the ``base_dir`` config value, in which case the filesystem storage will use the :py:class:`DagsterInstance`-provided default. ''' override_dir = init_context.system_storage_config.get('base_dir') if override_dir: file_manager = LocalFileManager(override_dir) intermediate_store = build_fs_intermediate_store( root_for_run_id=lambda _: override_dir, run_id=init_context.pipeline_run.run_id, type_storage_plugin_registry=init_context. type_storage_plugin_registry, ) else: file_manager = LocalFileManager.for_instance( init_context.instance, init_context.pipeline_run.run_id) intermediate_store = build_fs_intermediate_store( init_context.instance.intermediates_directory, run_id=init_context.pipeline_run.run_id, type_storage_plugin_registry=init_context. type_storage_plugin_registry, ) return SystemStorageData( file_manager=file_manager, intermediates_manager=IntermediateStoreIntermediatesManager( intermediate_store), )