def create_mem_system_storage_data(init_context):
return SystemStorageData(
intermediates_manager=InMemoryIntermediatesManager(),
file_manager=LocalFileManager.for_instance(
init_context.instance, init_context.pipeline_run.run_id
),
)
def create_mem_system_storage_data(init_context):
return SystemStorageData(
run_storage=InMemoryRunStorage(),
intermediates_manager=InMemoryIntermediatesManager(),
file_manager=LocalFileManager.for_run_id(
init_context.run_config.run_id),
)
def fs_system_storage(init_context):
override_dir = init_context.system_storage_config.get('base_dir')
if override_dir:
file_manager = LocalFileManager(override_dir)
intermediate_store = build_fs_intermediate_store(
root_for_run_id=lambda _: override_dir,
run_id=init_context.run_config.run_id,
type_storage_plugin_registry=init_context.
type_storage_plugin_registry,
)
else:
file_manager = LocalFileManager.for_instance(
init_context.instance, init_context.run_config.run_id)
intermediate_store = build_fs_intermediate_store(
init_context.instance.intermediates_directory,
run_id=init_context.run_config.run_id,
type_storage_plugin_registry=init_context.
type_storage_plugin_registry,
)
return SystemStorageData(
file_manager=file_manager,
intermediates_manager=IntermediateStoreIntermediatesManager(
intermediate_store),
)
def create_mem_system_storage_data(init_context):
return SystemStorageData(
intermediates_manager=build_in_mem_intermediates_storage(
init_context.pipeline_run.run_id,
type_storage_plugin_registry=init_context.
type_storage_plugin_registry,
),
file_manager=LocalFileManager.for_instance(
init_context.instance, init_context.pipeline_run.run_id),
)
def fs_system_storage(init_context):
base_dir = init_context.system_storage_config.get('base_dir')
return SystemStorageData(
file_manager=LocalFileManager.for_run_id(
init_context.run_config.run_id),
run_storage=FilesystemRunStorage(base_dir=base_dir),
intermediates_manager=IntermediateStoreIntermediatesManager(
FileSystemIntermediateStore(
run_id=init_context.run_config.run_id,
type_storage_plugin_registry=init_context.
type_storage_plugin_registry,
base_dir=base_dir,
)),
)
def fs_system_storage(init_context):
'''The default filesystem system storage.
Filesystem system storage is available by default on any :py:class:`ModeDefinition` that does
not provide custom system storages. To select it, include a fragment such as the following in
config:
.. code-block:: yaml
storage:
filesystem:
base_dir: '/path/to/dir/'
You may omit the ``base_dir`` config value, in which case the filesystem storage will use
the :py:class:`DagsterInstance`-provided default.
'''
override_dir = init_context.system_storage_config.get('base_dir')
if override_dir:
file_manager = LocalFileManager(override_dir)
intermediate_store = build_fs_intermediate_store(
root_for_run_id=lambda _: override_dir,
run_id=init_context.pipeline_run.run_id,
type_storage_plugin_registry=init_context.
type_storage_plugin_registry,
)
else:
file_manager = LocalFileManager.for_instance(
init_context.instance, init_context.pipeline_run.run_id)
intermediate_store = build_fs_intermediate_store(
init_context.instance.intermediates_directory,
run_id=init_context.pipeline_run.run_id,
type_storage_plugin_registry=init_context.
type_storage_plugin_registry,
)
return SystemStorageData(
file_manager=file_manager,
intermediates_manager=IntermediateStoreIntermediatesManager(
intermediate_store),
)