def test_file_system_intermediate_store_with_composite_type_storage_plugin():
run_id = str(uuid.uuid4())
intermediate_store = build_fs_intermediate_store(
DagsterInstance.ephemeral().intermediates_directory,
run_id=run_id,
type_storage_plugin_registry=TypeStoragePluginRegistry(
{RuntimeString.inst(): FancyStringFilesystemTypeStoragePlugin}),
)
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(List[String]),
['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(
Optional[String]), ['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(
['hello'], context,
resolve_to_runtime_type(List[Optional[String]]), ['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(
['hello'], context,
resolve_to_runtime_type(Optional[List[String]]), ['obj_name'])
def test_file_system_intermediate_store_with_composite_type_storage_plugin():
run_id = str(uuid.uuid4())
# FIXME need a dedicated test bucket
intermediate_store = FileSystemIntermediateStore(
run_id=run_id,
type_storage_plugin_registry=TypeStoragePluginRegistry(
{RuntimeString.inst(): FancyStringFilesystemTypeStoragePlugin}),
)
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(List[String]),
['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(
Optional[String]), ['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(
['hello'], context,
resolve_to_runtime_type(List[Optional[String]]), ['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(
['hello'], context,
resolve_to_runtime_type(Optional[List[String]]), ['obj_name'])
def test_file_system_intermediate_store_with_type_storage_plugin():
run_id = str(uuid.uuid4())
# FIXME need a dedicated test bucket
intermediate_store = FileSystemIntermediateStore(
run_id=run_id,
type_storage_plugin_registry=TypeStoragePluginRegistry(
{RuntimeString.inst(): FancyStringFilesystemTypeStoragePlugin}),
)
with yield_empty_pipeline_context(run_id=run_id) as context:
try:
intermediate_store.set_value('hello', context,
RuntimeString.inst(), ['obj_name'])
assert intermediate_store.has_object(context, ['obj_name'])
assert (intermediate_store.get_value(context, RuntimeString.inst(),
['obj_name']) == 'hello')
finally:
intermediate_store.rm_object(context, ['obj_name'])
def test_s3_intermediate_store_with_type_storage_plugin():
run_id = str(uuid.uuid4())
# FIXME need a dedicated test bucket
intermediate_store = S3IntermediateStore(
run_id=run_id,
s3_bucket='dagster-airflow-scratch',
types_to_register={RuntimeString.inst(): FancyStringS3TypeStoragePlugin},
)
with yield_empty_pipeline_context(run_id=run_id) as context:
try:
intermediate_store.set_value('hello', context, RuntimeString.inst(), ['obj_name'])
assert intermediate_store.has_object(context, ['obj_name'])
assert (
intermediate_store.get_value(context, RuntimeString.inst(), ['obj_name']) == 'hello'
)
finally:
intermediate_store.rm_object(context, ['obj_name'])
def test_s3_intermediate_store_with_type_storage_plugin(s3_bucket):
run_id = str(uuid.uuid4())
intermediate_store = S3IntermediateStore(
run_id=run_id,
s3_bucket=s3_bucket,
type_storage_plugin_registry=TypeStoragePluginRegistry(
{RuntimeString.inst(): FancyStringS3TypeStoragePlugin}),
)
with yield_empty_pipeline_context(run_id=run_id) as context:
try:
intermediate_store.set_value('hello', context,
RuntimeString.inst(), ['obj_name'])
assert intermediate_store.has_object(context, ['obj_name'])
assert (intermediate_store.get_value(context, RuntimeString.inst(),
['obj_name']) == 'hello')
finally:
intermediate_store.rm_object(context, ['obj_name'])
def test_s3_object_store_with_type_storage_plugin():
run_id = str(uuid.uuid4())
# FIXME need a dedicated test bucket
object_store = S3ObjectStore(
run_id=run_id,
s3_bucket='dagster-airflow-scratch',
types_to_register={String.inst(): FancyStringS3TypeStoragePlugin},
)
with yield_pipeline_execution_context(PipelineDefinition([]), {},
RunConfig(run_id=run_id)) as context:
try:
object_store.set_value('hello', context, String.inst(),
['obj_name'])
assert object_store.has_object(context, ['obj_name'])
assert object_store.get_value(context, String.inst(),
['obj_name']) == 'hello'
finally:
object_store.rm_object(context, ['obj_name'])
def test_file_system_intermediate_store_with_type_storage_plugin():
run_id = str(uuid.uuid4())
instance = DagsterInstance.ephemeral()
intermediate_store = build_fs_intermediate_store(
instance.intermediates_directory,
run_id=run_id,
type_storage_plugin_registry=TypeStoragePluginRegistry(
{RuntimeString.inst(): FancyStringFilesystemTypeStoragePlugin}),
)
with yield_empty_pipeline_context(run_id=run_id,
instance=instance) as context:
try:
intermediate_store.set_value('hello', context,
RuntimeString.inst(), ['obj_name'])
assert intermediate_store.has_object(context, ['obj_name'])
assert (intermediate_store.get_value(context, RuntimeString.inst(),
['obj_name']) == 'hello')
finally:
intermediate_store.rm_object(context, ['obj_name'])
def test_s3_intermediate_store_with_composite_type_storage_plugin():
run_id = str(uuid.uuid4())
# FIXME need a dedicated test bucket
intermediate_store = S3IntermediateStore(
run_id=run_id,
s3_bucket='dagster-airflow-scratch',
types_to_register={RuntimeString.inst(): FancyStringS3TypeStoragePlugin},
)
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(
['hello'], context, resolve_to_runtime_type(List(String)), ['obj_name']
)
def test_s3_intermediate_store_with_composite_type_storage_plugin(s3_bucket):
run_id = str(uuid.uuid4())
intermediate_store = S3IntermediateStore(
run_id=run_id,
s3_bucket=s3_bucket,
type_storage_plugin_registry=TypeStoragePluginRegistry(
{RuntimeString.inst(): FancyStringS3TypeStoragePlugin}),
)
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(List[String]),
['obj_name'])
def test_s3_object_store_with_composite_type_storage_plugin():
run_id = str(uuid.uuid4())
# FIXME need a dedicated test bucket
object_store = S3ObjectStore(
run_id=run_id,
s3_bucket='dagster-airflow-scratch',
types_to_register={String.inst(): FancyStringS3TypeStoragePlugin},
)
with yield_pipeline_execution_context(PipelineDefinition([]), {},
RunConfig(run_id=run_id)) as context:
with pytest.raises(check.NotImplementedCheckError):
object_store.set_value(['hello'], context,
resolve_to_runtime_type(List_(String_)),
['obj_name'])
def test_file_system_object_store_with_composite_type_storage_plugin():
run_id = str(uuid.uuid4())
# FIXME need a dedicated test bucket
object_store = FileSystemObjectStore(
run_id=run_id,
types_to_register={
String.inst(): FancyStringFilesystemTypeStoragePlugin
})
with yield_pipeline_execution_context(PipelineDefinition([]), {},
RunConfig(run_id=run_id)) as context:
with pytest.raises(check.NotImplementedCheckError):
object_store.set_value(['hello'], context,
resolve_to_runtime_type(List_(String_)),
['obj_name'])
with yield_pipeline_execution_context(PipelineDefinition([]), {},
RunConfig(run_id=run_id)) as context:
with pytest.raises(check.NotImplementedCheckError):
object_store.set_value(['hello'], context,
resolve_to_runtime_type(Nullable_(String_)),
['obj_name'])
with yield_pipeline_execution_context(PipelineDefinition([]), {},
RunConfig(run_id=run_id)) as context:
with pytest.raises(check.NotImplementedCheckError):
object_store.set_value(['hello'], context,
resolve_to_runtime_type(
List_(Nullable_(String_))),
['obj_name'])
with yield_pipeline_execution_context(PipelineDefinition([]), {},
RunConfig(run_id=run_id)) as context:
with pytest.raises(check.NotImplementedCheckError):
object_store.set_value(['hello'], context,
resolve_to_runtime_type(
Nullable_(List_(String_))),
['obj_name'])
