def test_file_system_intermediate_store_with_composite_type_storage_plugin(): run_id = str(uuid.uuid4()) intermediate_store = build_fs_intermediate_store( DagsterInstance.ephemeral().intermediates_directory, run_id=run_id, type_storage_plugin_registry=TypeStoragePluginRegistry( {RuntimeString.inst(): FancyStringFilesystemTypeStoragePlugin}), ) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value(['hello'], context, resolve_to_runtime_type(List[String]), ['obj_name']) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value(['hello'], context, resolve_to_runtime_type( Optional[String]), ['obj_name']) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value( ['hello'], context, resolve_to_runtime_type(List[Optional[String]]), ['obj_name']) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value( ['hello'], context, resolve_to_runtime_type(Optional[List[String]]), ['obj_name'])
def test_file_system_intermediate_store_with_composite_type_storage_plugin(): run_id = str(uuid.uuid4()) # FIXME need a dedicated test bucket intermediate_store = FileSystemIntermediateStore( run_id=run_id, type_storage_plugin_registry=TypeStoragePluginRegistry( {RuntimeString.inst(): FancyStringFilesystemTypeStoragePlugin}), ) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value(['hello'], context, resolve_to_runtime_type(List[String]), ['obj_name']) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value(['hello'], context, resolve_to_runtime_type( Optional[String]), ['obj_name']) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value( ['hello'], context, resolve_to_runtime_type(List[Optional[String]]), ['obj_name']) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value( ['hello'], context, resolve_to_runtime_type(Optional[List[String]]), ['obj_name'])
def test_file_system_intermediate_store_with_type_storage_plugin(): run_id = str(uuid.uuid4()) # FIXME need a dedicated test bucket intermediate_store = FileSystemIntermediateStore( run_id=run_id, type_storage_plugin_registry=TypeStoragePluginRegistry( {RuntimeString.inst(): FancyStringFilesystemTypeStoragePlugin}), ) with yield_empty_pipeline_context(run_id=run_id) as context: try: intermediate_store.set_value('hello', context, RuntimeString.inst(), ['obj_name']) assert intermediate_store.has_object(context, ['obj_name']) assert (intermediate_store.get_value(context, RuntimeString.inst(), ['obj_name']) == 'hello') finally: intermediate_store.rm_object(context, ['obj_name'])
def test_s3_intermediate_store_with_type_storage_plugin(): run_id = str(uuid.uuid4()) # FIXME need a dedicated test bucket intermediate_store = S3IntermediateStore( run_id=run_id, s3_bucket='dagster-airflow-scratch', types_to_register={RuntimeString.inst(): FancyStringS3TypeStoragePlugin}, ) with yield_empty_pipeline_context(run_id=run_id) as context: try: intermediate_store.set_value('hello', context, RuntimeString.inst(), ['obj_name']) assert intermediate_store.has_object(context, ['obj_name']) assert ( intermediate_store.get_value(context, RuntimeString.inst(), ['obj_name']) == 'hello' ) finally: intermediate_store.rm_object(context, ['obj_name'])
def test_s3_intermediate_store_with_type_storage_plugin(s3_bucket): run_id = str(uuid.uuid4()) intermediate_store = S3IntermediateStore( run_id=run_id, s3_bucket=s3_bucket, type_storage_plugin_registry=TypeStoragePluginRegistry( {RuntimeString.inst(): FancyStringS3TypeStoragePlugin}), ) with yield_empty_pipeline_context(run_id=run_id) as context: try: intermediate_store.set_value('hello', context, RuntimeString.inst(), ['obj_name']) assert intermediate_store.has_object(context, ['obj_name']) assert (intermediate_store.get_value(context, RuntimeString.inst(), ['obj_name']) == 'hello') finally: intermediate_store.rm_object(context, ['obj_name'])
def test_s3_object_store_with_type_storage_plugin(): run_id = str(uuid.uuid4()) # FIXME need a dedicated test bucket object_store = S3ObjectStore( run_id=run_id, s3_bucket='dagster-airflow-scratch', types_to_register={String.inst(): FancyStringS3TypeStoragePlugin}, ) with yield_pipeline_execution_context(PipelineDefinition([]), {}, RunConfig(run_id=run_id)) as context: try: object_store.set_value('hello', context, String.inst(), ['obj_name']) assert object_store.has_object(context, ['obj_name']) assert object_store.get_value(context, String.inst(), ['obj_name']) == 'hello' finally: object_store.rm_object(context, ['obj_name'])
def test_file_system_intermediate_store_with_type_storage_plugin(): run_id = str(uuid.uuid4()) instance = DagsterInstance.ephemeral() intermediate_store = build_fs_intermediate_store( instance.intermediates_directory, run_id=run_id, type_storage_plugin_registry=TypeStoragePluginRegistry( {RuntimeString.inst(): FancyStringFilesystemTypeStoragePlugin}), ) with yield_empty_pipeline_context(run_id=run_id, instance=instance) as context: try: intermediate_store.set_value('hello', context, RuntimeString.inst(), ['obj_name']) assert intermediate_store.has_object(context, ['obj_name']) assert (intermediate_store.get_value(context, RuntimeString.inst(), ['obj_name']) == 'hello') finally: intermediate_store.rm_object(context, ['obj_name'])
def test_s3_intermediate_store_with_composite_type_storage_plugin(): run_id = str(uuid.uuid4()) # FIXME need a dedicated test bucket intermediate_store = S3IntermediateStore( run_id=run_id, s3_bucket='dagster-airflow-scratch', types_to_register={RuntimeString.inst(): FancyStringS3TypeStoragePlugin}, ) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value( ['hello'], context, resolve_to_runtime_type(List(String)), ['obj_name'] )
def test_s3_intermediate_store_with_composite_type_storage_plugin(s3_bucket): run_id = str(uuid.uuid4()) intermediate_store = S3IntermediateStore( run_id=run_id, s3_bucket=s3_bucket, type_storage_plugin_registry=TypeStoragePluginRegistry( {RuntimeString.inst(): FancyStringS3TypeStoragePlugin}), ) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value(['hello'], context, resolve_to_runtime_type(List[String]), ['obj_name'])
def test_s3_object_store_with_composite_type_storage_plugin(): run_id = str(uuid.uuid4()) # FIXME need a dedicated test bucket object_store = S3ObjectStore( run_id=run_id, s3_bucket='dagster-airflow-scratch', types_to_register={String.inst(): FancyStringS3TypeStoragePlugin}, ) with yield_pipeline_execution_context(PipelineDefinition([]), {}, RunConfig(run_id=run_id)) as context: with pytest.raises(check.NotImplementedCheckError): object_store.set_value(['hello'], context, resolve_to_runtime_type(List_(String_)), ['obj_name'])
def test_file_system_object_store_with_composite_type_storage_plugin(): run_id = str(uuid.uuid4()) # FIXME need a dedicated test bucket object_store = FileSystemObjectStore( run_id=run_id, types_to_register={ String.inst(): FancyStringFilesystemTypeStoragePlugin }) with yield_pipeline_execution_context(PipelineDefinition([]), {}, RunConfig(run_id=run_id)) as context: with pytest.raises(check.NotImplementedCheckError): object_store.set_value(['hello'], context, resolve_to_runtime_type(List_(String_)), ['obj_name']) with yield_pipeline_execution_context(PipelineDefinition([]), {}, RunConfig(run_id=run_id)) as context: with pytest.raises(check.NotImplementedCheckError): object_store.set_value(['hello'], context, resolve_to_runtime_type(Nullable_(String_)), ['obj_name']) with yield_pipeline_execution_context(PipelineDefinition([]), {}, RunConfig(run_id=run_id)) as context: with pytest.raises(check.NotImplementedCheckError): object_store.set_value(['hello'], context, resolve_to_runtime_type( List_(Nullable_(String_))), ['obj_name']) with yield_pipeline_execution_context(PipelineDefinition([]), {}, RunConfig(run_id=run_id)) as context: with pytest.raises(check.NotImplementedCheckError): object_store.set_value(['hello'], context, resolve_to_runtime_type( Nullable_(List_(String_))), ['obj_name'])