Python Blob Examples

Example #1

def test_blob_download_unmanaged():
    with AutoDeletingTempDir('test') as wd:
        with AutoDeletingTempDir('test2') as t:
            tmp_name = wd.get_named_tempfile('source')
            tmp_sink = t.get_named_tempfile('sink')
            with open(tmp_name, 'wb') as w:
                w.write("hello".encode('utf-8'))

            b = blobs.Blob(tmp_name)
            b.download(tmp_sink)
            assert b.local_path == tmp_sink
            assert b.remote_location == tmp_name
            assert b.mode == 'rb'
            assert b.metadata.type.format == ""
            assert b.metadata.type.dimensionality == _core_types.BlobType.BlobDimensionality.SINGLE
            with b as r:
                assert r.read() == "hello".encode('utf-8')

            b = blobs.Blob(tmp_name)
            with pytest.raises(_user_exceptions.FlyteAssertion):
                b.download(tmp_sink)

            with open(tmp_name, 'wb') as w:
                w.write("bye".encode('utf-8'))

            b2 = blobs.Blob(tmp_name)
            b2.download(tmp_sink, overwrite=True)
            with b2 as r:
                assert r.read() == "bye".encode('utf-8')

Example #2

def test_blob_download_managed():
    with AutoDeletingTempDir('test') as wd:
        with test_utils.LocalTestFileSystem() as t:
            tmp_name = wd.get_named_tempfile('tmp')
            with open(tmp_name, 'wb') as w:
                w.write("hello".encode('utf-8'))

            b = blobs.Blob(tmp_name)
            b.download()
            assert b.local_path.startswith(t.name)
            assert b.remote_location == tmp_name
            assert b.mode == 'rb'
            assert b.metadata.type.format == ""
            assert b.metadata.type.dimensionality == _core_types.BlobType.BlobDimensionality.SINGLE
            with b as r:
                assert r.read() == "hello".encode('utf-8')

            b2 = blobs.Blob(tmp_name)
            with pytest.raises(_user_exceptions.FlyteAssertion):
                b2.download(b.local_path)

            with open(tmp_name, 'wb') as w:
                w.write("bye".encode('utf-8'))

            b2 = blobs.Blob(tmp_name)
            b2.download(local_path=b.local_path, overwrite=True)
            with b2 as r:
                assert r.read() == "bye".encode('utf-8')

        b = blobs.Blob(tmp_name)
        with pytest.raises(_user_exceptions.FlyteAssertion):
            b.download()

Example #3

def test_blob():
    b = blobs.Blob("/tmp/fake")
    assert b.remote_location == "/tmp/fake"
    assert b.local_path is None
    assert b.mode == "rb"
    assert b.metadata.type.format == ""
    assert b.metadata.type.dimensionality == _core_types.BlobType.BlobDimensionality.SINGLE

Example #4

def test_blob_double_enter():
    with test_utils.LocalTestFileSystem():
        with AutoDeletingTempDir('test') as wd:
            b = blobs.Blob(wd.get_named_tempfile("sink"), mode='wb')
            with b:
                with pytest.raises(_user_exceptions.FlyteAssertion):
                    with b:
                        pass

Example #5

def test_blob_from_python_std():
    with test_utils.LocalTestFileSystem() as t:
        with AutoDeletingTempDir('test') as wd:
            tmp_name = wd.get_named_tempfile("from_python_std")
            with open(tmp_name, 'wb') as w:
                w.write("hello hello".encode('utf-8'))
            b = blobs.Blob.from_python_std(tmp_name)
            assert b.mode == "wb"
            assert b.metadata.type.format == ""
            assert b.metadata.type.dimensionality == _core_types.BlobType.BlobDimensionality.SINGLE
            assert b.remote_location.startswith(t.name)
            assert b.local_path == tmp_name
            with open(b.remote_location, 'rb') as r:
                assert r.read() == "hello hello".encode('utf-8')

    b = blobs.Blob("/tmp/fake")
    b2 = blobs.Blob.from_python_std(b)
    assert b == b2

    with pytest.raises(_user_exceptions.FlyteTypeException):
        blobs.Blob.from_python_std(3)

Example #6

File: test_blobs.py Project: ybubnov/flytekit

def test_blob_to_python_std():
    impl = blob_impl.Blob("some/path", format="something")
    b = blobs.Blob(impl).to_python_std()
    assert b.metadata.type.format == "something"
    assert b.metadata.type.dimensionality == _core_types.BlobType.BlobDimensionality.SINGLE
    assert b.uri == "some/path"

Example #7

]

LIST_OF_SCALARS_AND_PYTHON_VALUES = [
    (literals.Scalar(primitive=literals.Primitive(integer=100)), 100),
    (literals.Scalar(primitive=literals.Primitive(float_value=500.0)), 500.0),
    (literals.Scalar(primitive=literals.Primitive(boolean=True)), True),
    (literals.Scalar(primitive=literals.Primitive(string_value='hello')),
     'hello'),
    (literals.Scalar(primitive=literals.Primitive(duration=timedelta(
        seconds=5))), timedelta(seconds=5)),
    (literals.Scalar(none_type=literals.Void()), None),
    (literals.Scalar(blob=literals.Blob(
        literals.BlobMetadata(
            _core_types.BlobType(
                "csv", _core_types.BlobType.BlobDimensionality.SINGLE)),
        "s3://some/where")), _blob_impl.Blob("s3://some/where", format="csv")),
    (literals.Scalar(blob=literals.Blob(
        literals.BlobMetadata(
            _core_types.BlobType(
                "", _core_types.BlobType.BlobDimensionality.SINGLE)),
        "s3://some/where")), _blob_impl.Blob("s3://some/where")),
    (literals.Scalar(blob=literals.Blob(
        literals.BlobMetadata(
            _core_types.BlobType(
                "csv", _core_types.BlobType.BlobDimensionality.MULTIPART)),
        "s3://some/where/")),
     _blob_impl.MultiPartBlob("s3://some/where/", format="csv")),
    (literals.Scalar(blob=literals.Blob(
        literals.BlobMetadata(
            _core_types.BlobType(
                "", _core_types.BlobType.BlobDimensionality.MULTIPART)),