def test_blob_download_unmanaged():
with AutoDeletingTempDir('test') as wd:
with AutoDeletingTempDir('test2') as t:
tmp_name = wd.get_named_tempfile('source')
tmp_sink = t.get_named_tempfile('sink')
with open(tmp_name, 'wb') as w:
w.write("hello".encode('utf-8'))
b = blobs.Blob(tmp_name)
b.download(tmp_sink)
assert b.local_path == tmp_sink
assert b.remote_location == tmp_name
assert b.mode == 'rb'
assert b.metadata.type.format == ""
assert b.metadata.type.dimensionality == _core_types.BlobType.BlobDimensionality.SINGLE
with b as r:
assert r.read() == "hello".encode('utf-8')
b = blobs.Blob(tmp_name)
with pytest.raises(_user_exceptions.FlyteAssertion):
b.download(tmp_sink)
with open(tmp_name, 'wb') as w:
w.write("bye".encode('utf-8'))
b2 = blobs.Blob(tmp_name)
b2.download(tmp_sink, overwrite=True)
with b2 as r:
assert r.read() == "bye".encode('utf-8')
def test_blob_download_managed():
with AutoDeletingTempDir('test') as wd:
with test_utils.LocalTestFileSystem() as t:
tmp_name = wd.get_named_tempfile('tmp')
with open(tmp_name, 'wb') as w:
w.write("hello".encode('utf-8'))
b = blobs.Blob(tmp_name)
b.download()
assert b.local_path.startswith(t.name)
assert b.remote_location == tmp_name
assert b.mode == 'rb'
assert b.metadata.type.format == ""
assert b.metadata.type.dimensionality == _core_types.BlobType.BlobDimensionality.SINGLE
with b as r:
assert r.read() == "hello".encode('utf-8')
b2 = blobs.Blob(tmp_name)
with pytest.raises(_user_exceptions.FlyteAssertion):
b2.download(b.local_path)
with open(tmp_name, 'wb') as w:
w.write("bye".encode('utf-8'))
b2 = blobs.Blob(tmp_name)
b2.download(local_path=b.local_path, overwrite=True)
with b2 as r:
assert r.read() == "bye".encode('utf-8')
b = blobs.Blob(tmp_name)
with pytest.raises(_user_exceptions.FlyteAssertion):
b.download()
def test_blob():
b = blobs.Blob("/tmp/fake")
assert b.remote_location == "/tmp/fake"
assert b.local_path is None
assert b.mode == "rb"
assert b.metadata.type.format == ""
assert b.metadata.type.dimensionality == _core_types.BlobType.BlobDimensionality.SINGLE
def test_blob_double_enter():
with test_utils.LocalTestFileSystem():
with AutoDeletingTempDir('test') as wd:
b = blobs.Blob(wd.get_named_tempfile("sink"), mode='wb')
with b:
with pytest.raises(_user_exceptions.FlyteAssertion):
with b:
pass
def test_blob_from_python_std():
with test_utils.LocalTestFileSystem() as t:
with AutoDeletingTempDir('test') as wd:
tmp_name = wd.get_named_tempfile("from_python_std")
with open(tmp_name, 'wb') as w:
w.write("hello hello".encode('utf-8'))
b = blobs.Blob.from_python_std(tmp_name)
assert b.mode == "wb"
assert b.metadata.type.format == ""
assert b.metadata.type.dimensionality == _core_types.BlobType.BlobDimensionality.SINGLE
assert b.remote_location.startswith(t.name)
assert b.local_path == tmp_name
with open(b.remote_location, 'rb') as r:
assert r.read() == "hello hello".encode('utf-8')
b = blobs.Blob("/tmp/fake")
b2 = blobs.Blob.from_python_std(b)
assert b == b2
with pytest.raises(_user_exceptions.FlyteTypeException):
blobs.Blob.from_python_std(3)
def test_blob_to_python_std():
impl = blob_impl.Blob("some/path", format="something")
b = blobs.Blob(impl).to_python_std()
assert b.metadata.type.format == "something"
assert b.metadata.type.dimensionality == _core_types.BlobType.BlobDimensionality.SINGLE
assert b.uri == "some/path"
]
LIST_OF_SCALARS_AND_PYTHON_VALUES = [
(literals.Scalar(primitive=literals.Primitive(integer=100)), 100),
(literals.Scalar(primitive=literals.Primitive(float_value=500.0)), 500.0),
(literals.Scalar(primitive=literals.Primitive(boolean=True)), True),
(literals.Scalar(primitive=literals.Primitive(string_value='hello')),
'hello'),
(literals.Scalar(primitive=literals.Primitive(duration=timedelta(
seconds=5))), timedelta(seconds=5)),
(literals.Scalar(none_type=literals.Void()), None),
(literals.Scalar(blob=literals.Blob(
literals.BlobMetadata(
_core_types.BlobType(
"csv", _core_types.BlobType.BlobDimensionality.SINGLE)),
"s3://some/where")), _blob_impl.Blob("s3://some/where", format="csv")),
(literals.Scalar(blob=literals.Blob(
literals.BlobMetadata(
_core_types.BlobType(
"", _core_types.BlobType.BlobDimensionality.SINGLE)),
"s3://some/where")), _blob_impl.Blob("s3://some/where")),
(literals.Scalar(blob=literals.Blob(
literals.BlobMetadata(
_core_types.BlobType(
"csv", _core_types.BlobType.BlobDimensionality.MULTIPART)),
"s3://some/where/")),
_blob_impl.MultiPartBlob("s3://some/where/", format="csv")),
(literals.Scalar(blob=literals.Blob(
literals.BlobMetadata(
_core_types.BlobType(
"", _core_types.BlobType.BlobDimensionality.MULTIPART)),
