def test_multipart_blob():
b = blobs.MultiPartBlob("/tmp/fake", mode='w', format='csv')
assert b.remote_location == "/tmp/fake/"
assert b.local_path is None
assert b.mode == "w"
assert b.metadata.type.format == "csv"
assert b.metadata.type.dimensionality == _core_types.BlobType.BlobDimensionality.MULTIPART
def test_multipart_blob_from_python_std():
with test_utils.LocalTestFileSystem() as t:
with AutoDeletingTempDir('test') as wd:
_generate_multipart_blob_data(wd)
b = blobs.MultiPartBlob.from_python_std(wd.name)
assert b.mode == "wb"
assert b.metadata.type.format == ""
assert b.metadata.type.dimensionality == _core_types.BlobType.BlobDimensionality.MULTIPART
assert b.remote_location.startswith(t.name)
assert b.local_path == wd.name
with open(os.path.join(b.remote_location, '0'), 'rb') as r:
assert r.read() == "part0".encode('utf-8')
with open(os.path.join(b.remote_location, '1'), 'rb') as r:
assert r.read() == "part1".encode('utf-8')
with open(os.path.join(b.remote_location, '2'), 'rb') as r:
assert r.read() == "part2".encode('utf-8')
b = blobs.MultiPartBlob("/tmp/fake/")
b2 = blobs.MultiPartBlob.from_python_std(b)
assert b == b2
with pytest.raises(_user_exceptions.FlyteTypeException):
blobs.MultiPartBlob.from_python_std(3)
(literals.Scalar(blob=literals.Blob(
literals.BlobMetadata(
_core_types.BlobType(
"csv", _core_types.BlobType.BlobDimensionality.SINGLE)),
"s3://some/where")), _blob_impl.Blob("s3://some/where", format="csv")),
(literals.Scalar(blob=literals.Blob(
literals.BlobMetadata(
_core_types.BlobType(
"", _core_types.BlobType.BlobDimensionality.SINGLE)),
"s3://some/where")), _blob_impl.Blob("s3://some/where")),
(literals.Scalar(blob=literals.Blob(
literals.BlobMetadata(
_core_types.BlobType(
"csv", _core_types.BlobType.BlobDimensionality.MULTIPART)),
"s3://some/where/")),
_blob_impl.MultiPartBlob("s3://some/where/", format="csv")),
(literals.Scalar(blob=literals.Blob(
literals.BlobMetadata(
_core_types.BlobType(
"", _core_types.BlobType.BlobDimensionality.MULTIPART)),
"s3://some/where/")), _blob_impl.MultiPartBlob("s3://some/where/")),
(literals.Scalar(schema=literals.Schema(
"s3://some/where/",
types.SchemaType([
types.SchemaType.SchemaColumn(
"a", types.SchemaType.SchemaColumn.SchemaColumnType.INTEGER),
types.SchemaType.SchemaColumn(
"b", types.SchemaType.SchemaColumn.SchemaColumnType.BOOLEAN),
types.SchemaType.SchemaColumn(
"c", types.SchemaType.SchemaColumn.SchemaColumnType.DATETIME),
types.SchemaType.SchemaColumn(