def test_blob_promote_from_model():
m = _literal_models.Literal(scalar=_literal_models.Scalar(
blob=_literal_models.Blob(
_literal_models.BlobMetadata(
_core_types.BlobType(format="f",
dimensionality=_core_types.BlobType.
BlobDimensionality.SINGLE)),
"some/path")))
b = blobs.Blob.promote_from_model(m)
assert b.value.blob.uri == "some/path"
assert b.value.blob.metadata.type.format == "f"
assert b.value.blob.metadata.type.dimensionality == _core_types.BlobType.BlobDimensionality.SINGLE
def __init__(self, remote_path, mode='rb', format=None):
"""
:param Text remote_path: Path to location where the Blob should be synced to.
:param Text mode: File access mode. 'a' and '+' are forbidden. A blob can only be written or read at a time.
:param Text format: Format
"""
if '+' in mode or 'a' in mode or ('w' in mode and 'r' in mode):
raise _user_exceptions.FlyteAssertion(
"A blob cannot be read and written at the same time")
self._mode = mode
self._local_path = None
self._file = None
super(Blob, self).__init__(
_literal_models.BlobMetadata(type=_core_types.BlobType(
format or "", _core_types.BlobType.BlobDimensionality.SINGLE)),
remote_path)
def __init__(self, remote_path, mode='rb', format=None):
"""
:param Text remote_path: Path to location where the Blob should be synced to.
:param Text mode: File access mode. 'a' and '+' are forbidden. A blob can only be written or read at a time.
:param Text format: Format of underlying blob pieces.
"""
remote_path = remote_path.strip().rstrip('/') + '/'
super(MultiPartBlob, self).__init__(
_literal_models.BlobMetadata(type=_core_types.BlobType(
format or "",
_core_types.BlobType.BlobDimensionality.MULTIPART)),
remote_path)
self._is_managed = False
self._blobs = []
self._directory = None
self._mode = mode
task.TaskClosure(task.CompiledTask(template))
for template in LIST_OF_TASK_TEMPLATES
]
LIST_OF_SCALARS_AND_PYTHON_VALUES = [
(literals.Scalar(primitive=literals.Primitive(integer=100)), 100),
(literals.Scalar(primitive=literals.Primitive(float_value=500.0)), 500.0),
(literals.Scalar(primitive=literals.Primitive(boolean=True)), True),
(literals.Scalar(primitive=literals.Primitive(string_value='hello')),
'hello'),
(literals.Scalar(primitive=literals.Primitive(duration=timedelta(
seconds=5))), timedelta(seconds=5)),
(literals.Scalar(none_type=literals.Void()), None),
(literals.Scalar(blob=literals.Blob(
literals.BlobMetadata(
_core_types.BlobType(
"csv", _core_types.BlobType.BlobDimensionality.SINGLE)),
"s3://some/where")), _blob_impl.Blob("s3://some/where", format="csv")),
(literals.Scalar(blob=literals.Blob(
literals.BlobMetadata(
_core_types.BlobType(
"", _core_types.BlobType.BlobDimensionality.SINGLE)),
"s3://some/where")), _blob_impl.Blob("s3://some/where")),
(literals.Scalar(blob=literals.Blob(
literals.BlobMetadata(
_core_types.BlobType(
"csv", _core_types.BlobType.BlobDimensionality.MULTIPART)),
"s3://some/where/")),
_blob_impl.MultiPartBlob("s3://some/where/", format="csv")),
(literals.Scalar(blob=literals.Blob(
literals.BlobMetadata(
