def __init__(self, filename, internal=False):
with tf.name_scope("FeatherIOTensor") as scope:
resource, columns = core_ops.io_feather_readable_init(
filename,
container=scope,
shared_name="{}/{}".format(filename,
uuid.uuid4().hex),
)
columns = [column.decode() for column in columns.numpy().tolist()]
elements = []
for column in columns:
shape, dtype = core_ops.io_feather_readable_spec(
resource, column)
shape = tf.TensorShape(shape.numpy())
dtype = tf.as_dtype(dtype.numpy())
spec = tf.TensorSpec(shape, dtype, column)
function = io_tensor_ops._IOTensorComponentFunction( # pylint: disable=protected-access
core_ops.io_feather_readable_read, resource, column, shape,
dtype)
elements.append(
io_tensor_ops.BaseIOTensor(spec,
function,
internal=internal))
spec = tuple([e.spec for e in elements])
super().__init__(spec, columns, elements, internal=internal)
def __init__(self, filename, schema, internal=False):
with tf.name_scope("AvroIOTensor") as scope:
metadata = ["schema: %s" % schema]
resource, columns = core_ops.avro_readable_init(
filename,
metadata=metadata,
container=scope,
shared_name="%s/%s" % (filename, uuid.uuid4().hex))
columns = [column.decode() for column in columns.numpy().tolist()]
elements = []
for column in columns:
shape, dtype = core_ops.avro_readable_spec(resource, column)
shape = tf.TensorShape(shape.numpy())
dtype = tf.as_dtype(dtype.numpy())
spec = tf.TensorSpec(shape, dtype, column)
function = io_tensor_ops._IOTensorComponentFunction( # pylint: disable=protected-access
core_ops.avro_readable_read, resource, column, shape,
dtype)
elements.append(
io_tensor_ops.BaseIOTensor(spec,
function,
internal=internal))
spec = tuple([e.spec for e in elements])
super(AvroIOTensor, self).__init__(spec,
columns,
elements,
internal=internal)
def __init__(self, filename, internal=False):
with tf.name_scope("HDF5IOTensor") as scope:
# TODO: unique shared_name might be removed if HDF5 is thead-safe?
resource, columns = core_ops.io_hdf5_readable_init(
filename,
container=scope,
shared_name="%s/%s" % (filename, uuid.uuid4().hex))
columns = [column.decode() for column in columns.numpy().tolist()]
elements = []
for column in columns:
shape, dtype = core_ops.io_hdf5_readable_spec(resource, column)
shape = tf.TensorShape(shape.numpy())
dtype = tf.as_dtype(dtype.numpy())
spec = tf.TensorSpec(shape, dtype, column)
if shape.rank == 0:
value = core_ops.io_hdf5_readable_read(
resource, 0, shape, column, dtype)
elements.append(
io_tensor_ops.ScalarIOTensor(spec,
value,
internal=internal))
else:
function = _HDF5IOTensorFunction(
core_ops.io_hdf5_readable_read, resource, column,
shape, dtype)
elements.append(
io_tensor_ops.BaseIOTensor(spec,
function,
internal=internal))
spec = tuple([e.spec for e in elements])
super(HDF5IOTensor, self).__init__(spec,
columns,
elements,
internal=internal)
def isnull(self, column):
"""Return a BaseIOTensor of bool for null values in `column`"""
column_index = self.columns.index(
next(e for e in self.columns if e == column))
spec = tf.nest.flatten(self.spec)[column_index]
# change spec to bool
spec = tf.TensorSpec(spec.shape, tf.bool)
class _Function(object):
def __init__(self, func, spec, column):
self._func = func
self._shape = tf.TensorShape([None
]).concatenate(spec.shape[1:])
self._dtype = spec.dtype
self._component = column
def __call__(self, resource, start, stop):
return self._func(resource,
start=start,
stop=stop,
component=self._component,
filter=['label'],
shape=self._shape,
dtype=self._dtype)
return io_tensor_ops.BaseIOTensor(spec,
self._resource,
_Function(
core_ops.csv_indexable_read,
spec, column),
partitions=None,
internal=True)
def isnull(self, column):
"""Return a BaseIOTensor of bool for null values in `column`"""
column_index = self.columns.index(
next(e for e in self.columns if e == column))
spec = tf.nest.flatten(self.spec)[column_index]
# change spec to bool
spec = tf.TensorSpec(spec.shape, tf.bool)
function = _IOTensorComponentLabelFunction(core_ops.csv_readable_read,
self._resource, column,
spec.shape, spec.dtype)
return io_tensor_ops.BaseIOTensor(spec, function, internal=True)
def __init__(self, query, endpoint=None, internal=False):
with tf.name_scope("PrometheusIOTensor") as scope:
metadata = [] if endpoint is None else ["endpoint: %s" % endpoint]
resource = core_golang_ops.io_prometheus_readable_init(
query,
metadata=metadata,
container=scope,
shared_name="%s/%s" % (query, uuid.uuid4().hex))
index_shape, index_dtype = core_golang_ops.io_prometheus_readable_spec(
resource, "index")
value_shape, value_dtype = core_golang_ops.io_prometheus_readable_spec(
resource, "value")
index_shape = tf.TensorShape(index_shape.numpy())
index_dtype = tf.as_dtype(index_dtype.numpy())
value_shape = tf.TensorShape(value_shape.numpy())
value_dtype = tf.as_dtype(value_dtype.numpy())
spec = tuple([
tf.TensorSpec(index_shape, index_dtype),
tf.TensorSpec(value_shape, value_dtype)
])
index = io_tensor_ops.BaseIOTensor(
spec[0],
_PrometheusIOTensorFunction(
core_golang_ops.io_prometheus_readable_read, resource,
"index", index_shape, index_dtype),
internal=internal)
value = io_tensor_ops.BaseIOTensor(
spec[1],
_PrometheusIOTensorFunction(
core_golang_ops.io_prometheus_readable_read, resource,
"value", value_shape, value_dtype),
internal=internal)
super(PrometheusIOTensor, self).__init__(spec,
index,
value,
internal=internal)
def __init__(self, table, internal=False):
with tf.name_scope("ArrowIOTensor") as scope:
# Hold reference to table and schema buffer for life of this op
self._table = table
self._schema_buffer = table.schema.serialize()
# Get buffer addresses as long ints
schema_buffer_addr = self._schema_buffer.address
schema_buffer_size = self._schema_buffer.size
array_buffer_addrs, array_buffer_sizes, array_lengths = \
_extract_table_arrays(table)
# Create the Arrow readable resource
resource = core_ops.io_arrow_readable_from_memory_init(
schema_buffer_addr,
schema_buffer_size,
array_buffer_addrs,
array_buffer_sizes,
array_lengths,
container=scope,
shared_name="pyarrow.Table%s/%s" %
(table.schema.names, uuid.uuid4().hex))
# Create a BaseIOTensor for each column
elements = []
columns = table.column_names
for column_index, column in enumerate(columns):
shape, dtype = core_ops.io_arrow_readable_spec(
resource, column_index)
shape = tf.TensorShape(shape.numpy())
dtype = tf.as_dtype(dtype.numpy())
spec = tf.TensorSpec(shape, dtype, column)
function = _ArrowIOTensorComponentFunction( # pylint: disable=protected-access
core_ops.io_arrow_readable_read, resource, column,
column_index, shape, dtype)
elements.append(
io_tensor_ops.BaseIOTensor(spec,
function,
internal=internal))
spec = tuple([e.spec for e in elements])
super(ArrowIOTensor, self).__init__(spec,
columns,
elements,
internal=internal)
def __init__(self, filename, internal=False):
with tf.name_scope("HDF5IOTensor") as scope:
class _Function(object):
def __init__(self, func, spec, key):
self._func = func
self._shape = tf.TensorShape([None]).concatenate(
spec.shape[1:])
self._dtype = spec.dtype
self._component = key
def __call__(self, resource, start, stop):
return self._func(resource,
start=start,
stop=stop,
component=self._component,
shape=self._shape,
dtype=self._dtype)
resource, columns = core_ops.hdf5_indexable_init(
filename,
container=scope,
shared_name="%s/%s" % (filename, uuid.uuid4().hex))
columns = [column.decode() for column in columns.numpy().tolist()]
elements = []
for column in columns:
shape, dtype = core_ops.hdf5_indexable_spec(resource, column)
shape = tf.TensorShape(shape.numpy())
dtype = tf.as_dtype(dtype.numpy())
spec = tf.TensorSpec(shape, dtype, column)
elements.append(
io_tensor_ops.BaseIOTensor(
spec,
resource,
_Function(core_ops.hdf5_indexable_read, spec, column),
partitions=None,
internal=True))
spec = tuple([e.spec for e in elements])
super(HDF5IOTensor, self).__init__(spec,
columns,
elements,
internal=internal)
def __init__(self, filename, internal=False):
with tf.name_scope("FFmpegIOTensor") as scope:
from tensorflow_io.core.python.ops import ffmpeg_ops
resource, columns = ffmpeg_ops.ffmpeg_indexable_init(
filename,
container=scope,
shared_name="%s/%s" % (filename, uuid.uuid4().hex))
columns = [column.decode() for column in columns.numpy().tolist()]
elements = []
for column in columns:
shape, dtype, rate = ffmpeg_ops.ffmpeg_indexable_spec(
resource, column)
shape = tf.TensorShape(
[None if e < 0 else e for e in shape.numpy()])
dtype = tf.as_dtype(dtype.numpy())
spec = tf.TensorSpec(shape, dtype, column)
if column.startswith("a:"):
rate = rate.numpy()
elements.append(
audio_io_tensor_ops.AudioIOTensor(rate,
spec,
resource,
None,
partitions=None,
internal=internal))
else:
elements.append(
io_tensor_ops.BaseIOTensor(spec,
resource,
None,
partitions=None,
internal=internal))
spec = tuple([e.spec for e in elements])
super(FFmpegIOTensor, self).__init__(spec,
columns,
elements,
internal=internal)
