def __init__(self, filename, schema, internal=False):
with tf.name_scope("AvroIOTensor") as scope:
metadata = ["schema: %s" % schema]
resource, columns = core_ops.io_avro_readable_init(
filename,
metadata=metadata,
container=scope,
shared_name=f"{filename}/{uuid.uuid4().hex}",
)
columns = [column.decode() for column in columns.numpy().tolist()]
elements = []
for column in columns:
shape, dtype = core_ops.io_avro_readable_spec(resource, column)
shape = tf.TensorShape(shape.numpy())
dtype = tf.as_dtype(dtype.numpy())
spec = tf.TensorSpec(shape, dtype, column)
function = io_tensor_ops._IOTensorComponentFunction( # pylint: disable=protected-access
core_ops.io_avro_readable_read, resource, column, shape,
dtype)
elements.append(
io_tensor_ops.BaseIOTensor(spec,
function,
internal=internal))
spec = tuple(e.spec for e in elements)
super().__init__(spec, columns, elements, internal=internal)
def __init__(self, filename, mode=None, internal=False):
with tf.name_scope("JSONIOTensor") as scope:
metadata = [] if mode is None else ["mode: %s" % mode]
resource, columns = core_ops.io_json_readable_init(
filename,
metadata=metadata,
container=scope,
shared_name="{}/{}".format(filename,
uuid.uuid4().hex),
)
columns = [column.decode() for column in columns.numpy().tolist()]
elements = []
for column in columns:
shape, dtype = core_ops.io_json_readable_spec(resource, column)
shape = tf.TensorShape(shape.numpy())
dtype = tf.as_dtype(dtype.numpy())
spec = tf.TensorSpec(shape, dtype, column)
function = io_tensor_ops._IOTensorComponentFunction( # pylint: disable=protected-access
core_ops.io_json_readable_read, resource, column, shape,
dtype)
elements.append(
io_tensor_ops.BaseIOTensor(spec,
function,
internal=internal))
spec = tuple([e.spec for e in elements])
super().__init__(spec, columns, elements, internal=internal)
def isnull(self, column):
"""Return a BaseIOTensor of bool for null values in `column`"""
column_index = self.columns.index(next(e for e in self.columns if e == column))
spec = tf.nest.flatten(self.spec)[column_index]
# change spec to bool
spec = tf.TensorSpec(spec.shape, tf.bool)
function = _IOTensorComponentLabelFunction(
core_ops.io_csv_readable_read,
self._resource,
column,
spec.shape,
spec.dtype,
)
return io_tensor_ops.BaseIOTensor(spec, function, internal=True)