def __init__(self, filename, internal=False): with tf.name_scope("FeatherIOTensor") as scope: resource, columns = core_ops.io_feather_readable_init( filename, container=scope, shared_name="{}/{}".format(filename, uuid.uuid4().hex), ) columns = [column.decode() for column in columns.numpy().tolist()] elements = [] for column in columns: shape, dtype = core_ops.io_feather_readable_spec( resource, column) shape = tf.TensorShape(shape.numpy()) dtype = tf.as_dtype(dtype.numpy()) spec = tf.TensorSpec(shape, dtype, column) function = io_tensor_ops._IOTensorComponentFunction( # pylint: disable=protected-access core_ops.io_feather_readable_read, resource, column, shape, dtype) elements.append( io_tensor_ops.BaseIOTensor(spec, function, internal=internal)) spec = tuple([e.spec for e in elements]) super().__init__(spec, columns, elements, internal=internal)
def __init__(self, filename, schema, internal=False): with tf.name_scope("AvroIOTensor") as scope: metadata = ["schema: %s" % schema] resource, columns = core_ops.avro_readable_init( filename, metadata=metadata, container=scope, shared_name="%s/%s" % (filename, uuid.uuid4().hex)) columns = [column.decode() for column in columns.numpy().tolist()] elements = [] for column in columns: shape, dtype = core_ops.avro_readable_spec(resource, column) shape = tf.TensorShape(shape.numpy()) dtype = tf.as_dtype(dtype.numpy()) spec = tf.TensorSpec(shape, dtype, column) function = io_tensor_ops._IOTensorComponentFunction( # pylint: disable=protected-access core_ops.avro_readable_read, resource, column, shape, dtype) elements.append( io_tensor_ops.BaseIOTensor(spec, function, internal=internal)) spec = tuple([e.spec for e in elements]) super(AvroIOTensor, self).__init__(spec, columns, elements, internal=internal)
def __init__(self, filename, internal=False): with tf.name_scope("HDF5IOTensor") as scope: # TODO: unique shared_name might be removed if HDF5 is thead-safe? resource, columns = core_ops.io_hdf5_readable_init( filename, container=scope, shared_name="%s/%s" % (filename, uuid.uuid4().hex)) columns = [column.decode() for column in columns.numpy().tolist()] elements = [] for column in columns: shape, dtype = core_ops.io_hdf5_readable_spec(resource, column) shape = tf.TensorShape(shape.numpy()) dtype = tf.as_dtype(dtype.numpy()) spec = tf.TensorSpec(shape, dtype, column) if shape.rank == 0: value = core_ops.io_hdf5_readable_read( resource, 0, shape, column, dtype) elements.append( io_tensor_ops.ScalarIOTensor(spec, value, internal=internal)) else: function = _HDF5IOTensorFunction( core_ops.io_hdf5_readable_read, resource, column, shape, dtype) elements.append( io_tensor_ops.BaseIOTensor(spec, function, internal=internal)) spec = tuple([e.spec for e in elements]) super(HDF5IOTensor, self).__init__(spec, columns, elements, internal=internal)
def isnull(self, column): """Return a BaseIOTensor of bool for null values in `column`""" column_index = self.columns.index( next(e for e in self.columns if e == column)) spec = tf.nest.flatten(self.spec)[column_index] # change spec to bool spec = tf.TensorSpec(spec.shape, tf.bool) class _Function(object): def __init__(self, func, spec, column): self._func = func self._shape = tf.TensorShape([None ]).concatenate(spec.shape[1:]) self._dtype = spec.dtype self._component = column def __call__(self, resource, start, stop): return self._func(resource, start=start, stop=stop, component=self._component, filter=['label'], shape=self._shape, dtype=self._dtype) return io_tensor_ops.BaseIOTensor(spec, self._resource, _Function( core_ops.csv_indexable_read, spec, column), partitions=None, internal=True)
def isnull(self, column): """Return a BaseIOTensor of bool for null values in `column`""" column_index = self.columns.index( next(e for e in self.columns if e == column)) spec = tf.nest.flatten(self.spec)[column_index] # change spec to bool spec = tf.TensorSpec(spec.shape, tf.bool) function = _IOTensorComponentLabelFunction(core_ops.csv_readable_read, self._resource, column, spec.shape, spec.dtype) return io_tensor_ops.BaseIOTensor(spec, function, internal=True)
def __init__(self, query, endpoint=None, internal=False): with tf.name_scope("PrometheusIOTensor") as scope: metadata = [] if endpoint is None else ["endpoint: %s" % endpoint] resource = core_golang_ops.io_prometheus_readable_init( query, metadata=metadata, container=scope, shared_name="%s/%s" % (query, uuid.uuid4().hex)) index_shape, index_dtype = core_golang_ops.io_prometheus_readable_spec( resource, "index") value_shape, value_dtype = core_golang_ops.io_prometheus_readable_spec( resource, "value") index_shape = tf.TensorShape(index_shape.numpy()) index_dtype = tf.as_dtype(index_dtype.numpy()) value_shape = tf.TensorShape(value_shape.numpy()) value_dtype = tf.as_dtype(value_dtype.numpy()) spec = tuple([ tf.TensorSpec(index_shape, index_dtype), tf.TensorSpec(value_shape, value_dtype) ]) index = io_tensor_ops.BaseIOTensor( spec[0], _PrometheusIOTensorFunction( core_golang_ops.io_prometheus_readable_read, resource, "index", index_shape, index_dtype), internal=internal) value = io_tensor_ops.BaseIOTensor( spec[1], _PrometheusIOTensorFunction( core_golang_ops.io_prometheus_readable_read, resource, "value", value_shape, value_dtype), internal=internal) super(PrometheusIOTensor, self).__init__(spec, index, value, internal=internal)
def __init__(self, table, internal=False): with tf.name_scope("ArrowIOTensor") as scope: # Hold reference to table and schema buffer for life of this op self._table = table self._schema_buffer = table.schema.serialize() # Get buffer addresses as long ints schema_buffer_addr = self._schema_buffer.address schema_buffer_size = self._schema_buffer.size array_buffer_addrs, array_buffer_sizes, array_lengths = \ _extract_table_arrays(table) # Create the Arrow readable resource resource = core_ops.io_arrow_readable_from_memory_init( schema_buffer_addr, schema_buffer_size, array_buffer_addrs, array_buffer_sizes, array_lengths, container=scope, shared_name="pyarrow.Table%s/%s" % (table.schema.names, uuid.uuid4().hex)) # Create a BaseIOTensor for each column elements = [] columns = table.column_names for column_index, column in enumerate(columns): shape, dtype = core_ops.io_arrow_readable_spec( resource, column_index) shape = tf.TensorShape(shape.numpy()) dtype = tf.as_dtype(dtype.numpy()) spec = tf.TensorSpec(shape, dtype, column) function = _ArrowIOTensorComponentFunction( # pylint: disable=protected-access core_ops.io_arrow_readable_read, resource, column, column_index, shape, dtype) elements.append( io_tensor_ops.BaseIOTensor(spec, function, internal=internal)) spec = tuple([e.spec for e in elements]) super(ArrowIOTensor, self).__init__(spec, columns, elements, internal=internal)
def __init__(self, filename, internal=False): with tf.name_scope("HDF5IOTensor") as scope: class _Function(object): def __init__(self, func, spec, key): self._func = func self._shape = tf.TensorShape([None]).concatenate( spec.shape[1:]) self._dtype = spec.dtype self._component = key def __call__(self, resource, start, stop): return self._func(resource, start=start, stop=stop, component=self._component, shape=self._shape, dtype=self._dtype) resource, columns = core_ops.hdf5_indexable_init( filename, container=scope, shared_name="%s/%s" % (filename, uuid.uuid4().hex)) columns = [column.decode() for column in columns.numpy().tolist()] elements = [] for column in columns: shape, dtype = core_ops.hdf5_indexable_spec(resource, column) shape = tf.TensorShape(shape.numpy()) dtype = tf.as_dtype(dtype.numpy()) spec = tf.TensorSpec(shape, dtype, column) elements.append( io_tensor_ops.BaseIOTensor( spec, resource, _Function(core_ops.hdf5_indexable_read, spec, column), partitions=None, internal=True)) spec = tuple([e.spec for e in elements]) super(HDF5IOTensor, self).__init__(spec, columns, elements, internal=internal)
def __init__(self, filename, internal=False): with tf.name_scope("FFmpegIOTensor") as scope: from tensorflow_io.core.python.ops import ffmpeg_ops resource, columns = ffmpeg_ops.ffmpeg_indexable_init( filename, container=scope, shared_name="%s/%s" % (filename, uuid.uuid4().hex)) columns = [column.decode() for column in columns.numpy().tolist()] elements = [] for column in columns: shape, dtype, rate = ffmpeg_ops.ffmpeg_indexable_spec( resource, column) shape = tf.TensorShape( [None if e < 0 else e for e in shape.numpy()]) dtype = tf.as_dtype(dtype.numpy()) spec = tf.TensorSpec(shape, dtype, column) if column.startswith("a:"): rate = rate.numpy() elements.append( audio_io_tensor_ops.AudioIOTensor(rate, spec, resource, None, partitions=None, internal=internal)) else: elements.append( io_tensor_ops.BaseIOTensor(spec, resource, None, partitions=None, internal=internal)) spec = tuple([e.spec for e in elements]) super(FFmpegIOTensor, self).__init__(spec, columns, elements, internal=internal)