def load_dependency_and_library(p):
for library in p:
# First try load all dependencies with RTLD_LOCAL
entries = []
for dependency in p[library]:
try:
entries.append(ctypes.CDLL(dependency))
except OSError as e:
pass
if len(entries) == len(p[library]):
# Dependencies has been satisfied, load dependencies again with RTLD_GLOBAL, no error is expected
for dependency in p[library]:
ctypes.CDLL(dependency, mode=ctypes.RTLD_GLOBAL)
# Load video_op
return _load_library(library)
# Otherwise we dlclose and retry
entries.reverse()
for entry in entries:
_ctypes.dlclose(entry._handle)
raise NotImplementedError("could not find ffmpeg after search through ", p)
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""SequenceFile Dataset."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import tensorflow
from tensorflow import dtypes
from tensorflow.compat.v1 import data
from tensorflow_io import _load_library
hadoop_ops = _load_library('_hadoop_ops.so')
class SequenceFileDataset(data.Dataset):
"""A Sequence File Dataset that reads the sequence file."""
def __init__(self, filenames):
"""Create a `SequenceFileDataset`.
`SequenceFileDataset` allows a user to read data from a hadoop sequence
file. A sequence file consists of (key value) pairs sequentially. At
the moment, `org.apache.hadoop.io.Text` is the only serialization type
being supported, and there is no compression support.
For example:
```python
For background on Cloud BigQuery, see: https://cloud.google.com/bigquery .
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from tensorflow.python.data.experimental.ops import interleave_ops
from tensorflow.python.data.ops import dataset_ops
from tensorflow.python.data.util import structure
from tensorflow.python.framework import dtypes
from tensorflow_io import _load_library
_bigquery_so = _load_library("_bigquery.so")
class BigQueryClient(object):
"""BigQueryClient is the entrypoint for interacting with Cloud BigQuery in TF.
BigQueryClient encapsulates a connection to Cloud BigQuery, and exposes the
`readSession` method to initiate a BigQuery read session.
"""
def __init__(self, client_resource=None):
"""Creates a BigQueryClient to start BigQuery read sessions.
Args:
client_resource: client resource (optional).
"""
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""OSS File System Support
OSS is an Object Storage Service provided by Alibaba Cloud. This module
implements a filesystem on top of it.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from tensorflow_io import _load_library
_load_library("_oss_ops.so")
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""CIFAR Dataset."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow
from tensorflow import dtypes
from tensorflow.compat.v1 import data
from tensorflow_io import _load_library
cifar_ops = _load_library('_cifar_ops.so')
class _CIFAR10Dataset(data.Dataset):
"""A CIFAR File Dataset that reads the cifar file."""
def __init__(self, filename, filters, batch=None):
"""Create a `CIFARDataset`.
Args:
filename: A `tf.string` tensor containing one or more filenames.
"""
self._data_input = cifar_ops.cifar10_input(filename, filters)
self._batch = 0 if batch is None else batch
super(_CIFAR10Dataset, self).__init__()
def _inputs(self):
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""LMDBDataset."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow
from tensorflow import dtypes
from tensorflow.compat.v1 import data
from tensorflow_io import _load_library
lmdb_ops = _load_library('_lmdb_ops.so')
class LMDBDataset(data.Dataset):
"""A LMDB Dataset that reads the lmdb file."""
def __init__(self, filenames):
"""Create a `LMDBDataset`.
`LMDBDataset` allows a user to read data from a mdb file as
(key value) pairs sequentially.
For example:
```python
tf.enable_eager_execution()
dataset = LMDBDataset("/foo/bar.mdb")
# Prints the (key, value) pairs inside a lmdb file.
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Parquet Dataset."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow
from tensorflow import dtypes
from tensorflow.compat.v1 import data
from tensorflow_io import _load_library
parquet_ops = _load_library('_parquet_ops.so')
if hasattr(tensorflow, "nest"):
from tensorflow import nest # pylint: disable=ungrouped-imports
else:
from tensorflow.python.data.util import nest # pylint: disable=ungrouped-imports
class ParquetDataset(data.Dataset):
"""A Parquet Dataset that reads the parquet file."""
def __init__(self, filenames, columns, output_types):
"""Create a `ParquetDataset`.
`ParquetDataset` allows a user to read data from a parquet file.
For example:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from six import iteritems
from six import string_types
from tensorflow.python.data.experimental.ops import interleave_ops
from tensorflow.python.data.ops import dataset_ops
from tensorflow.python.data.util import nest
from tensorflow.python.data.util import structure
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import tensor_shape
from tensorflow_io import _load_library
_bigtable_so = _load_library("_bigtable.so")
class BigtableClient(object): # pylint: disable=useless-object-inheritance
"""BigtableClient is the entrypoint for interacting with Cloud Bigtable in TF.
BigtableClient encapsulates a connection to Cloud Bigtable, and exposes the
`table` method to open a Bigtable table.
"""
def __init__(self,
project_id,
instance_id,
connection_pool_size=None,
max_receive_message_size=None):
"""Creates a BigtableClient that can be used to open connections to tables.
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""TextInput/TextOutput."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow
from tensorflow import dtypes
from tensorflow.compat.v1 import data
from tensorflow_io import _load_library
text_ops = _load_library('_text_ops.so')
class TextDataset(data.Dataset):
"""A Text Dataset
"""
def __init__(self, filename):
"""Create a Text Reader.
Args:
filename: A `tf.string` tensor containing one or more filenames.
"""
self._data_input = text_ops.text_input(filename, ["none", "gz"])
super(TextDataset, self).__init__()
def _inputs(self):
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""BigQuery reading support for TensorFlow."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from tensorflow.python.framework import ops
from tensorflow.python.ops import io_ops
from tensorflow_io import _load_library
_bigquery_reader_so = _load_library("_bigquery_reader_ops.so")
class BigQueryReader(io_ops.ReaderBase):
"""A Reader that outputs keys and tf.Example values from a BigQuery table.
Example use:
```python
# Assume a BigQuery has the following schema,
# name STRING,
# age INT,
# state STRING
# Create the parse_examples list of features.
features = dict(
name=tf.FixedLenFeature([1], tf.string),
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""PubSub Dataset."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow as tf
from tensorflow import dtypes
from tensorflow.compat.v1 import data
from tensorflow_io import _load_library
pubsub_ops = _load_library('_pubsub_ops.so')
class PubSubDataset(data.Dataset):
"""A PubSub Dataset that consumes the message.
"""
def __init__(self, subscriptions, server=None, eof=False, timeout=1000):
"""Create a PubSubDataset.
Args:
subscriptions: A `tf.string` tensor containing one or more subscriptions.
server: The pubsub server.
eof: If True, the pubsub reader will stop on EOF.
timeout: The timeout value for the PubSub to wait
(in millisecond).
"""
"""LibSVM Dataset."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from tensorflow import sparse
from tensorflow_io import _load_library
gen_libsvm_ops = _load_library('_libsvm_ops.so')
def decode_libsvm(content, num_features, dtype=None, label_dtype=None):
"""Convert Libsvm records to a tensor of label and a tensor of feature.
Args:
content: A `Tensor` of type `string`. Each string is a record/row in
the Libsvm format.
num_features: The number of features.
dtype: The type of the output feature tensor. Default to tf.float32.
label_dtype: The type of the output label tensor. Default to tf.int64.
Returns:
features: A `SparseTensor` of the shape `[input_shape, num_features]`.
labels: A `Tensor` of the same shape as content.
"""
labels, indices, values, shape = gen_libsvm_ops.decode_libsvm(
content, num_features, dtype=dtype, label_dtype=label_dtype)
return sparse.SparseTensor(indices, values, shape), labels
def make_libsvm_dataset(file_names,
num_features,
dtype=None,
label_dtype=None,
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Ignite File System for checkpointing and communication with TensorBoard.
Apache Ignite is a memory-centric distributed database, caching, and
processing platform for transactional, analytical, and streaming workloads,
delivering in-memory speeds at petabyte scale. In addition to database
functionality Apache Ignite provides a distributed file system called
IGFS (https://ignite.apache.org/features/igfs.html). IGFS delivers a similar
functionality to Hadoop HDFS, but only in-memory. In fact, in addition to
its own APIs, IGFS implements Hadoop FileSystem API and can be transparently
plugged into Hadoop or Spark deployments. This contrib package contains an
integration between IGFS and TensorFlow.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from tensorflow_io import _load_library
_load_library("_ignite_ops.so", "file_system")
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Audio Dataset."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow
from tensorflow import dtypes
from tensorflow.compat.v1 import data
from tensorflow_io import _load_library
audio_ops = _load_library('_audio_ops.so')
class WAVDataset(data.Dataset):
"""A WAV Dataset
"""
def __init__(self, filenames, batch=None):
"""Create a WAVDataset.
Args:
filenames: A `tf.string` tensor containing one or more filenames.
"""
self._data_input = audio_ops.wav_input(filenames)
self._batch = 0 if batch is None else batch
super(WAVDataset, self).__init__()
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Image Dataset."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow
from tensorflow import dtypes
from tensorflow.compat.v1 import data
from tensorflow_io import _load_library
image_ops = _load_library('_image_ops.so')
class WebPDataset(data.Dataset):
"""A WebP Image File Dataset that reads the WebP file."""
def __init__(self, filenames):
"""Create a `WebPDataset`.
filenames: A `tf.string` tensor containing one or more filenames.
"""
self._filenames = tensorflow.convert_to_tensor(filenames,
dtype=dtypes.string,
name="filenames")
super(WebPDataset, self).__init__()
def _inputs(self):
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""MNIST Dataset."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow
from tensorflow import dtypes
from tensorflow.compat.v1 import data
from tensorflow_io import _load_library
mnist_ops = _load_library('_mnist_ops.so')
class _MNISTBaseDataset(data.Dataset):
"""A MNIST Dataset
"""
def __init__(self, mnist_op_class, filenames, compression_type=None):
"""Create a MNISTReader.
Args:
mnist_op_class: The op of the dataset, either
mnist_ops.mnist_image_dataset or mnist_ops.mnist_label_dataset.
filenames: A `tf.string` tensor containing one or more filenames.
compression_type: (Optional.) A `tf.string` scalar evaluating to one of
`""` (no compression), `"ZLIB"`, or `"GZIP"`.
"""
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""GRPCInput."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow as tf
from tensorflow.compat.v1 import data
from tensorflow_io import _load_library
grpc_ops = _load_library('_grpc_ops.so')
class GRPCDataset(data.Dataset):
"""A GRPC Dataset
"""
def __init__(self, endpoint, shape, dtype, batch=None):
"""Create a GRPC Reader.
Args:
endpoint: A `tf.string` tensor containing one or more endpoints.
"""
self._data_input = grpc_ops.grpc_input(endpoint)
self._batch = 0 if batch is None else batch
shape[0] = None
self._output_shapes = tuple([
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""PcapDataset"""
import tensorflow as tf
from tensorflow_io.core.python.ops import data_ops as data_ops
from tensorflow_io import _load_library
pcap_ops = _load_library('_pcap_ops.so')
class PcapDataset(data_ops.Dataset):
"""A pcap Dataset. Pcap is a popular file format for capturing network packets.
"""
def __init__(self, filenames, batch=None):
"""Create a pcap Reader.
Args:
filenames: A `tf.string` tensor containing one or more filenames.
"""
batch = 0 if batch is None else batch
dtypes = [tf.float64, tf.string]
shapes = [
"""Video Dataset."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import tensorflow
from tensorflow import dtypes
from tensorflow import errors
from tensorflow.compat.v1 import data
from tensorflow_io import _load_library
video_ops = None
for f in ['_video_ops_ffmpeg_3.4.so', '_video_ops_ffmpeg_2.8.so', '_video_ops_libav_9.20.so']:
try:
video_ops = _load_library(f)
break;
except NotImplementedError as e:
print(e)
class VideoDataset(data.Dataset):
"""A Video File Dataset that reads the video file."""
def __init__(self, filenames):
"""Create a `VideoDataset`.
`VideoDataset` allows a user to read data from a video file with
ffmpeg. The output of VideoDataset is a sequence of (height, weight, 3)
tensor in rgb24 format.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Arrow Dataset."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import io
import tensorflow
from tensorflow import dtypes
from tensorflow.compat.v1 import data
from tensorflow_io import _load_library
arrow_ops = _load_library('_arrow_ops.so')
if hasattr(tensorflow, "nest"):
from tensorflow import nest # pylint: disable=ungrouped-imports
else:
from tensorflow.python.data.util import nest # pylint: disable=ungrouped-imports
def arrow_to_tensor_type(pa_t):
"""Convert Arrow type to tuple of (Tensor dtype, shape dims).
This function requires pyarrow to be installed.
"""
import pyarrow as pa
shape_dims = [] # initialize shape as scalar
if pa.types.is_boolean(pa_t):
tf_t = dtypes.bool
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Kinesis Dataset."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import tensorflow
from tensorflow import dtypes
from tensorflow.compat.v1 import data
from tensorflow_io import _load_library
kinesis_ops = _load_library('_kinesis_ops.so')
class KinesisDataset(data.Dataset):
"""A Kinesis Dataset that consumes the message.
Kinesis is a managed service provided by AWS for data streaming.
This dataset reads messages from Kinesis with each message presented
as a `tf.string`.
For example, we can construct and use the KinesisDataset as follows:
```python
dataset = KinesisDataset(
"kinesis_stream_name", read_indefinitely=False)
next = dataset.make_one_shot_iterator().get_next()
with tf.Session() as sess:
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Dataset."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from tensorflow_io import _load_library
core_ops = _load_library('libtensorflowio.so')
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""KafkaOutputSequence."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from tensorflow_io import _load_library
kafka_ops = _load_library('_kafka_ops.so')
class KafkaOutputSequence(object):
"""KafkaOutputSequence"""
def __init__(self, topic, servers="localhost"):
"""Create a `KafkaOutputSequence`.
"""
self._topic = topic
self._resource = kafka_ops.kafka_output_sequence(
topic=topic, servers=servers)
def setitem(self, index, item):
kafka_ops.kafka_output_sequence_set_item(self._resource, index, item)
from __future__ import division
from __future__ import print_function
import abc
import socket
import ssl
import struct
import six
import tensorflow as tf
from tensorflow import dtypes
from tensorflow.compat.v1 import data
from tensorflow_io import _load_library
ignite_ops = _load_library("_ignite_ops.so")
@six.add_metaclass(abc.ABCMeta) # pylint: disable=useless-object-inheritance
class Readable(object):
"""Readable abstract class that exposes methods to do reading-related
operations.
"""
@abc.abstractmethod
def __init__(self):
pass
def read_byte(self):
"""Reads and returnes byte."""
return self._read("b", 1)
import tensorflow as tf
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.training import training
from tensorflow_io import _load_library
# Some GCS operations may be pre-defined and available via tf.contrib in
# earlier TF versions. Because these ops are pre-registered, they will not be
# visible from the _gcs_config_ops library. In this case we use the tf.contrib
# version instead.
tf_v1 = tf.version.VERSION.startswith('1')
if not tf_v1:
_gcs_config_so = _load_library("_gcs_config_ops.so")
gcs_configure_credentials = _gcs_config_so.gcs_configure_credentials
gcs_configure_block_cache = _gcs_config_so.gcs_configure_block_cache
class BlockCacheParams(object): # pylint: disable=useless-object-inheritance
"""BlockCacheParams is a struct used for configuring the GCS Block Cache."""
def __init__(self, block_size=None, max_bytes=None, max_staleness=None):
self._block_size = block_size or 128 * 1024 * 1024
self._max_bytes = max_bytes or 2 * self._block_size
self._max_staleness = max_staleness or 0
@property
def block_size(self):
return self._block_size
