def default_hparams():
"""Returns a dicitionary of default hyperparameters.
.. code-block:: python
{
# (1) Hyperparams specific to scalar dataset
"dataset": {
"files": [],
"compression_type": None,
"data_type": "int",
"other_transformations": [],
"data_name": None,
}
# (2) General hyperparams
"num_epochs": 1,
"batch_size": 64,
"allow_smaller_final_batch": True,
"shuffle": True,
"shuffle_buffer_size": None,
"shard_and_shuffle": False,
"num_parallel_calls": 1,
"prefetch_buffer_size": 0,
"max_dataset_size": -1,
"seed": None,
"name": "scalar_data",
}
Here:
1. For the hyperparameters in the :attr:`"dataset"` field:
"files": str or list
A (list of) file path(s).
Each line contains a single scalar number.
"compression_type": str, optional
One of "" (no compression), "ZLIB", or "GZIP".
"data_type": str
The scalar type. Currently supports "int" and "float".
"other_transformations": list
A list of transformation functions or function names/paths to
further transform each single data instance.
(More documentations to be added.)
"data_name": str
Name of the dataset.
2. For the **general** hyperparameters, see
:meth:`texar.tf.data.DataBase.default_hparams` for details.
"""
hparams = DataBase.default_hparams()
hparams["name"] = "scalar_data"
hparams.update({"dataset": _default_scalar_dataset_hparams()})
return hparams
def default_hparams():
"""Returns a dictionary of default hyperparameters.
See the specific subclasses for the details.
"""
hparams = DataBase.default_hparams()
hparams.update({
"bucket_boundaries": [],
"bucket_batch_sizes": None,
"bucket_length_fn": None})
return hparams
def default_hparams():
"""Returns a dicitionary of default hyperparameters.
.. code-block:: python
{
# (1) Hyperparams specific to TFRecord dataset
'dataset': {
'files': [],
'feature_original_types': {},
'feature_convert_types': {},
'image_options': {},
"num_shards": None,
"shard_id": None,
"other_transformations": [],
"data_name": None,
}
# (2) General hyperparams
"num_epochs": 1,
"batch_size": 64,
"allow_smaller_final_batch": True,
"shuffle": True,
"shuffle_buffer_size": None,
"shard_and_shuffle": False,
"num_parallel_calls": 1,
"prefetch_buffer_size": 0,
"max_dataset_size": -1,
"seed": None,
"name": "tfrecord_data",
}
Here:
1. For the hyperparameters in the :attr:`"dataset"` field:
"files": str or list
A (list of) TFRecord file path(s).
"feature_original_types": dict
The feature names (str) with their data types and length types,
key and value in pair
`feature_name: [dtype, feature_len_type, len]`,
- `dtype` is a :tf_main:`TF Dtype <dtypes/DType>` such as\
`tf.string` and `tf.int32`, or its string name such as \
'tf.string' and 'tf.int32'. The feature will be read from the\
files and parsed into this dtype.
- `feature_len_type` is of type `str`, and can be either \
'FixedLenFeature' or 'VarLenFeature' for fixed length \
features and non-fixed length features, respectively.
- `len` is an `int` and is optional. It is the length for \
'FixedLenFeature'. Ignored if 'VarLenFeature' is used.
Example:
.. code-block:: python
feature_original_types = {
"input_ids": ["tf.int64", "FixedLenFeature", 128],
"label_ids": ["tf.int64", "FixedLenFeature"],
"name_lists": ["tf.string", "VarLenFeature"],
}
"feature_convert_types": dict, optional
Specifies dtype converting after reading the data files. This
`dict` maps feature names to desired data dtypes. For example,
you can first read a feature into dtype `tf.float64` by
specifying in "feature_original_types" above, and convert
the feature to dtype "tf.int64" by specifying here.
Features not specified here will not do dtype-convert.
- `dtype` is a :tf_main:`TF Dtype <dtypes/DType>` such as\
`tf.string` and `tf.int32`, or its string name such as \
'tf.string' and 'tf.int32'.
Be noticed that this converting process is after all the data
are restored, `feature_original_types` has to be set firstly.
Example:
.. code-block:: python
feature_convert_types = {
"input_ids": "tf.int32",
"label_ids": "tf.int32",
}
"image_options": dict, optional
Specifies the image feature name and performs image resizing,
includes three fields:
- "image_feature_name":
A `str`, the name of the feature which contains
the image data. If set, the image data
will be restored in format `numpy.ndarray`.
- "resize_height":
A `int`, the height of the image after resizing.
- "resize_width":
A `int`, the width of the image after resizing
If either `resize_height` or `resize_width` is not set,
image data will be restored with original shape.
"num_shards": int, optional
The number of data shards in distributed mode. Usually set to
the number of processes in distributed computing.
Used in combination with :attr:`"shard_id"`.
"shard_id": int, optional
Sets the unique id to identify a shard. The module will
processes only the corresponding shard of the whole data.
Used in combination with :attr:`"num_shards"`.
E.g., in a case of distributed computing on 2 GPUs, the hparams
of the data module for the two processes can be as below,
respectively.
For gpu 0:
.. code-block:: python
dataset: {
...
"num_shards": 2,
"shard_id": 0
}
For gpu 1:
.. code-block:: python
dataset: {
...
"num_shards": 2,
"shard_id": 1
}
Also refer to `examples/bert` for a use case.
"other_transformations": list
A list of transformation functions or function names/paths to
further transform each single data instance.
"data_name": str
Name of the dataset.
2. For the **general** hyperparameters, see
:meth:`texar.tf.data.DataBase.default_hparams` for details.
"""
hparams = DataBase.default_hparams()
hparams["name"] = "tfrecord_data"
hparams.update({"dataset": _default_tfrecord_dataset_hparams()})
return hparams
def __init__(self, hparams):
DataBase.__init__(self, hparams)
with tf.name_scope(self.name, self.default_hparams()["name"]):
self._make_data()
def __init__(self, hparams):
DataBase.__init__(self, hparams)