def fit(self,
x=None,
y=None,
batch_size=None,
epochs=1,
validation_split=0.,
validation_data=None,
distributed=False,
**kwargs):
if isinstance(x, TFDataset):
# todo check arguments
if not x.has_batch:
raise ValueError("The batch_size of TFDataset must be " +
"specified when used in KerasModel fit.")
if isinstance(x, TFNdarrayDataset):
x = _standarize_feature_label_dataset(x, self.model)
self._fit_distributed(x, validation_split, epochs, **kwargs)
elif distributed:
dataset = TFDataset.from_ndarrays((x, y),
val_tensors=validation_data)
self._fit_distributed(dataset, validation_split, epochs, **kwargs)
else:
self.model.fit(x=x,
y=y,
batch_size=batch_size,
epochs=epochs,
validation_split=validation_split,
validation_data=validation_data,
**kwargs)
def test_tf_net_predict_dataset(self):
resource_path = os.path.join(os.path.split(__file__)[0], "../../resources")
tfnet_path = os.path.join(resource_path, "tfnet")
net = TFNet.from_export_folder(tfnet_path)
dataset = TFDataset.from_ndarrays((np.random.rand(16, 4),))
output = net.predict(dataset)
output = np.stack(output.collect())
assert output.shape == (16, 2)
def evaluate(self,
x=None,
y=None,
batch_per_thread=None,
distributed=False):
"""
Evaluate a model on a given dataset
:param x: Input data. It could be:
- a TFDataset object
- A Numpy array (or array-like), or a list of arrays
(in case the model has multiple inputs).
- A dict mapping input names to the corresponding array/tensors,
if the model has named inputs.
:param y: Target data. Like the input data `x`,
It should be consistent with `x` (you cannot have Numpy inputs and
tensor targets, or inversely). If `x` is a TFDataset, `y` should
not be specified (since targets will be obtained from `x`).
:param batch_per_thread:
The default value is 1.
When distributed is True,the total batch size is batch_per_thread * rdd.getNumPartitions.
When distributed is False the total batch size is batch_per_thread * numOfCores.
:param distributed: Boolean. Whether to do prediction in distributed mode or local mode.
Default is True. In local mode, x must be a Numpy array.
"""
if isinstance(x, TFDataset):
if not x.has_batch:
raise ValueError("The batch_per_thread of TFDataset must be " +
"specified when used in KerasModel evaluate.")
if isinstance(x, TFNdarrayDataset):
x = _standarize_feature_label_dataset(x, self.model)
# todo check arguments
return self._evaluate_distributed(x)
else:
if distributed:
dataset = TFDataset.from_ndarrays(
(x, y),
batch_per_thread=-1
if batch_per_thread is None else batch_per_thread)
return self._evaluate_distributed(dataset)
else:
return self.model.evaluate(x=x,
y=y,
batch_size=batch_per_thread)
def evaluate(self,
x=None,
y=None,
batch_per_thread=None,
distributed=False):
if isinstance(x, TFDataset):
if not x.has_batch:
raise ValueError("The batch_per_thread of TFDataset must be " +
"specified when used in KerasModel evaluate.")
if isinstance(x, TFNdarrayDataset):
x = _standarize_feature_label_dataset(x, self.model)
# todo check arguments
return self._evaluate_distributed(x)
else:
if distributed:
dataset = TFDataset.from_ndarrays(
(x, y),
batch_per_thread=-1
if batch_per_thread is None else batch_per_thread)
return self._evaluate_distributed(dataset)
else:
return self.model.evaluate(x=x,
y=y,
batch_size=batch_per_thread)
def fit(self,
x=None,
y=None,
batch_size=None,
epochs=1,
validation_split=0.,
validation_data=None,
distributed=False,
**kwargs):
"""
Train the model for a fixed num of epochs
Arguments:
:param x: Input data. It could be:
- a TFDataset object
- A Numpy array (or array-like), or a list of arrays
(in case the model has multiple inputs).
- A dict mapping input names to the corresponding array/tensors,
if the model has named inputs.
:param y: Target data. Like the input data `x`,
It should be consistent with `x` (you cannot have Numpy inputs and
tensor targets, or inversely). If `x` is a TFDataset, `y` should
not be specified (since targets will be obtained from `x`).
:param batch_size: Integer or `None`.
Number of samples per gradient update.
If `x` is a TFDataset, you do not need to specify batch_size.
:param epochs: Integer. Number of epochs to train the model.
An epoch is an iteration over the entire `x` and `y`
data provided.
:param validation_split: Float between 0 and 1.
Fraction of the training data to be used as validation data.
The model will set apart this fraction of the training data,
will not train on it, and will evaluate
the loss and any model metrics
on this data at the end of each epoch.
:param validation_data: Data on which to evaluate
the loss and any model metrics at the end of each epoch.
The model will not be trained on this data.
`validation_data` will override `validation_split`.
`validation_data` could be:
- tuple `(x_val, y_val)` of Numpy arrays or tensors
- `TFDataset`
:param distributed: Boolean. Whether to do prediction in distributed mode or local mode.
Default is True. In local mode, x must be a Numpy array.
"""
if isinstance(x, TFDataset):
# todo check arguments
if not x.has_batch:
raise ValueError("The batch_size of TFDataset must be " +
"specified when used in KerasModel fit.")
if isinstance(x, TFNdarrayDataset):
x = _standarize_feature_label_dataset(x, self.model)
self._fit_distributed(x, validation_split, epochs, **kwargs)
elif distributed:
dataset = TFDataset.from_ndarrays((x, y),
val_tensors=validation_data)
self._fit_distributed(dataset, validation_split, epochs, **kwargs)
else:
self.model.fit(x=x,
y=y,
batch_size=batch_size,
epochs=epochs,
validation_split=validation_split,
validation_data=validation_data,
**kwargs)