def __init__(self):
self.ffconfig = ff.FFConfig()
self.ffconfig.parse_args()
print("Python API batchSize(%d) workersPerNodes(%d) numNodes(%d)" %
(self.ffconfig.get_batch_size(),
self.ffconfig.get_workers_per_node(),
self.ffconfig.get_num_nodes()))
self.ffmodel = ff.FFModel(self.ffconfig)
self.ffoptimizer = 0
self._layers = dict()
self._nb_layers = 0
self.input_tensors = []
self.output_tensor = 0
self.label_tensor = 0
self.full_input_tensors = []
self.full_label_tensor = 0
self.num_samples = 0
self.input_dataloaders = []
self.input_dataloaders_dim = []
self.label_dataloader = 0
self.label_dataloader_dim = 0
def compile(self,
optimizer,
loss=None,
metrics=None,
loss_weights=None,
weighted_metrics=None,
run_eagerly=None,
comp_mode=ff.CompMode.TRAINING,
**kwargs):
if loss_weights != None:
assert 0, "loss_weights is not supported"
if weighted_metrics != None:
assert 0, "weighted_metrics is not supported"
if run_eagerly != None:
assert 0, "run_eagerly is not supported"
assert loss != None, "loss is None"
if isinstance(loss, keras_losses.Loss) == True:
self._loss = loss
elif loss == 'categorical_crossentropy':
self._loss = keras_losses.CategoricalCrossentropy()
elif loss == 'sparse_categorical_crossentropy':
self._loss = keras_losses.SparseCategoricalCrossentropy()
self._label_type = ff.DataType.DT_INT32
elif loss == 'mean_squared_error':
self._loss = keras_losses.MeanSquaredError()
else:
assert 0, 'Unsupported loss'
assert metrics != None, "metrics is None"
assert isinstance(metrics, list) == True, 'Metrics should be a list'
for metric in metrics:
if isinstance(metric, keras_metrics.Metric) == True:
self._metrics.append(metric)
elif metric == 'accuracy':
self._metrics.append(keras_metrics.Accuracy())
elif metric == 'categorical_crossentropy':
self._metrics.append(keras_metrics.CategoricalCrossentropy())
elif metric == 'sparse_categorical_crossentropy':
self._metrics.append(
keras_metrics.SparseCategoricalCrossentropy())
elif metric == 'mean_squared_error':
self._metrics.append(keras_metrics.MeanSquaredError())
elif metric == 'root_mean_squared_error':
self._metrics.append(keras_metrics.RootMeanSquaredError())
elif metric == 'mean_absolute_error':
self._metrics.append(keras_metrics.MeanAbsoluteError())
else:
assert 0, 'Unsupported metric'
self._ffmodel = ff.FFModel(self._ffconfig)
self._create_input_tensors()
self._create_flexflow_layers()
self._verify_output_tensors()
self._verify_input_tensors()
self._ffoptimizer = optimizer
self._create_optimizer()
metrics_type = []
for metric in self._metrics:
metrics_type.append(metric.type)
self._ffmodel.optimizer = optimizer.ffhandle
self._ffmodel.compile(loss_type=self._loss.type,
metrics=metrics_type,
comp_mode=comp_mode)
self._create_label_tensor()
fflogger.debug("%s, %s, %s, %s" %
(str(self._input_tensors[0]), str(self._output_tensor),
str(self._input_tensors[0].ffhandle),
str(self._output_tensor.ffhandle)))
def compile(self,
optimizer,
loss=None,
metrics=None,
loss_weights=None,
weighted_metrics=None,
run_eagerly=None,
comp_mode=None,
**kwargs):
if loss_weights != None:
assert 0, "loss_weights is not supported"
if weighted_metrics != None:
assert 0, "weighted_metrics is not supported"
if run_eagerly != None:
assert 0, "run_eagerly is not supported"
assert loss != None, "loss is None"
if loss == 'categorical_crossentropy':
self._loss = ff_keras_losses.CategoricalCrossentropy()
elif loss == 'sparse_categorical_crossentropy':
self._loss = ff_keras_losses.SparseCategoricalCrossentropy()
self._label_type = ff.DataType.DT_INT32
elif loss == 'mean_squared_error':
self._loss = ff_keras_losses.MeanSquaredError()
else:
assert 0, 'Unsupported loss'
assert metrics != None, "metrics is None"
assert isinstance(metrics, list) == True, 'Metrics should be a list'
for metric in metrics:
if metric == 'accuracy':
self._metrics.append(ff_keras_metrics.Accuracy())
elif metric == 'categorical_crossentropy':
self._metrics.append(
ff_keras_metrics.CategoricalCrossentropy())
elif metric == 'sparse_categorical_crossentropy':
self._metrics.append(
ff_keras_metrics.SparseCategoricalCrossentropy())
elif metric == 'mean_squared_error':
self._metrics.append(ff_keras_metrics.MeanSquaredError())
elif metric == 'root_mean_squared_error':
self._metrics.append(ff_keras_metrics.RootMeanSquaredError())
elif metric == 'mean_absolute_error':
self._metrics.append(ff_keras_metrics.MeanAbsoluteError())
else:
assert 0, 'Unsupported metric'
self._ffmodel = ff.FFModel(self._ffconfig)
self._create_input_tensors()
self._create_flexflow_layers()
layers = self._ffmodel.get_layers()
for l in layers:
print(l, layers[l])
if isinstance(optimizer, tf_keras_optimizer.Optimizer) == True:
if isinstance(optimizer, tf_keras_optimizer.SGD) == True:
self._ffoptimizer = ff_keras_optimizer.SGD(
learning_rate=optimizer.learning_rate.numpy(),
momentum=optimizer.momentum.numpy(),
nesterov=optimizer.nesterov)
elif isinstance(optimizer, tf_keras_optimizer.Adam) == True:
self._ffoptimizer = ff_keras_optimizer.Adam(
learning_rate=optimizer.learning_rate.numpy(),
beta_1=optimizer.beta_1.numpy(),
beta_2=optimizer.beta_2.numpy(),
epsilon=optimizer.epsilon.numpy())
else:
assert 0, "Unsupported optimizer"
elif type(optimizer) == str:
if optimizer == 'SGD':
self._ffoptimizer = ff_keras_optimizer.SGD()
elif optimizer == 'Adam':
self._ffoptimizer = ff_keras_optimizer.Adam()
else:
assert 0, "Unsupported optimizer"
else:
assert 0, "Unsupported optimizer"
self._create_optimizer()
metrics_type = []
for metric in self._metrics:
metrics_type.append(metric.type)
self._ffmodel.compile(optimizer=self._ffoptimizer.ffhandle,
loss_type=self._loss.type,
metrics=metrics_type,
comp_mode=comp_mode)
self._create_label_tensor()
def __init__(self): super(Module, self).__init__() self._ffconfig = ff.FFConfig() self._ffconfig.parse_args() self._ffmodel = ff.FFModel(self._ffconfig) self._graph = None
