def compile(self,
optimizer=None,
loss=None,
metrics=None,
params=None,
train_hooks=None,
val_hooks=None,
checkpoint_dir=None,
**kwargs):
self.optimizer = optimizer
self.loss = loss
self.metrics = metrics
self.params = params
if train_hooks is not None:
self.train_hooks.extend(to_list(train_hooks))
if val_hooks is not None:
self.val_hooks.extend(to_list(val_hooks))
self._checkpoint_dir = checkpoint_dir
self._session_kwargs = kwargs
if not self.built:
logging.info("=>Model function was not built, fully compile will"
" delay after first call(after fit|evaluate|predict)")
return
self._is_compiled = True
self._compile_args_with_mode()
if self._mode != ExecutorMode.PREDICT:
self._compile_metrics(metrics)
self._compile_summary()
self._compile_environment(EnvironmentConfig(**self._session_kwargs))
def __call__(self, *args, **kwargs):
with ops.name_scope(self.name):
updates = to_list(self.update_state(*args, **kwargs))
with fops.control_dependencies(updates):
result = self.result()
# We are adding the metric object as metadata on every result tensor.
# This metric instance will later be used to reset variable state after
# each epoch of training.
for res in to_list(nest.flatten(result)):
setattr(res, '_metric_obj', self)
return result
def forward(self, inputs, initial_state=None):
if isinstance(inputs, list):
initial_state = inputs[1:]
inputs = inputs[0]
elif self.stateful:
initial_state = self.states
else:
initial_state = self.get_initial_state(inputs)
if len(initial_state) != len(self.states):
raise ValueError("Expect %d states, but received %d" %
(len(self.states), len(initial_state)))
time_steps = engine.int_shape(inputs)[1]
if self.unroll and time_steps in [None, 1]:
raise ValueError("To unroll a RNN, time dimension in inputs"
" must be known, and must larger than 1")
last_output, outputs, states = F.rnn(step_fn=self.cell.forward,
inputs=inputs,
initial_states=initial_state,
go_backwards=self.go_backwards,
unroll=self.unroll,
input_length=time_steps)
if self.stateful:
with fops.control_dependencies(self.states):
updates = [
state_ops.assign(self.states[i], states[i])
for i in range(len(self.states))
]
fops.add_to_collection(fops.GraphKeys.UPDATE_OPS, updates)
if not self.return_sequence:
outputs = last_output
if self.return_state:
return [outputs] + to_list(states)
else:
return outputs
def values(self, ignore_params=None):
params = self._flatten()
if ignore_params is not None:
ignore_params = to_list(ignore_params)
if any([not isinstance(param, str) for param in ignore_params]):
raise TypeError("Value in `ignore_params` must be str")
for param in ignore_params:
params.pop(param)
return params
def get_initial_state(self, inputs):
with ops.name_scope('initial_state'):
initial_state = array_ops.zeros_like(inputs) # (b, t, i)
initial_state = math_ops.reduce_sum(initial_state,
axis=(1, 2)) # (b,)
initial_state = array_ops.expand_dims(initial_state,
axis=1) # (b, 1)
return [
array_ops.tile(initial_state, [1, dim])
for dim in to_list(self.cell.state_size)
]
def __new__(cls,
outputs,
feed_inputs=None,
loss=None,
metrics=None,
params=None,
train_hooks=None,
val_hooks=None):
if outputs is None:
raise ValueError("Model output must be specified")
outputs = list(flatten_list(to_list(outputs)))
if train_hooks is None:
train_hooks = []
if val_hooks is None:
val_hooks = []
if feed_inputs is not None:
feed_inputs = to_list(feed_inputs)
return super(ExecutorSpec, cls).__new__(
cls, outputs=outputs, feed_inputs=feed_inputs,
loss=loss, metrics=metrics, params=params,
train_hooks=train_hooks, val_hooks=val_hooks)
def valid_data(data):
if data is None:
return []
if isinstance(data, dict):
data = [data[key] for key in list(sorted(data.keys()))]
else:
data = to_list(data)
if not all(isinstance(x, np.ndarray)
or F.is_tensor(x) for x in data):
raise ValueError("All elements should be instances"
" of numpy.ndarray or tensorflow.Tensor, but"
" received: " + str([type(x) for x in data]))
return data
def __new__(cls,
outputs=None,
loss=None,
metrics=None,
train_hooks=None,
val_hooks=None):
if outputs is not None:
outputs = nest.flatten(to_list(outputs))
if train_hooks is None:
train_hooks = []
if val_hooks is None:
val_hooks = []
return super(EstimatorSpec, cls).__new__(
cls, outputs=outputs, loss=loss,
metrics=metrics, train_hooks=train_hooks,
val_hooks=val_hooks)
def _valid_data(data, name='data'):
values = []
names = []
if isinstance(data, dict):
for name, value in data.items():
names.append(name)
values.append(value)
else:
values = to_list(data)
names = [name + '_%d' % i for i in range(1, len(values) + 1)]
if not all(isinstance(x, np.ndarray)
or F.is_tensor(x) for x in values):
raise ValueError("All elements should be instances"
" of numpy.ndarray or tensorflow.Tensor, but"
" received: " + str(values))
return names, values
def _compile_args(self, args, tag, default=None):
if isinstance(args, dict):
ret = []
for arg in args:
if arg not in self.output_names:
raise ValueError("Unknown entry in %s dictionary: %s."
"Only expected the following keys: %s"
% (tag, str(arg), str(self.output_names)))
for name in self.output_names:
ret.append(args.get(name, default))
else:
args = to_list(args)
if len(args) != len(self.outputs):
raise ValueError("Mismatch length between %s and outputs"
" with %d vs %d" % (tag, len(args), len(self.outputs)))
ret = args
return ret
def build(self, input_shape):
self.cell.build(input_shape)
self.batch_size = input_shape[0]
if self.stateful:
if self.batch_size is None:
raise ValueError("If a RNN is stateful, the last state for"
" each sample at index i in a batch will"
" be used as initial state for the sample"
" of index i in the following batch, that"
" means it needs to know its batch size")
self._states = [
self.add_weight(initial_value=array_ops.zeros(
self.batch_size, dim),
shape=(self.batch_size, dim),
dtype=self.dtype,
trainable=False,
name='state_%d' % i)
for i, dim in enumerate(to_list(self.cell.state_size))
]
def __init__(self,
thresholds=None,
top_k=None,
class_id=None,
name=None,
dtype=None):
super(Precision, self).__init__(name=name, dtype=dtype)
self.init_thresholds = thresholds
self.top_k = top_k
self.class_id = class_id
default_threshold = 0.5 if top_k else NEG_INF
self.thresholds = [valid_range(th or default_threshold, (0, 1))
for th in to_list(thresholds)]
self.true_positives = self.add_weight(
name='true_positives',
shape=(len(self.thresholds),),
initializer='zeros')
self.false_positives = self.add_weight(
name='false_positives',
shape=(len(self.thresholds),),
initializer='zeros')
def _compile_metrics(self, metrics):
logging.info("=>Compiling metrics...")
self.metric_names = ['loss']
self.metric_tensors = []
self.stateful_metrics = set()
self.stateful_metric_names = []
if isinstance(metrics, dict):
for name, metric in metrics.items():
self.metric_names.append(name)
self.metric_tensors.append(metric)
if hasattr(metric, '_metric_obj'):
self.stateful_metrics.add(getattr(metric, '_metric_obj'))
self.stateful_metric_names.append(name)
else:
for i, metric in enumerate(to_list(metrics)):
self.metric_tensors.append(metric)
name = 'metric_%d' % (i + 1)
if hasattr(metric, '_metric_obj'):
name = getattr(metric, '_metric_obj').name
self.stateful_metrics.add(getattr(metric, '_metric_obj'))
self.stateful_metric_names.append(name)
self.metric_names.append(name)
def states(self):
if self.stateful:
return self._states
else:
return [None] * len(to_list(self.cell.state_size))
def build_model(self, x, y=None, training=None):
x_keys, valid_x = utils.valid_data(x)
y_keys, valid_y = utils.valid_data(y) # y is [] if y=None
if self.inputs is None:
self._build_feed_inputs(valid_x)
if self.model_fn is None:
if has_arg(self.forward, 'training'):
self._uses_learning_phase = True
self.outputs = to_list(self(*self.inputs, training=training))
else:
self.outputs = to_list(self(*self.inputs))
elif y is not None:
self._build_feed_targets(valid_y)
logging.info('=>Calling model_fn...')
result = self.model_fn(
self, utils.nest_data(
self.inputs, x_keys, x),
utils.nest_data(
self.targets, y_keys, y))
logging.info('=>Finish calling model_fn...')
if not isinstance(result, EstimatorSpec):
raise ValueError("Result returned from `model_fn` must be"
"an instance of `EstimatorSpec`")
self.train_hooks.extend(result.train_hooks)
self.val_hooks.extend(result.val_hooks)
self.loss = result.loss
self.metrics = result.metrics
self.outputs = result.outputs
else: # graph-model, inputs and outputs already satisfied
self._input_names = []
self._feed_inputs = []
self._feed_input_names = []
self._feed_input_shapes = []
for i, x in enumerate(self.inputs):
name = 'input_%d' % (i + 1)
self._input_names.append(name)
self._feed_inputs.append(x)
self._feed_input_names.append(name)
self._feed_input_shapes.append(F.int_shape(x))
if self.model_fn is not None:
self._build_feed_targets(valid_y)
logging.info('=>Calling model_fn...')
result = self.model_fn(
self, None, utils.nest_data(
self.targets, y_keys, y))
logging.info('=>Finish calling model_fn...')
if not isinstance(result, EstimatorSpec):
raise ValueError("Result returned from `model_fn` must be"
"an instance of `EstimatorSpec`")
self.train_hooks.extend(result.train_hooks)
self.val_hooks.extend(result.val_hooks)
self.loss = result.loss
self.metrics = result.metrics
self.outputs = result.outputs
self._output_names = [
'output_%d' % i for i in range(1, len(self.outputs) + 1)]
if not self.uses_learning_phase:
self._uses_learning_phase = any(getattr(x, '_uses_learning_phase', False)
for x in self.outputs)
self._is_built = True
return valid_x, valid_y
def compile(self,
model_fn=None,
optimizer=None,
loss=None,
loss_weights=None,
metrics=None,
train_hooks=None,
val_hooks=None,
checkpoint_dir=None,
targets=None,
session_cfg=None,
**kwargs):
"""
:param model_fn: Function implemented by user when using estimator-style
execution mechanism, format as:
Params:
model: Instance reference of this model, you must call this model
to generate computation graph
inputs: list or dict, input data
labels: list or dict, labels
return: lib.training.EstimatorSpec
def model_fn(model, inputs, labels):
# data preprocess, hook preparation...
outputs = model(inputs)
# calculate loss, metrics, ....
return lib.training.EstimatorSpec(....)
:param optimizer: An instance of tf.train.Optimizer
:param loss: An instance of lib.training.Loss or predefined name of
lib.training.Loss (e.g. mse) when in keras-style execution mechanism,
otherwise, tensor computed from model_fn
:param loss_weights: Optional list or dict specifying scalar
coefficients (Python floats) to weight the loss contributions
of different model outputs.
:param metrics: Nested list with compatible instances of lib.training.Metric
or predefined name of lib.training.Metric (e.g. [['acc', 'mse'], ['acc']])
to self.outputs when in keras-style execution mechanism, otherwise, list or
dict with item as tensor computed from model_fn
:param train_hooks: List of instances of lib.training.SessRunHook for training,
it can be passed from model_fn
:param val_hooks: List of instances of lib.training.SessRunHook for evaluating,
it can be passed from model_fn
:param checkpoint_dir: Directory where execution results store in
:param targets: List or dict target data when in keras-style execution mechanism,
otherwise, None
:param session_cfg: Dict or file path contains session config should match content in './env_cfg.toml'
:param kwargs: Optional function parameters
"""
self.model_fn = model_fn
self.optimizer = optimizer
self.loss = loss
self.loss_weights = loss_weights
self.metrics = metrics
if train_hooks is not None:
self.train_hooks.extend(to_list(train_hooks))
if val_hooks is not None:
self.val_hooks.extend(to_list(val_hooks))
self._checkpoint_dir = checkpoint_dir
self._session_cfg = session_cfg
self._function_kwargs = kwargs
if not self.is_built:
logging.info("=>Model function was not built, fully compile will"
" delay after first call(after fit|evaluate|predict)")
return
start = time.time()
logging.info("=>Start compiling......")
self._is_compiled = True
self._compile_loss(loss=loss,
loss_weights=loss_weights,
targets=targets)
self._compile_metrics(metrics)
self._compile_summary()
self._compile_environment(self._session_cfg)
logging.info("=>Finish compiling in %.4fs" % (time.time() - start))
def _compile_metrics(self, metrics):
"""
Compile metrics to desired format
each output map with a list of metrics
item inside metrics can be an instance of `training.Metric` or a tensor
Note:
when metrics if class-format, we will do formation check between metrics
and `self.outputs` to make sure enough number of metrics to compatible with
`self.outputs` and `self.targets`
when metrics if tensor-format, we will not do formation check, cause metric
calculation already handled by users themselves inside `model_fn`
:param metrics: None or a nested list or dict
"""
logging.info("=>Compiling metrics...")
is_tensor = False
if not metrics:
metrics = [[]] * len(self.outputs)
elif isinstance(metrics, list):
if not F.is_tensor(metrics[0]):
if not is_tensor and len(metrics) != len(self.outputs):
raise ValueError("Number of metric inside `metrics`"
" %d is not compatible with number"
" of `self.outputs` %d" % (
len(metrics), len(self.outputs)))
else:
is_tensor = True
metrics = [('metric_%d' % (i+1), m) for i, m in enumerate(metrics)]
elif isinstance(metrics, dict):
if not F.is_tensor(metrics[list(metrics.keys())[0]]):
metrics = [metrics.get(name, [])
for name in self.output_names]
else:
is_tensor = True
metrics = list(metrics.items())
else:
raise TypeError("Unexpected type of metrics: " + str(type(metrics)))
with ops.name_scope('compile_metric'):
if is_tensor:
self._compile_metric_tensors(metrics)
else:
# Must handle sparse situation carefully!
def _compile_metric(m, loss_fn):
if isinstance(loss_fn, losses.SparseCategoricalCrossEntropy):
if m in {'accuracy', 'acc'}:
m = metric_module.SparseCategoricalAccuracy()
return m
m = metric_module.get(m)
return m
metric_tensors = []
for i in range(len(self.outputs)):
if i in self._skip_target_indices:
continue
target = self.targets[i]
output = self.outputs[i]
output_metrics = to_list(metrics[i])
loss_function = self.loss_functions[i]
for j, metric in enumerate(output_metrics):
metric = _compile_metric(metric, loss_function)
metric_name = getattr(metric, 'name', 'metric_%d' % j)
metric_result = metric(target, output)
if len(self.output_names) > 1:
metric_name = self.output_names[i] + '_' + metric_name
metric_tensors.append((metric_name, metric_result))
self._compile_metric_tensors(metric_tensors)
