def compile(self, optimizer=None, loss=None, metrics=None, params=None, train_hooks=None, val_hooks=None, checkpoint_dir=None, **kwargs): self.optimizer = optimizer self.loss = loss self.metrics = metrics self.params = params if train_hooks is not None: self.train_hooks.extend(to_list(train_hooks)) if val_hooks is not None: self.val_hooks.extend(to_list(val_hooks)) self._checkpoint_dir = checkpoint_dir self._session_kwargs = kwargs if not self.built: logging.info("=>Model function was not built, fully compile will" " delay after first call(after fit|evaluate|predict)") return self._is_compiled = True self._compile_args_with_mode() if self._mode != ExecutorMode.PREDICT: self._compile_metrics(metrics) self._compile_summary() self._compile_environment(EnvironmentConfig(**self._session_kwargs))
def __call__(self, *args, **kwargs): with ops.name_scope(self.name): updates = to_list(self.update_state(*args, **kwargs)) with fops.control_dependencies(updates): result = self.result() # We are adding the metric object as metadata on every result tensor. # This metric instance will later be used to reset variable state after # each epoch of training. for res in to_list(nest.flatten(result)): setattr(res, '_metric_obj', self) return result
def forward(self, inputs, initial_state=None): if isinstance(inputs, list): initial_state = inputs[1:] inputs = inputs[0] elif self.stateful: initial_state = self.states else: initial_state = self.get_initial_state(inputs) if len(initial_state) != len(self.states): raise ValueError("Expect %d states, but received %d" % (len(self.states), len(initial_state))) time_steps = engine.int_shape(inputs)[1] if self.unroll and time_steps in [None, 1]: raise ValueError("To unroll a RNN, time dimension in inputs" " must be known, and must larger than 1") last_output, outputs, states = F.rnn(step_fn=self.cell.forward, inputs=inputs, initial_states=initial_state, go_backwards=self.go_backwards, unroll=self.unroll, input_length=time_steps) if self.stateful: with fops.control_dependencies(self.states): updates = [ state_ops.assign(self.states[i], states[i]) for i in range(len(self.states)) ] fops.add_to_collection(fops.GraphKeys.UPDATE_OPS, updates) if not self.return_sequence: outputs = last_output if self.return_state: return [outputs] + to_list(states) else: return outputs
def values(self, ignore_params=None): params = self._flatten() if ignore_params is not None: ignore_params = to_list(ignore_params) if any([not isinstance(param, str) for param in ignore_params]): raise TypeError("Value in `ignore_params` must be str") for param in ignore_params: params.pop(param) return params
def get_initial_state(self, inputs): with ops.name_scope('initial_state'): initial_state = array_ops.zeros_like(inputs) # (b, t, i) initial_state = math_ops.reduce_sum(initial_state, axis=(1, 2)) # (b,) initial_state = array_ops.expand_dims(initial_state, axis=1) # (b, 1) return [ array_ops.tile(initial_state, [1, dim]) for dim in to_list(self.cell.state_size) ]
def __new__(cls, outputs, feed_inputs=None, loss=None, metrics=None, params=None, train_hooks=None, val_hooks=None): if outputs is None: raise ValueError("Model output must be specified") outputs = list(flatten_list(to_list(outputs))) if train_hooks is None: train_hooks = [] if val_hooks is None: val_hooks = [] if feed_inputs is not None: feed_inputs = to_list(feed_inputs) return super(ExecutorSpec, cls).__new__( cls, outputs=outputs, feed_inputs=feed_inputs, loss=loss, metrics=metrics, params=params, train_hooks=train_hooks, val_hooks=val_hooks)
def valid_data(data): if data is None: return [] if isinstance(data, dict): data = [data[key] for key in list(sorted(data.keys()))] else: data = to_list(data) if not all(isinstance(x, np.ndarray) or F.is_tensor(x) for x in data): raise ValueError("All elements should be instances" " of numpy.ndarray or tensorflow.Tensor, but" " received: " + str([type(x) for x in data])) return data
def __new__(cls, outputs=None, loss=None, metrics=None, train_hooks=None, val_hooks=None): if outputs is not None: outputs = nest.flatten(to_list(outputs)) if train_hooks is None: train_hooks = [] if val_hooks is None: val_hooks = [] return super(EstimatorSpec, cls).__new__( cls, outputs=outputs, loss=loss, metrics=metrics, train_hooks=train_hooks, val_hooks=val_hooks)
def _valid_data(data, name='data'): values = [] names = [] if isinstance(data, dict): for name, value in data.items(): names.append(name) values.append(value) else: values = to_list(data) names = [name + '_%d' % i for i in range(1, len(values) + 1)] if not all(isinstance(x, np.ndarray) or F.is_tensor(x) for x in values): raise ValueError("All elements should be instances" " of numpy.ndarray or tensorflow.Tensor, but" " received: " + str(values)) return names, values
def _compile_args(self, args, tag, default=None): if isinstance(args, dict): ret = [] for arg in args: if arg not in self.output_names: raise ValueError("Unknown entry in %s dictionary: %s." "Only expected the following keys: %s" % (tag, str(arg), str(self.output_names))) for name in self.output_names: ret.append(args.get(name, default)) else: args = to_list(args) if len(args) != len(self.outputs): raise ValueError("Mismatch length between %s and outputs" " with %d vs %d" % (tag, len(args), len(self.outputs))) ret = args return ret
def build(self, input_shape): self.cell.build(input_shape) self.batch_size = input_shape[0] if self.stateful: if self.batch_size is None: raise ValueError("If a RNN is stateful, the last state for" " each sample at index i in a batch will" " be used as initial state for the sample" " of index i in the following batch, that" " means it needs to know its batch size") self._states = [ self.add_weight(initial_value=array_ops.zeros( self.batch_size, dim), shape=(self.batch_size, dim), dtype=self.dtype, trainable=False, name='state_%d' % i) for i, dim in enumerate(to_list(self.cell.state_size)) ]
def __init__(self, thresholds=None, top_k=None, class_id=None, name=None, dtype=None): super(Precision, self).__init__(name=name, dtype=dtype) self.init_thresholds = thresholds self.top_k = top_k self.class_id = class_id default_threshold = 0.5 if top_k else NEG_INF self.thresholds = [valid_range(th or default_threshold, (0, 1)) for th in to_list(thresholds)] self.true_positives = self.add_weight( name='true_positives', shape=(len(self.thresholds),), initializer='zeros') self.false_positives = self.add_weight( name='false_positives', shape=(len(self.thresholds),), initializer='zeros')
def _compile_metrics(self, metrics): logging.info("=>Compiling metrics...") self.metric_names = ['loss'] self.metric_tensors = [] self.stateful_metrics = set() self.stateful_metric_names = [] if isinstance(metrics, dict): for name, metric in metrics.items(): self.metric_names.append(name) self.metric_tensors.append(metric) if hasattr(metric, '_metric_obj'): self.stateful_metrics.add(getattr(metric, '_metric_obj')) self.stateful_metric_names.append(name) else: for i, metric in enumerate(to_list(metrics)): self.metric_tensors.append(metric) name = 'metric_%d' % (i + 1) if hasattr(metric, '_metric_obj'): name = getattr(metric, '_metric_obj').name self.stateful_metrics.add(getattr(metric, '_metric_obj')) self.stateful_metric_names.append(name) self.metric_names.append(name)
def states(self): if self.stateful: return self._states else: return [None] * len(to_list(self.cell.state_size))
def build_model(self, x, y=None, training=None): x_keys, valid_x = utils.valid_data(x) y_keys, valid_y = utils.valid_data(y) # y is [] if y=None if self.inputs is None: self._build_feed_inputs(valid_x) if self.model_fn is None: if has_arg(self.forward, 'training'): self._uses_learning_phase = True self.outputs = to_list(self(*self.inputs, training=training)) else: self.outputs = to_list(self(*self.inputs)) elif y is not None: self._build_feed_targets(valid_y) logging.info('=>Calling model_fn...') result = self.model_fn( self, utils.nest_data( self.inputs, x_keys, x), utils.nest_data( self.targets, y_keys, y)) logging.info('=>Finish calling model_fn...') if not isinstance(result, EstimatorSpec): raise ValueError("Result returned from `model_fn` must be" "an instance of `EstimatorSpec`") self.train_hooks.extend(result.train_hooks) self.val_hooks.extend(result.val_hooks) self.loss = result.loss self.metrics = result.metrics self.outputs = result.outputs else: # graph-model, inputs and outputs already satisfied self._input_names = [] self._feed_inputs = [] self._feed_input_names = [] self._feed_input_shapes = [] for i, x in enumerate(self.inputs): name = 'input_%d' % (i + 1) self._input_names.append(name) self._feed_inputs.append(x) self._feed_input_names.append(name) self._feed_input_shapes.append(F.int_shape(x)) if self.model_fn is not None: self._build_feed_targets(valid_y) logging.info('=>Calling model_fn...') result = self.model_fn( self, None, utils.nest_data( self.targets, y_keys, y)) logging.info('=>Finish calling model_fn...') if not isinstance(result, EstimatorSpec): raise ValueError("Result returned from `model_fn` must be" "an instance of `EstimatorSpec`") self.train_hooks.extend(result.train_hooks) self.val_hooks.extend(result.val_hooks) self.loss = result.loss self.metrics = result.metrics self.outputs = result.outputs self._output_names = [ 'output_%d' % i for i in range(1, len(self.outputs) + 1)] if not self.uses_learning_phase: self._uses_learning_phase = any(getattr(x, '_uses_learning_phase', False) for x in self.outputs) self._is_built = True return valid_x, valid_y
def compile(self, model_fn=None, optimizer=None, loss=None, loss_weights=None, metrics=None, train_hooks=None, val_hooks=None, checkpoint_dir=None, targets=None, session_cfg=None, **kwargs): """ :param model_fn: Function implemented by user when using estimator-style execution mechanism, format as: Params: model: Instance reference of this model, you must call this model to generate computation graph inputs: list or dict, input data labels: list or dict, labels return: lib.training.EstimatorSpec def model_fn(model, inputs, labels): # data preprocess, hook preparation... outputs = model(inputs) # calculate loss, metrics, .... return lib.training.EstimatorSpec(....) :param optimizer: An instance of tf.train.Optimizer :param loss: An instance of lib.training.Loss or predefined name of lib.training.Loss (e.g. mse) when in keras-style execution mechanism, otherwise, tensor computed from model_fn :param loss_weights: Optional list or dict specifying scalar coefficients (Python floats) to weight the loss contributions of different model outputs. :param metrics: Nested list with compatible instances of lib.training.Metric or predefined name of lib.training.Metric (e.g. [['acc', 'mse'], ['acc']]) to self.outputs when in keras-style execution mechanism, otherwise, list or dict with item as tensor computed from model_fn :param train_hooks: List of instances of lib.training.SessRunHook for training, it can be passed from model_fn :param val_hooks: List of instances of lib.training.SessRunHook for evaluating, it can be passed from model_fn :param checkpoint_dir: Directory where execution results store in :param targets: List or dict target data when in keras-style execution mechanism, otherwise, None :param session_cfg: Dict or file path contains session config should match content in './env_cfg.toml' :param kwargs: Optional function parameters """ self.model_fn = model_fn self.optimizer = optimizer self.loss = loss self.loss_weights = loss_weights self.metrics = metrics if train_hooks is not None: self.train_hooks.extend(to_list(train_hooks)) if val_hooks is not None: self.val_hooks.extend(to_list(val_hooks)) self._checkpoint_dir = checkpoint_dir self._session_cfg = session_cfg self._function_kwargs = kwargs if not self.is_built: logging.info("=>Model function was not built, fully compile will" " delay after first call(after fit|evaluate|predict)") return start = time.time() logging.info("=>Start compiling......") self._is_compiled = True self._compile_loss(loss=loss, loss_weights=loss_weights, targets=targets) self._compile_metrics(metrics) self._compile_summary() self._compile_environment(self._session_cfg) logging.info("=>Finish compiling in %.4fs" % (time.time() - start))
def _compile_metrics(self, metrics): """ Compile metrics to desired format each output map with a list of metrics item inside metrics can be an instance of `training.Metric` or a tensor Note: when metrics if class-format, we will do formation check between metrics and `self.outputs` to make sure enough number of metrics to compatible with `self.outputs` and `self.targets` when metrics if tensor-format, we will not do formation check, cause metric calculation already handled by users themselves inside `model_fn` :param metrics: None or a nested list or dict """ logging.info("=>Compiling metrics...") is_tensor = False if not metrics: metrics = [[]] * len(self.outputs) elif isinstance(metrics, list): if not F.is_tensor(metrics[0]): if not is_tensor and len(metrics) != len(self.outputs): raise ValueError("Number of metric inside `metrics`" " %d is not compatible with number" " of `self.outputs` %d" % ( len(metrics), len(self.outputs))) else: is_tensor = True metrics = [('metric_%d' % (i+1), m) for i, m in enumerate(metrics)] elif isinstance(metrics, dict): if not F.is_tensor(metrics[list(metrics.keys())[0]]): metrics = [metrics.get(name, []) for name in self.output_names] else: is_tensor = True metrics = list(metrics.items()) else: raise TypeError("Unexpected type of metrics: " + str(type(metrics))) with ops.name_scope('compile_metric'): if is_tensor: self._compile_metric_tensors(metrics) else: # Must handle sparse situation carefully! def _compile_metric(m, loss_fn): if isinstance(loss_fn, losses.SparseCategoricalCrossEntropy): if m in {'accuracy', 'acc'}: m = metric_module.SparseCategoricalAccuracy() return m m = metric_module.get(m) return m metric_tensors = [] for i in range(len(self.outputs)): if i in self._skip_target_indices: continue target = self.targets[i] output = self.outputs[i] output_metrics = to_list(metrics[i]) loss_function = self.loss_functions[i] for j, metric in enumerate(output_metrics): metric = _compile_metric(metric, loss_function) metric_name = getattr(metric, 'name', 'metric_%d' % j) metric_result = metric(target, output) if len(self.output_names) > 1: metric_name = self.output_names[i] + '_' + metric_name metric_tensors.append((metric_name, metric_result)) self._compile_metric_tensors(metric_tensors)