def test_loop(model, inputs, targets, verbose=0, steps=None): """evaluate method to validate a model that uses DistributionStrategy. Arguments: model: Keras Model instance. inputs: List of input arrays. targets: List of target arrays. verbose: verbosity mode. steps: Total number of steps (batches of samples) before declaring predictions finished. Ignored with the default value of `None`. Returns: Scalar loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ current_strategy = model._distribution_strategy def _per_device_test_function(model): model._make_test_function() return (model.test_function.inputs, model.test_function.outputs, model.test_function.updates_op, model.test_function.session_kwargs) with current_strategy.scope(): (grouped_inputs, grouped_outputs, grouped_updates, grouped_session_args) = current_strategy.call_for_each_tower( _per_device_test_function, model._grouped_model) (all_inputs, all_outputs, all_updates, all_session_args) = distributed_training_utils.unwrap_values( current_strategy, grouped_inputs, grouped_outputs, grouped_updates, grouped_session_args, with_loss_tensor=True) dataset_inputs = distributed_training_utils.flatten_perdevice_values( current_strategy, inputs) dataset_targets = distributed_training_utils.flatten_perdevice_values( current_strategy, targets) distributed_test_function = K.Function( all_inputs, all_outputs, updates=all_updates, name='distributed_test_function', **all_session_args) # We need to set sample_weights to None since there are sample weight # placeholders that are created with default values. sample_weights = [None for _ in range(len(model.outputs) * current_strategy.num_towers)] if model.uses_learning_phase and not isinstance(K.learning_phase(), int): ins = dataset_inputs + dataset_targets + sample_weights + [0] else: ins = dataset_inputs + dataset_targets outs = [] if verbose == 1: progbar = Progbar(target=steps) # Copy the weights from the original model to each of the replicated models. orig_model_weights = model.get_weights() with current_strategy.scope(): distributed_model = current_strategy.unwrap(model._grouped_model)[0] distributed_training_utils.set_weights( current_strategy, distributed_model, orig_model_weights) if steps is not None: for step in range(steps): batch_outs = distributed_test_function(ins) batch_outs = _aggregate_metrics_across_towers( current_strategy.num_towers, model.metrics_names, batch_outs) if isinstance(batch_outs, list): if step == 0: for _ in enumerate(batch_outs): outs.append(0.) for i, batch_out in enumerate(batch_outs): outs[i] += batch_out else: if step == 0: outs.append(0.) outs[0] += batch_outs if verbose == 1: progbar.update(step + 1) for i in range(len(outs)): outs[i] /= steps if len(outs) == 1: return outs[0] return outs
def predict_loop(model, inputs, verbose=0, steps=None): """Abstract method to loop over some data in batches. Arguments: model: Keras Model instance. inputs: list of tensors to be fed to `f`. verbose: verbosity mode. steps: Total number of steps (batches of samples) before declaring `_predict_loop` finished. Ignored with the default value of `None`. Returns: Array of predictions (if the model has a single output) or list of arrays of predictions (if the model has multiple outputs). """ current_strategy = model._distribution_strategy def _per_device_predict_function(model): model._make_predict_function() return (model.predict_function.inputs, model.predict_function.outputs, model.predict_function.updates_op, model.predict_function.session_kwargs) with current_strategy.scope(): (grouped_inputs, grouped_outputs, grouped_updates, grouped_session_args) = current_strategy.call_for_each_tower( _per_device_predict_function, model._grouped_model) (all_inputs, all_outputs, all_updates, all_session_args) = distributed_training_utils.unwrap_values( current_strategy, grouped_inputs, grouped_outputs, grouped_updates, grouped_session_args) dataset_inputs = distributed_training_utils.flatten_perdevice_values( current_strategy, inputs) distributed_predict_function = K.Function( all_inputs, all_outputs, updates=all_updates, name='distributed_predict_function', **all_session_args) if model.uses_learning_phase and not isinstance(K.learning_phase(), int): ins = dataset_inputs + [0] else: ins = dataset_inputs if verbose == 1: progbar = Progbar(target=steps) # Copy the weights from the original model to each of the replicated models. orig_model_weights = model.get_weights() with current_strategy.scope(): distributed_model = current_strategy.unwrap(model._grouped_model)[0] distributed_training_utils.set_weights( current_strategy, distributed_model, orig_model_weights) if steps is not None: # Since we do not know how many samples we will see, we cannot pre-allocate # the returned Numpy arrays. Instead, we store one array per batch seen # and concatenate them upon returning. unconcatenated_outs = [] for step in range(steps): batch_outs = distributed_predict_function(ins) if not isinstance(batch_outs, list): batch_outs = [batch_outs] if step == 0: for _ in batch_outs: unconcatenated_outs.append([]) for i, batch_out in enumerate(batch_outs): unconcatenated_outs[i].append(batch_out) if verbose == 1: progbar.update(step + 1) if len(unconcatenated_outs) == 1: return np.concatenate(unconcatenated_outs[0], axis=0) return [ np.concatenate(unconcatenated_outs[i], axis=0) for i in range(len(unconcatenated_outs)) ]
def experimental_predict_loop(model, iterator, verbose=0, steps=None): """Predict loop for predicting with TPU DistributionStrategy. Arguments: model: Keras Model instance. iterator: Iterator for input data. verbose: Integer, Verbosity mode 0 or 1. steps: Total number of steps (batches of samples) before declaring `_predict_loop` finished. Ignored with the default value of `None`. Returns: Array of predictions (if the model has a single output) or list of arrays of predictions (if the model has multiple outputs). """ current_strategy = model._distribution_strategy K.get_session().run(current_strategy.initialize()) # TODO(priyag, sourabhbajaj): This should likely not be hardcoded here. K.set_learning_phase(0) def _per_device_predict_function(model): model._make_predict_function() return (model.predict_function.inputs, model.predict_function.outputs, model.predict_function.updates_op, model.predict_function.session_kwargs) def step_fn(ctx, inputs): """Clones the model and calls make_predict_function.""" # TODO(priyag, sourabhbajaj): The model gets cloned every time # fit/test/predict is called. We should look into caching this keyed on # input shapes. clone_model_on_replicas(model, current_strategy, make_callback_model=False, inputs=inputs, mode=_Mode.PREDICT) (grouped_inputs, grouped_outputs, grouped_updates, grouped_session_args ) = current_strategy.extended.call_for_each_replica( _per_device_predict_function, args=(model._grouped_model_predict, )) (all_inputs, all_outputs, all_updates, all_session_args) = distributed_training_utils.unwrap_values( current_strategy, grouped_inputs, grouped_outputs, grouped_updates, grouped_session_args) combined_fn = K.function(all_inputs, all_outputs, updates=all_updates, name='distributed_predict_function', **all_session_args) for label, output in zip(model.output_names, combined_fn.outputs): ctx.set_last_step_output(label, output) return combined_fn.updates_op # Add initial dummy values for outputs. initial_loop_values = {} batch_dimension = distributed_training_utils.get_batch_dimension(iterator) for name, tensor in zip(model.output_names, model.outputs): # TODO(priyag): This is a workaround as we do not know the batch dimension # of the model's output at this point. shape = tensor_shape.TensorShape(tensor.shape.dims) shape.dims = [batch_dimension] + shape.dims[1:] initial_loop_values[name] = array_ops.zeros(shape, tensor.dtype) with current_strategy.scope(): # TODO(priyag, sourabhbajaj): Support steps_per_run if/when we add outfeed. ctx = current_strategy.extended.experimental_run_steps_on_iterator( step_fn, iterator, iterations=1, initial_loop_values=initial_loop_values) predict_op = ctx.run_op output_tensors = ctx.last_step_outputs if verbose == 1: progbar = Progbar(target=steps) # Copy the weights from the original model to each of the replicated models. with current_strategy.scope(): _copy_weights_to_distributed_model(model, model._grouped_model_predict) assert steps is not None # Since we do not know how many samples we will see, we cannot pre-allocate # the returned Numpy arrays. Instead, we store one array per batch seen # and concatenate them upon returning. unconcatenated_outs = [[] for _ in model.outputs] for step in range(steps): _, batch_outs = K.get_session().run([predict_op, output_tensors]) # TODO(priyag): maybe need to unwrap the outputs first for MirroredStrategy. for i, label in enumerate(model.output_names): unconcatenated_outs[i].extend(batch_outs[label]) if verbose >= 1: progbar.update(step + 1) K.get_session().run(current_strategy.finalize()) if len(unconcatenated_outs) == 1: return np.concatenate(unconcatenated_outs[0], axis=0) return [ np.concatenate(unconcatenated_outs[i], axis=0) for i in range(len(unconcatenated_outs)) ]
or list of arrays of predictions (if the model has multiple outputs). """ model._make_predict_function() f = model.predict_function # Callbacks batch end. batch_logs = cbks.make_logs(model, batch_logs, batch_outs, mode) callbacks._call_batch_hook(mode, 'end', batch_index, batch_logs) progbar.on_batch_end(batch_index, batch_logs) num_samples = training_utils.check_num_samples( inputs, batch_size, steps, 'steps') if verbose == 1: if steps is not None: progbar = Progbar(target=steps) else: progbar = Progbar(target=num_samples) indices_for_conversion_to_dense = [] for i in range(len(model._feed_inputs)): if (issparse is not None and issparse(inputs[i]) and not K.is_sparse(model._feed_inputs[i])): indices_for_conversion_to_dense.append(i) if steps is not None: # Step-based predictions. # Since we do not know how many samples # we will see, we cannot pre-allocate # the returned Numpy arrays. # Instead, we store one array per batch seen
def experimental_tpu_predict_loop(model, dataset, verbose=0, steps=None, callbacks=None): """Predict loop for predicting with TPU tf.distribute.Strategy. Arguments: model: Keras Model instance. dataset: Dataset for input data. verbose: Integer, Verbosity mode 0 or 1. steps: Total number of steps (batches of samples) before declaring `_predict_loop` finished. Ignored with the default value of `None`. callbacks: List of callbacks to be called during training Returns: Array of predictions (if the model has a single output) or list of arrays of predictions (if the model has multiple outputs). """ mode = ModeKeys.PREDICT dataset_fully_shaped = dist_utils.is_dataset_shape_fully_defined(dataset) padding_handler = None if not dataset_fully_shaped: # TODO(hongjunchoi): Investigate whether operations from # PartialBatchPaddingHandler are unnecessarily pruned out # during graph optimization. padding_handler = padding_util.PartialBatchPaddingHandler( model._feed_output_shapes) batch_size, _, prefetch_buffer = input_lib._get_dataset_attributes( dataset) padding_handler.padded_batch_size = batch_size padding_handler.padding_mask = dataset.reduce( padding_handler.padding_mask, padding_handler.update_mask) dataset = dataset.map(padding_handler.pad_batch) dataset = dataset.unbatch() # Upon this point, it is guaranteed that the dataset does not # have partial batches. Thus, we set `drop_remainder=True` to # get static shape information about the elements in the dataset. dataset = dataset.batch(batch_size, drop_remainder=True) if prefetch_buffer is not None: dataset = dataset.prefetch(prefetch_buffer) current_strategy = model._distribution_strategy iterator = dist_utils.get_iterator(dataset, current_strategy) scope = dist_utils.distributed_scope(strategy=current_strategy, learning_phase=0) scope.__enter__() def _predict_step_fn(inputs): """A fn that returns output of single prediction step.""" (distribution_strategy_context.get_replica_context().merge_call( _build_model, args=(model, mode, inputs))) (_, outputs, updates, _) = _per_replica_execution_function( dist_utils.get_distributed_model(model, mode), mode) with ops.control_dependencies([updates]): return [array_ops.identity(out) for out in outputs] # TODO(hongjunchoi): When numpy array is passed as an input to `predict()` # use numpy arrays directly to avoid cumulating unnecessary input pipeline # ops. predict_input_data = iterator.get_next() per_replica_outputs = current_strategy.run(_predict_step_fn, args=(predict_input_data, )) output_tensors = dist_utils.flatten_per_replica_values( current_strategy, per_replica_outputs) if verbose >= 1: progbar = Progbar(target=steps) if model._compile_distribution: dist_utils._copy_weights_to_distributed_model(model, mode) dist_utils._reset_metrics(model) callbacks = cbks.configure_callbacks(callbacks, model, do_validation=False, epochs=1, steps_per_epoch=steps, verbose=verbose, count_mode='steps', mode=mode) callbacks._call_begin_hook(mode) # Since we do not know how many samples we will see, we cannot pre-allocate # the returned Numpy arrays. Instead, we store one array per batch seen # and concatenate them upon returning. num_model_outputs = len(model.output_names) unconcatenated_outs = [[] for _ in range(num_model_outputs)] if steps is not None: target_steps = steps else: raise ValueError( 'Number of steps could not be inferred from the data, ' 'please pass the steps argument.') current_step = 0 while current_step < target_steps: batch_logs = {'batch': current_step, 'size': 1} callbacks._call_batch_hook(mode, 'begin', current_step, batch_logs) try: predict_ops = control_flow_ops.group(output_tensors) _, batch_outs = K.batch_get_value([predict_ops, output_tensors]) except errors.OutOfRangeError: warning_msg = 'Make sure that your dataset can generate at least ' '`steps` batches (in this case, {} batches).'.format(steps) logging.warning('Your dataset iterator ran out of data; ' 'interrupting evaluation. ' + warning_msg) break # TODO(priyag): maybe need to unwrap the outputs first for MirroredStrategy. for i in range(num_model_outputs): output_start_index = i * current_strategy.num_replicas_in_sync output_end_index = (output_start_index + current_strategy.num_replicas_in_sync) single_model_output = batch_outs[ output_start_index:output_end_index] unconcatenated_outs[i].extend(single_model_output) batch_logs = cbks.make_logs(model, batch_logs, batch_outs, mode) callbacks._call_batch_hook(mode, 'end', current_step, batch_logs) if verbose == 1: progbar.update(current_step + 1) current_step += 1 if verbose >= 1: # Progress bar finishes at the end. progbar.update(current_step) callbacks._call_end_hook(mode) scope.__exit__(None, None, None) if len(unconcatenated_outs) == 1: prediction_result = np.concatenate(unconcatenated_outs[0], axis=0) else: prediction_result = [ np.concatenate(out, axis=0) for out in unconcatenated_outs ] if padding_handler: prediction_result = padding_handler.apply_mask(prediction_result) return prediction_result
def predict_generator(model, generator, steps=None, max_queue_size=10, workers=1, use_multiprocessing=False, verbose=0): """See docstring for `Model.predict_generator`.""" if not context.executing_eagerly(): model._make_predict_function() steps_done = 0 all_outs = [] is_sequence = isinstance(generator, data_utils.Sequence) if not is_sequence and use_multiprocessing and workers > 1: logging.warning( UserWarning('Using a generator with `use_multiprocessing=True`' ' and multiple workers may duplicate your data.' ' Please consider using the`keras.utils.Sequence' ' class.')) if steps is None: if is_sequence: steps = len(generator) else: raise ValueError('Please specify the `steps` argument.') enqueuer = None try: if workers > 0: if is_sequence: enqueuer = data_utils.OrderedEnqueuer( generator, use_multiprocessing=use_multiprocessing) else: enqueuer = data_utils.GeneratorEnqueuer( generator, use_multiprocessing=use_multiprocessing) enqueuer.start(workers=workers, max_queue_size=max_queue_size) output_generator = enqueuer.get() else: if is_sequence: output_generator = data_utils.iter_sequence_infinite(generator) else: output_generator = generator if verbose == 1: progbar = Progbar(target=steps) while steps_done < steps: generator_output = next(output_generator) if isinstance(generator_output, tuple): # Compatibility with the generators # used for training. if len(generator_output) == 2: x, _ = generator_output elif len(generator_output) == 3: x, _, _ = generator_output else: raise ValueError('Output of generator should be ' 'a tuple `(x, y, sample_weight)` ' 'or `(x, y)`. Found: ' + str(generator_output)) else: # Assumes a generator that only # yields inputs (not targets and sample weights). x = generator_output outs = model.predict_on_batch(x) if not isinstance(outs, list): outs = [outs] if not all_outs: for out in outs: all_outs.append([]) for i, out in enumerate(outs): all_outs[i].append(out) steps_done += 1 if verbose == 1: progbar.update(steps_done) except (errors.OutOfRangeError, StopIteration): logging.warning( 'Your dataset iterator ran out of data interrupting testing. ' 'Make sure that your dataset can generate at least `steps` ' 'batches (in this case, %d batches). You may need to use the ' 'repeat() function when building your dataset.', steps) finally: if enqueuer is not None: enqueuer.stop() if len(all_outs) == 1: if steps_done == 1: return all_outs[0][0] else: return np.concatenate(all_outs[0]) if steps_done == 1: return [out[0] for out in all_outs] else: return [np.concatenate(out) for out in all_outs]
def test_loop(model, inputs, targets, sample_weights=None, batch_size=None, verbose=0, steps=None): """Abstract method to loop over some data in batches. Arguments: model: Keras Model instance. inputs: List of input arrays. targets: List of target arrays. sample_weights: Optional list of sample weight arrays. batch_size: integer batch size or `None`. verbose: verbosity mode. steps: Total number of steps (batches of samples) before declaring predictions finished. Ignored with the default value of `None`. Returns: Scalar loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ model._make_test_function() f = model.test_function sample_weights = sample_weights or [] if model.uses_learning_phase and not isinstance(K.learning_phase(), int): ins = inputs + targets + sample_weights + [0] else: ins = inputs + targets + sample_weights if hasattr(model, 'metrics'): for m in model.stateful_metric_functions: m.reset_states() stateful_metric_indices = [ i for i, name in enumerate(model.metrics_names) if str(name) in model.stateful_metric_names ] else: stateful_metric_indices = [] num_samples = training_utils.check_num_samples(ins, batch_size, steps, 'steps') outs = [] if verbose == 1: if steps is not None: progbar = Progbar(target=steps) else: progbar = Progbar(target=num_samples) # To prevent a slowdown, we find beforehand the arrays that need conversion. feed = model._feed_inputs + model._feed_targets + model._feed_sample_weights indices_for_conversion_to_dense = [] for i in range(len(feed)): if issparse is not None and issparse( ins[i]) and not K.is_sparse(feed[i]): indices_for_conversion_to_dense.append(i) if steps is not None: for step in range(steps): batch_outs = f(ins) if isinstance(batch_outs, list): if step == 0: for _ in enumerate(batch_outs): outs.append(0.) for i, batch_out in enumerate(batch_outs): if i in stateful_metric_indices: outs[i] = batch_out else: outs[i] += batch_out else: if step == 0: outs.append(0.) outs[0] += batch_outs if verbose == 1: progbar.update(step + 1) for i in range(len(outs)): if i not in stateful_metric_indices: outs[i] /= steps else: batches = make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] if isinstance(ins[-1], int): # Do not slice the training phase flag. ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = slice_arrays(ins, batch_ids) for i in indices_for_conversion_to_dense: ins_batch[i] = ins_batch[i].toarray() batch_outs = f(ins_batch) if isinstance(batch_outs, list): if batch_index == 0: outs.extend([0.] * len(batch_outs)) for i, batch_out in enumerate(batch_outs): if i in stateful_metric_indices: outs[i] = batch_out else: outs[i] += batch_out * len(batch_ids) else: if batch_index == 0: outs.append(0.) outs[0] += batch_outs * len(batch_ids) if verbose == 1: progbar.update(batch_end) for i in range(len(outs)): if i not in stateful_metric_indices: outs[i] /= num_samples if len(outs) == 1: return outs[0] return outs
def experimental_tpu_predict_loop(model, dataset, verbose=0, steps=None, callbacks=None): """Predict loop for predicting with TPU DistributionStrategy. Arguments: model: Keras Model instance. dataset: Dataset for input data. verbose: Integer, Verbosity mode 0 or 1. steps: Total number of steps (batches of samples) before declaring `_predict_loop` finished. Ignored with the default value of `None`. callbacks: List of callbacks to be called during training Returns: Array of predictions (if the model has a single output) or list of arrays of predictions (if the model has multiple outputs). """ mode = ModeKeys.PREDICT dataset_fully_shaped = (distributed_training_utils. is_dataset_shape_fully_defined(dataset)) padding_handler = None if not dataset_fully_shaped: # TODO(hongjunchoi): Investigate whether operations from # PartialBatchPaddingHandler are unnecessarily pruned out # during graph optimization. padding_handler = padding_util.PartialBatchPaddingHandler( model._feed_output_shapes) batched_dataset = input_lib._get_batched_dataset(dataset) batch_size, _, prefetch_buffer = input_lib._get_batched_dataset_attributes( batched_dataset) padding_handler.padded_batch_size = batch_size padding_handler.padding_mask = dataset.reduce(padding_handler.padding_mask, padding_handler.update_mask) dataset = dataset.map(padding_handler.pad_batch) dataset = dataset.apply(batching.unbatch()) # Upon this point, it is guaranteed that the dataset does not # have partial batches. Thus, we set `drop_remainder=True` to # get static shape information about the elements in the dataset. dataset = dataset.batch(batch_size, drop_remainder=True) if prefetch_buffer is not None: dataset = dataset.prefetch(prefetch_buffer) current_strategy = model._distribution_strategy iterator = distributed_training_utils.get_iterator(dataset, current_strategy) scope = distributed_training_utils.distributed_scope( strategy=current_strategy, learning_phase=0) scope.__enter__() def _per_device_predict_function(model): model._make_predict_function() return (model.predict_function.inputs, model.predict_function.outputs, model.predict_function.updates_op, model.predict_function.session_kwargs) def step_fn(ctx, inputs): """Clones the model and calls make_predict_function.""" if model._compile_distribution: distributed_training_utils.clone_model_on_replicas( model, current_strategy, mode, inputs=inputs) else: distributed_training_utils._build_distributed_network( model, current_strategy, mode, inputs) (grouped_inputs, grouped_outputs, grouped_updates, grouped_session_args) = current_strategy.extended.call_for_each_replica( _per_device_predict_function, args=(model._distributed_model_predict,)) (all_inputs, all_outputs, all_updates, all_session_args) = distributed_training_utils.unwrap_values( current_strategy, grouped_inputs, grouped_outputs, grouped_updates, grouped_session_args) combined_fn = K.function( all_inputs, all_outputs, updates=all_updates, name='distributed_predict_function', **all_session_args) for label, output in zip(model.output_names, combined_fn.outputs): ctx.set_last_step_output(label, output) return combined_fn.updates_op # Add initial dummy values for outputs. initial_loop_values = {} batch_dimension = distributed_training_utils.get_batch_dimension(iterator) for name, tensor in zip(model.output_names, model.outputs): # TODO(priyag): This is a workaround as we do not know the batch dimension # of the model's output at this point. shape = tensor_shape.TensorShape(tensor.shape.dims) shape.dims = [batch_dimension] + shape.dims[1:] initial_loop_values[name] = array_ops.zeros(shape, tensor.dtype) # TODO(priyag, sourabhbajaj): Support steps_per_run if/when we add outfeed. ctx = current_strategy.extended.experimental_run_steps_on_iterator( step_fn, iterator, iterations=1, initial_loop_values=initial_loop_values) predict_op = ctx.run_op output_tensors = ctx.last_step_outputs if verbose == 1: progbar = Progbar(target=steps) if model._compile_distribution: distributed_training_utils._copy_weights_to_distributed_model(model, mode) distributed_training_utils._reset_metrics(model) callbacks = cbks.configure_callbacks( callbacks, model, do_validation=False, epochs=1, steps_per_epoch=steps, verbose=verbose, count_mode='steps', mode=mode) callbacks._call_begin_hook(mode) assert steps is not None # Since we do not know how many samples we will see, we cannot pre-allocate # the returned Numpy arrays. Instead, we store one array per batch seen # and concatenate them upon returning. unconcatenated_outs = [[] for _ in model.outputs] for step in range(steps): batch_logs = {'batch': step, 'size': 1} callbacks._call_batch_hook(mode, 'begin', step, batch_logs) _, batch_outs = K.get_session().run([predict_op, output_tensors]) # TODO(priyag): maybe need to unwrap the outputs first for MirroredStrategy. for i, label in enumerate(model.output_names): unconcatenated_outs[i].extend(batch_outs[label]) batch_logs = cbks.make_logs(model, batch_logs, batch_outs, mode) callbacks._call_batch_hook(mode, 'end', step, batch_logs) if verbose >= 1: progbar.update(step + 1) callbacks._call_end_hook(mode) scope.__exit__(None, None, None) if len(unconcatenated_outs) == 1: prediction_result = np.concatenate(unconcatenated_outs[0], axis=0) else: prediction_result = [ np.concatenate(unconcatenated_outs[i], axis=0) for i in range(len(unconcatenated_outs)) ] if padding_handler: prediction_result = padding_handler.apply_mask(prediction_result) return prediction_result
def predict_generator(model, generator, steps=None, max_queue_size=10, workers=1, use_multiprocessing=False, verbose=0): """See docstring for `Model.predict_generator`.""" if not context.executing_eagerly(): model._make_predict_function() steps_done = 0 wait_time = 0.01 all_outs = [] is_sequence = isinstance(generator, Sequence) if not is_sequence and use_multiprocessing and workers > 1: logging.warning( UserWarning('Using a generator with `use_multiprocessing=True`' ' and multiple workers may duplicate your data.' ' Please consider using the`keras.utils.Sequence' ' class.')) if steps is None: if is_sequence: steps = len(generator) else: raise ValueError('`steps=None` is only valid for a generator' ' based on the `keras.utils.Sequence` class.' ' Please specify `steps` or use the' ' `keras.utils.Sequence` class.') enqueuer = None try: if workers > 0: if is_sequence: enqueuer = OrderedEnqueuer( generator, use_multiprocessing=use_multiprocessing) else: enqueuer = GeneratorEnqueuer( generator, use_multiprocessing=use_multiprocessing, wait_time=wait_time) enqueuer.start(workers=workers, max_queue_size=max_queue_size) output_generator = enqueuer.get() else: if is_sequence: output_generator = iter(generator) else: output_generator = generator if verbose == 1: progbar = Progbar(target=steps) while steps_done < steps: generator_output = next(output_generator) if isinstance(generator_output, tuple): # Compatibility with the generators # used for training. if len(generator_output) == 2: x, _ = generator_output elif len(generator_output) == 3: x, _, _ = generator_output else: raise ValueError('Output of generator should be ' 'a tuple `(x, y, sample_weight)` ' 'or `(x, y)`. Found: ' + str(generator_output)) else: # Assumes a generator that only # yields inputs (not targets and sample weights). x = generator_output outs = model.predict_on_batch(x) if not isinstance(outs, list): outs = [outs] if not all_outs: for out in outs: all_outs.append([]) for i, out in enumerate(outs): all_outs[i].append(out) steps_done += 1 if verbose == 1: progbar.update(steps_done) finally: if enqueuer is not None: enqueuer.stop() if len(all_outs) == 1: if steps_done == 1: return all_outs[0][0] else: return np.concatenate(all_outs[0]) if steps_done == 1: return [out[0] for out in all_outs] else: return [np.concatenate(out) for out in all_outs]
def predict_loop(model, inputs, batch_size=32, verbose=0, steps=None): """Abstract method to loop over some data in batches. Arguments: model: Keras Model instance. inputs: list of tensors to be fed to `f`. batch_size: integer batch size. verbose: verbosity mode. steps: Total number of steps (batches of samples) before declaring `_predict_loop` finished. Ignored with the default value of `None`. Returns: Array of predictions (if the model has a single output) or list of arrays of predictions (if the model has multiple outputs). """ model._make_predict_function() f = model.predict_function if model.uses_learning_phase and not isinstance(K.learning_phase(), int): ins = inputs + [0] else: ins = inputs num_samples = training_utils.check_num_samples(inputs, batch_size, steps, 'steps') if verbose == 1: if steps is not None: progbar = Progbar(target=steps) else: progbar = Progbar(target=num_samples) indices_for_conversion_to_dense = [] for i in range(len(model._feed_inputs)): if (issparse is not None and issparse(inputs[i]) and not K.is_sparse(model._feed_inputs[i])): indices_for_conversion_to_dense.append(i) if steps is not None: # Step-based predictions. # Since we do not know how many samples # we will see, we cannot pre-allocate # the returned Numpy arrays. # Instead, we store one array per batch seen # and concatenate them upon returning. unconcatenated_outs = [] for step in range(steps): batch_outs = f(ins) if not isinstance(batch_outs, list): batch_outs = [batch_outs] if step == 0: for batch_out in batch_outs: unconcatenated_outs.append([]) for i, batch_out in enumerate(batch_outs): unconcatenated_outs[i].append(batch_out) if verbose == 1: progbar.update(step + 1) if len(unconcatenated_outs) == 1: return np.concatenate(unconcatenated_outs[0], axis=0) return [ np.concatenate(unconcatenated_outs[i], axis=0) for i in range(len(unconcatenated_outs)) ] else: # Sample-based predictions. outs = [] batches = make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] if ins and isinstance(ins[-1], int): # Do not slice the training phase flag. ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = slice_arrays(ins, batch_ids) for i in indices_for_conversion_to_dense: ins_batch[i] = ins_batch[i].toarray() batch_outs = f(ins_batch) if not isinstance(batch_outs, list): batch_outs = [batch_outs] if batch_index == 0: # Pre-allocate the results arrays. for batch_out in batch_outs: shape = (num_samples, ) + batch_out.shape[1:] outs.append(np.zeros(shape, dtype=batch_out.dtype)) for i, batch_out in enumerate(batch_outs): outs[i][batch_start:batch_end] = batch_out if verbose == 1: progbar.update(batch_end) if len(outs) == 1: return outs[0] return outs
def evaluate_generator(model, generator, steps=None, max_queue_size=10, workers=1, use_multiprocessing=False, verbose=0): """See docstring for `Model.evaluate_generator`.""" if not context.executing_eagerly(): model._make_test_function() if hasattr(model, 'metrics'): for m in model.stateful_metric_functions: m.reset_states() stateful_metric_indices = [ i for i, name in enumerate(model.metrics_names) if str(name) in model.stateful_metric_names ] else: stateful_metric_indices = [] steps_done = 0 wait_time = 0.01 all_outs = [] batch_sizes = [] is_sequence = isinstance(generator, Sequence) if not is_sequence and use_multiprocessing and workers > 1: logging.warning( UserWarning('Using a generator with `use_multiprocessing=True`' ' and multiple workers may duplicate your data.' ' Please consider using the`keras.utils.Sequence' ' class.')) if steps is None: if is_sequence: steps = len(generator) else: raise ValueError('`steps=None` is only valid for a generator' ' based on the `keras.utils.Sequence` class.' ' Please specify `steps` or use the' ' `keras.utils.Sequence` class.') enqueuer = None try: if workers > 0: if is_sequence: enqueuer = OrderedEnqueuer( generator, use_multiprocessing=use_multiprocessing) else: enqueuer = GeneratorEnqueuer( generator, use_multiprocessing=use_multiprocessing, wait_time=wait_time) enqueuer.start(workers=workers, max_queue_size=max_queue_size) output_generator = enqueuer.get() else: if is_sequence: output_generator = iter(generator) else: output_generator = generator if verbose == 1: progbar = Progbar(target=steps) while steps_done < steps: generator_output = next(output_generator) if not hasattr(generator_output, '__len__'): raise ValueError('Output of generator should be a tuple ' '(x, y, sample_weight) ' 'or (x, y). Found: ' + str(generator_output)) if len(generator_output) == 2: x, y = generator_output sample_weight = None elif len(generator_output) == 3: x, y, sample_weight = generator_output else: raise ValueError('Output of generator should be a tuple ' '(x, y, sample_weight) ' 'or (x, y). Found: ' + str(generator_output)) outs = model.test_on_batch(x, y, sample_weight=sample_weight) if isinstance(x, list): batch_size = x[0].shape[0] elif isinstance(x, dict): batch_size = list(x.values())[0].shape[0] else: batch_size = x.shape[0] if batch_size == 0: raise ValueError('Received an empty batch. ' 'Batches should at least contain one item.') all_outs.append(outs) steps_done += 1 batch_sizes.append(batch_size) if verbose == 1: progbar.update(steps_done) finally: if enqueuer is not None: enqueuer.stop() if not isinstance(outs, list): return np.average(np.asarray(all_outs), weights=batch_sizes) else: averages = [] for i in range(len(outs)): if i not in stateful_metric_indices: averages.append( np.average([out[i] for out in all_outs], weights=batch_sizes)) else: averages.append(np.float64(all_outs[-1][i])) return averages
def _fun_generator(self, x, generator, state): self._model.optimizer.clear() self._update_weights(x) callbacks = state['callbacks'] if not state['in_epoch']: callbacks.on_epoch_begin(state['epoch']) state['in_epoch'] = True cost_sum = 0 iterator = range(len(generator)) if state['verbose'] == 1 and len(generator) > 1: # iterator = trange(len(generator)) progress_bar = Progbar(len(generator)) else: # iterator = range(len(generator)) progress_bar = False state['epoch_logs'] = {} epoch_logs = state['epoch_logs'] for batch_index in iterator: inputs, outputs, sample_weights = generator[batch_index] if isinstance(inputs, list): isize = inputs[0].shape[0] else: isize = inputs.shape[0] batch_logs = {'batch': batch_index, 'size': isize} callbacks.on_batch_begin(batch_index, batch_logs) outs = self._model.train_on_batch(inputs, outputs, sample_weights) if not isinstance(outs, list): outs = [outs] for lbl, v in zip(self._model.metrics_names, outs): batch_logs[lbl] = v epoch_logs[lbl] = epoch_logs.get(lbl, 0.0) + v callbacks.on_batch_end(batch_index, batch_logs) batch_cost = batch_logs['loss'] if progress_bar: progress_bar.update(batch_index + 1) # iterator.set_postfix(cost=batch_cost) cost_sum += batch_cost generator.on_epoch_end() epoch_logs = state['epoch_logs'] if state['verbose'] > 0: print( 'itr:', state['epoch'], ', '.join([ '{0}: {1:.4e}'.format(k, v) for k, v in epoch_logs.items() ])) elif state['verbose'] < 0: print('itr:', state['epoch'], ', loss: {0:.6e}'.format(epoch_logs['loss'])) # average the metrics for lbl in self._model.metrics_names: epoch_logs[lbl] = epoch_logs.get(lbl) / len(iterator) cost = cost_sum / len(iterator) gradients = self._model.optimizer.get_gradient_values() x_grad = np.empty(x.shape) x_offset = 0 for grad in gradients: w_size = grad.size x_grad[x_offset:x_offset + w_size] = grad.reshape(-1) x_offset += w_size assert x_offset == self._weights_size self._cost = cost self._gradients = x_grad return cost, x_grad
def train(self, train_data, val_data=None, epochs=200, early_stopping=None, verbose=1, save_best=True, ckpt_path=None, as_model=False, monitor='val_accuracy', early_stop_metric='val_loss', callbacks=None, **kwargs): """Train the model for the input `train_data` of nodes or `sequence`. Note: ---------- You must compile your model before training/testing/predicting. Use `model.build()`. Parameters: ---------- train_data: Numpy array-like, `list`, Integer scalar or `graphgallery.Sequence` The index of nodes (or sequence) that will be used during training. val_data: Numpy array-like, `list`, Integer scalar or `graphgallery.Sequence`, optional The index of nodes (or sequence) that will be used for validation. (default :obj: `None`, i.e., do not use validation during training) epochs: Positive integer The number of epochs of training.(default :obj: `200`) early_stopping: Positive integer or None The number of early stopping patience during training. (default :obj: `None`, i.e., do not use early stopping during training) verbose: int in {0, 1, 2, 3, 4} 'verbose=0': not verbose; 'verbose=1': Progbar (one line, detailed); 'verbose=2': Progbar (one line, omitted); 'verbose=3': Progbar (multi line, detailed); 'verbose=4': Progbar (multi line, omitted); (default :obj: 1) save_best: bool Whether to save the best weights (accuracy of loss depend on `monitor`) of training or validation (depend on `validation` is `False` or `True`). (default :bool: `True`) ckpt_path: String or None The path of saved weights/model. (default :obj: `None`, i.e., `./log/{self.name}_weights`) as_model: bool Whether to save the whole model or weights only, if `True`, the `self.custom_objects` must be speficied if you are using custom `layer` or `loss` and so on. monitor: String One of (val_loss, val_acc, loss, acc), it determines which metric will be used for `save_best`. (default :obj: `val_acc`) early_stop_metric: String One of (val_loss, val_acc, loss, acc), it determines which metric will be used for early stopping. (default :obj: `val_loss`) callbacks: tensorflow.keras.callbacks. (default :obj: `None`) kwargs: other keyword Parameters. Return: ---------- A `tf.keras.callbacks.History` object. Its `History.history` attribute is a record of training loss values and metrics values at successive epochs, as well as validation loss values and validation metrics values (if applicable). """ raise_if_kwargs(kwargs) if not (isinstance(verbose, int) and 0 <= verbose <= 4): raise ValueError("'verbose=0': not verbose" "'verbose=1': Progbar(one line, detailed), " "'verbose=2': Progbar(one line, omitted), " "'verbose=3': Progbar(multi line, detailed), " "'verbose=4': Progbar(multi line, omitted), " f"but got {verbose}") model = self.model # Check if model has been built if model is None: raise RuntimeError( 'You must compile your model before training/testing/predicting. Use `model.build()`.' ) metrics_names = getattr(model, "metrics_names", None) # FIXME: This would be return '[]' for tensorflow>=2.2.0 # See <https://github.com/tensorflow/tensorflow/issues/37990> if not metrics_names: raise RuntimeError( f"Please specify the attribute 'metrics_names' for the model.") if not isinstance(train_data, Sequence): train_data = self.train_sequence(train_data) self.train_data = train_data validation = val_data is not None if validation: if not isinstance(val_data, Sequence): val_data = self.test_sequence(val_data) self.val_data = val_data metrics_names = metrics_names + [ "val_" + metric for metric in metrics_names ] if not isinstance(callbacks, callbacks_module.CallbackList): callbacks = callbacks_module.CallbackList(callbacks) history = History() callbacks.append(history) if early_stopping: es_callback = EarlyStopping(monitor=early_stop_metric, patience=early_stopping, mode='auto', verbose=kwargs.pop('es_verbose', 1)) callbacks.append(es_callback) if save_best: if not ckpt_path: ckpt_path = self.ckpt_path else: self.ckpt_path = ckpt_path makedirs_from_filepath(ckpt_path) if not ckpt_path.endswith(gg.file_ext()): ckpt_path = ckpt_path + gg.file_ext() assert monitor in metrics_names, f"'{monitor}' are not included in the metrics names." mc_callback = ModelCheckpoint(ckpt_path, monitor=monitor, save_best_only=True, save_weights_only=not as_model, verbose=0) callbacks.append(mc_callback) callbacks.set_model(model) model.stop_training = False metrics_names = metrics_names + ["Duration"] if verbose: stateful_metrics = set(metrics_names) if verbose <= 2: progbar = Progbar(target=epochs, verbose=verbose, stateful_metrics=stateful_metrics) print("Training...") logs = BunchDict() begin_time = time.perf_counter() callbacks.on_train_begin() try: for epoch in range(epochs): if verbose > 2: progbar = Progbar(target=len(train_data), verbose=verbose - 2, stateful_metrics=stateful_metrics) callbacks.on_epoch_begin(epoch) callbacks.on_train_batch_begin(0) train_logs = self.train_step(train_data) train_data.on_epoch_end() logs.update(train_logs) if validation: valid_logs = self.test_step(val_data) logs.update({("val_" + k): v for k, v in valid_logs.items()}) val_data.on_epoch_end() callbacks.on_train_batch_end(len(train_data), logs) callbacks.on_epoch_end(epoch, logs) time_passed = time.perf_counter() - begin_time logs["Duration"] = time_passed if verbose > 2: print(f"Epoch {epoch+1}/{epochs}") progbar.update(len(train_data), logs.items()) elif verbose: progbar.update(epoch + 1, logs.items()) if model.stop_training: print(f"Early Stopping in Epoch {epoch}", file=sys.stderr) break callbacks.on_train_end() self.load(ckpt_path, as_model=as_model) finally: # to avoid unexpected termination of the model self.remove_weights() return history
def train(self, idx_train, idx_val=None, epochs=200, early_stopping=None, verbose=0, save_best=True, weight_path=None, as_model=False, monitor='val_acc', early_stop_metric='val_loss', callbacks=None, **kwargs): """Train the model for the input `idx_train` of nodes or `sequence`. Note: ---------- You must compile your model before training/testing/predicting. Use `model.build()`. Parameters: ---------- idx_train: Numpy array-like, `list`, Integer scalar or `graphgallery.Sequence` The index of nodes (or sequence) that will be used during training. idx_val: Numpy array-like, `list`, Integer scalar or `graphgallery.Sequence`, optional The index of nodes (or sequence) that will be used for validation. (default :obj: `None`, i.e., do not use validation during training) epochs: Positive integer The number of epochs of training.(default :obj: `200`) early_stopping: Positive integer or None The number of early stopping patience during training. (default :obj: `None`, i.e., do not use early stopping during training) verbose: int in {0, 1, 2, 3, 4} 'verbose=0': not verbose; 'verbose=1': Progbar (one line, detailed); 'verbose=2': Progbar (one line, omitted); 'verbose=3': Progbar (multi line, detailed); 'verbose=4': Progbar (multi line, omitted); (default :obj: 0) save_best: bool Whether to save the best weights (accuracy of loss depend on `monitor`) of training or validation (depend on `validation` is `False` or `True`). (default :bool: `True`) weight_path: String or None The path of saved weights/model. (default :obj: `None`, i.e., `./log/{self.name}_weights`) as_model: bool Whether to save the whole model or weights only, if `True`, the `self.custom_objects` must be speficied if you are using customized `layer` or `loss` and so on. monitor: String One of (val_loss, val_acc, loss, acc), it determines which metric will be used for `save_best`. (default :obj: `val_acc`) early_stop_metric: String One of (val_loss, val_acc, loss, acc), it determines which metric will be used for early stopping. (default :obj: `val_loss`) callbacks: tensorflow.keras.callbacks. (default :obj: `None`) kwargs: other keyword Parameters. Return: ---------- A `tf.keras.callbacks.History` object. Its `History.history` attribute is a record of training loss values and metrics values at successive epochs, as well as validation loss values and validation metrics values (if applicable). """ raise_if_kwargs(kwargs) if not (isinstance(verbose, int) and 0 <= verbose <= 4): raise ValueError("'verbose=0': not verbose" "'verbose=1': Progbar(one line, detailed), " "'verbose=2': Progbar(one line, omitted), " "'verbose=3': Progbar(multi line, detailed), " "'verbose=4': Progbar(multi line, omitted), " f"but got {verbose}") model = self.model # Check if model has been built if model is None: raise RuntimeError( 'You must compile your model before training/testing/predicting. Use `model.build()`.' ) if isinstance(idx_train, Sequence): train_data = idx_train else: idx_train = asintarr(idx_train) train_data = self.train_sequence(idx_train) self.idx_train = idx_train validation = idx_val is not None if validation: if isinstance(idx_val, Sequence): val_data = idx_val else: idx_val = asintarr(idx_val) val_data = self.test_sequence(idx_val) self.idx_val = idx_val else: monitor = 'acc' if monitor[:3] == 'val' else monitor if not isinstance(callbacks, callbacks_module.CallbackList): callbacks = callbacks_module.CallbackList(callbacks) history = History() callbacks.append(history) if early_stopping: es_callback = EarlyStopping(monitor=early_stop_metric, patience=early_stopping, mode='auto', verbose=kwargs.pop('es_verbose', 1)) callbacks.append(es_callback) if save_best: if not weight_path: weight_path = self.weight_path else: self.weight_path = weight_path makedirs_from_path(weight_path) if not weight_path.endswith(POSTFIX): weight_path = weight_path + POSTFIX mc_callback = ModelCheckpoint(weight_path, monitor=monitor, save_best_only=True, save_weights_only=not as_model, verbose=0) callbacks.append(mc_callback) callbacks.set_model(model) model.stop_training = False callbacks.on_train_begin() if verbose: stateful_metrics = {"acc", 'loss', 'val_acc', 'val_loss', 'time'} if verbose <= 2: progbar = Progbar(target=epochs, verbose=verbose, stateful_metrics=stateful_metrics) print("Training...") begin_time = time.perf_counter() for epoch in range(epochs): if verbose > 2: progbar = Progbar(target=len(train_data), verbose=verbose - 2, stateful_metrics=stateful_metrics) callbacks.on_epoch_begin(epoch) callbacks.on_train_batch_begin(0) loss, accuracy = self.train_step(train_data) training_logs = {'loss': loss, 'acc': accuracy} if validation: val_loss, val_accuracy = self.test_step(val_data) training_logs.update({ 'val_loss': val_loss, 'val_acc': val_accuracy }) val_data.on_epoch_end() callbacks.on_train_batch_end(len(train_data), training_logs) callbacks.on_epoch_end(epoch, training_logs) train_data.on_epoch_end() if verbose: time_passed = time.perf_counter() - begin_time training_logs.update({'time': time_passed}) if verbose > 2: print(f"Epoch {epoch+1}/{epochs}") progbar.update(len(train_data), training_logs.items()) else: progbar.update(epoch + 1, training_logs.items()) if model.stop_training: break callbacks.on_train_end() if save_best: self.load(weight_path, as_model=as_model) self.remove_weights() return history
def test_loop(model, iterator, verbose=0, steps=None): """Test loop for evaluating with DistributionStrategy. Arguments: model: Keras Model instance. iterator: Iterator for input data. verbose: Integer, Verbosity mode 0 or 1. steps: Total number of steps (batches of samples) before declaring predictions finished. Ignored with the default value of `None`. Returns: Scalar loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs and/or metrics). The attribute `model.metrics_names` will give you the display labels for the outputs. """ current_strategy = model._distribution_strategy # TODO(priyag, sourabhbajaj): Remove this when the codepaths are merged. if current_strategy.__class__.__name__ == 'TPUStrategy': return _experimental_test_loop(model, iterator, verbose, steps) if not model._grouped_model: clone_model_on_towers(model, current_strategy) def _per_device_test_function(model): model._make_test_function() return (model.test_function.inputs, model.test_function.outputs, model.test_function.updates_op, model.test_function.session_kwargs) inputs, targets = _get_input_from_iterator(iterator, model) with current_strategy.scope(): (grouped_inputs, grouped_outputs, grouped_updates, grouped_session_args) = current_strategy.call_for_each_tower( _per_device_test_function, model._grouped_model) (all_inputs, all_outputs, all_updates, all_session_args) = distributed_training_utils.unwrap_values( current_strategy, grouped_inputs, grouped_outputs, grouped_updates, grouped_session_args, with_loss_tensor=True) dataset_inputs = distributed_training_utils.flatten_perdevice_values( current_strategy, inputs) dataset_targets = distributed_training_utils.flatten_perdevice_values( current_strategy, targets) distributed_test_function = K.Function( all_inputs, all_outputs, updates=all_updates, name='distributed_test_function', **all_session_args) # We need to set sample_weights to None since there are sample weight # placeholders that are created with default values. sample_weights = [ None for _ in range(len(model.outputs) * current_strategy.num_towers) ] if model.uses_learning_phase and not isinstance( K.learning_phase(), int): ins = dataset_inputs + dataset_targets + sample_weights + [0] else: ins = dataset_inputs + dataset_targets for m in model.stateful_metric_functions: m.reset_states() stateful_metric_indices = [ i for i, name in enumerate(model.metrics_names) if str(name) in model.stateful_metric_names ] outs = [] if verbose == 1: progbar = Progbar(target=steps) # Copy the weights from the original model to each of the replicated models. orig_model_weights = model.get_weights() distributed_model = current_strategy.unwrap(model._grouped_model)[0] distributed_training_utils.set_weights(current_strategy, distributed_model, orig_model_weights) assert steps is not None for step in range(steps): batch_outs = distributed_test_function(ins) batch_outs = _aggregate_metrics_across_towers( current_strategy.num_towers, model.metrics_names, model.stateful_metric_names, batch_outs) if isinstance(batch_outs, list): if step == 0: outs = [0.] * len(batch_outs) for i, batch_out in enumerate(batch_outs): if i in stateful_metric_indices: outs[i] = batch_out else: outs[i] += batch_out else: if step == 0: outs.append(0.) outs[0] += batch_outs if verbose >= 1: progbar.update(step + 1) for i in range(len(outs)): if i not in stateful_metric_indices: outs[i] /= steps if len(outs) == 1: return outs[0] return outs
def experimental_tpu_test_loop(model, dataset, verbose=0, steps=None, callbacks=None): """Test loop for evaluating with TPU DistributionStrategy. Arguments: model: Keras Model instance. dataset: Dataset for input data. verbose: Integer, Verbosity mode 0 or 1. steps: Total number of steps (batches of samples) before declaring predictions finished. Ignored with the default value of `None`. callbacks: List of callbacks to be called during training Returns: Scalar loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs and/or metrics). The attribute `model.metrics_names` will give you the display labels for the outputs. """ mode = ModeKeys.TEST current_strategy = model._distribution_strategy iterator = distributed_training_utils.get_iterator(dataset, current_strategy) scope = distributed_training_utils.distributed_scope( strategy=current_strategy, learning_phase=0) scope.__enter__() def _per_device_eval_function(model): model._make_eval_function() return (model._eval_function.inputs, model._eval_function.outputs, model._eval_function.updates_op, model._eval_function.session_kwargs) def step_fn(ctx, inputs): """Clones the model and calls make_eval_function.""" inputs, targets = inputs if model._compile_distribution: distributed_training_utils.clone_model_on_replicas( model, current_strategy, mode=mode, inputs=inputs, targets=targets) else: distributed_training_utils._build_distributed_network( model, current_strategy, mode, inputs, targets) (grouped_inputs, grouped_outputs, grouped_updates, grouped_session_args) = current_strategy.extended.call_for_each_replica( _per_device_eval_function, args=(model._distributed_model_test,)) (all_inputs, all_outputs, all_updates, all_session_args) = distributed_training_utils.unwrap_values( current_strategy, grouped_inputs, grouped_outputs, grouped_updates, grouped_session_args) combined_fn = K.function( all_inputs, all_outputs, updates=all_updates, name='distributed_test_function', **all_session_args) for label, output in zip(model.metrics_names, combined_fn.outputs): if label == 'loss': reduce_op = distribute_lib.get_loss_reduction() else: # We reduce all other metrics using mean for now. This is temporary # workaround until new metrics are in place. reduce_op = ds_reduce_util.ReduceOp.MEAN ctx.set_last_step_output(label, output, reduce_op) return combined_fn.updates_op # Add initial dummy values for loss and other metric tensors. initial_loop_values = {} initial_loop_values['loss'] = constant_op.constant(1e7) for name in model.metrics_names[1:]: tensor = model._all_stateful_metrics_tensors[name] initial_loop_values[name] = array_ops.zeros(tensor.shape, tensor.dtype) # TODO(priyag): Use steps_per_run when we use new metrics as they will # allow handling metric computation at each step using variables. ctx = current_strategy.extended.experimental_run_steps_on_iterator( step_fn, iterator, iterations=1, initial_loop_values=initial_loop_values) test_op = ctx.run_op output_tensors = ctx.last_step_outputs if verbose == 1: progbar = Progbar(target=steps) if model._compile_distribution: distributed_training_utils._copy_weights_to_distributed_model(model, mode) distributed_training_utils._reset_metrics(model) callbacks = cbks.configure_callbacks( callbacks, model, do_validation=False, epochs=1, steps_per_epoch=steps, verbose=verbose, count_mode='steps', mode=ModeKeys.TEST) callbacks._call_begin_hook(mode) assert steps is not None outs = [0.] * len(model.metrics_names) for step in range(steps): batch_logs = {'batch': step, 'size': 1} callbacks._call_batch_hook(mode, 'begin', step, batch_logs) _, batch_outs = K.get_session().run([test_op, output_tensors]) for i, label in enumerate(model.metrics_names): if i == 0: # Loss is stateless metrics. outs[i] += batch_outs[label] else: # For all stateful metrics, the aggregation is handled by mirrored vars. outs[i] = batch_outs[label] batch_logs = cbks.make_logs(model, batch_logs, outs, mode) callbacks._call_batch_hook(mode, 'end', step, batch_logs) if verbose >= 1: progbar.update(step + 1) callbacks._call_end_hook(mode) scope.__exit__(None, None, None) if len(outs) >= 0: outs[0] /= (steps) if len(outs) == 1: return outs[0] return outs
def predict_loop(model, iterator, verbose=0, steps=None): """Predict loop for predicting with DistributionStrategy. Arguments: model: Keras Model instance. iterator: Iterator for input data. verbose: Integer, Verbosity mode 0 or 1. steps: Total number of steps (batches of samples) before declaring `_predict_loop` finished. Ignored with the default value of `None`. Returns: Array of predictions (if the model has a single output) or list of arrays of predictions (if the model has multiple outputs). """ current_strategy = model._distribution_strategy # TODO(priyag, sourabhbajaj): Remove this when the codepaths are merged. if current_strategy.__class__.__name__ == 'TPUStrategy': return _experimental_predict_loop(model, iterator, verbose, steps) if not model._grouped_model: clone_model_on_towers(model, current_strategy) def _per_device_predict_function(model): model._make_predict_function() return (model.predict_function.inputs, model.predict_function.outputs, model.predict_function.updates_op, model.predict_function.session_kwargs) inputs, _ = _get_input_from_iterator(iterator, model) with current_strategy.scope(): (grouped_inputs, grouped_outputs, grouped_updates, grouped_session_args) = current_strategy.call_for_each_tower( _per_device_predict_function, model._grouped_model) (all_inputs, all_outputs, all_updates, all_session_args) = distributed_training_utils.unwrap_values( current_strategy, grouped_inputs, grouped_outputs, grouped_updates, grouped_session_args) dataset_inputs = distributed_training_utils.flatten_perdevice_values( current_strategy, inputs) distributed_predict_function = K.Function( all_inputs, all_outputs, updates=all_updates, name='distributed_predict_function', **all_session_args) if model.uses_learning_phase and not isinstance( K.learning_phase(), int): ins = dataset_inputs + [0] else: ins = dataset_inputs if verbose == 1: progbar = Progbar(target=steps) # Copy the weights from the original model to each of the replicated models. orig_model_weights = model.get_weights() distributed_model = current_strategy.unwrap(model._grouped_model)[0] distributed_training_utils.set_weights(current_strategy, distributed_model, orig_model_weights) if steps is not None: # Since we do not know how many samples we will see, we cannot # pre-allocate the returned Numpy arrays. Instead, we store one array per # batch seen and concatenate them upon returning. unconcatenated_outs = [] for step in range(steps): batch_outs = distributed_predict_function(ins) if not isinstance(batch_outs, list): batch_outs = [batch_outs] if step == 0: for _ in batch_outs: unconcatenated_outs.append([]) # TODO(anjalisridhar): Should combine the outputs from multiple towers # correctly here. for i, batch_out in enumerate(batch_outs): unconcatenated_outs[i].append(batch_out) if verbose >= 1: progbar.update(step + 1) if len(unconcatenated_outs) == 1: return np.concatenate(unconcatenated_outs[0], axis=0) return [ np.concatenate(unconcatenated_outs[i], axis=0) for i in range(len(unconcatenated_outs)) ]
def experimental_tpu_test_loop(model, dataset, verbose=0, steps=None, callbacks=None): """Test loop for evaluating with TPU tf.distribute.Strategy. Arguments: model: Keras Model instance. dataset: Dataset for input data. verbose: Integer, Verbosity mode 0 or 1. steps: Total number of steps (batches of samples) before declaring predictions finished. Ignored with the default value of `None`. callbacks: List of callbacks to be called during training Returns: Scalar loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs and/or metrics). The attribute `model.metrics_names` will give you the display labels for the outputs. """ mode = ModeKeys.TEST current_strategy = model._distribution_strategy iterator = dist_utils.get_iterator(dataset, current_strategy) scope = dist_utils.distributed_scope(strategy=current_strategy, learning_phase=0) scope.__enter__() out_labels = model.metrics_names def _test_step_fn(inputs): """A fn that returns output of single test step.""" if isinstance(inputs, (tuple, list)) and len(inputs) == 2: inputs, targets = inputs else: targets = None (distribution_strategy_context.get_replica_context().merge_call( _build_model, args=(model, mode, inputs, targets))) (_, outputs, updates, _) = _per_replica_execution_function( dist_utils.get_distributed_model(model, mode), mode) with ops.control_dependencies([updates]): return [array_ops.identity(out) for out in outputs] test_input_data = iterator.get_next() per_replica_outputs = current_strategy.run(_test_step_fn, args=(test_input_data, )) output_tensors = {} for label, output in zip(out_labels, per_replica_outputs): if label == 'loss': reduce_op = ds_reduce_util.ReduceOp.SUM else: # We reduce all other metrics using mean for now. This is temporary # workaround until new metrics are in place. reduce_op = ds_reduce_util.ReduceOp.MEAN output_tensors[label] = current_strategy.reduce(reduce_op, output, axis=None) test_op = control_flow_ops.group(list(output_tensors.values())) if verbose >= 1: progbar = Progbar(target=steps) if model._compile_distribution: dist_utils._copy_weights_to_distributed_model(model, mode) dist_utils._reset_metrics(model) callbacks = cbks.configure_callbacks(callbacks, model, do_validation=False, epochs=1, steps_per_epoch=steps, verbose=verbose, count_mode='steps', mode=ModeKeys.TEST) callbacks._call_begin_hook(mode) outs = [0.] * len(model.metrics_names) if steps is not None: target_steps = steps else: raise ValueError( 'Number of steps could not be inferred from the data, ' 'please pass the steps argument.') current_step = 0 while current_step < target_steps: batch_logs = {'batch': current_step, 'size': 1} callbacks._call_batch_hook(mode, 'begin', current_step, batch_logs) try: _, batch_outs = K.batch_get_value([test_op, output_tensors]) except errors.OutOfRangeError: warning_msg = 'Make sure that your dataset can generate at least ' '`steps` batches (in this case, {} batches).'.format(steps) logging.warning('Your dataset iterator ran out of data; ' 'interrupting evaluation. ' + warning_msg) target_steps = current_step break for i, label in enumerate(model.metrics_names): if i == 0: # Loss is stateless metrics. outs[i] += batch_outs[label] else: # For all stateful metrics, the aggregation is handled by mirrored vars. outs[i] = batch_outs[label] batch_logs = cbks.make_logs(model, batch_logs, outs, mode) callbacks._call_batch_hook(mode, 'end', current_step, batch_logs) if verbose == 1: progbar.update(current_step + 1) current_step += 1 if verbose >= 1: # Progress bar finishes at the end. progbar.update(target_steps) callbacks._call_end_hook(mode) scope.__exit__(None, None, None) if len(outs) >= 0: outs[0] /= (target_steps) if len(outs) == 1: return outs[0] return outs
def experimental_test_loop(model, iterator, verbose=0, steps=None, initialize_finalize_strategy=True): """Test loop for evaluating with TPU DistributionStrategy. Arguments: model: Keras Model instance. iterator: Iterator for input data. verbose: Integer, Verbosity mode 0 or 1. steps: Total number of steps (batches of samples) before declaring predictions finished. Ignored with the default value of `None`. initialize_finalize_strategy: Should the strategy initialize and finalize functions be called. Returns: Scalar loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs and/or metrics). The attribute `model.metrics_names` will give you the display labels for the outputs. """ current_strategy = model._distribution_strategy if initialize_finalize_strategy: K.get_session().run(current_strategy.initialize()) def _per_device_eval_function(model): model._make_eval_function() return (model._eval_function.inputs, model._eval_function.outputs, model._eval_function.updates_op, model._eval_function.session_kwargs) # TODO(priyag, sourabhbajaj): This should likely not be hardcoded here. K.set_learning_phase(0) def step_fn(ctx, inputs): """Clones the model and calls make_eval_function.""" # TODO(priyag, sourabhbajaj): The model gets cloned every time # fit/test/predict is called. We should look into caching this keyed on # input shapes. inputs, targets = inputs clone_model_on_replicas(model, current_strategy, make_callback_model=False, inputs=inputs, targets=targets, mode=_Mode.TEST) (grouped_inputs, grouped_outputs, grouped_updates, grouped_session_args ) = current_strategy.extended.call_for_each_replica( _per_device_eval_function, args=(model._grouped_model_test, )) (all_inputs, all_outputs, all_updates, all_session_args) = distributed_training_utils.unwrap_values( current_strategy, grouped_inputs, grouped_outputs, grouped_updates, grouped_session_args) combined_fn = K.function(all_inputs, all_outputs, updates=all_updates, name='distributed_test_function', **all_session_args) for label, output in zip(model.metrics_names, combined_fn.outputs): if label == 'loss': reduce_op = distribute_lib.get_loss_reduction() else: # We reduce all other metrics using mean for now. This is temporary # workaround until new metrics are in place. reduce_op = ds_reduce_util.ReduceOp.MEAN ctx.set_last_step_output(label, output, reduce_op) return combined_fn.updates_op # Add initial dummy values for loss and other metric tensors. initial_loop_values = {} initial_loop_values['loss'] = constant_op.constant(1e7) for name in model.metrics_names[1:]: tensor = model._all_stateful_metrics_tensors[name] initial_loop_values[name] = array_ops.zeros(tensor.shape, tensor.dtype) with current_strategy.scope(): # TODO(priyag): Use steps_per_run when we use new metrics as they will # allow handling metric computation at each step using variables. ctx = current_strategy.extended.experimental_run_steps_on_iterator( step_fn, iterator, iterations=1, initial_loop_values=initial_loop_values) test_op = ctx.run_op output_tensors = ctx.last_step_outputs if verbose == 1: progbar = Progbar(target=steps) # Copy the weights from the original model to each of the replicated models. with current_strategy.scope(): _copy_weights_to_distributed_model(model, model._grouped_model_test) assert steps is not None outs = [0.] * len(model.metrics_names) for step in range(steps): _, batch_outs = K.get_session().run([test_op, output_tensors]) for i, label in enumerate(model.metrics_names): outs[i] += batch_outs[label] if verbose >= 1: progbar.update(step + 1) for i in range(len(outs)): outs[i] /= (steps) if initialize_finalize_strategy: K.get_session().run(current_strategy.finalize()) if len(outs) == 1: return outs[0] return outs
def read_CVA_waveform(CVA_list, save_dir): progbar = Progbar(len(CVA_list)) for q, list in enumerate(CVA_list): f = open(list, 'r') lines = f.readlines() count = 0 t = [] e = [] n = [] z = [] filename = str(list[-24:-4]).replace('/', '') if os.path.exists(f'{save_dir}{filename}.mseed'): progbar.add(1) continue try: for line in lines: if count == 0: station_code = line.strip()[14:20] if len(station_code) > 4: station_code = seisnn.io.change_station_code( station_code) network = 'TSMIP' else: network = 'CWBSN' elif count == 2: start_time = obspy.UTCDateTime(line.strip()[12:-1]) elif count == 3: duration = float(line.strip()[20:28]) elif count == 4: samplerate = int(line.strip()[17:20]) elif count == 1: pass elif count == 5: pass elif count == 6: pass elif count == 7: pass elif count == 8: pass elif count == 9: pass elif count == 10: pass else: try: tt = float(line[0:10]) except ValueError: tt = 0 try: te = float(line[10:20]) except ValueError: te = 0 try: tn = float(line[20:30]) except ValueError: tn = 0 try: tz = float(line[30:40]) except ValueError: tz = 0 t.append(tt) e.append(te) n.append(tn) z.append(tz) count = count + 1 traceE = obspy.core.trace.Trace(np.array(e)) traceN = obspy.core.trace.Trace(np.array(n)) traceZ = obspy.core.trace.Trace(np.array(z)) for i, trace in enumerate([traceE, traceN, traceZ]): try: trace.stats.network = network trace.stats.station = station_code trace.stats.starttime = start_time trace.stats.sampling_rate = samplerate trace.stats.npts = len(t) trace.stats.delta = t[1] - t[0] except IndexError: print(list) if i == 0: trace.stats.channel = 'EHE' if i == 1: trace.stats.channel = 'EHN' if i == 2: trace.stats.channel = 'EHZ' st = obspy.core.stream.Stream([traceE, traceN, traceZ]) progbar.add(1) st.write(f'{save_dir}{filename}.mseed', format='MSEED') f.close() except ValueError: print(list)
def evaluate_generator(model, generator, steps=None, max_queue_size=10, workers=1, use_multiprocessing=False, verbose=0): """See docstring for `Model.evaluate_generator`.""" if not context.executing_eagerly(): model._make_test_function() if hasattr(model, '_compile_metrics'): for m in model.metrics: m.reset_states() steps_done = 0 all_outs = [] batch_sizes = [] is_sequence = isinstance(generator, data_utils.Sequence) if not is_sequence and use_multiprocessing and workers > 1: logging.warning( UserWarning('Using a generator with `use_multiprocessing=True`' ' and multiple workers may duplicate your data.' ' Please consider using the`keras.utils.Sequence' ' class.')) if steps is None: if is_sequence: steps = len(generator) else: raise ValueError('Please specify the `steps` argument.') enqueuer = None try: if workers > 0: if is_sequence: enqueuer = data_utils.OrderedEnqueuer( generator, use_multiprocessing=use_multiprocessing) else: enqueuer = data_utils.GeneratorEnqueuer( generator, use_multiprocessing=use_multiprocessing) enqueuer.start(workers=workers, max_queue_size=max_queue_size) output_generator = enqueuer.get() else: if is_sequence: output_generator = data_utils.iter_sequence_infinite(generator) else: output_generator = generator if verbose == 1: progbar = Progbar(target=steps) while steps_done < steps: generator_output = next(output_generator) if not hasattr(generator_output, '__len__'): raise ValueError('Output of generator should be a tuple ' '(x, y, sample_weight) ' 'or (x, y). Found: ' + str(generator_output)) if len(generator_output) == 2: x, y = generator_output sample_weight = None elif len(generator_output) == 3: x, y, sample_weight = generator_output else: raise ValueError('Output of generator should be a tuple ' '(x, y, sample_weight) ' 'or (x, y). Found: ' + str(generator_output)) outs = model.test_on_batch(x, y, sample_weight=sample_weight) if isinstance(x, list): batch_size = int(x[0].shape[0]) elif isinstance(x, dict): batch_size = int(list(x.values())[0].shape[0]) else: batch_size = int(x.shape[0]) if batch_size == 0: raise ValueError('Received an empty batch. ' 'Batches should at least contain one item.') all_outs.append(outs) steps_done += 1 batch_sizes.append(batch_size) if verbose == 1: progbar.update(steps_done) except (errors.OutOfRangeError, StopIteration): logging.warning( 'Your dataset iterator ran out of data interrupting testing. ' 'Make sure that your dataset can generate at least `steps` ' 'batches (in this case, %d batches). You may need to use the ' 'repeat() function when building your dataset.', steps) finally: if enqueuer is not None: enqueuer.stop() if not isinstance(outs, list): return np.average(np.asarray(all_outs), weights=batch_sizes) else: averages = [float(all_outs[-1][0])] # index 0 = 'loss' averages.extend([ np.average([out[i] for out in all_outs], weights=batch_sizes) for i in range(1, len(outs)) ]) return averages