def split_data(training_data): X_q, X_p, X_n, Y = training_data X = [np.asarray(X_q), np.asarray(X_p), np.asarray(X_n)] Y = np.array(Y) split_at = int(len(Y) * (1.0 - 0.1)) train_X, val_X = training._slice_arrays(X, 0, split_at), training._slice_arrays( X, split_at) train_Y = Y[0:split_at] val_Y = Y[split_at:] #train_Y, val_Y = training._slice_arrays(Y, 0, split_at), training._slice_arrays(Y, split_at) return train_X, val_X, train_Y, val_Y
def split_train_and_valid(cls, data, mask, validation_split, shuffle=False): print('Shuffle & split...') if shuffle: data, mask = cls.shuffle_train(data, mask) split_at = int(len(data) * (1. - validation_split)) x_train, x_valid = (_slice_arrays(data, 0, split_at), _slice_arrays(data, split_at)) y_train, y_valid = (_slice_arrays(mask, 0, split_at), _slice_arrays(mask, split_at)) cls.save_valid_idx(range(len(data))[split_at:]) # print('x_train像素点之和:{}'.format(x_train.sum())) # print('y_train_mask像素点之和:{}'.format(y_train.sum())) return (x_train, y_train), (x_valid, y_valid)
def test_slice_arrays(): input_a = np.random.random((10, 3)) _slice_arrays(None) _slice_arrays(input_a, 0) _slice_arrays(input_a, 0, 1) _slice_arrays(input_a, stop=2) input_a = [None, [1, 1], None, [1, 1]] _slice_arrays(input_a, 0) _slice_arrays(input_a, 0, 1) _slice_arrays(input_a, stop=2) input_a = [None] _slice_arrays(input_a, 0) _slice_arrays(input_a, 0, 1) _slice_arrays(input_a, stop=2) input_a = None _slice_arrays(input_a, 0) _slice_arrays(input_a, 0, 1) _slice_arrays(input_a, stop=2)
X = np.zeros((len(questions), MAXLEN, len(chars)), dtype=np.bool) y = np.zeros((len(questions), DIGITS + 1, len(chars)), dtype=np.bool) for i, sentence in enumerate(questions): X[i] = ctable.encode(sentence, maxlen=MAXLEN) for i, sentence in enumerate(expected): y[i] = ctable.encode(sentence, maxlen=DIGITS + 1) # Shuffle (X, y) in unison as the later parts of X will almost all be larger digits indices = np.arange(len(y)) np.random.shuffle(indices) X = X[indices] y = y[indices] # Explicitly set apart 20% for validation data that we never train over split_at = int(len(X) - len(X) / 20) (X_train, X_val) = (_slice_arrays(X, 0, split_at), _slice_arrays(X, split_at)) (y_train, y_val) = (y[:split_at], y[split_at:]) print(X_train.shape) print(y_train.shape) print('Build model...') # from https://gist.github.com/rouseguy/1122811f2375064d009dac797d59bae9 model = Sequential() # "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE # note: in a situation where your input sequences have a variable length, # use input_shape=(None, nb_feature). model.add(RNN(HIDDEN_SIZE, return_sequences=True, input_shape=(MAXLEN, len(chars)))) # todo : for trying multi-layer
def _fit_loop(self, f, ins, out_labels=None, batch_size=32, epochs=100, verbose=1, callbacks=None, val_f=None, val_ins=None, shuffle=True, callback_metrics=None, initial_epoch=0, steps_per_epoch=None, validation_steps=None): """Abstract fit function for f(ins). Assume that f returns a list, labeled by out_labels. # Arguments f: Keras function returning a list of tensors ins: List of tensors to be fed to `f` out_labels: List of strings, display names of the outputs of `f` batch_size: Integer batch size or None if unknown. epochs: Number of times to iterate over the data verbose: Verbosity mode, 0, 1 or 2 callbacks: List of callbacks to be called during training val_f: Keras function to call for validation val_ins: List of tensors to be fed to `val_f` shuffle: Whether to shuffle the data at the beginning of each epoch callback_metrics: List of strings, the display names of the metrics passed to the callbacks. They should be the concatenation of list the display names of the outputs of `f` and the list of display names of the outputs of `f_val`. initial_epoch: Epoch at which to start training (useful for resuming a previous training run) steps_per_epoch: Total number of steps (batches of samples) before declaring one epoch finished and starting the next epoch. Ignored with the default value of `None`. validation_steps: Number of steps to run validation for (only if doing validation from data tensors). Ignored with the default value of `None`. # Returns `History` object. [A tweaked version.] """ do_validation = False if val_f and val_ins: do_validation = True if verbose and ins and hasattr(ins[0], 'shape') and hasattr( val_ins[0], 'shape'): print('Train on %d samples, validate on %d samples' % (ins[0].shape[0], val_ins[0].shape[0])) if validation_steps: do_validation = True if steps_per_epoch is None: raise ValueError('Can only use `validation_steps` ' 'when doing step-wise ' 'training, i.e. `steps_per_epoch` ' 'must be set.') num_train_samples = self._check_num_samples(ins, batch_size, steps_per_epoch, 'steps_per_epoch') if num_train_samples is not None: index_array = np.arange(num_train_samples) self.history = cbks.History() callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history] if verbose: if steps_per_epoch is not None: count_mode = 'steps' else: count_mode = 'samples' callbacks += [cbks.ProgbarLogger(count_mode)] callbacks = cbks.CallbackList(callbacks) out_labels = out_labels or [] # it's possible to callback a different model than self # (used by Sequential models) if hasattr(self, 'callback_model') and self.callback_model: callback_model = self.callback_model else: callback_model = self callbacks.set_model(callback_model) callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, 'steps': steps_per_epoch, 'samples': num_train_samples, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics or [], }) callbacks.on_train_begin() callback_model.stop_training = False # for cbk in callbacks: # cbk.validation_data = val_ins for epoch in range(initial_epoch, epochs): callbacks.on_epoch_begin(epoch) epoch_logs = {} if steps_per_epoch is not None: for step_index in range(steps_per_epoch): batch_logs = {} batch_logs['batch'] = step_index batch_logs['size'] = 1 callbacks.on_batch_begin(step_index, batch_logs) outs = f(ins) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(step_index, batch_logs) if callback_model.stop_training: break if do_validation: val_outs = self._test_loop(val_f, val_ins, batch_size=batch_size, steps=validation_steps, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o else: if shuffle == 'batch': index_array = _batch_shuffle(index_array, batch_size) elif shuffle: np.random.shuffle(index_array) batches = _make_batches(num_train_samples, batch_size) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] try: if isinstance(ins[-1], float): # do not slice the training phase flag ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = _slice_arrays(ins, batch_ids) except TypeError: raise TypeError('TypeError while preparing batch. ' 'If using HDF5 input data, ' 'pass shuffle="batch".') batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = len(batch_ids) batch_logs['ids'] = batch_ids callbacks.on_batch_begin(batch_index, batch_logs) outs = f(ins_batch) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) if callback_model.stop_training: break if batch_index == len(batches) - 1: # last batch. if do_validation: val_outs = self._test_loop(val_f, val_ins, batch_size=batch_size, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # same labels assumed for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) if callback_model.stop_training: break callbacks.on_train_end() return self.history
def fit(model, x, y, batch_size=32, nb_epoch=10, verbose=1, callbacks=[], validation_split=0., validation_data=None, shuffle=True, class_weight=None, sample_weight=None): '''Trains the model for a fixed number of epochs (iterations on a dataset). # Arguments x: Numpy array of training data, or list of Numpy arrays if the model has multiple inputs. If all inputs in the model are named, you can also pass a dictionary mapping input names to Numpy arrays. y: Numpy array of target data, or list of Numpy arrays if the model has multiple outputs. If all outputs in the model are named, you can also pass a dictionary mapping output names to Numpy arrays. batch_size: integer. Number of samples per gradient update. nb_epoch: integer, the number of times to iterate over the training data arrays. verbose: 0, 1, or 2. Verbosity mode. 0 = silent, 1 = verbose, 2 = one log line per epoch. callbacks: list of callbacks to be called during training. See [callbacks](/callbacks). validation_split: float between 0 and 1: fraction of the training data to be used as validation data. The model will set apart this fraction of the training data, will not train on it, and will evaluate the loss and any model metrics on this data at the end of each epoch. validation_data: data on which to evaluate the loss and any model metrics at the end of each epoch. The model will not be trained on this data. This could be a tuple (x_val, y_val) or a tuple (val_x, val_y, val_sample_weights). shuffle: boolean, whether to shuffle the training data before each epoch. class_weight: optional dictionary mapping class indices (integers) to a weight (float) to apply to the model's loss for the samples from this class during training. This can be useful to tell the model to "pay more attention" to samples from an under-represented class. sample_weight: optional array of the same length as x, containing weights to apply to the model's loss for each sample. In the case of temporal data, you can pass a 2D array with shape (samples, sequence_length), to apply a different weight to every timestep of every sample. In this case you should make sure to specify sample_weight_mode="temporal" in compile(). # Returns A `History` instance. Its `history` attribute contains all information collected during training. ''' # validate user data ''' We need to use the custom standardize_user_data function defined above to skip checking for the array_length. Everything else is the same as the original fit code Note: We did not use model._standardize_user_data(...) but instead used the above standardize_user_data(...) method to bypass the original standardize code in keras. ''' x, y, sample_weights = _standardize_user_data(model, x, y, sample_weight=sample_weight, class_weight=class_weight, check_batch_dim=False, batch_size=batch_size) # prepare validation data if validation_data: do_validation = True if len(validation_data) == 2: val_x, val_y = validation_data val_sample_weight = None elif len(validation_data) == 3: val_x, val_y, val_sample_weight = validation_data else: raise val_x, val_y, val_sample_weights = model._standardize_user_data( val_x, val_y, sample_weight=val_sample_weight, check_batch_dim=False, batch_size=batch_size) model._make_test_function() val_f = model.test_function if model.uses_learning_phase and type(K.learning_phase()) is not int: val_ins = val_x + val_y + val_sample_weights + [0.] else: val_ins = val_x + val_y + val_sample_weights elif validation_split and 0. < validation_split < 1.: do_validation = True split_at = int(len(x[0]) * (1. - validation_split)) x, val_x = (_slice_arrays(x, 0, split_at), _slice_arrays(x, split_at)) y, val_y = (_slice_arrays(y, 0, split_at), _slice_arrays(y, split_at)) sample_weights, val_sample_weights = (_slice_arrays( sample_weights, 0, split_at), _slice_arrays(sample_weights, split_at)) model._make_test_function() val_f = model.test_function if model.uses_learning_phase and type(K.learning_phase()) is not int: val_ins = val_x + val_y + val_sample_weights + [0.] else: val_ins = val_x + val_y + val_sample_weights else: do_validation = False val_f = None val_ins = None # prepare input arrays and training function if model.uses_learning_phase and type(K.learning_phase()) is not int: ins = x + y + sample_weights + [1.] else: ins = x + y + sample_weights model._make_train_function() f = model.train_function # prepare display labels out_labels = model.metrics_names # rename duplicated metrics name # (can happen with an output layer shared among multiple dataflows) deduped_out_labels = [] for i, label in enumerate(out_labels): new_label = label if out_labels.count(label) > 1: dup_idx = out_labels[:i].count(label) new_label += '_' + str(dup_idx + 1) deduped_out_labels.append(new_label) out_labels = deduped_out_labels if do_validation: callback_metrics = copy.copy(out_labels) + [ 'val_' + n for n in out_labels ] else: callback_metrics = copy.copy(out_labels) # delegate logic to _fit_loop return model._fit_loop(f, ins, out_labels=out_labels, batch_size=batch_size, nb_epoch=nb_epoch, verbose=verbose, callbacks=callbacks, val_f=val_f, val_ins=val_ins, shuffle=shuffle, callback_metrics=callback_metrics)
def test_slice_arrays(): input_a = np.random.random((10, 3)) _slice_arrays(None) _slice_arrays(input_a, 0) _slice_arrays(input_a, 0, 1) _slice_arrays(input_a, stop=2) input_a = [None, [1, 1], None, [1, 1]] _slice_arrays(input_a, 0) _slice_arrays(input_a, 0, 1) _slice_arrays(input_a, stop=2) input_a = [None] _slice_arrays(input_a, 0) _slice_arrays(input_a, 0, 1) _slice_arrays(input_a, stop=2) input_a = None _slice_arrays(input_a, 0) _slice_arrays(input_a, 0, 1) _slice_arrays(input_a, stop=2)
def fit_tfrecord(self, x=None, y=None, batch_size=32, epochs=1, verbose=1, callbacks=None, validation_split=0., validation_data=None, shuffle=True, class_weight=None, sample_weight=None, initial_epoch=0, steps_per_epoch=None, **kwargs): """Trains the model for a fixed number of epochs (iterations on a dataset). # Arguments x: Numpy array of training data, or list of Numpy arrays if the model has multiple inputs. If all inputs in the model are named, you can also pass a dictionary mapping input names to Numpy arrays. y: Numpy array of target data, or list of Numpy arrays if the model has multiple outputs. If all outputs in the model are named, you can also pass a dictionary mapping output names to Numpy arrays. batch_size: integer. Number of samples per gradient update. epochs: integer, the number of times to iterate over the training data arrays. verbose: 0, 1, or 2. Verbosity mode. 0 = silent, 1 = verbose, 2 = one log line per epoch. callbacks: list of callbacks to be called during training. See [callbacks](/callbacks). validation_split: float between 0 and 1: fraction of the training data to be used as validation data. The model will set apart this fraction of the training data, will not train on it, and will evaluate the loss and any model metrics on this data at the end of each epoch. validation_data: data on which to evaluate the loss and any model metrics at the end of each epoch. The model will not be trained on this data. This could be a tuple (x_val, y_val) or a tuple (x_val, y_val, val_sample_weights). shuffle: boolean, whether to shuffle the training data before each epoch. class_weight: optional dictionary mapping class indices (integers) to a weight (float) to apply to the model's loss for the samples from this class during training. This can be useful to tell the model to "pay more attention" to samples from an under-represented class. sample_weight: optional array of the same length as x, containing weights to apply to the model's loss for each sample. In the case of temporal data, you can pass a 2D array with shape (samples, sequence_length), to apply a different weight to every timestep of every sample. In this case you should make sure to specify sample_weight_mode="temporal" in compile(). initial_epoch: epoch at which to start training (useful for resuming a previous training run) steps_per_epoch: Total number of steps (batches of samples) before declaring one epoch finished and starting the next epoch. The default `None` is equal to the number of unique samples in your dataset divided by the batch size, or 1 if that cannot be determined. # Returns A `History` instance. Its `history` attribute contains all information collected during training. # Raises ValueError: In case of mismatch between the provided input data and what the model expects. """ # Legacy support if 'nb_epoch' in kwargs: warnings.warn( 'The `nb_epoch` argument in `fit` ' 'has been renamed `epochs`.', stacklevel=2) epochs = kwargs.pop('nb_epoch') if kwargs: raise TypeError('Unrecognized keyword arguments: ' + str(kwargs)) # Validate user data. x, y, sample_weights = self._standardize_user_data( x, y, sample_weight=sample_weight, class_weight=class_weight, check_batch_axis=False, batch_size=batch_size) # Prepare validation data. if validation_data: do_validation = True if len(validation_data) == 2: val_x, val_y = validation_data val_sample_weight = None elif len(validation_data) == 3: val_x, val_y, val_sample_weight = validation_data else: raise ValueError('When passing validation_data, ' 'it must contain 2 (x_val, y_val) ' 'or 3 (x_val, y_val, val_sample_weights) ' 'items, however it contains %d items' % len(validation_data)) val_x, val_y, val_sample_weights = self._standardize_user_data( val_x, val_y, sample_weight=val_sample_weight, check_batch_axis=False, batch_size=batch_size) self._make_test_function() val_f = self.test_function if self.uses_learning_phase and \ not isinstance(K.learning_phase(), int): val_ins = val_x + val_y + val_sample_weights + [0.] else: val_ins = val_x + val_y + val_sample_weights elif validation_split and 0. < validation_split < 1.: do_validation = True if hasattr(x[0], 'shape'): split_at = int(x[0].shape[0] * (1. - validation_split)) else: split_at = int(len(x[0]) * (1. - validation_split)) x, val_x = (_slice_arrays(x, 0, split_at), _slice_arrays(x, split_at)) y, val_y = (_slice_arrays(y, 0, split_at), _slice_arrays(y, split_at)) sample_weights, val_sample_weights = (_slice_arrays( sample_weights, 0, split_at), _slice_arrays(sample_weights, split_at)) self._make_test_function() val_f = self.test_function if self.uses_learning_phase and \ not isinstance(K.learning_phase(), int): val_ins = val_x + val_y + val_sample_weights + [0.] else: val_ins = val_x + val_y + val_sample_weights else: do_validation = False val_f = None val_ins = None # Prepare input arrays and training function. if self.uses_learning_phase and \ not isinstance(K.learning_phase(), int): ins = x + y + sample_weights + [1.] else: ins = x + y + sample_weights self._make_train_function() f = self.train_function # Prepare display labels. out_labels = self._get_deduped_metrics_names() if do_validation: callback_metrics = \ copy.copy(out_labels) + ['val_' + n for n in out_labels] else: callback_metrics = copy.copy(out_labels) # Delegate logic to `_fit_loop`. return self._fit_loop(f, ins, out_labels=out_labels, batch_size=batch_size, epochs=epochs, verbose=verbose, callbacks=callbacks, val_f=val_f, val_ins=val_ins, shuffle=shuffle, callback_metrics=callback_metrics, initial_epoch=initial_epoch, steps_per_epoch=steps_per_epoch)
def _fit_loop(self, f, ins, out_labels=None, batch_size=32, epochs=100, verbose=1, callbacks=None, val_f=None, val_ins=None, shuffle=True, callback_metrics=None, initial_epoch=0, steps_per_epoch=None): """Abstract fit function for `f(ins)`. Assume that f returns a list, labeled by out_labels. # Arguments f: Keras function returning a list of tensors ins: list of tensors to be fed to `f` out_labels: list of strings, display names of the outputs of `f` batch_size: integer batch size epochs: number of times to iterate over the data verbose: verbosity mode, 0, 1 or 2 callbacks: list of callbacks to be called during training val_f: Keras function to call for validation val_ins: list of tensors to be fed to `val_f` shuffle: whether to shuffle the data at the beginning of each epoch callback_metrics: list of strings, the display names of the metrics passed to the callbacks. They should be the concatenation of list the display names of the outputs of `f` and the list of display names of the outputs of `f_val`. initial_epoch: epoch at which to start training (useful for resuming a previous training run) steps_per_epoch: Total number of steps (batches of samples) before declaring one epoch finished and starting the next epoch. The default `None` is equal to the number of unique samples in your dataset divided by the batch size, or 1 if that cannot be determined. # Returns `History` object. """ do_validation = False if val_f and val_ins: do_validation = True if verbose and ins and hasattr(ins[0], 'shape'): print('Train on %d samples, validate on %d samples' % (ins[0].shape[0], val_ins[0].shape[0])) if steps_per_epoch is not None: num_train_samples = steps_per_epoch else: if ins and hasattr(ins[0], 'shape'): num_train_samples = ins[0].shape[0] else: # May happen if we are running `fit` without Numpy input data, # i.e. if all inputs to the models are data tensors # instead of placeholders. # In that case we will run `fit` over a single batch. num_train_samples = batch_size verbose = 2 index_array = np.arange(num_train_samples) self.history = cbks.History() callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history] if verbose: # callbacks += [cbks.ProgbarLogger()] callbacks += [ProgbarLogger_TFRecord()] callbacks = cbks.CallbackList(callbacks) out_labels = out_labels or [] # it's possible to callback a different model than self # (used by Sequential models) if hasattr(self, 'callback_model') and self.callback_model: callback_model = self.callback_model else: callback_model = self callbacks.set_model(callback_model) callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, 'samples': num_train_samples, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics or [], }) callbacks.on_train_begin() callback_model.stop_training = False for cbk in callbacks: cbk.validation_data = val_ins for epoch in range(initial_epoch, epochs): callbacks.on_epoch_begin(epoch) if shuffle == 'batch': index_array = _batch_shuffle(index_array, batch_size) elif shuffle: np.random.shuffle(index_array) batches = _make_batches(num_train_samples, batch_size) epoch_logs = {} for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] try: if isinstance(ins[-1], float): # Do not slice the training phase flag. ins_batch = \ _slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = _slice_arrays(ins, batch_ids) except TypeError: raise TypeError('TypeError while preparing batch. ' 'If using HDF5 input data, ' 'pass shuffle="batch".') batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = len(batch_ids) callbacks.on_batch_begin(batch_index, batch_logs) outs = f(ins_batch) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) if callback_model.stop_training: break if batch_index == len(batches) - 1: # Last batch. if do_validation: val_outs = self._test_loop(val_f, val_ins, batch_size=batch_size, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) if callback_model.stop_training: break callbacks.on_train_end() return self.history
def batch_data(self, model, batch_size): if self.batched == False: x, y, sample_weights = model._standardize_user_data( self.X_train, self.Y_train, sample_weight=None, class_weight=None, check_batch_axis=False, batch_size=batch_size) val_x, val_y, val_sample_weights = model._standardize_user_data( self.X_test, self.Y_test, sample_weight=None, check_batch_axis=False, batch_size=batch_size) ins = x + y + sample_weights val_ins = val_x + val_y + val_sample_weights if model.uses_learning_phase and not isinstance( K.learning_phase(), int): ins += [1.] val_ins += [0.] from keras.engine.training import _slice_arrays self.n_batch_train = ins[0].shape[0] // batch_size self.train_batches = [] index_arr = range(ins[0].shape[0]) for batch_index in range(self.n_batch_train): batch_ids = index_arr[batch_index * batch_size:(batch_index + 1) * batch_size] if isinstance(ins[-1], float): # do not slice the training phase flag ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = _slice_arrays(ins, batch_ids) self.train_batches.append(ins_batch) self.n_batch_val = val_ins[0].shape[0] // batch_size self.val_batches = [] index_arr = range(val_ins[0].shape[0]) for batch_index in range(self.n_batch_val): batch_ids = index_arr[batch_index * batch_size:(batch_index + 1) * batch_size] if isinstance(val_ins[-1], float): # do not slice the training phase flag ins_batch = _slice_arrays(val_ins[:-1], batch_ids) + [val_ins[-1]] else: ins_batch = _slice_arrays(val_ins, batch_ids) self.val_batches.append(ins_batch) self.batched = True
def batch_data(self, model, batch_size): if self.batched==False: x, y, sample_weights = model._standardize_user_data( self.X_train, self.Y_train, sample_weight=None, class_weight=None, check_batch_axis=False, batch_size=batch_size) val_x, val_y, val_sample_weights = model._standardize_user_data( self.X_test, self.Y_test, sample_weight=None, check_batch_axis=False, batch_size=batch_size) ins = x + y + sample_weights val_ins = val_x + val_y + val_sample_weights if model.uses_learning_phase and not isinstance(K.learning_phase(), int): ins+=[1.] val_ins+=[0.] from keras.engine.training import _slice_arrays self.n_batch_train = ins[0].shape[0]//batch_size self.train_batches = [] index_arr = range(ins[0].shape[0]) for batch_index in range(self.n_batch_train): batch_ids = index_arr[batch_index * batch_size: (batch_index+1)*batch_size] if isinstance(ins[-1], float): # do not slice the training phase flag ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = _slice_arrays(ins, batch_ids) self.train_batches.append(ins_batch) self.n_batch_val = val_ins[0].shape[0]//batch_size self.val_batches = [] index_arr = range(val_ins[0].shape[0]) for batch_index in range(self.n_batch_val): batch_ids = index_arr[batch_index * batch_size: (batch_index+1)*batch_size] if isinstance(val_ins[-1], float): # do not slice the training phase flag ins_batch = _slice_arrays(val_ins[:-1], batch_ids) + [val_ins[-1]] else: ins_batch = _slice_arrays(val_ins, batch_ids) self.val_batches.append(ins_batch) self.batched=True
X_new, y_new = np.array(X_new), np.array(y_new) # Shuffle (X_new, y_new) in unison as the later parts of X_new # will almost all be larger digits indices = np.arange(len(y_new)) np.random.shuffle(indices) X_new = X_new[indices] y_new = y_new[indices] # Save memory! del indices # Explicitly set apart 10% for test data that we never train over split_at = int(len(X_new) - len(X_new) / 10) (X_train_new, X_test_new) = (_slice_arrays(X_new, 0, split_at), _slice_arrays(X_new, split_at)) (y_train_new, y_test_new) = (y_new[:split_at], y_new[split_at:]) # Save memory!! del X_new, y_new print(X_train_new.shape, y_train_new.shape) # (You don't need to change this part of the code) from __future__ import print_function import numpy as np np.random.seed(1234) import keras
def _fit_loop(self, f: callable, ins: List[numpy.array], out_labels: List[str] = None, batch_size: int = 32, epochs: int = 100, verbose: int = 1, callbacks: List[Callback] = None, val_f: callable = None, val_ins: List[numpy.array] = None, shuffle: bool = True, callback_metrics: List[str] = None, initial_epoch: int = 0): """ Abstract fit function which preprocesses and batches data before training a model. We override this keras backend function to support multi-gpu training via splitting a large batch size across multiple gpus. This function is broadly the same as the Keras backend version aside from this - changed elements have corresponding comments attached. Note that this should not be called directly - it is used by calling model.fit(). Assume that step_function returns a list, labeled by out_labels. Parameters ---------- f: A callable ``Step`` or a Keras ``Function``, required. A DeepQA Step or Keras Function returning a list of tensors. ins: List[numpy.array], required. The list of tensors to be fed to ``step_function``. out_labels: List[str], optional (default = None). The display names of the outputs of ``step_function``. batch_size: int, optional (default = 32). The integer batch size. epochs: int, optional (default = 100). Number of times to iterate over the data. verbose: int, optional, (default = 1) Verbosity mode, 0, 1 or 2. callbacks: List[Callback], optional (default = None). A list of Keras callbacks to be called during training. val_f: A callable ``Step`` or a Keras ``Function``, optional (default = None). The Keras function to call for validation. val_ins: List[numpy.array], optional (default) A list of tensors to be fed to ``val_f``. shuffle: bool, optional (default = True). whether to shuffle the data at the beginning of each epoch callback_metrics: List[str], optional, (default = None). A list of strings, the display names of the validation metrics. passed to the callbacks. They should be the concatenation of list the display names of the outputs of ``f`` and the list of display names of the outputs of ``f_val``. initial_epoch: int, optional (default = 0). The epoch at which to start training (useful for resuming a previous training run). Returns ------- A Keras ``History`` object. """ do_validation = False if val_f and val_ins: do_validation = True if verbose: print('Train on %d samples, validate on %d samples' % (ins[0].shape[0], val_ins[0].shape[0])) if ins and hasattr(ins[0], 'shape'): num_train_samples = ins[0].shape[0] else: # May happen if we are running `fit` without Numpy input data, # i.e. if all inputs to the models are data tensors # instead of placeholders. # In that case we will run `fit` over a single batch. num_train_samples = batch_size verbose = 2 index_array = numpy.arange(num_train_samples) out_labels = out_labels or [] callbacks, callback_model = self._prepare_callbacks( callbacks, val_ins, epochs, batch_size, num_train_samples, callback_metrics, do_validation, verbose) for epoch in range(initial_epoch, epochs): callbacks.on_epoch_begin(epoch) if shuffle == 'batch': index_array = _batch_shuffle(index_array, batch_size) elif shuffle: numpy.random.shuffle(index_array) batches = _make_batches(num_train_samples, batch_size) epoch_logs = {} for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] try: if isinstance(ins[-1], float): # Do not slice the training phase flag. ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = _slice_arrays(ins, batch_ids) except TypeError: raise TypeError('TypeError while preparing batch. ' 'If using HDF5 input data, ' 'pass shuffle="batch".') # Here is the main difference between a single gpu model and one split # across multiple gpus. In our multiple gpu model, all of the inputs # are replicated num_gpus times, so we need to split our large batch # into the corresponding sets of smaller batches for each model. if self.num_gpus > 1: # The Keras learning phase is a global variable used across model towers. # If it is present, we remove it before splitting up the inputs # and add it back on afterwards. if isinstance(ins_batch[-1], float): model_inputs = self._multi_gpu_batch(ins_batch[:-1]) model_inputs.append(ins_batch[-1]) else: model_inputs = self._multi_gpu_batch(ins_batch) ins_batch = model_inputs batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = len(batch_ids) callbacks.on_batch_begin(batch_index, batch_logs) outs = f(ins_batch) if not isinstance(outs, list): outs = [outs] for label, output in zip(out_labels, outs): batch_logs[label] = output callbacks.on_batch_end(batch_index, batch_logs) if batch_index == len(batches) - 1: # Last batch. if do_validation: # If we are using multiple gpus, our batch size will be # scaled up accordingly. However, validation will run # on a single gpu, so we divide by the number of gpus # to avoid OOM errors. if self.num_gpus > 1: val_batch_size = int(batch_size / self.num_gpus) # pylint: disable=no-member else: val_batch_size = batch_size val_outs = self._test_loop(val_f, val_ins, batch_size=val_batch_size, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for label, output in zip(out_labels, val_outs): epoch_logs['val_' + label] = output callbacks.on_epoch_end(epoch, epoch_logs) if callback_model.stop_training: # pylint: disable=no-member break callbacks.on_train_end() return self.history