def test_validate_callbacks_predefined_callbacks(self): supported_predefined_callbacks = [ callbacks.TensorBoard(), callbacks.CSVLogger(filename='./log.csv'), callbacks.EarlyStopping(), callbacks.ModelCheckpoint(filepath='./checkpoint'), callbacks.TerminateOnNaN(), callbacks.ProgbarLogger(), callbacks.History(), callbacks.RemoteMonitor() ] distributed_training_utils_v1.validate_callbacks( supported_predefined_callbacks, adam.Adam()) unsupported_predefined_callbacks = [ callbacks.ReduceLROnPlateau(), callbacks.LearningRateScheduler(schedule=lambda epoch: 0.001) ] for callback in unsupported_predefined_callbacks: with self.assertRaisesRegex( ValueError, 'You must specify a Keras Optimizer V2'): distributed_training_utils_v1.validate_callbacks( [callback], tf.compat.v1.train.AdamOptimizer())
def __init__(self, model, optimizer, comm, batch_iterator, batch_size, num_replicas=None, warmup_steps=1000, lr=0.01, num_batches_minimum=100): # random.seed(task_index) self.epoch = 0 self.num_so_far = 0 self.num_so_far_accum = 0 self.num_so_far_indiv = 0 self.model = model self.optimizer = optimizer self.max_lr = 0.1 self.DUMMY_LR = 0.001 self.comm = comm self.batch_size = batch_size self.batch_iterator = batch_iterator self.set_batch_iterator_func() self.warmup_steps = warmup_steps self.num_batches_minimum = num_batches_minimum self.num_workers = comm.Get_size() self.task_index = comm.Get_rank() self.history = cbks.History() if num_replicas is None or num_replicas < 1 or num_replicas > self.num_workers: self.num_replicas = self.num_workers else: self.num_replicas = num_replicas self.lr = lr / (1.0 + self.num_replicas / 100.0) if ( lr < self.max_lr) else self.max_lr / (1.0 + self.num_replicas / 100.0)
def setup_callback_list(self, model_name): if model_name in self.callback_lists: return self.callback_lists[model_name] model = self.models[model_name] callbacks = self.callbacks[model_name] \ if model_name in self.callbacks else [] # Prepare callbacks for autoencoder model all_callbacks = [cbks.BaseLogger()] + callbacks + [cbks.History()] all_callbacks = cbks.CallbackList(all_callbacks) out_labels = model.metrics_names if self.do_validation: callback_metrics = copy.copy(out_labels) + \ ["val_" + l for l in out_labels] else: callback_metrics = copy.copy(out_labels) callback_list = cbks.CallbackList(all_callbacks) callback_list.set_params({ 'batch_size': self.batch_size, 'epochs': self.epochs, 'verbose': 2, 'do_validation': model_name in self.do_validation, 'metrics': callback_metrics or [], }) callback_list.set_model(model) return callback_list
def cb(): # pl = callbacks.ProgbarLogger(count_mode='steps') history = callbacks.History() ch = callbacks.ModelCheckpoint( './weights/weights.{epoch:02d}-{val_loss:.2f}.hdf5', monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=False, mode='auto', period=1) es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=0, mode='auto') cb = [history, ch, es] #学習率の変更をしたいならこれ #keras.callbacks.LearningRateScheduler(schedule, verbose=0) #テンソルボードに記述したいならこれ #keras.callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=32, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) #評価値の改善が止まった時に学習率を減らします. #モデルは訓練が停滞した時に学習率を2〜10で割ることで恩恵を受けることがあります. このコールバックは評価値を監視し,'patience'で指定されたエポック数の間改善が見られなかった場合,学習率を減らします. #keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=10, verbose=0, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0) return cb, history
def __init__(self, filepath, verbose=0, period=1, targets=None, is_each=True): super(HistoryCheckpoint, self).__init__() self.__verbose = verbose self.__filepath = filepath self.__period = period self.__epochs_since_last_save = 0 self.__history_callback = KC.History() self.__targets = [TargetHistory.Loss] if isinstance(targets, list): self.__targets = targets self.__is_each = is_each
def get_callbacks(self, opt): # ModelCheckpoints: saving model after each epoch fn1 = (os.path.basename(self.model_params_file_path).replace( '.json', '')) fn = (f'{fn1}___{self.start_time}' f'___model_%s{"_TEST" if opt.test else ""}.h5' % ('{epoch:02d}')) filepath = os.path.join(self.path_nn_model, self.model.name, fn) del fn checkpoint = callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=opt.verbose) # TerminateOnNaN tonan = callbacks.TerminateOnNaN() # History history = callbacks.History() # CSV logger: saves epoch train and valid loss to a log file fn1 = (os.path.basename(self.model_params_file_path).replace( '.json', '')) fn = (f'{fn1}___{self.start_time}' f'___training{"_TEST" if opt.test else ""}.log') filepath = os.path.join(self.path_nn_model, self.model.name, fn) csv_logger = callbacks.CSVLogger(filepath, separator=',', append=True) # Learning rate scheduler def exp_decay(epoch, initial_lrate=self.model_params['keras_train']['lr'], decay=self.model_params['keras_train']['lr_decay']): lrate = initial_lrate * np.exp(-decay * epoch) return lrate def learning_rate_decay( epoch, initial_lrate=self.model_params['keras_train']['lr'], decay=self.model_params['keras_train']['lr_decay']): lrate = initial_lrate * (1 - decay)**epoch return lrate lrs = callbacks.LearningRateScheduler(learning_rate_decay) callbacks_list = [ tonan, checkpoint, history, csv_logger, csv_logger, lrs ] # Early stopping: stops training if validation loss does not improves if (self.model_params['keras_train'].get('early_stopping_n') is not None): es = callbacks.EarlyStopping( monitor='val_loss', min_delta=0, patience=self.model_params['keras_train']['early_stopping_n'], verbose=opt.verbose) callbacks_list.append(es) return callbacks_list
def fit(self, freeze_indices, optimizers, warmup_epochs=5): # callbacks filepath = 'models/transfer_CNN_reg.h5' mc = callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='auto', period=1) hist = callbacks.History() es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=2, verbose=1, mode='auto') if not os.path.exists('tensorboard_logs/transfer_CNN_tensorboard_reg'): os.makedirs('tensorboard_logs/transfer_CNN_tensorboard_reg') tensorboard = callbacks.TensorBoard( log_dir='tensorboard_logs/transfer_CNN_tensorboard_reg', histogram_freq=0, batch_size=self.batch_size, write_graph=True, embeddings_freq=0, write_images=False) # change head from default self._create_transfer_model() # train head, then chunks histories = [] for i, freeze in enumerate(freeze_indices): if i == 0: e = warmup_epochs opt = optimizers[0] else: e = self.epochs opt = optimizers[1] self._change_trainable_layers(freeze) self.model.compile(optimizer=opt, loss=root_mean_squared_error) history = self.model.fit_generator( self.train_generator, steps_per_epoch=len(self.train_generator), epochs=e, validation_data=self.validation_generator, validation_steps=len(self.validation_generator), callbacks=[mc, tensorboard, hist]) histories.append(history.history) return histories
def _fit(self, f, nb_train_sample, nb_batches, batch_size=128, nb_epoch=100, verbose=1, callbacks=[], shuffle=True, metrics=[]): """ Abstract fit function for f(*ins). Assume that f returns a list, labelled by out_labels. """ history = cbks.History() callbacks = [cbks.BaseLogger()] + callbacks + [history] if verbose: callbacks = callbacks + [cbks.ProgbarLogger()] callbacks = cbks.CallbackList(callbacks) callbacks._set_model(self) callbacks._set_params({ 'batch_size': nb_train_sample // nb_batches, 'nb_epoch': nb_epoch, 'nb_sample': nb_train_sample, 'verbose': verbose, 'do_validation': False, 'metrics': metrics, }) callbacks.on_train_begin() self.stop_training = False for epoch in range(nb_epoch): callbacks.on_epoch_begin(epoch) for batch_index in range(nb_batches): batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = batch_size callbacks.on_batch_begin(batch_index, batch_logs) f(self, batch_index, batch_logs) callbacks.on_batch_end(batch_index, batch_logs) epoch_logs = {} callbacks.on_epoch_end(epoch, epoch_logs) if self.stop_training: break callbacks.on_train_end() return history
def train_model(input_to_softmax, pickle_path, save_model_path, train_json='train_corpus.json', valid_json='valid_corpus.json', minibatch_size=16, # You will want to change this depending on the GPU you are training on spectrogram=True, mfcc_dim=13, optimizer=Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False, clipnorm=1, clipvalue=.5), epochs=30, # You will want to change this depending on the model you are training and data you are using verbose=1, sort_by_duration=False, max_duration=10.0): # Obtain batches of data audio_gen = AudioGenerator(minibatch_size=minibatch_size, spectrogram=spectrogram, mfcc_dim=mfcc_dim, max_duration=max_duration, sort_by_duration=sort_by_duration) # Load the datasets audio_gen.load_train_data(train_json) audio_gen.load_validation_data(valid_json) # Calculate steps per epoch num_train_examples=len(audio_gen.train_audio_paths) steps_per_epoch = num_train_examples//minibatch_size # Calculate validation steps num_valid_samples = len(audio_gen.valid_audio_paths) validation_steps = num_valid_samples//minibatch_size # Add custom CTC loss function to the nn model = add_ctc_loss(input_to_softmax) # Dummy lambda function for loss since CTC loss is implemented above model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=optimizer) # Make initial results/ directory for saving model pickles if not os.path.exists('results'): os.makedirs('results') # Add callbacks checkpointer = ModelCheckpoint(filepath='results/'+save_model_path, verbose=0) terminator = callbacks.TerminateOnNaN() time_machiner = callbacks.History() logger = callbacks.CSVLogger('training.log') tensor_boarder = callbacks.TensorBoard(log_dir='./logs', batch_size=16, write_graph=True, write_grads=True, write_images=True,) # Fit/train model hist = model.fit_generator(generator=audio_gen.next_train(), steps_per_epoch=steps_per_epoch, epochs=epochs, validation_data=audio_gen.next_valid(), validation_steps=validation_steps, callbacks=[checkpointer, terminator, logger, time_machiner, tensor_boarder], verbose=verbose) # Save model loss with open('results/'+pickle_path, 'wb') as f: pickle.dump(hist.history, f)
def fit_generator(self, generator, nb_epoch, nb_batches_per_epoch, callbacks=[], batch_size=None, verbose=False): if batch_size is None: batch_size = 2 * len(next(generator)[0]) out_labels = ['g', 'd', 'm'] self.history = cbks.History() callbacks = [cbks.BaseLogger()] + callbacks + [self.history] if verbose: callbacks += [cbks.ProgbarLogger()] callbacks = cbks.CallbackList(callbacks) callbacks.set_model(self) callbacks.set_params({ 'nb_epoch': nb_epoch, 'nb_sample': nb_batches_per_epoch * batch_size, 'verbose': verbose, 'metrics': out_labels, }) callbacks.on_train_begin() for e in range(nb_epoch): callbacks.on_epoch_begin(e) for batch_index, (seq_input, real) in enumerate(generator): callbacks.on_batch_begin(batch_index) batch_logs = dict() batch_logs['batch'] = batch_index batch_logs['size'] = len(real) + len(seq_input) outs = self.train_on_batch(seq_input, real) for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) if batch_index + 1 == nb_batches_per_epoch: break callbacks.on_epoch_end(e) callbacks.on_train_end()
def __init__(self, model, optimizer, comm, batch_iterator, batch_size, num_replicas=None, warmup_steps=1000, lr=0.01, num_batches_minimum=100, conf=None): random.seed(g.task_index) np.random.seed(g.task_index) self.conf = conf self.start_time = time.time() self.epoch = 0 self.num_so_far = 0 self.num_so_far_accum = 0 self.num_so_far_indiv = 0 self.model = model self.optimizer = optimizer self.max_lr = 0.1 self.DUMMY_LR = 0.001 self.batch_size = batch_size self.batch_iterator = batch_iterator self.set_batch_iterator_func() self.warmup_steps = warmup_steps self.num_batches_minimum = num_batches_minimum # TODO(KGF): duplicate/may be in conflict with global_vars.py self.comm = comm self.num_workers = comm.Get_size() self.task_index = comm.Get_rank() self.history = cbks.History() self.model.stop_training = False if (num_replicas is None or num_replicas < 1 or num_replicas > self.num_workers): self.num_replicas = self.num_workers else: self.num_replicas = num_replicas self.lr = (lr / (1.0 + self.num_replicas / 100.0) if (lr < self.max_lr) else self.max_lr / (1.0 + self.num_replicas / 100.0))
def online_fit(autoencoder, X_train, model_name, test_size=0.2, batch_size = 8, epochs = 100, verbose=True) \ -> Model: ''' Fits the model :param autoencoder: the autoencoder to fit :param X_train: :param model_name: :param test_size: :param batch_size: :param epochs: :param verbose: :return: the fit model ''' dt = datetime.today() currentDate = ''.join([str(dt.year), str(dt.month), str(dt.day)]) bestModelName = ''.join( [currentDate, model_name, '_AutoEncoder', '.h5']) bestModelFilepath = os.path.join('.', 'models', bestModelName) checkpoint = callbacks.ModelCheckpoint(bestModelFilepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min') early_stopping = callbacks.EarlyStopping(monitor='val_loss', patience=3, verbose=True) history = callbacks.History() autoencoder.fit(X_train, X_train, batch_size=batch_size, epochs=epochs, callbacks=[early_stopping, checkpoint, history], verbose=verbose, validation_split=test_size) return autoencoder
def callbacks(model, callbacks, params): model.history = cbks.History() _callbacks = [ cbks.BaseLogger(stateful_metrics=model.stateful_metric_names) ] _callbacks.append( cbks.ProgbarLogger(count_mode='steps', stateful_metrics=model.stateful_metric_names)) _callbacks += (callbacks or []) + [model.history] callbacks = cbks.CallbackList(_callbacks) if hasattr(model, 'callback_model') and model.callback_model: callback_model = model.callback_model else: callback_model = model callbacks.set_model(callback_model) out_labels = model.metrics_names callback_metrics = out_labels + ['val_' + n for n in out_labels] callbacks.set_params({ **params, 'metrics': callback_metrics, }) return callbacks
def fit(self): filepath = 'models/Simple_CNN.h5' mc = callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='auto', period=1) hist = callbacks.History() es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=4, verbose=1, mode='auto') if not os.path.exists('tensorboard_logs/Simple_CNN_tensorboard'): os.makedirs('tensorboard_logs/Simple_CNN_tensorboard') tensorboard = callbacks.TensorBoard( log_dir='tensorboard_logs/Simple_CNN_tensorboard', histogram_freq=0, batch_size=self.batch_size, write_graph=True, embeddings_freq=0, write_images=False) self._find_class_weights() self.history = self.model.fit_generator( self.train_generator, steps_per_epoch=len(self.train_generator), epochs=self.nb_epoch, class_weight=self.class_weights, validation_data=self.validation_generator, validation_steps=len(self.validation_generator), callbacks=[mc, hist, es, tensorboard]) return self.history
def init_callbacks(self, for_worker=False): """Prepares all keras callbacks to be used in training. Automatically attaches a History callback to the end of the callback list. If for_worker is True, leaves out callbacks that only make sense with validation enabled.""" import keras.callbacks as cbks remove_for_worker = [cbks.EarlyStopping, cbks.ModelCheckpoint] if for_worker: for obj in remove_for_worker: self.callbacks_list = [ c for c in self.callbacks_list if not isinstance(c, obj) ] self.model.history = cbks.History() self.callbacks = cbks.CallbackList(self.callbacks_list + [self.model.history]) # it's possible to callback a different model than self # (used by Sequential models) if hasattr(self.model, 'callback_model') and self.model.callback_model: self.callback_model = self.model.callback_model else: self.callback_model = self.model self.callbacks.set_model(self.callback_model) self.callback_model.stop_training = False
""" from __future__ import print_function from keras.models import Sequential from keras.layers import Dense import pandas import matplotlib.pyplot as plt from keras import optimizers from keras import callbacks import numpy import keras.regularizers as regularizers from sklearn.preprocessing import StandardScaler batch_size = 300 history = callbacks.History() print('Loading data...') dataframeTrain = pandas.DataFrame(pandas.read_csv('train.txt', sep = ' ')) dataframeTest = pandas.DataFrame(pandas.read_csv('test.txt', sep = ' ')) train = dataframeTrain.values test = dataframeTest.values # split into input (X) and output (Y) variables X_train = train[0:20000,6:26] y_train = train[0:20000,27] X_test = test[53:209,6:26] y_test = test[53:209,27] #Trainingsdatensatz Standardisieren
def run(GP): # set the seed if GP['seed']: np.random.seed(GP['seed']) else: np.random.seed(np.random.randint(10000)) # Set paths if not os.path.isdir(GP['home_dir']): print('Keras home directory not set') sys.exit(0) sys.path.append(GP['home_dir']) # Setup loggin args = candle.ArgumentStruct(**GP) # set_seed(args.rng_seed) # ext = extension_from_parameters(args) candle.verify_path(args.save_path) prefix = args.save_path # + ext logfile = args.logfile if args.logfile else prefix + '.log' candle.set_up_logger(logfile, logger, False) #args.verbose logger.info('Params: {}'.format(GP)) import p2b1 as hf reload(hf) #import keras_model_utils as KEU #reload(KEU) #reload(p2ck) #reload(p2ck.optimizers) maps = hf.autoencoder_preprocess() from keras.optimizers import SGD, RMSprop, Adam from keras.datasets import mnist from keras.callbacks import LearningRateScheduler, ModelCheckpoint from keras import callbacks from keras.layers.advanced_activations import ELU from keras.preprocessing.image import ImageDataGenerator # GP=hf.ReadConfig(opts.config_file) batch_size = GP['batch_size'] learning_rate = GP['learning_rate'] kerasDefaults = candle.keras_default_config() ##### Read Data ######## import helper (data_files, fields) = p2b1.get_list_of_data_files(GP) # Read from local directoy #(data_files, fields) = helper.get_local_files('/p/gscratchr/brainusr/datasets/cancer/pilot2/3k_run16_10us.35fs-DPPC.20-DIPC.60-CHOL.20.dir/') #(data_files, fields) = helper.get_local_files('3k_run16', '/p/lscratchf/brainusr/datasets/cancer/pilot2/') # Define datagenerator datagen = hf.ImageNoiseDataGenerator(corruption_level=GP['noise_factor']) # get data dimension ## num_samples = 0 for f in data_files: # Seperate different arrays from the data (X, nbrs, resnums) = helper.get_data_arrays(f) num_samples += X.shape[0] (X, nbrs, resnums) = helper.get_data_arrays(data_files[0]) print('\nData chunk shape: ', X.shape) molecular_hidden_layers = GP['molecular_num_hidden'] if not molecular_hidden_layers: X_train = hf.get_data(X, case=GP['case']) input_dim = X_train.shape[1] else: # computing input dimension for outer AE input_dim = X.shape[1] * molecular_hidden_layers[-1] print('\nState AE input/output dimension: ', input_dim) # get data dimension for molecular autoencoder molecular_nbrs = np.int(GP['molecular_nbrs']) num_molecules = X.shape[1] num_beads = X.shape[2] if GP['nbr_type'] == 'relative': # relative x, y, z positions num_loc_features = 3 loc_feat_vect = ['rel_x', 'rel_y', 'rel_z'] elif GP['nbr_type'] == 'invariant': # relative distance and angle num_loc_features = 2 loc_feat_vect = ['rel_dist', 'rel_angle'] else: print('Invalid nbr_type!!') exit() if not GP['type_bool']: # only consider molecular location coordinates num_type_features = 0 type_feat_vect = [] else: num_type_features = 5 type_feat_vect = list(fields.keys())[3:8] num_features = num_loc_features + num_type_features + num_beads dim = np.prod([num_beads, num_features, molecular_nbrs + 1]) bead_kernel_size = num_features molecular_input_dim = dim mol_kernel_size = num_beads feature_vector = loc_feat_vect + type_feat_vect + list(fields.keys())[8:] print('\nMolecular AE input/output dimension: ', molecular_input_dim) print( '\nData Format:\n[Frames (%s), Molecules (%s), Beads (%s), %s (%s)]' % (num_samples, num_molecules, num_beads, feature_vector, num_features)) ### Define Model, Solver and Compile ########## print('\nDefine the model and compile') opt = candle.build_optimizer(GP['optimizer'], learning_rate, kerasDefaults) model_type = 'mlp' memo = '%s_%s' % (GP['base_memo'], model_type) ######## Define Molecular Model, Solver and Compile ######### molecular_nonlinearity = GP['molecular_nonlinearity'] len_molecular_hidden_layers = len(molecular_hidden_layers) conv_bool = GP['conv_bool'] full_conv_bool = GP['full_conv_bool'] if conv_bool: molecular_model, molecular_encoder = AE_models.conv_dense_mol_auto( bead_k_size=bead_kernel_size, mol_k_size=mol_kernel_size, weights_path=None, input_shape=(1, molecular_input_dim, 1), nonlinearity=molecular_nonlinearity, hidden_layers=molecular_hidden_layers, l2_reg=GP['l2_reg'], drop=float(GP['drop_prob'])) elif full_conv_bool: molecular_model, molecular_encoder = AE_models.full_conv_mol_auto( bead_k_size=bead_kernel_size, mol_k_size=mol_kernel_size, weights_path=None, input_shape=(1, molecular_input_dim, 1), nonlinearity=molecular_nonlinearity, hidden_layers=molecular_hidden_layers, l2_reg=GP['l2_reg'], drop=float(GP['drop_prob'])) else: molecular_model, molecular_encoder = AE_models.dense_auto( weights_path=None, input_shape=(molecular_input_dim, ), nonlinearity=molecular_nonlinearity, hidden_layers=molecular_hidden_layers, l2_reg=GP['l2_reg'], drop=float(GP['drop_prob'])) if GP['loss'] == 'mse': loss_func = 'mse' elif GP['loss'] == 'custom': loss_func = helper.combined_loss molecular_model.compile( optimizer=opt, loss=loss_func, metrics=['mean_squared_error', 'mean_absolute_error']) print('\nModel Summary: \n') molecular_model.summary() ##### set up callbacks and cooling for the molecular_model ########## drop = 0.5 mb_epochs = GP['epochs'] initial_lrate = GP['learning_rate'] epochs_drop = 1 + int(np.floor(mb_epochs / 3)) def step_decay(epoch): global initial_lrate, epochs_drop, drop lrate = initial_lrate * np.power(drop, np.floor((1 + epoch) / epochs_drop)) return lrate lr_scheduler = LearningRateScheduler(step_decay) history = callbacks.History() # callbacks=[history,lr_scheduler] history_logger = candle.LoggingCallback(logger.debug) candleRemoteMonitor = candle.CandleRemoteMonitor(params=GP) timeoutMonitor = candle.TerminateOnTimeOut(TIMEOUT) callbacks = [history, history_logger, candleRemoteMonitor, timeoutMonitor] loss = 0. #### Save the Model to disk if GP['save_path'] != None: save_path = GP['save_path'] if not os.path.exists(save_path): os.makedirs(save_path) else: save_path = '.' model_json = molecular_model.to_json() with open(save_path + '/model.json', "w") as json_file: json_file.write(model_json) encoder_json = molecular_encoder.to_json() with open(save_path + '/encoder.json', "w") as json_file: json_file.write(encoder_json) print('Saved model to disk') #### Train the Model if GP['train_bool']: ct = hf.Candle_Molecular_Train( molecular_model, molecular_encoder, data_files, mb_epochs, callbacks, batch_size=batch_size, nbr_type=GP['nbr_type'], save_path=GP['save_path'], len_molecular_hidden_layers=len_molecular_hidden_layers, molecular_nbrs=molecular_nbrs, conv_bool=conv_bool, full_conv_bool=full_conv_bool, type_bool=GP['type_bool'], sampling_density=GP['sampling_density']) frame_loss, frame_mse = ct.train_ac() else: frame_mse = [] frame_loss = [] return frame_loss, frame_mse
def fit_tfrecord(self, steps_per_epoch, epochs=1, verbose=1, callbacks=None, validation_steps=None, initial_epoch=0): epoch = initial_epoch self._make_tfrecord_train_function() do_validation = bool(len(self.val_inputs) > 0) if do_validation and not validation_steps: raise ValueError('When using a validation batch, ' 'you must specify a value for ' '`validation_steps`.') # Prepare display labels. out_labels = self._get_deduped_metrics_names() if do_validation: callback_metrics = copy.copy(out_labels) + [ 'val_' + n for n in out_labels ] else: callback_metrics = copy.copy(out_labels) # prepare callbacks self.history = cbks.History() callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history] if verbose: callbacks += [cbks.ProgbarLogger(count_mode='steps')] callbacks = cbks.CallbackList(callbacks) # it's possible to callback a different model than self: if hasattr(self, 'callback_model') and self.callback_model: callback_model = self.callback_model else: callback_model = self callbacks.set_model(callback_model) callbacks.set_params({ 'epochs': epochs, 'steps': steps_per_epoch, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics, }) callbacks.on_train_begin() if do_validation: val_sample_weight = None for cbk in callbacks: cbk.validation_data = [ self.val_inputs, self.y_val, val_sample_weight ] try: sess = K.get_session() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) callback_model.stop_training = False while epoch < epochs: callbacks.on_epoch_begin(epoch) steps_done = 0 batch_index = 0 while steps_done < steps_per_epoch: # build batch logs batch_logs = { 'batch': batch_index, 'size': self.inputs[0].shape[0].value } callbacks.on_batch_begin(batch_index, batch_logs) if self.uses_learning_phase and not isinstance( K.learning_phase(), int): ins = [1.] else: ins = [] outs = self.train_function(ins) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) # Construct epoch logs. epoch_logs = {} batch_index += 1 steps_done += 1 # Epoch finished. if steps_done >= steps_per_epoch and do_validation: val_outs = self._validate_tfrecord( steps=validation_steps) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) epoch += 1 if callback_model.stop_training: break finally: # TODO: If you close the queue, you can't open it again.. # coord.request_stop() # coord.join(threads) pass callbacks.on_train_end() return self.history
def _fit(self, f, ins, out_labels=[], batch_size=128, nb_epoch=100, verbose=1, callbacks=[], val_f=None, val_ins=None, shuffle=True, metrics=[]): ''' Abstract fit function for f(*ins). Assume that f returns a list, labelled by out_labels. ''' do_validation = False if val_f and val_ins: do_validation = True if verbose: print("Train on %d samples, validate on %d samples" % (len(ins[0]), len(val_ins[0]))) nb_train_sample = len(ins[0]) index_array = np.arange(nb_train_sample) history = cbks.History() if verbose: callbacks = [history, cbks.BaseLogger()] + callbacks else: callbacks = [history] + callbacks callbacks = cbks.CallbackList(callbacks) callbacks._set_model(self) callbacks._set_params({ 'batch_size': batch_size, 'nb_epoch': nb_epoch, 'nb_sample': nb_train_sample, 'verbose': verbose, 'do_validation': do_validation, 'metrics': metrics, }) callbacks.on_train_begin() self.stop_training = False for epoch in range(nb_epoch): callbacks.on_epoch_begin(epoch) if shuffle == 'batch': index_array = batch_shuffle(index_array, batch_size) elif shuffle: np.random.shuffle(index_array) batches = make_batches(nb_train_sample, batch_size) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] try: ins_batch = slice_X(ins, batch_ids) except TypeError as err: raise Exception('TypeError while preparing batch. \ If using HDF5 input data, pass shuffle="batch".\n') batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = len(batch_ids) callbacks.on_batch_begin(batch_index, batch_logs) outs = f(*ins_batch) if type(outs) != list: outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) epoch_logs = {} if batch_index == len(batches) - 1: # last batch # validation if do_validation: # replace with self._evaluate val_outs = self._test_loop(val_f, val_ins, batch_size=batch_size, verbose=0) if type(val_outs) != list: val_outs = [val_outs] # same labels assumed for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) if self.stop_training: break callbacks.on_train_end() return history
def _fit_loop(self, f, ins, out_labels=None, batch_size=32, epochs=100, verbose=1, callbacks=None, val_f=None, val_ins=None, shuffle=True, callback_metrics=None, initial_epoch=0, steps_per_epoch=None, validation_steps=None): """Abstract fit function for f(ins). Assume that f returns a list, labeled by out_labels. # Arguments f: Keras function returning a list of tensors ins: List of tensors to be fed to `f` out_labels: List of strings, display names of the outputs of `f` batch_size: Integer batch size or None if unknown. epochs: Number of times to iterate over the data verbose: Verbosity mode, 0, 1 or 2 callbacks: List of callbacks to be called during training val_f: Keras function to call for validation val_ins: List of tensors to be fed to `val_f` shuffle: Whether to shuffle the data at the beginning of each epoch callback_metrics: List of strings, the display names of the metrics passed to the callbacks. They should be the concatenation of list the display names of the outputs of `f` and the list of display names of the outputs of `f_val`. initial_epoch: Epoch at which to start training (useful for resuming a previous training run) steps_per_epoch: Total number of steps (batches of samples) before declaring one epoch finished and starting the next epoch. Ignored with the default value of `None`. validation_steps: Number of steps to run validation for (only if doing validation from data tensors). Ignored with the default value of `None`. # Returns `History` object. [A tweaked version.] """ do_validation = False if val_f and val_ins: do_validation = True if verbose and ins and hasattr(ins[0], 'shape') and hasattr( val_ins[0], 'shape'): print('Train on %d samples, validate on %d samples' % (ins[0].shape[0], val_ins[0].shape[0])) if validation_steps: do_validation = True if steps_per_epoch is None: raise ValueError('Can only use `validation_steps` ' 'when doing step-wise ' 'training, i.e. `steps_per_epoch` ' 'must be set.') num_train_samples = self._check_num_samples(ins, batch_size, steps_per_epoch, 'steps_per_epoch') if num_train_samples is not None: index_array = np.arange(num_train_samples) self.history = cbks.History() callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history] if verbose: if steps_per_epoch is not None: count_mode = 'steps' else: count_mode = 'samples' callbacks += [cbks.ProgbarLogger(count_mode)] callbacks = cbks.CallbackList(callbacks) out_labels = out_labels or [] # it's possible to callback a different model than self # (used by Sequential models) if hasattr(self, 'callback_model') and self.callback_model: callback_model = self.callback_model else: callback_model = self callbacks.set_model(callback_model) callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, 'steps': steps_per_epoch, 'samples': num_train_samples, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics or [], }) callbacks.on_train_begin() callback_model.stop_training = False # for cbk in callbacks: # cbk.validation_data = val_ins for epoch in range(initial_epoch, epochs): callbacks.on_epoch_begin(epoch) epoch_logs = {} if steps_per_epoch is not None: for step_index in range(steps_per_epoch): batch_logs = {} batch_logs['batch'] = step_index batch_logs['size'] = 1 callbacks.on_batch_begin(step_index, batch_logs) outs = f(ins) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(step_index, batch_logs) if callback_model.stop_training: break if do_validation: val_outs = self._test_loop(val_f, val_ins, batch_size=batch_size, steps=validation_steps, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o else: if shuffle == 'batch': index_array = _batch_shuffle(index_array, batch_size) elif shuffle: np.random.shuffle(index_array) batches = _make_batches(num_train_samples, batch_size) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] try: if isinstance(ins[-1], float): # do not slice the training phase flag ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = _slice_arrays(ins, batch_ids) except TypeError: raise TypeError('TypeError while preparing batch. ' 'If using HDF5 input data, ' 'pass shuffle="batch".') batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = len(batch_ids) batch_logs['ids'] = batch_ids callbacks.on_batch_begin(batch_index, batch_logs) outs = f(ins_batch) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) if callback_model.stop_training: break if batch_index == len(batches) - 1: # last batch. if do_validation: val_outs = self._test_loop(val_f, val_ins, batch_size=batch_size, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # same labels assumed for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) if callback_model.stop_training: break callbacks.on_train_end() return self.history
def fit_generator_Ndiff(model, generator, steps_per_epoch=None, batch_size=1, N_diff=5, margin=0.5, epochs=1, verbose=1, callbacks=None, validation_data=None, validation_steps=None, class_weight=None, max_queue_size=10, workers=1, use_multiprocessing=False, shuffle=True, initial_epoch=0): """Trains the model on data yielded batch-by-batch by a Python generator. The generator is run in parallel to the model, for efficiency. For instance, this allows you to do real-time data augmentation on images on CPU in parallel to training your model on GPU. The use of `keras.utils.Sequence` guarantees the ordering and guarantees the single use of every input per epoch when using `use_multiprocessing=True`. # Arguments generator: A generator or an instance of `Sequence` (`keras.utils.Sequence`) object in order to avoid duplicate data when using multiprocessing. The output of the generator must be either - a tuple `(inputs, targets)` - a tuple `(inputs, targets, sample_weights)`. This tuple (a single output of the generator) makes a single batch. Therefore, all arrays in this tuple must have the same length (equal to the size of this batch). Different batches may have different sizes. For example, the last batch of the epoch is commonly smaller than the others, if the size of the dataset is not divisible by the batch size. The generator is expected to loop over its data indefinitely. An epoch finishes when `steps_per_epoch` batches have been seen by the model. steps_per_epoch: Integer. Total number of steps (batches of samples) to yield from `generator` before declaring one epoch finished and starting the next epoch. It should typically be equal to the number of samples of your dataset divided by the batch size. Optional for `Sequence`: if unspecified, will use the `len(generator)` as a number of steps. epochs: Integer. Number of epochs to train the model. An epoch is an iteration over the entire data provided, as defined by `steps_per_epoch`. Note that in conjunction with `initial_epoch`, `epochs` is to be understood as "final epoch". The model is not trained for a number of iterations given by `epochs`, but merely until the epoch of index `epochs` is reached. verbose: Integer. 0, 1, or 2. Verbosity mode. 0 = silent, 1 = progress bar, 2 = one line per epoch. callbacks: List of `keras.callbacks.Callback` instances. List of callbacks to apply during training. See [callbacks](/callbacks). validation_data: This can be either - a generator for the validation data - tuple `(x_val, y_val)` - tuple `(x_val, y_val, val_sample_weights)` on which to evaluate the loss and any model metrics at the end of each epoch. The model will not be trained on this data. validation_steps: Only relevant if `validation_data` is a generator. Total number of steps (batches of samples) to yield from `validation_data` generator before stopping. Optional for `Sequence`: if unspecified, will use the `len(validation_data)` as a number of steps. class_weight: Optional dictionary mapping class indices (integers) to a weight (float) value, used for weighting the loss function (during training only). This can be useful to tell the model to "pay more attention" to samples from an under-represented class. max_queue_size: Integer. Maximum size for the generator queue. If unspecified, `max_queue_size` will default to 10. workers: Integer. Maximum number of processes to spin up when using process based threading. If unspecified, `workers` will default to 1. If 0, will execute the generator on the main thread. use_multiprocessing: Boolean. If True, use process based threading. If unspecified, `use_multiprocessing` will default to False. Note that because this implementation relies on multiprocessing, you should not pass non picklable arguments to the generator as they can't be passed easily to children processes. shuffle: Boolean. Whether to shuffle the training data in batch-sized chunks before each epoch. Only used with instances of `Sequence` (`keras.utils.Sequence`). initial_epoch: Integer. Epoch at which to start training (useful for resuming a previous training run). # Returns A `History` object. Its `History.history` attribute is a record of training loss values and metrics values at successive epochs, as well as validation loss values and validation metrics values (if applicable). # Example ```python def generate_arrays_from_file(path): while 1: with open(path) as f: for line in f: # create numpy arrays of input data # and labels, from each line in the file x1, x2, y = process_line(line) yield ({'input_1': x1, 'input_2': x2}, {'output': y}) model.fit_generator(generate_arrays_from_file('/my_file.txt'), steps_per_epoch=10000, epochs=10) ``` # Raises ValueError: In case the generator yields data in an invalid format. """ wait_time = 0.01 # in seconds epoch = initial_epoch do_validation = bool(validation_data) # self._make_train_function() # if do_validation: # self._make_test_function() is_sequence = isinstance(generator, Sequence) # do_validation = True if is_sequence else False if not is_sequence and use_multiprocessing and workers > 1: warnings.warn( UserWarning('Using a generator with `use_multiprocessing=True`' ' and multiple workers may duplicate your data.' ' Please consider using the`keras.utils.Sequence' ' class.')) if steps_per_epoch is None: if is_sequence: steps_per_epoch = len(generator) else: raise ValueError('`steps_per_epoch=None` is only valid for a' ' generator based on the `keras.utils.Sequence`' ' class. Please specify `steps_per_epoch` or use' ' the `keras.utils.Sequence` class.') # python 2 has 'next', 3 has '__next__' # avoid any explicit version checks val_gen = (hasattr(validation_data, 'next') or hasattr(validation_data, '__next__') or isinstance(validation_data, Sequence)) if (val_gen and not isinstance(validation_data, Sequence) and not validation_steps): raise ValueError('`validation_steps=None` is only valid for a' ' generator based on the `keras.utils.Sequence`' ' class. Please specify `validation_steps` or use' ' the `keras.utils.Sequence` class.') # Prepare display labels. out_labels = model._get_deduped_metrics_names() callback_metrics = out_labels + ['val_' + n for n in out_labels] # prepare callbacks history = cbks.History() callbacks = [cbks.BaseLogger()] + (callbacks or []) + [history] if verbose: callbacks += [cbks.ProgbarLogger(count_mode='steps')] callbacks = cbks.CallbackList(callbacks) # # it's possible to callback a different model than self: if hasattr(model, 'callback_model') and model.callback_model: callback_model = model.callback_model else: callback_model = model callbacks.set_model(callback_model) callbacks.set_params({ 'epochs': epochs, 'steps': steps_per_epoch, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics, }) callbacks.on_train_begin() enqueuer = None val_enqueuer = None try: if do_validation: if val_gen: if workers > 0: if isinstance(validation_data, Sequence): val_enqueuer = OrderedEnqueuer( validation_data, use_multiprocessing=use_multiprocessing) if validation_steps is None: validation_steps = len(validation_data) else: val_enqueuer = GeneratorEnqueuer( validation_data, use_multiprocessing=use_multiprocessing, wait_time=wait_time) val_enqueuer.start(workers=workers, max_queue_size=max_queue_size) validation_generator = val_enqueuer.get() else: validation_generator = validation_data else: pass # if len(validation_data) == 2: # val_x, val_y = validation_data # val_sample_weights = None # elif len(validation_data) == 3: # val_x, val_y, val_sample_weights = validation_data # else: # raise ValueError('`validation_data` should be a tuple ' # '`(val_x, val_y, val_sample_weight)` ' # 'or `(val_x, val_y)`. Found: ' + # str(validation_data)) # val_x, val_y, val_sample_weights = _standardize_user_data( # val_x, val_y, val_sample_weight) # val_data = val_x + val_y + val_sample_weights # if self.uses_learning_phase and not isinstance(K.learning_phase(), int): # val_data += [0.] # for cbk in callbacks: # cbk.validation_data = val_data if workers > 0: if is_sequence: enqueuer = OrderedEnqueuer( generator, use_multiprocessing=use_multiprocessing, shuffle=shuffle) else: enqueuer = GeneratorEnqueuer( generator, use_multiprocessing=use_multiprocessing, wait_time=wait_time) enqueuer.start(workers=workers, max_queue_size=max_queue_size) output_generator = enqueuer.get() else: output_generator = generator callback_model.stop_training = False # Construct epoch logs. epoch_logs = {} while epoch < epochs: callbacks.on_epoch_begin(epoch) steps_done = 0 batch_index = 0 while steps_done < steps_per_epoch: generator_output = next(output_generator) if not hasattr(generator_output, '__len__'): raise ValueError('Output of generator should be ' 'batch_size lists ' + str(generator_output)) if len(generator_output) == batch_size: # ii_ndiff: the index of the negative sample gen_out = generator_output sample_weight = None else: raise ValueError('Output of generator should be ' 'batch_size lists ' + str(generator_output)) # build batch logs batch_logs = {} # if isinstance(x, list): # batch_size = x[0].shape[0] # elif isinstance(x, dict): # batch_size = list(x.values())[0].shape[0] # else: # batch_size = x.shape[0] batch_logs['batch'] = batch_index batch_logs['size'] = batch_size callbacks.on_batch_begin(batch_index, batch_logs) # aggregate the losses by inner index n_diff loss_mat = np.zeros((batch_size, N_diff)) for ii_ndiff in range(N_diff): # get the maximum sequence length len_anchor_max, len_same_max, len_diff_max = \ get_maximum_length(batch_size=batch_size, generator_output=gen_out, index=[ii_ndiff]*batch_size) print(len_anchor_max, len_same_max, len_diff_max) # organize the input for the prediction input_anchor, input_same, input_diff = \ make_same_length_batch(batch_size=batch_size, len_anchor_max=len_anchor_max, len_same_max=len_same_max, len_diff_max=len_diff_max, generator_output=gen_out, index=[ii_ndiff]*batch_size) output_batch_pred = model.predict_on_batch( [input_anchor, input_same, input_diff]) loss = K.eval( triplet_loss_no_mean(output_batch_pred, margin)) loss_mat[:, ii_ndiff] = loss # this the index of the input which has the maximum loss for each N_diff pairs index_max_loss = np.argmax(loss_mat, axis=-1) len_anchor_max, len_same_max, len_diff_max = get_maximum_length( batch_size=batch_size, generator_output=gen_out, index=index_max_loss) input_anchor, input_same, input_diff = \ make_same_length_batch(batch_size=batch_size, len_anchor_max=len_anchor_max, len_same_max=len_same_max, len_diff_max=len_diff_max, generator_output=gen_out, index=index_max_loss) outs = model.train_on_batch( [input_anchor, input_same, input_diff], None, sample_weight=sample_weight, class_weight=class_weight) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) batch_index += 1 steps_done += 1 # Epoch finished. if steps_done >= steps_per_epoch and do_validation: if val_gen: val_outs = evaluate_generator( model=model, generator=validation_generator, steps=validation_steps, batch_size=batch_size, margin=margin, N_diff=N_diff, workers=0) else: pass # # No need for try/except because # # data has already been validated. # val_outs = model.evaluate( # val_x, val_y, # batch_size=batch_size, # sample_weight=val_sample_weights, # verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o if callback_model.stop_training: break callbacks.on_epoch_end(epoch, epoch_logs) epoch += 1 if callback_model.stop_training: break finally: try: if enqueuer is not None: enqueuer.stop() finally: if val_enqueuer is not None: val_enqueuer.stop() callbacks.on_train_end() return history
def fit_generator_autosized( model, generator, epochs=1, #steps_per_epoch=None, verbose=1, callbacks=None, validation_data=None, validation_steps=None, validation_callbacks=None, class_weight=None, max_queue_size=10, workers=1, use_multiprocessing=False, shuffle=True, initial_epoch=0): """See docstring for `Model.fit_generator`.""" wait_time = 0.01 # in seconds epoch = initial_epoch do_validation = bool(validation_data) model._make_train_function() if do_validation: model._make_test_function() is_sequence = isinstance(generator, Sequence) if not is_sequence and use_multiprocessing and workers > 1: warnings.warn( UserWarning('Using a generator with `use_multiprocessing=True`' ' and multiple workers may duplicate your data.' ' Please consider using the`keras.utils.Sequence' ' class.')) # if steps_per_epoch is None: # if is_sequence: # steps_per_epoch = len(generator) # else: # raise ValueError('`steps_per_epoch=None` is only valid for a' # ' generator based on the ' # '`keras.utils.Sequence`' # ' class. Please specify `steps_per_epoch` ' # 'or use the `keras.utils.Sequence` class.') # python 2 has 'next', 3 has '__next__' # avoid any explicit version checks val_gen = (hasattr(validation_data, 'next') or hasattr(validation_data, '__next__') or isinstance(validation_data, Sequence)) # if (val_gen and not isinstance(validation_data, Sequence) and # not validation_steps): # raise ValueError('`validation_steps=None` is only valid for a' # ' generator based on the `keras.utils.Sequence`' # ' class. Please specify `validation_steps` or use' # ' the `keras.utils.Sequence` class.') # Prepare display labels. out_labels = model.metrics_names callback_metrics = out_labels + ['val_' + n for n in out_labels] # prepare callbacks model.history = cbks.History() _callbacks = [ cbks.BaseLogger(stateful_metrics=model.stateful_metric_names) ] # instead of ProgbarLogger (but only for first epoch): if verbose: print('Epoch 1/%d' % epochs) progbar = Progbar(target=None, verbose=1, stateful_metrics=model.stateful_metric_names) _callbacks += (callbacks or []) + [model.history] callbacks = cbks.CallbackList(_callbacks) # it's possible to callback a different model than self: if hasattr(model, 'callback_model') and model.callback_model: callback_model = model.callback_model else: callback_model = model callbacks.set_model(callback_model) callbacks.set_params({ 'epochs': epochs, 'steps': None, # will be refined during first epoch 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics, }) callbacks.on_train_begin() enqueuer = None val_enqueuer = None try: if do_validation and not val_gen: # Prepare data for validation if len(validation_data) == 2: val_x, val_y = validation_data val_sample_weight = None elif len(validation_data) == 3: val_x, val_y, val_sample_weight = validation_data else: raise ValueError('`validation_data` should be a tuple ' '`(val_x, val_y, val_sample_weight)` ' 'or `(val_x, val_y)`. Found: ' + str(validation_data)) val_x, val_y, val_sample_weights = model._standardize_user_data( val_x, val_y, val_sample_weight) val_data = val_x + val_y + val_sample_weights if model.uses_learning_phase and not isinstance( K.learning_phase(), int): val_data += [0.] for cbk in callbacks: cbk.validation_data = val_data if workers > 0: if is_sequence: enqueuer = OrderedEnqueuer( generator, use_multiprocessing=use_multiprocessing, shuffle=shuffle) else: enqueuer = GeneratorEnqueuer( generator, use_multiprocessing=use_multiprocessing, wait_time=wait_time) enqueuer.start(workers=workers, max_queue_size=max_queue_size) output_generator = enqueuer.get() else: if is_sequence: output_generator = iter(generator) else: output_generator = generator callback_model.stop_training = False # Construct epoch logs. epoch_logs = {} while epoch < epochs: for m in model.stateful_metric_functions: m.reset_states() callbacks.on_epoch_begin(epoch) steps_done = 0 batch_index = 0 for generator_output in output_generator: if not generator_output: # end of epoch? break if not hasattr(generator_output, '__len__'): raise ValueError('Output of generator should be ' 'a tuple `(x, y, sample_weight)` ' 'or `(x, y)`. Found: ' + str(generator_output)) if len(generator_output) == 2: x, y = generator_output sample_weight = None elif len(generator_output) == 3: x, y, sample_weight = generator_output else: raise ValueError('Output of generator should be ' 'a tuple `(x, y, sample_weight)` ' 'or `(x, y)`. Found: ' + str(generator_output)) # build batch logs batch_logs = {} if not x: # Handle data tensors support when no input given # step-size = 1 for data tensors batch_size = 1 elif isinstance(x, list): batch_size = x[0].shape[0] elif isinstance(x, dict): batch_size = list(x.values())[0].shape[0] else: batch_size = x.shape[0] batch_logs['batch'] = batch_index batch_logs['size'] = batch_size callbacks.on_batch_begin(batch_index, batch_logs) outs = model.train_on_batch(x, y, sample_weight=sample_weight, class_weight=class_weight) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) if epoch == initial_epoch and verbose: log_values = [] for k in callback_metrics: if k in batch_logs: log_values.append((k, batch_logs[k])) progbar.update(steps_done, log_values) batch_index += 1 steps_done += 1 if callback_model.stop_training: break if epoch == initial_epoch: if verbose: log_values = [] for k in callback_metrics: if k in batch_logs: log_values.append((k, batch_logs[k])) progbar.update(steps_done, log_values) # Epoch finished. if do_validation: if val_gen: val_outs, validation_steps = evaluate_generator_autosized( model, validation_data, steps=validation_steps, callbacks=validation_callbacks, workers=workers, use_multiprocessing=use_multiprocessing, max_queue_size=max_queue_size, verbose=1) else: # No need for try/except because # data has already been validated. val_outs = model.evaluate(val_x, val_y, batch_size=batch_size, sample_weight=val_sample_weights, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o if callback_model.stop_training: break callbacks.on_epoch_end(epoch, epoch_logs) if epoch == initial_epoch: if verbose: print() progbar = cbks.ProgbarLogger( count_mode='steps', stateful_metrics=model.stateful_metric_names) progbar.set_model(callback_model) callbacks.append(progbar) callbacks.set_params({ 'epochs': epochs, 'steps': steps_done, # refine 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics, }) if verbose: progbar.on_train_begin() epoch += 1 if callback_model.stop_training: break finally: try: if enqueuer is not None: enqueuer.stop() finally: if val_enqueuer is not None: val_enqueuer.stop() callbacks.on_train_end() return model.history
async def fit_generator(model, generator, steps_per_epoch=None, epochs=1, verbose=1, callbacks=None, validation_data=None, validation_steps=None, class_weight=None, shuffle=True, initial_epoch=0): """See docstring for `Model.fit_generator`.""" epoch = initial_epoch do_validation = bool(validation_data) model._make_train_function() if do_validation: model._make_test_function() if steps_per_epoch is None: steps_per_epoch = len(generator) # Prepare display labels. out_labels = model.metrics_names callback_metrics = out_labels + ['val_' + n for n in out_labels] # prepare callbacks model.history = cbks.History() _callbacks = [ cbks.BaseLogger(stateful_metrics=model.stateful_metric_names) ] if verbose: _callbacks.append( cbks.ProgbarLogger(count_mode='steps', stateful_metrics=model.stateful_metric_names)) _callbacks += (callbacks or []) + [model.history] callbacks = cbks.CallbackList(_callbacks) # it's possible to callback a different model than self: if hasattr(model, 'callback_model') and model.callback_model: callback_model = model.callback_model else: callback_model = model callbacks.set_model(callback_model) callbacks.set_params({ 'epochs': epochs, 'steps': steps_per_epoch, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics, }) callbacks.on_train_begin() output_generator = generator.async_next callback_model.stop_training = False # Construct epoch logs. epoch_logs = {} while epoch < epochs: for m in model.stateful_metric_functions: m.reset_states() callbacks.on_epoch_begin(epoch) steps_done = 0 batch_index = 0 while steps_done < steps_per_epoch: generator_output = await output_generator() if not hasattr(generator_output, '__len__'): raise ValueError('Output of generator should be ' 'a tuple `(x, y, sample_weight)` ' 'or `(x, y)`. Found: ' + str(generator_output)) if len(generator_output) == 2: x, y = generator_output sample_weight = None elif len(generator_output) == 3: x, y, sample_weight = generator_output else: raise ValueError('Output of generator should be ' 'a tuple `(x, y, sample_weight)` ' 'or `(x, y)`. Found: ' + str(generator_output)) # build batch logs batch_logs = {} if x is None or len(x) == 0: # Handle data tensors support when no input given # step-size = 1 for data tensors batch_size = 1 elif isinstance(x, list): batch_size = x[0].shape[0] elif isinstance(x, dict): batch_size = list(x.values())[0].shape[0] else: batch_size = x.shape[0] batch_logs['batch'] = batch_index batch_logs['size'] = batch_size callbacks.on_batch_begin(batch_index, batch_logs) outs = model.train_on_batch(x, y, sample_weight=sample_weight, class_weight=class_weight) outs = to_list(outs) for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) batch_index += 1 steps_done += 1 # Epoch finished. if steps_done >= steps_per_epoch and do_validation: val_outs = await evaluate_generator(model, validation_data, validation_steps) val_outs = to_list(val_outs) # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o if callback_model.stop_training: break generator.on_epoch_end() callbacks.on_epoch_end(epoch, epoch_logs) epoch += 1 if callback_model.stop_training: break callbacks.on_train_end() return model.history
def _fit_loop(self, f, ins, out_labels=None, batch_size=32, epochs=100, verbose=1, callbacks=None, val_f=None, val_ins=None, shuffle=True, callback_metrics=None, initial_epoch=0, steps_per_epoch=None): """Abstract fit function for `f(ins)`. Assume that f returns a list, labeled by out_labels. # Arguments f: Keras function returning a list of tensors ins: list of tensors to be fed to `f` out_labels: list of strings, display names of the outputs of `f` batch_size: integer batch size epochs: number of times to iterate over the data verbose: verbosity mode, 0, 1 or 2 callbacks: list of callbacks to be called during training val_f: Keras function to call for validation val_ins: list of tensors to be fed to `val_f` shuffle: whether to shuffle the data at the beginning of each epoch callback_metrics: list of strings, the display names of the metrics passed to the callbacks. They should be the concatenation of list the display names of the outputs of `f` and the list of display names of the outputs of `f_val`. initial_epoch: epoch at which to start training (useful for resuming a previous training run) steps_per_epoch: Total number of steps (batches of samples) before declaring one epoch finished and starting the next epoch. The default `None` is equal to the number of unique samples in your dataset divided by the batch size, or 1 if that cannot be determined. # Returns `History` object. """ do_validation = False if val_f and val_ins: do_validation = True if verbose and ins and hasattr(ins[0], 'shape'): print('Train on %d samples, validate on %d samples' % (ins[0].shape[0], val_ins[0].shape[0])) if steps_per_epoch is not None: num_train_samples = steps_per_epoch else: if ins and hasattr(ins[0], 'shape'): num_train_samples = ins[0].shape[0] else: # May happen if we are running `fit` without Numpy input data, # i.e. if all inputs to the models are data tensors # instead of placeholders. # In that case we will run `fit` over a single batch. num_train_samples = batch_size verbose = 2 index_array = np.arange(num_train_samples) self.history = cbks.History() callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history] if verbose: # callbacks += [cbks.ProgbarLogger()] callbacks += [ProgbarLogger_TFRecord()] callbacks = cbks.CallbackList(callbacks) out_labels = out_labels or [] # it's possible to callback a different model than self # (used by Sequential models) if hasattr(self, 'callback_model') and self.callback_model: callback_model = self.callback_model else: callback_model = self callbacks.set_model(callback_model) callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, 'samples': num_train_samples, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics or [], }) callbacks.on_train_begin() callback_model.stop_training = False for cbk in callbacks: cbk.validation_data = val_ins for epoch in range(initial_epoch, epochs): callbacks.on_epoch_begin(epoch) if shuffle == 'batch': index_array = _batch_shuffle(index_array, batch_size) elif shuffle: np.random.shuffle(index_array) batches = _make_batches(num_train_samples, batch_size) epoch_logs = {} for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] try: if isinstance(ins[-1], float): # Do not slice the training phase flag. ins_batch = \ _slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = _slice_arrays(ins, batch_ids) except TypeError: raise TypeError('TypeError while preparing batch. ' 'If using HDF5 input data, ' 'pass shuffle="batch".') batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = len(batch_ids) callbacks.on_batch_begin(batch_index, batch_logs) outs = f(ins_batch) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) if callback_model.stop_training: break if batch_index == len(batches) - 1: # Last batch. if do_validation: val_outs = self._test_loop(val_f, val_ins, batch_size=batch_size, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) if callback_model.stop_training: break callbacks.on_train_end() return self.history
def fit_models(callback_model, models, generators, metrics_names, batch_size, steps_per_epoch=None, epochs=1, verbose=1, callbacks=None, initial_epoch=0): epoch = initial_epoch # Prepare display labels. callback_metrics = [n for m in metrics_names for n in m.keys()] # prepare callbacks stateful_metric_names = [] for model in models: model.history = cbks.History() try: stateful_metric_names.extend(model.stateful_metric_names) except AttributeError: stateful_metric_names.extend(model.model.stateful_metric_names) _callbacks = [cbks.BaseLogger(stateful_metrics=stateful_metric_names)] if verbose: _callbacks.append( cbks.ProgbarLogger(count_mode='steps', stateful_metrics=stateful_metric_names)) _callbacks += (callbacks or []) + [model.history for model in models] callbacks = cbks.CallbackList(_callbacks) # it's possible to callback a different model than self: callbacks.set_model(callback_model) callbacks.set_params({ 'epochs': epochs, 'steps': steps_per_epoch, 'verbose': verbose, 'do_validation': False, 'metrics': callback_metrics, }) callbacks.on_train_begin() try: callback_model.stop_training = False # Construct epoch logs. epoch_logs = {} while epoch < epochs: for model in models: try: stateful_metric_functions = model.stateful_metric_functions except AttributeError: stateful_metric_functions = model.model.stateful_metric_functions for m in stateful_metric_functions: m.reset_states() callbacks.on_epoch_begin(epoch) steps_done = 0 batch_index = 0 while steps_done < steps_per_epoch: # build batch logs batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = batch_size callbacks.on_batch_begin(batch_index, batch_logs) for model, output_generator, metrics in zip( models, generators, metrics_names): generator_output = next(output_generator) if not hasattr(generator_output, '__len__'): raise ValueError('Output of generator should be ' 'a tuple `(x, y, sample_weight)` ' 'or `(x, y)`. Found: ' + str(generator_output)) if len(generator_output) == 2: x, y = generator_output sample_weight = None elif len(generator_output) == 3: x, y, sample_weight = generator_output else: raise ValueError('Output of generator should be ' 'a tuple `(x, y, sample_weight)` ' 'or `(x, y)`. Found: ' + str(generator_output)) outs = model.train_on_batch(x, y, sample_weight=sample_weight) if not isinstance(outs, list): outs = [outs] for name, i in metrics.items(): batch_logs[name] = outs[i] callbacks.on_batch_end(batch_index, batch_logs) batch_index += 1 steps_done += 1 # Epoch finished. if callback_model.stop_training: break callbacks.on_epoch_end(epoch, epoch_logs) epoch += 1 if callback_model.stop_training: break finally: pass callbacks.on_train_end() return [model.history for model in models]
def _fit_loop(self, f, ins, out_labels=None, batch_size=32, nb_epoch=100, verbose=1, callbacks=None, val_f=None, val_ins=None, shuffle=True, callback_metrics=None, initial_epoch=0): """Abstract fit function for f(ins). Assume that f returns a list, labeled by out_labels. # Arguments f: Keras function returning a list of tensors ins: list of tensors to be fed to `f` out_labels: list of strings, display names of the outputs of `f` batch_size: integer batch size nb_epoch: number of times to iterate over the data verbose: verbosity mode, 0, 1 or 2 callbacks: list of callbacks to be called during training val_f: Keras function to call for validation val_ins: list of tensors to be fed to `val_f` shuffle: whether to shuffle the data at the beginning of each epoch callback_metrics: list of strings, the display names of the metrics passed to the callbacks. They should be the concatenation of list the display names of the outputs of `f` and the list of display names of the outputs of `f_val`. initial_epoch: epoch at which to start training (useful for resuming a previous training run) # Returns `History` object. [A tweaked version.] """ do_validation = False if val_f and val_ins: do_validation = True if verbose: print('Train on %d samples, validate on %d samples' % (ins[0].shape[0], val_ins[0].shape[0])) nb_train_sample = ins[0].shape[0] index_array = np.arange(nb_train_sample) self.history = cbks.History() callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history] if verbose: callbacks += [cbks.ProgbarLogger()] callbacks = cbks.CallbackList(callbacks) out_labels = out_labels or [] # it's possible to callback a different model than self # (used by Sequential models) if hasattr(self, 'callback_model') and self.callback_model: callback_model = self.callback_model else: callback_model = self callbacks.set_model(callback_model) callbacks.set_params({ 'batch_size': batch_size, 'nb_epoch': nb_epoch, 'nb_sample': nb_train_sample, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics or [], }) callbacks.on_train_begin() callback_model.stop_training = False self.validation_data = val_ins for epoch in range(initial_epoch, nb_epoch): callbacks.on_epoch_begin(epoch) if shuffle == 'batch': index_array = batch_shuffle(index_array, batch_size) elif shuffle: np.random.shuffle(index_array) batches = make_batches(nb_train_sample, batch_size) epoch_logs = {} for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] try: if isinstance(ins[-1], float): # do not slice the training phase flag ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = slice_X(ins, batch_ids) except TypeError: raise TypeError('TypeError while preparing batch. ' 'If using HDF5 input data, ' 'pass shuffle="batch".') batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = len(batch_ids) batch_logs['ids'] = batch_ids callbacks.on_batch_begin(batch_index, batch_logs) outs = f(ins_batch) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) if batch_index == len(batches) - 1: # last batch # validation if do_validation: # replace with self._evaluate val_outs = self._test_loop(val_f, val_ins, batch_size=batch_size, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # same labels assumed for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) if callback_model.stop_training: break callbacks.on_train_end() return self.history
def fit_dataflow(self, dflow, steps_per_epoch, epochs=1, verbose=1, callbacks=None, validation_data=None, validation_steps=None, class_weight=None, max_q_size=10, workers=1, pickle_safe=False, initial_epoch=0): """Fits the model on data yielded batch-by-batch by a Python generator. The generator is run in parallel to the model, for efficiency. For instance, this allows you to do real-time data augmentation on images on CPU in parallel to training your model on GPU. # Arguments dflow: a dataflow object a-la-carte Tensorpack. The output of the generator must be either - a tuple (inputs, targets) - a tuple (inputs, targets, sample_weights). All arrays should contain the same number of samples. The generator is expected to loop over its data indefinitely. An epoch finishes when `steps_per_epoch` samples have been seen by the model. steps_per_epoch: Total number of steps (batches of samples) to yield from `generator` before declaring one epoch finished and starting the next epoch. It should typically be equal to the number of unique samples if your dataset divided by the batch size. epochs: integer, total number of iterations on the data. verbose: verbosity mode, 0, 1, or 2. callbacks: list of callbacks to be called during training. validation_data: this can be either - a generator for the validation data - a tuple (inputs, targets) - a tuple (inputs, targets, sample_weights). validation_steps: Only relevant if `validation_data` is a generator. Total number of steps (batches of samples) to yield from `generator` before stopping. class_weight: dictionary mapping class indices to a weight for the class. max_q_size: maximum size for the generator queue workers: maximum number of processes to spin up when using process based threading pickle_safe: if True, use process based threading. Note that because this implementation relies on multiprocessing, you should not pass non picklable arguments to the generator as they can't be passed easily to children processes. initial_epoch: epoch at which to start training (useful for resuming a previous training run) # Returns A `History` object. # Example ```python def generate_arrays_from_file(path): while 1: f = open(path) for line in f: # create numpy arrays of input data # and labels, from each line in the file x1, x2, y = process_line(line) yield ({'input_1': x1, 'input_2': x2}, {'output': y}) f.close() model.fit_generator(generate_arrays_from_file('/my_file.txt'), steps_per_epoch=10000, epochs=10) ``` # Raises ValueError: In case the generator yields data in an invalid format. """ # wait_time = 0.01 # in seconds epoch = initial_epoch do_validation = bool(validation_data) self._make_train_function() if do_validation: self._make_test_function() # python 2 has 'next', 3 has '__next__' # avoid any explicit version checks val_gen = (hasattr(validation_data, 'next') or hasattr(validation_data, '__next__')) if val_gen and not validation_steps: raise ValueError('When using a generator for validation data, ' 'you must specify a value for ' '`validation_steps`.') out_labels = self.metrics_names callback_metrics = out_labels + ['val_' + n for n in out_labels] # prepare callbacks self.history = cbks.History() callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history] if verbose: callbacks += [cbks.ProgbarLogger(count_mode='steps')] callbacks = cbks.CallbackList(callbacks) # it's possible to callback a different model than self: if hasattr(self, 'callback_model') and self.callback_model: callback_model = self.callback_model else: callback_model = self callbacks.set_model(callback_model) callbacks.set_params({ 'epochs': epochs, 'steps': steps_per_epoch, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics, }) callbacks.on_train_begin() if do_validation and not val_gen: if len(validation_data) == 2: val_x, val_y = validation_data val_sample_weight = None elif len(validation_data) == 3: val_x, val_y, val_sample_weight = validation_data else: raise ValueError('validation_data should be a tuple ' '`(val_x, val_y, val_sample_weight)` ' 'or `(val_x, val_y)`. Found: ' + str(validation_data)) val_x, val_y, val_sample_weights = self._standardize_user_data( val_x, val_y, val_sample_weight) for cbk in callbacks: cbk.validation_data = val_x + [val_y, val_sample_weights] # enqueuer = None # TODO: Tensorpack does some kind of acceleratn using # QueueInputTrainer, QueueInput, and EnqueueThread. The # implementation below corresponds to SimpleTrainer which # Tensorpack notes as being slow. I still cannot decipher what # exactly is going on in Tensorpack. For the same per-GPU batchsize # the runtime per epoch seems on par. Perhaps with Tensorpack # implementation using Queue+Thread for datafalow the feed_dict # would be faster. The keras fit_generator does use an enqueuer, # but I did not notice performance difference between using # fit_generator or this mixed-in fit_dataflow method. try: # enqueuer = GeneratorEnqueuer(generator, pickle_safe=pickle_safe) # enqueuer.start(max_q_size=max_q_size, workers=workers) dflow.reset_state() _generator = dflow.get_data() callback_model.stop_training = False while epoch < epochs: callbacks.on_epoch_begin(epoch) steps_done = 0 batch_index = 0 while steps_done < steps_per_epoch: # generator_output = None generator_output = next(_generator) # while enqueuer.is_running(): # if not enqueuer.queue.empty(): # generator_output = enqueuer.queue.get() # break # else: # time.sleep(wait_time) if not hasattr(generator_output, '__len__'): raise ValueError('output of generator should be ' 'a tuple `(x, y, sample_weight)` ' 'or `(x, y)`. Found: ' + str(generator_output)) if len(generator_output) == 2: x, y = generator_output sample_weight = None elif len(generator_output) == 3: x, y, sample_weight = generator_output else: raise ValueError('output of generator should be ' 'a tuple `(x, y, sample_weight)` ' 'or `(x, y)`. Found: ' + str(generator_output)) # build batch logs batch_logs = {} if isinstance(x, list): batch_size = x[0].shape[0] elif isinstance(x, dict): batch_size = list(x.values())[0].shape[0] else: batch_size = x.shape[0] batch_logs['batch'] = batch_index batch_logs['size'] = batch_size callbacks.on_batch_begin(batch_index, batch_logs) outs = self.train_on_batch(x, y, sample_weight=sample_weight, class_weight=class_weight) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) # Construct epoch logs. epoch_logs = {} batch_index += 1 steps_done += 1 # Epoch finished. if steps_done >= steps_per_epoch and do_validation: if val_gen: val_outs = self.evaluate_generator( validation_data, validation_steps, max_q_size=max_q_size, workers=workers, pickle_safe=pickle_safe) else: # No need for try/except because # data has already been validated. val_outs = self.evaluate( val_x, val_y, batch_size=batch_size, sample_weight=val_sample_weights, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) epoch += 1 if callback_model.stop_training: break finally: # if enqueuer is not None: # enqueuer.stop() pass callbacks.on_train_end() return self.history
def fit(self, model_dir_path, image_label_pairs, epochs=None, batch_size=None, snapshot_dir_path=None, snapshot_interval=None): # Change the epoch value if epochs is None: epochs = 100 if batch_size is None: batch_size = 64 if snapshot_interval is None: snapshot_interval = 20 self.config = dict() self.config['img_width'] = self.img_width self.config['img_height'] = self.img_height self.config['random_input_dim'] = self.random_input_dim self.config['text_input_dim'] = self.text_input_dim self.config['img_channels'] = self.img_channels self.config['glove_source_dir_path'] = self.glove_source_dir_path self.glove_model.load(data_dir_path=self.glove_source_dir_path, embedding_dim=self.text_input_dim) config_file_path = Capsules.get_config_file_path(model_dir_path) np.save(config_file_path, self.config) noise = np.zeros((batch_size, self.random_input_dim)) text_batch = np.zeros((batch_size, self.text_input_dim)) self.create_model() batch_count = int(image_label_pairs.shape[0] / batch_size) print(batch_count) #exp_replay = [] # array to store sample for experience replay for epoch in range(epochs): cum_d_loss = 0 cum_g_loss = 0 cum_g_acc = 0 cum_d_acc = 0 print('-' * 15, 'Epoch %d' % epoch, '-' * 15) for batch_index in tqdm(range(batch_count)): # Step 1: train the discriminator image_label_pair_batch = image_label_pairs[batch_index * batch_size: (batch_index + 1) * batch_size] image_batch = [] for index in range(batch_size): image_label_pair = image_label_pair_batch[index] normalized_img = image_label_pair[0] text = image_label_pair[1] image_batch.append(normalized_img) text_batch[index, :] = self.glove_model.encode_doc( text, self.text_input_dim) noise[index, :] = np.random.uniform( -1, 1, self.random_input_dim) image_batch = np.array(image_batch) #image_batch = np.transpose(image_batch, (0, 2, 3, 1)) generated_images = self.generator.predict([noise, text_batch], verbose=0) # Train on soft targets (add noise to targets as well) noise_prop = 0.05 # Randomly flip 5% of targets # Prepare labels for real data true_labels = np.zeros((batch_size, 1)) + np.random.uniform( low=0.0, high=0.1, size=(batch_size, 1)) flipped_idx = np.random.choice(np.arange(len(true_labels)), size=int(noise_prop * len(true_labels))) true_labels[flipped_idx] = 1 - true_labels[flipped_idx] # Prepare labels for generated data gene_labels = np.ones((batch_size, 1)) - np.random.uniform( low=0.0, high=0.1, size=(batch_size, 1)) flipped_idx = np.random.choice(np.arange(len(gene_labels)), size=int(noise_prop * len(gene_labels))) gene_labels[flipped_idx] = 1 - gene_labels[flipped_idx] ''' # Store a random point for experience replay r_idx = np.random.randint(batch_size) exp_replay.append([generated_images[r_idx], text_batch[r_idx], gene_labels[r_idx]]) ''' if ( epoch * batch_size + batch_index ) % snapshot_interval == 0 and snapshot_dir_path is not None: self.save_snapshots(generated_images, snapshot_dir_path=snapshot_dir_path, epoch=epoch, batch_index=batch_index) self.discriminator.trainable = True # Train discriminator on real data d_loss_true = self.discriminator.train_on_batch( [image_batch, text_batch], true_labels) # Train discriminator on generated data d_loss_gene = self.discriminator.train_on_batch( [generated_images, text_batch], gene_labels) d_loss = ((np.asarray(d_loss_true) + np.asarray(d_loss_gene)) * 0.5).tolist() cum_d_loss += d_loss[0] cum_d_acc += d_loss[1] ''' #Adversarial Model Training #If we have enough points, do experience replay if len(exp_replay) == batch_size: generated_images = np.array([p[0] for p in exp_replay]) text_batch = np.array([p[1] for p in exp_replay]) gene_labels = np.array([p[2] for p in exp_replay]) expprep_loss_gene = self.discriminator.train_on_batch([generated_images, text_batch], gene_labels) exp_replay = [] break ''' #step 2: train the generator for index in range(batch_size): image_label_pair = image_label_pair_batch[index] text = image_label_pair[1] text_batch[index, :] = self.glove_model.encode_doc( text, self.text_input_dim) noise[index, :] = np.random.uniform( -1, 1, self.random_input_dim) self.discriminator.trainable = False g_loss = self.model.train_on_batch([noise, text_batch], np.zeros((batch_size, 1))) cum_g_loss += g_loss[0] cum_g_acc += g_loss[1] #for index in range(batch_size): #noise[index, :] = np.random.uniform(-1, 1, self.random_input_dim) #g_loss = self.model.train_on_batch([noise, text_batch], np.array([1] * batch_size)) #print('\tEpoch: {}, Generator Loss: {}, Discriminator Loss: {}'.format(epoch+1, cum_g_loss/batch_count, cum_d_loss/batch_count)) print( '\tEpoch: {}, Generator Loss: {}, Generator Accuracy: {}, Discriminator Loss: {}, Disciminator Accuracy: {}' .format(epoch + 1, cum_g_loss / batch_count, cum_g_acc / batch_count, cum_d_loss / batch_count, cum_d_acc / batch_count)) D_L.append(cum_d_loss / batch_count) D_A.append(cum_d_acc / batch_count) G_L.append(cum_g_loss / batch_count) G_A.append(cum_g_acc / batch_count) #if (epoch * batch_size + batch_index) % 10 == 9: self.generator.save_weights( Capsules.get_weight_file_path(model_dir_path, 'generator'), True) self.discriminator.save_weights( Capsules.get_weight_file_path(model_dir_path, 'discriminator'), True) self.generator.save_weights( Capsules.get_weight_file_path(model_dir_path, 'generator'), True) self.discriminator.save_weights( Capsules.get_weight_file_path(model_dir_path, 'discriminator'), True) callbacks.History() callbacks.ModelCheckpoint(os.path.join(model_dir_path, 'capgans.h5'), monitor='cum_d_loss', verbose=0, save_best_only=False, save_weights_only=False, mode='auto', period=1)
def fit_and_predict_generator_with_sceneinst_metrics( model, generator, params, multithreading_metrics=False, steps_per_epoch=None, epochs=1, verbose=1, callbacks=None, validation_data=None, validation_steps=None, max_queue_size=10, workers=1, use_multiprocessing=False, shuffle=True, initial_epoch=0): """See docstring for `Model.fit_generator`.""" wait_time = 0.01 # in seconds epoch = initial_epoch do_validation = bool(validation_data) model._make_train_function() if do_validation: model._make_test_function() is_sequence = isinstance(generator, Sequence) if not is_sequence and use_multiprocessing and workers > 1: warnings.warn( UserWarning('Using a generator with `use_multiprocessing=True`' ' and multiple workers may duplicate your data.' ' Please consider using the`keras.utils.Sequence' ' class.')) if steps_per_epoch is None: if is_sequence: steps_per_epoch = len(generator) else: raise ValueError('`steps_per_epoch=None` is only valid for a' ' generator based on the ' '`keras.utils.Sequence`' ' class. Please specify `steps_per_epoch` ' 'or use the `keras.utils.Sequence` class.') # python 2 has 'next', 3 has '__next__' # avoid any explicit version checks val_gen = (hasattr(validation_data, 'next') or hasattr(validation_data, '__next__') or isinstance(validation_data, Sequence)) if (val_gen and not isinstance(validation_data, Sequence) and not validation_steps): raise ValueError('`validation_steps=None` is only valid for a' ' generator based on the `keras.utils.Sequence`' ' class. Please specify `validation_steps` or use' ' the `keras.utils.Sequence` class.') # Prepare display labels. out_labels = model.metrics_names callback_metrics = out_labels + ['val_' + n for n in out_labels] # prepare callbacks model.history = cbks.History() _callbacks = [ cbks.BaseLogger(stateful_metrics=model.stateful_metric_names) ] if verbose: _callbacks.append( cbks.ProgbarLogger(count_mode='steps', stateful_metrics=model.stateful_metric_names)) _callbacks += (callbacks or []) + [model.history] callbacks = cbks.CallbackList(_callbacks) # it's possible to callback a different model than self: if hasattr(model, 'callback_model') and model.callback_model: callback_model = model.callback_model else: callback_model = model callbacks.set_model(callback_model) callbacks.set_params({ 'epochs': epochs, 'steps': steps_per_epoch, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics, }) callbacks.on_train_begin() enqueuer = None val_enqueuer = None try: if do_validation: if val_gen and workers > 0: # Create an Enqueuer that can be reused val_data = validation_data if isinstance(val_data, Sequence): val_enqueuer = OrderedEnqueuer( val_data, use_multiprocessing=use_multiprocessing) validation_steps = len(val_data) else: val_enqueuer = GeneratorEnqueuer( val_data, use_multiprocessing=use_multiprocessing) val_enqueuer.start(workers=workers, max_queue_size=max_queue_size) val_enqueuer_gen = val_enqueuer.get() elif val_gen: val_data = validation_data if isinstance(val_data, Sequence): val_enqueuer_gen = iter(val_data) else: val_enqueuer_gen = val_data else: # Prepare data for validation if len(validation_data) == 2: val_x, val_y = validation_data val_sample_weight = None elif len(validation_data) == 3: val_x, val_y, val_sample_weight = validation_data else: raise ValueError('`validation_data` should be a tuple ' '`(val_x, val_y, val_sample_weight)` ' 'or `(val_x, val_y)`. Found: ' + str(validation_data)) val_x, val_y, val_sample_weights = model._standardize_user_data( val_x, val_y, val_sample_weight) val_data = val_x + val_y + val_sample_weights if model.uses_learning_phase and not isinstance( K.learning_phase(), int): val_data += [0.] for cbk in callbacks: cbk.validation_data = val_data if workers > 0: if is_sequence: enqueuer = OrderedEnqueuer( generator, use_multiprocessing=use_multiprocessing, shuffle=shuffle) else: enqueuer = GeneratorEnqueuer( generator, use_multiprocessing=use_multiprocessing, wait_time=wait_time) enqueuer.start(workers=workers, max_queue_size=max_queue_size) output_generator = enqueuer.get() else: if is_sequence: output_generator = iter(generator) else: output_generator = generator callback_model.stop_training = False # Construct epoch logs. epoch_logs = {} while epoch < epochs: # setup scene instance dictionary model.scene_instance_id_metrics_dict_train = {} # create thread for asynchronous batch metrics calculation (one thread per epoch, joined before final metrics calculation) if multithreading_metrics: label_queue = queue.Queue( ) # threadsafe queue into which we will push (y_pred, y) tuples trainmetrics_thread = threading.Thread( target=metrics_per_batch_thread_handler, args=(label_queue, model.scene_instance_id_metrics_dict_train, params['mask_value'], steps_per_epoch)) trainmetrics_thread.start() #print('thread for calculating the batch train metrics has been started') for m in model.stateful_metric_functions: m.reset_states() callbacks.on_epoch_begin(epoch) steps_done = 0 batch_index = 0 runtime_generator_cumulated = 0. runtime_train_and_predict_on_batch_cumulated = 0. runtime_class_accuracies_cumulated = 0. skip_runtime_avg = 5 # skipping the first few batches to reduce bias due to inital extra time while steps_done < steps_per_epoch: t_start_batch = time() t_start = time() generator_output = next(output_generator) runtime_generator_next = time() - t_start if batch_index >= skip_runtime_avg: runtime_generator_cumulated += runtime_generator_next if not hasattr(generator_output, '__len__'): raise ValueError('Output of generator should be ' 'a tuple `(x, y, sample_weight)` ' 'or `(x, y)`. Found: ' + str(generator_output)) if len(generator_output) == 2: x, y = generator_output sample_weight = None elif len(generator_output) == 3: x, y, sample_weight = generator_output else: raise ValueError('Output of generator should be ' 'a tuple `(x, y, sample_weight)` ' 'or `(x, y)`. Found: ' + str(generator_output)) # build batch logs batch_logs = {} if x is None or len(x) == 0: # Handle data tensors support when no input given # step-size = 1 for data tensors batch_size = 1 elif isinstance(x, list): batch_size = x[0].shape[0] elif isinstance(x, dict): batch_size = list(x.values())[0].shape[0] else: batch_size = x.shape[0] batch_logs['batch'] = batch_index batch_logs['size'] = batch_size t_start = time() callbacks.on_batch_begin(batch_index, batch_logs) runtime_callbacks_on_batch_begin = time() - t_start # remark on label shape: last (fourth) dimension contains in 0 the true labels, in 1 the corresponding sceneinstid (millioncode) t_start = time() # set sample weights if params['nosceneinstweights']: sample_weight = None else: sample_weight = heiner_calculate_sample_weights_batch( y[:, :, 0, 1], generator.length_dict, generator.scene_instance_ids_dict, 'train') # run forward and backward pass and do the gradient descent step batch_loss, y_pred_logits, gradient_norm = heiner_train_and_predict_on_batch( model, x, y[:, :, :, 0], sample_weight=sample_weight, calc_global_gradient_norm=not params['nocalcgradientnorm']) runtime_train_and_predict_on_batch = time() - t_start if batch_index >= skip_runtime_avg: runtime_train_and_predict_on_batch_cumulated += runtime_train_and_predict_on_batch batch_logs['loss'] = batch_loss model.gradient_norm = gradient_norm t_start = time() # from logits to predicted class probabilities y_pred_probs = sigmoid(y_pred_logits, out=y_pred_logits) # last arg: inplace # from probabilities to hard class decisions y_pred = np.greater_equal( y_pred_probs, params['outputthreshold'], out=y_pred_probs) # last arg: inplace # increment metrics for scene instances in batch if multithreading_metrics: # the following two arrays need to be unchanged in order for being thread-safe # assumption 1: batchloader yields array copies (true for moritz loader) # assumption 2: *_and_predict_on_batch return newly allocated arrays label_queue.put((y_pred, y)) else: heiner_calculate_class_accuracies_metrics_per_scene_instance_in_batch( model.scene_instance_id_metrics_dict_train, y_pred, y, params['mask_value']) runtime_class_accuracies = time() - t_start if batch_index >= skip_runtime_avg: runtime_class_accuracies_cumulated += runtime_class_accuracies t_start = time() callbacks.on_batch_end(batch_index, batch_logs) runtime_callbacks_on_batch_end = time() - t_start runtime_batch = time() - t_start_batch # print((' ----> batch {} in epoch {} took in total {:.2f} sec => generator {:.2f} ' + # 'train_and_predict {:.2f}, metrics {:.2f}') # .format(batch_index + 1, epoch + 1, runtime_batch, runtime_generator_next, # runtime_train_and_predict_on_batch, # runtime_class_accuracies)) batch_index += 1 steps_done += 1 if steps_done > skip_runtime_avg and steps_done == steps_per_epoch - 1: print( ' --> batch {} we have average runtimes: generator {:.2f}, train_predict {:.2f}, metrics {:.2f}' .format( batch_index, runtime_generator_cumulated / (steps_done - skip_runtime_avg), runtime_train_and_predict_on_batch_cumulated / (steps_done - skip_runtime_avg), runtime_class_accuracies_cumulated / (steps_done - skip_runtime_avg))) # Epoch finished. if steps_done >= steps_per_epoch and do_validation: if val_gen: val_outs = evaluate_and_predict_generator_with_sceneinst_metrics( model, val_enqueuer_gen, params, multithreading_metrics, validation_steps, workers=0, verbose=1) else: # No need for try/except because # data has already been validated. val_outs = model.evaluate( val_x, val_y, batch_size=batch_size, sample_weight=val_sample_weights, verbose=0) val_outs = to_list(val_outs) # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o if callback_model.stop_training: break if multithreading_metrics: trainmetrics_thread.join() print( ' --> both threads for calculating the batch metrics -- training and validation -- finished all their work' ) callbacks.on_epoch_end(epoch, epoch_logs) epoch += 1 if callback_model.stop_training: break finally: try: if enqueuer is not None: enqueuer.stop() finally: if val_enqueuer is not None: val_enqueuer.stop() if multithreading_metrics: trainmetrics_thread.join() # joined again (harmless) callbacks.on_train_end() return model.history
def fit_with_pseudo_label(self, steps_per_epoch, validation_steps=None, use_checkpoints=True, class_labels=None, verbose=1, use_multiprocessing=False, shuffle=False, workers=1, max_queue_size=10): # Default value if validation steps is none if (validation_steps == None): validation_steps = self.validation_generator.samples // self.batch_size wait_time = 0.01 # in seconds self.model._make_train_function() # Create a checkpoint callback checkpoint = ModelCheckpoint("../models_checkpoints/" + str(self.h5_filename) + ".h5", monitor='val_acc', verbose=1, save_best_only=True, save_weights_only=True, mode='auto', period=1) # Generate callbacks callback_list = [] if use_checkpoints: callback_list.append(checkpoint) # Init train counters epoch = 0 validation_data = self.validation_generator do_validation = bool(validation_data) self.model._make_train_function() if do_validation: self.model._make_test_function() val_gen = (hasattr(validation_data, 'next') or hasattr(validation_data, '__next__') or isinstance(validation_data, Sequence)) if (val_gen and not isinstance(validation_data, Sequence) and not validation_steps): raise ValueError('`validation_steps=None` is only valid for a' ' generator based on the `keras.utils.Sequence`' ' class. Please specify `validation_steps` or use' ' the `keras.utils.Sequence` class.') # Prepare display labels. out_labels = self.model.metrics_names callback_metrics = out_labels + ['val_' + n for n in out_labels] # Prepare train callbacks self.model.history = cbks.History() callbacks = [cbks.BaseLogger()] + (callback_list or []) + \ [self.model.history] if verbose: callbacks += [cbks.ProgbarLogger(count_mode='steps')] callbacks = cbks.CallbackList(callbacks) # it's possible to callback a different model than self: if hasattr(self.model, 'callback_model') and self.model.callback_model: callback_model = self.model.callback_model else: callback_model = self.model callbacks.set_model(callback_model) is_sequence = isinstance(self.train_generator, Sequence) if not is_sequence and use_multiprocessing and workers > 1: warnings.warn( UserWarning('Using a generator with `use_multiprocessing=True`' ' and multiple workers may duplicate your data.' ' Please consider using the`keras.utils.Sequence' ' class.')) if is_sequence: steps_per_epoch = len(self.train_generator) enqueuer = None val_enqueuer = None callbacks.set_params({ 'epochs': self.epochs, 'steps': steps_per_epoch, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics, }) callbacks.on_train_begin() try: if do_validation and not val_gen: # Prepare data for validation if len(validation_data) == 2: val_x, val_y = validation_data val_sample_weight = None elif len(validation_data) == 3: val_x, val_y, val_sample_weight = validation_data else: raise ValueError('`validation_data` should be a tuple ' '`(val_x, val_y, val_sample_weight)` ' 'or `(val_x, val_y)`. Found: ' + str(validation_data)) val_x, val_y, val_sample_weights = self.model._standardize_user_data( val_x, val_y, val_sample_weight) val_data = val_x + val_y + val_sample_weights if self.model.uses_learning_phase and not isinstance( K.learning_phase(), int): val_data += [0.] for cbk in callbacks: cbk.validation_data = val_data if is_sequence: enqueuer = OrderedEnqueuer( self.train_generator, use_multiprocessing=use_multiprocessing, shuffle=shuffle) else: enqueuer = GeneratorEnqueuer( self.train_generator, use_multiprocessing=use_multiprocessing, wait_time=wait_time) enqueuer.start(workers=workers, max_queue_size=max_queue_size) output_generator = enqueuer.get() # Train the model # Construct epoch logs. epoch_logs = {} # Epochs while epoch < self.epochs: callbacks.on_epoch_begin(epoch) steps_done = 0 batch_index = 0 # Steps per epoch while steps_done < steps_per_epoch: generator_output = next(output_generator) if len(generator_output) == 2: x, y = generator_output sample_weight = None elif len(generator_output) == 3: x, y, sample_weight = generator_output else: raise ValueError('Output of generator should be ' 'a tuple `(x, y, sample_weight)` ' 'or `(x, y)`. Found: ' + str(generator_output)) #========================== # Mini-batch #========================== if (self.print_pseudo_generate): print '' print 'Generating pseudo-labels...' verbose = 1 else: verbose = 0 if self.no_label_generator.samples > 0: no_label_output = self.model.predict_generator( self.no_label_generator, self.no_label_generator.samples, verbose=verbose) # One-hot encoded self.no_label_generator.classes = np.argmax( no_label_output, axis=1) # Concat Pseudo labels with true labels x_pseudo, y_pseudo = next(self.no_label_generator) x, y = np.concatenate((x, x_pseudo), axis=0), np.concatenate( (y, y_pseudo), axis=0) # build batch logs batch_logs = {} if isinstance(x, list): batch_size = x[0].shape[0] elif isinstance(x, dict): batch_size = list(x.values())[0].shape[0] else: batch_size = x.shape[0] batch_logs['batch'] = batch_index batch_logs['size'] = batch_size callbacks.on_batch_begin(batch_index, batch_logs) # Runs a single gradient update on a single batch of data scalar_training_loss = self.model.train_on_batch(x=x, y=y) if not isinstance(scalar_training_loss, list): scalar_training_loss = [scalar_training_loss] for l, o in zip(out_labels, scalar_training_loss): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) #========================== # end Mini-batch #========================== batch_index += 1 steps_done += 1 if steps_done >= steps_per_epoch and do_validation: if val_gen: val_outs = self.model.evaluate_generator( validation_data, validation_steps, workers=workers, use_multiprocessing=use_multiprocessing, max_queue_size=max_queue_size) else: # No need for try/except because # data has already been validated. val_outs = self.model.evaluate( val_x, val_y, batch_size=batch_size, sample_weight=val_sample_weights, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o # Epoch finished. callbacks.on_epoch_end(epoch, epoch_logs) epoch += 1 finally: try: if enqueuer is not None: enqueuer.stop() finally: if val_enqueuer is not None: val_enqueuer.stop() callbacks.on_train_end() return self.model.history