コード例 #1
0
def build_callbacks(config, generator, n_samples, other_generators=None):
    # For this model, we want to monitor F1 for early stopping and
    # model checkpointing.  The way to do that is for the metrics callback
    # compute F1, put it in the logs dictionary that's passed to
    # on_epoch_end, and to pass that to the early stopping and model
    # checkpointing callbacks.
    controller_callbacks = []
    controller_callbacks.append(
            keras.callbacks.EarlyStopping(
                monitor=config.callback_monitor,
                mode=config.callback_monitor_mode,
                patience=config.patience,
                verbose=1))

    if 'persistent' in config.mode:
        controller_callbacks.append(
                keras.callbacks.ModelCheckpoint(
                    monitor=config.callback_monitor,
                    mode=config.callback_monitor_mode,
                    filepath=config.model_path + config.model_checkpoint_file,
                    save_best_only=config.save_best_only,
                    verbose=1))

    controller = MetricsCallback(config, generator, n_samples,
            callbacks=controller_callbacks,
            other_generators=other_generators)

    callbacks = []
    callbacks.append(controller)
    callbacks.append(modeling.callbacks.DenseWeightNormCallback(config))

    return callbacks
コード例 #2
0
def build_callbacks(config, generator, n_samples, dictionary, target_map):
    callbacks = []
    mc = MetricsCallback(config, generator, n_samples, dictionary, target_map)
    wn = DenseWeightNormCallback(config)
    es = keras.callbacks.EarlyStopping(patience=config.patience, verbose=1)
    callbacks.extend([mc, wn, es])
    if 'persistent' in config.mode:
        cp = keras.callbacks.ModelCheckpoint(
                filepath=config.model_path + 'model.h5',
                save_best_only=True)
        callbacks.append(cp)

    return callbacks
コード例 #3
0
def build_callbacks(model, config, X_validation=None, y_validation=None, marshaller=None):
    callbacks = []

    callbacks.append(keras.callbacks.EarlyStopping(
            patience=config.patience, verbose=1))

    callbacks.append(DenseWeightNormCallback(config))

    if 'persistent' in config.mode:
        callbacks.append(keras.callbacks.ModelCheckpoint(
                filepath=os.path.join(config.model_path, config.checkpoint_name),
                save_best_only=config.save_best_only,
                verbose=1))

    pred_cb = PredictionCallback(
            X_validation,
            logger=config.logger,
            marshaller=marshaller,
            batch_size=config.batch_size)

    pred_callbacks = []

    if config.confusion_matrix:
        if isinstance(model, keras.models.Graph):
            assert marshaller is not None
        assert X_validation is not None
        assert y_validation is not None
        confusion_cb = ConfusionMatrix(
                X_validation, y_validation,
                logger=config.logger,
                marshaller=marshaller)
        pred_callbacks.append(confusion_cb)

    if config.classification_report:
        if isinstance(model, keras.models.Graph):
            assert marshaller is not None
        assert X_validation is not None
        assert y_validation is not None
        report_cb = ClassificationReport(
                X_validation,
                y_validation,
                logger=config.logger,
                marshaller=marshaller)
        pred_callbacks.append(report_cb)

    for pcb in pred_callbacks:
        pred_cb.add(pcb)

    if len(pred_callbacks):
        callbacks.append(pred_cb)

    return callbacks
コード例 #4
0
def build_callbacks(config, generator, n_samples, dictionary, target_map, pool):
    callbacks = []
    mc = MetricsCallback(config, generator, n_samples, dictionary, target_map)
    wn = DenseWeightNormCallback(config)
    es = keras.callbacks.EarlyStopping(patience=config.patience, verbose=1)
    callbacks.extend([mc, wn, es])
    if 'persistent' in config.mode:
        cp = keras.callbacks.ModelCheckpoint(
                filepath=config.model_path + 'model.h5',
                save_best_only=True)
        callbacks.append(cp)

    if pool is not None:
        cc = CurriculumCallback(config.logger, pool,
                threshold=config.length_curriculum_change_threshold,
                frequency=config.length_curriculum_change_frequency,
                monitor=config.length_curriculum_monitor)
        callbacks.append(cc)

    return callbacks
コード例 #5
0
ファイル: train_keras.py プロジェクト: joshloyal/modeling
def main(args):
    model_id = build_model_id(args)
    model_path = build_model_path(args, model_id)
    setup_model_dir(args, model_path)
    sys.stdout, sys.stderr = setup_logging(args, model_path)

    x_train, y_train = load_model_data(args.train_file,
            args.data_name, args.target_name)
    x_validation, y_validation = load_model_data(
            args.validation_file,
            args.data_name, args.target_name)

    rng = np.random.RandomState(args.seed)

    if args.n_classes > -1:
        n_classes = args.n_classes
    else:
        n_classes = max(y_train)+1

    n_classes, target_names, class_weight = load_target_data(args, n_classes)

    if len(class_weight) == 0:
        n_samples = len(y_train)
        print('n_samples', n_samples)
        print('classes', range(n_classes))
        print('weights', n_samples / (n_classes * np.bincount(y_train)))
        class_weight = dict(zip(range(n_classes),
            n_samples / (n_classes * np.bincount(y_train))))
    print('class_weight', class_weight)

    logging.debug("n_classes {0} min {1} max {2}".format(
        n_classes, min(y_train), max(y_train)))

    y_train_one_hot = np_utils.to_categorical(y_train, n_classes)
    y_validation_one_hot = np_utils.to_categorical(y_validation, n_classes)

    logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
    logging.debug("x_train " + str(x_train.shape))

    min_vocab_index = np.min(x_train)
    max_vocab_index = np.max(x_train)
    logging.debug("min vocab index {0} max vocab index {1}".format(
        min_vocab_index, max_vocab_index))

    json_cfg = load_model_json(args, x_train, n_classes)

    logging.debug("loading model")

    sys.path.append(args.model_dir)
    import model
    from model import build_model

    #######################################################################      
    # Subsetting
    #######################################################################      
    if args.subsetting_function:
        subsetter = getattr(model, args.subsetting_function)
    else:
        subsetter = None

    def take_subset(subsetter, path, x, y, y_one_hot, n):
        if subsetter is None:
            return x[0:n], y[0:n], y_one_hot[0:n]
        else:
            mask = subsetter(path)
            idx = np.where(mask)[0]
            idx = idx[0:n]
        return x[idx], y[idx], y_one_hot[idx]

    x_train, y_train, y_train_one_hot = take_subset(
            subsetter, args.train_file,
            x_train, y_train, y_train_one_hot,
            n=args.n_train)

    x_validation, y_validation, y_validation_one_hot = take_subset(
            subsetter, args.validation_file,
            x_validation, y_validation, y_validation_one_hot,
            n=args.n_validation)

    #######################################################################      
    # Preprocessing
    #######################################################################      
    if args.preprocessing_class:
        preprocessor = getattr(model, args.preprocessing_class)(seed=args.seed)
    else:
        preprocessor = modeling.preprocess.NullPreprocessor()

    logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
    logging.debug("x_train " + str(x_train.shape))

    model_cfg = ModelConfig(**json_cfg)
    logging.info("model_cfg " + str(model_cfg))
    model = build_model(model_cfg)
    setattr(model, 'stop_training', False)

    logging.info('model has {n_params} parameters'.format(
        n_params=count_parameters(model)))

    if len(args.extra_train_file) > 1:
        callbacks = keras.callbacks.CallbackList()
    else:
        callbacks = []

    save_model_info(args, model_path, model_cfg)

    if not args.no_save:
        if args.save_all_checkpoints:
            filepath = model_path + '/model-{epoch:04d}.h5'
        else:
            filepath = model_path + '/model.h5'
        callbacks.append(ModelCheckpoint(
            filepath=filepath,
            verbose=1,
            save_best_only=not args.save_every_epoch))

    callback_logger = logging.info if args.log else callable_print

    if args.n_epochs < sys.maxsize:
        # Number of epochs overrides patience.  If the number of epochs
        # is specified on the command line, the model is trained for
        # exactly that number; otherwise, the model is trained with
        # early stopping using the patience specified in the model 
        # configuration.
        callbacks.append(EarlyStopping(
            monitor='val_loss', patience=model_cfg.patience, verbose=1))

    if args.classification_report:
        cr = ClassificationReport(x_validation, y_validation,
                callback_logger,
                target_names=target_names)
        callbacks.append(cr)

    if model_cfg.optimizer == 'SGD':
        callbacks.append(SingleStepLearningRateSchedule(patience=10))

    if len(args.extra_train_file) > 1:
        args.extra_train_file.append(args.train_file)
        logging.info("Using the following files for training: " +
                ','.join(args.extra_train_file))

        train_file_iter = itertools.cycle(args.extra_train_file)
        current_train = args.train_file

        callbacks._set_model(model)
        callbacks.on_train_begin(logs={})

        epoch = batch = 0

        while True:
            x_train, y_train_one_hot = preprocessor.fit_transform(
                    x_train, y_train_one_hot)
            x_validation, y_validation_one_hot = preprocessor.transform(
                    x_validation, y_validation_one_hot)

            iteration = batch % len(args.extra_train_file)

            logging.info("epoch {epoch} iteration {iteration} - training with {train_file}".format(
                    epoch=epoch, iteration=iteration, train_file=current_train))
            callbacks.on_epoch_begin(epoch, logs={})

            n_train = x_train.shape[0]

            callbacks.on_batch_begin(batch, logs={'size': n_train})

            index_array = np.arange(n_train)
            if args.shuffle:
                rng.shuffle(index_array)

            batches = keras.models.make_batches(n_train, model_cfg.batch_size)
            logging.info("epoch {epoch} iteration {iteration} - starting {n_batches} batches".format(
                    epoch=epoch, iteration=iteration, n_batches=len(batches)))

            avg_train_loss = avg_train_accuracy = 0.
            for batch_index, (batch_start, batch_end) in enumerate(batches):
                batch_ids = index_array[batch_start:batch_end]

                if isinstance(model, keras.models.Graph):
                    data = {
                            'input': x_train[batch_ids],
                            'output': y_train_one_hot[batch_ids]
                            }
                    train_loss = model.train_on_batch(data, class_weight=class_weight)
                    train_accuracy = 0.
                else:
                    train_loss, train_accuracy = model.train_on_batch(
                            x_train[batch_ids], y_train_one_hot[batch_ids],
                            accuracy=True, class_weight=class_weight)

                batch_end_logs = {'loss': train_loss, 'accuracy': train_accuracy}

                avg_train_loss = (avg_train_loss * batch_index + train_loss)/(batch_index + 1)
                avg_train_accuracy = (avg_train_accuracy * batch_index + train_accuracy)/(batch_index + 1)

                callbacks.on_batch_end(batch,
                        logs={'loss': train_loss, 'accuracy': train_accuracy})

            logging.info("epoch {epoch} iteration {iteration} - finished {n_batches} batches".format(
                    epoch=epoch, iteration=iteration, n_batches=len(batches)))

            logging.info("epoch {epoch} iteration {iteration} - loss: {loss} - acc: {acc}".format(
                    epoch=epoch, iteration=iteration, loss=avg_train_loss, acc=avg_train_accuracy))

            batch += 1

            # Validation frequency (this if-block) doesn't necessarily
            # occur in the same iteration as beginning of an epoch
            # (next if-block), so model.evaluate appears twice here.
            kwargs = { 'verbose': 0 if args.log else 1 }
            pargs = []
            validation_data = {}
            if isinstance(model, keras.models.Graph):
                validation_data = {
                        'input': x_validation,
                        'output': y_validation_one_hot
                        }
                pargs = [validation_data]
            else:
                pargs = [x_validation, y_validation_one_hot]
                kwargs['show_accuracy'] = True

            if (iteration + 1) % args.validation_freq == 0:
                if isinstance(model, keras.models.Graph):
                    val_loss = model.evaluate(*pargs, **kwargs)
                    y_hat = model.predict(validation_data)
                    val_acc = accuracy_score(y_validation, np.argmax(y_hat['output'], axis=1))
                else:
                    val_loss, val_acc = model.evaluate(
                            *pargs, **kwargs)
                logging.info("epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}".format(
                        epoch=epoch, iteration=iteration, val_loss=val_loss, val_acc=val_acc))
                epoch_end_logs = {'iteration': iteration, 'val_loss': val_loss, 'val_acc': val_acc}
                callbacks.on_epoch_end(epoch, epoch_end_logs)

            if batch % len(args.extra_train_file) == 0:
                if isinstance(model, keras.models.Graph):
                    val_loss = model.evaluate(*pargs, **kwargs)
                    y_hat = model.predict(validation_data)
                    val_acc = accuracy_score(y_validation, np.argmax(y_hat['output'], axis=1))
                else:
                    val_loss, val_acc = model.evaluate(
                            *pargs, **kwargs)
                logging.info("epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}".format(
                        epoch=epoch, iteration=iteration, val_loss=val_loss, val_acc=val_acc))
                epoch_end_logs = {'iteration': iteration, 'val_loss': val_loss, 'val_acc': val_acc}
                epoch += 1
                callbacks.on_epoch_end(epoch, epoch_end_logs)

            if model.stop_training:
                logging.info("epoch {epoch} iteration {iteration} - done training".format(
                    epoch=epoch, iteration=iteration))
                break

            current_train = next(train_file_iter)
            x_train, y_train = load_model_data(current_train,
                    args.data_name, args.target_name)
            y_train_one_hot = np_utils.to_categorical(y_train, n_classes)

            if epoch > args.n_epochs:
                break

        callbacks.on_train_end(logs={})
    else:
        x_train, y_train_one_hot = preprocessor.fit_transform(
                x_train, y_train_one_hot)
        x_validation, y_validation_one_hot = preprocessor.transform(
                x_validation, y_validation_one_hot)
        if isinstance(model, keras.models.Graph):
            data = {
                    'input': x_train,
                    'output': y_train_one_hot
                    }
            validation_data = {
                    'input': x_validation,
                    'output': y_validation_one_hot
                    }
            model.fit(data,
                shuffle=args.shuffle,
                nb_epoch=args.n_epochs,
                batch_size=model_cfg.batch_size,
                validation_data=validation_data,
                callbacks=callbacks,
                class_weight=class_weight,
                verbose=2 if args.log else 1)
            y_hat = model.predict(validation_data)
            print('val_acc %.04f' % 
                    accuracy_score(y_validation, np.argmax(y_hat['output'], axis=1)))
        else:
            model.fit(x_train, y_train_one_hot,
                shuffle=args.shuffle,
                nb_epoch=args.n_epochs,
                batch_size=model_cfg.batch_size,
                show_accuracy=True,
                validation_data=(x_validation, y_validation_one_hot),
                callbacks=callbacks,
                class_weight=class_weight,
                verbose=2 if args.log else 1)
コード例 #6
0
if missinglink_callback.rm_active:
    checkpoints_directory = '/output/checkpoints'
    if not os.path.exists(checkpoints_directory):
        os.mkdir(checkpoints_directory)
    checkpoint_format = checkpoints_directory + '/weights_epoch-{epoch:02d}_loss-{loss:.4f}.h5'
    save_models_callback = keras.callbacks.ModelCheckpoint(
        checkpoint_format,
        monitor='val_loss',
        verbose=1,
        save_best_only=True,
        save_weights_only=True,
        mode='auto',
        period=5,
    )
    callbacks.append(save_models_callback)

    tensor_board_path = '/output/tensorboard'
    save_tb_callback = keras.callbacks.TensorBoard(log_dir=tensor_board_path)
    callbacks.append(save_tb_callback)


def get_transfer_model():
    # import inception with pre-trained weights. do not include fully #connected layers
    if MODEL == "resnet50":
        base_model = applications.ResNet50(weights='imagenet',
                                           include_top=False)
    elif MODEL == "mobilenet":
        base_model = applications.MobileNet(weights='imagenet',
                                            include_top=False,
                                            input_shape=input_shape)
コード例 #7
0
    def minimize(self, seed_input=None, max_iter=200,
                 input_modifiers=None, grad_modifier=None,
                 callbacks=None, verbose=True):
        """Performs gradient descent on the input image with respect to defined losses.
        Args:
            seed_input: An N-dim numpy array of shape: `(samples, channels, image_dims...)` if `image_data_format=
                channels_first` or `(samples, image_dims..., channels)` if `image_data_format=channels_last`.
                Seeded with random noise if set to None. (Default value = None)
            max_iter: The maximum number of gradient descent iterations. (Default value = 200)
            input_modifiers: A list of [InputModifier](vis.input_modifiers#inputmodifier) instances specifying
                how to make `pre` and `post` changes to the optimized input during the optimization process.
                `pre` is applied in list order while `post` is applied in reverse order. For example,
                `input_modifiers = [f, g]` means that `pre_input = g(f(inp))` and `post_input = f(g(inp))`
            grad_modifier: gradient modifier to use. See [grad_modifiers](vis.grad_modifiers.md). If you don't
                specify anything, gradients are unchanged. (Default value = None)
            callbacks: A list of [OptimizerCallback](vis.callbacks#optimizercallback) instances to trigger.
            verbose: Logs individual losses at the end of every gradient descent iteration.
                Very useful to estimate loss weight factor(s). (Default value = True)
        Returns:
            The tuple of `(optimized input, grads with respect to wrt, wrt_value)` after gradient descent iterations.
        """
        seed_input = self._get_seed_input(seed_input)
        input_modifiers = input_modifiers or []
        grad_modifier = _identity if grad_modifier is None else get(grad_modifier)

        callbacks = callbacks or []
        if verbose:
            callbacks.append(_PRINT_CALLBACK)

        cache = None
        best_loss = float('inf')
        best_input = None

        grads = None
        wrt_value = None

        all_losses = []
        for i in range(max_iter):
            # Apply modifiers `pre` step
            for modifier in input_modifiers:
                seed_input = modifier.pre(seed_input)

            # 0 learning phase for 'test'
            computed_values = self.compute_fn([seed_input, 0])
            losses = computed_values[:len(self.loss_names)]
            named_losses = list(zip(self.loss_names, losses))
            overall_loss, grads, wrt_value = computed_values[len(self.loss_names):]

            # TODO: theano grads shape is inconsistent for some reason. Patch for now and investigate later.
            if grads.shape != wrt_value.shape:
                grads = np.reshape(grads, wrt_value.shape)

            # Apply grad modifier.
            grads = grad_modifier(grads)

            # Trigger callbacks
            for c in (callbacks):
                c.callback(i, self, named_losses, overall_loss, grads, wrt_value)

            # Gradient descent update.
            # It only makes sense to do this if wrt_tensor is input_tensor. Otherwise shapes wont match for the update.
            if self.wrt_tensor is self.input_tensor:
                step, cache = self._rmsprop(grads, cache)
                seed_input += step

            # Apply modifiers `post` step
            for modifier in reversed(input_modifiers):
                seed_input = modifier.post(seed_input)

            all_losses.append(named_losses)
            if overall_loss < best_loss:
                best_loss = overall_loss.copy()
                best_input = seed_input.copy()

        # Trigger on_end
        for c in callbacks:
            c.on_end()

        img = best_input[0]
        #img = utils.deprocess_input(best_input[0], self.input_range)

        return img, grads, wrt_value,all_losses,named_losses,overall_loss
コード例 #8
0
ファイル: dotnum_image.py プロジェクト: mattroos/seq2seq
model.summary()

## Train the model
epochs = 75
# Save the model architecture
basename = 'epoch{:03d}_seed'.format(epochs) + '{:04d}'.format(
    seed) + '_usdot_real_reluDecode'
json_string = model.to_json()
open('model_dotnum_' + basename + '_arch.json', 'w').write(json_string)

# history = History()
callbacks = [History()]
weights_file = 'model_dotnum_' + basename + '_weights.h5'  # where weights will be saved
callbacks.append(
    ModelCheckpoint(filepath=weights_file,
                    monitor='val_loss',
                    save_best_only=True))

print('\nTraining the model...')
start = time.time()
model.fit(dataTrain,
          labelsTrain,
          batch_size=32,
          epochs=epochs,
          verbose=1,
          callbacks=callbacks,
          validation_split=0.0,
          validation_data=(dataVal, labelsVal),
          shuffle=True,
          class_weight=None,
          sample_weight=None,
コード例 #9
0
ファイル: train_keras.py プロジェクト: Libardo1/modeling
def main(args):
    model_id = build_model_id(args)
    model_path = build_model_path(args, model_id)
    setup_model_dir(args, model_path)
    sys.stdout, sys.stderr = setup_logging(args, model_path)

    x_train, y_train = load_model_data(args.train_file,
            args.data_name, args.target_name)
    x_validation, y_validation = load_model_data(
            args.validation_file,
            args.data_name, args.target_name)

    rng = np.random.RandomState(args.seed)

    if args.n_classes > -1:
        n_classes = args.n_classes
    else:
        n_classes = max(y_train)+1

    n_classes, target_names, class_weight = load_target_data(args, n_classes)

    if class_weight is None and args.class_weight_auto:
        n_samples = len(y_train)
        weights = float(n_samples) / (n_classes * np.bincount(y_train))
        if args.class_weight_exponent:
            weights = weights**args.class_weight_exponent
        class_weight = dict(zip(range(n_classes), weights))

    if args.verbose:
        logging.debug("n_classes {0} min {1} max {2}".format(
            n_classes, min(y_train), max(y_train)))

    y_train_one_hot = np_utils.to_categorical(y_train, n_classes)
    y_validation_one_hot = np_utils.to_categorical(y_validation, n_classes)

    if args.verbose:
        logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
        logging.debug("x_train " + str(x_train.shape))

    min_vocab_index = np.min(x_train)
    max_vocab_index = np.max(x_train)

    if args.verbose:
        logging.debug("min vocab index {0} max vocab index {1}".format(
            min_vocab_index, max_vocab_index))

    json_cfg = load_model_json(args, x_train, n_classes)

    if args.verbose:
        logging.debug("loading model")

    sys.path.append(args.model_dir)
    import model
    from model import build_model

    #######################################################################      
    # Subsetting
    #######################################################################      
    if args.subsetting_function:
        subsetter = getattr(M, args.subsetting_function)
    else:
        subsetter = None

    def take_subset(subsetter, path, x, y, y_one_hot, n):
        if subsetter is None:
            return x[0:n], y[0:n], y_one_hot[0:n]
        else:
            mask = subsetter(path)
            idx = np.where(mask)[0]
            idx = idx[0:n]
        return x[idx], y[idx], y_one_hot[idx]

    x_train, y_train, y_train_one_hot = take_subset(
            subsetter, args.train_file,
            x_train, y_train, y_train_one_hot,
            n=args.n_train)

    x_validation, y_validation, y_validation_one_hot = take_subset(
            subsetter, args.validation_file,
            x_validation, y_validation, y_validation_one_hot,
            n=args.n_validation)

    #######################################################################      
    # Preprocessing
    #######################################################################      
    if args.preprocessing_class:
        preprocessor = getattr(M, args.preprocessing_class)(seed=args.seed)
    else:
        preprocessor = modeling.preprocess.NullPreprocessor()

    if args.verbose:
        logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
        logging.debug("x_train " + str(x_train.shape))

    model_cfg = ModelConfig(**json_cfg)
    if args.verbose:
        logging.info("model_cfg " + str(model_cfg))
    net = build_model(model_cfg)
    setattr(net, 'stop_training', False)

    marshaller = None
    if isinstance(net, keras.models.Graph):
        marshaller = getattr(model, args.graph_marshalling_class)()

    logging.info('model has {n_params} parameters'.format(
        n_params=count_parameters(net)))

    if len(args.extra_train_file) > 1:
        callbacks = keras.callbacks.CallbackList()
    else:
        callbacks = []

    save_model_info(args, model_path, model_cfg)


    callback_logger = logging.info if args.log else callable_print

    #######################################################################      
    # Callbacks that need validation set predictions.
    #######################################################################      

    pc = PredictionCallback(x_validation, callback_logger,
            marshaller=marshaller, batch_size=model_cfg.batch_size)
    callbacks.append(pc)

    if args.classification_report:
        cr = ClassificationReport(x_validation, y_validation,
                callback_logger,
                target_names=target_names)
        pc.add(cr)
    
    if args.confusion_matrix:
        cm = ConfusionMatrix(x_validation, y_validation,
                callback_logger)
        pc.add(cm)

    def get_mode(metric_name):
        return {
                'val_loss': 'min',
                'val_acc': 'max',
                'val_f1': 'max',
                'val_f2': 'max',
                'val_f0.5': 'max'
                }[metric_name]

    if args.early_stopping or args.early_stopping_metric is not None:
        es = EarlyStopping(monitor=args.early_stopping_metric,
                mode=get_mode(args.early_stopping_metric),
                patience=model_cfg.patience,
                verbose=1)
        cb = DelegatingMetricCallback(
                x_validation, y_validation, callback_logger,
                delegate=es,
                metric_name=args.early_stopping_metric,
                marshaller=marshaller)
        pc.add(cb)

    if not args.no_save:
        if args.save_all_checkpoints:
            filepath = model_path + '/model-{epoch:04d}.h5'
        else:
            filepath = model_path + '/model.h5'
        mc = ModelCheckpoint(
            filepath=filepath,
            mode=get_mode(args.checkpoint_metric),
            verbose=1,
            monitor=args.checkpoint_metric,
            save_best_only=not args.save_every_epoch)
        cb = DelegatingMetricCallback(
                x_validation, y_validation, callback_logger,
                delegate=mc,
                metric_name=args.checkpoint_metric,
                marshaller=marshaller)
        pc.add(cb)

    if model_cfg.optimizer == 'SGD':
        callbacks.append(SingleStepLearningRateSchedule(patience=10))

    if len(args.extra_train_file) > 1:
        args.extra_train_file.append(args.train_file)
        logging.info("Using the following files for training: " +
                ','.join(args.extra_train_file))

        train_file_iter = itertools.cycle(args.extra_train_file)
        current_train = args.train_file

        callbacks._set_model(net)
        callbacks.on_train_begin(logs={})

        epoch = batch = 0

        while True:
            x_train, y_train_one_hot = preprocessor.fit_transform(
                    x_train, y_train_one_hot)
            x_validation, y_validation_one_hot = preprocessor.transform(
                    x_validation, y_validation_one_hot)

            iteration = batch % len(args.extra_train_file)

            logging.info("epoch {epoch} iteration {iteration} - training with {train_file}".format(
                    epoch=epoch, iteration=iteration, train_file=current_train))
            callbacks.on_epoch_begin(epoch, logs={})

            n_train = x_train.shape[0]

            callbacks.on_batch_begin(batch, logs={'size': n_train})

            index_array = np.arange(n_train)
            if args.shuffle:
                rng.shuffle(index_array)

            batches = keras.models.make_batches(n_train, model_cfg.batch_size)
            logging.info("epoch {epoch} iteration {iteration} - starting {n_batches} batches".format(
                    epoch=epoch, iteration=iteration, n_batches=len(batches)))

            avg_train_loss = avg_train_accuracy = 0.
            for batch_index, (batch_start, batch_end) in enumerate(batches):
                batch_ids = index_array[batch_start:batch_end]

                if isinstance(net, keras.models.Graph):
                    train_data = marshaller.marshal(
                            x_train[batch_ids], y_train_one_hot[batch_ids])
                    train_loss = net.train_on_batch(
                            train_data, class_weight=class_weight)
                    # It looks like train_on_batch returns a different
                    # type for graph than sequential models.
                    train_loss = train_loss[0]
                    train_accuracy = 0.
                else:
                    train_loss, train_accuracy = net.train_on_batch(
                            x_train[batch_ids], y_train_one_hot[batch_ids],
                            accuracy=True, class_weight=class_weight)

                batch_end_logs = {'loss': train_loss, 'accuracy': train_accuracy}

                avg_train_loss = (avg_train_loss * batch_index + train_loss)/(batch_index + 1)
                avg_train_accuracy = (avg_train_accuracy * batch_index + train_accuracy)/(batch_index + 1)

                callbacks.on_batch_end(batch,
                        logs={'loss': train_loss, 'accuracy': train_accuracy})

            logging.info("epoch {epoch} iteration {iteration} - finished {n_batches} batches".format(
                    epoch=epoch, iteration=iteration, n_batches=len(batches)))

            logging.info("epoch {epoch} iteration {iteration} - loss: {loss} - acc: {acc}".format(
                    epoch=epoch, iteration=iteration, loss=avg_train_loss, acc=avg_train_accuracy))

            batch += 1

            # Validation frequency (this if-block) doesn't necessarily
            # occur in the same iteration as beginning of an epoch
            # (next if-block), so net.evaluate appears twice here.
            kwargs = {
                    'batch_size': model_cfg.batch_size,
                    'verbose': 0 if args.log else 1 
                    }
            pargs = []
            validation_data = {}
            if isinstance(net, keras.models.Graph):
                validation_data = marshaller.marshal(
                        x_validation, y_validation_one_hot)
                pargs = [validation_data]
            else:
                pargs = [x_validation, y_validation_one_hot]
                kwargs['show_accuracy'] = True

            if (iteration + 1) % args.validation_freq == 0:
                if isinstance(net, keras.models.Graph):
                    val_loss = net.evaluate(*pargs, **kwargs)
                    y_hat = net.predict(validation_data, batch_size=model_cfg.batch_size)
                    val_acc = accuracy_score(y_validation, np.argmax(y_hat['output'], axis=1))
                else:
                    val_loss, val_acc = net.evaluate(
                            *pargs, **kwargs)
                logging.info("epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}".format(
                        epoch=epoch, iteration=iteration, val_loss=val_loss, val_acc=val_acc))
                epoch_end_logs = {'iteration': iteration, 'val_loss': val_loss, 'val_acc': val_acc}
                callbacks.on_epoch_end(epoch, epoch_end_logs)

            if batch % len(args.extra_train_file) == 0:
                if isinstance(net, keras.models.Graph):
                    val_loss = net.evaluate(*pargs, **kwargs)
                    y_hat = net.predict(validation_data, batch_size=model_cfg.batch_size)
                    val_acc = accuracy_score(y_validation, np.argmax(y_hat['output'], axis=1))
                else:
                    val_loss, val_acc = net.evaluate(
                            *pargs, **kwargs)
                logging.info("epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}".format(
                        epoch=epoch, iteration=iteration, val_loss=val_loss, val_acc=val_acc))
                epoch_end_logs = {'iteration': iteration, 'val_loss': val_loss, 'val_acc': val_acc}
                epoch += 1
                callbacks.on_epoch_end(epoch, epoch_end_logs)

            if net.stop_training:
                logging.info("epoch {epoch} iteration {iteration} - done training".format(
                    epoch=epoch, iteration=iteration))
                break

            current_train = next(train_file_iter)
            x_train, y_train = load_model_data(current_train,
                    args.data_name, args.target_name)
            y_train_one_hot = np_utils.to_categorical(y_train, n_classes)

            if epoch > args.n_epochs:
                break

        callbacks.on_train_end(logs={})
    else:
        x_train, y_train_one_hot = preprocessor.fit_transform(
                x_train, y_train_one_hot)
        x_validation, y_validation_one_hot = preprocessor.transform(
                x_validation, y_validation_one_hot)

        if isinstance(net, keras.models.Graph):
            train_data = marshaller.marshal(
                    x_train, y_train_one_hot)
            validation_data = marshaller.marshal(
                    x_validation, y_validation_one_hot)
            net.fit(train_data,
                shuffle=args.shuffle,
                nb_epoch=args.n_epochs,
                batch_size=model_cfg.batch_size,
                validation_data=validation_data,
                callbacks=callbacks,
                class_weight=class_weight,
                verbose=2 if args.log else 1)
        else:
            net.fit(x_train, y_train_one_hot,
                shuffle=args.shuffle,
                nb_epoch=args.n_epochs,
                batch_size=model_cfg.batch_size,
                show_accuracy=True,
                validation_data=(x_validation, y_validation_one_hot),
                callbacks=callbacks,
                class_weight=class_weight,
                verbose=2 if args.log else 1)
コード例 #10
0
def fit(model, *fit_args, plots=['loss'], **fit_kwargs):
    """
    Interactive model training in jupyter notebook
    
    Just replace
        model.fit(model, x, y, nb_epoch=42, ...)
    with
        fit(model, x, y, nb_epoch=42, ...)
    to get interactive training
    """
    nbagg_backend = matplotlib.backends.backend.lower() == 'nbagg'
    if not nbagg_backend:
        warnings.warn(
            '''\nUse nbAgg backend if possible to prevent stacking display of training plots.
        Add import matplotlib; matplotlib.use(\'nbagg\') and restart kernel''')

    nb_epoch = fit_kwargs.get('nb_epoch', 10)

    progress_bar = tqdm.tqdm_notebook(total=nb_epoch)

    class LossHistory(keras.callbacks.Callback):
        def __init__(self):
            super().__init__()
            self.fig = plt.figure()
            self.vals = dict()

        def on_epoch_end(self, epoch, logs={}):
            progress_bar.update()
            for k in logs.keys():
                val_list = self.vals.get(k, [])
                val_list.append(logs[k])
                self.vals[k] = val_list

        def on_train_end(self, logs={}):
            progress_bar.close()

    cancel_button = ipywidgets.Button(description='Cancel')

    def cancel_fitting():
        model.stop_training = True
        learning_thread.join(5)
        print('Cancelled')

    cancel_button.on_click(lambda sender: cancel_fitting())

    loss_history = LossHistory()

    display(cancel_button)

    if nbagg_backend:
        axis = None

        def plot_loss(sender):
            nonlocal axis
            if axis is None:
                axis = plt.gca()
            axis.clear()
            for n, l in loss_history.vals.items():
                if n in plots:
                    axis.semilogy(l, label=n)
            axis.legend(loc='best', frameon=False)
            plt.show()
    else:

        def plot_loss(sender):
            for n, l in loss_history.vals.items():
                if n in plots:
                    plt.semilogy(l, label=n)
            plt.legend(loc='best', frameon=False)
            plt.show()

    plot_button = ipywidgets.Button(description='Plot')
    plot_button.on_click(plot_loss)

    display(plot_button)

    callbacks = fit_kwargs.get('callbacks', [])
    callbacks = callbacks.copy()
    callbacks.append(loss_history)
    fit_kwargs['callbacks'] = callbacks
    fit_kwargs['verbose'] = False
    learning_thread = threading.Thread(
        target=lambda: model.fit(*fit_args, **fit_kwargs))
    learning_thread.start()
コード例 #11
0
ファイル: train_cnn.py プロジェクト: vishalbelsare/crowdastro
def train(training_h5, model_json, weights_path, epochs, batch_size, s3=False,
          s3_bucket=None, using_dataset=False, test_set_index=0):
    """Trains a CNN.

    training_h5: Training HDF5 file.
    model_json: JSON model file.
    weights_path: Output weights HDF5 file.
    epochs: Number of training epochs.
    batch_size: Batch size.
    s3: Whether to periodically dump to Amazon S3. Default False.
    s3_bucket: Name of the bucket to dump to. Must be specified iff s3 is True.
    using_dataset: Whether the given training file is the Zenodo crowdastro
        dataset. Default False (i.e. we are using the generated training file).
    test_set_index: Index of the test set to not use for training. Default 0.
    """
    if s3 and not s3_bucket:
        raise ValueError('Must specify s3_bucket to dump to S3.')
    if not s3 and s3_bucket:
        raise ValueError('s3_bucket was specified but s3 is False.')

    import keras.callbacks
    import keras.models
    from keras.preprocessing.image import ImageDataGenerator
    model = keras.models.model_from_json(model_json.read())
    model.compile(loss='binary_crossentropy', optimizer='adadelta')

    if not using_dataset:
        ir_survey = training_h5.attrs['ir_survey']
        n_nonimage_features = config['surveys'][ir_survey]['n_features']
        features_name = 'raw_features'
    else:
        ir_survey = 'wise'
        n_nonimage_features = 5
        features_name = 'features'

    test_set = set(training_h5['test_sets'][test_set_index, :])
    n_examples = training_h5[features_name].shape[0]
    train_set = [i for i in range(n_examples) if i not in test_set]
    del test_set  # Just to make sure we don't use it.

    training_inputs = training_h5[features_name].value[train_set, :]
    training_outputs = training_h5['labels'].value[train_set]
    assert training_inputs.shape[0] == training_outputs.shape[0]

    # Downsample for class balance.
    zero_indices = (training_outputs == 0).nonzero()[0]
    one_indices = (training_outputs == 1).nonzero()[0]
    subset_zero_indices = numpy.random.choice(
        zero_indices, size=(len(one_indices,)), replace=False)
    all_indices = numpy.hstack([subset_zero_indices, one_indices])
    all_indices.sort()

    training_inputs = training_inputs[all_indices]
    training_outputs = training_outputs[all_indices]
    assert (training_outputs == 1).sum() == (training_outputs == 0).sum()

    class DumpToS3(keras.callbacks.Callback):
        def __init__(self, weights_path, bucket, period=50):
            super().__init__()
            self.weights_path = weights_path
            self.bucket = bucket
            self.period = period

        def on_train_begin(self, logs={}):
            self.epochs = 0

        def on_epoch_end(self, epoch, logs={}):
            self.epochs += 1
            if self.epochs % self.period == 0:
                # Every 50 epochs...
                logging.debug('Dumping to S3...')
                res = subprocess.check_call(
                    ['aws', 's3', 'cp', self.weights_path,
                     's3://' + self.bucket + '/', '--region', 'us-east-1'])
                logging.info('Dumped to S3: {}'.format(res))

    try:
        model.load_weights(weights_path)
        logging.info('Loaded weights.')
    except OSError:
        logging.warning('Couldn\'t load weights file. Creating new file...')
        pass

    callbacks = [
        keras.callbacks.ModelCheckpoint(
                weights_path,
                monitor='val_loss',
                verbose=1,
                save_best_only=False,
                save_weights_only=True,
                mode='auto'),
    ]

    if s3:
        callbacks.append(DumpToS3(weights_path, s3_bucket))

    def create_generator(X, Y):
        """Yields generated images and auxiliary inputs.

        https://github.com/fchollet/keras/issues/3386
        """

        X_im = X[:, n_nonimage_features:].reshape(
            (-1, 1, PATCH_DIAMETER, PATCH_DIAMETER))
        X_au = X[:, :n_nonimage_features]

        while True:
            # Shuffle indices.
            idx = numpy.random.permutation(X.shape[0])
            # Standard image generator.
            datagen = ImageDataGenerator(
                    data_format='channels_first',
                    horizontal_flip=True,
                    vertical_flip=True)
            datagen.fit(X_im)
            # Shuffle the data before batching using known indices.
            batches = datagen.flow(X_im[idx], Y[idx], batch_size=batch_size,
                                   shuffle=False)
            idx0 = 0
            for batch in batches:
                idx1 = idx0 + batch[0].shape[0]

                # Yield ((image, aux), label) tuples.
                to_yield = ([X_au[idx[idx0:idx1]], batch[0]], batch[1])
                yield to_yield

                idx0 = idx1
                if idx1 >= X.shape[0]:
                    break

    model.fit_generator(create_generator(training_inputs, training_outputs),
                        steps_per_epoch=training_inputs.shape[0] // batch_size,
                        epochs=epochs,
                        callbacks=callbacks,
                        workers=1)

    model.save_weights(weights_path, overwrite=True)
コード例 #12
0
ファイル: train.py プロジェクト: s059ff/segnet-camvid-keras
def main():

    # Parse arguments.
    parser = argparse.ArgumentParser()
    kwargs = {
        'type': int,
        'default': 100,
        'help': 'The number of times of learning. default: 100'
    }
    parser.add_argument('-e', '--epochs', **kwargs)
    kwargs = {
        'type': int,
        'default': 10,
        'help': 'The frequency of saving model. default: 10'
    }
    parser.add_argument('-c', '--checkpoint_interval', **kwargs)
    kwargs = {
        'type': int,
        'default': 1,
        'help': 'The number of samples contained per mini batch. default: 1'
    }
    parser.add_argument('-b', '--batch_size', **kwargs)
    kwargs = {
        'default':
        False,
        'action':
        'store_true',
        'help':
        'Whether store all data to GPU. If not specified this option, use both CPU memory and GPU memory.'
    }
    parser.add_argument('--onmemory', **kwargs)
    args = parser.parse_args()

    # Prepare training data.
    dataset = np.load('./temp/dataset.npz')
    train_x = dataset['train_x']
    train_y = dataset['train_y']
    test_x = dataset['test_x']
    test_y = dataset['test_y']

    # Prepare tensorflow.
    config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))
    session = tf.Session(config=config)
    keras.backend.tensorflow_backend.set_session(session)

    # Prepare model.
    model = SegNet(shape=(360, 480, 3))
    model.compile(loss='binary_crossentropy',
                  optimizer='adadelta',
                  metrics=['accuracy'])

    # Training.
    callbacks = []
    timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
    directory = f'./logs/{timestamp}/'
    os.makedirs(directory, exist_ok=True)
    callbacks.append(keras.callbacks.TensorBoard(log_dir=directory))

    filename = 'model-{epoch:04d}.h5'
    directory = f'./temp/{timestamp}/'
    os.makedirs(directory, exist_ok=True)
    callbacks.append(
        keras.callbacks.ModelCheckpoint(filepath=f'{directory}{filename}',
                                        monitor='val_loss',
                                        verbose=0,
                                        save_best_only=False,
                                        save_weights_only=False,
                                        mode='auto',
                                        period=args.checkpoint_interval))

    model.save_weights(f'{directory}{filename}'.format(epoch=0))

    if args.onmemory:
        model.fit(x=train_x,
                  y=train_y,
                  validation_data=(test_x, test_y),
                  epochs=args.epochs,
                  batch_size=args.batch_size,
                  class_weight='balanced',
                  shuffle=True,
                  verbose=1,
                  callbacks=callbacks)
    else:

        class Generator(keras.utils.Sequence):
            def __init__(self, x, y, batch_size, shuffle):
                self.x = x
                self.y = y
                self.batch_size = batch_size
                self.indices = np.arange(len(self.x))
                self.shuffle = shuffle
                assert len(self.x) == len(self.y)
                assert len(self.x) % self.batch_size == 0

            def __getitem__(self, index):
                i = index * self.batch_size
                indices = self.indices[i:i + self.batch_size]
                x = self.x[indices]
                y = self.y[indices]
                return x, y

            def __len__(self):
                return len(self.x) // self.batch_size

            def on_epoch_end(self):
                if self.shuffle:
                    self.indices = np.random.permutation(self.indices)

        model.fit_generator(generator=Generator(train_x, train_y,
                                                args.batch_size, True),
                            validation_data=Generator(test_x, test_y,
                                                      args.batch_size, False),
                            epochs=args.epochs,
                            class_weight='balanced',
                            shuffle=True,
                            verbose=1,
                            callbacks=callbacks)
コード例 #13
0
ファイル: train_keras.py プロジェクト: codeaudit/modeling
def main(args):
    model_id = build_model_id(args)
    model_path = build_model_path(args, model_id)
    setup_model_dir(args, model_path)
    sys.stdout, sys.stderr = setup_logging(args, model_path)

    x_train, y_train = load_model_data(args.train_file, args.data_name,
                                       args.target_name)
    x_validation, y_validation = load_model_data(args.validation_file,
                                                 args.data_name,
                                                 args.target_name)

    rng = np.random.RandomState(args.seed)

    if args.n_classes > -1:
        n_classes = args.n_classes
    else:
        n_classes = max(y_train) + 1

    n_classes, target_names, class_weight = load_target_data(args, n_classes)

    logging.debug("n_classes {0} min {1} max {2}".format(
        n_classes, min(y_train), max(y_train)))

    y_train_one_hot = np_utils.to_categorical(y_train, n_classes)
    y_validation_one_hot = np_utils.to_categorical(y_validation, n_classes)

    logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
    logging.debug("x_train " + str(x_train.shape))

    min_vocab_index = np.min(x_train)
    max_vocab_index = np.max(x_train)
    logging.debug("min vocab index {0} max vocab index {1}".format(
        min_vocab_index, max_vocab_index))

    json_cfg = load_model_json(args, x_train, n_classes)

    logging.debug("loading model")

    sys.path.append(args.model_dir)
    import model
    from model import build_model

    if args.subsetting_function:
        subsetter = getattr(model, args.subsetting_function)
    else:
        subsetter = None

    def take_subset(subsetter, path, x, y, y_one_hot, n):
        if subsetter is None:
            return x[0:n], y[0:n], y_one_hot[0:n]
        else:
            mask = subsetter(path)
            idx = np.where(mask)[0]
            idx = idx[0:n]
        return x[idx], y[idx], y_one_hot[idx]

    x_train, y_train, y_train_one_hot = take_subset(subsetter,
                                                    args.train_file,
                                                    x_train,
                                                    y_train,
                                                    y_train_one_hot,
                                                    n=args.n_train)

    x_validation, y_validation, y_validation_one_hot = take_subset(
        subsetter,
        args.validation_file,
        x_validation,
        y_validation,
        y_validation_one_hot,
        n=args.n_validation)

    logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
    logging.debug("x_train " + str(x_train.shape))

    model_cfg = ModelConfig(**json_cfg)
    logging.info("model_cfg " + str(model_cfg))
    model = build_model(model_cfg)
    setattr(model, 'stop_training', False)

    logging.info('model has {n_params} parameters'.format(
        n_params=count_parameters(model)))

    if len(args.extra_train_file) > 1:
        callbacks = keras.callbacks.CallbackList()
    else:
        callbacks = []

    save_model_info(args, model_path, model_cfg)

    if not args.no_save:
        callbacks.append(
            ModelCheckpoint(filepath=model_path + '/model-{epoch:04d}.h5',
                            verbose=1,
                            save_best_only=True))

    callback_logger = logging.info if args.log else callable_print

    if args.n_epochs < sys.maxsize:
        # Number of epochs overrides patience.  If the number of epochs
        # is specified on the command line, the model is trained for
        # exactly that number; otherwise, the model is trained with
        # early stopping using the patience specified in the model
        # configuration.
        callbacks.append(
            EarlyStopping(monitor='val_loss',
                          patience=model_cfg.patience,
                          verbose=1))

    if args.classification_report:
        cr = ClassificationReport(x_validation,
                                  y_validation,
                                  callback_logger,
                                  target_names=target_names)
        callbacks.append(cr)

    if model_cfg.optimizer == 'SGD':
        callbacks.append(SingleStepLearningRateSchedule(patience=10))

    if len(args.extra_train_file) > 1:
        args.extra_train_file.append(args.train_file)
        logging.info("Using the following files for training: " +
                     ','.join(args.extra_train_file))

        train_file_iter = itertools.cycle(args.extra_train_file)
        current_train = args.train_file

        callbacks._set_model(model)
        callbacks.on_train_begin(logs={})

        epoch = batch = 0

        while True:
            iteration = batch % len(args.extra_train_file)

            logging.info(
                "epoch {epoch} iteration {iteration} - training with {train_file}"
                .format(epoch=epoch,
                        iteration=iteration,
                        train_file=current_train))
            callbacks.on_epoch_begin(epoch, logs={})

            n_train = x_train.shape[0]

            callbacks.on_batch_begin(batch, logs={'size': n_train})

            index_array = np.arange(n_train)
            if args.shuffle:
                rng.shuffle(index_array)

            batches = keras.models.make_batches(n_train, model_cfg.batch_size)
            logging.info(
                "epoch {epoch} iteration {iteration} - starting {n_batches} batches"
                .format(epoch=epoch,
                        iteration=iteration,
                        n_batches=len(batches)))

            avg_train_loss = avg_train_accuracy = 0.
            for batch_index, (batch_start, batch_end) in enumerate(batches):
                batch_ids = index_array[batch_start:batch_end]

                if isinstance(model, keras.models.Graph):
                    data = {
                        'input': x_train[batch_ids],
                        'output': y_train_one_hot[batch_ids]
                    }
                    train_loss = model.train_on_batch(
                        data, class_weight=class_weight)
                    train_accuracy = 0.
                else:
                    train_loss, train_accuracy = model.train_on_batch(
                        x_train[batch_ids],
                        y_train_one_hot[batch_ids],
                        accuracy=True,
                        class_weight=class_weight)

                batch_end_logs = {
                    'loss': train_loss,
                    'accuracy': train_accuracy
                }

                avg_train_loss = (avg_train_loss * batch_index +
                                  train_loss) / (batch_index + 1)
                avg_train_accuracy = (avg_train_accuracy * batch_index +
                                      train_accuracy) / (batch_index + 1)

                callbacks.on_batch_end(batch,
                                       logs={
                                           'loss': train_loss,
                                           'accuracy': train_accuracy
                                       })

            logging.info(
                "epoch {epoch} iteration {iteration} - finished {n_batches} batches"
                .format(epoch=epoch,
                        iteration=iteration,
                        n_batches=len(batches)))

            logging.info(
                "epoch {epoch} iteration {iteration} - loss: {loss} - acc: {acc}"
                .format(epoch=epoch,
                        iteration=iteration,
                        loss=avg_train_loss,
                        acc=avg_train_accuracy))

            batch += 1

            # Validation frequency (this if-block) doesn't necessarily
            # occur in the same iteration as beginning of an epoch
            # (next if-block), so model.evaluate appears twice here.
            if (iteration + 1) % args.validation_freq == 0:
                val_loss, val_acc = model.evaluate(
                    x_validation,
                    y_validation_one_hot,
                    show_accuracy=True,
                    verbose=0 if args.log else 1)
                logging.info(
                    "epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}"
                    .format(epoch=epoch,
                            iteration=iteration,
                            val_loss=val_loss,
                            val_acc=val_acc))
                epoch_end_logs = {
                    'iteration': iteration,
                    'val_loss': val_loss,
                    'val_acc': val_acc
                }
                callbacks.on_epoch_end(epoch, epoch_end_logs)

            if batch % len(args.extra_train_file) == 0:
                val_loss, val_acc = model.evaluate(
                    x_validation,
                    y_validation_one_hot,
                    show_accuracy=True,
                    verbose=0 if args.log else 1)
                logging.info(
                    "epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}"
                    .format(epoch=epoch,
                            iteration=iteration,
                            val_loss=val_loss,
                            val_acc=val_acc))
                epoch_end_logs = {
                    'iteration': iteration,
                    'val_loss': val_loss,
                    'val_acc': val_acc
                }
                epoch += 1
                callbacks.on_epoch_end(epoch, epoch_end_logs)

            if model.stop_training:
                logging.info(
                    "epoch {epoch} iteration {iteration} - done training".
                    format(epoch=epoch, iteration=iteration))
                break

            current_train = next(train_file_iter)
            x_train, y_train = load_model_data(current_train, args.data_name,
                                               args.target_name)
            y_train_one_hot = np_utils.to_categorical(y_train, n_classes)

            if epoch > args.n_epochs:
                break

        callbacks.on_train_end(logs={})
    else:
        print('args.n_epochs', args.n_epochs)
        if isinstance(model, keras.models.Graph):
            data = {'input': x_train, 'output': y_train_one_hot}
            validation_data = {
                'input': x_validation,
                'output': y_validation_one_hot
            }
            model.fit(
                data,
                shuffle=args.shuffle,
                nb_epoch=args.n_epochs,
                batch_size=model_cfg.batch_size,
                #show_accuracy=True,
                validation_data=validation_data,
                callbacks=callbacks,
                class_weight=class_weight,
                verbose=2 if args.log else 1)
            y_hat = model.predict_classes(data)
            print('val_acc %.04f' % accuracy_score(y_validate, y_hat))
        else:
            model.fit(x_train,
                      y_train_one_hot,
                      shuffle=args.shuffle,
                      nb_epoch=args.n_epochs,
                      batch_size=model_cfg.batch_size,
                      show_accuracy=True,
                      validation_data=(x_validation, y_validation_one_hot),
                      callbacks=callbacks,
                      class_weight=class_weight,
                      verbose=2 if args.log else 1)
コード例 #14
0
ファイル: train.py プロジェクト: sidorenkobw/malia
def train(callback=None, out_weights='weights.h5'):
    reload(audiotransform)
    reload(speechmodel)

    model = speechmodel.makeModel()

    model.compile(loss='mean_squared_error',
                  optimizer=keras.optimizers.Nadam(lr=0.00002,
                                                   beta_1=0.9,
                                                   beta_2=0.999,
                                                   epsilon=1e-08,
                                                   schedule_decay=0.004),
                  metrics=['accuracy'])

    paths = []
    words = []
    for p in sampleSet2():  # or findSounds(words)
        try:
            raw = load(p, hz=speechmodel.rate)
        except:
            print "load failed", p
            continue

        try:
            crop = audiotransform.autoCrop(raw, rate=speechmodel.rate)
            print 'using %s autocropped to %s samples' % (p, len(crop))
        except audiotransform.TooQuiet:
            print '%s too quiet' % p
            continue
        paths.append(p)
        word = soundFields(p)['word']
        if word not in words:
            words.append(word)

    repeat = 2
    x = numpy.zeros((len(paths) * repeat, speechmodel.xWidth),
                    dtype=numpy.float)
    y = numpy.zeros((len(paths) * repeat, speechmodel.embedSize),
                    dtype=numpy.float)

    for row, p in enumerate(paths * repeat):
        audio = load(p, hz=speechmodel.rate)
        audio = audiotransform.autoCrop(audio, rate=speechmodel.rate)
        #audio = audiotransform.rightPad(audio, speechmodel.goalSize)
        audio = audiotransform.randomPad(audio, speechmodel.goalSize, path=p)
        audio = audiotransform.randomScale(audio)
        m = mfcc(audio, samplerate=speechmodel.rate)
        x[row, :] = m.reshape((1, speechmodel.xWidth))
        y[row, :] = np_utils.to_categorical(
            words.index(soundFields(p)['word']), speechmodel.embedSize)
        if callback:
            callback.loaded_sound(row, len(paths) * repeat)

    callbacks = []
    #callbacks.append(keras.callbacks.TensorBoard(log_dir='./logs', histogram_freq=1, write_graph=True))
    if callback:
        callbacks.append(callback)

    model.fit(x,
              y,
              batch_size=500,
              epochs=500,
              validation_split=.2,
              shuffle=True,
              callbacks=callbacks)

    model.save_weights(out_weights)
    with open(out_weights + '.words', 'w') as f:
        f.write(json.dumps(words) + '\n')
    if callback:
        callback.on_save(out_weights, fileSize=os.path.getsize(out_weights))
コード例 #15
0
def train(epochs, img_w, output, gs_output, steps_per_epoch, validation_steps,
          minibatch_size, **kwargs):
    # Input Parameters
    img_h = 64
    output_dir = output

    c.reset_dir(output_dir)

    # Network parameters
    pool_size = my_model.pool_size

    img_gen = TextImageGenerator(minibatch_size=minibatch_size,
                                 img_w=img_w,
                                 img_h=img_h,
                                 downsample_factor=(pool_size**2))

    input_data, y_pred = my_model.create_tensor_io(img_w, img_h,
                                                   img_gen.get_output_size())

    labels = Input(name='the_labels',
                   shape=[img_gen.absolute_max_string_len],
                   dtype='float32')
    input_length = Input(name='input_length', shape=[1], dtype='int64')
    label_length = Input(name='label_length', shape=[1], dtype='int64')
    # Keras doesn't currently support loss funcs with extra parameters
    # so CTC loss is implemented in a lambda layer
    loss_out = Lambda(ctc_lambda_func, output_shape=(1, ),
                      name='ctc')([y_pred, labels, input_length, label_length])

    # clipnorm seems to speeds up convergence
    sgd = SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5)

    model = Model(inputs=[input_data, labels, input_length, label_length],
                  outputs=loss_out)

    # the loss calc occurs elsewhere, so use a dummy lambda func for the loss
    model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd)
    # captures output of softmax so we can decode the output during visualization
    test_func = K.function([input_data], [y_pred])

    callbacks = []
    callbacks.append(
        ModelCheckpoint(
            filepath=os.path.join(output_dir, 'weight.{epoch:06d}.hdf5')))
    callbacks.append(
        ModelCheckpoint(filepath=os.path.join(output_dir, 'weight.best.hdf5'),
                        save_best_only=True))
    callbacks.append(CSVLogger(filename=os.path.join(output_dir, 'log.csv')))
    if gs_output != None:
        callbacks.append(
            g.Copy(os.path.join(output_dir, 'weight.{epoch:06d}.hdf5'),
                   os.path.join(gs_output, 'weight.{epoch:06d}.hdf5')))
        callbacks.append(
            g.Copy(os.path.join(output_dir, 'weight.best.hdf5'),
                   os.path.join(gs_output, 'weight.best.hdf5')))
        callbacks.append(
            g.Copy(os.path.join(output_dir, 'log.csv'),
                   os.path.join(gs_output, 'log.csv')))

    model.fit_generator(generator=img_gen.next_batch(),
                        steps_per_epoch=steps_per_epoch,
                        epochs=epochs,
                        validation_data=img_gen.next_batch(),
                        validation_steps=validation_steps,
                        callbacks=callbacks,
                        verbose=2)
コード例 #16
0
ファイル: train_cnn.py プロジェクト: chengsoonong/crowdastro
def train(
    training_h5,
    model_json,
    weights_path,
    epochs,
    batch_size,
    s3=False,
    s3_bucket=None,
    using_dataset=False,
    cnn_train_set_h5=None,
):
    """Trains a CNN.

    training_h5: Training HDF5 file.
    model_json: JSON model file.
    weights_path: Output weights HDF5 file.
    epochs: Number of training epochs.
    batch_size: Batch size.
    s3: Whether to periodically dump to Amazon S3. Default False.
    s3_bucket: Name of the bucket to dump to. Must be specified iff s3 is True.
    using_dataset: Whether the given training file is the Zenodo crowdastro
        dataset. Default False (i.e. we are using the generated training file).
    cnn_train_set_h5: HDF5 file specifying the CNN training set. Overrides the
        training file if specified. Default None (i.e. not specified).
    """
    if s3 and not s3_bucket:
        raise ValueError("Must specify s3_bucket to dump to S3.")
    if not s3 and s3_bucket:
        raise ValueError("s3_bucket was specified but s3 is False.")

    import keras.callbacks
    import keras.models

    model = keras.models.model_from_json(model_json.read())
    model.compile(loss="binary_crossentropy", optimizer="adadelta")

    if not cnn_train_set_h5:
        train_set = training_h5["cnn_train_set"].value
    else:
        train_set = cnn_train_set_h5["cnn_train_set"].value

    if not using_dataset:
        ir_survey = training_h5.attrs["ir_survey"]
        n_nonimage_features = config["surveys"][ir_survey]["n_features"]
        features_name = "raw_features"
    else:
        ir_survey = "wise"
        n_nonimage_features = 5
        features_name = "features"

    training_inputs = training_h5[features_name].value[train_set, n_nonimage_features:]

    training_inputs = training_inputs.reshape((-1, 1, PATCH_DIAMETER, PATCH_DIAMETER))
    training_outputs = training_h5["labels"].value[train_set]
    assert training_inputs.shape[0] == training_outputs.shape[0]

    # Downsample for class balance.
    zero_indices = (training_outputs == 0).nonzero()[0]
    one_indices = (training_outputs == 1).nonzero()[0]
    subset_zero_indices = numpy.random.choice(zero_indices, size=(len(one_indices)), replace=False)
    all_indices = numpy.hstack([subset_zero_indices, one_indices])
    all_indices.sort()

    training_inputs = training_inputs[all_indices]
    training_outputs = training_outputs[all_indices]
    assert (training_outputs == 1).sum() == (training_outputs == 0).sum()

    class DumpToS3(keras.callbacks.Callback):
        def __init__(self, weights_path, bucket, period=50):
            super().__init__()
            self.weights_path = weights_path
            self.bucket = bucket
            self.period = period

        def on_train_begin(self, logs={}):
            self.epochs = 0

        def on_epoch_end(self, epoch, logs={}):
            self.epochs += 1
            if self.epochs % self.period == 0:
                # Every 50 epochs...
                logging.debug("Dumping to S3...")
                res = subprocess.check_call(
                    ["aws", "s3", "cp", self.weights_path, "s3://" + self.bucket + "/", "--region", "us-east-1"]
                )
                logging.info("Dumped to S3: {}".format(res))

    try:
        model.load_weights(weights_path)
        logging.info("Loaded weights.")
    except OSError:
        logging.warning("Couldn't load weights file. Creating new file...")
        pass

    callbacks = [
        keras.callbacks.ModelCheckpoint(
            weights_path, monitor="val_loss", verbose=1, save_best_only=False, save_weights_only=True, mode="auto"
        )
    ]

    if s3:
        callbacks.append(DumpToS3(weights_path, s3_bucket))

    model.fit(training_inputs, training_outputs, batch_size=batch_size, nb_epoch=epochs, callbacks=callbacks)
    model.save_weights(weights_path, overwrite=True)
コード例 #17
0
ファイル: train_keras.py プロジェクト: ppyht2/modeling
def main(args):
    model_id = build_model_id(args)
    model_path = build_model_path(args, model_id)
    setup_model_dir(args, model_path)
    sys.stdout, sys.stderr = setup_logging(args, model_path)

    x_train, y_train = load_model_data(args.train_file, args.data_name,
                                       args.target_name)
    x_validation, y_validation = load_model_data(args.validation_file,
                                                 args.data_name,
                                                 args.target_name)

    rng = np.random.RandomState(args.seed)

    if args.n_classes > -1:
        n_classes = args.n_classes
    else:
        n_classes = max(y_train) + 1

    n_classes, target_names, class_weight = load_target_data(args, n_classes)

    if class_weight is None and args.class_weight_auto:
        n_samples = len(y_train)
        weights = float(n_samples) / (n_classes * np.bincount(y_train))
        if args.class_weight_exponent:
            weights = weights**args.class_weight_exponent
        class_weight = dict(zip(range(n_classes), weights))

    if args.verbose:
        logging.debug("n_classes {0} min {1} max {2}".format(
            n_classes, min(y_train), max(y_train)))

    y_train_one_hot = np_utils.to_categorical(y_train, n_classes)
    y_validation_one_hot = np_utils.to_categorical(y_validation, n_classes)

    if args.verbose:
        logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
        logging.debug("x_train " + str(x_train.shape))

    min_vocab_index = np.min(x_train)
    max_vocab_index = np.max(x_train)

    if args.verbose:
        logging.debug("min vocab index {0} max vocab index {1}".format(
            min_vocab_index, max_vocab_index))

    json_cfg = load_model_json(args, x_train, n_classes)

    if args.verbose:
        logging.debug("loading model")

    sys.path.append(args.model_dir)
    import model
    from model import build_model

    #######################################################################
    # Subsetting
    #######################################################################
    if args.subsetting_function:
        subsetter = getattr(M, args.subsetting_function)
    else:
        subsetter = None

    def take_subset(subsetter, path, x, y, y_one_hot, n):
        if subsetter is None:
            return x[0:n], y[0:n], y_one_hot[0:n]
        else:
            mask = subsetter(path)
            idx = np.where(mask)[0]
            idx = idx[0:n]
        return x[idx], y[idx], y_one_hot[idx]

    x_train, y_train, y_train_one_hot = take_subset(subsetter,
                                                    args.train_file,
                                                    x_train,
                                                    y_train,
                                                    y_train_one_hot,
                                                    n=args.n_train)

    x_validation, y_validation, y_validation_one_hot = take_subset(
        subsetter,
        args.validation_file,
        x_validation,
        y_validation,
        y_validation_one_hot,
        n=args.n_validation)

    #######################################################################
    # Preprocessing
    #######################################################################
    if args.preprocessing_class:
        preprocessor = getattr(M, args.preprocessing_class)(seed=args.seed)
    else:
        preprocessor = modeling.preprocess.NullPreprocessor()

    if args.verbose:
        logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
        logging.debug("x_train " + str(x_train.shape))

    model_cfg = ModelConfig(**json_cfg)
    if args.verbose:
        logging.info("model_cfg " + str(model_cfg))
    net = build_model(model_cfg)
    setattr(net, 'stop_training', False)

    marshaller = None
    if isinstance(net, keras.models.Graph):
        marshaller = getattr(model, args.graph_marshalling_class)()

    logging.info('model has {n_params} parameters'.format(
        n_params=count_parameters(net)))

    if len(args.extra_train_file) > 1:
        callbacks = keras.callbacks.CallbackList()
    else:
        callbacks = []

    save_model_info(args, model_path, model_cfg)

    callback_logger = logging.info if args.log else callable_print

    #######################################################################
    # Callbacks that need validation set predictions.
    #######################################################################

    pc = PredictionCallback(x_validation,
                            callback_logger,
                            marshaller=marshaller,
                            batch_size=model_cfg.batch_size)
    callbacks.append(pc)

    if args.classification_report:
        cr = ClassificationReport(x_validation,
                                  y_validation,
                                  callback_logger,
                                  target_names=target_names)
        pc.add(cr)

    if args.confusion_matrix:
        cm = ConfusionMatrix(x_validation, y_validation, callback_logger)
        pc.add(cm)

    def get_mode(metric_name):
        return {
            'val_loss': 'min',
            'val_acc': 'max',
            'val_f1': 'max',
            'val_f2': 'max',
            'val_f0.5': 'max'
        }[metric_name]

    if args.early_stopping or args.early_stopping_metric is not None:
        es = EarlyStopping(monitor=args.early_stopping_metric,
                           mode=get_mode(args.early_stopping_metric),
                           patience=model_cfg.patience,
                           verbose=1)
        cb = DelegatingMetricCallback(x_validation,
                                      y_validation,
                                      callback_logger,
                                      delegate=es,
                                      metric_name=args.early_stopping_metric,
                                      marshaller=marshaller)
        pc.add(cb)

    if not args.no_save:
        if args.save_all_checkpoints:
            filepath = model_path + '/model-{epoch:04d}.h5'
        else:
            filepath = model_path + '/model.h5'
        mc = ModelCheckpoint(filepath=filepath,
                             mode=get_mode(args.checkpoint_metric),
                             verbose=1,
                             monitor=args.checkpoint_metric,
                             save_best_only=not args.save_every_epoch)
        cb = DelegatingMetricCallback(x_validation,
                                      y_validation,
                                      callback_logger,
                                      delegate=mc,
                                      metric_name=args.checkpoint_metric,
                                      marshaller=marshaller)
        pc.add(cb)

    if model_cfg.optimizer == 'SGD':
        callbacks.append(SingleStepLearningRateSchedule(patience=10))

    if len(args.extra_train_file) > 1:
        args.extra_train_file.append(args.train_file)
        logging.info("Using the following files for training: " +
                     ','.join(args.extra_train_file))

        train_file_iter = itertools.cycle(args.extra_train_file)
        current_train = args.train_file

        callbacks._set_model(net)
        callbacks.on_train_begin(logs={})

        epoch = batch = 0

        while True:
            x_train, y_train_one_hot = preprocessor.fit_transform(
                x_train, y_train_one_hot)
            x_validation, y_validation_one_hot = preprocessor.transform(
                x_validation, y_validation_one_hot)

            iteration = batch % len(args.extra_train_file)

            logging.info(
                "epoch {epoch} iteration {iteration} - training with {train_file}"
                .format(epoch=epoch,
                        iteration=iteration,
                        train_file=current_train))
            callbacks.on_epoch_begin(epoch, logs={})

            n_train = x_train.shape[0]

            callbacks.on_batch_begin(batch, logs={'size': n_train})

            index_array = np.arange(n_train)
            if args.shuffle:
                rng.shuffle(index_array)

            batches = keras.models.make_batches(n_train, model_cfg.batch_size)
            logging.info(
                "epoch {epoch} iteration {iteration} - starting {n_batches} batches"
                .format(epoch=epoch,
                        iteration=iteration,
                        n_batches=len(batches)))

            avg_train_loss = avg_train_accuracy = 0.
            for batch_index, (batch_start, batch_end) in enumerate(batches):
                batch_ids = index_array[batch_start:batch_end]

                if isinstance(net, keras.models.Graph):
                    train_data = marshaller.marshal(x_train[batch_ids],
                                                    y_train_one_hot[batch_ids])
                    train_loss = net.train_on_batch(train_data,
                                                    class_weight=class_weight)
                    # It looks like train_on_batch returns a different
                    # type for graph than sequential models.
                    train_loss = train_loss[0]
                    train_accuracy = 0.
                else:
                    train_loss, train_accuracy = net.train_on_batch(
                        x_train[batch_ids],
                        y_train_one_hot[batch_ids],
                        accuracy=True,
                        class_weight=class_weight)

                batch_end_logs = {
                    'loss': train_loss,
                    'accuracy': train_accuracy
                }

                avg_train_loss = (avg_train_loss * batch_index +
                                  train_loss) / (batch_index + 1)
                avg_train_accuracy = (avg_train_accuracy * batch_index +
                                      train_accuracy) / (batch_index + 1)

                callbacks.on_batch_end(batch,
                                       logs={
                                           'loss': train_loss,
                                           'accuracy': train_accuracy
                                       })

            logging.info(
                "epoch {epoch} iteration {iteration} - finished {n_batches} batches"
                .format(epoch=epoch,
                        iteration=iteration,
                        n_batches=len(batches)))

            logging.info(
                "epoch {epoch} iteration {iteration} - loss: {loss} - acc: {acc}"
                .format(epoch=epoch,
                        iteration=iteration,
                        loss=avg_train_loss,
                        acc=avg_train_accuracy))

            batch += 1

            # Validation frequency (this if-block) doesn't necessarily
            # occur in the same iteration as beginning of an epoch
            # (next if-block), so net.evaluate appears twice here.
            kwargs = {
                'batch_size': model_cfg.batch_size,
                'verbose': 0 if args.log else 1
            }
            pargs = []
            validation_data = {}
            if isinstance(net, keras.models.Graph):
                validation_data = marshaller.marshal(x_validation,
                                                     y_validation_one_hot)
                pargs = [validation_data]
            else:
                pargs = [x_validation, y_validation_one_hot]
                kwargs['show_accuracy'] = True

            if (iteration + 1) % args.validation_freq == 0:
                if isinstance(net, keras.models.Graph):
                    val_loss = net.evaluate(*pargs, **kwargs)
                    y_hat = net.predict(validation_data,
                                        batch_size=model_cfg.batch_size)
                    val_acc = accuracy_score(
                        y_validation, np.argmax(y_hat['output'], axis=1))
                else:
                    val_loss, val_acc = net.evaluate(*pargs, **kwargs)
                logging.info(
                    "epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}"
                    .format(epoch=epoch,
                            iteration=iteration,
                            val_loss=val_loss,
                            val_acc=val_acc))
                epoch_end_logs = {
                    'iteration': iteration,
                    'val_loss': val_loss,
                    'val_acc': val_acc
                }
                callbacks.on_epoch_end(epoch, epoch_end_logs)

            if batch % len(args.extra_train_file) == 0:
                if isinstance(net, keras.models.Graph):
                    val_loss = net.evaluate(*pargs, **kwargs)
                    y_hat = net.predict(validation_data,
                                        batch_size=model_cfg.batch_size)
                    val_acc = accuracy_score(
                        y_validation, np.argmax(y_hat['output'], axis=1))
                else:
                    val_loss, val_acc = net.evaluate(*pargs, **kwargs)
                logging.info(
                    "epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}"
                    .format(epoch=epoch,
                            iteration=iteration,
                            val_loss=val_loss,
                            val_acc=val_acc))
                epoch_end_logs = {
                    'iteration': iteration,
                    'val_loss': val_loss,
                    'val_acc': val_acc
                }
                epoch += 1
                callbacks.on_epoch_end(epoch, epoch_end_logs)

            if net.stop_training:
                logging.info(
                    "epoch {epoch} iteration {iteration} - done training".
                    format(epoch=epoch, iteration=iteration))
                break

            current_train = next(train_file_iter)
            x_train, y_train = load_model_data(current_train, args.data_name,
                                               args.target_name)
            y_train_one_hot = np_utils.to_categorical(y_train, n_classes)

            if epoch > args.n_epochs:
                break

        callbacks.on_train_end(logs={})
    else:
        x_train, y_train_one_hot = preprocessor.fit_transform(
            x_train, y_train_one_hot)
        x_validation, y_validation_one_hot = preprocessor.transform(
            x_validation, y_validation_one_hot)

        if isinstance(net, keras.models.Graph):
            train_data = marshaller.marshal(x_train, y_train_one_hot)
            validation_data = marshaller.marshal(x_validation,
                                                 y_validation_one_hot)
            net.fit(train_data,
                    shuffle=args.shuffle,
                    nb_epoch=args.n_epochs,
                    batch_size=model_cfg.batch_size,
                    validation_data=validation_data,
                    callbacks=callbacks,
                    class_weight=class_weight,
                    verbose=2 if args.log else 1)
        else:
            net.fit(x_train,
                    y_train_one_hot,
                    shuffle=args.shuffle,
                    nb_epoch=args.n_epochs,
                    batch_size=model_cfg.batch_size,
                    show_accuracy=True,
                    validation_data=(x_validation, y_validation_one_hot),
                    callbacks=callbacks,
                    class_weight=class_weight,
                    verbose=2 if args.log else 1)
コード例 #18
0
                            test_samples = [samples[i] for i in test_index]

                            X_train, y_train = vectorize_samples(
                                train_samples, computed_params)
                            y_train = np.expand_dims(y_train, -1)

                            model = create_model(params, computed_params)

                            callbacks = []
                            model_checkpoint = ModelCheckpoint(
                                weights_path,
                                monitor='val_loss',
                                verbose=1,
                                save_best_only=True,
                                mode='auto')
                            callbacks.append(model_checkpoint)

                            early_stopping = EarlyStopping(monitor='val_loss',
                                                           patience=5,
                                                           verbose=1,
                                                           mode='auto')
                            callbacks.append(early_stopping)

                            model.fit(X_train,
                                      y_train,
                                      validation_split=0.1,
                                      epochs=100,
                                      verbose=2,
                                      batch_size=batch_size,
                                      callbacks=callbacks)
                            model.load_weights(weights_path)
コード例 #19
0
print("\tDone: " + str(end - start) + "seconds")

## Train the model
# Save the model architecture
basename = 'epoch{:03d}_seed'.format(epochs) + '{:04d}'.format(
    seed) + '_usdot_synth_ctc'
# basename = 'epoch{:03d}_seed'.format(epochs) + '{:04d}'.format(seed) + '_usdot_real_ctc'
json_string = model.to_json()
filename_model_arch = 'model_dotnum_' + basename + '_arch.json'
open(filename_model_arch, 'w').write(json_string)

# history = History()
callbacks = [History()]
filename_model_weights = 'model_dotnum_' + basename + '_weights.h5'  # where weights will be saved
callbacks.append(
    ModelCheckpoint(filepath=filename_model_weights,
                    monitor='val_loss',
                    save_best_only=True))

inTrain = {
    'the_input': dataTrain,
    'the_labels': labelsTrain,
    'input_length': inputLengthTrain,
    'label_length': labelLengthTrain,
    'source_str': lineStringsTrain  # used for visualization only
}
outTrain = {
    'ctc': np.zeros([len(dataTrain)])
}  # dummy data for dummy loss function
inVal = {
    'the_input': dataVal,
    'the_labels': labelsVal,
コード例 #20
0
ファイル: train.py プロジェクト: ksaid0203/Graduation-project
    logger.info("load weight:%s", args.weights_path)
    model.load_weights(args.weights_path, by_name=True)

gen = dataset.DataGenerator(config)
train_data_generator = gen.generate(os.path.join(args.data_dir, "train"), train_generator, train_discriminator)
val_data_generator = gen.generate(os.path.join(args.data_dir, "validate"), train_generator, train_discriminator)

# Call back preparation.
model_file_path = './nnmodel/glcic-stage{}-{}'.format(
    args.stage, '{epoch:02d}-{val_loss:.2f}.h5')
callbacks = [keras.callbacks.TerminateOnNaN(),
            keras.callbacks.TensorBoard(log_dir='./tb_log',histogram_freq=0,write_graph=True,write_images=False),
            keras.callbacks.ModelCheckpoint(filepath=model_file_path,verbose=1,save_weights_only=True,save_best_only=False,period=20)]

if args.testimage_path and not args.stage == 2:
    # epoch毎にgeneratorの出力を保存
    test_data_generator = gen.generate(args.testimage_path,
                                       train_generator, train_discriminator)
    inputs, _ = next(test_data_generator)
    callbacks.append(SaveGeneratorOutput(gen, config.batch_size, inputs))

model.fit_generator(train_data_generator,
                    steps_per_epoch=steps_per_epoch,
                    epochs=epochs,
                    verbose=1,max_queue_size=10,
                    callbacks=callbacks,
                    validation_data=val_data_generator,
                    validation_steps=5)

model_file_path = './nnmodel/glcic-latest-stage{}.h5'.format(args.stage)
model.save_weights(model_file_path)
コード例 #21
0
ファイル: train.py プロジェクト: poi890poi/a-projects
def train_or_load(model, input_shape, class_ids, model_dir, args):
    hp = Hyperparameters(args.model_prototype)

    num_classes = len(class_ids)

    weights_path = os.path.join(model_dir, 'weights.keras')
    if os.path.exists(weights_path) and not args.reset:
        # Load weights from file
        model.load_weights(weights_path)
        print()
        print('Weights loaded from', weights_path)

    if args.dummy:
        # Generate random dummy data for verification of model definition
        data = np.random.random((1000,)+input_shape)
        labels = np.random.randint(num_classes, size=(1000, 1))
        one_hot_labels = keras.utils.to_categorical(labels, num_classes=num_classes) # Convert labels to categorical one-hot encoding
        model.fit(data, one_hot_labels, batch_size=32, epochs=epochs, verbose=1)
    else:
        # Get list of proto-samples and shuffle them
        print()
        print('Getting sample file list...')
        sample_count = 0
        now = int(time.time())
        proto_samples = list() # List of all proto-samples (in the original dataset without mutation) sorted by classes
        for i in range(num_classes):
            proto_samples.append(list())
        srcdir = os.path.normpath(args.src)
        pathlist = sorted(Path(srcdir).glob('**/*.ppm'))
        for path in pathlist:
            path_in_str = str(path)
            sample_dir, sample_filename = os.path.split(path_in_str)
            class_id = int(os.path.split(sample_dir)[1])
            proto_samples[class_id].append(path_in_str)
            sample_count += 1
        for i in range(num_classes):
            random.shuffle(proto_samples[i])
        print('Total of', sample_count, 'samples found.')

        # These variables/objects outside of iteration loop maintain their status through the whole training process
        sample_offset = 0
        history = TrainHistory(args, model_dir)
        logdir = os.path.normpath(os.path.join(model_dir, 'logdir'))
        if not os.path.exists(logdir):
            os.makedirs(logdir)
        
        # The graph is inconsistent with keras
        #tensorboard = keras.callbacks.TensorBoard(log_dir=logdir, histogram_freq=epochs/4,
        #    write_graph=True, write_grads=True, write_images=True)

        # Iterations of training
        samples_per_class = int(hp.batch_size / num_classes)
        hp.batch_size = int(num_classes * samples_per_class) # Enforce strict classes balancing (same amount of samples for each class)
        sample_offset = 0
        fine_tune = False
        for iteration in range(hp.iterations_total):
            # Load training data from filesystem
            samples = list()

            phase = 0
            for i in range(len(hp.iterations)-1):
                if iteration > hp.iterations[i]:
                    phase = i
            print()
            print('Initial iteration:', iteration, 'in', hp.iterations_total, ', phase', phase)
            print('Time elapsed:', int(time.time())-now, 'sec')
            print('Loading', samples_per_class, 'samples per class with offset', sample_offset)
            for i in range(samples_per_class):
                batch_shuffled = list()
                for class_id in range(num_classes):
                    sample_pointer = (sample_offset + i) % len(proto_samples[class_id])
                    #print(class_id, len(samples[class_id]), sample_pointer)
                    #print(samples[class_id])
                    sample_path = proto_samples[class_id][sample_pointer]
                    sample = get_mutations(sample_path, 1, intensity=hp.mintensity[phase])[0]
                    sample = np.array(cv2.normalize(sample.astype('float'), None, 0.0, 1.0, cv2.NORM_MINMAX)).reshape(input_shape)
                    batch_shuffled.append([class_id, sample]) # Pair class_id and sample data so it's easily shuffled
                random.shuffle(batch_shuffled) # Shuffle in small chunk with balanced data (one sample for each class)
                samples += batch_shuffled

            labels = np.zeros((len(samples), 1))
            data = np.zeros((len(samples),)+input_shape)
            for i in range(len(samples)):
                labels[i] = samples[i][0]
                data[i] = samples[i][1]
            one_hot_labels = keras.utils.to_categorical(labels, num_classes=num_classes) # Convert labels to categorical one-hot encoding

            # Train the model
            callbacks = [
                history,
                #tensorboard, # The graph is messed and inconsistent with keras
                keras.callbacks.ModelCheckpoint(weights_path, monitor=hp.checkpoint_metric, save_best_only=True, verbose=1),
            ]
            if hp.patience:
                callbacks.append(keras.callbacks.EarlyStopping(monitor=hp.earlystop_metric, min_delta=hp.min_delta, patience=hp.patience, verbose=1))

            lr = hp.learn_rate[phase]
            print('Training with learn rate=', lr)
            K.update(model.optimizer.lr, lr)
            model.fit(data, one_hot_labels,
                batch_size=num_classes, epochs=hp.epochs,
                verbose=1, callbacks=callbacks,
                validation_split=hp.vs, shuffle=False) # Do not use keras internal shuffling so the logic can be controlled
            #model.train_on_batch(data, one_hot_labels)

            sample_offset += samples_per_class
            if sample_offset>=65535: sample_offset = 0
コード例 #22
0
#model.compile(optimizer='adam', loss='mse')
model.compile(loss='mean_squared_error',
              optimizer=Adam(lr=0.01,
                             beta_1=0.9,
                             beta_2=0.999,
                             epsilon=1e-08,
                             decay=0.0,
                             amsgrad=False),
              metrics=['mae', 'accuracy'])

#visualize the model
plot_model(model, to_file=path + '/model.png', show_shapes=True)

callbacks = []
# CSVLogger
callbacks.append(CSVLogger(path + "/log.csv"))

from keras.callbacks import ModelCheckpoint
checkpointer = ModelCheckpoint(filepath=path + "/best_weight.h5",
                               monitor="val_loss",
                               save_best_only=True,
                               save_weights_only=True)
callbacks = [checkpointer]
# fitting
history = model.fit(x_train,
                    y_train,
                    batch_size=243,
                    epochs=10000,
                    verbose=1,
                    validation_data=(x_test, y_test),
                    callbacks=callbacks)
コード例 #23
0
def train():

    logging.basicConfig(level=logging.INFO)

    args = cmd.opts.parse_arguments()

    logging.info("Loading dataset...")
    augmentation_args = {
        'rotation_range': args.rotation_range,
        'width_shift_range': args.width_shift_range,
        'height_shift_range': args.height_shift_range,
        'shear_range': args.shear_range,
        'zoom_range': args.zoom_range,
        'fill_mode': args.fill_mode,
    }
    train_generator, train_steps_per_epoch, val_generator, val_steps_per_epoch = datafeed.create_generators(
        args.batch_size,
        args.train_num,
        args.test_num,
        shuffle=args.shuffle,
        normalize_images=args.normalize,
        augment_training=args.augment_training,
        augment_validation=args.augment_validation,
        augmentation_args=augmentation_args)

    # ==========================================================
    # ======================Build model ========================

    print('=' * 40)
    print('Creating and compiling model...')
    print('=' * 40)
    imgs_train, mask_train = next(train_generator)
    _, height, width, channels = imgs_train.shape
    _, _, _, classes = mask_train.shape

    logging.info("Building model...")
    string_to_model = {
        "unet": models.unet,
        "dilated-unet": models.dilated_unet,
    }
    model = string_to_model[args.model]

    m = model(height=height,
              width=width,
              channels=channels,
              classes=classes,
              features=args.features,
              depth=args.depth,
              padding=args.padding,
              temperature=args.temperature,
              batchnorm=args.batchnorm,
              dropout=args.dropout)

    m.summary()

    #===============================================================
    #======================= Build metrics, lossfunc ===============

    if args.loss == 'pixel':

        def lossfunc(y_true, y_pred):
            return loss.weighted_categorical_crossentropy(
                y_true, y_pred, args.loss_weights)
    elif args.loss == 'dice':

        def lossfunc(y_true, y_pred):
            return loss.sorensen_dice_loss(y_true, y_pred, args.loss_weights)
    elif args.loss == 'jaccard':

        def lossfunc(y_true, y_pred):
            return loss.jaccard_loss(y_true, y_pred, args.loss_weights)
    else:
        raise Exception("Unknown loss ({})".format(args.loss))

    #===================== metrics =================================

    metrics = []
    if args.metrics_F1:
        metrics.append(custom_metrics.F1)
    if args.metrics_precision:
        metrics.append(custom_metrics.precision)
    if args.metrics_recall:
        metrics.append(custom_metrics.recall)
    if args.metrics_jaccard:
        metrics.append(custom_metrics.jaccard)

    #===================== parse for optimizer ====================

    if args.optimizer == 'sgd':
        opt = optimizers.SGD(lr=args.learning_rate,
                             decay=args.decay,
                             momentum=args.momentum)
    elif args.optimizer == 'adam':
        opt = optimizers.adam(lr=args.learning_rate, beta_1=0.9, beta_2=0.999)
    else:
        raise Exception("Unknown optimizer ({})".format(args.loss))

    model.compile(loss=lossfunc, optimizer=opt, metrics=metrics)

    # ========================================================
    # ==================  CallBacks     ======================

    print('=' * 40)
    print('Checkpoint config')
    print('=' * 40)

    checkpoint_folder = args.checkpoint_dir
    if not os.path.exists(checkpoint_folder):
        os.makedirs(checkpoint_folder)

    if args.checkpoint:
        if args.loss == 'dice':
            filepath = os.path.join(
                args.outdir,
                "weights-{epoch:02d}-{dice:.4f}--{val_dice:.4f}.hdf5")
            monitor = 'val_dice'
            mode = 'max'
        elif args.loss == 'jaccard':
            filepath = os.path.join(
                args.outdir,
                "weights-{epoch:02d}-{jaccard:.4f}--{val_jaccard:.4f}.hdf5")
            monitor = 'val_jaccard'
            mode = 'max'
        checkpoint = ModelCheckpoint(filepath,
                                     monitor=monitor,
                                     verbose=1,
                                     save_best_only=True,
                                     mode=mode)
        callbacks = [checkpoint]
    else:
        callbacks = []

    history = metricsHistory()
    callbacks.append(history)
    # ========================================================
    # ========================================================

    print('=' * 40)
    print('Fitting model...')
    print('=' * 40)

    logging.info("Begin training.")
    model.fit_generator(train_generator,
                        epochs=args.epochs,
                        steps_per_epoch=train_steps_per_epoch,
                        validation_data=val_generator,
                        validation_steps=val_steps_per_epoch,
                        callbacks=callbacks)

    m.save(os.path.join(args.outdir, args.outfile))

    np.save(checkpoint_folder + "/losses", history.losses)