def train_bgd(self):
        """
        Optimize the parameters of a model use batch gradient decent.
        Loss and trained models are stored in local file.
        Returns a tuple of:
        - best_model: The model with the lowest validation error.
        - train_loss_history: A list containing the value of loss
          at each iteration (update) during training.
        - val_loss_history: A list containing the value of loss
          at each epoch based on trained model.
        Others:
        - A logging file to store losses.
        - Some model files to store trained model at each epoch
        """
        # prepare the training and validation datasets
        data_info_file = self.data_info_file
        data_info = np.array([l.strip() for l in open(data_info_file).readlines()])
        # data_info = data_info[0:100]
        N = len(data_info)
        print (N)
        N_val = int(N * self.val_rate)
        N_train = N - N_val
        perm = np.random.permutation(N)
        train_info = data_info[perm[0: N_train]]
        val_info = data_info[perm[N_train:]]
        print 'Number of training data %d' % N_train
        print 'Number of validation data %d' % N_val
        # create log file and result directory
        log_file, result_dir = self.creat_result_dir_logfile()
        print(self.params)
        logging.info(self.params)  # log the params for training

        # get the optimizer
        self.optimizer = self.get_optimizer()
        lowest_val_loss = float('inf')
        best_model = None
        self.train_loss_batch_history = []
        train_loss_epoch_history = []
        val_loss_epoch_history = []

        # Add info into to log file
        logging.info('Use batch gradient decent!')
        logging.info('# of training data: {}'.format(N_train))
        logging.info('# of validation data: {}'.format(N_val))
        logging.info('# of batch gradient decent')
        logging.info('# of iteration: {}'.format(self.num_epochs))
        logging.info(time.strftime('%Y-%m-%d_%H-%M-%S'))
        logging.info('start training ...')

        for epoch in xrange(1, self.num_epochs + 1):
            start_time = time.time()
            input_data, label = self.all_data_load(train_info, result_dir)
            self.compute_loss_model_update_bgd(input_data, label, self.num_iters)

            # each epoch performs an evaluation on subset of training data
            # the same as number of validation datasets
            mask = np.random.choice(N_train, N_val, replace=False)
            train_test_data = input_data[mask]
            label_test = label[mask]
            train_loss_epoch = self.compute_loss_bgd(train_test_data, label_test)
            train_loss_epoch_history.append(train_loss_epoch)
            elapsed_time = time.time() - start_time
            log_msg = self.get_log_msg('training', train_loss_epoch,
                                       elapsed_time, epoch)
            logging.info(log_msg)
            print(log_msg)

            # each epoch perform an evaluation on validation set
            start_time = time.time()
            input_data, label = self.all_data_load(val_info, result_dir)
            val_loss_epoch = self.compute_loss_bgd(input_data, label)
            val_loss_epoch_history.append(val_loss_epoch)
            elapsed_time = time.time() - start_time
            log_msg = self.get_log_msg('validation', val_loss_epoch,
                                       elapsed_time, epoch)
            logging.info(log_msg)
            print(log_msg)

            if val_loss_epoch < lowest_val_loss:
                lowest_val_loss = val_loss_epoch
                best_model = copy.deepcopy(self.model)

            # draw loss curve
            draw_batch_loss(self.train_loss_batch_history,
                            result_dir + '/batch_loss.jpg')
            draw_loss_from_list(train_loss_epoch_history,
                                val_loss_epoch_history,
                                result_dir + '/loss.jpg')

            # store the trained model
            if epoch == 1 or epoch % self.snapshot == 0:
                model_fn = '%s/epoch_%d.chainermodel' % (result_dir, epoch)
                pickle.dump(self.model, open(model_fn, 'wb'), -1)

        logging.info(time.strftime('%Y-%m-%d_%H-%M-%S'))
        logging.info('End training')

        return (best_model, self.train_loss_batch_history,
                train_loss_epoch_history, val_loss_epoch_history)
    def train(self):
        """
        Optimize the parameters of a model to minimize the loss.
        Loss and trained models are stored in local file.
        Returns a tuple of:
        - best_model: The model with the lowest validation error.
        - train_loss_batch_history: A list all batch loss in all epochs
        - train_loss_epoch_history: A list containing the value of loss
          at each epoch (update) during training.
        - val_loss_epoch_history: A list containing the value of loss
          at each epoch based on trained model.
        Others:
        - A logging file to store losses.
        - Some model files to store trained model at each epoch
        """
        # prepare the training and validation datasets
        data_info_file = self.data_info_file
        data_info = np.array([l.strip() for l in open(data_info_file).readlines()])
        # data_info = data_info[0:100]
        N = len(data_info)
        print (N)
        N_val = int(N * self.val_rate)
        N_train = N - N_val
        perm = np.random.permutation(N)
        train_info = data_info[perm[0: N_train]]
        val_info = data_info[perm[N_train:]]
        print 'Number of training data %d' % N_train
        print 'Number of validation data %d' % N_val
        # create log file and result directory
        log_file, result_dir = self.creat_result_dir_logfile()
        print(self.params)
        logging.info(self.params)  # log the params for training

        # get the optimizer
        self.optimizer = self.get_optimizer()
        lowest_val_loss = float('inf')
        best_model = None
        self.train_loss_batch_history = []
        train_loss_epoch_history = []
        val_loss_epoch_history = []

        # Add info into to log file
        logging.info('# of training data: {}'.format(N_train))
        logging.info('# of validation data: {}'.format(N_val))
        logging.info('# of batch size: {}'.format(self.batch_size))
        logging.info('# of epoch: {}'.format(self.num_epochs))
        logging.info(time.strftime('%Y-%m-%d_%H-%M-%S'))
        logging.info('start training ...')

        for epoch in xrange(1, self.num_epochs + 1):
            # use multiprocess to speed up training and save memory
            # the model can be trained while the batch data is loading
            input_q = Queue()  # store data info for loading and transformation
            data_q = Queue()  # store transformed data

            # create process to load training data
            data_load_process = Process(target=self.data_load,
                                        args=(input_q, data_q, result_dir))
            data_load_process.start()
            start_time = time.time()
            self.compute_loss_model_update(train_info, input_q, data_q)
            input_q.put(None)
            data_load_process.join()

            # each epoch performs an evaluation on subset of training data
            # the same as number of validation datasets
            input_q = Queue()
            data_q = Queue()
            data_load_process = Process(target=self.data_load,
                                        args=(input_q, data_q, result_dir))
            data_load_process.start()
            train_loss_epoch = self.compute_loss(train_info, N_val, input_q, data_q)
            train_loss_epoch_history.append(train_loss_epoch)
            elapsed_time = time.time() - start_time
            log_msg = self.get_log_msg('training', train_loss_epoch,
                                       elapsed_time, epoch)
            logging.info(log_msg)
            print(log_msg)
            input_q.put(None)
            data_load_process.join()

            # each epoch perform an evaluation on validation set
            input_q = Queue()
            data_q = Queue()
            data_load_process = Process(target=self.data_load,
                                        args=(input_q, data_q, result_dir))
            data_load_process.start()
            start_time = time.time()
            val_loss_epoch = self.compute_loss(val_info, N_val, input_q, data_q)
            val_loss_epoch_history.append(val_loss_epoch)
            elapsed_time = time.time() - start_time
            log_msg = self.get_log_msg('validation',
                                       val_loss_epoch, elapsed_time, epoch)
            logging.info(log_msg)
            print(log_msg)
            if val_loss_epoch < lowest_val_loss:
                lowest_val_loss = val_loss_epoch
                best_model = copy.deepcopy(self.model)
            input_q.put(None)
            data_load_process.join()

            # draw loss curve
            draw_batch_loss(self.train_loss_batch_history, result_dir
                            + '/batch_loss.jpg')
            draw_loss_from_list(train_loss_epoch_history,
                                val_loss_epoch_history,
                                result_dir + '/val_loss.jpg')

            # store the trained model
            if epoch == 1 or epoch % self.snapshot == 0:
                model_fn = '%s/epoch_%d.chainermodel' % (result_dir, epoch)
                pickle.dump(self.model, open(model_fn, 'wb'), -1)

        logging.info(time.strftime('%Y-%m-%d_%H-%M-%S'))
        logging.info('End training')

        return (best_model, self.train_loss_batch_history,
                train_loss_epoch_history, val_loss_epoch_history)
Пример #3
0
    def train(self, X, y):
        """
        Optimize the parameters of a model to minimize the loss.
        Loss and trained models are stored in local file.
        Returns a tuple of:
        - best_model: The model with the lowest validation error.
        - train_loss_batch_history: A list all batch loss in all epochs
        - train_loss_epoch_history: A list containing the value of loss
          at each epoch (update) during training.
        - val_loss_epoch_history: A list containing the value of loss
          at each epoch based on trained model.
        Others:
        - A logging file to store losses.
        - Some model files to store trained model at each epoch
        """
        # prepare the training and validation datasets
        # chainer only support np.float32
        if X.dtype != np.float32:
            X = X.astype(np.float32)
        if y.dtype != np.float32:
            y = y.astype(np.float32)

        N = X.shape[0]
        N_val = int(N * self.val_rate)
        N_train = N - N_val
        perm = np.random.permutation(N)
        X_train = X[perm[0: N_train]]
        y_train = y[perm[0: N_train]]
        X_val = X[perm[N_train:]]
        y_val = y[perm[N_train:]]
        print 'Number of training data %d' % N_train
        print 'Number of validation data %d' % N_val
        # create log file and result directory
        log_file, result_dir = self.creat_result_dir_logfile()
        print(self.params)
        logging.info(self.params)  # log the params for training

        # get the optimizer
        self.optimizer = self.get_optimizer()
        lowest_val_loss = float('inf')
        best_model = None
        self.train_loss_batch_history = []
        train_loss_epoch_history = []
        val_loss_epoch_history = []

        # Add info into to log file
        logging.info('# of training data: {}'.format(N_train))
        logging.info('# of validation data: {}'.format(N_val))
        logging.info('# of batch size: {}'.format(self.batch_size))
        logging.info('# of epoch: {}'.format(self.num_epochs))
        logging.info(time.strftime('%Y-%m-%d_%H-%M-%S'))
        logging.info('start training ...')

        for epoch in xrange(1, self.num_epochs + 1):
            start_time = time.time()
            # update model
            self.compute_loss_model_update(X_train, y_train)

            # each epoch performs an evaluation on subset of training data
            # the same as number of validation datasets
            mask = np.random.choice(N_train, N_val, replace=False)
            train_test_data = X[mask]
            label_test = y[mask]
            # train_test_data = X_train
            # label_test = y_train
            train_loss_epoch = self.compute_loss(train_test_data, label_test)
            train_loss_epoch_history.append(train_loss_epoch)
            elapsed_time = time.time() - start_time
            log_msg = self.get_log_msg('training', train_loss_epoch,
                                       elapsed_time, epoch)
            logging.info(log_msg)
            print(log_msg)

            # each epoch perform an evaluation on validation set
            start_time = time.time()
            val_loss_epoch = self.compute_loss(X_val, y_val)
            val_loss_epoch_history.append(val_loss_epoch)
            elapsed_time = time.time() - start_time
            log_msg = self.get_log_msg('validation', val_loss_epoch,
                                       elapsed_time, epoch)
            logging.info(log_msg)
            print(log_msg)

            if val_loss_epoch < lowest_val_loss:
                lowest_val_loss = val_loss_epoch
                best_model = copy.deepcopy(self.model)

            # draw loss curve
            draw_batch_loss(self.train_loss_batch_history,
                            result_dir + '/batch_loss.jpg')
            draw_loss_from_list(train_loss_epoch_history,
                                val_loss_epoch_history,
                                result_dir + '/loss.jpg')

            # store the trained model
            if epoch == 1 or epoch % self.snapshot == 0:
                model_fn = '%s/epoch_%d.chainermodel' % (result_dir, epoch)
                pickle.dump(self.model, open(model_fn, 'wb'), -1)

        logging.info(time.strftime('%Y-%m-%d_%H-%M-%S'))
        logging.info('End training')

        return (best_model, self.train_loss_batch_history,
                train_loss_epoch_history, val_loss_epoch_history)