def run(self): best_valid_error = float(sys.maxint) valid_error = float(sys.maxint) train_cost = float(sys.maxint) valid_cost = float(sys.maxint) train_stats_values = [] valid_stats_values = [] epoch = 0 error_dcr = 0 self.best_epoch_last_update = 0 self.best_valid_last_update = float(sys.maxint) train_stats_names = ['train_' + name for name in self.train_stats_names] valid_stats_names = ['valid_' + name for name in self.test_stats_names] job_start = time.time() while (self.continue_learning(epoch, error_dcr, best_valid_error)): if epoch > 0: self.log.info("best_epoch_last_update: %d"%self.best_epoch_last_update) self.log.info("valid_error_decrease: %f"%error_dcr) self.log.info("best_valid_last_update: %f"%self.best_valid_last_update) self.log.info("========[ End of Epoch ]========\n\n") epoch += 1 start_time = time.time() num_train_examples = 0 total_train_cost = 0. train_stats_values = np.zeros(len(train_stats_names), dtype=floatX) num_valid_examples = 0 total_valid_cost = 0. total_valid_stopping_cost = 0. valid_stats_values = np.zeros(len(valid_stats_names), dtype=floatX) blk = 0 for block in self.dataset: block_time = time.time() blk += 1 train_set = block.get_train() valid_set = block.get_valid() #====================[ Training Progress ]====================# if train_set.dataset_size > 0: self.log.info('..training '+ self.dataset.__class__.__name__ + ' block %s/%s'%(blk, self.dataset.nblocks)) progbar = Progbar(target=train_set.dataset_size) blk_sz = 0 for idx in train_set: cost = self.training(train_set.X[idx], train_set.y[idx]) total_train_cost += cost * len(idx) num_train_examples += len(idx) train_stats_values += len(idx) * get_shared_values(self.train_stats_shared) blk_sz += len(idx) progbar.update(blk_sz) print #-------[ Update train best cost and error values ]-------# train_cost = total_train_cost / num_train_examples train_stats_values /= num_train_examples #===================[ Validating Progress ]===================# if valid_set.dataset_size > 0: self.log.info('..validating ' + self.dataset.__class__.__name__ + ' block %s/%s'%(blk, self.dataset.nblocks)) progbar = Progbar(target=valid_set.dataset_size) blk_sz = 0 for idx in valid_set: stopping_cost, cost = self.testing(valid_set.X[idx], valid_set.y[idx]) total_valid_cost += cost * len(idx) total_valid_stopping_cost += stopping_cost * len(idx) num_valid_examples += len(idx) valid_stats_values += len(idx) * get_shared_values(self.test_stats_shared) blk_sz += len(idx) progbar.update(blk_sz) print #-------[ Update valid best cost and error values ]-------# valid_error = total_valid_stopping_cost / num_valid_examples valid_cost = total_valid_cost / num_valid_examples valid_stats_values /= num_valid_examples if valid_error < best_valid_error: best_valid_error = valid_error self.log.info('..best validation error so far') if self.log.save_model: self.log._save_model(self.model) self.log.info('..model saved') if valid_error < self.best_valid_last_update: error_dcr = self.best_valid_last_update - valid_error else: error_dcr = 0 self.log.info('block time: %0.2fs'%(time.time()-block_time)) self.log.info(get_mem_usage()) #==============[ save to database, save epoch error]==============# if self.log.save_to_database: self.log._save_to_database(epoch, train_cost, valid_cost, best_valid_error) self.log.info('..sent to database: %s:%s' % (self.log.save_to_database['name'], self.log.experiment_name)) if self.log.save_epoch_error: self.log._save_epoch_error(epoch, valid_error) self.log.info('..epoch error saved') end_time = time.time() #=====================[ log outputs to file ]=====================# merged_train = merge_lists(train_stats_names, train_stats_values) merged_valid = merge_lists(valid_stats_names, valid_stats_values) outputs = [('epoch', epoch), ('runtime(s)', int(end_time-start_time)), ('train_' + self.train_cost.func_name, train_cost), ('valid_' + self.train_cost.func_name, valid_cost), ('valid_' + self.valid_cost.func_name, valid_error), ('best_valid_' + self.valid_cost.func_name, best_valid_error)] outputs += merged_train + merged_valid self.log._log_outputs(outputs) job_end = time.time() self.log.info('Job Completed on %s'%time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime(job_end))) ttl_time = int(job_end - job_start) dt = datetime.timedelta(seconds=ttl_time) self.log.info('Total Time Taken: %s'%str(dt)) self.log.info("========[ End of Job ]========\n\n")
def run(self): best_valid_error = float(sys.maxint) valid_error = float(sys.maxint) train_cost = float(sys.maxint) valid_cost = float(sys.maxint) train_stats_values = [] valid_stats_values = [] epoch = 0 error_dcr = 0 self.best_epoch_last_update = 0 self.best_valid_last_update = float(sys.maxint) train_stats_names = [ 'train_' + name for name in self.train_stats_names ] valid_stats_names = ['valid_' + name for name in self.test_stats_names] job_start = time.time() while (self.continue_learning(epoch, error_dcr, best_valid_error)): if epoch > 0: self.log.info("best_epoch_last_update: %d" % self.best_epoch_last_update) self.log.info("valid_error_decrease: %f" % error_dcr) self.log.info("best_valid_last_update: %f" % self.best_valid_last_update) self.log.info("========[ End of Epoch ]========\n\n") epoch += 1 start_time = time.time() num_train_examples = 0 total_train_cost = 0. train_stats_values = np.zeros(len(train_stats_names), dtype=floatX) num_valid_examples = 0 total_valid_cost = 0. total_valid_stopping_cost = 0. valid_stats_values = np.zeros(len(valid_stats_names), dtype=floatX) blk = 0 for block in self.dataset: block_time = time.time() blk += 1 train_set = block.get_train() valid_set = block.get_valid() #====================[ Training Progress ]====================# if train_set.dataset_size > 0: self.log.info('..training ' + self.dataset.__class__.__name__ + ' block %s/%s' % (blk, self.dataset.nblocks)) progbar = Progbar(target=train_set.dataset_size) for idx in train_set: cost = self.training(train_set.X[idx], train_set.y[idx]) total_train_cost += cost * len(idx) num_train_examples += len(idx) train_stats_values += len(idx) * get_shared_values( self.train_stats_shared) progbar.update(num_train_examples) print #-------[ Update train best cost and error values ]-------# train_cost = total_train_cost / num_train_examples train_stats_values /= num_train_examples #===================[ Validating Progress ]===================# if valid_set.dataset_size > 0: self.log.info('..validating ' + self.dataset.__class__.__name__ + ' block %s/%s' % (blk, self.dataset.nblocks)) progbar = Progbar(target=valid_set.dataset_size) for idx in valid_set: stopping_cost, cost = self.testing( valid_set.X[idx], valid_set.y[idx]) total_valid_cost += cost * len(idx) total_valid_stopping_cost += stopping_cost * len(idx) num_valid_examples += len(idx) valid_stats_values += len(idx) * get_shared_values( self.test_stats_shared) progbar.update(num_valid_examples) print #-------[ Update valid best cost and error values ]-------# valid_error = total_valid_stopping_cost / num_valid_examples valid_cost = total_valid_cost / num_valid_examples valid_stats_values /= num_valid_examples if valid_error < best_valid_error: best_valid_error = valid_error self.log.info('..best validation error so far') if self.log.save_model: self.log._save_model(self.model) self.log.info('..model saved') if valid_error < self.best_valid_last_update: error_dcr = self.best_valid_last_update - valid_error else: error_dcr = 0 self.log.info('block time: %0.2fs' % (time.time() - block_time)) self.log.info(get_mem_usage()) #==============[ save to database, save epoch error]==============# if self.log.save_to_database: self.log._save_to_database(epoch, train_cost, valid_cost, best_valid_error) self.log.info('..sent to database: %s:%s' % (self.log.save_to_database['name'], self.log.experiment_name)) if self.log.save_epoch_error: self.log._save_epoch_error(epoch, valid_error) self.log.info('..epoch error saved') end_time = time.time() #=====================[ log outputs to file ]=====================# merged_train = merge_lists(train_stats_names, train_stats_values) merged_valid = merge_lists(valid_stats_names, valid_stats_values) outputs = [('epoch', epoch), ('runtime(s)', int(end_time - start_time)), ('train_' + self.train_cost.func_name, train_cost), ('valid_' + self.train_cost.func_name, valid_cost), ('valid_' + self.valid_cost.func_name, valid_error), ('best_valid_' + self.valid_cost.func_name, best_valid_error)] outputs += merged_train + merged_valid self.log._log_outputs(outputs) job_end = time.time() self.log.info( 'Job Completed on %s' % time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime(job_end))) ttl_time = int(job_end - job_start) dt = datetime.timedelta(seconds=ttl_time) self.log.info('Total Time Taken: %s' % str(dt)) self.log.info("========[ End of Job ]========\n\n")
def dl_progress(count, block_size, total_size): global progbar if progbar is None: progbar = Progbar(total_size) else: progbar.update(count*block_size)
def dl_progress(count, block_size, total_size): global progbar if progbar is None: progbar = Progbar(total_size) else: progbar.update(count * block_size)