def create_dataloader(config, data, mode): dataset = create_dataset(config, data, mode) if mode == 'train': # create Sampler if dist.is_available() and dist.is_initialized(): train_RandomSampler = distributed.DistributedSampler(dataset) else: train_RandomSampler = sampler.RandomSampler(dataset, replacement=False) train_BatchSampler = sampler.BatchSampler(train_RandomSampler, batch_size=config.train.batch_size, drop_last=config.train.dataloader.drop_last) # Augment collator = get_collate_fn(config) # DataLoader data_loader = DataLoader(dataset=dataset, batch_sampler=train_BatchSampler, collate_fn=collator, pin_memory=config.train.dataloader.pin_memory, num_workers=config.train.dataloader.work_nums) elif mode == 'val': if dist.is_available() and dist.is_initialized(): val_SequentialSampler = distributed.DistributedSampler(dataset) else: val_SequentialSampler = sampler.SequentialSampler(dataset) val_BatchSampler = sampler.BatchSampler(val_SequentialSampler, batch_size=config.val.batch_size, drop_last=config.val.dataloader.drop_last) data_loader = DataLoader(dataset, batch_sampler=val_BatchSampler, pin_memory=config.val.dataloader.pin_memory, num_workers=config.val.dataloader.work_nums) else: if dist.is_available() and dist.is_initialized(): test_SequentialSampler = distributed.DistributedSampler(dataset) else: test_SequentialSampler = None data_loader = DataLoader(dataset, sampler=test_SequentialSampler, batch_size=config.test.batch_size, pin_memory=config.val.dataloader.pin_memory, num_workers=config.val.dataloader.work_nums) return data_loader
def input_fn(config, mode): if mode not in ('train', 'eval', 'pred'): raise ValueError('mode must be one of "train", "eval" and "pred"') mode_map = MODE_MAPPING[mode] numclasses = config['data']['numclasses'] if not isinstance(numclasses, int) or numclasses <= 0: raise ValueError('numclasses must be a positive integer.') tf.logging.info( 'Creating the input function for the estimator for {}.'.format( mode_map)) data_config = config['data'][mode_map] preprocessing_config = config['data']['preprocessing'] dataset = create_dataset(data_config, numclasses, preprocessing_config, mode) return dataset
args = parser.parse_args() options_path = args.options opt = options.parse(options_path, is_train=False) util.mkdirs((path for key, path in opt['path'].items() if not key == 'pretrain_model_G')) # Make all directories needed opt = options.dict2box(opt) from data.datasets import create_dataset from data.data_loader import create_dataloader from models.models import create_model # Create test dataset and dataloader test_loaders = [] test_set_names = [] for dataset_opt in opt.datasets: test_set = create_dataset(dataset_opt) test_loader = create_dataloader(test_set, dataset_opt) test_size = len(test_set) test_set_name = dataset_opt.name print('Number of test images in [%s]: %d' % (test_set_name, test_size)) test_loaders.append(test_loader) test_set_names.append(test_set_name) # Create model model = create_model(opt) # Path for log file test_log_path = os.path.join(opt.path.log, 'test_log.txt') if os.path.exists(test_log_path): os.remove(test_log_path) print('Old test log is removed.')
import tensorflow as tf from data.datasets import create_dataset from utils.writeconfig import DEFAULT_CONFIG tf.logging.set_verbosity(tf.logging.INFO) db = create_dataset(DEFAULT_CONFIG['data']['training'], 80, DEFAULT_CONFIG['data']['preprocessing'], 'train') dbiter = db.make_initializable_iterator() data = dbiter.get_next() init_op = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init_op) sess.run(dbiter.initializer) try: while True: data_out = sess.run(data) print(data['features'].shape) for key in data_out['labels']: print('{} - {}'.format(key, data_out['labels'][key].shape)) if key == 'object_bboxes': print(data_out['labels'][key][ 0, 0:data_out['labels']['numbboxes'][0], :]) except tf.errors.OutOfRangeError: print('Finished.')
def r_squared(self): mean_y = mean(self.ys) mean_ys = [mean_y for x in self.ys] total_sum_of_squares = sum((self.ys - mean_ys)**2) residual_sum_of_squares = sum((self.ys - self.regression_line)**2) return 1. - (residual_sum_of_squares / total_sum_of_squares) def draw(self, predict_x=None): style.use('ggplot') plt.scatter(self.xs, self.ys, color="#003D72", label="data") plt.plot(self.xs, self.regression_line, label="regression line") plt.legend(loc=2) if predict_x: plt.scatter(predict_x, self.predict(predict_x), color="g", label="predict") plt.legend(loc=4) plt.show() if __name__ == '__main__': # xs, ys = create_dataset(n=40, variance=40, step=2, correlation='pos') xs, ys = create_dataset(n=40, variance=5, step=2) regression = Regression(xs, ys) regression.draw(7) print(regression.r_squared()) # print(regression.best_fit_slope())
def main(): # Create train dataset train_set_opt = opt.datasets[0] train_set = create_dataset(train_set_opt) train_size = int(math.ceil(len(train_set) / train_set_opt.batch_size)) print('Number of train images: %d batches of size %d' % (train_size, train_set_opt.batch_size)) total_iters = int(opt.train.niter) total_epoches = int(math.ceil(total_iters / train_size)) print('Total epoches needed: %d' % total_epoches) # Create val dataset val_set_opt = opt.datasets[1] val_set = create_dataset(val_set_opt) val_size = len(val_set) print('Number of val images: %d' % val_size) # Create dataloader train_loader = create_dataloader(train_set, train_set_opt) val_loader = create_dataloader(val_set, val_set_opt) # Create model model = create_model(opt) # Create binarization module import bin global bin_op bin_op = bin.BinOp(model.netG) model.train() # Create logger logger = Logger(opt) current_step = 0 need_make_val_dir = True start_time = time.time() for epoch in range(total_epoches): for i, train_data in enumerate(train_loader): current_step += 1 if current_step > total_iters: break train_start_time = time.time() # Training model.feed_data(train_data) # optimize_parameters 함수를 분할해서 대체함 for binarization # model.optimize_parameters(current_step) bin_op.binarization() model.forward_G() model.optimizer_G.zero_grad() model.backward_G() bin_op.restore() bin_op.updateBinaryGradWeight() model.optimizer_G.step() train_duration = time.time() - train_start_time if current_step % opt.logger.print_freq == 0: losses = model.get_current_losses() logger.print_results(losses, epoch, current_step, train_duration, 'loss') if current_step % opt.logger.save_checkpoint_freq == 0: print('Saving the model at the end of current_step %d' % (current_step)) model.save(current_step) # Validation if current_step % opt.train.val_freq == 0: validate(val_loader, val_size, model, logger, epoch, current_step) model.update_learning_rate(step=current_step, scheme=opt.train.lr_scheme) print('End of Epoch %d' % epoch) print('Saving the final model') model.save('latest') print('End of Training \t Time Taken: %d sec' % (time.time() - start_time))