def test(dataset, data_split, label_split, model, logger, epoch): with torch.no_grad(): metric = Metric() model.train(False) for m in range(cfg['num_users']): data_loader = make_data_loader({'test': SplitDataset(dataset, data_split[m])})['test'] for i, input in enumerate(data_loader): input = collate(input) input_size = input['img'].size(0) input['label_split'] = torch.tensor(label_split[m]) input = to_device(input, cfg['device']) output = model(input) output['loss'] = output['loss'].mean() if cfg['world_size'] > 1 else output['loss'] evaluation = metric.evaluate(cfg['metric_name']['test']['Local'], input, output) logger.append(evaluation, 'test', input_size) data_loader = make_data_loader({'test': dataset})['test'] for i, input in enumerate(data_loader): input = collate(input) input_size = input['img'].size(0) input = to_device(input, cfg['device']) output = model(input) output['loss'] = output['loss'].mean() if cfg['world_size'] > 1 else output['loss'] evaluation = metric.evaluate(cfg['metric_name']['test']['Global'], input, output) logger.append(evaluation, 'test', input_size) info = {'info': ['Model: {}'.format(cfg['model_tag']), 'Test Epoch: {}({:.0f}%)'.format(epoch, 100.)]} logger.append(info, 'test', mean=False) logger.write('test', cfg['metric_name']['test']['Local'] + cfg['metric_name']['test']['Global']) return
def test(data_loader, model, logger, epoch): with torch.no_grad(): metric = Metric() model.train(False) for i, input in enumerate(data_loader): input = collate(input) input_size = input['img'].size(0) input = to_device(input, cfg['device']) output = model(input) output['loss'] = output['loss'].mean( ) if cfg['world_size'] > 1 else output['loss'] evaluation = metric.evaluate(cfg['metric_name']['test'], input, output) logger.append(evaluation, 'test', input_size) logger.append(evaluation, 'test') info = { 'info': [ 'Model: {}'.format(cfg['model_tag']), 'Test Epoch: {}({:.0f}%)'.format(epoch, 100.) ] } logger.append(info, 'test', mean=False) logger.write('test', cfg['metric_name']['test']) if cfg['show']: input['reconstruct'] = True input['z'] = output['z'] output = model.reverse(input) save_img(input['img'][:100], './output/vis/input_{}.png'.format(cfg['model_tag']), range=(-1, 1)) save_img(output['img'][:100], './output/vis/output_{}.png'.format(cfg['model_tag']), range=(-1, 1)) return
def runExperiment(): seed = int(cfg['model_tag'].split('_')[0]) torch.manual_seed(seed) torch.cuda.manual_seed(seed) dataset = fetch_dataset(cfg['data_name'], cfg['subset']) process_dataset(dataset['train']) if cfg['raw']: data_loader = make_data_loader(dataset)['train'] metric = Metric() img = [] for i, input in enumerate(data_loader): input = collate(input) img.append(input['img']) img = torch.cat(img, dim=0) output = {'img': img} evaluation = metric.evaluate(cfg['metric_name']['test'], None, output) is_result, fid_result = evaluation['InceptionScore'], evaluation['FID'] print('Inception Score ({}): {}'.format(cfg['data_name'], is_result)) print('FID ({}): {}'.format(cfg['data_name'], fid_result)) save(is_result, './output/result/is_generated_{}.npy'.format(cfg['data_name']), mode='numpy') save(fid_result, './output/result/fid_generated_{}.npy'.format(cfg['data_name']), mode='numpy') else: generated = np.load('./output/npy/generated_{}.npy'.format( cfg['model_tag']), allow_pickle=True) test(generated) return
def train(data_loader, model, optimizer, logger, epoch): metric = Metric() model.train(True) for i, input in enumerate(data_loader): start_time = time.time() input = collate(input) input_size = len(input['img']) input = to_device(input, config.PARAM['device']) model.zero_grad() output = model(input) output['loss'] = output['loss'].mean() if config.PARAM['world_size'] > 1 else output['loss'] output['loss'].backward() optimizer.step() if i % int((len(data_loader) * config.PARAM['log_interval']) + 1) == 0: batch_time = time.time() - start_time lr = optimizer.param_groups[0]['lr'] epoch_finished_time = datetime.timedelta(seconds=round(batch_time * (len(data_loader) - i - 1))) exp_finished_time = epoch_finished_time + datetime.timedelta( seconds=round((config.PARAM['num_epochs'] - epoch) * batch_time * len(data_loader))) info = {'info': ['Model: {}'.format(config.PARAM['model_tag']), 'Train Epoch: {}({:.0f}%)'.format(epoch, 100. * i / len(data_loader)), 'Learning rate: {}'.format(lr), 'Epoch Finished Time: {}'.format(epoch_finished_time), 'Experiment Finished Time: {}'.format(exp_finished_time)]} logger.append(info, 'train', mean=False) evaluation = metric.evaluate(config.PARAM['metric_names']['train'], input, output) logger.append(evaluation, 'train', n=input_size) logger.write('train', config.PARAM['metric_names']['train']) return
def test(data_loader, ae, model, logger, epoch): with torch.no_grad(): metric = Metric() ae.train(False) model.train(False) for i, input in enumerate(data_loader): input = collate(input) input_size = input['img'].size(0) input = to_device(input, cfg['device']) _, _, input['img'] = ae.encode(input['img']) input['img'] = input['img'].detach() output = model(input) output['loss'] = output['loss'].mean( ) if cfg['world_size'] > 1 else output['loss'] evaluation = metric.evaluate(cfg['metric_name']['test'], input, output) logger.append(evaluation, 'test', input_size) logger.append(evaluation, 'test') info = { 'info': [ 'Model: {}'.format(cfg['model_tag']), 'Test Epoch: {}({:.0f}%)'.format(epoch, 100.) ] } logger.append(info, 'test', mean=False) logger.write('test', cfg['metric_name']['test']) return
def runExperiment(): seed = int(cfg['model_tag'].split('_')[0]) torch.manual_seed(seed) torch.cuda.manual_seed(seed) dataset = fetch_dataset(cfg['data_name'], cfg['subset']) process_dataset(dataset['train']) if cfg['raw']: data_loader = make_data_loader(dataset)['train'] metric = Metric() img, label = [], [] for i, input in enumerate(data_loader): input = collate(input) img.append(input['img']) label.append(input['label']) img = torch.cat(img, dim=0) label = torch.cat(label, dim=0) output = {'img': img, 'label': label} evaluation = metric.evaluate(cfg['metric_name']['test'], None, output) dbi_result = evaluation['DBI'] print('Davies-Bouldin Index ({}): {}'.format(cfg['data_name'], dbi_result)) save(dbi_result, './output/result/dbi_created_{}.npy'.format(cfg['data_name']), mode='numpy') else: created = np.load('./output/npy/created_{}.npy'.format( cfg['model_tag']), allow_pickle=True) test(created) return
def test(data_loader, model): with torch.no_grad(): metric = Metric() model.train(False) for i, input in enumerate(data_loader): input = collate(input) input = to_device(input, config.PARAM['device']) output = model(input) output['loss'] = output['loss'].mean() if config.PARAM['world_size'] > 1 else output['loss'] evaluation = metric.evaluate(config.PARAM['metric_names']['test'], input, output) print(evaluation) return evaluation
def test(dataset, model, logger, epoch): with torch.no_grad(): metric = Metric() model.train(False) batch_dataset = BatchDataset(dataset, cfg['bptt']) for i, input in enumerate(batch_dataset): input_size = input['label'].size(0) input = to_device(input, cfg['device']) output = model(input) output['loss'] = output['loss'].mean() if cfg['world_size'] > 1 else output['loss'] evaluation = metric.evaluate(cfg['metric_name']['test'], input, output) logger.append(evaluation, 'test', input_size) info = {'info': ['Model: {}'.format(cfg['model_tag']), 'Test Epoch: {}({:.0f}%)'.format(epoch, 100.)]} logger.append(info, 'test', mean=False) logger.write('test', cfg['metric_name']['test']) return
def test(created): with torch.no_grad(): metric = Metric() created = torch.tensor(created / 255 * 2 - 1) valid_mask = torch.sum(torch.isnan(created), dim=(1, 2, 3)) == 0 created = created[valid_mask] label = torch.arange(cfg['classes_size']) label = label.repeat(cfg['generate_per_mode']) label = label[valid_mask] output = {'img': created, 'label': label} evaluation = metric.evaluate(cfg['metric_name']['test'], None, output) dbi_result = evaluation['DBI'] print('Davies-Bouldin Index ({}): {}'.format(cfg['model_tag'], dbi_result)) save(dbi_result, './output/result/dbi_created_{}.npy'.format(cfg['model_tag']), mode='numpy') return evaluation
def test(generated): with torch.no_grad(): metric = Metric() generated = torch.tensor(generated / 255 * 2 - 1) valid_mask = torch.sum(torch.isnan(generated), dim=(1, 2, 3)) == 0 generated = generated[valid_mask] output = {'img': generated} evaluation = metric.evaluate(cfg['metric_name']['test'], None, output) is_result, fid_result = evaluation['InceptionScore'], evaluation['FID'] print('Inception Score ({}): {}'.format(cfg['model_tag'], is_result)) print('FID ({}): {}'.format(cfg['model_tag'], fid_result)) save(is_result, './output/result/is_generated_{}.npy'.format(cfg['model_tag']), mode='numpy') save(fid_result, './output/result/fid_generated_{}.npy'.format(cfg['model_tag']), mode='numpy') return evaluation
def train(data_loader, ae, model, optimizer, logger, epoch): metric = Metric() ae.train(False) model.train(True) start_time = time.time() for i, input in enumerate(data_loader): input = collate(input) input_size = input['img'].size(0) input = to_device(input, cfg['device']) with torch.no_grad(): _, _, input['img'] = ae.encode(input['img']) input['img'] = input['img'].detach() optimizer.zero_grad() output = model(input) output['loss'] = output['loss'].mean( ) if cfg['world_size'] > 1 else output['loss'] output['loss'].backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 1) optimizer.step() evaluation = metric.evaluate(cfg['metric_name']['train'], input, output) logger.append(evaluation, 'train', n=input_size) if i % int((len(data_loader) * cfg['log_interval']) + 1) == 0: batch_time = (time.time() - start_time) / (i + 1) lr = optimizer.param_groups[0]['lr'] epoch_finished_time = datetime.timedelta( seconds=round(batch_time * (len(data_loader) - i - 1))) exp_finished_time = epoch_finished_time + datetime.timedelta( seconds=round((cfg['num_epochs'] - epoch) * batch_time * len(data_loader))) info = { 'info': [ 'Model: {}'.format(cfg['model_tag']), 'Train Epoch: {}({:.0f}%)'.format( epoch, 100. * i / len(data_loader)), 'Learning rate: {}'.format(lr), 'Epoch Finished Time: {}'.format(epoch_finished_time), 'Experiment Finished Time: {}'.format(exp_finished_time) ] } logger.append(info, 'train', mean=False) logger.write('train', cfg['metric_name']['train']) return
def train(self, local_parameters, lr, logger): metric = Metric() model = eval('models.{}(model_rate=self.model_rate).to(cfg["device"])'.format(cfg['model_name'])) model.load_state_dict(local_parameters) model.train(True) optimizer = make_optimizer(model, lr) for local_epoch in range(1, cfg['num_epochs']['local'] + 1): for i, input in enumerate(self.data_loader): input = collate(input) input_size = input['img'].size(0) input['label_split'] = torch.tensor(self.label_split) input = to_device(input, cfg['device']) optimizer.zero_grad() output = model(input) output['loss'].backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 1) optimizer.step() evaluation = metric.evaluate(cfg['metric_name']['train']['Local'], input, output) logger.append(evaluation, 'train', n=input_size) local_parameters = model.state_dict() return local_parameters
def test(model, logger, epoch): sample_per_iter = cfg['batch_size']['test'] with torch.no_grad(): metric = Metric() model.train(False) C = torch.arange(cfg['classes_size']) C = C.repeat(cfg['generate_per_mode']) cfg['z'] = torch.randn([C.size(0), cfg['gan']['latent_size']]) if 'z' not in cfg else cfg['z'] C_generated = torch.split(C, sample_per_iter) z_generated = torch.split(cfg['z'], sample_per_iter) generated = [] for i in range(len(C_generated)): C_generated_i = C_generated[i].to(cfg['device']) z_generated_i = z_generated[i].to(cfg['device']) generated_i = model.generate(C_generated_i, z_generated_i) generated.append(generated_i.cpu()) generated = torch.cat(generated) output = {'img': generated} evaluation = metric.evaluate(cfg['metric_name']['test'], None, output) logger.append(evaluation, 'test') info = {'info': ['Model: {}'.format(cfg['model_tag']), 'Test Epoch: {}({:.0f}%)'.format(epoch, 100.)]} logger.append(info, 'test', mean=False) logger.write('test', cfg['metric_name']['test']) return
def best_split(self, metric: Metric) -> Tuple[float, float]: vals = metric.evaluate(self) i = metric.best_index(vals) return self.uniq[i], vals[i]
def train(data_loader, model, optimizer, logger, epoch): metric = Metric() model.train(True) start_time = time.time() for i, input in enumerate(data_loader): input = collate(input) input_size = input['img'].size(0) input = to_device(input, cfg['device']) ############################ # (1) Update D network ########################### for _ in range(cfg['iter']['discriminator']): # train with real optimizer['discriminator'].zero_grad() optimizer['generator'].zero_grad() D_x = model.discriminate(input['img'], input[cfg['subset']]) # train with fake z1 = torch.randn(input['img'].size(0), cfg['gan']['latent_size'], device=cfg['device']) generated = model.generate(input[cfg['subset']], z1) D_G_z1 = model.discriminate(generated.detach(), input[cfg['subset']]) if cfg['loss_type'] == 'BCE': D_loss = torch.nn.functional.binary_cross_entropy_with_logits( D_x, torch.ones((input['img'].size(0), 1), device=cfg['device'])) + \ torch.nn.functional.binary_cross_entropy_with_logits( D_G_z1, torch.zeros((input['img'].size(0), 1), device=cfg['device'])) elif cfg['loss_type'] == 'Hinge': D_loss = torch.nn.functional.relu(1.0 - D_x).mean() + torch.nn.functional.relu(1.0 + D_G_z1).mean() else: raise ValueError('Not valid loss type') D_loss.backward() optimizer['discriminator'].step() ############################ # (2) Update G network ########################### for _ in range(cfg['iter']['generator']): optimizer['discriminator'].zero_grad() optimizer['generator'].zero_grad() z2 = torch.randn(input['img'].size(0), cfg['gan']['latent_size'], device=cfg['device']) generated = model.generate(input[cfg['subset']], z2) D_G_z2 = model.discriminate(generated, input[cfg['subset']]) if cfg['loss_type'] == 'BCE': G_loss = torch.nn.functional.binary_cross_entropy_with_logits( D_G_z2, torch.ones((input['img'].size(0), 1), device=cfg['device'])) elif cfg['loss_type'] == 'Hinge': G_loss = -D_G_z2.mean() else: raise ValueError('Not valid loss type') G_loss.backward() optimizer['generator'].step() output = {'loss': abs(D_loss - G_loss), 'loss_D': D_loss, 'loss_G': G_loss} evaluation = metric.evaluate(cfg['metric_name']['train'], input, output) logger.append(evaluation, 'train', n=input_size) if i % int((len(data_loader) * cfg['log_interval']) + 1) == 0: batch_time = (time.time() - start_time) / (i + 1) generator_lr, discriminator_lr = optimizer['generator'].param_groups[0]['lr'], \ optimizer['discriminator'].param_groups[0]['lr'] epoch_finished_time = datetime.timedelta(seconds=round(batch_time * (len(data_loader) - i - 1))) exp_finished_time = epoch_finished_time + datetime.timedelta( seconds=round((cfg['num_epochs'] - epoch) * batch_time * len(data_loader))) info = {'info': ['Model: {}'.format(cfg['model_tag']), 'Train Epoch: {}({:.0f}%)'.format(epoch, 100. * i / len(data_loader)), 'Learning rate : (G: {}, D: {})'.format(generator_lr, discriminator_lr), 'Epoch Finished Time: {}'.format(epoch_finished_time), 'Experiment Finished Time: {}'.format(exp_finished_time)]} logger.append(info, 'train', mean=False) logger.write('train', cfg['metric_name']['train']) return