def calibrate_sigw1_metric(config, x_future, x_past): sigs_past = config.compute_sig_past(x_past) sigs_future = config.compute_sig_future(x_future) assert sigs_past.size(0) == sigs_future.size(0) X, Y = to_numpy(sigs_past), to_numpy(sigs_future) lm = LinearRegression() lm.fit(X, Y) sigs_pred = torch.from_numpy(lm.predict(X)).float().to(x_future.device) return sigs_pred
def compute_predictive_score(x_past, x_future, x_fake): size = x_fake.shape[0] X = to_numpy(x_past.reshape(size, -1)) Y = to_numpy(x_fake.reshape(size, -1)) size = x_past.shape[0] X_test = X.copy() Y_test = to_numpy(x_future[:, :1].reshape(size, -1)) model = LinearRegression() model.fit(X, Y) # TSTR r2_tstr = model.score(X_test, Y_test) model = LinearRegression() model.fit(X_test, Y_test) # TRTR r2_trtr = model.score(X_test, Y_test) return dict(r2_tstr=r2_tstr, r2_trtr=r2_trtr, predictive_score=np.abs(r2_trtr - r2_tstr))
def _val_single_step(self, batch): """ """ start_step = time.time() self.model.eval() frontal = to_device(batch['frontal'], self.device) lateral = to_device(batch['lateral'], self.device) labels = to_device(batch['labels'], self.device) mask = to_device(batch['mask'], self.device) start_forward = time.time() logits = self.model(frontal, lateral) loss = self.criterion(logits, labels, mask) end_forward = time.time() mask = to_numpy(batch['mask']) labels = to_numpy(batch['labels'].long()) scores = to_numpy(torch.sigmoid(logits)) self.pr_meter.add_predictions(mask, scores, labels) self.auc_meter.add_predictions(mask, scores, labels) results = [] if self.save_predictions: for i in range(len(labels)): results.append({ 'patient': batch['patient'][i], 'study_id': batch['study_id'][i], 'frontal': batch['frontal_fn'][i], 'lateral': batch['lateral_fn'][i], 'labels': labels[i], 'mask': mask[i], 'scores': scores[i], }) end_step = time.time() meta = {} meta['val_time'] = end_step - start_step meta['forward_time'] = end_forward - start_forward meta['learning_rate'] = self.optimizer.param_groups[-1]['lr'] meta['memory_allocated'] = torch.cuda.memory_allocated() metrics = {} metrics['loss'] = loss.item() return {'meta': meta, 'metrics': metrics, 'results': results}
def test(): # lmdb_path = "/share/zhui/reg_dataset/NIPS2014" lmdb_path = "/share/zhui/reg_dataset/IIIT5K_3000" train_dataset = LmdbDataset(root=lmdb_path, voc_type='ALLCASES_SYMBOLS', max_len=50) batch_size = 1 train_dataloader = data.DataLoader( train_dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=AlignCollate(imgH=64, imgW=256, keep_ratio=False)) for i, (images, labels, label_lens) in enumerate(train_dataloader): # visualization of input image # toPILImage = transforms.ToPILImage() images = images.permute(0, 2, 3, 1) images = to_numpy(images) images = images * 0.5 + 0.5 images = images * 255 for id, (image, label, label_len) in enumerate(zip(images, labels, label_lens)): image = Image.fromarray(np.uint8(image)) # image = toPILImage(image) image.show() print(image.size) print( labels2strs(label, train_dataset.id2char, train_dataset.char2id)) print(label_len.item()) input()
def compare_cross_corr(x_real, x_fake): """ Computes cross correlation matrices of x_real and x_fake and plots them. """ x_real = x_real.reshape(-1, x_real.shape[2]) x_fake = x_fake.reshape(-1, x_fake.shape[2]) cc_real = np.corrcoef(to_numpy(x_real).T) cc_fake = np.corrcoef(to_numpy(x_fake).T) vmin = min(cc_fake.min(), cc_real.min()) vmax = max(cc_fake.max(), cc_real.max()) fig, axes = plt.subplots(1, 2) axes[0].matshow(cc_real, vmin=vmin, vmax=vmax) im = axes[1].matshow(cc_fake, vmin=vmin, vmax=vmax) axes[0].set_title('Real') axes[1].set_title('Generated') fig.subplots_adjust(right=0.8) cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.7]) fig.colorbar(im, cax=cbar_ax)
def select_action(self, s_t, episode): # assert episode >= self.warmup, 'Episode: {} warmup: {}'.format(episode, self.warmup) action = to_numpy(self.actor(to_tensor(np.array(s_t).reshape( 1, -1)))).squeeze(0) delta = self.init_delta * (self.delta_decay**(episode - self.warmup)) # action += self.is_training * max(self.epsilon, 0) * self.random_process.sample() action = self.sample_from_truncated_normal_distribution( lower=self.lbound, upper=self.rbound, mu=action, sigma=delta) action = np.clip(action, self.lbound, self.rbound) # self.a_t = action return action
def plot_summary(x_fake, x_real, max_lag=None, labels=None): if max_lag is None: max_lag = min(128, x_fake.shape[1]) from lib.test_metrics import skew_torch, kurtosis_torch dim = x_real.shape[2] _, axes = plt.subplots(dim, 3, figsize=(25, dim * 5)) if len(axes.shape) == 1: axes = axes[None, ...] for i in range(dim): x_real_i = x_real[..., i:i + 1] x_fake_i = x_fake[..., i:i + 1] compare_hists(x_real=to_numpy(x_real_i), x_fake=to_numpy(x_fake_i), ax=axes[i, 0]) def text_box(x, height, title): textstr = '\n'.join(( r'%s' % (title,), # t'abs_metric=%.2f' % abs_metric r'$s=%.2f$' % (skew_torch(x).item(),), r'$\kappa=%.2f$' % (kurtosis_torch(x).item(),)) ) props = dict(boxstyle='round', facecolor='wheat', alpha=0.5) axes[i, 0].text( 0.05, height, textstr, transform=axes[i, 0].transAxes, fontsize=14, verticalalignment='top', bbox=props ) text_box(x_real_i, 0.95, 'Historical') text_box(x_fake_i, 0.70, 'Generated') compare_hists(x_real=to_numpy(x_real_i), x_fake=to_numpy(x_fake_i), ax=axes[i, 1], log=True) compare_acf(x_real=x_real_i, x_fake=x_fake_i, ax=axes[i, 2], max_lag=max_lag, CI=False, dim=(0, 1))
def select_action(self, s_t, episode): # assert episode >= self.warmup, 'Episode: {} warmup: {}'.format(episode, self.warmup) # squeeze()可以删除但唯独条目,即把shape为1的维度去掉 action = to_numpy(self.actor(to_tensor(np.array(s_t).reshape(1, -1)))).squeeze(0) # 通过默认值为0.95的衰减率求得目前的delta作为截断正态分布的方差 delta = self.init_delta * (self.delta_decay ** (episode - self.warmup)) # action += self.is_training * max(self.epsilon, 0) * self.random_process.sample() # 根据action与delta得到一个截断正态分布的采样,相当于有加入了一定的随机性 action = self.sample_from_truncated_normal_distribution(lower=self.lbound, upper=self.rbound, mu=action, sigma=delta) # 没必要吧,毕竟上一句已经能够限定范围了 action = np.clip(action, self.lbound, self.rbound) # self.a_t = action return action
def evaluate(self, x_fake): for test_metric in self.test_metrics_list: with torch.no_grad(): test_metric(x_fake[:10000]) self.training_loss[test_metric.name].append( to_numpy(test_metric.loss_componentwise))
def test(): train_dataset = LmdbDataset( root=global_args.test_data_dir, voc_type=global_args.voc_type, max_len=global_args.max_len, num_samples=global_args.num_test, with_name=True) print(train_dataset.nSamples, 'samples') train_dataloader = data.DataLoader( train_dataset, batch_size=1, shuffle=False, num_workers=global_args.workers, collate_fn=AlignCollateWithNames(imgH=64, imgW=256, keep_ratio=False)) if global_args.image_path: out_html = open(os.path.join(global_args.image_path, 'index.html'), 'w') out_html.write('''<html> <body> <table> <tr><th>No</th><th>Image</th><th>Labels</th><th>Length</th><th>Name</th></tr> ''') else: out_html = None i = 1 max_len = 0 for images, labels, label_lens, image_names in train_dataloader: # visualization of input image # toPILImage = transforms.ToPILImage() images = images.permute(0,2,3,1) images = to_numpy(images) images = images * 0.5 + 0.5 images = images * 255 for image, label, label_len, image_name in zip(images, labels, label_lens, image_names): image = Image.fromarray(np.uint8(image)) label_str = labels2strs(label, train_dataset.id2char, train_dataset.char2id) if image_name is not None: image_name = image_name.decode('utf-8') else: image_name = '' l_len = label_len.item() if max_len < l_len: max_len = l_len if global_args.image_path: image_filename = f'image-{i:09d}.jpg' image.save(os.path.join(global_args.image_path, image_filename)) out_html.write( f'<tr><td>{i}</td>' f'<td><img src="{image_filename}" width="{image.width}" height="{image.height}" /></td>' f'<td>{label_str}</td><td>{l_len}</td><td>{image_name}</td></tr>\n') else: image.show() print(image.size) print(label_str, l_len) if image_name: print(image_name) input() i += 1 if out_html: out_html.write(f'</table>\n<p>The maximal label length is {max_len}.</p>\n</body>\n</html>\n') out_html.close()
def evaluate_generator(model_name, seed, experiment_dir, dataset, use_cuda=True): """ Args: model_name: experiment_dir: dataset: use_cuda: Returns: """ torch.random.manual_seed(0) if use_cuda: device = 'cuda' else: device = 'cpu' experiment_summary = dict() experiment_summary['model_id'] = model_name experiment_summary['seed'] = seed sig_config = get_algo_config(dataset, experiment_dir) # shorthands base_config = BaseConfig(device=device) p, q = base_config.p, base_config.q # ---------------------------------------------- # Load and prepare real path. # ---------------------------------------------- x_real = load_pickle(os.path.join(os.path.dirname(experiment_dir), 'x_real.torch')).to(device) x_past, x_future = x_real[:, :p], x_real[:, p:p + q] x_future = x_real[:, p:p + q] dim = x_real.shape[-1] # ---------------------------------------------- # Load generator weights and hyperparameters # ---------------------------------------------- G_weights = load_pickle(os.path.join(experiment_dir, 'G_weights.torch')) G = SimpleGenerator(dim * p, dim, 3 * (50,), dim).to(device) G.load_state_dict(G_weights) # ---------------------------------------------- # Compute predictive score - TSTR (train on synthetic, test on real) # ---------------------------------------------- with torch.no_grad(): x_fake = G.sample(1, x_past) predict_score_dict = compute_predictive_score(x_past, x_future, x_fake) experiment_summary.update(predict_score_dict) # ---------------------------------------------- # Compute metrics and scores of the unconditional distribution. # ---------------------------------------------- with torch.no_grad(): x_fake = G.sample(q, x_past) test_metrics_dict = compute_test_metrics(x_fake, x_real) experiment_summary.update(test_metrics_dict) # ---------------------------------------------- # Compute Sig-W_1 distance. # ---------------------------------------------- if dataset in ['VAR', 'ARCH']: x_past = x_past[::10] x_future = x_future[::10] sigs_pred = calibrate_sigw1_metric(sig_config, x_future, x_past) # generate fake paths sigs_conditional = list() with torch.no_grad(): steps = 100 size = x_past.size(0) // steps for i in range(steps): x_past_sample = x_past[i * size:(i + 1) * size] if i < (steps - 1) else x_past[i * size:] sigs_fake_ce = sample_sig_fake(G, q, sig_config, x_past_sample)[0] sigs_conditional.append(sigs_fake_ce) sigs_conditional = torch.cat(sigs_conditional, dim=0) sig_w1_metric = sigcwgan_loss(sigs_pred, sigs_conditional) experiment_summary['sig_w1_metric'] = sig_w1_metric.item() # ---------------------------------------------- # Create the relevant summary plots. # ---------------------------------------------- with torch.no_grad(): _x_past = x_past.clone().repeat(5, 1, 1) if dataset in ['STOCKS', 'ECG'] else x_past.clone() x_fake_future = G.sample(q, _x_past) plot_summary(x_fake=x_fake_future, x_real=x_real, max_lag=q) plt.savefig(os.path.join(experiment_dir, 'summary.png')) plt.close() if is_multivariate(x_real): compare_cross_corr(x_fake=x_fake_future, x_real=x_real) plt.savefig(os.path.join(experiment_dir, 'cross_correl.png')) plt.close() # ---------------------------------------------- # Generate long paths # ---------------------------------------------- with torch.no_grad(): x_fake = G.sample(8000, x_past[0:1]) plot_summary(x_fake=x_fake, x_real=x_real, max_lag=q) plt.savefig(os.path.join(experiment_dir, 'summary_long.png')) plt.close() plt.plot(to_numpy(x_fake[0, :1000])) plt.savefig(os.path.join(experiment_dir, 'long_path.png')) plt.close() return experiment_summary
def plot_signature(signature_tensor, alpha=0.2): plt.plot(to_numpy(signature_tensor).T, alpha=alpha, linestyle='None', marker='o') plt.grid()