def __normalize(self, data): max = data.max(axis=0, keepdims=True) min = data.min(axis=0, keepdims=True) data = (data - min) / (max - min) return (data - 0.5) / 0.5
def normalize(data, old_min=None, old_max=None, new_min=0, new_max=1, dim='time'): # Function to remove seasonality from data # Returns de-seasonalized data with same shape as input if 'time' in data.dims: # get year and month as separate dimension data = unstack_month_and_year(data) if dim == 'time': data = data.stack(time=['year', 'month']) if old_min is None: old_min = data.min(dim=dim) old_max = data.max(dim=dim) data.values = np.float32( minmax_scaler(data, old_min=old_min, new_min=new_min, old_max=old_max, new_max=new_max)) return data.unstack(), old_min, old_max
def experiment_gan_plot(data, samples, title, ax, is_spiral=False): if is_spiral: ax.scatter(data[:, 0], data[:, 1], label='real') ax.scatter(samples[:, 0], samples[:, 1], label='fake') else: data_grid = np.linspace(data.min(), data.max(), 1000) sample_grid = np.linspace(samples.min(), samples.max(), 1000) bandwidth = 0.1 data_p_n = kde(data, data_grid, bandwidth=bandwidth) sample_p_n = kde(samples, sample_grid, bandwidth=bandwidth) ax.fill_between(x=data_grid, y1=data_p_n, y2=0, alpha=0.7, label='real') ax.fill_between(x=sample_grid, y1=sample_p_n, y2=0, alpha=0.7, label='fake') ax.legend(prop={'size': 40}) ax.grid() ax.set_title(title, fontsize=45) ax.tick_params(axis="x", labelsize=40) ax.tick_params(axis="y", labelsize=40)
def get_mri_images(file): img = nib.load(file) data = img.get_fdata() maxx = data.max() data = data / maxx return data, data.shape[-1]
def normalize(attr_data): data = np.array([i for item in attr_data for i in item]) data_min = data.min() data_max = data.max() std_attr_data = (data - data_min) / (data_max - data_min) std_attr_data = std_attr_data.astype(np.float32) return std_attr_data
def annotate_heatmap(im, data=None, valfmt="{x:.2f}", textcolors=("black", "white"), threshold=None, **textkw): """ A function to annotate a heatmap. Parameters ---------- im The AxesImage to be labeled. data Data used to annotate. If None, the image's data is used. Optional. valfmt The format of the annotations inside the heatmap. This should either use the string format method, e.g. "$ {x:.2f}", or be a `matplotlib.ticker.Formatter`. Optional. textcolors A pair of colors. The first is used for values below a threshold, the second for those above. Optional. threshold Value in data units according to which the colors from textcolors are applied. If None (the default) uses the middle of the colormap as separation. Optional. **kwargs All other arguments are forwarded to each call to `text` used to create the text labels. """ if not isinstance(data, (list, np.ndarray)): data = im.get_array() # Normalize the threshold to the images color range. if threshold is not None: threshold = im.norm(threshold) else: threshold = im.norm(data.max()) / 2. # Set default alignment to center, but allow it to be # overwritten by textkw. kw = dict(horizontalalignment="center", verticalalignment="center") kw.update(textkw) # Get the formatter in case a string is supplied if isinstance(valfmt, str): valfmt = matplotlib.ticker.StrMethodFormatter(valfmt) # Loop over the data and create a `Text` for each "pixel". # Change the text's color depending on the data. texts = [] for i in range(data.shape[0]): for j in range(data.shape[1]): kw.update(color=textcolors[int(im.norm(data[i, j]) > threshold)]) text = im.axes.text(j, i, valfmt(data[i, j], None), **kw) texts.append(text) return texts
def bytescale(data, high=255, low=0): """ Byte scales an array (image). Byte scaling means converting the input image to uint8 dtype and scaling the range to ``(low, high)`` (default 0-255). If the input image already has dtype uint8, no scaling is done. This function is only available if Python Imaging Library (PIL) is installed. Parameters ---------- data : ndarray PIL image data array. cmin : scalar, optional Bias scaling of small values. Default is ``data.min()``. cmax : scalar, optional Bias scaling of large values. Default is ``data.max()``. high : scalar, optional Scale max value to `high`. Default is 255. low : scalar, optional Scale min value to `low`. Default is 0. Returns ------- img_array : uint8 ndarray The byte-scaled array. Examples -------- >>> from scipy.misc import bytescale >>> img = np.array([[ 91.06794177, 3.39058326, 84.4221549 ], ... [ 73.88003259, 80.91433048, 4.88878881], ... [ 51.53875334, 34.45808177, 27.5873488 ]]) >>> bytescale(img) array([[255, 0, 236], [205, 225, 4], [140, 90, 70]], dtype=uint8) >>> bytescale(img, high=200, low=100) array([[200, 100, 192], [180, 188, 102], [155, 135, 128]], dtype=uint8) >>> bytescale(img, cmin=0, cmax=255) array([[91, 3, 84], [74, 81, 5], [52, 34, 28]], dtype=uint8) """ if data.dtype == np.uint8: return data if high > 255 or low < 0 or high < low: raise ValueError("check high low values") cmin = data.min() cmax = data.max() cscale = cmax - cmin if cscale == 0: cscale = 1 scale = float(high - low) / cscale bytedata = (data - cmin) * scale + low return (bytedata.clip(low, high) + 0.5).astype(np.uint8)
def test(args, cfg): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model_file_name = os.path.split(args.model_path)[1] model_name = model_file_name[:model_file_name.find("_")] IMG_Path = Path(args.img_path) IMG_File = natsort.natsorted(list(IMG_Path.glob("*.png")), alg=natsort.PATH) IMG_Str = [] for i in IMG_File: IMG_Str.append(str(i)) # Setup image print("Read Input Image from : {}".format(args.img_path)) data_loader = get_loader(args.dataset) data_path = get_data_path(args.dataset, config_file=cfg) loader = data_loader(data_path, is_transform=True, img_norm=args.img_norm) n_classes = loader.n_classes # Setup Model model = get_model(cfg['model'], n_classes) state = convert_state_dict(torch.load(args.model_path)["model_state"]) # state=torch.load(args.model_path)["model_state"] model.load_state_dict(state) model.eval() model.to(device) for j in tqdm(range(len(IMG_Str))): img_path = IMG_Str[j] img = misc.imread(img_path) # img = img[:, :, ::-1] img = img.astype(np.float64) # img -= loader.mean if args.img_norm: img = img.astype(float) / 255.0 # NHWC -> NCHW img = img.transpose(2, 0, 1) img = np.expand_dims(img, 0) img = torch.from_numpy(img).float() images = img.to(device) outputs = model(images) outputs_probability = F.softmax(outputs) data = outputs_probability.data data_max = data.max(1) prob = data_max[0] prob_img_format = np.squeeze(prob.cpu().numpy(), axis=0) avg_prob = np.mean(prob_img_format) print("Confidence Score for %s: \n%f" % (img_path, avg_prob)) pred = np.squeeze(outputs.data.max(1)[1].cpu().numpy(), axis=0) decoded = loader.decode_segmap(pred) out_path = "test_out/test_confidence/out/" + Path(img_path).name decoded_bgr = cv.cvtColor(decoded, cv.COLOR_RGB2BGR) # misc.imsave(out_path, decoded) cv.imwrite(out_path, decoded_bgr)
def normalize_min_max(self, data): data = np.array(data) denominator = data.max(axis=0) - data.min(axis=0) denominator[denominator == 0] = 1 data = (data - data.min(axis=0)) / denominator return data
def normalize_dataset(data, normalizer, column_wise=False): if normalizer == 'max01': if column_wise: minimum = data.min(axis=0, keepdims=True) maximum = data.max(axis=0, keepdims=True) else: minimum = data.min() maximum = data.max() scaler = MinMax01Scaler(minimum, maximum) data = scaler.transform(data) print('Normalize the dataset by MinMax01 Normalization') elif normalizer == 'max11': if column_wise: minimum = data.min(axis=0, keepdims=True) maximum = data.max(axis=0, keepdims=True) else: minimum = data.min() maximum = data.max() scaler = MinMax11Scaler(minimum, maximum) data = scaler.transform(data) print('Normalize the dataset by MinMax11 Normalization') elif normalizer == 'std': if column_wise: mean = data.mean(axis=0, keepdims=True) std = data.std(axis=0, keepdims=True) else: mean = data.mean() std = data.std() scaler = StandardScaler(mean, std) data = scaler.transform(data) print('Normalize the dataset by Standard Normalization') elif normalizer == 'None': scaler = NScaler() data = scaler.transform(data) print('Does not normalize the dataset') elif normalizer == 'cmax': #column min max, to be depressed #note: axis must be the spatial dimension, please check ! scaler = ColumnMinMaxScaler(data.min(axis=0), data.max(axis=0)) data = scaler.transform(data) print('Normalize the dataset by Column Min-Max Normalization') else: raise ValueError return data, scaler
def load_data(data_source, seqlen, date_op, date_ed, normalize=True): # data shape: batch * seqlen * feature with open(mapped_edge_filepath, 'r') as f: mapped_edges = json.load(f) totle_time_interval_num = day_time_interval_num * (int(date_ed) - int(date_op) + 1) data = np.zeros((edge_num, totle_time_interval_num)) speedfiles = os.listdir(datadir + data_source + '/') for speedfile in speedfiles: if speedfile.startswith('.') or speedfile.split( '.')[0] > date_ed or speedfile.split('.')[0] < date_op: continue day = int(speedfile.split('.')[0][-2:]) - int(date_op[-2:]) with open(datadir + data_source + '/' + speedfile, 'r') as f: speed = json.load(f) for edge in speed: # current edge not in the selected region if edge not in mapped_edges: continue for time_interval in speed[edge]: edge_id = mapped_edges[edge] data[edge_id, day * day_time_interval_num + int(time_interval)] = speed[edge][time_interval] ub_index = np.where(data == 0) if normalize: data = (data - data.min()) / data.max() data[ub_index] = 0 output = [] for i in range(totle_time_interval_num - seqlen): output.append(data[:, i:i + seqlen]) output = torch.FloatTensor(output).permute(0, 2, 1) filled_rate = len(np.where(output.numpy() != 0)[0]) / ( output.shape[0] * output.shape[1] * output.shape[2]) print( len(np.where(output.numpy()[:72, :, :] != 0)[0]) / output[:72, :, :].numel()) print( len(np.where(output.numpy()[72:120, :, :] != 0)[0]) / output[72:120, :, :].numel()) print( len(np.where(output.numpy()[120:204, :, :] != 0)[0]) / output[120:204, :, :].numel()) print( len(np.where(output.numpy()[204:240, :, :] != 0)[0]) / output[204:240, :, :].numel()) print( len(np.where(output.numpy()[240:288, :, :] != 0)[0]) / output[240:288, :, :].numel()) print(data_source, 'data shape:', output.shape, 'filled rate:', filled_rate) return output
def pil_and_hdf5_loader(path): # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835) # for hdf5 if(path.endswith('.hdf5')): with h5py.File(path, 'r') as f: data = f[HDF5_DATASET_NAME][:].astype(np.float32) # normalize it to 0 to 1 data /= (data.max() - data.min() + 0.0001) # normalize to -1 to 1 data *= 2 data -= 1 # the torrchvision.transforms.toTensor rescales input to the range -1 to 1 in certain conditions, # we want to scale -1 to 1 # so scale in the dataloader itself! # link: https://pytorch.org/docs/stable/torchvision/transforms.html return data # note: # DONOT USE: np.array(f[hdf5_dataset_name]) it was much slower in testing # for other types else: with open(path, 'rb') as f: img = Image.open(f) data = np.array(img.convert('RGB')).astype(np.float32) # here too we want the scaling to be from -1 to 1 # the to tensor normalizes 0 to 1 only if the numpy array is of type uint8 # so return float32 image instead # link: https://pytorch.org/docs/stable/torchvision/transforms.html # normalize it to 0 to 1 data /= (data.max() - data.min() + 0.0001) # normalize to -1 to 1 data *= 2 data -= 1 return data
def __init__(self, data, ground_truth, semi=False): super(HyperX, self).__init__() # Normalize the data in [0,1] data = (data - data.min()) / (data.max() - data.min()) self.data = data self.gt = ground_truth self.n_classes = len(np.unique(ground_truth)) if semi: # Semi-supervision, include neighbours at 50px x_pos, y_pos = np.nonzero(morphology.dilation(ground_truth > 0, morphology.disk(50))) else: x_pos, y_pos = np.nonzero(ground_truth) self.indices = [idx for idx in zip(x_pos, y_pos)]
def square_plot(data, path): if type(data) == list: data = np.concatentate(data) data = (data - data.min()) / (data.max() - data.min()) n = int(np.ceil(np.sqrt(data.shape[0]))) padding = (((0, n**2 - data.shape[0]), (0, 1), (0, 1)) + ((0, 0), ) * (data.ndim - 3)) data = np.pad(data, padding, mode='constant', constant_values=1) data = data.reshape((n, n) + data.shape[1:]).transpose( (0, 2, 1, 3) + tuple(range(4, data.ndim + 1))) data = data.reshape((n * data.shape[1], n * data.shape[3]) + data.shape[4:]) plt.imsave(path, data, cmap='gray')
def min_max_scale(data, center=False): min_ = data.min(axis=0) max_ = data.max(axis=0) denom = np.broadcast_to((max_ - min_), data.shape) scaled_data = np.divide((data - min_), denom, where=(~np.isclose(denom, 0))) if center: mean_ = scaled_data.mean(axis=0) scaled_data = scaled_data - mean_ else: mean_ = None return scaled_data, min_, max_, mean_
def bytescale(data, cmin=None, cmax=None, high=255, low=0): """ Byte scales an array (image). Byte scaling means converting the input image to uint8 dtype and scaling the range to ``(low, high)`` (default 0-255). If the input image already has dtype uint8, no scaling is done. This function is only available if Python Imaging Library (PIL) is installed. Parameters ---------- data : ndarray PIL image data array. cmin : scalar, optional Bias scaling of small values. Default is ``data.min()``. cmax : scalar, optional Bias scaling of large values. Default is ``data.max()``. high : scalar, optional Scale max value to `high`. Default is 255. low : scalar, optional Scale min value to `low`. Default is 0. Returns ------- img_array : uint8 ndarray The byte-scaled array. """ if data.dtype == np.uint8: return data if high > 255: raise ValueError("`high` should be less than or equal to 255.") if low < 0: raise ValueError("`low` should be greater than or equal to 0.") if high < low: raise ValueError("`high` should be greater than or equal to `low`.") if cmin is None: cmin = data.min() if cmax is None: cmax = data.max() cscale = cmax - cmin if cscale < 0: raise ValueError("`cmax` should be larger than `cmin`.") elif cscale == 0: cscale = 1 scale = float(high - low) / cscale bytedata = (data - cmin) * scale + low return (bytedata.clip(low, high) + 0.5).astype(np.uint8)
def transform_inputs(loader, batch_size=batch_size): encoded_inputs = [] labels = [] tq = tqdm(loader) with torch.no_grad(): for batch_idx, (data, label) in enumerate(tq): data = Variable(data.squeeze().transpose(0, 1)) data = (data - data.min().item()) / (data.max().item() - data.min().item()) h = model.predict(data) for i in range(h.shape[1]): encoded_inputs.append(h[:, i, :].flatten().numpy()) labels.append(label[i].item()) return torch.utils.data.DataLoader(torch.utils.data.TensorDataset( torch.Tensor(encoded_inputs), torch.Tensor(labels)), batch_size=batch_size, shuffle=True)
def hdf5_loader(path): # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835) # for hdf5 if(path.endswith('.hdf5')): with h5py.File(path, 'r') as f: data = f[HDF5_DATASET_NAME][:].astype(np.float32) # normalize it to 0 to 1 data /= (data.max() - data.min() + 0.0001) # normalize to -1 to 1 data *= 2 data -= 1 # the torrchvision.transforms.toTensor rescales input to the range -1 to 1 in certain conditions, # we want to scale -1 to 1 # so scale in the dataloader itself! # link: https://pytorch.org/docs/stable/torchvision/transforms.html return data
def visualize_gradients(viz, data, caption='', zoom=4): batchSize = data.size(0) rows = int(math.sqrt(batchSize)) toPIL = transforms.ToPILImage() # normalize it data = data.cpu() dmin = data.min() dmax = data.max() width = dmax - dmin if (width > 0.0): data = data.add(-dmin).div(width) data_imgs = utils.make_grid(data, nrow=rows) pimg = toPIL(data_imgs) pimg = pimg.resize((pimg.height * zoom, pimg.width * zoom), Image.NEAREST) imgarray = np.array(pimg) new_image = torch.from_numpy(imgarray) assert (new_image.dim() == 3)
def train(epoch): train_loss = 0 for batch_idx, data in enumerate(train_loader): #transforming data #data = Variable(data) #to remove eventually data = Variable(torch.unsqueeze(data['frame'], 1)).float().cuda() data = (data - data.min().item()) / (data.max().item() - data.min().item()) #forward + backward + optimize optimizer.zero_grad() kld_loss, nll_loss, _, _ = model(data) loss = kld_loss + nll_loss loss.backward() optimizer.step() #grad norm clipping, only in pytorch version >= 1.10 nn.utils.clip_grad_norm_(model.parameters(), clip) #sample = model.sample(batch_size, 14) #print('sample') #print(sample) #plt.imshow(sample.numpy()) #plt.pause(1e-6) #printing if batch_idx % print_every == 0: print( 'Train Epoch: {} [{}/{} ({:.0f}%)]\t KLD Loss: {:.6f} \t NLL Loss: {:.6f}' .format(epoch, batch_idx * len(data), len(train_loader.dataset), 100. * batch_idx / len(train_loader), kld_loss.item() / batch_size, nll_loss.item() / batch_size)) train_loss += loss.item() print('====> Epoch: {} Average loss: {:.4f}'.format( epoch, train_loss / len(train_loader.dataset))) return
def visualize_batch(viz, data, caption='', normalize=True, gammacorrect=False, window=None): if(gammacorrect): gamma = 2.20 data = data.pow(1.0/gamma) if(normalize == False): #data = data.mul(0.5).add(0.5).clamp(0, 1) data = data.clamp(0, 1) else: dmin = data.min() dmax = data.max() width = dmax - dmin if (width > 0.0): data = data.add(-dmin).div(width) #data_imgs = utils.make_grid(data).permute(1, 2, 0) data_imgs = utils.make_grid(data) #viz.showImage(data_imgs, caption, window=window)
def test(epoch): """uses test data to evaluate likelihood of the model""" mean_kld_loss, mean_nll_loss = 0, 0 for i, (data, _) in enumerate(test_loader): #data = Variable(data) data = Variable(data.squeeze().transpose(0, 1)) data = (data - data.min().item()) / (data.max().item() - data.min().item()) kld_loss, nll_loss, _, _ = model(data) mean_kld_loss += kld_loss.item() mean_nll_loss += nll_loss.item() mean_kld_loss /= len(test_loader.dataset) mean_nll_loss /= len(test_loader.dataset) print('====> Test set loss: KLD Loss = {:.4f}, NLL Loss = {:.4f} '.format( mean_kld_loss, mean_nll_loss))
def train(epoch): train_loss = 0 for batch_idx, (data, _) in enumerate(train_loader): #transforming data #data = Variable(data) #to remove eventually #data, _ = data.to(device, dtype=torch.float), _.to(device, dtype=torch.float) data = Variable(data.squeeze().transpose(0, 1)) data = (data - data.min().data.item()) / (data.max().data.item() - data.min().data.item()) #data = data.to(device) #forward + backward + optimize optimizer.zero_grad() kld_loss, nll_loss, _, _ = model(data) loss = kld_loss + nll_loss loss.backward() optimizer.step() #grad norm clipping, only in pytorch version >= 1.10 nn.utils.clip_grad_norm(model.parameters(), clip) #printing if batch_idx % print_every == 0: print( 'Train Epoch: {} [{}/{} ({:.0f}%)]\t KLD Loss: {:.6f} \t NLL Loss: {:.6f}' .format(epoch, batch_idx * len(data), len(train_loader.dataset), 100. * batch_idx / len(train_loader), kld_loss.data.item() / batch_size, nll_loss.data.item() / batch_size)) sample = model.sample(28) plt.imshow(sample.numpy()) plt.pause(1e-6) train_loss += loss.data.item() print('====> Epoch: {} Average loss: {:.4f}'.format( epoch, train_loss / len(train_loader.dataset)))
def train(epoch): train_loss = 0 tq = tqdm(train_loader) for batch_idx, (data, _) in enumerate(tq): data = Variable(data.squeeze().transpose(0, 1)) data = (data - data.min().item()) / (data.max().item() - data.min().item()) #forward + backward + optimize optimizer.zero_grad() kld_loss, nll_loss, _, _ = model(data) loss = kld_loss + nll_loss loss.backward() optimizer.step() #grad norm clipping, only in pytorch version >= 1.10 nn.utils.clip_grad_norm(model.parameters(), clip) tq.set_postfix(kld_loss=(kld_loss.item() / batch_size), nll_loss=(nll_loss.item() / batch_size)) train_loss += loss.item() return
def visualize_experiment_dataset(is_spiral=False, modes=1, param_modes=[(0, 1)]): data = experiment_data(is_spiral=is_spiral, n_modes=modes, params=param_modes) plt.figure(figsize=(20, 20)) plt.rc("text", usetex=True) plt.title("Initial Distribution", fontsize=45) if is_spiral: plt.scatter(data[:, 0], data[:, 1], label='train spiral data') else: data_grid = np.linspace(data.min(), data.max(), 1000) bandwidth = 0.1 data_p_n = kde(data, data_grid, bandwidth=bandwidth) plt.fill_between(x=data_grid, y1=data_p_n, y2=0, alpha=0.7) plt.xticks(fontsize=40) plt.yticks(fontsize=40) plt.grid() savefig("results/intial_distr.pdf") plt.show()
def dataset_to_list(dataset: torch.utils.data.Dataset, label_only: bool = False, force: bool = True) -> tuple[list, list[int]]: if not force: if label_only and 'targets' in dataset.__dict__.keys(): return None, list(dataset.targets) if 'data' in dataset.__dict__.keys() and 'targets' in dataset.__dict__.keys(): data = dataset.data if isinstance(data, np.ndarray): data = torch.as_tensor(data) if isinstance(data, torch.Tensor): if data.max() > 2: data = data.to(dtype=torch.float) / 255 data = [img for img in data] return data, list(dataset.targets) data, targets = list(zip(*dataset))[:2] if label_only: data = None else: data = list(data) targets = list(targets) return data, targets
def __init__(self, k, root, train=True, transform=None, target_transform=None, download=False): if train: transform = torchvision.transforms.Compose([ transforms.RandomApply([transforms.RandomAffine( degrees=10.0, translate=(0.1, 0.1), scale=(0.8, 1.2), shear=5.0, resample=False )], p=0.5), # will scale [0, 1] transforms.ToTensor(), # dequantize transforms.Lambda(lambda im: ((255.0 * im + torch.rand_like(im)) / 256.0).clamp(1e-3, 1 - 1e-3)), ]) else: # will scale [0, 1] transform = transforms.ToTensor() super().__init__( root=root, train=train, transform=transform, target_transform=target_transform, download=download, ) self.k = k if self.train: data = self.train_data else: data = self.test_data X = data.type(torch.float32).view((data.shape[0], -1)) / data.max() self.pca = PCA(n_components=k) self.pca.fit(X) print("k = {k} captures {var:.3f} variance of dataset".format( k=self.k, var=self.pca.explained_variance_ratio_.sum()))
def vis_square(data, name_fig): '''Take an array of shape (n, height, width) or (n, height, width, 3) #and visualize each (height, width) thing in a grid of size approx. sqrt(n) by sqrt(n)''' # normalize data for display data = (data - data.min()) / (data.max() - data.min()) # force the number of filters to be square n = int(np.ceil(np.sqrt(data.shape[0]))) padding = (((0, n ** 2 - data.shape[0]), (0, 1), (0, 1)) # add some space between filters + ((0, 0),) * (data.ndim - 3)) # don't pad the last dimension (if there is one) data = np.pad(data, padding, mode='constant', constant_values=1) # pad with ones (white) # tile the filters into an image data = data.reshape((n, n) + data.shape[1:]).transpose((0, 2, 1, 3) + tuple(range(4, data.ndim + 1))) data = data.reshape((n * data.shape[1], n * data.shape[3]) + data.shape[4:]) plt.imshow(data) plt.show() plt.imsave(name_fig,data) return data
def run_fid(data, sample): assert data.max() <=1 and data.min() >= 0 assert sample.max() <=1 and sample.min() >= 0 data = 2*data - 1 if data.shape[1] == 1: data = data.repeat(1,3,1,1) data = data.detach() with torch.no_grad(): iss, _, _, acts_real = inception_score(data, cuda=True, batch_size=32, resize=True, splits=10, return_preds=True) sample = 2*sample - 1 if sample.shape[1] == 1: sample = sample.repeat(1,3,1,1) sample = sample.detach() with torch.no_grad(): issf, _, _, acts_fake = inception_score(sample, cuda=True, batch_size=32, resize=True, splits=10, return_preds=True) # idxs_ = np.argsort(np.abs(acts_fake).sum(-1))[:1800] # filter the ones with super large values # acts_fake = acts_fake[idxs_] m1, s1 = calculate_activation_statistics(acts_real) m2, s2 = calculate_activation_statistics(acts_fake) fid_value = calculate_frechet_distance(m1, s1, m2, s2) return fid_value
def get_dataloader(dataset, batch_size=128, window=12, horizon=1, val_days=10, test_days=10, normalizer='max'): if dataset == 'SYDNEY': data = Load_Sydney_Demand_Data( os.path.join(base_dir, '1h_data_new3.csv')) print(data.shape) print('Load Sydney Dataset Successfully!') if normalizer == 'max': scaler = MinMaxScaler(data.min(), data.max()) data = scaler.transform(data) print('Normalize the dataset by MinMax Normalization') elif normalizer == 'std': scaler = StandardScaler(data.mean(), data.std()) data = scaler.transform(data) print('Normalize the dataset by Standard Normalization') else: scaler = None X, Y = Add_Window_Horizon(data, window, horizon) print(X.shape, Y.shape) x_tra, x_val, x_test = split_train_val_test(X, val_days, test_days) y_tra, y_val, y_test = split_train_val_test(Y, val_days, test_days) print(x_tra.shape, y_tra.shape) print(x_val.shape, y_val.shape) print(x_test.shape, y_test.shape) train_dataloader = data_loader(x_tra, y_tra, batch_size, 'train') val_dataloader = data_loader(x_val, y_val, batch_size, 'val') test_dataloader = data_loader(x_test, y_test, batch_size, 'test') dataloader = data_loader(X, Y, batch_size, 'all') return train_dataloader, val_dataloader, test_dataloader, scaler