def backward(ctx, grad_output): dev = grad_output.device if ctx.batched: # Batched Gradient vols, tfs, lfs = ctx.saved_tensors # Volume Grad Shape (BS, W, H, D) volume_grad = torch.zeros(ctx.bs, *ctx.vr.volume.shape, dtype=torch.float32, device=dev) # TF Grad Shape (BS, W, C) tf_grad = torch.zeros(ctx.bs, *ctx.vr.tf_tex.shape, ctx.vr.tf_tex.n, dtype=torch.float32, device=dev) for i, vol, tf, lf in zip(range(ctx.bs), vols, tfs, lfs): ctx.vr.clear_grad() ctx.vr.set_cam_pos(lf) ctx.vr.set_volume(vol) ctx.vr.set_tf_tex(tf) ctx.vr.clear_framebuffer() ctx.vr.compute_entry_exit(ctx.sampling_rate, ctx.jitter) ctx.vr.raycast(ctx.sampling_rate) ctx.vr.get_final_image() ctx.vr.output_rgba.grad.from_torch(grad_output[i]) ctx.vr.get_final_image.grad() ctx.vr.raycast.grad(ctx.sampling_rate) volume_grad[i] = torch.nan_to_num(ctx.vr.volume.grad.to_torch(device=dev)) tf_grad[i] = torch.nan_to_num(ctx.vr.tf_tex.grad.to_torch(device=dev)) return None, volume_grad, tf_grad, None, None, None, None else: # Non-batched, single item ctx.vr.clear_grad() ctx.vr.output_rgba.grad.from_torch(grad_output) ctx.vr.get_final_image.grad() ctx.vr.raycast.grad(ctx.sampling_rate) return None, \ torch.nan_to_num(ctx.vr.volume.grad.to_torch(device=dev)), \ torch.nan_to_num(ctx.vr.tf_tex.grad.to_torch(device=dev)), \ None, None, None, None
def decode(self, z): x_dcg = torch.nan_to_num(self.dec_dcg1(z), self.eps) x_ecg = torch.nan_to_num(self.dec_ecg1(z), self.eps) x_dcg = self.actv(x_dcg) x_ecg = self.actv(x_ecg) x_dcg = self.bn_dec_dcg1(x_dcg) x_ecg = self.bn_dec_ecg1(x_ecg) x_dcg = torch.nan_to_num(self.dec_dcg2(x_dcg), self.eps) x_ecg = torch.nan_to_num(self.dec_ecg2(x_ecg), self.eps) x_dcg = self.actv(x_dcg) x_ecg = self.actv(x_ecg) x_dcg = self.bn_dec_dcg2(x_dcg) x_ecg = self.bn_dec_ecg2(x_ecg) x_dcg = torch.nan_to_num(self.dec_dcg3(x_dcg), self.eps) x_ecg = torch.nan_to_num(self.dec_ecg3(x_ecg), self.eps) x_dcg = self.actv(x_dcg) x_ecg = self.actv(x_ecg) x_dcg = self.bn_dec_dcg3(x_dcg) x_ecg = self.bn_dec_ecg3(x_ecg) x_dcg = torch.nan_to_num(self.dec_dcg4(x_dcg), self.eps) x_ecg = torch.nan_to_num(self.dec_ecg4(x_ecg), self.eps) x_dcg = self.actv(x_dcg) x_ecg = self.actv(x_ecg) x_dcg = self.bn_dec_dcg4(x_dcg) x_ecg = self.bn_dec_ecg4(x_ecg) x_dcg = torch.nan_to_num(self.dec_dcg5(x_dcg), self.eps) x_ecg = torch.nan_to_num(self.dec_ecg5(x_ecg), self.eps) re_dcg = torch.sigmoid(x_dcg) re_ecg = torch.sigmoid(x_ecg) return re_dcg, re_ecg
def __init__(self, a, b, validate_args=None): self.a, self.b = broadcast_all(a, b) if isinstance(a, Number) and isinstance(b, Number): batch_shape = torch.Size() else: batch_shape = self.a.size() super(TruncatedStandardNormal, self).__init__( batch_shape, validate_args=validate_args) if self.a.dtype != self.b.dtype: raise ValueError('Truncation bounds types are different') if any((self.a >= self.b).view(-1, ).tolist()): raise ValueError('Incorrect truncation range') eps = torch.finfo(self.a.dtype).eps self._dtype_min_gt_0 = eps self._dtype_max_lt_1 = 1 - eps self._little_phi_a = self._little_phi(self.a) self._little_phi_b = self._little_phi(self.b) self._big_phi_a = self._big_phi(self.a) self._big_phi_b = self._big_phi(self.b) self._Z = (self._big_phi_b - self._big_phi_a).clamp_min(eps) self._log_Z = self._Z.log() little_phi_coeff_a = torch.nan_to_num(self.a, nan=math.nan) little_phi_coeff_b = torch.nan_to_num(self.b, nan=math.nan) self._lpbb_m_lpaa_d_Z = (self._little_phi_b * little_phi_coeff_b - self._little_phi_a * little_phi_coeff_a) / self._Z self._mean = -(self._little_phi_b - self._little_phi_a) / self._Z self._variance = 1 - self._lpbb_m_lpaa_d_Z - \ ((self._little_phi_b - self._little_phi_a) / self._Z) ** 2 self._entropy = CONST_LOG_SQRT_2PI_E + self._log_Z - 0.5 * self._lpbb_m_lpaa_d_Z
def test_nan_to_num(self): mode = FakeTensorMode(inner=None) with enable_torch_dispatch_mode(mode): for dtype in [torch.float16, torch.float32]: x = torch.rand([4], dtype=dtype) y = torch.nan_to_num(x, nan=None) z = torch.nan_to_num(x, 0.0) self.assertEqual(dtype, y.dtype) self.assertEqual(dtype, z.dtype)
def test_sharded_tensor_nan_to_num(self): specs = _chunk_sharding_specs_list_for_test([0, 1], seed=10) for spec in specs: tensor = torch.rand(16, 12).cuda(self.rank) tensor[:, :2] = float('nan') tensor[:, 4:5] = float('inf') tensor[:, 10:] = -float('inf') st = _shard_tensor(tensor, spec) st_expected = _shard_tensor(torch.nan_to_num(tensor), spec) st = torch.nan_to_num(st) self.assertTrue(torch.allclose(st, st_expected))
def compute(self): if self.another_macro_f1: macro_prec = torch.mean( torch.nan_to_num(self.tp_sum / self.preds_sum, posinf=0.)) macro_recall = torch.mean( torch.nan_to_num(self.tp_sum / self.target_sum, posinf=0.)) return 2 * (macro_prec * macro_recall) / (macro_prec + macro_recall + 1e-10) else: label_f1 = 2 * self.tp_sum / (self.preds_sum + self.target_sum + 1e-10) return torch.mean(label_f1)
def encode(self, x): x = torch.nan_to_num(self.enc1(x), nan=self.eps) x = self.actv(x) x = self.bn_enc1(x) x = torch.nan_to_num(self.enc2(x), nan=self.eps) x = self.actv(x) x = self.bn_enc2(x) x = torch.nan_to_num(self.enc3(x), nan=self.eps) x = self.actv(x) x = self.bn_enc3(x) mu = self.fc_mu(x) logvar = self.fc_logvar(x) return mu, logvar
def read_exr(filename): """ Read color data from EXR image file. Set negative values and nans to 0. :param filename: string describing file path :return: RGB image in float32 format (with 1xCxHxW layout) """ exrfile = exr.InputFile(filename) header = exrfile.header() dw = header['dataWindow'] isize = (dw.max.y - dw.min.y + 1, dw.max.x - dw.min.x + 1) channelData = dict() # Convert all channels in the image to numpy arrays for c in header['channels']: if c != 'A': C = exrfile.channel(c, Imath.PixelType(Imath.PixelType.FLOAT)) C = np.frombuffer(C, dtype=np.float32) C = np.reshape(C, isize) channelData[c] = C colorChannels = ['R', 'G', 'B'] img_np = np.concatenate( [channelData[c][..., np.newaxis] for c in colorChannels], axis=2) img_torch = torch.max(torch.nan_to_num(torch.from_numpy( np.array(img_np))), torch.tensor( [0.0])).cuda() # added maximum to avoid negative values in images return img_torch.permute(2, 0, 1).unsqueeze(0)
def visualise_posterior(model, X_test, Y_test, Y_test_pred, mixture=True, title=None, new_fig=False): ''' Visualising posterior predictive (a single distribution) ''' if new_fig: plt.figure(figsize=(5,5)) #f, ax = plt.subplots(1, 1, figsize=(8, 8)) if mixture: sample_means, sample_stds = get_posterior_predictive_means_stds(Y_test_pred) sample_stds = torch.nan_to_num(sample_stds, nan=1e-5) lower, upper = get_posterior_predictive_uncertainty_intervals(sample_means, sample_stds) pred_mean = sample_means.mean(dim=0) else: lower, upper = Y_test_pred.confidence_region() lower=lower.detach().numpy() upper=upper.detach().numpy() pred_mean = Y_test_pred.mean.numpy() plt.plot(X_test.numpy(), pred_mean, 'b-', label='Mean') plt.scatter(model.covar_module.inducing_points.detach(), [-2.5]*model.num_inducing, c='r', marker='x', label='Inducing') plt.fill_between(X_test.detach().numpy(), lower, upper, alpha=0.5, label=r'$\pm$2\sigma', color='g') plt.scatter(model.train_x.numpy(), model.train_y.numpy(), c='k', marker='x', alpha=0.7, label='Train') plt.plot(X_test, Y_test, color='b', linestyle='dashed', alpha=0.7, labeL='True') #ax.set_ylim([-3, 3]) plt.legend(fontsize='small') plt.title(title)
def forward(self, x): """ Forward pass. """ window = self.window x = expand_dims(x) padding = [(w // 2, w - w // 2 - 1) for w in window] num = F.pad(x, (0, 0, *padding[1], *padding[0], 0, 0, 0, 0)) num = F.conv3d(num, self.kernels[0])**2 num = F.pad(num, (*padding[2], 0, 0, 0, 0, 0, 0, 0, 0)) num = F.conv3d(num, self.kernels[1]) denum = F.pad(x, (*padding[2], *padding[1], *padding[0], 0, 0, 0, 0)) denum = F.conv3d(denum**2, self.kernels[2]) normilizing = torch.ones(x.shape[:-1], dtype=x.dtype, requires_grad=False).to(x.device) normilizing = F.pad(normilizing, (*padding[1], *padding[0], 0, 0, 0, 0)) normilizing = F.conv2d(normilizing, self.kernels[3]) denum *= normilizing.view(*normilizing.shape, 1) result = torch.nan_to_num(num / denum, nan=self.fill_value) return squueze(result, self.ndim)
def local_norm(img, kernel_size=31, cutoff_percent=80): ''' Performs local normalization of a given image relative to the neighborhood of size (kernel_size, kernel_size) using a global cutoffself. # Arguments: - img: tensor with the image of shape (height, width) - kernel_size: size of the averaging kernel or tuple of (kernel_height, kernel_width). Should be odd ints. - cutoff_percentile: int between 0 and 100. Global percentile cut-off, preventing over-amplification of noise. # Returns: - norm_img: image of the same size as the input img, with values locally normalized. ''' kernel_size = hp.misc.ensure_list(kernel_size) if len(kernel_size) == 1: kernel_size = (kernel_size[0], kernel_size[0]) norm = ttf.gaussian_blur(img.unsqueeze(0), kernel_size, [k/3 for k in kernel_size]).squeeze(0) cutoff = torch.max(norm) * np.power(cutoff_percent/100, 3) norm_img = img / torch.maximum(norm, cutoff) norm_img = torch.nan_to_num(norm_img) img = (img-torch.min(img))/(torch.max(img)-torch.min(img)) norm_img = (norm_img-torch.min(norm_img))/(torch.max(norm_img)-torch.min(norm_img)) return (img+norm_img)/2
def forward(self, v: TorchscriptPreprocessingInput) -> torch.Tensor: if torch.jit.isinstance(v, List[torch.Tensor]): v = torch.stack(v) if not torch.jit.isinstance(v, torch.Tensor): raise ValueError(f"Unsupported input: {v}") v = torch.nan_to_num(v, nan=self.computed_fill_value) v = v.long() outputs: List[torch.Tensor] = [] for v_i in v: components = h3_to_components(v_i) header: List[int] = [ components.mode, components.edge, components.resolution, components.base_cell, ] cells_padding: List[int] = [self.h3_padding_value] * ( self.max_h3_resolution - len(components.cells)) output = torch.tensor(header + components.cells + cells_padding, dtype=torch.uint8, device=v.device) outputs.append(output) return torch.stack(outputs)
def _calc_loss(self, samples: Dict[str, torch.Tensor], clip_range: float) -> torch.Tensor: """## PPO Loss""" # Sampled observations are fed into the model to get $\pi_\theta(a_t|s_t)$ and $V^{\pi_\theta}(s_t)$; pi_val, value = self.model(samples['obs'], False) pi = Categorical(logits=pi_val) # #### Policy log_pi = pi.log_prob(samples['actions']) # *this is different from rewards* $r_t$. ratio = torch.exp(log_pi - samples['log_pis']) # The ratio is clipped to be close to 1. # Using the normalized advantage # $\bar{A_t} = \frac{\hat{A_t} - \mu(\hat{A_t})}{\sigma(\hat{A_t})}$ # introduces a bias to the policy gradient estimator, # but it reduces variance a lot. clipped_ratio = ratio.clamp(min=1.0 - clip_range, max=1.0 + clip_range) # advantages are normalized policy_reward = torch.min(ratio * samples['advantages'], clipped_ratio * samples['advantages']) policy_reward = policy_reward.mean() # #### Entropy Bonus entropy_bonus = pi.entropy() entropy_bonus = entropy_bonus.mean() # add regularization to logits to revive previously-seen impossible moves max_logit = pi_val.max().item() prob_reg = -(torch.nan_to_num(pi_val - max_logit, neginf=0)** 2).mean() * self.cur_prob_reg_weight # #### Value # Clipping makes sure the value function $V_\theta$ doesn't deviate # significantly from $V_{\theta_{OLD}}$. clipped_value = samples['values'][:, :2] + ( value[:, :2] - samples['values'][:, :2]).clamp(min=-clip_range, max=clip_range) vf_loss = torch.max((value[:, :2] - samples['returns'])**2, (clipped_value - samples['returns'])**2) vf_loss[:, 1] *= self.cur_target_prob_weight vf_loss_score = 0.5 * vf_loss[:, 0].mean() vf_loss = 0.5 * vf_loss.sum(-1).mean() # we want to maximize $\mathcal{L}^{CLIP+VF+EB}(\theta)$ # so we take the negative of it as the loss loss = -(policy_reward - self.c.vf_weight * vf_loss + \ self.cur_entropy_weight * (entropy_bonus + prob_reg)) # for monitoring approx_kl_divergence = .5 * ((samples['log_pis'] - log_pi)**2).mean() clip_fraction = (abs( (ratio - 1.0)) > clip_range).to(torch.float).mean() tracker.add({ 'policy_reward': policy_reward, 'vf_loss': vf_loss**0.5, 'vf_loss_1': vf_loss_score**0.5, 'entropy_bonus': entropy_bonus, 'kl_div': approx_kl_divergence, 'clip_fraction': clip_fraction }) return loss
def calculate_equalization_scale(input_obs: _InputEqualizationObserver, weight_obs: _WeightEqualizationObserver) -> torch.Tensor: r""" Calculates the equalization scale and sets the equalization_scale value in the observers. Args: input_obs: Observer that tracks the ranges for the input columns weight_obs: Observer that tracks the ranges for the weight columns """ (min_inputs, max_inputs) = input_obs.get_input_minmax() (min_weights, max_weights) = weight_obs.get_weight_col_minmax() if not (check_min_max_valid(min_inputs, max_inputs) and check_min_max_valid(min_weights, max_weights)): warnings.warn( "Must run observer before calling calculate_equalization_scale. " + "Returning default equalization scale torch.tensor(1)." ) return torch.tensor(1) if not (min_inputs.shape == min_weights.shape): raise ValueError( "Input and Weight must have the same column dimension. " + f"Found {min_inputs.shape} and {min_weights.shape} shapes instead." ) equalization_scale = torch.sqrt((max_weights - min_weights) / (max_inputs - min_inputs)) # Replace all 'inf', 'nan', 0's with 1s to prevent errors equalization_scale[equalization_scale == 0.] = 1 equalization_scale = torch.nan_to_num(equalization_scale, nan=1, posinf=1, neginf=1) return equalization_scale
def forward(self, input_im, target_im, mask, num_ch): if num_ch > 1: mask = torch.cat([mask for _ in range(num_ch)], dim=1) neg_mask = torch.logical_not(mask) diff = torch.abs(input_im - target_im) diff_in = diff[mask] loss_in = torch.sum(diff_in) / torch.numel(diff) loss_in = torch.nan_to_num(loss_in) diff_out = diff[neg_mask] loss_out = torch.sum(diff_out) / torch.numel(diff) loss_out = torch.nan_to_num(loss_out) return loss_in, loss_out
def _ce_compute(confidences: Tensor, accuracies: Tensor, bin_boundaries: Tensor, norm: str = "l1", debias: bool = True) -> Tensor: conf_bin = torch.zeros_like(bin_boundaries) acc_bin = torch.zeros_like(bin_boundaries) prop_bin = torch.zeros_like(bin_boundaries) for i, (bin_lower, bin_upper) in enumerate(zip(bin_boundaries[:-1], bin_boundaries[1:])): # Calculated confidence and accuracy in each bin in_bin = confidences.gt(bin_lower.item()) * confidences.le(bin_upper.item()) prop_in_bin = in_bin.float().mean() if prop_in_bin.item() > 0: acc_bin[i] = accuracies[in_bin].float().mean() conf_bin[i] = confidences[in_bin].mean() prop_bin[i] = prop_in_bin if norm == "l1": ce = torch.sum(torch.abs(acc_bin - conf_bin) * prop_bin) print(ce) elif norm == "max": ce = torch.max(torch.abs(acc_bin - conf_bin)) elif norm == "l2": ce = torch.sum(torch.pow(acc_bin - conf_bin, 2) * prop_bin) if debias: # the order here (acc_bin - 1 ) vs (1 - acc_bin) is flipped from # the equation in Verified Uncertainty Prediction (Kumar et al 2019)/ debias_bins = (acc_bin * (acc_bin - 1) * prop_bin) / (prop_bin * accuracies.size()[0] - 1) ce += torch.sum(torch.nan_to_num(debias_bins)) # replace nans with zeros if nothing appeared in a bin ce = torch.sqrt(ce) if ce > 0 else torch.tensor(0) else: raise ValueError(f"Norm {norm} is not supported. Please select from l1, l2, or max. ") return ce
def make_base(tt, w, tau, model_coh: bool = False) -> torch.Tensor: """ Calculates the basis for the linear least squares problem Parameters ---------- tt : ndarry 2D-time points w : float System response tau : ndarray 1D decay times model_coh : bool If true, appends a gaussian and its first two dervitates to model coherent behavior at time-zero. Returns ------- [type] [description] """ k = 1 / (tau[None, None, ...]) t = (tt)[..., None] scaled_tt = tt / w if False: A = torch.exp(-k * tt) else: nw = w[:, None] A = 0.5 * torch.erfc(-t / nw + nw * k / (2.0)) A *= torch.exp(k * (nw * nw * k / (4.0) - t)) if model_coh: exp_half = torch.exp(0.5) scaled_tt = tt / w coh = torch.exp(-0.5 * scaled_tt * scaled_tt) coh = coh[:, :, None].repeat((1, 1, 3)) coh[..., 1] *= (-scaled_tt * exp_half) coh[..., 2] *= (scaled_tt - 1) A = torch.cat((A, coh), dim=-1) #if torch.isnan(A).any(): # print(A) torch.nan_to_num(A, out=A) return A
def _calc_norm(self, f, a, b, avg=False): """ Calculates the normalising term, Z. """ # if is allowed to be 0 inftest1 = (a==0).any() inftest2 = (b==0).any() if inftest1: print(f"a is 0; avg = {avg}") if inftest2: print(f"b is 0; avg = {avg}") # if any of a is zero, then add EPS i_0 = 1/a[...,0] * t.exp(a[...,0] * f[...,0] + b[...,0]) i_1 = 1/a[...,1:-1] * t.exp(b[...,1:-1]) * ( t.exp(a[...,1:-1] * f[...,1:]) - t.exp(a[...,1:-1] * f[...,:-1]) ) i_2 = 1/a[...,-1] * t.exp(a[...,-1] * f[...,-1] + b[...,-1]) inftest0 = t.isinf(i_0).any() inftest1 = t.isinf(i_1).any() inftest2 = t.isinf(i_2).any() if inftest0: print(f"i0 is inf; avg = {avg}") i_0 = t.nan_to_num(i_0) if inftest1: print(f"i1 is inf; avg = {avg}") i_1 = t.nan_to_num(i_1) if inftest2: print(f"i2 is inf; avg = {avg}") i_2 = t.nan_to_num(i_2) if avg: self.i_0_avg = i_0 self.i_1_avg = i_1 self.i_2_avg = i_2 self.Z_avg = i_0 + i_1.sum(-1) - i_2 else: self.i_0 = i_0 self.i_1 = i_1 self.i_2 = i_2 self.Z = i_0 + i_1.sum(-1) - i_2 nantest = t.isnan(self.Z).any() if nantest: print("Z has inf")
def forward(self, v: TorchscriptPreprocessingInput) -> torch.Tensor: if not torch.jit.isinstance(v, torch.Tensor): raise ValueError(f"Unsupported input: {v}") v = torch.nan_to_num(v, nan=self.computed_fill_value) v = v.to(dtype=torch.float32) return self.numeric_transformer.transform_inference(v)
def compute(self): target = torch.cat(self.target, dim=0).cpu().long().numpy() preds = torch.nan_to_num(torch.cat(self.preds, dim=0).float().cpu()).numpy() if len(preds.shape) > 1: preds = 1 - preds[:, 0] target = (target != 0).astype(int) if len(np.unique(target)) == 1: return torch.tensor(0, device=self.preds[0].device) return torch.tensor(roc_auc_score(target, preds), device=self.preds[0].device)
def create_stft(self, sample): # Create spectrogram spec = self.amp_to_db( self.spectrogram(sample.clone().detach().requires_grad_(True))) for i in range(spec.shape[0]): spec[i] -= spec[i].min(1, keepdim=True)[0] spec[i] /= spec[i].max(1, keepdim=True)[0] spec = torch.nan_to_num(spec, nan=0.0) return spec
def f1(x): x = torch.DoubleTensor(x).cuda() x[:, 0] = torch.abs(x[:, 0]) i1 = 1/(np.pi*self.sigma0**2) * torch.exp(-torch.norm(x, p=2, dim=1)**2 / (2*self.sigma0**2)) g = self.compute_set(x, eps, k1, log_k2, return_mean=False) r = x[:, 0] * norm1 integral = i1 * (r * np.exp(1/(dim-2) * log_k) * (g * 1.))**(dim-2) integral = torch.nan_to_num(integral) return np.double(integral.detach().cpu().numpy())
def update(self, preds, target): assert preds.shape == target.shape binary_topk_preds = select_topk(preds, self.top_k) target = target.to(dtype=torch.int) num_relevant = torch.sum(binary_topk_preds & target, dim=-1) top_ks = torch.tensor([self.top_k] * preds.shape[0]).to(preds.device) self.score += torch.nan_to_num(num_relevant / torch.min(top_ks, target.sum(dim=-1)), posinf=0.).sum() self.num_sample += len(preds)
def _format(self, x): x[x == 0] = np.nan x2 = self.normalize(x, 2) x3 = self.normalize(x, 3) #x2[xbl] = 0 #x3[xbl] = 0 x = torch.cat((x2, x3), dim=1) x = torch.nan_to_num(x, nan=0) return x
def forward(self): # compare inplace if self.op == "nan_to_num": if self.replace_inf: output = torch.nan_to_num(self.input, nan=1.0) else: output = torch.nan_to_num(self.input, nan=1.0, posinf=math.inf, neginf=-math.inf) else: if self.replace_inf: output = torch.nan_to_num_(self.input, nan=1.0) else: output = torch.nan_to_num_(self.input, nan=1.0, posinf=math.inf, neginf=-math.inf) return output
def __call__(self, image): if torch.isnan(image).any().item(): if not self.nan_detected: logger.warning( "NaN values were found in your images and will be removed." ) self.nan_detected = True return torch.nan_to_num(image) else: return image
def score_goals(self, sampled_ags, info): # sampled_ags is np.array of shape NUM_ENVS x NUM_SAMPLED_GOALS (both arbitrary) num_envs, num_sampled_ags = sampled_ags.shape[:2] scores = self.success_predictor(info.states).reshape(num_envs, num_sampled_ags) # these are predicted success % scores = self.torch(scores) # entropy to achieve goals entropy = -scores * torch.log(scores) -(1-scores)*torch.log(1-scores) return torch.nan_to_num(entropy)
def read_parent_by_id(id: int, expr_root: str = None, args = None) -> dict: """ based on the id, find the directory of the parent and parse its relevant data for wt inheritance. Args: id: one of the parent ids saved on the individual after traced mating expr_root: this should match args.save_path Returns: A dictionary describing the parent """ if main_config.distributed_cloud: task = get_task_by_network_id(id, run_code=args.code) if task is None: logger.warning("Could not find expected parent id") spec = pickle_load_from_str(task.pkl_data) results = pickle_load_from_str(task.result_pkl_data) genome = micro_encoding.decode(spec["genome"]) flops = results["flops"] acc = results["valid_acc"] wt_path = download_blob(results["wt_blob_name"]) else: # parent is a dict arch_path = os.path.join(expr_root, "arch_{}".format(id)) path = os.path.join(arch_path, "log.txt") wt_path = os.path.join(arch_path, "weights.pt") with open(path, "r") as f: s = f.read() genome = eval(s.split("\n")[1][15:]) flops = float(s.split("\n")[3][8:-2]) acc = float(s.split("\n")[4][11:]) error = 100 - acc weights = torch.load(wt_path) has_nan = False for k, v in weights.items(): if torch.any(torch.isnan(v.cpu())): has_nan = True weights[k] = torch.nan_to_num(v.cpu()) if has_nan: logger.warning("Nan values detected when trying to load weights for parent with id %i" % id) parent = {"genome": genome, "flops": flops, "error": error, "weights": weights} return parent
def Hb(p: Tensor): """ Binary entropy. Args: p: Tensor of probabilities. Returns: Binary entropy for each probability. """ epsilon = np.finfo(float).eps p = torch.clamp(p, min=epsilon, max=1 - epsilon) return -torch.nan_to_num(p * torch.log2(p) + (1 - p) * torch.log2(1 - p))
def test_nan_to_num(self, device, dtype): for contiguous in [False, True]: x = make_tensor((64, 64), low=0., high=100., dtype=dtype, device=device) if dtype.is_floating_point: # Add extremal values. extremals = [float('nan'), float('inf'), -float('inf')] for idx, extremal in zip(torch.randint(0, 63, (3,)), extremals): x[idx, :] = extremal if not contiguous: x = x.T # With args nan = random.random() posinf = random.random() * 5 neginf = random.random() * 10 self.compare_with_numpy(lambda x: x.nan_to_num(nan=nan, posinf=posinf), lambda x: np.nan_to_num(x, nan=nan, posinf=posinf), x) self.compare_with_numpy(lambda x: x.nan_to_num(posinf=posinf, neginf=neginf), lambda x: np.nan_to_num(x, posinf=posinf, neginf=neginf), x) # Out Variant out = torch.empty_like(x) result = torch.nan_to_num(x) torch.nan_to_num(x, out=out) self.assertEqual(result, out) result = torch.nan_to_num(x, nan=nan, posinf=posinf, neginf=neginf) torch.nan_to_num(x, out=out, nan=nan, posinf=posinf, neginf=neginf) self.assertEqual(result, out)