def creluBoxy(self): if self.errors is None: if self.beta is None: return self.new(F.relu(self.head), None, None) er = self.beta mx = F.relu(self.head + er) mn = F.relu(self.head - er) return self.new((mn + mx) / 2, (mx - mn) / 2, None) aber = torch.abs(self.errors) sm = torch.sum(aber, 0) if not self.beta is None: sm += self.beta mx = self.head + sm mn = self.head - sm should_box = mn.lt(0) * mx.gt(0) gtz = self.head.gt(0).float() mx /= 2 newhead = h.ifThenElse(should_box, mx, gtz * self.head) newbeta = h.ifThenElse( should_box, mx, gtz * (self.beta if not self.beta is None else 0)) newerr = (1 - should_box.float()) * gtz * self.errors return self.new(newhead, newbeta, newerr)
def creluSwitch(dom): if dom.errors is None: if dom.beta is None: return dom.new(F.relu(dom.head), None, None) er = dom.beta mx = F.relu(dom.head + er) mn = F.relu(dom.head - er) return dom.new((mn + mx) / 2, (mx - mn) / 2 , None) aber = torch.abs(dom.errors) sm = torch.sum(aber, 0) if not dom.beta is None: sm += dom.beta mn = dom.head - sm mx = sm mx += dom.head should_box = mn.lt(0) * mx.gt(0) gtz = dom.head.gt(0) mn.neg_() should_boxer = mn.gt(mx) mn /= 2 newhead = h.ifThenElse(should_box, h.ifThenElse(should_boxer, mx / 2, dom.head + mn ), gtz.float() * dom.head) zbet = dom.beta if not dom.beta is None else 0 newbeta = h.ifThenElse(should_box, h.ifThenElse(should_boxer, mx / 2, mn + zbet), gtz.float() * zbet) newerr = h.ifThenElseL(should_box, 1 - should_boxer, gtz).float() * dom.errors return dom.new(newhead, newbeta , newerr)
def creluBoxySound(dom): if dom.errors is None: if dom.beta is None: return dom.new(F.relu(dom.head), None, None) er = dom.beta mx = F.relu(dom.head + er) mn = F.relu(dom.head - er) return dom.new((mn + mx) / 2, (mx - mn) / 2 + 2e-6, None) aber = torch.abs(dom.errors) sm = torch.sum(aber, 0) if not dom.beta is None: sm += dom.beta mx = dom.head + sm mn = dom.head - sm should_box = mn.lt(0) * mx.gt(0) gtz = dom.head.gt(0).to_dtype() mx /= 2 newhead = h.ifThenElse(should_box, mx, gtz * dom.head) newbeta = h.ifThenElse(should_box, mx + 2e-6, gtz * (dom.beta if not dom.beta is None else 0)) newerr = (1 - should_box.to_dtype()) * gtz * dom.errors return dom.new(newhead, newbeta, newerr)
def softplus(self): if self.errors is None: if self.beta is None: return self.new(F.softplus(self.head), None, None) tp = F.softplus(self.head + self.beta) bt = F.softplus(self.head - self.beta) return self.new((tp + bt) / 2, (tp - bt) / 2, None) errors = self.concreteErrors() o = h.ones(self.head.size()) def sp(hd): return F.softplus( hd) # torch.log(o + torch.exp(hd)) # not very stable def spp(hd): ehd = torch.exp(hd) return ehd.div(ehd + o) def sppp(hd): ehd = torch.exp(hd) md = ehd + o return ehd.div(md.mul(md)) fa = sp(self.head) fpa = spp(self.head) a = self.head k = torch.sum(errors.abs(), 0) def evalG(r): return r.mul(r).mul(sppp(a + r)) m = torch.max(evalG(h.zeros(k.size())), torch.max(evalG(k), evalG(-k))) m = h.ifThenElse(a.abs().lt(k), torch.max(m, torch.max(evalG(a), evalG(-a))), m) m /= 2 return self.new(fa, m if self.beta is None else m + self.beta.mul(fpa), None if self.errors is None else self.errors.mul(fpa))