def __init__(self,
in_features,
out_features,
andor="*",
modinf=False,
regular_deriv=False,
min_input=0.0,
max_input=1.0,
min_slope=0.001,
max_slope=10.0):
"""
Implementation of RBF module with logloss.
:param in_features: Number of input features.
:param out_features: Number of output features.
:param andor: '^' for and, 'v' for or, '*' for mixed.
:param modinf: Whether to aggregate using max (if True) of sum (if False).
:param regular_deriv: Whether to use regular derivatives or not.
:param min_input: minimum value for w (and therefore min value for input)
:param max_input: max, as above.
:param min_slope: min value for u, defining the slope.
:param max_slope: max value for u, defining the slope.
"""
super(RBFI, self).__init__()
self.in_features = in_features
self.out_features = out_features
self.andor = andor
self.modinf = modinf
self.regular_deriv = regular_deriv
self.w = BoundedParameter(torch.Tensor(out_features, in_features),
lower_bound=min_input,
upper_bound=max_input)
self.u = BoundedParameter(torch.Tensor(out_features, in_features),
lower_bound=min_slope,
upper_bound=max_slope)
if andor == 'v':
self.andor01 = Parameter(torch.ones((1, out_features)))
elif andor == '^':
self.andor01 = Parameter(torch.zeros((
1,
out_features,
)))
else:
self.andor01 = Parameter(torch.Tensor(
1,
out_features,
))
self.andor01.data.random_(0, 2)
self.andor01.requires_grad = False
self.w.data.uniform_(min_input, max_input)
# Initialization of u.
self.u.data.uniform_(0.2, 0.7) # These could be parameters.
self.u.data.clamp_(min_slope, max_slope)
def hook(o):
meta_module, meta_class = None, o.get('meta_class')
if meta_class in ('Datetime', 'datetime.datetime'):
# 'Datetime' included for backward compatibility
try:
tmp = datetime.datetime.strptime(
o['date'], '%Y-%m-%dT%H:%M:%S.%f')
except Exception as e:
tmp = datetime.datetime.strptime(
o['date'], '%Y-%m-%dT%H:%M:%S')
return tmp
elif meta_class == 'set':
# Set.
return set(o['set'])
# Numpy arrays.
elif meta_class == 'numpy.ndarray':
data = base64.b64decode(o['data'])
dtype = o['dtype']
shape = o['shape']
v = numpy.frombuffer(data, dtype=dtype)
v = v.reshape(shape)
return v
# Numpy numbers.
elif meta_class == 'numpy.number':
data = base64.b64decode(o['data'])
dtype = o['dtype']
v = numpy.frombuffer(data, dtype=dtype)[0]
return v
# Parameters
elif meta_class == 'Parameter':
p = Parameter(torch.tensor(o['data']), requires_grad=o['requires_grad'])
return p.to(device)
elif meta_class == 'BoundedParameter':
p = BoundedParameter(torch.tensor(o['data']),
requires_grad=o['requires_grad'],
lower_bound=o['lower_bound'],
upper_bound=o['upper_bound'])
return p.to(device)
elif meta_class == 'Storage':
p = Storage(o['d'])
return p
elif meta_class and '.' in meta_class:
# correct for classes that have migrated from one module to another
meta_class = remapper.get(meta_class, meta_class)
# separate the module name from the actual class name
meta_module, meta_class = meta_class.rsplit('.',1)
if meta_class is not None:
del o['meta_class']
# this option is for backward compatibility in case a module is not specified
if meta_class in fallback:
meta_module = fallback.get(meta_class)
if meta_module is not None and objectify:
try:
module = importlib.import_module(meta_module)
cls = getattr(module, meta_class)
# Figures out parameters for intializer.
try:
args = inspect.getargspec(cls.__init__)[0]
args = [x for x in args if x in o.keys()]
dd = [o[x] for x in args]
obj = cls(*dd)
except:
print("Incomplete rebuild:", traceback.format_exc())
obj = cls()
obj.__dict__.update(o)
# Restores modules.
if isinstance(obj, Module):
for k, v in obj._modules.items():
setattr(obj, k, v)
o = obj
except Exception as e:
# We need to allow the case where the class is now obsolete.
print(traceback.format_exc())
print("Could not restore: %r %r", (meta_module, meta_class))
o = None
elif type(o).__name__ == 'dict':
o = Storage(o)
return o
class MWD_trimmable(nn.Module, Serializable):
def __init__(
self,
in_features,
out_features,
andor="*",
pseudo=1.0,
min_input=0.0,
max_input=1.0,
min_slope=0.001,
max_slope=10.0,
):
"""
Implementation of MWD.
:param in_features: Number of input features.
:param out_features: Number of output features.
:param andor: '^' for and, 'v' for or, '*' for mixed.
:param modinf: Whether to aggregate using max (if True) of sum (if False).
:param regular_deriv: Whether to use regular derivatives or not.
:param min_input: minimum value for w (and therefore min value for input)
:param max_input: max, as above.
:param min_slope: min value for u, defining the slope.
:param max_slope: max value for u, defining the slope.
"""
super(MWD_trimmable, self).__init__()
self.in_features = in_features
self.out_features = out_features
self.andor = andor
self.pseudo = pseudo
self.w = BoundedParameter(torch.Tensor(out_features, in_features),
lower_bound=min_input,
upper_bound=max_input)
self.u = BoundedParameter(torch.Tensor(out_features, in_features),
lower_bound=min_slope,
upper_bound=max_slope)
if andor == 'v':
self.andor01 = Parameter(torch.ones((1, out_features)))
elif andor == '^':
self.andor01 = Parameter(torch.zeros((
1,
out_features,
)))
else:
self.andor01 = Parameter(torch.Tensor(
1,
out_features,
))
self.andor01.data.random_(0, 2)
self.andor01.requires_grad = False
self.w.data.uniform_(min_input, max_input)
# Initialization of u.
self.u.data.uniform_(0., 0.7) # These could be parameters.
self.u.data.clamp_(min_slope, max_slope)
def forward(self, x):
# Let n be the input size, and m the output size.
# The tensor x is of shape * n. To make room for the output,
# we view it as of shape * 1 n.
xx = x.unsqueeze(-2)
xuw = self.u * (xx - self.w)
xuwsq = xuw * xuw
# Aggregates into a modulus.
# We want to get the largest square, which is the min one as we changed signs.
z = SharedFeedbackMaxExpPseudo.apply(xuwsq, self.pseudo)
y = LargeAttractorExpPseudo2.apply(z, self.pseudo)
# Takes into account and-orness.
if self.andor == '^':
return y
elif self.andor == 'v':
return 1.0 - y
else:
return y + self.andor01 * (1.0 - 2.0 * y)
def interval_forward(self, x_min, x_max):
xx_min = x_min.unsqueeze(-2)
xx_max = x_max.unsqueeze(-2)
xuw1 = self.u * (xx_min - self.w)
xuwsq1 = xuw1 * xuw1
xuw2 = self.u * (xx_max - self.w)
xuwsq2 = xuw2 * xuw2
sq_max = torch.max(xuwsq1, xuwsq2)
sq_min = torch.min(xuwsq1, xuwsq2)
# If w is between x_min and x_max, then sq_min should be 0.
# So we multiply sq_min by something that is 0 if x_min < w < x_max.
sq_min = sq_min * ((xx_min > self.w) + (self.w > xx_max)).float()
y_min = torch.exp(-torch.max(sq_max, -1)[0])
y_max = torch.exp(-torch.max(sq_min, -1)[0])
# Takes into account and-orness.
if self.andor == '^':
return y_min, y_max
elif self.andor == 'v':
return 1.0 - y_max, 1.0 - y_min
else:
y1 = y_min + self.andor01 * (1.0 - 2.0 * y_min)
y2 = y_max + self.andor01 * (1.0 - 2.0 * y_max)
y_min = torch.min(y1, y2)
y_max = torch.max(y1, y2)
return y_min, y_max
def overall_sensitivity(self):
"""Returns the sensitivity to adversarial examples of the layer."""
s = torch.max(torch.max(self.u, -1)[0], -1)[0].item()
s *= np.sqrt(2. / np.e)
return s
def sensitivity(self, previous_layer):
"""Given the sensitivity of the previous layer (a vector of length equal
to the number of inputs), it computes the sensitivity to adversarial examples
of the current layer, as a vector of length equal to the output size of the
layer. If the input sensitivity of the previous layer is None, then unit
sensitivity is assumed."""
if previous_layer is None:
previous_layer = self.w.new(1, self.in_features)
previous_layer.fill_(1.)
else:
previous_layer = previous_layer.view(1, self.in_features)
u_prod = previous_layer * self.u
s = SharedFeedbackMaxExpPseudo.apply(u_prod, 1.0)
s = s * np.sqrt(2. / np.e)
return s
class RBFI(nn.Module):
def __init__(self,
in_features,
out_features,
andor="*",
modinf=False,
regular_deriv=False,
min_input=0.0,
max_input=1.0,
min_slope=0.001,
max_slope=10.0):
"""
Implementation of RBF module with logloss.
:param in_features: Number of input features.
:param out_features: Number of output features.
:param andor: '^' for and, 'v' for or, '*' for mixed.
:param modinf: Whether to aggregate using max (if True) of sum (if False).
:param regular_deriv: Whether to use regular derivatives or not.
:param min_input: minimum value for w (and therefore min value for input)
:param max_input: max, as above.
:param min_slope: min value for u, defining the slope.
:param max_slope: max value for u, defining the slope.
"""
super(RBFI, self).__init__()
self.in_features = in_features
self.out_features = out_features
self.andor = andor
self.modinf = modinf
self.regular_deriv = regular_deriv
self.w = BoundedParameter(torch.Tensor(out_features, in_features),
lower_bound=min_input,
upper_bound=max_input)
self.u = BoundedParameter(torch.Tensor(out_features, in_features),
lower_bound=min_slope,
upper_bound=max_slope)
if andor == 'v':
self.andor01 = Parameter(torch.ones((1, out_features)))
elif andor == '^':
self.andor01 = Parameter(torch.zeros((
1,
out_features,
)))
else:
self.andor01 = Parameter(torch.Tensor(
1,
out_features,
))
self.andor01.data.random_(0, 2)
self.andor01.requires_grad = False
self.w.data.uniform_(min_input, max_input)
# Initialization of u.
self.u.data.uniform_(0.2, 0.7) # These could be parameters.
self.u.data.clamp_(min_slope, max_slope)
def dumps(self):
"""Writes itself to a string."""
# Creates a dictionary
d = dict(
in_features=self.in_features,
out_features=self.out_features,
min_input=self.w.lower_bound,
max_input=self.w.upper_bound,
min_slope=self.u.lower_bound,
max_slope=self.u.upper_bound,
modinf=self.modinf,
regular_deriv=self.regular_deriv,
andor=self.andor,
andor01=self.andor01.cpu().numpy(),
u=self.u.data.cpu().numpy(),
w=self.w.data.cpu().numpy(),
)
return Serializable.dumps(d)
@staticmethod
def loads(s, device):
"""Reads itself from string s."""
d = Serializable.loads(s)
m = RBFI(d['in_features'],
d['out_features'],
andor=d['andor'],
modinf=d['modinf'],
regular_deriv=d['regular_deriv'],
min_input=d['min_input'],
max_input=d['max_input'],
min_slope=d['min_slope'],
max_slope=d['max_slope'])
m.u.data = torch.from_numpy(d['u']).to(device)
m.w.data = torch.from_numpy(d['w']).to(device)
m.andor01.data = torch.from_numpy(d['andor01']).to(device)
return m
def forward(self, x):
# Let n be the input size, and m the output size.
# The tensor x is of shape * n. To make room for the output,
# we view it as of shape * 1 n.
s = list(x.shape)
new_s = s[:-1] + [1, s[-1]]
xx = x.view(*new_s)
xuw = self.u * (xx - self.w)
xuwsq = xuw * xuw
# Aggregates into a modulus.
if self.modinf:
# We want to get the largest square, which is the min one as we changed signs.
if self.regular_deriv:
z, _ = torch.max(xuwsq, -1)
y = torch.exp(-z)
else:
z = SharedFeedbackMax.apply(xuwsq)
y = LargeAttractorExp.apply(z)
else:
z = torch.sum(xuwsq, -1)
if self.regular_deriv:
y = torch.exp(-z)
else:
y = LargeAttractorExp.apply(z)
# Takes into account and-orness.
if self.andor == '^':
return y
elif self.andor == 'v':
return 1.0 - y
else:
return y + self.andor01 * (1.0 - 2.0 * y)
def overall_sensitivity(self):
"""Returns the sensitivity to adversarial examples of the layer."""
if self.modinf:
s = torch.max(torch.max(self.u, -1)[0], -1)[0].item()
else:
s = torch.max(torch.sqrt(torch.sum(self.u * self.u, -1)))[0].item()
s *= np.sqrt(2. / np.e)
return s
def sensitivity(self, previous_layer):
"""Given the sensitivity of the previous layer (a vector of length equal
to the number of inputs), it computes the sensitivity to adversarial examples
of the current layer, as a vector of length equal to the output size of the
layer. If the input sensitivity of the previous layer is None, then unit
sensitivity is assumed."""
if previous_layer is None:
previous_layer = self.w.new(1, self.in_features)
previous_layer.fill_(1.)
else:
previous_layer = previous_layer.view(1, self.in_features)
u_prod = previous_layer * self.u
if self.modinf:
# s = torch.max(u_prod, -1)[0]
s = SharedFeedbackMax.apply(u_prod)
else:
s = torch.sqrt(torch.sum(u_prod * u_prod, -1))
s = s * np.sqrt(2. / np.e)
return s
class RBFI(nn.Module):
def __init__(self,
in_features,
out_features,
andor="*",
min_input=0.0,
max_input=1.0,
min_slope=0.001,
max_slope=10.0):
"""
Implementation of RBF module with logloss.
:param in_features: Number of input features.
:param out_features: Number of output features.
:param andor: '^' for and, 'v' for or, '*' for mixed.
:param min_input: minimum value for w (and therefore min value for input)
:param max_input: max, as above.
:param min_slope: min value for u, defining the slope.
:param max_slope: max value for u, defining the slope.
"""
super(RBFI, self).__init__()
self.in_features = in_features
self.out_features = out_features
self.andor = andor
self.w = BoundedParameter(torch.Tensor(out_features, in_features),
lower_bound=min_input,
upper_bound=max_input)
self.u = BoundedParameter(torch.Tensor(out_features, in_features),
lower_bound=min_slope,
upper_bound=max_slope)
if andor == 'v':
self.andor01 = Parameter(torch.ones((1, out_features)))
elif andor == '^':
self.andor01 = Parameter(torch.zeros((
1,
out_features,
)))
else:
self.andor01 = Parameter(torch.Tensor(
1,
out_features,
))
self.andor01.data.random_(0, 2)
self.andor01.requires_grad = False
self.w.data.uniform_(min_input, max_input)
# Initialization of u.
self.u.data.uniform_(0.2, 0.7) # These could be parameters.
self.u.data.clamp_(min_slope, max_slope)
def forward(self, x):
# Let n be the input size, and m the output size.
# The tensor x is of shape * n. To make room for the output,
# we view it as of shape * 1 n.
s = list(x.shape)
new_s = s[:-1] + [1, s[-1]]
xx = x.view(*new_s)
xuw = self.u * (xx - self.w)
xuwsq = xuw * xuw
# Aggregates into a modulus.
z = SharedFeedbackMax.apply(xuwsq)
y = LargeAttractorExp.apply(z)
# Takes into account and-orness.
if self.andor == '^':
return y
elif self.andor == 'v':
return 1.0 - y
else:
return y + self.andor01 * (1.0 - 2.0 * y)
def sensitivity(self, previous_layer):
"""Given the sensitivity of the previous layer (a vector of length equal
to the number of inputs), it computes the sensitivity to adversarial examples
of the current layer, as a vector of length equal to the output size of the
layer. If the input sensitivity of the previous layer is None, then unit
sensitivity is assumed."""
if previous_layer is None:
previous_layer = self.w.new(1, self.in_features)
previous_layer.fill_(1.)
else:
previous_layer = previous_layer.view(1, self.in_features)
u = previous_layer * self.u
s = torch.max(u, -1)[0]
s *= np.sqrt(2. / np.e)
return s