def inv_transform(self,y):
inv_transform = lbann.WeightedSum(
lbann.SafeDivide(
lbann.Add(lbann.Constant(value=1.0, hint_layer=y),lbann.Identity(y)),
lbann.Subtract(lbann.Constant(value=1.0, hint_layer=y),lbann.Identity(y))),
scaling_factors=str(self.datascale))
linear_scale = 1/self.linear_scaler
CH2 = lbann.Tanh(lbann.WeightedSum(inv_transform,scaling_factors=str(linear_scale)))
return CH2
def construct_model():
"""Construct LBANN model.
Pilot1 Combo model
"""
import lbann
# Layer graph
data = lbann.Input(data_field='samples')
responses = lbann.Input(data_field='responses')
pred = combo.Combo()(data)
mse = lbann.MeanSquaredError([responses, pred])
SS_res = lbann.Reduction(lbann.Square(lbann.Subtract(responses, pred)),
mode='sum')
#SS_tot = var(x) = mean((x-mean(x))^2)
mini_batch_size = lbann.MiniBatchSize()
mean = lbann.Divide(lbann.BatchwiseReduceSum(responses), mini_batch_size)
SS_tot = lbann.Divide(
lbann.BatchwiseReduceSum(lbann.Square(lbann.Subtract(responses,
mean))),
mini_batch_size)
eps = lbann.Constant(value=1e-07, hint_layer=SS_tot)
r2 = lbann.Subtract(lbann.Constant(value=1, num_neurons='1'),
lbann.Divide(SS_res, lbann.Add(SS_tot, eps)))
metrics = [lbann.Metric(mse, name='mse')]
metrics.append(lbann.Metric(r2, name='r2'))
callbacks = [lbann.CallbackPrint(), lbann.CallbackTimer()]
# Construct model
num_epochs = 100
layers = list(lbann.traverse_layer_graph([data, responses]))
return lbann.Model(num_epochs,
layers=layers,
metrics=metrics,
objective_function=mse,
callbacks=callbacks)
def f_invtransform(y, scale=4.0): ### Transform to original space
'''
The inverse of the transformation function that scales the data before training
'''
inv_transform = lbann.WeightedSum(lbann.SafeDivide(
lbann.Add(lbann.Constant(value=1.0, hint_layer=y), lbann.Identity(y)),
lbann.Subtract(lbann.Constant(value=1.0, hint_layer=y),
lbann.Identity(y))),
scaling_factors=str(scale))
return inv_transform
def forward(self, inputs):
if len(inputs) != 2:
raise ValueError('expected two inputs: predictions and labels')
pred = inputs[0]
label = inputs[1]
ones = p.Constant(hint_layer=pred, value=1.0)
term1 = lbann.Multiply(
[label, lbann.Log(lbann.Subtract([ones, pred]))])
term2 = lbann.Log(pred)
full = lbann.WeightedSum([term1, term2], scaling_factors='-1.0 -1.0')
return lbann.Reduction(full)
def inv_transform(self, y): ### Original transformation
'''
The inverse of the transformation function that scales the data before training
'''
inv_transform = lbann.WeightedSum(lbann.SafeDivide(
lbann.Add(lbann.Constant(value=1.0, hint_layer=y),
lbann.Identity(y)),
lbann.Subtract(lbann.Constant(value=1.0, hint_layer=y),
lbann.Identity(y))),
scaling_factors=str(self.datascale))
return inv_transform
def inv_transform(self, y):
'''
The inverse of the transformation function that scales the data before training
'''
inv_transform = lbann.WeightedSum(lbann.SafeDivide(
lbann.Add(lbann.Constant(value=1.0, hint_layer=y),
lbann.Identity(y)),
lbann.Subtract(lbann.Constant(value=1.0, hint_layer=y),
lbann.Identity(y))),
scaling_factors=str(self.datascale))
#linear_scale = 1/self.linear_scaler
#CH2 = lbann.Tanh(lbann.WeightedSum(inv_transform,scaling_factors=str(linear_scale)))
#return CH2
return inv_transform
def inv_transform(self, y): ### Original transformation
'''
The inverse of the transformation function that scales the data before training
'''
inv_transform = lbann.WeightedSum(lbann.SafeDivide(
lbann.Add(lbann.Constant(value=1.0, hint_layer=y),
lbann.Identity(y)),
lbann.Subtract(lbann.Constant(value=1.0, hint_layer=y),
lbann.Identity(y))),
scaling_factors=str(self.datascale))
return inv_transform
# def inv_transform(self, y):### New tranformation : log-linear
# threshold = lbann.Constant(value=0.5, hint_layer=y)
# is_above_threshold = lbann.Greater(y, threshold)
# is_below_threshold = lbann.LogicalNot(is_above_threshold)
# below = lbann.SafeDivide(
# lbann.Subtract(y, lbann.Constant(value=1, hint_layer=y)),
# lbann.Constant(value=0.03, hint_layer=y),
# )
# above = lbann.Exp(lbann.SafeDivide(
# lbann.Subtract(
# y,
# lbann.Constant(value=0.5-0.5/math.log(300)*math.log(50), hint_layer=y)),
# lbann.Constant(value=0.5/math.log(300), hint_layer=y),
# ))
# return lbann.Add(
# lbann.Multiply(is_above_threshold, above),
# lbann.Multiply(is_below_threshold, below),
# )
# def f_invtransform_new(y):
# if y<=0.5:
# a=0.03;b=-1.0
# return (y-b)/a
# elif y>0.5:
# a=0.5/np.log(300)
# b=0.5-a*np.log(50)
# return np.exp((y-b)/a)
def compute_loss(self, x, y):
# y[:, :-1]
y = lbann.Slice(
y,
axis=0,
slice_points=str_list([0, self.input_feature_dims - 1]),
)
y = lbann.Identity(y)
# x[:, 1:]
x = lbann.Slice(
x,
slice_points=str_list([1, self.input_feature_dims]),
)
x = lbann.Identity(x)
# Convert indices in x to one-hot representation
# Note: Ignored indices result in zero vectors
ignore_mask = lbann.Equal(
x,
self.constant(self.label_to_ignore, hint_layer=x),
)
keep_mask = lbann.LogicalNot(ignore_mask)
length = lbann.Reduction(keep_mask, mode='sum')
length = lbann.Max(length, self.constant(1, [1]))
x = lbann.Add(
lbann.Multiply(keep_mask, x),
lbann.Multiply(ignore_mask, self.constant(-1, hint_layer=x)),
)
x = lbann.Slice(x,
slice_points=str_list(range(self.input_feature_dims)))
x = [lbann.Identity(x) for _ in range(self.input_feature_dims - 1)]
x = [lbann.OneHot(xi, size=self.dictionary_size) for xi in x]
x = [
lbann.Reshape(xi, dims=str_list([1, self.dictionary_size]))
for xi in x
]
x = lbann.Concatenation(x, axis=0)
# recon_loss = F.cross_entropy(
# y[:, :-1].contiguous().view(-1, y.size(-1)),
# x[:, 1:].contiguous().view(-1),
# ignore_index=self.pad
# )
# Note: Ideally we'd shift y by y.max(-1) for numerical stability
shifts = lbann.MatMul(
lbann.Max(y, self.constant(0, hint_layer=y)),
self.constant(
1 / math.sqrt(self.dictionary_size),
[self.dictionary_size, self.dictionary_size],
),
)
y = lbann.Subtract(y, shifts)
z = lbann.MatMul(
lbann.Exp(y),
self.constant(1, [self.dictionary_size, 1]),
)
z = lbann.Log(z)
z = lbann.MatMul(
lbann.Reshape(keep_mask, dims=str_list([1, -1])),
z,
)
recon_loss = lbann.MatMul(
lbann.Reshape(y, dims=str_list([1, -1])),
lbann.Reshape(x, dims=str_list([1, -1])),
transpose_b=True,
)
recon_loss = lbann.Subtract(z, recon_loss)
recon_loss = lbann.Reshape(recon_loss, dims=str_list([1]))
recon_loss = lbann.Divide(recon_loss, length)
return recon_loss
def compute_loss(self, x, y):
# y[:, :-1]
y = lbann.Slice(
y,
axis=0,
slice_points=str_list([0, self.input_feature_dims-1]),
)
y = lbann.Identity(y)
# x[:, 1:]
x = lbann.Slice(
x,
slice_points=str_list([1, self.input_feature_dims]),
)
x = lbann.Identity(x)
# Figure out entries in x to ignore
ignore_mask = lbann.Equal(
x,
self.constant(self.label_to_ignore, hint_layer=x),
)
keep_mask = lbann.LogicalNot(ignore_mask)
length = lbann.Reduction(keep_mask, mode='sum')
length = lbann.Max(length, self.constant(1, [1]))
# Convert entries in x to indices in y
# Note: Ignored entries correspond to an index of -1.
offsets = [
row*self.dictionary_size
for row in range(self.input_feature_dims-1)
]
offsets = lbann.Weights(
initializer=lbann.ValueInitializer(values=str_list(offsets)),
optimizer=lbann.NoOptimizer(),
)
offsets = lbann.WeightsLayer(
dims=str_list([self.input_feature_dims-1]),
weights=offsets,
)
y_inds = lbann.Add(x, offsets)
y_inds = lbann.Add(
lbann.Multiply(keep_mask, y_inds),
lbann.Multiply(
ignore_mask,
self.constant(-1, hint_layer=y_inds),
),
)
# recon_loss = F.cross_entropy(
# y[:, :-1].contiguous().view(-1, y.size(-1)),
# x[:, 1:].contiguous().view(-1),
# ignore_index=self.pad
# )
# Shift y for numerical stability
# Note: We'd prefer to shift by y.max(-1)
shifts = lbann.MatMul(
lbann.Max(y, self.constant(0, hint_layer=y)),
self.constant(
1 / math.sqrt(self.dictionary_size),
[self.dictionary_size, self.dictionary_size],
),
)
y = lbann.Subtract(y, shifts)
# Compute log of softmax denominator and sum
z = lbann.MatMul(
lbann.Exp(y),
self.constant(1, [self.dictionary_size, 1]),
)
z = lbann.Log(z)
z = lbann.MatMul(
lbann.Reshape(keep_mask, dims=str_list([1, -1])),
z,
)
z = lbann.Reshape(z, dims=str_list([1]))
# Compute cross entropy
recon_loss = lbann.Gather(
lbann.Reshape(y, dims=str_list([-1])),
y_inds,
)
recon_loss = lbann.Reduction(recon_loss, mode='sum')
recon_loss = lbann.Subtract(z, recon_loss)
recon_loss = lbann.Divide(recon_loss, length)
return recon_loss
