def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Sum with a weights layer so that gradient checking will
# verify that error signals are correct.
x_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input_weights')
x = lbann.Sum(
lbann.Reshape(lbann.Input(), dims=tools.str_list(_sample_dims)),
lbann.WeightsLayer(weights=x_weights,
dims=tools.str_list(_sample_dims)))
x_lbann = x
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# ------------------------------------------
# Data-parallel layout
# ------------------------------------------
# LBANN implementation
x = x_lbann
y = lbann.DFTAbs(x)
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='data-parallel layout'))
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 0
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Layer graph
weight = lbann.Weights(initializer=lbann.UniformInitializer(min=0, max=1))
weight = lbann.WeightsLayer(weights=weight, dims=tools.str_list([1]))
rand = lbann.Input(data_field='samples')
layers = list(lbann.traverse_layer_graph([weight, rand]))
for l in layers:
l.device = 'CPU'
# Model objects
metrics = [
lbann.Metric(weight, name='weight'),
lbann.Metric(rand, name='random'),
]
callbacks = [
lbann.CallbackPrint(),
]
# Construct model
return lbann.Model(_num_epochs,
layers=layers,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Multiply with a weights layer so that gradient checking
# will verify that error signals are correct. We multiply instead
# of adding so that each batch sample contributes a different
# gradient.
x_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=1.0),
name='input_weights')
x = lbann.Multiply(
lbann.Input(data_field='samples'),
lbann.WeightsLayer(weights=x_weights,
dims=tools.str_list(_sample_size)),
)
# Compute variance along batch dimension
sum_x = lbann.BatchwiseReduceSum(x)
sum_x2 = lbann.BatchwiseReduceSum(lbann.Square(x))
mini_batch_size = lbann.Tessellate(lbann.MiniBatchSize(), hint_layer=x)
mean_x = lbann.Divide(sum_x, mini_batch_size)
mean_x2 = lbann.Divide(sum_x2, mini_batch_size)
var = lbann.Subtract(mean_x2, lbann.Square(mean_x))
obj = lbann.L2Norm2(var)
# Objects for LBANN model
layers = list(lbann.traverse_layer_graph(x))
metric = lbann.Metric(obj, name='obj')
obj = lbann.ObjectiveFunction(obj)
callbacks = []
# Compute expected metric value
var = np.var(_samples, axis=0)
val = tools.numpy_l2norm2(var)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metric.name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# Gradient checking
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# Construct model
num_epochs = 0
return lbann.Model(num_epochs,
layers=layers,
objective_function=obj,
metrics=[metric],
callbacks=callbacks)
def create_bn_weights(layer_name, num_channels):
weights_ary = []
for i in range(4):
val = make_random_array((num_channels, ), 15 + i)
weights_ary.append(
lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(val))),
name='{}_{}'.format(layer_name, i)))
return weights_ary
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Convenience function to compute L2 norm squared with NumPy
def l2_norm2(x):
x = x.reshape(-1)
return np.inner(x, x)
# LBANN implementation
x = lbann.Reshape(lbann.Input(data_field='samples'),
dims=tools.str_list(_sample_dims))
y = lbann.Argmax(x, device='cpu')
z = lbann.L2Norm2(y)
# Objects for LBANN model
obj = z
metric = lbann.Metric(z, name='obj')
layers = list(lbann.traverse_layer_graph(z))
callbacks = []
# Get expected metric value from NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).reshape(_sample_dims).astype(np.float64)
y = np.argmax(x)
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metric.name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# Construct model
num_epochs = 0
return lbann.Model(num_epochs,
layers=layers,
objective_function=obj,
metrics=metric,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
x = lbann.Input(data_field='samples')
x_lbann = x
# Objects for LBANN model
metrics = []
callbacks = []
# ------------------------------------------
# Data-parallel weights layer
# ------------------------------------------
# Note: Weights are stored in one column of (STAR,STAR)
# distributed matrix
# Weights
weights_values = np.random.normal(size=_sample_dims).astype(np.float32)
weights_file = os.path.join(weights_dir, 'dataparallel_weights.npy')
np.save(weights_file, weights_values)
# LBANN implementation
x = lbann.Reshape(x_lbann, dims=tools.str_list(_sample_dims))
weights = lbann.Weights(
initializer=lbann.NumpyInitializer(file=weights_file),
)
weights = lbann.WeightsLayer(
weights=weights,
dims=tools.str_list(_sample_dims),
)
y = lbann.Multiply(x, weights)
z = lbann.L2Norm2(y)
metrics.append(lbann.Metric(z, name='data-parallel weights layer'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).reshape(_sample_dims).astype(np.float64)
y = x * weights_values
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Data-parallel FC layer
# ------------------------------------------
# Note: Weights are stored in (STAR,STAR) distributed matrix
# Weights
output_size = 7
linearity = np.random.normal(size=(output_size, _sample_size)).astype(np.float32)
linearity = linearity.astype(np.float64)
bias = np.random.normal(size=output_size).astype(np.float32)
linearity_file = os.path.join(weights_dir, 'dataparallel_fc_linearity.npy')
bias_file = os.path.join(weights_dir, 'dataparallel_fc_bias.npy')
np.save(linearity_file, linearity)
np.save(bias_file, bias)
# LBANN implementation
x = x_lbann
linearity_weights \
= lbann.Weights(initializer=lbann.NumpyInitializer(file=linearity_file))
bias_weights \
= lbann.Weights(initializer=lbann.NumpyInitializer(file=bias_file))
y = lbann.FullyConnected(
x,
weights=(linearity_weights, bias_weights),
data_layout='data_parallel',
num_neurons=output_size,
has_bias=True,
transpose=False)
z = lbann.L2Norm2(y)
metrics.append(lbann.Metric(z, name='data-parallel FC layer'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).astype(np.float64)
y = np.matmul(linearity, x) + bias
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Model-parallel FC layer
# ------------------------------------------
# Note: Weights are stored in (MC,MR) distributed matrix
# Weights
output_size = 9
linearity = np.random.normal(size=(output_size, _sample_size)).astype(np.float32)
bias = np.random.normal(size=output_size).astype(np.float32)
bias = bias.astype(np.float64)
linearity_file = os.path.join(weights_dir, 'modelparallel_fc_linearity.npy')
bias_file = os.path.join(weights_dir, 'modelparallel_fc_bias.npy')
np.save(linearity_file, linearity)
np.save(bias_file, bias)
# LBANN implementation
x = x_lbann
linearity_weights \
= lbann.Weights(initializer=lbann.NumpyInitializer(file=linearity_file))
bias_weights \
= lbann.Weights(initializer=lbann.NumpyInitializer(file=bias_file))
y = lbann.FullyConnected(
x,
weights=(linearity_weights, bias_weights),
data_layout='model_parallel',
num_neurons=output_size,
has_bias=True,
transpose=False)
z = lbann.L2Norm2(y)
metrics.append(lbann.Metric(z, name='model-parallel FC layer'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).astype(np.float64)
y = np.matmul(linearity, x) + bias
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 0
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x_lbann),
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Sum with a weights layer so that gradient checking will
# verify that error signals are correct.
x_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input_weights')
x = lbann.Sum(
lbann.Reshape(lbann.Input(data_field='samples'),
dims=tools.str_list(_sample_dims)),
lbann.WeightsLayer(weights=x_weights,
dims=tools.str_list(_sample_dims)))
x_lbann = x
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
branches = 2
# ------------------------------------------
# Data-parallel layout
# ------------------------------------------
# LBANN implementation
x = x_lbann
y = lbann.Identity(x, data_layout='data_parallel')
slice_points = (0, 8, 16)
x_slice = lbann.Slice(x,
axis=2,
slice_points=tools.str_list(slice_points),
parallel_strategy={
'sub_branch_tag': 0,
'enable_subgraph': True
})
branch1 = lbann.Identity(x_slice,
data_layout='data_parallel',
parallel_strategy={
'sub_branch_tag': 1,
'enable_subgraph': True
})
branch2 = lbann.Identity(x_slice,
data_layout='data_parallel',
parallel_strategy={
'sub_branch_tag': 2,
'enable_subgraph': True
})
grid_slice = lbann.Cross_Grid_Sum_Slice([branch1, branch2])
branch1 = lbann.Identity(grid_slice)
branch2 = lbann.Identity(grid_slice)
sum_branch = lbann.Sum([branch1, branch2],
parallel_strategy={
'sub_branch_tag': 0,
'enable_subgraph': True
})
z = lbann.L2Norm2(sum_branch)
obj.append(z)
metrics.append(lbann.Metric(z, name='data-parallel layout'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).reshape(_sample_dims).astype(np.float64)
y = []
cross_sum = 0
for j in range(len(slice_points) - 1):
x_slice = x[:, :, slice_points[j]:slice_points[j + 1]]
if (j == 0):
cross_sum = x_slice
else:
cross_sum += x_slice
last_dim = int(_sample_dims[-1] / (branches * branches))
sum_slices = None
for j in range(branches):
if (j == 0):
sum_slices = cross_sum[:, :, :last_dim]
else:
sum_slices += cross_sum[:, :, j * last_dim:(j + 1) * last_dim]
z = tools.numpy_l2norm2(sum_slices)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Gradient checking
# ------------------------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 0
return lbann.Model(
num_epochs,
subgraph_communication=lbann.SubgraphCommunication.COLL_OPT,
layers=lbann.traverse_layer_graph(x_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Sum with a weights layer so that gradient checking will
# verify that error signals are correct.
x_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input_weights')
x = lbann.Sum(lbann.Reshape(lbann.Input(data_field='samples'),
dims=tools.str_list(_sample_size)),
lbann.WeightsLayer(weights=x_weights,
dims=tools.str_list(_sample_size)))
x_lbann = x
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# ------------------------------------------
# Data-parallel layout
# ------------------------------------------
num_height_groups = tools.gpus_per_node(lbann)
if num_height_groups == 0:
e = 'this test requires GPUs.'
print('Skip - ' + e)
pytest.skip(e)
# LBANN implementation
x = x_lbann
x = lbann.Reshape(x, dims="4 2 6")
y = lbann.LeakyRelu(x, negative_slope=0.01,
data_layout='data_parallel',
parallel_strategy=create_parallel_strategy(
num_height_groups))
y = lbann.Reshape(y, dims=str(sample_dims()))
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='data-parallel layout'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).astype(np.float64)
y = np.where(x > 0, x, 0.01*x)
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Model-parallel layout
# ------------------------------------------
# LBANN implementation
x = x_lbann
x = lbann.Reshape(x, dims="4 2 6")
y = lbann.LeakyRelu(x, negative_slope=2,
data_layout='model_parallel',
parallel_strategy=create_parallel_strategy(
num_height_groups))
y = lbann.Reshape(y, dims=str(sample_dims()))
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='model-parallel layout'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).astype(np.float64)
y = np.where(x > 0, x, 2*x)
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Gradient checking
# ------------------------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 0
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Sum with a weights layer so that gradient checking will
# verify that error signals are correct.
x_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input_weights')
x0 = lbann.WeightsLayer(weights=x_weights,
dims=tools.str_list(_sample_dims))
x1 = lbann.Reshape(lbann.Input(), dims=tools.str_list(_sample_dims))
x = lbann.Sum(x0, x1)
# Apply channel-wise scale/bias
scale_values = tools.str_list(np.nditer(_scale))
bias_values = tools.str_list(np.nditer(_bias))
scalebias_weights = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(values='{} {}'.format(scale_values,
bias_values)),
name='scalebias_weights'
)
y = lbann.ChannelwiseScaleBias(x, weights=scalebias_weights)
z = lbann.L2Norm2(y)
# Objects for LBANN model
obj = z
metric = lbann.Metric(z, name='obj')
layers = list(lbann.traverse_layer_graph(z))
callbacks = []
# Get expected metric value from NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).reshape(_sample_dims).astype(np.float64)
y = _scale.astype(np.float64) * x + _bias.astype(np.float64)
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metric.name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# Gradient checking
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# Construct model
num_epochs = 0
return lbann.Model(num_epochs,
layers=layers,
objective_function=obj,
metrics=metric,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Sum with a weights layer so that gradient checking will
# verify that error signals are correct.
x_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input_weights')
x = lbann.Sum(lbann.Reshape(lbann.Input(data_field='samples'),
dims=tools.str_list(_sample_size)),
lbann.WeightsLayer(weights=x_weights,
dims=tools.str_list(_sample_size)))
x_lbann = x
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# ------------------------------------------
# Data-parallel layout
# ------------------------------------------
# LBANN implementation
x = x_lbann
y = lbann.Identity(x, data_layout='data_parallel')
branch1 = lbann.Identity(y, data_layout='data_parallel',parallel_strategy = {'sub_branch_tag':1,'enable_subgraph':True})
branch2 = lbann.Identity(y, data_layout='data_parallel',parallel_strategy = {'sub_branch_tag':2,'enable_subgraph':True})
sum_branch = lbann.Sum([branch1,branch2],parallel_strategy = {'sub_branch_tag':0,'enable_subgraph':True})
z = lbann.L2Norm2(sum_branch)
obj.append(z)
metrics.append(lbann.Metric(z, name='data-parallel layout'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).astype(np.float64)
y = x
z = tools.numpy_l2norm2(2*y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Gradient checking
# ------------------------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 0
return lbann.Model(num_epochs,subgraph_communication=lbann.SubgraphCommunication.COLL_OPT,
layers=lbann.traverse_layer_graph(x_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Sum with a weights layer so that gradient checking will
# verify that error signals are correct.
x = lbann.Input(data_field='samples')
x_slice = lbann.Slice(
x,
slice_points=tools.str_list([0, input_size, input_size + output_size]),
)
x0_weights = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input_weights',
)
x0 = lbann.Sum(
lbann.Identity(x_slice),
lbann.WeightsLayer(weights=x0_weights,
dims=tools.str_list(input_size)),
)
x1 = lbann.Identity(x_slice)
x0_lbann = x0
x1_lbann = x1
# Apply gather
y0 = lbann.Gather(x0, x1)
y1 = lbann.Concatenation([
lbann.Constant(value=i + 1, num_neurons='1')
for i in range(output_size)
])
y = lbann.Multiply(y0, y1)
z = lbann.L2Norm2(y)
# Objects for LBANN model
metrics = []
callbacks = []
objs = []
metrics.append(lbann.Metric(z, name='1D_obj'))
objs.append(lbann.ObjectiveFunction(z))
# Compute expected metric value
vals = []
for i in range(num_samples()):
sample = get_sample(i)
x0 = sample[:input_size]
x1 = sample[input_size:]
y0 = np.zeros(output_size)
for i in range(output_size):
if 0 <= x1[i] < input_size:
y0[i] = x0[int(x1[i])]
z = 0
for i in range(output_size):
z += ((i + 1) * y0[i])**2
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
######################################################################
#
# 2D Values , 1D Input, Axis = 0
#
######################################################################
x0 = lbann.Reshape(x0_lbann, dims=tools.str_list([num_rows, num_columns]))
x1 = lbann.Identity(x1_lbann, name="indices_2D_axis_0")
y0 = lbann.Gather(x0, x1, name="Gather_2D_axis_0", axis=0)
y1 = lbann.Concatenation([
lbann.Constant(value=i + 1, num_neurons='1')
for i in range(num_columns * output_size)
])
y0 = lbann.Reshape(y0, dims=tools.str_list([num_columns * output_size]))
y1 = lbann.Reshape(y1, dims=tools.str_list([num_columns * output_size]))
y = lbann.Multiply(y0, y1)
z = lbann.L2Norm2(y)
objs.append(z)
metrics.append(lbann.Metric(z, name="2D_obj_axis_0"))
vals = []
for i in range(num_samples()):
sample = get_sample(i)
x0 = np.array(sample[:input_size]).reshape((num_rows, num_columns))
x1 = sample[input_size:input_size + output_size]
y0 = np.zeros((output_size, num_columns))
for i in range(output_size):
if 0 <= x1[i] <= num_rows:
for j in range(num_columns):
y0[i][j] = x0[int(x1[i])][j]
z = 0
for i in range(num_columns * output_size):
z += ((i + 1) * y0.flatten()[i])**2
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
######################################################################
#
# 2D Values , 1D Input, Axis = 1
#
######################################################################
x0 = lbann.Reshape(x0_lbann, dims=tools.str_list([num_rows, num_columns]))
x1 = lbann.Identity(x1_lbann, name="Indices_2D")
y0 = lbann.Gather(x0, x1, name="Gather_2D", axis=1)
y1 = lbann.Concatenation([
lbann.Constant(value=i + 1, num_neurons='1')
for i in range(num_rows * output_size)
])
y0 = lbann.Reshape(y0, dims=tools.str_list([num_rows * output_size]))
y1 = lbann.Reshape(y1, dims=tools.str_list([num_rows * output_size]))
y = lbann.Multiply(y0, y1)
z = lbann.L2Norm2(y)
objs.append(z)
metrics.append(lbann.Metric(z, name="2D_obj_axis_1"))
vals = []
for i in range(num_samples()):
sample = get_sample(i)
x0 = np.array(sample[:input_size]).reshape((num_rows, num_columns))
x1 = sample[input_size:input_size + output_size]
y0 = np.zeros((num_rows, output_size))
for i in range(num_rows):
for j in range(output_size):
if 0 <= x1[j] <= num_columns:
y0[i][j] = x0[i][int(x1[j])]
z = 0
for i in range(num_rows * output_size):
z += ((i + 1) * y0.flatten()[i])**2
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# Gradient checking
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# Construct model
num_epochs = 0
layers = list(lbann.traverse_layer_graph(x))
return lbann.Model(num_epochs,
layers=layers,
objective_function=objs,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Sum with a weights layer so that gradient checking will
# verify that error signals are correct.
x_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input_weights')
x = lbann.Sum(
lbann.Reshape(lbann.Input(), dims=tools.str_list(_sample_dims)),
lbann.WeightsLayer(weights=x_weights,
dims=tools.str_list(_sample_dims)))
x_lbann = x
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# ------------------------------------------
# Pooling
# ------------------------------------------
num_height_groups = tools.gpus_per_node(lbann)
if num_height_groups == 0:
e = 'this test requires GPUs.'
print('Skip - ' + e)
pytest.skip(e)
pool_configs = []
# 3x3 pooling with same padding
for mode, val in [
("average", 700.1066377082393), # _num_samples=8
("max", 1255.4813455546334), # _num_samples=8
# ("average", 830.2573008820838), # _num_samples=23
# ("max", 1167.667676299899), # _num_samples=23
]:
pool_configs.append({
"name": "3x3 {} pooling".format(mode),
"kernel_dims": (3, 3),
"strides": (1, 1),
"pads": (0, 0),
"pool_mode": mode,
"val": val,
})
# 2x2 strided pooling
for mode, val in [
("average", 263.76437243059104), # _num_samples=23
("max", 358.66104389177207), # _num_samples=23
# ("average", 293.61402789516825), # _num_samples=23
# ("max", 351.4916288366334), # _num_samples=23
]:
pool_configs.append({
"name": "2x2 {} pooling".format(mode),
"kernel_dims": (2, 2),
"strides": (2, 2),
"pads": (0, 0),
"pool_mode": mode,
"val": val,
})
# 2x2x2 3D pooling
for mode, val in [
("average", 59.3851451701403), # _num_samples=8
("max", 216.75871475407558), # _num_samples=8
# ("average", 89.61246528381926), # _num_samples=23
# ("max", 198.65624293856985), # _num_samples=23
]:
pool_configs.append({
"name": "2x2x2 {} pooling".format(mode),
"kernel_dims": (2, 2, 2),
"strides": (2, 2, 2),
"pads": (0, 0, 0),
"pool_mode": mode,
"val": val,
})
for p in pool_configs:
# Apply pooling
x = x_lbann
if len(p["kernel_dims"]) == 3:
x = lbann.Reshape(x, dims=tools.str_list(_sample_dims_3d))
y = lbann.Pooling(
x,
num_dims=len(p["kernel_dims"]),
has_vectors=True,
pool_dims=tools.str_list(p["kernel_dims"]),
pool_strides=tools.str_list(p["strides"]),
pool_pads=tools.str_list(p["pads"]),
pool_mode=p["pool_mode"],
parallel_strategy=create_parallel_strategy(num_height_groups))
z = lbann.L2Norm2(y)
# Since max pooling is not differentiable, we only use average pooling.
if p["pool_mode"] == "average":
obj.append(z)
metrics.append(lbann.Metric(z, name=p["name"]))
# PyTorch implementation
try:
x = _samples
if len(p["kernel_dims"]) == 3:
x = np.reshape(x, [_num_samples] + _sample_dims_3d)
y = pytorch_pooling(
x,
p["kernel_dims"],
p["pool_mode"],
stride=p["strides"],
padding=p["pads"],
)
z = tools.numpy_l2norm2(y) / _num_samples
val = z
except:
# Precomputed value
val = p["val"]
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Gradient checking
# ------------------------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 0
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Sum with a weights layer so that gradient checking will
# verify that error signals are correct.
x_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input_weights')
x = lbann.Sum(lbann.Reshape(lbann.Input(data_field='samples'),
dims=tools.str_list(_sample_dims)),
lbann.WeightsLayer(weights=x_weights,
dims=tools.str_list(_sample_dims)))
x_lbann = x
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# --------------------------
# Slice along axis 0
# --------------------------
# LBANN implementation
slice_points = (2, 3, 6, 7)
x = x_lbann
x_slice = lbann.Slice(x, axis=0, slice_points=tools.str_list(slice_points))
y = []
for _ in range(len(slice_points)-1):
y.append(lbann.L2Norm2(x_slice))
z = lbann.Add(y[0], y[2])
obj.append(z)
metrics.append(lbann.Metric(z, name='axis0'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).reshape(_sample_dims).astype(np.float64)
y = []
for j in range(len(slice_points)-1):
x_slice = x[slice_points[j]:slice_points[j+1],:,:]
y.append(tools.numpy_l2norm2(x_slice))
z = y[0] + y[2]
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# --------------------------
# Slice along axis 1
# --------------------------
# LBANN implementation
slice_points = (0, 2, 3, 4)
x = x_lbann
x_slice = lbann.Slice(x, axis=1, slice_points=tools.str_list(slice_points))
y = []
for _ in range(len(slice_points)-1):
y.append(lbann.L2Norm2(x_slice))
z = lbann.Add(y[0], y[2])
obj.append(z)
metrics.append(lbann.Metric(z, name='axis1'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).reshape(_sample_dims).astype(np.float64)
y = []
for j in range(len(slice_points)-1):
x_slice = x[:,slice_points[j]:slice_points[j+1],:]
y.append(tools.numpy_l2norm2(x_slice))
z = y[0] + y[2]
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# --------------------------
# Slice along axis 2
# --------------------------
# LBANN implementation
slice_points = (1, 3)
x = x_lbann
x_slice = lbann.Slice(x, axis=2, slice_points=tools.str_list(slice_points))
y = []
for _ in range(len(slice_points)-1):
y.append(lbann.L2Norm2(x_slice))
z = y[0]
obj.append(z)
metrics.append(lbann.Metric(z, name='axis2'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).reshape(_sample_dims).astype(np.float64)
y = []
for j in range(len(slice_points)-1):
x_slice = x[:,:,slice_points[j]:slice_points[j+1]]
y.append(tools.numpy_l2norm2(x_slice))
z = y[0]
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# --------------------------
# Model-parallel
# --------------------------
# LBANN implementation
slice_points = (31, 54, 56, 57)
x = lbann.Reshape(x_lbann, dims=tools.str_list([105]))
x_slice = lbann.Slice(x, slice_points=tools.str_list(slice_points),
data_layout='model_parallel')
y = []
for _ in range(len(slice_points)-1):
y.append(lbann.L2Norm2(x_slice))
z = lbann.Add(y[0], y[2])
obj.append(z)
metrics.append(lbann.Metric(z, name='model-parallel'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).reshape(-1).astype(np.float64)
y = []
for j in range(len(slice_points)-1):
x_slice = x[slice_points[j]:slice_points[j+1]]
y.append(tools.numpy_l2norm2(x_slice))
z = y[0] + y[2]
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# --------------------------
# Gradient checking
# --------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# --------------------------
# Construct model
# --------------------------
num_epochs = 0
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Sum with a weights layer so that gradient checking will
# verify that error signals are correct.
x_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input_weights')
x = lbann.Sum(
lbann.Input(),
lbann.WeightsLayer(weights=x_weights,
dims=tools.str_list(_sample_size)))
x_lbann = x
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# --------------------------
# Concatenate along axis 0
# --------------------------
# LBANN implementation
x = x_lbann
x = lbann.Reshape(x, dims=tools.str_list([5, 3, 4]))
x_slice = lbann.Slice(x, axis=0, slice_points=tools.str_list([0, 1, 3, 5]))
x1 = lbann.Identity(x_slice)
x2 = lbann.Identity(x_slice)
x3 = lbann.Identity(x_slice)
y = lbann.Concatenation(x3, x2, x1, axis=0)
z = lbann.L2Norm2(lbann.Multiply(x, y))
obj.append(z)
metrics.append(lbann.Metric(z, name='axis0'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).reshape([5, 3, 4]).astype(np.float64)
x1 = x[0:1, :, :]
x2 = x[1:3, :, :]
x3 = x[3:5, :, :]
y = np.concatenate((x3, x2, x1), axis=0)
z = tools.numpy_l2norm2(x * y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# --------------------------
# Slice along axis 1
# --------------------------
# LBANN implementation
x = x_lbann
x = lbann.Reshape(x, dims=tools.str_list([3, 4, 5]))
x_slice = lbann.Slice(x, axis=1, slice_points=tools.str_list([0, 1, 3, 4]))
x1 = lbann.Identity(x_slice)
x2 = lbann.Identity(x_slice)
x3 = lbann.Identity(x_slice)
y = lbann.Concatenation(x2, x1, x3, axis=1)
z = lbann.L2Norm2(lbann.Multiply(x, y))
obj.append(z)
metrics.append(lbann.Metric(z, name='axis1'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).reshape([3, 4, 5]).astype(np.float64)
x1 = x[:, 0:1, :]
x2 = x[:, 1:3, :]
x3 = x[:, 3:4, :]
y = np.concatenate((x2, x1, x3), axis=1)
z = tools.numpy_l2norm2(x * y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# --------------------------
# Slice along axis 2
# --------------------------
# LBANN implementation
x = x_lbann
x = lbann.Reshape(x, dims=tools.str_list([3, 4, 5]))
x_slice = lbann.Slice(x,
axis=2,
slice_points=tools.str_list([0, 1, 2, 3, 5]))
x1 = lbann.Identity(x_slice)
x2 = lbann.Identity(x_slice)
x3 = lbann.Identity(x_slice)
x4 = lbann.Identity(x_slice)
y = lbann.Concatenation(x2, x4, x1, x3, axis=2)
z = lbann.L2Norm2(lbann.Multiply(x, y))
obj.append(z)
metrics.append(lbann.Metric(z, name='axis2'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).reshape([3, 4, 5]).astype(np.float64)
x1 = x[:, :, 0:1]
x2 = x[:, :, 1:2]
x3 = x[:, :, 2:3]
x4 = x[:, :, 3:5]
y = np.concatenate((x2, x4, x1, x3), axis=2)
z = tools.numpy_l2norm2(x * y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# --------------------------
# Model-parallel
# --------------------------
# LBANN implementation
x = x_lbann
x = lbann.Reshape(x, dims=tools.str_list([60]))
x_slice = lbann.Slice(x, slice_points=tools.str_list([0, 22, 23, 60]))
x1 = lbann.Identity(x_slice)
x2 = lbann.Identity(x_slice)
x3 = lbann.Identity(x_slice)
y = lbann.Concatenation(x3, x1, x2, data_layout='model_parallel')
z = lbann.L2Norm2(lbann.Multiply(x, y))
obj.append(z)
metrics.append(lbann.Metric(z, name='model-parallel'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).reshape([60]).astype(np.float64)
x1 = x[0:22]
x2 = x[22:23]
x3 = x[23:60]
y = np.concatenate((x3, x1, x2))
z = tools.numpy_l2norm2(x * y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# --------------------------
# Gradient checking
# --------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# --------------------------
# Construct model
# --------------------------
num_epochs = 0
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
from lbann.modules.rnn import ChannelwiseGRU
# Input data
# Note: Sum with a weights layer so that gradient checking will
# verify that error signals are correct.
x_weights = lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0),
name='input')
h_weights = lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0),
name='inital_hidden')
input_ = lbann.Input(data_field='samples')
input_slice = lbann.Slice(
input_,
slice_points=tools.str_list(
[0, _num_channels * _input_size, _sample_size]),
)
x = lbann.Reshape(input_slice,
dims=tools.str_list([_num_channels, _input_size]),
name="input_reshape")
x = lbann.Sum(x,
lbann.WeightsLayer(weights=x_weights,
dims=tools.str_list(
[_num_channels, _input_size])),
name="input_sum")
h = lbann.Reshape(input_slice,
dims=tools.str_list([_num_channels, _hidden_size]),
name="hidden_reshape")
h = lbann.Sum(h,
lbann.WeightsLayer(weights=h_weights,
dims=tools.str_list(
[_num_channels, _hidden_size])),
name="input_hidden_sum")
x_lbann = x
h_lbann = h
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# Weights
rnn_weights_numpy = []
ih_matrix = np.random.uniform(
low=-1,
high=1,
size=(3 * _hidden_size, _input_size),
)
hh_matrix = np.random.uniform(
low=-1,
high=1,
size=(3 * _hidden_size, _hidden_size),
)
ih_bias = np.random.uniform(low=-1, high=1, size=(3 * _hidden_size, ))
hh_bias = np.random.uniform(low=-1, high=1, size=(3 * _hidden_size, ))
rnn_weights_numpy.extend([ih_matrix, ih_bias, hh_matrix, hh_bias])
rnn_weights_lbann = [
lbann.Weights(initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(w, order='F'))))
for w in rnn_weights_numpy
]
# LBANN implementation
x = x_lbann
h = h_lbann
channelwise_GRU_cell = ChannelwiseGRU(num_channels=_num_channels,
size=_hidden_size,
weights=rnn_weights_lbann)
y = channelwise_GRU_cell(x, h)
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(
lbann.Metric(z, name="Multi-channel, Unidirectional, GRU Cell"))
# NumPy implementation
vals = []
for i in range(num_samples()):
input_ = get_sample(i).astype(np.float64)
x = input_[:_num_channels * _input_size].reshape(
(_num_channels, _input_size))
h = input_[_num_channels * _input_size:].reshape(
(_num_channels, _hidden_size))
y = numpy_gru_cell(x, h, rnn_weights_numpy)
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackPrintModelDescription())
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Gradient checking
# ------------------------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 0
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Weights matrix
height = _sample_size
width = 7
mat = np.random.normal(size=(height, width)).astype(np.float32)
# Input data
x_lbann = lbann.Input(data_field='samples')
# Objects for LBANN model
layers = [x_lbann]
obj = []
metrics = []
callbacks = []
# ------------------------------------------
# Compute expected metric values with NumPy
# ------------------------------------------
vals = []
for i in range(num_samples()):
x = get_sample(i).astype(np.float64)
y = np.linalg.norm(mat.astype(np.float64), axis=1)
z = np.inner(x, y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
# ------------------------------------------
# Data-parallel layout
# ------------------------------------------
w = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(mat, order='F')),
),
)
dummy = lbann.FullyConnected(
lbann.Constant(num_neurons=tools.str_list([width])),
weights=w,
num_neurons=height,
data_layout='data_parallel',
)
x = x_lbann
y = lbann.RowwiseWeightsNorms(
weights=w,
hint_layer=dummy,
data_layout='data_parallel',
)
z = lbann.MatMul(
lbann.Reshape(x, dims=tools.str_list([1,-1])),
lbann.Reshape(y, dims=tools.str_list([1,-1])),
transpose_b=True,
)
layers.append(dummy)
obj.append(z)
metrics.append(lbann.Metric(z, name='data-parallel layout'))
callbacks.append(
lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test',
)
)
# ------------------------------------------
# Model-parallel layout
# ------------------------------------------
w = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(mat, order='F')),
),
)
dummy = lbann.FullyConnected(
lbann.Constant(num_neurons=tools.str_list([width])),
weights=w,
num_neurons=height,
data_layout='model_parallel',
)
x = x_lbann
y = lbann.RowwiseWeightsNorms(
weights=w,
hint_layer=dummy,
data_layout='model_parallel',
)
z = lbann.MatMul(
lbann.Reshape(x, dims=tools.str_list([1,-1])),
lbann.Reshape(y, dims=tools.str_list([1,-1])),
transpose_b=True,
)
layers.append(dummy)
obj.append(z)
metrics.append(lbann.Metric(z, name='model-parallel layout'))
callbacks.append(
lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test',
)
)
# ------------------------------------------
# Gradient checking
# ------------------------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 0
return lbann.Model(num_epochs,
layers=layers,
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Sum with a weights layer so that gradient checking will
# verify that error signals are correct.
x_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input_weights')
x = lbann.Sum(
lbann.Reshape(lbann.Input(), dims=tools.str_list(_sample_dims)),
lbann.WeightsLayer(weights=x_weights,
dims=tools.str_list(_sample_dims)))
x_lbann = x
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# ------------------------------------------
# Basic 3^n convolution
# ------------------------------------------
# 3^n conv, stride=1, pad=1, dilation=1, bias
num_height_groups = tools.gpus_per_node(lbann)
if num_height_groups == 0:
e = 'this test requires GPUs.'
print('Skip - ' + e)
pytest.skip(e)
for num_dims, reference_val in [(2, 11913.852660080756),
(3, 9952.365297083174)]:
# Convolution settings
kernel_dims = [
5,
_sample_dims[0] if num_dims == 2 else _sample_dims_3d[0],
] + [3] * num_dims
strides = [1] * num_dims
pads = [1] * num_dims
dilations = [1] * num_dims
kernel = make_random_array(kernel_dims, 11)
# Apply convolution
kernel_weights = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(kernel))),
name='kernel1_{}d'.format(num_dims))
x = x_lbann
if num_dims == 3:
x = lbann.Reshape(x, dims=tools.str_list(_sample_dims_3d))
y = lbann.Convolution(
x,
weights=(kernel_weights, ),
num_dims=num_dims,
num_output_channels=kernel_dims[0],
has_vectors=True,
conv_dims=tools.str_list(kernel_dims[2:]),
conv_strides=tools.str_list(strides),
conv_pads=tools.str_list(pads),
conv_dilations=tools.str_list(dilations),
has_bias=False,
parallel_strategy=create_parallel_strategy(num_height_groups))
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(
lbann.Metric(z, name='basic {}D 3^n convolution'.format(num_dims)))
# PyTorch implementation
try:
x = _samples
if num_dims == 3:
x = np.reshape(x, [_num_samples] + _sample_dims_3d)
y = pytorch_convolution(x,
kernel,
stride=strides,
padding=pads,
dilation=dilations)
z = tools.numpy_l2norm2(y) / _num_samples
val = z
except:
# Precomputed value
val = reference_val
# val = 398.6956458317758 # _num_samples=8, 6 channels
# val = 381.7401227915947 # _num_samples=23, 6 channels
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Gradient checking
# ------------------------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 0
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
x_lbann = lbann.Input(data_field='samples')
y_numpy = np.random.normal(size=one_hot_size).astype(np.float32)
y_numpy[:] = 1 ### @todo Remove
y_lbann = lbann.Weights(initializer=lbann.ValueInitializer(
values=tools.str_list(y_numpy)))
y_lbann = lbann.WeightsLayer(
weights=y_lbann,
dims=tools.str_list([one_hot_size]),
)
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# ------------------------------------------
# Compute expected metric values with NumPy
# ------------------------------------------
vals = []
for i in range(num_samples()):
x = int(np.floor(get_sample(i)[0]))
y = y_numpy
z = y[x] if (0 <= x < one_hot_size) else 0
vals.append(z)
val = np.mean(vals, dtype=np.float64)
tol = np.abs(8 * val * np.finfo(np.float32).eps)
# ------------------------------------------
# Data-parallel layout
# ------------------------------------------
x = x_lbann
y = y_lbann
x_onehot = lbann.OneHot(
x,
size=one_hot_size,
data_layout='data_parallel',
)
z = lbann.MatMul(
lbann.Reshape(x_onehot, dims='1 -1'),
lbann.Reshape(y, dims='1 -1'),
transpose_b=True,
)
obj.append(z)
metrics.append(lbann.Metric(z, name='data-parallel layout'))
callbacks.append(
lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test',
))
# ------------------------------------------
# Model-parallel layout
# ------------------------------------------
x = x_lbann
y = y_lbann
x_onehot = lbann.OneHot(
x,
size=one_hot_size,
data_layout='model_parallel',
)
z = lbann.MatMul(
lbann.Reshape(x_onehot, dims='1 -1'),
lbann.Reshape(y, dims='1 -1'),
transpose_b=True,
)
obj.append(z)
metrics.append(lbann.Metric(z, name='model-parallel layout'))
callbacks.append(
lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test',
))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 0
return lbann.Model(num_epochs,
layers=x_lbann,
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: We want to use gradient checking to verify that error
# signals are correct. To do this, we zero-initialize a weights
# object, construct a zero-valued tensor, and add it to the
# input. To make sure that batchnorm is non-trivial, we multiply
# the zero-valued tensor by the mini-batch index.
x = lbann.Reshape(lbann.Input(), dims=tools.str_list(_sample_dims))
x_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input_weights')
x0 = lbann.WeightsLayer(weights=x_weights,
dims=tools.str_list(_sample_dims))
x1 = lbann.Divide(lbann.MiniBatchIndex(), lbann.MiniBatchSize())
x1 = lbann.Tessellate(lbann.Reshape(x1, dims='1 1 1'),
dims=tools.str_list(_sample_dims))
x = lbann.Sum(x, lbann.Multiply(x0, x1))
x_lbann = x
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# ------------------------------------------
# Data-parallel layout
# ------------------------------------------
# LBANN implementation
decay = 0.9
epsilon = 1e-5
x = x_lbann
y = lbann.EntrywiseBatchNormalization(x,
decay=decay,
epsilon=epsilon,
data_layout='data_parallel')
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='data-parallel layout'))
# ------------------------------------------
# Model-parallel layout
# ------------------------------------------
# LBANN implementation
decay = 0.9
epsilon = 1e-5
x = x_lbann
y = lbann.EntrywiseBatchNormalization(x,
decay=decay,
epsilon=epsilon,
data_layout='model_parallel')
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='model-parallel layout'))
# ------------------------------------------
# Gradient checking
# ------------------------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 1
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Sum with a weights layer so that gradient checking will
# verify that error signals are correct.
x_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input_weights')
x = lbann.Sum(
lbann.Reshape(lbann.Input(), dims=tools.str_list(_sample_dims)),
lbann.WeightsLayer(weights=x_weights,
dims=tools.str_list(_sample_dims)))
x_lbann = x
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# ------------------------------------------
# Pooling
# ------------------------------------------
pool_configs = []
# 3x3 pooling with same padding
for mode, val in [("average", 10.44275178160873),
("max", 57.139962751295094)]:
pool_configs.append({
"name": "3x3 {} pooling".format(mode),
"kernel_dims": (3, 3),
"strides": (1, 1),
"pads": (1, 1),
"pool_mode": mode,
"val": val,
})
# 2x2 strided pooling
for mode, val in [("average", 3.679837210617764),
("max", 8.836797848269207)]:
pool_configs.append({
"name": "2x2 {} pooling".format(mode),
"kernel_dims": (2, 2),
"strides": (2, 2),
"pads": (0, 0),
"pool_mode": mode,
"val": val,
})
# 2x4 pooling
for mode, val in [("average", 2.936493136591308),
("max", 10.950791583798338)]:
pool_configs.append({
"name": "2x4 {} pooling".format(mode),
"kernel_dims": (2, 4),
"strides": (3, 1),
"pads": (1, 0),
"pool_mode": mode,
"val": val,
})
# 2x2x2 3D pooling
for mode, val in [("average", 0.5709457972093812),
("max", 3.886491271066883)]:
pool_configs.append({
"name": "2x2x2 {} pooling".format(mode),
"kernel_dims": (2, 2, 2),
"strides": (2, 2, 2),
"pads": (0, 0, 0),
"pool_mode": mode,
"val": val,
})
for p in pool_configs:
# Apply pooling
x = x_lbann
if len(p["kernel_dims"]) == 3:
x = lbann.Reshape(x, dims=tools.str_list(_sample_dims_3d))
y = lbann.Pooling(x,
num_dims=len(p["kernel_dims"]),
has_vectors=True,
pool_dims=tools.str_list(p["kernel_dims"]),
pool_strides=tools.str_list(p["strides"]),
pool_pads=tools.str_list(p["pads"]),
pool_mode=p["pool_mode"])
z = lbann.L2Norm2(y)
# Since max pooling is not differentiable, we only use average pooling.
if p["pool_mode"] == "average":
obj.append(z)
metrics.append(lbann.Metric(z, name=p["name"]))
# PyTorch implementation
try:
x = _samples
if len(p["kernel_dims"]) == 3:
x = np.reshape(x, [_num_samples] + _sample_dims_3d)
y = pytorch_pooling(
x,
p["kernel_dims"],
p["pool_mode"],
stride=p["strides"],
padding=p["pads"],
)
z = tools.numpy_l2norm2(y) / _num_samples
val = z
except:
# Precomputed value
val = p["val"]
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Gradient checking
# ------------------------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 0
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Sum with weights layers so that gradient checking will
# verify that error signals are correct.
x0_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input0_weights')
x1_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input1_weights')
x_slice = lbann.Slice(lbann.Input(),
slice_points=tools.str_list([0, _m*_k, _m*_k+_k*_n]))
x0 = lbann.Sum(x_slice,
lbann.WeightsLayer(weights=x0_weights, dims=str(_m*_k)))
x1 = lbann.Sum(x_slice,
lbann.WeightsLayer(weights=x1_weights, dims=str(_k*_n)))
x0_lbann = x0
x1_lbann = x1
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# ------------------------------------------
# NN GEMM
# ------------------------------------------
# LBANN implementation
x0 = lbann.Reshape(x0_lbann, dims=tools.str_list([_m, _k]))
x1 = lbann.Reshape(x1_lbann, dims=tools.str_list([_k, _n]))
y = lbann.MatMul(x0, x1, data_layout='data_parallel')
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='NN GEMM'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).astype(np.float64)
x0 = x[:_m*_k].reshape([_m,_k])
x1 = x[_m*_k:].reshape([_k,_n])
y = np.matmul(x0, x1)
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# TN GEMM
# ------------------------------------------
# LBANN implementation
x0 = lbann.Reshape(x0_lbann, dims=tools.str_list([_k, _m]))
x1 = lbann.Reshape(x1_lbann, dims=tools.str_list([_k, _n]))
y = lbann.MatMul(x0, x1, transpose_a=True, data_layout='data_parallel')
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='TN GEMM'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).astype(np.float64)
x0 = x[:_m*_k].reshape([_k,_m])
x1 = x[_m*_k:].reshape([_k,_n])
y = np.matmul(x0.transpose(), x1)
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# NT GEMM
# ------------------------------------------
# LBANN implementation
x0 = lbann.Reshape(x0_lbann, dims=tools.str_list([_m, _k]))
x1 = lbann.Reshape(x1_lbann, dims=tools.str_list([_n, _k]))
y = lbann.MatMul(x0, x1, transpose_b=True, data_layout='data_parallel')
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='NT GEMM'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).astype(np.float64)
x0 = x[:_m*_k].reshape([_m,_k])
x1 = x[_m*_k:].reshape([_n,_k])
y = np.matmul(x0, x1.transpose())
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# TT GEMM
# ------------------------------------------
# LBANN implementation
x0 = lbann.Reshape(x0_lbann, dims=tools.str_list([_k, _m]))
x1 = lbann.Reshape(x1_lbann, dims=tools.str_list([_n, _k]))
y = lbann.MatMul(x0, x1, transpose_a=True, transpose_b=True,
data_layout='data_parallel')
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='TT GEMM'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).astype(np.float64)
x0 = x[:_m*_k].reshape([_k,_m])
x1 = x[_m*_k:].reshape([_n,_k])
y = np.matmul(x0.transpose(), x1.transpose())
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Gradient checking
# ------------------------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 0
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x0_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
x = lbann.Identity(lbann.Input())
x_lbann = x
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# ------------------------------------------
# GPU
# ------------------------------------------
# Embeddings
np.random.seed(_seed)
embedding_dim = 7
embeddings = np.random.normal(size=(_num_embeddings,embedding_dim))
# LBANN implementation
embedding_weights = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(values=tools.str_list(np.nditer(embeddings)))
)
x = x_lbann
y = lbann.DistEmbedding(x,
weights=embedding_weights,
num_embeddings=_num_embeddings,
embedding_dim=embedding_dim,
barrier_in_forward_prop=True,
device='gpu')
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='GPU'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i)
y = embeddings[x,:]
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# CPU
# ------------------------------------------
# Embeddings
np.random.seed(_seed)
embedding_dim = 5
embeddings = np.random.normal(size=(_num_embeddings,embedding_dim))
# LBANN implementation
embedding_weights = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(values=tools.str_list(np.nditer(embeddings)))
)
x = x_lbann
y = lbann.DistEmbedding(x,
weights=embedding_weights,
num_embeddings=_num_embeddings,
embedding_dim=embedding_dim,
barrier_in_forward_prop=True,
device='cpu')
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='CPU'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i)
y = embeddings[x,:]
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Gradient checking
# ------------------------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
# Construct model
num_epochs = 0
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Sum with a weights layer so that gradient checking will
# verify that error signals are correct.
x_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input_weights')
x0 = lbann.WeightsLayer(weights=x_weights,
dims=tools.str_list(_sample_dims))
x1 = lbann.Reshape(lbann.Input(data_field='samples'), dims=tools.str_list(_sample_dims))
x = lbann.Sum(x0, x1)
x_lbann = x
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# ------------------------------------------
# Compute expected metric values with NumPy
# ------------------------------------------
# Input and output dimensions
input_channel_dims = _sample_dims[1:]
output_channel_dims = (2,5)
input_channel_size = functools.reduce(operator.mul, input_channel_dims)
output_channel_size = functools.reduce(operator.mul, output_channel_dims)
# Weight values
linearity = np.random.normal(
size=(output_channel_size,input_channel_size)
).astype(np.float32)
bias = np.random.normal(size=(output_channel_size,1)).astype(np.float32)
# With bias
x = (_samples
.reshape((-1,input_channel_size))
.transpose()
.astype(np.float64))
y = np.matmul(linearity.astype(np.float64), x) + bias.astype(np.float64)
z = tools.numpy_l2norm2(y) / _num_samples
val_with_bias = z
# Without bias
x = (_samples
.reshape((-1,input_channel_size))
.transpose()
.astype(np.float64))
y = np.matmul(linearity.astype(np.float64), x)
z = tools.numpy_l2norm2(y) / _num_samples
val_without_bias = z
# ------------------------------------------
# Data-parallel layout, non-transpose, bias
# ------------------------------------------
# LBANN implementation
linearity_weights = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(linearity, order='F'))
)
)
bias_weights = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(bias))
)
)
x = x_lbann
y = lbann.ChannelwiseFullyConnected(
x,
weights=(linearity_weights, bias_weights),
output_channel_dims=output_channel_dims,
)
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='data-parallel layout, non-transpose, bias'))
# NumPy implementation
tol = 8 * val_with_bias * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val_with_bias-tol,
upper_bound=val_with_bias+tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Data-parallel layout, non-transpose, no bias
# ------------------------------------------
# LBANN implementation
linearity_weights = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(linearity, order='F'))
)
)
x = x_lbann
y = lbann.ChannelwiseFullyConnected(
x,
weights=(linearity_weights),
output_channel_dims=output_channel_dims,
bias=False,
)
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='data-parallel layout, non-transpose, no bias'))
# NumPy implementation
tol = 8 * val_without_bias * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val_without_bias-tol,
upper_bound=val_without_bias+tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Data-parallel layout, transpose, bias
# ------------------------------------------
# LBANN implementation
linearity_weights = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(linearity, order='C'))
)
)
bias_weights = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(bias))
)
)
x = x_lbann
y = lbann.ChannelwiseFullyConnected(
x,
weights=(linearity_weights, bias_weights),
output_channel_dims=output_channel_dims,
transpose=True,
)
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='data-parallel layout, transpose, bias'))
# NumPy implementation
tol = 8 * val_with_bias * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val_with_bias-tol,
upper_bound=val_with_bias+tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Data-parallel layout, transpose, no bias
# ------------------------------------------
# LBANN implementation
linearity_weights = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(linearity, order='C'))
)
)
x = x_lbann
y = lbann.ChannelwiseFullyConnected(
x,
weights=(linearity_weights),
output_channel_dims=output_channel_dims,
bias=False,
transpose=True,
)
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='data-parallel layout, transpose, no bias'))
# NumPy implementation
tol = 8 * val_without_bias * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val_without_bias-tol,
upper_bound=val_without_bias+tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Gradient checking
# ------------------------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 0
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Sum with a weights layer so that gradient checking will
# verify that error signals are correct.
x_weights = lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0),
name='input')
h_weights = lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0),
name='inital_hidden')
input_ = lbann.Input(data_field='samples')
input_slice = lbann.Slice(
input_,
slice_points=tools.str_list(
[0, _sequence_length * _input_size, _sample_size]),
)
x = lbann.Reshape(input_slice,
dims=tools.str_list([_sequence_length, _input_size]))
x = lbann.Sum(x, lbann.WeightsLayer(weights=x_weights, hint_layer=x))
h = lbann.Reshape(
input_slice,
dims=tools.str_list([_num_layers, _input_size]),
)
h = lbann.Sum(h, lbann.WeightsLayer(weights=h_weights, hint_layer=h))
x_lbann = x
h_lbann = h
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# ------------------------------------------
# Multi-layer, unidirectional GRU
# ------------------------------------------
# Note: input_size=hidden_size due to a limitation in oneDNN
# Weights
rnn_weights_numpy = []
for i in range(_num_layers):
ih_matrix = np.random.uniform(
low=-1,
high=1,
size=(3 * _input_size, _input_size),
)
hh_matrix = np.random.uniform(
low=-1,
high=1,
size=(3 * _input_size, _input_size),
)
ih_bias = np.random.uniform(low=-1, high=1, size=(3 * _input_size, ))
hh_bias = np.random.uniform(low=-1, high=1, size=(3 * _input_size, ))
rnn_weights_numpy.extend([ih_matrix, hh_matrix, ih_bias, hh_bias])
rnn_weights_numpy = [w.astype(np.float32) for w in rnn_weights_numpy]
rnn_weights_lbann = [
lbann.Weights(initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(w, order='F'))))
for w in rnn_weights_numpy
]
# LBANN implementation
x = x_lbann
h = h_lbann
y = lbann.GRU(
x,
h,
hidden_size=_input_size,
num_layers=_num_layers,
weights=rnn_weights_lbann,
)
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='Multi-layer, unidirectional'))
# NumPy implementation
vals = []
for i in range(num_samples()):
input_ = get_sample(i).astype(np.float64)
x = input_[:_sequence_length * _input_size].reshape(
(_sequence_length, _input_size))
h = input_[_sequence_length * _input_size:].reshape(
(_num_layers, _input_size))
y = numpy_gru(x, h, rnn_weights_numpy)
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Single-layer, unidirectional GRU
# ------------------------------------------
# Weights
rnn_weights_numpy = []
hidden_size = 7
ih_matrix = np.random.uniform(
low=-1,
high=1,
size=(3 * hidden_size, _input_size),
)
hh_matrix = np.random.uniform(
low=-1,
high=1,
size=(3 * hidden_size, hidden_size),
)
ih_bias = np.random.uniform(low=-1, high=1, size=(3 * hidden_size, ))
hh_bias = np.random.uniform(low=-1, high=1, size=(3 * hidden_size, ))
rnn_weights_numpy.extend([ih_matrix, hh_matrix, ih_bias, hh_bias])
rnn_weights_numpy = [w.astype(np.float32) for w in rnn_weights_numpy]
rnn_weights_lbann = [
lbann.Weights(initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(w, order='F'))))
for w in rnn_weights_numpy
]
# LBANN implementation
x = x_lbann
h = h_lbann
h = lbann.Reshape(
lbann.Slice(
lbann.Reshape(h, dims='-1'),
slice_points=tools.str_list([0, hidden_size]),
),
dims='1 -1',
)
y = lbann.GRU(
x,
h,
hidden_size=hidden_size,
num_layers=1,
weights=rnn_weights_lbann,
)
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='Single-layer, unidirectional'))
# NumPy implementation
vals = []
for i in range(num_samples()):
input_ = get_sample(i).astype(np.float64)
x = input_[:_sequence_length * _input_size].reshape(
(_sequence_length, _input_size))
h = input_[_sequence_length * _input_size:].reshape(
(_num_layers, _input_size))
h = h.flatten()[:hidden_size].reshape((1, hidden_size))
y = numpy_gru(x, h, rnn_weights_numpy)
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Gradient checking
# ------------------------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 0
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Sum with a weights layer so that gradient checking will
# verify that error signals are correct.
x_weights = lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0),
name='input')
h_weights = lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0),
name='inital_hidden')
input_ = lbann.Identity(lbann.Input())
input_slice = lbann.Slice(
input_,
slice_points=tools.str_list(
[0, _sequence_length * _input_size, _sample_size]),
)
x = lbann.Reshape(input_slice,
dims=tools.str_list([_sequence_length, _input_size]))
x = lbann.Sum(x, lbann.WeightsLayer(weights=x_weights, hint_layer=x))
h = lbann.Reshape(
input_slice,
dims=tools.str_list([_hidden_size]),
)
h = lbann.Sum(h, lbann.WeightsLayer(weights=h_weights, hint_layer=h))
x_lbann = x
h_lbann = h
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# ------------------------------------------
# 1-layer, uni-directional GRU
# ------------------------------------------
# Weights
ih_matrix = np.random.normal(size=(3 * _hidden_size,
_input_size)).astype(np.float32)
hh_matrix = np.random.normal(size=(3 * _hidden_size,
_hidden_size)).astype(np.float32)
ih_bias = np.random.normal(size=(3 * _hidden_size, )).astype(np.float32)
hh_bias = np.random.normal(size=(3 * _hidden_size, )).astype(np.float32)
ih_matrix_weights = lbann.Weights(initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(ih_matrix, order='F'))))
hh_matrix_weights = lbann.Weights(initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(hh_matrix, order='F'))))
ih_bias_weights = lbann.Weights(initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(ih_bias))))
hh_bias_weights = lbann.Weights(initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(hh_bias))))
# LBANN implementation
x = x_lbann
h = h_lbann
y = lbann.GRU(
x,
h,
hidden_size=_hidden_size,
weights=[
ih_matrix_weights, hh_matrix_weights, ih_bias_weights,
hh_bias_weights
],
)
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='1-layer, unidirectional'))
# NumPy implementation
vals = []
for i in range(num_samples()):
input_ = get_sample(i).astype(np.float64)
x = input_[:_sequence_length * _input_size].reshape(
(_sequence_length, _input_size))
h = input_[_sequence_length * _input_size:]
y = numpy_gru(x, h, ih_matrix, hh_matrix, ih_bias, hh_bias)
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Gradient checking
# ------------------------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 0
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
x = lbann.Input(data_field='samples')
x_lbann = x
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# ------------------------------------------
# No padding index
# ------------------------------------------
# Embeddings
np.random.seed(20191015)
embedding_dim = 5
embeddings = np.random.normal(size=(_num_embeddings, embedding_dim))
# LBANN implementation
embedding_weights = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(embeddings))))
x = x_lbann
y = lbann.Embedding(x,
weights=embedding_weights,
num_embeddings=_num_embeddings,
embedding_dim=embedding_dim)
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='no padding index'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i)
y = embeddings[x, :]
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Padding index 0
# ------------------------------------------
# Embeddings
np.random.seed(201910152)
embedding_dim = 7
padding_idx = 0
embeddings = np.random.normal(size=(_num_embeddings, embedding_dim))
# LBANN implementation
# Note: Embedding layer gradients are not exact if a padding index
# is set. Avoid gradient checking by not using an optimizer.
embedding_weights = lbann.Weights(
optimizer=None,
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(embeddings))))
x = x_lbann
y = lbann.Embedding(x,
weights=embedding_weights,
num_embeddings=_num_embeddings,
embedding_dim=embedding_dim,
padding_idx=padding_idx)
z = lbann.L2Norm2(y)
metrics.append(lbann.Metric(z, name='padding index = 0'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i)
y = np.where((x == padding_idx).reshape((-1, 1)), 0, embeddings[x, :])
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Gradient checking
# ------------------------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
# Construct model
num_epochs = 0
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Sum with weights layers so that gradient checking will
# verify that error signals are correct.
slice_size = _samples.shape[-1]
x0_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input0_weights')
x1_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input1_weights')
x_slice = lbann.Slice(lbann.Input(),
slice_points=tools.str_list([0, slice_size, 2*slice_size]))
x0 = lbann.Sum(x_slice,
lbann.WeightsLayer(weights=x0_weights, dims=str(slice_size)))
x1 = lbann.Sum(x_slice,
lbann.WeightsLayer(weights=x1_weights, dims=str(slice_size)))
x0_lbann = x0
x1_lbann = x1
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# ------------------------------------------
# Data-parallel layout
# ------------------------------------------
# LBANN implementation
x0 = x0_lbann
x1 = x1_lbann
y = lbann.CrossEntropy(x0, x1, data_layout='data_parallel')
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='data-parallel layout'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).astype(np.float64)
x0 = x[:slice_size]
x1 = x[slice_size:]
y = -np.inner(x1, np.log(x0))
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Model-parallel layout
# ------------------------------------------
# LBANN implementation
x0 = x0_lbann
x1 = x1_lbann
y = lbann.CrossEntropy(x0, x1, data_layout='model_parallel')
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='model-parallel layout'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).astype(np.float64)
x0 = x[:slice_size]
x1 = x[slice_size:]
y = -np.inner(x1, np.log(x0))
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(lbann.CallbackCheckMetric(
metric=metrics[-1].name,
lower_bound=val-tol,
upper_bound=val+tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Gradient checking
# ------------------------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 0
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x0_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Sum with a weights layer so that gradient checking will
# verify that error signals are correct.
x_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input_weights')
x = lbann.Sum(
lbann.Reshape(lbann.Input(), dims=tools.str_list(_sample_dims)),
lbann.WeightsLayer(weights=x_weights,
dims=tools.str_list(_sample_dims)))
x_lbann = x
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# ------------------------------------------
# Basic 3x3 convolution
# ------------------------------------------
# 3x3 conv, stride=1, pad=1, dilation=1, bias
# Convolution settings
kernel_dims = (5, _sample_dims[0], 3, 3)
strides = (1, 1)
pads = (1, 1)
dilations = (1, 1)
kernel = make_random_array(kernel_dims, 11)
bias = make_random_array([kernel_dims[0]], 123)
# Apply convolution
kernel_weights = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(kernel))),
name='kernel1')
bias_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(bias))),
name='bias1')
x = x_lbann
y = lbann.Convolution(x,
weights=(kernel_weights, bias_weights),
num_dims=3,
num_output_channels=kernel_dims[0],
has_vectors=True,
conv_dims=tools.str_list(kernel_dims[2:]),
conv_strides=tools.str_list(strides),
conv_pads=tools.str_list(pads),
conv_dilations=tools.str_list(dilations),
has_bias=True)
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='basic 3x3 convolution'))
# PyTorch implementation
try:
x = _samples
y = pytorch_convolution(x,
kernel,
bias=bias,
stride=strides,
padding=pads,
dilation=dilations)
z = tools.numpy_l2norm2(y) / _num_samples
val = z
except:
# Precomputed value
val = 153.84937996554953
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# 2x4 strided convolution
# ------------------------------------------
# Convolution settings
kernel_dims = (3, _sample_dims[0], 2, 4)
strides = (3, 1)
pads = (3, 0)
dilations = (1, 1)
num_groups = 1
kernel = make_random_array(kernel_dims, 19)
# Apply convolution
kernel_weights = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(kernel))),
name='kernel2')
x = x_lbann
y = lbann.Convolution(x,
weights=(kernel_weights),
num_dims=3,
num_output_channels=kernel_dims[0],
has_vectors=True,
conv_dims=tools.str_list(kernel_dims[2:]),
conv_strides=tools.str_list(strides),
conv_pads=tools.str_list(pads),
conv_dilations=tools.str_list(dilations),
num_groups=num_groups,
has_bias=False)
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='2x4 convolution'))
# PyTorch implementation
try:
x = _samples
y = pytorch_convolution(x,
kernel,
bias=None,
stride=strides,
padding=pads,
dilation=dilations,
groups=num_groups)
z = tools.numpy_l2norm2(y) / _num_samples
val = z
except:
# Precomputed value
val = 19.24587403346207
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Gradient checking
# ------------------------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 0
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
# Input data
# Note: Sum with a weights layer so that gradient checking will
# verify that error signals are correct.
x_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input_weights')
x = lbann.Sum(
lbann.Reshape(lbann.Input(data_field='samples'),
dims=tools.str_list(_sample_size)),
lbann.WeightsLayer(weights=x_weights,
dims=tools.str_list(_sample_size)))
x_lbann = x
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# ------------------------------------------
# Data-parallel layout
# ------------------------------------------
# LBANN implementation
x = x_lbann
y = lbann.Elu(x, alpha=1, data_layout='data_parallel')
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='data-parallel layout'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).astype(np.float64)
y = np.where(x < 0, np.expm1(x), x)
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Model-parallel layout
# ------------------------------------------
# LBANN implementation
x = x_lbann
y = lbann.Elu(x, alpha=0.5, data_layout='model_parallel')
z = lbann.L2Norm2(y)
obj.append(z)
metrics.append(lbann.Metric(z, name='model-parallel layout'))
# NumPy implementation
vals = []
for i in range(num_samples()):
x = get_sample(i).astype(np.float64)
y = np.where(x < 0, 0.5 * np.expm1(x), x)
z = tools.numpy_l2norm2(y)
vals.append(z)
val = np.mean(vals)
tol = 8 * val * np.finfo(np.float32).eps
callbacks.append(
lbann.CallbackCheckMetric(metric=metrics[-1].name,
lower_bound=val - tol,
upper_bound=val + tol,
error_on_failure=True,
execution_modes='test'))
# ------------------------------------------
# Gradient checking
# ------------------------------------------
callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True))
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 0
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x_lbann),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
def construct_model(lbann):
"""Construct LBANN model.
Args:
lbann (module): Module for LBANN Python frontend
"""
gpus_per_node = tools.gpus_per_node(lbann)
if gpus_per_node == 0:
e = 'this test requires GPUs.'
print('Skip - ' + e)
pytest.skip(e)
# Input data
# Note: Sum with a weights layer so that gradient checking will
# verify that error signals are correct.
x_weights = lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ConstantInitializer(value=0.0),
name='input_weights')
x = lbann.Sum(
lbann.Reshape(lbann.Input(data_field='samples'),
dims=tools.str_list(_sample_dims)),
lbann.WeightsLayer(weights=x_weights,
dims=tools.str_list(_sample_dims)))
# Objects for LBANN model
obj = []
metrics = []
callbacks = []
# Convolution/FC settings
kernel_dims = (5, _sample_dims[0], 3, 3)
strides = (1, 1)
pads = (1, 1)
dilations = (1, 1)
kernel = make_random_array(kernel_dims, 11)
fc_input_size = kernel_dims[0] * np.prod(_sample_dims[1:])
linearity1 = make_random_array((_output_size, fc_input_size), 13)
linearity2 = make_random_array((_output_size, _output_size), 17)
linearity_no_opt = make_random_array((_output_size, _output_size), 19)
biases = make_random_array((_output_size, ), 19)
# Weight values
kernel_weights = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(kernel))),
name='kernel1')
linearity1_weights = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(linearity1, order='F'))))
linearity2_weights = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(linearity2, order='F'))))
linearity_no_opt_weights = lbann.Weights(
optimizer=lbann.NoOptimizer(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(linearity_no_opt, order='F'))))
biases_weights = lbann.Weights(
optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(biases, order='F'))))
def create_bn_weights(layer_name, num_channels):
weights_ary = []
for i in range(4):
val = make_random_array((num_channels, ), 15 + i)
weights_ary.append(
lbann.Weights(optimizer=lbann.SGD(),
initializer=lbann.ValueInitializer(
values=tools.str_list(np.nditer(val))),
name='{}_{}'.format(layer_name, i)))
return weights_ary
y = lbann.Convolution(x,
weights=(kernel_weights, ),
num_dims=3,
num_output_channels=kernel_dims[0],
has_vectors=True,
conv_dims=tools.str_list(kernel_dims[2:]),
conv_strides=tools.str_list(strides),
conv_pads=tools.str_list(pads),
conv_dilations=tools.str_list(dilations),
has_bias=False)
# y = lbann.BatchNormalization(
# y,
# weights=create_bn_weights("bn1", kernel_dims[0]))
y = lbann.Relu(y)
y = lbann.FullyConnected(y,
weights=(linearity1_weights, ),
data_layout='data_parallel',
num_neurons=_output_size,
has_bias=False)
# y = lbann.BatchNormalization(
# y,
# weights=create_bn_weights("bn2", _output_size))
y = lbann.Relu(y)
y = lbann.FullyConnected(y,
weights=(linearity2_weights, biases_weights),
data_layout='data_parallel',
num_neurons=_output_size,
has_bias=True)
# y = lbann.BatchNormalization(
# y,
# weights=create_bn_weights("bn3", _output_size))
y = lbann.Relu(y)
y = lbann.FullyConnected(y,
weights=(linearity_no_opt_weights),
data_layout='data_parallel',
num_neurons=_output_size,
has_bias=False)
z = lbann.L2Norm2(y)
obj.append(z)
# ------------------------------------------
# Construct model
# ------------------------------------------
num_epochs = 1 ### @todo Remove
return lbann.Model(num_epochs,
layers=lbann.traverse_layer_graph(x),
objective_function=obj,
metrics=metrics,
callbacks=callbacks)
