def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum( lbann.Input(), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_size))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # -------------------------- # Concatenate along axis 0 # -------------------------- # LBANN implementation x = x_lbann x = lbann.Reshape(x, dims=tools.str_list([5, 3, 4])) x_slice = lbann.Slice(x, axis=0, slice_points=tools.str_list([0, 1, 3, 5])) x1 = lbann.Identity(x_slice) x2 = lbann.Identity(x_slice) x3 = lbann.Identity(x_slice) y = lbann.Concatenation(x3, x2, x1, axis=0) z = lbann.L2Norm2(lbann.Multiply(x, y)) obj.append(z) metrics.append(lbann.Metric(z, name='axis0')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape([5, 3, 4]).astype(np.float64) x1 = x[0:1, :, :] x2 = x[1:3, :, :] x3 = x[3:5, :, :] y = np.concatenate((x3, x2, x1), axis=0) z = tools.numpy_l2norm2(x * y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # -------------------------- # Slice along axis 1 # -------------------------- # LBANN implementation x = x_lbann x = lbann.Reshape(x, dims=tools.str_list([3, 4, 5])) x_slice = lbann.Slice(x, axis=1, slice_points=tools.str_list([0, 1, 3, 4])) x1 = lbann.Identity(x_slice) x2 = lbann.Identity(x_slice) x3 = lbann.Identity(x_slice) y = lbann.Concatenation(x2, x1, x3, axis=1) z = lbann.L2Norm2(lbann.Multiply(x, y)) obj.append(z) metrics.append(lbann.Metric(z, name='axis1')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape([3, 4, 5]).astype(np.float64) x1 = x[:, 0:1, :] x2 = x[:, 1:3, :] x3 = x[:, 3:4, :] y = np.concatenate((x2, x1, x3), axis=1) z = tools.numpy_l2norm2(x * y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # -------------------------- # Slice along axis 2 # -------------------------- # LBANN implementation x = x_lbann x = lbann.Reshape(x, dims=tools.str_list([3, 4, 5])) x_slice = lbann.Slice(x, axis=2, slice_points=tools.str_list([0, 1, 2, 3, 5])) x1 = lbann.Identity(x_slice) x2 = lbann.Identity(x_slice) x3 = lbann.Identity(x_slice) x4 = lbann.Identity(x_slice) y = lbann.Concatenation(x2, x4, x1, x3, axis=2) z = lbann.L2Norm2(lbann.Multiply(x, y)) obj.append(z) metrics.append(lbann.Metric(z, name='axis2')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape([3, 4, 5]).astype(np.float64) x1 = x[:, :, 0:1] x2 = x[:, :, 1:2] x3 = x[:, :, 2:3] x4 = x[:, :, 3:5] y = np.concatenate((x2, x4, x1, x3), axis=2) z = tools.numpy_l2norm2(x * y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # -------------------------- # Model-parallel # -------------------------- # LBANN implementation x = x_lbann x = lbann.Reshape(x, dims=tools.str_list([60])) x_slice = lbann.Slice(x, slice_points=tools.str_list([0, 22, 23, 60])) x1 = lbann.Identity(x_slice) x2 = lbann.Identity(x_slice) x3 = lbann.Identity(x_slice) y = lbann.Concatenation(x3, x1, x2, data_layout='model_parallel') z = lbann.L2Norm2(lbann.Multiply(x, y)) obj.append(z) metrics.append(lbann.Metric(z, name='model-parallel')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape([60]).astype(np.float64) x1 = x[0:22] x2 = x[22:23] x3 = x[23:60] y = np.concatenate((x3, x1, x2)) z = tools.numpy_l2norm2(x * y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # -------------------------- # Gradient checking # -------------------------- callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # -------------------------- # Construct model # -------------------------- num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with weights layers so that gradient checking will # verify that error signals are correct. x0_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input0_weights') x1_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input1_weights') x_slice = lbann.Slice(lbann.Input(data_field='samples'), slice_points=tools.str_list( [0, _m * _k, _m * _k + _k * _n])) x0 = lbann.Sum(x_slice, lbann.WeightsLayer(weights=x0_weights, dims=str(_m * _k))) x1 = lbann.Sum(x_slice, lbann.WeightsLayer(weights=x1_weights, dims=str(_k * _n))) x0_lbann = x0 x1_lbann = x1 # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # NN GEMM # ------------------------------------------ # LBANN implementation x0 = lbann.Reshape(x0_lbann, dims=tools.str_list([_m, _k])) x1 = lbann.Reshape(x1_lbann, dims=tools.str_list([_k, _n])) y = lbann.MatMul(x0, x1, data_layout='data_parallel') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='NN GEMM')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) x0 = x[:_m * _k].reshape([_m, _k]) x1 = x[_m * _k:].reshape([_k, _n]) y = np.matmul(x0, x1) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # TN GEMM # ------------------------------------------ # LBANN implementation x0 = lbann.Reshape(x0_lbann, dims=tools.str_list([_k, _m])) x1 = lbann.Reshape(x1_lbann, dims=tools.str_list([_k, _n])) y = lbann.MatMul(x0, x1, transpose_a=True, data_layout='data_parallel') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='TN GEMM')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) x0 = x[:_m * _k].reshape([_k, _m]) x1 = x[_m * _k:].reshape([_k, _n]) y = np.matmul(x0.transpose(), x1) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # NT GEMM # ------------------------------------------ # LBANN implementation x0 = lbann.Reshape(x0_lbann, dims=tools.str_list([_m, _k])) x1 = lbann.Reshape(x1_lbann, dims=tools.str_list([_n, _k])) y = lbann.MatMul(x0, x1, transpose_b=True, data_layout='data_parallel') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='NT GEMM')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) x0 = x[:_m * _k].reshape([_m, _k]) x1 = x[_m * _k:].reshape([_n, _k]) y = np.matmul(x0, x1.transpose()) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # TT GEMM # ------------------------------------------ # LBANN implementation x0 = lbann.Reshape(x0_lbann, dims=tools.str_list([_k, _m])) x1 = lbann.Reshape(x1_lbann, dims=tools.str_list([_n, _k])) y = lbann.MatMul(x0, x1, transpose_a=True, transpose_b=True, data_layout='data_parallel') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='TT GEMM')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) x0 = x[:_m * _k].reshape([_k, _m]) x1 = x[_m * _k:].reshape([_n, _k]) y = np.matmul(x0.transpose(), x1.transpose()) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x0_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum( lbann.Reshape(lbann.Input(), dims=tools.str_list(_sample_size)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_size))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Basic case # ------------------------------------------ # LBANN implementation x = x_lbann y = lbann.ErfInv(x) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='erf')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) y = scipy.special.erfinv(x) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ from lbann.modules.rnn import ChannelwiseGRU # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0), name='input') h_weights = lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0), name='inital_hidden') input_ = lbann.Input(data_field='samples') input_slice = lbann.Slice( input_, slice_points=tools.str_list( [0, _num_channels * _input_size, _sample_size]), ) x = lbann.Reshape(input_slice, dims=tools.str_list([_num_channels, _input_size]), name="input_reshape") x = lbann.Sum(x, lbann.WeightsLayer(weights=x_weights, dims=tools.str_list( [_num_channels, _input_size])), name="input_sum") h = lbann.Reshape(input_slice, dims=tools.str_list([_num_channels, _hidden_size]), name="hidden_reshape") h = lbann.Sum(h, lbann.WeightsLayer(weights=h_weights, dims=tools.str_list( [_num_channels, _hidden_size])), name="input_hidden_sum") x_lbann = x h_lbann = h # Objects for LBANN model obj = [] metrics = [] callbacks = [] # Weights rnn_weights_numpy = [] ih_matrix = np.random.uniform( low=-1, high=1, size=(3 * _hidden_size, _input_size), ) hh_matrix = np.random.uniform( low=-1, high=1, size=(3 * _hidden_size, _hidden_size), ) ih_bias = np.random.uniform(low=-1, high=1, size=(3 * _hidden_size, )) hh_bias = np.random.uniform(low=-1, high=1, size=(3 * _hidden_size, )) rnn_weights_numpy.extend([ih_matrix, ih_bias, hh_matrix, hh_bias]) rnn_weights_lbann = [ lbann.Weights(initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(w, order='F')))) for w in rnn_weights_numpy ] # LBANN implementation x = x_lbann h = h_lbann channelwise_GRU_cell = ChannelwiseGRU(num_channels=_num_channels, size=_hidden_size, weights=rnn_weights_lbann) y = channelwise_GRU_cell(x, h) z = lbann.L2Norm2(y) obj.append(z) metrics.append( lbann.Metric(z, name="Multi-channel, Unidirectional, GRU Cell")) # NumPy implementation vals = [] for i in range(num_samples()): input_ = get_sample(i).astype(np.float64) x = input_[:_num_channels * _input_size].reshape( (_num_channels, _input_size)) h = input_[_num_channels * _input_size:].reshape( (_num_channels, _hidden_size)) y = numpy_gru_cell(x, h, rnn_weights_numpy) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackPrintModelDescription()) callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data x = lbann.Identity(lbann.Input()) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # No padding index # ------------------------------------------ # Embeddings np.random.seed(20191015) embedding_dim = 5 embeddings = np.random.normal(size=(_num_embeddings, embedding_dim)) # LBANN implementation embedding_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(embeddings)))) x = x_lbann y = lbann.Embedding(x, weights=embedding_weights, num_embeddings=_num_embeddings, embedding_dim=embedding_dim) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='no padding index')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i) y = embeddings[x, :] z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Padding index 0 # ------------------------------------------ # Embeddings np.random.seed(201910152) embedding_dim = 7 padding_idx = 0 embeddings = np.random.normal(size=(_num_embeddings, embedding_dim)) # LBANN implementation # Note: Embedding layer gradients are not exact if a padding index # is set. Avoid gradient checking by not using an optimizer. embedding_weights = lbann.Weights( optimizer=None, initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(embeddings)))) x = x_lbann y = lbann.Embedding(x, weights=embedding_weights, num_embeddings=_num_embeddings, embedding_dim=embedding_dim, padding_idx=padding_idx) z = lbann.L2Norm2(y) metrics.append(lbann.Metric(z, name='padding index = 0')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i) y = np.where((x == padding_idx).reshape((-1, 1)), 0, embeddings[x, :]) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ # Construct model num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum( lbann.Reshape(lbann.Input(data_field='samples'), dims=tools.str_list(_sample_dims)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Pooling # ------------------------------------------ num_height_groups = tools.gpus_per_node(lbann) if num_height_groups == 0: e = 'this test requires GPUs.' print('Skip - ' + e) pytest.skip(e) pool_configs = [] # 3x3 pooling with same padding for mode, val in [ ("average", 700.1066377082393), # _num_samples=8 ("max", 1255.4813455546334), # _num_samples=8 # ("average", 830.2573008820838), # _num_samples=23 # ("max", 1167.667676299899), # _num_samples=23 ]: pool_configs.append({ "name": "3x3 {} pooling".format(mode), "kernel_dims": (3, 3), "strides": (1, 1), "pads": (0, 0), "pool_mode": mode, "val": val, }) # 2x2 strided pooling for mode, val in [ ("average", 263.76437243059104), # _num_samples=23 ("max", 358.66104389177207), # _num_samples=23 # ("average", 293.61402789516825), # _num_samples=23 # ("max", 351.4916288366334), # _num_samples=23 ]: pool_configs.append({ "name": "2x2 {} pooling".format(mode), "kernel_dims": (2, 2), "strides": (2, 2), "pads": (0, 0), "pool_mode": mode, "val": val, }) # 2x2x2 3D pooling for mode, val in [ ("average", 59.3851451701403), # _num_samples=8 ("max", 216.75871475407558), # _num_samples=8 # ("average", 89.61246528381926), # _num_samples=23 # ("max", 198.65624293856985), # _num_samples=23 ]: pool_configs.append({ "name": "2x2x2 {} pooling".format(mode), "kernel_dims": (2, 2, 2), "strides": (2, 2, 2), "pads": (0, 0, 0), "pool_mode": mode, "val": val, }) for p in pool_configs: # Apply pooling x = x_lbann if len(p["kernel_dims"]) == 3: x = lbann.Reshape(x, dims=tools.str_list(_sample_dims_3d)) y = lbann.Pooling( x, num_dims=len(p["kernel_dims"]), has_vectors=True, pool_dims=tools.str_list(p["kernel_dims"]), pool_strides=tools.str_list(p["strides"]), pool_pads=tools.str_list(p["pads"]), pool_mode=p["pool_mode"], parallel_strategy=create_parallel_strategy(num_height_groups)) z = lbann.L2Norm2(y) # Since max pooling is not differentiable, we only use average pooling. if p["pool_mode"] == "average": obj.append(z) metrics.append(lbann.Metric(z, name=p["name"])) # PyTorch implementation try: x = _samples if len(p["kernel_dims"]) == 3: x = np.reshape(x, [_num_samples] + _sample_dims_3d) y = pytorch_pooling( x, p["kernel_dims"], p["pool_mode"], stride=p["strides"], padding=p["pads"], ) z = tools.numpy_l2norm2(y) / _num_samples val = z except: # Precomputed value val = p["val"] tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum( lbann.Reshape(lbann.Input(), dims=tools.str_list(_sample_dims)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Data-parallel layout # ------------------------------------------ # LBANN implementation output_dims = (7, 4, 3) x = x_lbann y = lbann.Tessellate(x, dims=tools.str_list(output_dims), data_layout='data_parallel') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='data-parallel layout')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape(_sample_dims).astype(np.float64) y = np.tile(x, (3, 4, 1))[:7, :4, :3] z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Model-parallel layout # ------------------------------------------ # LBANN implementation output_dims = (2, 1, 9) x = x_lbann y = lbann.Tessellate(x, dims=tools.str_list(output_dims), data_layout='model_parallel') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='model-parallel layout')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape(_sample_dims).astype(np.float64) y = np.tile(x, (1, 1, 3))[:2, :1, :9] z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # -------------------------- # Gradient checking # -------------------------- callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # -------------------------- # Construct model # -------------------------- num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data x = lbann.Input(data_field='samples') x_lbann = x # Objects for LBANN model metrics = [] callbacks = [] # ------------------------------------------ # Data-parallel weights layer # ------------------------------------------ # Note: Weights are stored in one column of (STAR,STAR) # distributed matrix # Weights weights_values = np.random.normal(size=_sample_dims).astype(np.float32) weights_file = os.path.join(weights_dir, 'dataparallel_weights.npy') np.save(weights_file, weights_values) # LBANN implementation x = lbann.Reshape(x_lbann, dims=tools.str_list(_sample_dims)) weights = lbann.Weights( initializer=lbann.NumpyInitializer(file=weights_file), ) weights = lbann.WeightsLayer( weights=weights, dims=tools.str_list(_sample_dims), ) y = lbann.Multiply(x, weights) z = lbann.L2Norm2(y) metrics.append(lbann.Metric(z, name='data-parallel weights layer')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape(_sample_dims).astype(np.float64) y = x * weights_values z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Data-parallel FC layer # ------------------------------------------ # Note: Weights are stored in (STAR,STAR) distributed matrix # Weights output_size = 7 linearity = np.random.normal(size=(output_size, _sample_size)).astype(np.float32) linearity = linearity.astype(np.float64) bias = np.random.normal(size=output_size).astype(np.float32) linearity_file = os.path.join(weights_dir, 'dataparallel_fc_linearity.npy') bias_file = os.path.join(weights_dir, 'dataparallel_fc_bias.npy') np.save(linearity_file, linearity) np.save(bias_file, bias) # LBANN implementation x = x_lbann linearity_weights \ = lbann.Weights(initializer=lbann.NumpyInitializer(file=linearity_file)) bias_weights \ = lbann.Weights(initializer=lbann.NumpyInitializer(file=bias_file)) y = lbann.FullyConnected( x, weights=(linearity_weights, bias_weights), data_layout='data_parallel', num_neurons=output_size, has_bias=True, transpose=False) z = lbann.L2Norm2(y) metrics.append(lbann.Metric(z, name='data-parallel FC layer')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) y = np.matmul(linearity, x) + bias z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Model-parallel FC layer # ------------------------------------------ # Note: Weights are stored in (MC,MR) distributed matrix # Weights output_size = 9 linearity = np.random.normal(size=(output_size, _sample_size)).astype(np.float32) bias = np.random.normal(size=output_size).astype(np.float32) bias = bias.astype(np.float64) linearity_file = os.path.join(weights_dir, 'modelparallel_fc_linearity.npy') bias_file = os.path.join(weights_dir, 'modelparallel_fc_bias.npy') np.save(linearity_file, linearity) np.save(bias_file, bias) # LBANN implementation x = x_lbann linearity_weights \ = lbann.Weights(initializer=lbann.NumpyInitializer(file=linearity_file)) bias_weights \ = lbann.Weights(initializer=lbann.NumpyInitializer(file=bias_file)) y = lbann.FullyConnected( x, weights=(linearity_weights, bias_weights), data_layout='model_parallel', num_neurons=output_size, has_bias=True, transpose=False) z = lbann.L2Norm2(y) metrics.append(lbann.Metric(z, name='model-parallel FC layer')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) y = np.matmul(linearity, x) + bias z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with weights layers so that gradient checking will # verify that error signals are correct. slice_size = _samples.shape[-1] x0_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input0_weights') x1_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input1_weights') x_slice = lbann.Slice(lbann.Input(data_field='samples'), slice_points=tools.str_list([0, slice_size, 2*slice_size])) x0 = lbann.Sum(x_slice, lbann.WeightsLayer(weights=x0_weights, dims=str(slice_size))) x1 = lbann.Sum(x_slice, lbann.WeightsLayer(weights=x1_weights, dims=str(slice_size))) x0_lbann = x0 x1_lbann = x1 # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Data-parallel layout # ------------------------------------------ # LBANN implementation x0 = x0_lbann x1 = x1_lbann y = lbann.SigmoidBinaryCrossEntropy(x0, x1, data_layout='data_parallel') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='data-parallel layout')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) x0 = x[:slice_size] x1 = x[slice_size:] y = -x1 * np.log1p(np.exp(-x0)) - (1-x1) * np.log1p(np.exp(x0)) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Model-parallel layout # ------------------------------------------ # LBANN implementation x0 = x0_lbann x1 = x1_lbann y = lbann.SigmoidBinaryCrossEntropy(x0, x1, data_layout='model_parallel') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='model-parallel layout')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) x0 = x[:slice_size] x1 = x[slice_size:] y = -x1 * np.log1p(np.exp(-x0)) - (1-x1) * np.log1p(np.exp(x0)) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x0_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum(lbann.Reshape(lbann.Input(data_field='samples'), dims=tools.str_list(_sample_dims)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Data-parallel layout # ------------------------------------------ # LBANN implementation x = x_lbann y = lbann.Identity(x, data_layout='data_parallel') slice_points = (0, 4, 8) x_slice = lbann.Slice(x, axis=2, slice_points=tools.str_list(slice_points),parallel_strategy = {'sub_branch_tag':0,'enable_subgraph':True}) branch1 = lbann.Identity(x_slice, data_layout='data_parallel',parallel_strategy = {'sub_branch_tag':1,'enable_subgraph':True}) branch2 = lbann.Identity(x_slice, data_layout='data_parallel',parallel_strategy = {'sub_branch_tag':2,'enable_subgraph':True}) grid_sum = lbann.Cross_Grid_Sum([branch1,branch2],parallel_strategy = {'sub_branch_tag':0,'enable_subgraph':True}) branch1 = lbann.Identity(grid_sum) branch2 = lbann.Identity(grid_sum) sum_branch = lbann.Sum([branch1,branch2],parallel_strategy = {'sub_branch_tag':0,'enable_subgraph':True}) z = lbann.L2Norm2(sum_branch) obj.append(z) metrics.append(lbann.Metric(z, name='data-parallel layout')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape(_sample_dims).astype(np.float64) y = [] cross_sum = 0 for j in range(len(slice_points)-1): x_slice = x[:,:,slice_points[j]:slice_points[j+1]] if(j==0): cross_sum = x_slice else: cross_sum += x_slice z = 2*cross_sum z = tools.numpy_l2norm2(z) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs,subgraph_communication=lbann.SubgraphCommunication.COLL_OPT, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum( lbann.Reshape(lbann.Input(data_field='samples'), dims=tools.str_list(_sample_dims)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Basic 3^n convolution # ------------------------------------------ # 3^n conv, stride=1, pad=1, dilation=1, bias num_height_groups = tools.gpus_per_node(lbann) if num_height_groups == 0: e = 'this test requires GPUs.' print('Skip - ' + e) pytest.skip(e) for num_dims, reference_val in [(2, 11913.852660080756), (3, 9952.365297083174)]: # Convolution settings kernel_dims = [ 5, _sample_dims[0] if num_dims == 2 else _sample_dims_3d[0], ] + [3] * num_dims strides = [1] * num_dims pads = [1] * num_dims dilations = [1] * num_dims kernel = make_random_array(kernel_dims, 11) # Apply convolution kernel_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(kernel))), name='kernel1_{}d'.format(num_dims)) x = x_lbann if num_dims == 3: x = lbann.Reshape(x, dims=tools.str_list(_sample_dims_3d)) y = lbann.Convolution( x, weights=(kernel_weights, ), num_dims=num_dims, num_output_channels=kernel_dims[0], has_vectors=True, conv_dims=tools.str_list(kernel_dims[2:]), conv_strides=tools.str_list(strides), conv_pads=tools.str_list(pads), conv_dilations=tools.str_list(dilations), has_bias=False, parallel_strategy=create_parallel_strategy(num_height_groups)) z = lbann.L2Norm2(y) obj.append(z) metrics.append( lbann.Metric(z, name='basic {}D 3^n convolution'.format(num_dims))) # PyTorch implementation try: x = _samples if num_dims == 3: x = np.reshape(x, [_num_samples] + _sample_dims_3d) y = pytorch_convolution(x, kernel, stride=strides, padding=pads, dilation=dilations) z = tools.numpy_l2norm2(y) / _num_samples val = z except: # Precomputed value val = reference_val # val = 398.6956458317758 # _num_samples=8, 6 channels # val = 381.7401227915947 # _num_samples=23, 6 channels tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x0 = lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims)) x1 = lbann.Reshape(lbann.Input(data_field='samples'), dims=tools.str_list(_sample_dims), name="Input_layer") x = lbann.Sum(x0, x1, name="Adding_weight_layer") x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] num_channel_groups = tools.gpus_per_node(lbann) if num_channel_groups == 0: e = 'this test requires GPUs.' print('Skip - ' + e) pytest.skip(e) # ------------------------------------------ # Compute expected metric values with NumPy # ------------------------------------------ # Input and output dimensions input_channel_dims = _sample_dims[1:] output_channel_dims = (1, 3) input_channel_size = functools.reduce(operator.mul, input_channel_dims) output_channel_size = functools.reduce(operator.mul, output_channel_dims) # Weight values linearity = np.random.normal(size=(output_channel_size, input_channel_size)).astype(np.float32) bias = np.random.normal(size=(output_channel_size, 1)).astype(np.float32) # With bias x = (_samples.reshape( (-1, input_channel_size)).transpose().astype(np.float64)) y = np.matmul(linearity.astype(np.float64), x) + bias.astype(np.float64) z = tools.numpy_l2norm2(y) / _num_samples val_with_bias = z # Without bias x = (_samples.reshape( (-1, input_channel_size)).transpose().astype(np.float64)) y = np.matmul(linearity.astype(np.float64), x) z = tools.numpy_l2norm2(y) / _num_samples val_without_bias = z # ------------------------------------------ # Non-transpose, bias # ------------------------------------------ # LBANN implementation linearity_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(linearity, order='F')))) bias_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(bias)))) x = x_lbann y = lbann.ChannelwiseFullyConnected( x, weights=(linearity_weights, bias_weights), output_channel_dims=output_channel_dims, parallel_strategy=create_parallel_strategy(num_channel_groups), name="bias") z = lbann.L2Norm2(y, name="L2Norm") obj.append(z) metrics.append(lbann.Metric(z, name='non-transpose, bias')) # NumPy implementation tol = 8 * val_with_bias * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val_with_bias - tol, upper_bound=val_with_bias + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Non-transpose, no bias # ------------------------------------------ # LBANN implementation linearity_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(linearity, order='F')))) x = x_lbann y = lbann.ChannelwiseFullyConnected( x, weights=(linearity_weights), output_channel_dims=output_channel_dims, parallel_strategy=create_parallel_strategy(num_channel_groups), name="no_bias") z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='non-transpose, no bias')) # NumPy implementation tol = 8 * val_without_bias * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val_without_bias - tol, upper_bound=val_without_bias + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Transpose, bias # ------------------------------------------ # LBANN implementation linearity_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(linearity, order='C')))) bias_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(bias)))) x = x_lbann y = lbann.ChannelwiseFullyConnected( x, weights=(linearity_weights, bias_weights), output_channel_dims=output_channel_dims, parallel_strategy=create_parallel_strategy(num_channel_groups), transpose=True, ) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='transpose, bias')) # NumPy implementation tol = 8 * val_with_bias * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val_with_bias - tol, upper_bound=val_with_bias + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Transpose, no bias # ------------------------------------------ # LBANN implementation linearity_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(linearity, order='C')))) x = x_lbann y = lbann.ChannelwiseFullyConnected( x, weights=(linearity_weights), output_channel_dims=output_channel_dims, parallel_strategy=create_parallel_strategy(num_channel_groups), bias=False, transpose=True) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='Non-transpose, no bias')) # NumPy implementation tol = 8 * val_without_bias * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val_without_bias - tol, upper_bound=val_without_bias + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 1 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x0 = lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims)) x1 = lbann.Reshape(lbann.Input(data_field='samples'), dims=tools.str_list(_sample_dims)) x = lbann.Sum(x0, x1) # Apply channel-wise scale/bias scale_values = tools.str_list(np.nditer(_scale)) bias_values = tools.str_list(np.nditer(_bias)) scalebias_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer(values='{} {}'.format(scale_values, bias_values)), name='scalebias_weights' ) y = lbann.ChannelwiseScaleBias(x, weights=scalebias_weights) z = lbann.L2Norm2(y) # Objects for LBANN model obj = z metric = lbann.Metric(z, name='obj') layers = list(lbann.traverse_layer_graph(z)) callbacks = [] # Get expected metric value from NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape(_sample_dims).astype(np.float64) y = _scale.astype(np.float64) * x + _bias.astype(np.float64) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metric.name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # Gradient checking callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # Construct model num_epochs = 0 return lbann.Model(num_epochs, layers=layers, objective_function=obj, metrics=metric, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum(lbann.Reshape(lbann.Input(), dims=tools.str_list(_sample_size)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_size))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Data-parallel layout # ------------------------------------------ num_height_groups = tools.gpus_per_node(lbann) if num_height_groups == 0: e = 'this test requires GPUs.' print('Skip - ' + e) pytest.skip(e) # LBANN implementation x = x_lbann x = lbann.Reshape(x, dims="4 2 6") y = lbann.LeakyRelu(x, negative_slope=0.01, data_layout='data_parallel', parallel_strategy=create_parallel_strategy( num_height_groups)) y = lbann.Reshape(y, dims=str(sample_dims())) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='data-parallel layout')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) y = np.where(x > 0, x, 0.01*x) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Model-parallel layout # ------------------------------------------ # LBANN implementation x = x_lbann x = lbann.Reshape(x, dims="4 2 6") y = lbann.LeakyRelu(x, negative_slope=2, data_layout='model_parallel', parallel_strategy=create_parallel_strategy( num_height_groups)) y = lbann.Reshape(y, dims=str(sample_dims())) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='model-parallel layout')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) y = np.where(x > 0, x, 2*x) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum(lbann.Reshape(lbann.Input(), dims=tools.str_list(_sample_dims)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Basic 3x3 convolution # ------------------------------------------ # 3x3 conv, stride=1, pad=1, dilation=1, bias # Convolution settings kernel_dims = (5, _sample_dims[0], 3, 3) strides = (1, 1) pads = (1, 1) dilations = (1, 1) kernel = make_random_array(kernel_dims, 11) bias = make_random_array([kernel_dims[0]], 123) # Apply convolution kernel_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer(values=tools.str_list(np.nditer(kernel))), name='kernel1' ) bias_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer(values=tools.str_list(np.nditer(bias))), name='bias1' ) x = x_lbann y = lbann.Convolution(x, weights=(kernel_weights, bias_weights), num_dims=3, num_output_channels=kernel_dims[0], has_vectors=True, conv_dims=tools.str_list(kernel_dims[2:]), conv_strides=tools.str_list(strides), conv_pads=tools.str_list(pads), conv_dilations=tools.str_list(dilations), has_bias=True, parallel_strategy=create_parallel_strategy(4)) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='basic 3x3 convolution')) # PyTorch implementation try: x = _samples y = pytorch_convolution( x, kernel, bias=bias, stride=strides, padding=pads, dilation=dilations ) z = tools.numpy_l2norm2(y) / _num_samples val = z except: # Precomputed value val = 153.84937996554953 tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # 2x4 strided convolution # ------------------------------------------ # Convolution settings kernel_dims = (3, _sample_dims[0], 2, 4) strides = (3, 1) pads = (3, 0) dilations = (1, 1) num_groups = 1 kernel = make_random_array(kernel_dims, 19) # Apply convolution kernel_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer(values=tools.str_list(np.nditer(kernel))), name='kernel2' ) x = x_lbann y = lbann.Convolution(x, weights=(kernel_weights), num_dims=3, num_output_channels=kernel_dims[0], has_vectors=True, conv_dims=tools.str_list(kernel_dims[2:]), conv_strides=tools.str_list(strides), conv_pads=tools.str_list(pads), conv_dilations=tools.str_list(dilations), num_groups=num_groups, has_bias=False, parallel_strategy=create_parallel_strategy(4)) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='2x4 convolution')) # PyTorch implementation try: x = _samples y = pytorch_convolution( x, kernel, bias=None, stride=strides, padding=pads, dilation=dilations, groups=num_groups ) z = tools.numpy_l2norm2(y) / _num_samples val = z except: # Precomputed value val = 19.24587403346207 tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum(lbann.Reshape(lbann.Input(data_field='samples'), dims=tools.str_list(_sample_dims)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # -------------------------- # Slice along axis 0 # -------------------------- # LBANN implementation slice_points = (2, 3, 6, 7) x = x_lbann x_slice = lbann.Slice(x, axis=0, slice_points=tools.str_list(slice_points)) y = [] for _ in range(len(slice_points)-1): y.append(lbann.L2Norm2(x_slice)) z = lbann.Add(y[0], y[2]) obj.append(z) metrics.append(lbann.Metric(z, name='axis0')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape(_sample_dims).astype(np.float64) y = [] for j in range(len(slice_points)-1): x_slice = x[slice_points[j]:slice_points[j+1],:,:] y.append(tools.numpy_l2norm2(x_slice)) z = y[0] + y[2] vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # -------------------------- # Slice along axis 1 # -------------------------- # LBANN implementation slice_points = (0, 2, 3, 4) x = x_lbann x_slice = lbann.Slice(x, axis=1, slice_points=tools.str_list(slice_points)) y = [] for _ in range(len(slice_points)-1): y.append(lbann.L2Norm2(x_slice)) z = lbann.Add(y[0], y[2]) obj.append(z) metrics.append(lbann.Metric(z, name='axis1')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape(_sample_dims).astype(np.float64) y = [] for j in range(len(slice_points)-1): x_slice = x[:,slice_points[j]:slice_points[j+1],:] y.append(tools.numpy_l2norm2(x_slice)) z = y[0] + y[2] vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # -------------------------- # Slice along axis 2 # -------------------------- # LBANN implementation slice_points = (1, 3) x = x_lbann x_slice = lbann.Slice(x, axis=2, slice_points=tools.str_list(slice_points)) y = [] for _ in range(len(slice_points)-1): y.append(lbann.L2Norm2(x_slice)) z = y[0] obj.append(z) metrics.append(lbann.Metric(z, name='axis2')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape(_sample_dims).astype(np.float64) y = [] for j in range(len(slice_points)-1): x_slice = x[:,:,slice_points[j]:slice_points[j+1]] y.append(tools.numpy_l2norm2(x_slice)) z = y[0] vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # -------------------------- # Model-parallel # -------------------------- # LBANN implementation slice_points = (31, 54, 56, 57) x = lbann.Reshape(x_lbann, dims=tools.str_list([105])) x_slice = lbann.Slice(x, slice_points=tools.str_list(slice_points), data_layout='model_parallel') y = [] for _ in range(len(slice_points)-1): y.append(lbann.L2Norm2(x_slice)) z = lbann.Add(y[0], y[2]) obj.append(z) metrics.append(lbann.Metric(z, name='model-parallel')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape(-1).astype(np.float64) y = [] for j in range(len(slice_points)-1): x_slice = x[slice_points[j]:slice_points[j+1]] y.append(tools.numpy_l2norm2(x_slice)) z = y[0] + y[2] vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # -------------------------- # Gradient checking # -------------------------- callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # -------------------------- # Construct model # -------------------------- num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum( lbann.Reshape(lbann.Input(data_field='samples'), dims=tools.str_list(_input_size)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_input_size))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Compute expected metric values with NumPy # ------------------------------------------ # Weight values linearity = np.random.normal(size=(_output_size, _input_size)).astype(np.float32) bias = np.random.normal(size=(_output_size, 1)).astype(np.float32) # With bias x = _samples.transpose().astype(np.float64) y = np.matmul(linearity.astype(np.float64), x) + bias.astype(np.float64) z = tools.numpy_l2norm2(y) / _num_samples val_with_bias = z # Without bias x = _samples.transpose().astype(np.float64) y = np.matmul(linearity.astype(np.float64), x) z = tools.numpy_l2norm2(y) / _num_samples val_without_bias = z # ------------------------------------------ # Data-parallel layout, non-transpose, bias # ------------------------------------------ # LBANN implementation linearity_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(linearity, order='F')))) bias_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(bias)))) x = x_lbann y = lbann.FullyConnected(x, weights=(linearity_weights, bias_weights), data_layout='data_parallel', num_neurons=_output_size, has_bias=True, transpose=False) z = lbann.L2Norm2(y) obj.append(z) metrics.append( lbann.Metric(z, name='data-parallel layout, non-transpose, bias')) # NumPy implementation val = val_with_bias tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Model-parallel layout, non-transpose, bias # ------------------------------------------ # LBANN implementation linearity_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(linearity, order='F')))) bias_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(bias)))) x = x_lbann y = lbann.FullyConnected(x, weights=(linearity_weights, bias_weights), data_layout='model_parallel', num_neurons=_output_size, has_bias=True, transpose=False) z = lbann.L2Norm2(y) obj.append(z) metrics.append( lbann.Metric(z, name='model-parallel layout, non-transpose, bias')) # NumPy implementation val = val_with_bias tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Data-parallel layout, transpose, no bias # ------------------------------------------ # LBANN implementation linearity_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(linearity, order='C')))) x = x_lbann y = lbann.FullyConnected(x, weights=linearity_weights, data_layout='data_parallel', num_neurons=_output_size, has_bias=False, transpose=True) z = lbann.L2Norm2(y) obj.append(z) metrics.append( lbann.Metric(z, name='data-parallel layout, transpose, no bias')) # NumPy implementation val = val_without_bias tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Model-parallel layout, transpose, no bias # ------------------------------------------ # LBANN implementation linearity_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(linearity, order='C')))) x = x_lbann y = lbann.FullyConnected(x, weights=linearity_weights, data_layout='model_parallel', num_neurons=_output_size, has_bias=False, transpose=True) z = lbann.L2Norm2(y) obj.append(z) metrics.append( lbann.Metric(z, name='data-parallel layout, transpose, no bias')) # NumPy implementation val = val_without_bias tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)