def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum( lbann.Reshape(lbann.Input(), dims=tools.str_list(_sample_dims)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Data-parallel layout # ------------------------------------------ # LBANN implementation x = x_lbann y = lbann.DFTAbs(x) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='data-parallel layout')) callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Layer graph weight = lbann.Weights(initializer=lbann.UniformInitializer(min=0, max=1)) weight = lbann.WeightsLayer(weights=weight, dims=tools.str_list([1])) rand = lbann.Input(data_field='samples') layers = list(lbann.traverse_layer_graph([weight, rand])) for l in layers: l.device = 'CPU' # Model objects metrics = [ lbann.Metric(weight, name='weight'), lbann.Metric(rand, name='random'), ] callbacks = [ lbann.CallbackPrint(), ] # Construct model return lbann.Model(_num_epochs, layers=layers, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Multiply with a weights layer so that gradient checking # will verify that error signals are correct. We multiply instead # of adding so that each batch sample contributes a different # gradient. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=1.0), name='input_weights') x = lbann.Multiply( lbann.Input(data_field='samples'), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_size)), ) # Compute variance along batch dimension sum_x = lbann.BatchwiseReduceSum(x) sum_x2 = lbann.BatchwiseReduceSum(lbann.Square(x)) mini_batch_size = lbann.Tessellate(lbann.MiniBatchSize(), hint_layer=x) mean_x = lbann.Divide(sum_x, mini_batch_size) mean_x2 = lbann.Divide(sum_x2, mini_batch_size) var = lbann.Subtract(mean_x2, lbann.Square(mean_x)) obj = lbann.L2Norm2(var) # Objects for LBANN model layers = list(lbann.traverse_layer_graph(x)) metric = lbann.Metric(obj, name='obj') obj = lbann.ObjectiveFunction(obj) callbacks = [] # Compute expected metric value var = np.var(_samples, axis=0) val = tools.numpy_l2norm2(var) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metric.name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # Gradient checking callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # Construct model num_epochs = 0 return lbann.Model(num_epochs, layers=layers, objective_function=obj, metrics=[metric], callbacks=callbacks)
def create_bn_weights(layer_name, num_channels): weights_ary = [] for i in range(4): val = make_random_array((num_channels, ), 15 + i) weights_ary.append( lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(val))), name='{}_{}'.format(layer_name, i))) return weights_ary
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Convenience function to compute L2 norm squared with NumPy def l2_norm2(x): x = x.reshape(-1) return np.inner(x, x) # LBANN implementation x = lbann.Reshape(lbann.Input(data_field='samples'), dims=tools.str_list(_sample_dims)) y = lbann.Argmax(x, device='cpu') z = lbann.L2Norm2(y) # Objects for LBANN model obj = z metric = lbann.Metric(z, name='obj') layers = list(lbann.traverse_layer_graph(z)) callbacks = [] # Get expected metric value from NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape(_sample_dims).astype(np.float64) y = np.argmax(x) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metric.name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # Construct model num_epochs = 0 return lbann.Model(num_epochs, layers=layers, objective_function=obj, metrics=metric, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data x = lbann.Input(data_field='samples') x_lbann = x # Objects for LBANN model metrics = [] callbacks = [] # ------------------------------------------ # Data-parallel weights layer # ------------------------------------------ # Note: Weights are stored in one column of (STAR,STAR) # distributed matrix # Weights weights_values = np.random.normal(size=_sample_dims).astype(np.float32) weights_file = os.path.join(weights_dir, 'dataparallel_weights.npy') np.save(weights_file, weights_values) # LBANN implementation x = lbann.Reshape(x_lbann, dims=tools.str_list(_sample_dims)) weights = lbann.Weights( initializer=lbann.NumpyInitializer(file=weights_file), ) weights = lbann.WeightsLayer( weights=weights, dims=tools.str_list(_sample_dims), ) y = lbann.Multiply(x, weights) z = lbann.L2Norm2(y) metrics.append(lbann.Metric(z, name='data-parallel weights layer')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape(_sample_dims).astype(np.float64) y = x * weights_values z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Data-parallel FC layer # ------------------------------------------ # Note: Weights are stored in (STAR,STAR) distributed matrix # Weights output_size = 7 linearity = np.random.normal(size=(output_size, _sample_size)).astype(np.float32) linearity = linearity.astype(np.float64) bias = np.random.normal(size=output_size).astype(np.float32) linearity_file = os.path.join(weights_dir, 'dataparallel_fc_linearity.npy') bias_file = os.path.join(weights_dir, 'dataparallel_fc_bias.npy') np.save(linearity_file, linearity) np.save(bias_file, bias) # LBANN implementation x = x_lbann linearity_weights \ = lbann.Weights(initializer=lbann.NumpyInitializer(file=linearity_file)) bias_weights \ = lbann.Weights(initializer=lbann.NumpyInitializer(file=bias_file)) y = lbann.FullyConnected( x, weights=(linearity_weights, bias_weights), data_layout='data_parallel', num_neurons=output_size, has_bias=True, transpose=False) z = lbann.L2Norm2(y) metrics.append(lbann.Metric(z, name='data-parallel FC layer')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) y = np.matmul(linearity, x) + bias z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Model-parallel FC layer # ------------------------------------------ # Note: Weights are stored in (MC,MR) distributed matrix # Weights output_size = 9 linearity = np.random.normal(size=(output_size, _sample_size)).astype(np.float32) bias = np.random.normal(size=output_size).astype(np.float32) bias = bias.astype(np.float64) linearity_file = os.path.join(weights_dir, 'modelparallel_fc_linearity.npy') bias_file = os.path.join(weights_dir, 'modelparallel_fc_bias.npy') np.save(linearity_file, linearity) np.save(bias_file, bias) # LBANN implementation x = x_lbann linearity_weights \ = lbann.Weights(initializer=lbann.NumpyInitializer(file=linearity_file)) bias_weights \ = lbann.Weights(initializer=lbann.NumpyInitializer(file=bias_file)) y = lbann.FullyConnected( x, weights=(linearity_weights, bias_weights), data_layout='model_parallel', num_neurons=output_size, has_bias=True, transpose=False) z = lbann.L2Norm2(y) metrics.append(lbann.Metric(z, name='model-parallel FC layer')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) y = np.matmul(linearity, x) + bias z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum( lbann.Reshape(lbann.Input(data_field='samples'), dims=tools.str_list(_sample_dims)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] branches = 2 # ------------------------------------------ # Data-parallel layout # ------------------------------------------ # LBANN implementation x = x_lbann y = lbann.Identity(x, data_layout='data_parallel') slice_points = (0, 8, 16) x_slice = lbann.Slice(x, axis=2, slice_points=tools.str_list(slice_points), parallel_strategy={ 'sub_branch_tag': 0, 'enable_subgraph': True }) branch1 = lbann.Identity(x_slice, data_layout='data_parallel', parallel_strategy={ 'sub_branch_tag': 1, 'enable_subgraph': True }) branch2 = lbann.Identity(x_slice, data_layout='data_parallel', parallel_strategy={ 'sub_branch_tag': 2, 'enable_subgraph': True }) grid_slice = lbann.Cross_Grid_Sum_Slice([branch1, branch2]) branch1 = lbann.Identity(grid_slice) branch2 = lbann.Identity(grid_slice) sum_branch = lbann.Sum([branch1, branch2], parallel_strategy={ 'sub_branch_tag': 0, 'enable_subgraph': True }) z = lbann.L2Norm2(sum_branch) obj.append(z) metrics.append(lbann.Metric(z, name='data-parallel layout')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape(_sample_dims).astype(np.float64) y = [] cross_sum = 0 for j in range(len(slice_points) - 1): x_slice = x[:, :, slice_points[j]:slice_points[j + 1]] if (j == 0): cross_sum = x_slice else: cross_sum += x_slice last_dim = int(_sample_dims[-1] / (branches * branches)) sum_slices = None for j in range(branches): if (j == 0): sum_slices = cross_sum[:, :, :last_dim] else: sum_slices += cross_sum[:, :, j * last_dim:(j + 1) * last_dim] z = tools.numpy_l2norm2(sum_slices) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model( num_epochs, subgraph_communication=lbann.SubgraphCommunication.COLL_OPT, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum(lbann.Reshape(lbann.Input(data_field='samples'), dims=tools.str_list(_sample_size)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_size))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Data-parallel layout # ------------------------------------------ num_height_groups = tools.gpus_per_node(lbann) if num_height_groups == 0: e = 'this test requires GPUs.' print('Skip - ' + e) pytest.skip(e) # LBANN implementation x = x_lbann x = lbann.Reshape(x, dims="4 2 6") y = lbann.LeakyRelu(x, negative_slope=0.01, data_layout='data_parallel', parallel_strategy=create_parallel_strategy( num_height_groups)) y = lbann.Reshape(y, dims=str(sample_dims())) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='data-parallel layout')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) y = np.where(x > 0, x, 0.01*x) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Model-parallel layout # ------------------------------------------ # LBANN implementation x = x_lbann x = lbann.Reshape(x, dims="4 2 6") y = lbann.LeakyRelu(x, negative_slope=2, data_layout='model_parallel', parallel_strategy=create_parallel_strategy( num_height_groups)) y = lbann.Reshape(y, dims=str(sample_dims())) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='model-parallel layout')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) y = np.where(x > 0, x, 2*x) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x0 = lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims)) x1 = lbann.Reshape(lbann.Input(), dims=tools.str_list(_sample_dims)) x = lbann.Sum(x0, x1) # Apply channel-wise scale/bias scale_values = tools.str_list(np.nditer(_scale)) bias_values = tools.str_list(np.nditer(_bias)) scalebias_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer(values='{} {}'.format(scale_values, bias_values)), name='scalebias_weights' ) y = lbann.ChannelwiseScaleBias(x, weights=scalebias_weights) z = lbann.L2Norm2(y) # Objects for LBANN model obj = z metric = lbann.Metric(z, name='obj') layers = list(lbann.traverse_layer_graph(z)) callbacks = [] # Get expected metric value from NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape(_sample_dims).astype(np.float64) y = _scale.astype(np.float64) * x + _bias.astype(np.float64) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metric.name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # Gradient checking callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # Construct model num_epochs = 0 return lbann.Model(num_epochs, layers=layers, objective_function=obj, metrics=metric, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum(lbann.Reshape(lbann.Input(data_field='samples'), dims=tools.str_list(_sample_size)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_size))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Data-parallel layout # ------------------------------------------ # LBANN implementation x = x_lbann y = lbann.Identity(x, data_layout='data_parallel') branch1 = lbann.Identity(y, data_layout='data_parallel',parallel_strategy = {'sub_branch_tag':1,'enable_subgraph':True}) branch2 = lbann.Identity(y, data_layout='data_parallel',parallel_strategy = {'sub_branch_tag':2,'enable_subgraph':True}) sum_branch = lbann.Sum([branch1,branch2],parallel_strategy = {'sub_branch_tag':0,'enable_subgraph':True}) z = lbann.L2Norm2(sum_branch) obj.append(z) metrics.append(lbann.Metric(z, name='data-parallel layout')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) y = x z = tools.numpy_l2norm2(2*y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs,subgraph_communication=lbann.SubgraphCommunication.COLL_OPT, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x = lbann.Input(data_field='samples') x_slice = lbann.Slice( x, slice_points=tools.str_list([0, input_size, input_size + output_size]), ) x0_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights', ) x0 = lbann.Sum( lbann.Identity(x_slice), lbann.WeightsLayer(weights=x0_weights, dims=tools.str_list(input_size)), ) x1 = lbann.Identity(x_slice) x0_lbann = x0 x1_lbann = x1 # Apply gather y0 = lbann.Gather(x0, x1) y1 = lbann.Concatenation([ lbann.Constant(value=i + 1, num_neurons='1') for i in range(output_size) ]) y = lbann.Multiply(y0, y1) z = lbann.L2Norm2(y) # Objects for LBANN model metrics = [] callbacks = [] objs = [] metrics.append(lbann.Metric(z, name='1D_obj')) objs.append(lbann.ObjectiveFunction(z)) # Compute expected metric value vals = [] for i in range(num_samples()): sample = get_sample(i) x0 = sample[:input_size] x1 = sample[input_size:] y0 = np.zeros(output_size) for i in range(output_size): if 0 <= x1[i] < input_size: y0[i] = x0[int(x1[i])] z = 0 for i in range(output_size): z += ((i + 1) * y0[i])**2 vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) ###################################################################### # # 2D Values , 1D Input, Axis = 0 # ###################################################################### x0 = lbann.Reshape(x0_lbann, dims=tools.str_list([num_rows, num_columns])) x1 = lbann.Identity(x1_lbann, name="indices_2D_axis_0") y0 = lbann.Gather(x0, x1, name="Gather_2D_axis_0", axis=0) y1 = lbann.Concatenation([ lbann.Constant(value=i + 1, num_neurons='1') for i in range(num_columns * output_size) ]) y0 = lbann.Reshape(y0, dims=tools.str_list([num_columns * output_size])) y1 = lbann.Reshape(y1, dims=tools.str_list([num_columns * output_size])) y = lbann.Multiply(y0, y1) z = lbann.L2Norm2(y) objs.append(z) metrics.append(lbann.Metric(z, name="2D_obj_axis_0")) vals = [] for i in range(num_samples()): sample = get_sample(i) x0 = np.array(sample[:input_size]).reshape((num_rows, num_columns)) x1 = sample[input_size:input_size + output_size] y0 = np.zeros((output_size, num_columns)) for i in range(output_size): if 0 <= x1[i] <= num_rows: for j in range(num_columns): y0[i][j] = x0[int(x1[i])][j] z = 0 for i in range(num_columns * output_size): z += ((i + 1) * y0.flatten()[i])**2 vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) ###################################################################### # # 2D Values , 1D Input, Axis = 1 # ###################################################################### x0 = lbann.Reshape(x0_lbann, dims=tools.str_list([num_rows, num_columns])) x1 = lbann.Identity(x1_lbann, name="Indices_2D") y0 = lbann.Gather(x0, x1, name="Gather_2D", axis=1) y1 = lbann.Concatenation([ lbann.Constant(value=i + 1, num_neurons='1') for i in range(num_rows * output_size) ]) y0 = lbann.Reshape(y0, dims=tools.str_list([num_rows * output_size])) y1 = lbann.Reshape(y1, dims=tools.str_list([num_rows * output_size])) y = lbann.Multiply(y0, y1) z = lbann.L2Norm2(y) objs.append(z) metrics.append(lbann.Metric(z, name="2D_obj_axis_1")) vals = [] for i in range(num_samples()): sample = get_sample(i) x0 = np.array(sample[:input_size]).reshape((num_rows, num_columns)) x1 = sample[input_size:input_size + output_size] y0 = np.zeros((num_rows, output_size)) for i in range(num_rows): for j in range(output_size): if 0 <= x1[j] <= num_columns: y0[i][j] = x0[i][int(x1[j])] z = 0 for i in range(num_rows * output_size): z += ((i + 1) * y0.flatten()[i])**2 vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # Gradient checking callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # Construct model num_epochs = 0 layers = list(lbann.traverse_layer_graph(x)) return lbann.Model(num_epochs, layers=layers, objective_function=objs, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum( lbann.Reshape(lbann.Input(), dims=tools.str_list(_sample_dims)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Pooling # ------------------------------------------ num_height_groups = tools.gpus_per_node(lbann) if num_height_groups == 0: e = 'this test requires GPUs.' print('Skip - ' + e) pytest.skip(e) pool_configs = [] # 3x3 pooling with same padding for mode, val in [ ("average", 700.1066377082393), # _num_samples=8 ("max", 1255.4813455546334), # _num_samples=8 # ("average", 830.2573008820838), # _num_samples=23 # ("max", 1167.667676299899), # _num_samples=23 ]: pool_configs.append({ "name": "3x3 {} pooling".format(mode), "kernel_dims": (3, 3), "strides": (1, 1), "pads": (0, 0), "pool_mode": mode, "val": val, }) # 2x2 strided pooling for mode, val in [ ("average", 263.76437243059104), # _num_samples=23 ("max", 358.66104389177207), # _num_samples=23 # ("average", 293.61402789516825), # _num_samples=23 # ("max", 351.4916288366334), # _num_samples=23 ]: pool_configs.append({ "name": "2x2 {} pooling".format(mode), "kernel_dims": (2, 2), "strides": (2, 2), "pads": (0, 0), "pool_mode": mode, "val": val, }) # 2x2x2 3D pooling for mode, val in [ ("average", 59.3851451701403), # _num_samples=8 ("max", 216.75871475407558), # _num_samples=8 # ("average", 89.61246528381926), # _num_samples=23 # ("max", 198.65624293856985), # _num_samples=23 ]: pool_configs.append({ "name": "2x2x2 {} pooling".format(mode), "kernel_dims": (2, 2, 2), "strides": (2, 2, 2), "pads": (0, 0, 0), "pool_mode": mode, "val": val, }) for p in pool_configs: # Apply pooling x = x_lbann if len(p["kernel_dims"]) == 3: x = lbann.Reshape(x, dims=tools.str_list(_sample_dims_3d)) y = lbann.Pooling( x, num_dims=len(p["kernel_dims"]), has_vectors=True, pool_dims=tools.str_list(p["kernel_dims"]), pool_strides=tools.str_list(p["strides"]), pool_pads=tools.str_list(p["pads"]), pool_mode=p["pool_mode"], parallel_strategy=create_parallel_strategy(num_height_groups)) z = lbann.L2Norm2(y) # Since max pooling is not differentiable, we only use average pooling. if p["pool_mode"] == "average": obj.append(z) metrics.append(lbann.Metric(z, name=p["name"])) # PyTorch implementation try: x = _samples if len(p["kernel_dims"]) == 3: x = np.reshape(x, [_num_samples] + _sample_dims_3d) y = pytorch_pooling( x, p["kernel_dims"], p["pool_mode"], stride=p["strides"], padding=p["pads"], ) z = tools.numpy_l2norm2(y) / _num_samples val = z except: # Precomputed value val = p["val"] tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum(lbann.Reshape(lbann.Input(data_field='samples'), dims=tools.str_list(_sample_dims)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # -------------------------- # Slice along axis 0 # -------------------------- # LBANN implementation slice_points = (2, 3, 6, 7) x = x_lbann x_slice = lbann.Slice(x, axis=0, slice_points=tools.str_list(slice_points)) y = [] for _ in range(len(slice_points)-1): y.append(lbann.L2Norm2(x_slice)) z = lbann.Add(y[0], y[2]) obj.append(z) metrics.append(lbann.Metric(z, name='axis0')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape(_sample_dims).astype(np.float64) y = [] for j in range(len(slice_points)-1): x_slice = x[slice_points[j]:slice_points[j+1],:,:] y.append(tools.numpy_l2norm2(x_slice)) z = y[0] + y[2] vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # -------------------------- # Slice along axis 1 # -------------------------- # LBANN implementation slice_points = (0, 2, 3, 4) x = x_lbann x_slice = lbann.Slice(x, axis=1, slice_points=tools.str_list(slice_points)) y = [] for _ in range(len(slice_points)-1): y.append(lbann.L2Norm2(x_slice)) z = lbann.Add(y[0], y[2]) obj.append(z) metrics.append(lbann.Metric(z, name='axis1')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape(_sample_dims).astype(np.float64) y = [] for j in range(len(slice_points)-1): x_slice = x[:,slice_points[j]:slice_points[j+1],:] y.append(tools.numpy_l2norm2(x_slice)) z = y[0] + y[2] vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # -------------------------- # Slice along axis 2 # -------------------------- # LBANN implementation slice_points = (1, 3) x = x_lbann x_slice = lbann.Slice(x, axis=2, slice_points=tools.str_list(slice_points)) y = [] for _ in range(len(slice_points)-1): y.append(lbann.L2Norm2(x_slice)) z = y[0] obj.append(z) metrics.append(lbann.Metric(z, name='axis2')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape(_sample_dims).astype(np.float64) y = [] for j in range(len(slice_points)-1): x_slice = x[:,:,slice_points[j]:slice_points[j+1]] y.append(tools.numpy_l2norm2(x_slice)) z = y[0] vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # -------------------------- # Model-parallel # -------------------------- # LBANN implementation slice_points = (31, 54, 56, 57) x = lbann.Reshape(x_lbann, dims=tools.str_list([105])) x_slice = lbann.Slice(x, slice_points=tools.str_list(slice_points), data_layout='model_parallel') y = [] for _ in range(len(slice_points)-1): y.append(lbann.L2Norm2(x_slice)) z = lbann.Add(y[0], y[2]) obj.append(z) metrics.append(lbann.Metric(z, name='model-parallel')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape(-1).astype(np.float64) y = [] for j in range(len(slice_points)-1): x_slice = x[slice_points[j]:slice_points[j+1]] y.append(tools.numpy_l2norm2(x_slice)) z = y[0] + y[2] vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # -------------------------- # Gradient checking # -------------------------- callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # -------------------------- # Construct model # -------------------------- num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum( lbann.Input(), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_size))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # -------------------------- # Concatenate along axis 0 # -------------------------- # LBANN implementation x = x_lbann x = lbann.Reshape(x, dims=tools.str_list([5, 3, 4])) x_slice = lbann.Slice(x, axis=0, slice_points=tools.str_list([0, 1, 3, 5])) x1 = lbann.Identity(x_slice) x2 = lbann.Identity(x_slice) x3 = lbann.Identity(x_slice) y = lbann.Concatenation(x3, x2, x1, axis=0) z = lbann.L2Norm2(lbann.Multiply(x, y)) obj.append(z) metrics.append(lbann.Metric(z, name='axis0')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape([5, 3, 4]).astype(np.float64) x1 = x[0:1, :, :] x2 = x[1:3, :, :] x3 = x[3:5, :, :] y = np.concatenate((x3, x2, x1), axis=0) z = tools.numpy_l2norm2(x * y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # -------------------------- # Slice along axis 1 # -------------------------- # LBANN implementation x = x_lbann x = lbann.Reshape(x, dims=tools.str_list([3, 4, 5])) x_slice = lbann.Slice(x, axis=1, slice_points=tools.str_list([0, 1, 3, 4])) x1 = lbann.Identity(x_slice) x2 = lbann.Identity(x_slice) x3 = lbann.Identity(x_slice) y = lbann.Concatenation(x2, x1, x3, axis=1) z = lbann.L2Norm2(lbann.Multiply(x, y)) obj.append(z) metrics.append(lbann.Metric(z, name='axis1')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape([3, 4, 5]).astype(np.float64) x1 = x[:, 0:1, :] x2 = x[:, 1:3, :] x3 = x[:, 3:4, :] y = np.concatenate((x2, x1, x3), axis=1) z = tools.numpy_l2norm2(x * y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # -------------------------- # Slice along axis 2 # -------------------------- # LBANN implementation x = x_lbann x = lbann.Reshape(x, dims=tools.str_list([3, 4, 5])) x_slice = lbann.Slice(x, axis=2, slice_points=tools.str_list([0, 1, 2, 3, 5])) x1 = lbann.Identity(x_slice) x2 = lbann.Identity(x_slice) x3 = lbann.Identity(x_slice) x4 = lbann.Identity(x_slice) y = lbann.Concatenation(x2, x4, x1, x3, axis=2) z = lbann.L2Norm2(lbann.Multiply(x, y)) obj.append(z) metrics.append(lbann.Metric(z, name='axis2')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape([3, 4, 5]).astype(np.float64) x1 = x[:, :, 0:1] x2 = x[:, :, 1:2] x3 = x[:, :, 2:3] x4 = x[:, :, 3:5] y = np.concatenate((x2, x4, x1, x3), axis=2) z = tools.numpy_l2norm2(x * y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # -------------------------- # Model-parallel # -------------------------- # LBANN implementation x = x_lbann x = lbann.Reshape(x, dims=tools.str_list([60])) x_slice = lbann.Slice(x, slice_points=tools.str_list([0, 22, 23, 60])) x1 = lbann.Identity(x_slice) x2 = lbann.Identity(x_slice) x3 = lbann.Identity(x_slice) y = lbann.Concatenation(x3, x1, x2, data_layout='model_parallel') z = lbann.L2Norm2(lbann.Multiply(x, y)) obj.append(z) metrics.append(lbann.Metric(z, name='model-parallel')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).reshape([60]).astype(np.float64) x1 = x[0:22] x2 = x[22:23] x3 = x[23:60] y = np.concatenate((x3, x1, x2)) z = tools.numpy_l2norm2(x * y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # -------------------------- # Gradient checking # -------------------------- callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # -------------------------- # Construct model # -------------------------- num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ from lbann.modules.rnn import ChannelwiseGRU # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0), name='input') h_weights = lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0), name='inital_hidden') input_ = lbann.Input(data_field='samples') input_slice = lbann.Slice( input_, slice_points=tools.str_list( [0, _num_channels * _input_size, _sample_size]), ) x = lbann.Reshape(input_slice, dims=tools.str_list([_num_channels, _input_size]), name="input_reshape") x = lbann.Sum(x, lbann.WeightsLayer(weights=x_weights, dims=tools.str_list( [_num_channels, _input_size])), name="input_sum") h = lbann.Reshape(input_slice, dims=tools.str_list([_num_channels, _hidden_size]), name="hidden_reshape") h = lbann.Sum(h, lbann.WeightsLayer(weights=h_weights, dims=tools.str_list( [_num_channels, _hidden_size])), name="input_hidden_sum") x_lbann = x h_lbann = h # Objects for LBANN model obj = [] metrics = [] callbacks = [] # Weights rnn_weights_numpy = [] ih_matrix = np.random.uniform( low=-1, high=1, size=(3 * _hidden_size, _input_size), ) hh_matrix = np.random.uniform( low=-1, high=1, size=(3 * _hidden_size, _hidden_size), ) ih_bias = np.random.uniform(low=-1, high=1, size=(3 * _hidden_size, )) hh_bias = np.random.uniform(low=-1, high=1, size=(3 * _hidden_size, )) rnn_weights_numpy.extend([ih_matrix, ih_bias, hh_matrix, hh_bias]) rnn_weights_lbann = [ lbann.Weights(initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(w, order='F')))) for w in rnn_weights_numpy ] # LBANN implementation x = x_lbann h = h_lbann channelwise_GRU_cell = ChannelwiseGRU(num_channels=_num_channels, size=_hidden_size, weights=rnn_weights_lbann) y = channelwise_GRU_cell(x, h) z = lbann.L2Norm2(y) obj.append(z) metrics.append( lbann.Metric(z, name="Multi-channel, Unidirectional, GRU Cell")) # NumPy implementation vals = [] for i in range(num_samples()): input_ = get_sample(i).astype(np.float64) x = input_[:_num_channels * _input_size].reshape( (_num_channels, _input_size)) h = input_[_num_channels * _input_size:].reshape( (_num_channels, _hidden_size)) y = numpy_gru_cell(x, h, rnn_weights_numpy) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackPrintModelDescription()) callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Weights matrix height = _sample_size width = 7 mat = np.random.normal(size=(height, width)).astype(np.float32) # Input data x_lbann = lbann.Input(data_field='samples') # Objects for LBANN model layers = [x_lbann] obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Compute expected metric values with NumPy # ------------------------------------------ vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) y = np.linalg.norm(mat.astype(np.float64), axis=1) z = np.inner(x, y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps # ------------------------------------------ # Data-parallel layout # ------------------------------------------ w = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(mat, order='F')), ), ) dummy = lbann.FullyConnected( lbann.Constant(num_neurons=tools.str_list([width])), weights=w, num_neurons=height, data_layout='data_parallel', ) x = x_lbann y = lbann.RowwiseWeightsNorms( weights=w, hint_layer=dummy, data_layout='data_parallel', ) z = lbann.MatMul( lbann.Reshape(x, dims=tools.str_list([1,-1])), lbann.Reshape(y, dims=tools.str_list([1,-1])), transpose_b=True, ) layers.append(dummy) obj.append(z) metrics.append(lbann.Metric(z, name='data-parallel layout')) callbacks.append( lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test', ) ) # ------------------------------------------ # Model-parallel layout # ------------------------------------------ w = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(mat, order='F')), ), ) dummy = lbann.FullyConnected( lbann.Constant(num_neurons=tools.str_list([width])), weights=w, num_neurons=height, data_layout='model_parallel', ) x = x_lbann y = lbann.RowwiseWeightsNorms( weights=w, hint_layer=dummy, data_layout='model_parallel', ) z = lbann.MatMul( lbann.Reshape(x, dims=tools.str_list([1,-1])), lbann.Reshape(y, dims=tools.str_list([1,-1])), transpose_b=True, ) layers.append(dummy) obj.append(z) metrics.append(lbann.Metric(z, name='model-parallel layout')) callbacks.append( lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test', ) ) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=layers, objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum( lbann.Reshape(lbann.Input(), dims=tools.str_list(_sample_dims)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Basic 3^n convolution # ------------------------------------------ # 3^n conv, stride=1, pad=1, dilation=1, bias num_height_groups = tools.gpus_per_node(lbann) if num_height_groups == 0: e = 'this test requires GPUs.' print('Skip - ' + e) pytest.skip(e) for num_dims, reference_val in [(2, 11913.852660080756), (3, 9952.365297083174)]: # Convolution settings kernel_dims = [ 5, _sample_dims[0] if num_dims == 2 else _sample_dims_3d[0], ] + [3] * num_dims strides = [1] * num_dims pads = [1] * num_dims dilations = [1] * num_dims kernel = make_random_array(kernel_dims, 11) # Apply convolution kernel_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(kernel))), name='kernel1_{}d'.format(num_dims)) x = x_lbann if num_dims == 3: x = lbann.Reshape(x, dims=tools.str_list(_sample_dims_3d)) y = lbann.Convolution( x, weights=(kernel_weights, ), num_dims=num_dims, num_output_channels=kernel_dims[0], has_vectors=True, conv_dims=tools.str_list(kernel_dims[2:]), conv_strides=tools.str_list(strides), conv_pads=tools.str_list(pads), conv_dilations=tools.str_list(dilations), has_bias=False, parallel_strategy=create_parallel_strategy(num_height_groups)) z = lbann.L2Norm2(y) obj.append(z) metrics.append( lbann.Metric(z, name='basic {}D 3^n convolution'.format(num_dims))) # PyTorch implementation try: x = _samples if num_dims == 3: x = np.reshape(x, [_num_samples] + _sample_dims_3d) y = pytorch_convolution(x, kernel, stride=strides, padding=pads, dilation=dilations) z = tools.numpy_l2norm2(y) / _num_samples val = z except: # Precomputed value val = reference_val # val = 398.6956458317758 # _num_samples=8, 6 channels # val = 381.7401227915947 # _num_samples=23, 6 channels tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data x_lbann = lbann.Input(data_field='samples') y_numpy = np.random.normal(size=one_hot_size).astype(np.float32) y_numpy[:] = 1 ### @todo Remove y_lbann = lbann.Weights(initializer=lbann.ValueInitializer( values=tools.str_list(y_numpy))) y_lbann = lbann.WeightsLayer( weights=y_lbann, dims=tools.str_list([one_hot_size]), ) # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Compute expected metric values with NumPy # ------------------------------------------ vals = [] for i in range(num_samples()): x = int(np.floor(get_sample(i)[0])) y = y_numpy z = y[x] if (0 <= x < one_hot_size) else 0 vals.append(z) val = np.mean(vals, dtype=np.float64) tol = np.abs(8 * val * np.finfo(np.float32).eps) # ------------------------------------------ # Data-parallel layout # ------------------------------------------ x = x_lbann y = y_lbann x_onehot = lbann.OneHot( x, size=one_hot_size, data_layout='data_parallel', ) z = lbann.MatMul( lbann.Reshape(x_onehot, dims='1 -1'), lbann.Reshape(y, dims='1 -1'), transpose_b=True, ) obj.append(z) metrics.append(lbann.Metric(z, name='data-parallel layout')) callbacks.append( lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test', )) # ------------------------------------------ # Model-parallel layout # ------------------------------------------ x = x_lbann y = y_lbann x_onehot = lbann.OneHot( x, size=one_hot_size, data_layout='model_parallel', ) z = lbann.MatMul( lbann.Reshape(x_onehot, dims='1 -1'), lbann.Reshape(y, dims='1 -1'), transpose_b=True, ) obj.append(z) metrics.append(lbann.Metric(z, name='model-parallel layout')) callbacks.append( lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test', )) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=x_lbann, objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: We want to use gradient checking to verify that error # signals are correct. To do this, we zero-initialize a weights # object, construct a zero-valued tensor, and add it to the # input. To make sure that batchnorm is non-trivial, we multiply # the zero-valued tensor by the mini-batch index. x = lbann.Reshape(lbann.Input(), dims=tools.str_list(_sample_dims)) x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x0 = lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims)) x1 = lbann.Divide(lbann.MiniBatchIndex(), lbann.MiniBatchSize()) x1 = lbann.Tessellate(lbann.Reshape(x1, dims='1 1 1'), dims=tools.str_list(_sample_dims)) x = lbann.Sum(x, lbann.Multiply(x0, x1)) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Data-parallel layout # ------------------------------------------ # LBANN implementation decay = 0.9 epsilon = 1e-5 x = x_lbann y = lbann.EntrywiseBatchNormalization(x, decay=decay, epsilon=epsilon, data_layout='data_parallel') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='data-parallel layout')) # ------------------------------------------ # Model-parallel layout # ------------------------------------------ # LBANN implementation decay = 0.9 epsilon = 1e-5 x = x_lbann y = lbann.EntrywiseBatchNormalization(x, decay=decay, epsilon=epsilon, data_layout='model_parallel') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='model-parallel layout')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 1 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum( lbann.Reshape(lbann.Input(), dims=tools.str_list(_sample_dims)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Pooling # ------------------------------------------ pool_configs = [] # 3x3 pooling with same padding for mode, val in [("average", 10.44275178160873), ("max", 57.139962751295094)]: pool_configs.append({ "name": "3x3 {} pooling".format(mode), "kernel_dims": (3, 3), "strides": (1, 1), "pads": (1, 1), "pool_mode": mode, "val": val, }) # 2x2 strided pooling for mode, val in [("average", 3.679837210617764), ("max", 8.836797848269207)]: pool_configs.append({ "name": "2x2 {} pooling".format(mode), "kernel_dims": (2, 2), "strides": (2, 2), "pads": (0, 0), "pool_mode": mode, "val": val, }) # 2x4 pooling for mode, val in [("average", 2.936493136591308), ("max", 10.950791583798338)]: pool_configs.append({ "name": "2x4 {} pooling".format(mode), "kernel_dims": (2, 4), "strides": (3, 1), "pads": (1, 0), "pool_mode": mode, "val": val, }) # 2x2x2 3D pooling for mode, val in [("average", 0.5709457972093812), ("max", 3.886491271066883)]: pool_configs.append({ "name": "2x2x2 {} pooling".format(mode), "kernel_dims": (2, 2, 2), "strides": (2, 2, 2), "pads": (0, 0, 0), "pool_mode": mode, "val": val, }) for p in pool_configs: # Apply pooling x = x_lbann if len(p["kernel_dims"]) == 3: x = lbann.Reshape(x, dims=tools.str_list(_sample_dims_3d)) y = lbann.Pooling(x, num_dims=len(p["kernel_dims"]), has_vectors=True, pool_dims=tools.str_list(p["kernel_dims"]), pool_strides=tools.str_list(p["strides"]), pool_pads=tools.str_list(p["pads"]), pool_mode=p["pool_mode"]) z = lbann.L2Norm2(y) # Since max pooling is not differentiable, we only use average pooling. if p["pool_mode"] == "average": obj.append(z) metrics.append(lbann.Metric(z, name=p["name"])) # PyTorch implementation try: x = _samples if len(p["kernel_dims"]) == 3: x = np.reshape(x, [_num_samples] + _sample_dims_3d) y = pytorch_pooling( x, p["kernel_dims"], p["pool_mode"], stride=p["strides"], padding=p["pads"], ) z = tools.numpy_l2norm2(y) / _num_samples val = z except: # Precomputed value val = p["val"] tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with weights layers so that gradient checking will # verify that error signals are correct. x0_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input0_weights') x1_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input1_weights') x_slice = lbann.Slice(lbann.Input(), slice_points=tools.str_list([0, _m*_k, _m*_k+_k*_n])) x0 = lbann.Sum(x_slice, lbann.WeightsLayer(weights=x0_weights, dims=str(_m*_k))) x1 = lbann.Sum(x_slice, lbann.WeightsLayer(weights=x1_weights, dims=str(_k*_n))) x0_lbann = x0 x1_lbann = x1 # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # NN GEMM # ------------------------------------------ # LBANN implementation x0 = lbann.Reshape(x0_lbann, dims=tools.str_list([_m, _k])) x1 = lbann.Reshape(x1_lbann, dims=tools.str_list([_k, _n])) y = lbann.MatMul(x0, x1, data_layout='data_parallel') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='NN GEMM')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) x0 = x[:_m*_k].reshape([_m,_k]) x1 = x[_m*_k:].reshape([_k,_n]) y = np.matmul(x0, x1) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # TN GEMM # ------------------------------------------ # LBANN implementation x0 = lbann.Reshape(x0_lbann, dims=tools.str_list([_k, _m])) x1 = lbann.Reshape(x1_lbann, dims=tools.str_list([_k, _n])) y = lbann.MatMul(x0, x1, transpose_a=True, data_layout='data_parallel') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='TN GEMM')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) x0 = x[:_m*_k].reshape([_k,_m]) x1 = x[_m*_k:].reshape([_k,_n]) y = np.matmul(x0.transpose(), x1) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # NT GEMM # ------------------------------------------ # LBANN implementation x0 = lbann.Reshape(x0_lbann, dims=tools.str_list([_m, _k])) x1 = lbann.Reshape(x1_lbann, dims=tools.str_list([_n, _k])) y = lbann.MatMul(x0, x1, transpose_b=True, data_layout='data_parallel') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='NT GEMM')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) x0 = x[:_m*_k].reshape([_m,_k]) x1 = x[_m*_k:].reshape([_n,_k]) y = np.matmul(x0, x1.transpose()) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # TT GEMM # ------------------------------------------ # LBANN implementation x0 = lbann.Reshape(x0_lbann, dims=tools.str_list([_k, _m])) x1 = lbann.Reshape(x1_lbann, dims=tools.str_list([_n, _k])) y = lbann.MatMul(x0, x1, transpose_a=True, transpose_b=True, data_layout='data_parallel') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='TT GEMM')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) x0 = x[:_m*_k].reshape([_k,_m]) x1 = x[_m*_k:].reshape([_n,_k]) y = np.matmul(x0.transpose(), x1.transpose()) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x0_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data x = lbann.Identity(lbann.Input()) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # GPU # ------------------------------------------ # Embeddings np.random.seed(_seed) embedding_dim = 7 embeddings = np.random.normal(size=(_num_embeddings,embedding_dim)) # LBANN implementation embedding_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer(values=tools.str_list(np.nditer(embeddings))) ) x = x_lbann y = lbann.DistEmbedding(x, weights=embedding_weights, num_embeddings=_num_embeddings, embedding_dim=embedding_dim, barrier_in_forward_prop=True, device='gpu') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='GPU')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i) y = embeddings[x,:] z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # CPU # ------------------------------------------ # Embeddings np.random.seed(_seed) embedding_dim = 5 embeddings = np.random.normal(size=(_num_embeddings,embedding_dim)) # LBANN implementation embedding_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer(values=tools.str_list(np.nditer(embeddings))) ) x = x_lbann y = lbann.DistEmbedding(x, weights=embedding_weights, num_embeddings=_num_embeddings, embedding_dim=embedding_dim, barrier_in_forward_prop=True, device='cpu') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='CPU')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i) y = embeddings[x,:] z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ # Construct model num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x0 = lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims)) x1 = lbann.Reshape(lbann.Input(data_field='samples'), dims=tools.str_list(_sample_dims)) x = lbann.Sum(x0, x1) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Compute expected metric values with NumPy # ------------------------------------------ # Input and output dimensions input_channel_dims = _sample_dims[1:] output_channel_dims = (2,5) input_channel_size = functools.reduce(operator.mul, input_channel_dims) output_channel_size = functools.reduce(operator.mul, output_channel_dims) # Weight values linearity = np.random.normal( size=(output_channel_size,input_channel_size) ).astype(np.float32) bias = np.random.normal(size=(output_channel_size,1)).astype(np.float32) # With bias x = (_samples .reshape((-1,input_channel_size)) .transpose() .astype(np.float64)) y = np.matmul(linearity.astype(np.float64), x) + bias.astype(np.float64) z = tools.numpy_l2norm2(y) / _num_samples val_with_bias = z # Without bias x = (_samples .reshape((-1,input_channel_size)) .transpose() .astype(np.float64)) y = np.matmul(linearity.astype(np.float64), x) z = tools.numpy_l2norm2(y) / _num_samples val_without_bias = z # ------------------------------------------ # Data-parallel layout, non-transpose, bias # ------------------------------------------ # LBANN implementation linearity_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(linearity, order='F')) ) ) bias_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(bias)) ) ) x = x_lbann y = lbann.ChannelwiseFullyConnected( x, weights=(linearity_weights, bias_weights), output_channel_dims=output_channel_dims, ) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='data-parallel layout, non-transpose, bias')) # NumPy implementation tol = 8 * val_with_bias * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val_with_bias-tol, upper_bound=val_with_bias+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Data-parallel layout, non-transpose, no bias # ------------------------------------------ # LBANN implementation linearity_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(linearity, order='F')) ) ) x = x_lbann y = lbann.ChannelwiseFullyConnected( x, weights=(linearity_weights), output_channel_dims=output_channel_dims, bias=False, ) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='data-parallel layout, non-transpose, no bias')) # NumPy implementation tol = 8 * val_without_bias * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val_without_bias-tol, upper_bound=val_without_bias+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Data-parallel layout, transpose, bias # ------------------------------------------ # LBANN implementation linearity_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(linearity, order='C')) ) ) bias_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(bias)) ) ) x = x_lbann y = lbann.ChannelwiseFullyConnected( x, weights=(linearity_weights, bias_weights), output_channel_dims=output_channel_dims, transpose=True, ) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='data-parallel layout, transpose, bias')) # NumPy implementation tol = 8 * val_with_bias * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val_with_bias-tol, upper_bound=val_with_bias+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Data-parallel layout, transpose, no bias # ------------------------------------------ # LBANN implementation linearity_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(linearity, order='C')) ) ) x = x_lbann y = lbann.ChannelwiseFullyConnected( x, weights=(linearity_weights), output_channel_dims=output_channel_dims, bias=False, transpose=True, ) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='data-parallel layout, transpose, no bias')) # NumPy implementation tol = 8 * val_without_bias * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val_without_bias-tol, upper_bound=val_without_bias+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0), name='input') h_weights = lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0), name='inital_hidden') input_ = lbann.Input(data_field='samples') input_slice = lbann.Slice( input_, slice_points=tools.str_list( [0, _sequence_length * _input_size, _sample_size]), ) x = lbann.Reshape(input_slice, dims=tools.str_list([_sequence_length, _input_size])) x = lbann.Sum(x, lbann.WeightsLayer(weights=x_weights, hint_layer=x)) h = lbann.Reshape( input_slice, dims=tools.str_list([_num_layers, _input_size]), ) h = lbann.Sum(h, lbann.WeightsLayer(weights=h_weights, hint_layer=h)) x_lbann = x h_lbann = h # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Multi-layer, unidirectional GRU # ------------------------------------------ # Note: input_size=hidden_size due to a limitation in oneDNN # Weights rnn_weights_numpy = [] for i in range(_num_layers): ih_matrix = np.random.uniform( low=-1, high=1, size=(3 * _input_size, _input_size), ) hh_matrix = np.random.uniform( low=-1, high=1, size=(3 * _input_size, _input_size), ) ih_bias = np.random.uniform(low=-1, high=1, size=(3 * _input_size, )) hh_bias = np.random.uniform(low=-1, high=1, size=(3 * _input_size, )) rnn_weights_numpy.extend([ih_matrix, hh_matrix, ih_bias, hh_bias]) rnn_weights_numpy = [w.astype(np.float32) for w in rnn_weights_numpy] rnn_weights_lbann = [ lbann.Weights(initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(w, order='F')))) for w in rnn_weights_numpy ] # LBANN implementation x = x_lbann h = h_lbann y = lbann.GRU( x, h, hidden_size=_input_size, num_layers=_num_layers, weights=rnn_weights_lbann, ) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='Multi-layer, unidirectional')) # NumPy implementation vals = [] for i in range(num_samples()): input_ = get_sample(i).astype(np.float64) x = input_[:_sequence_length * _input_size].reshape( (_sequence_length, _input_size)) h = input_[_sequence_length * _input_size:].reshape( (_num_layers, _input_size)) y = numpy_gru(x, h, rnn_weights_numpy) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Single-layer, unidirectional GRU # ------------------------------------------ # Weights rnn_weights_numpy = [] hidden_size = 7 ih_matrix = np.random.uniform( low=-1, high=1, size=(3 * hidden_size, _input_size), ) hh_matrix = np.random.uniform( low=-1, high=1, size=(3 * hidden_size, hidden_size), ) ih_bias = np.random.uniform(low=-1, high=1, size=(3 * hidden_size, )) hh_bias = np.random.uniform(low=-1, high=1, size=(3 * hidden_size, )) rnn_weights_numpy.extend([ih_matrix, hh_matrix, ih_bias, hh_bias]) rnn_weights_numpy = [w.astype(np.float32) for w in rnn_weights_numpy] rnn_weights_lbann = [ lbann.Weights(initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(w, order='F')))) for w in rnn_weights_numpy ] # LBANN implementation x = x_lbann h = h_lbann h = lbann.Reshape( lbann.Slice( lbann.Reshape(h, dims='-1'), slice_points=tools.str_list([0, hidden_size]), ), dims='1 -1', ) y = lbann.GRU( x, h, hidden_size=hidden_size, num_layers=1, weights=rnn_weights_lbann, ) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='Single-layer, unidirectional')) # NumPy implementation vals = [] for i in range(num_samples()): input_ = get_sample(i).astype(np.float64) x = input_[:_sequence_length * _input_size].reshape( (_sequence_length, _input_size)) h = input_[_sequence_length * _input_size:].reshape( (_num_layers, _input_size)) h = h.flatten()[:hidden_size].reshape((1, hidden_size)) y = numpy_gru(x, h, rnn_weights_numpy) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0), name='input') h_weights = lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0), name='inital_hidden') input_ = lbann.Identity(lbann.Input()) input_slice = lbann.Slice( input_, slice_points=tools.str_list( [0, _sequence_length * _input_size, _sample_size]), ) x = lbann.Reshape(input_slice, dims=tools.str_list([_sequence_length, _input_size])) x = lbann.Sum(x, lbann.WeightsLayer(weights=x_weights, hint_layer=x)) h = lbann.Reshape( input_slice, dims=tools.str_list([_hidden_size]), ) h = lbann.Sum(h, lbann.WeightsLayer(weights=h_weights, hint_layer=h)) x_lbann = x h_lbann = h # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # 1-layer, uni-directional GRU # ------------------------------------------ # Weights ih_matrix = np.random.normal(size=(3 * _hidden_size, _input_size)).astype(np.float32) hh_matrix = np.random.normal(size=(3 * _hidden_size, _hidden_size)).astype(np.float32) ih_bias = np.random.normal(size=(3 * _hidden_size, )).astype(np.float32) hh_bias = np.random.normal(size=(3 * _hidden_size, )).astype(np.float32) ih_matrix_weights = lbann.Weights(initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(ih_matrix, order='F')))) hh_matrix_weights = lbann.Weights(initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(hh_matrix, order='F')))) ih_bias_weights = lbann.Weights(initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(ih_bias)))) hh_bias_weights = lbann.Weights(initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(hh_bias)))) # LBANN implementation x = x_lbann h = h_lbann y = lbann.GRU( x, h, hidden_size=_hidden_size, weights=[ ih_matrix_weights, hh_matrix_weights, ih_bias_weights, hh_bias_weights ], ) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='1-layer, unidirectional')) # NumPy implementation vals = [] for i in range(num_samples()): input_ = get_sample(i).astype(np.float64) x = input_[:_sequence_length * _input_size].reshape( (_sequence_length, _input_size)) h = input_[_sequence_length * _input_size:] y = numpy_gru(x, h, ih_matrix, hh_matrix, ih_bias, hh_bias) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data x = lbann.Input(data_field='samples') x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # No padding index # ------------------------------------------ # Embeddings np.random.seed(20191015) embedding_dim = 5 embeddings = np.random.normal(size=(_num_embeddings, embedding_dim)) # LBANN implementation embedding_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(embeddings)))) x = x_lbann y = lbann.Embedding(x, weights=embedding_weights, num_embeddings=_num_embeddings, embedding_dim=embedding_dim) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='no padding index')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i) y = embeddings[x, :] z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Padding index 0 # ------------------------------------------ # Embeddings np.random.seed(201910152) embedding_dim = 7 padding_idx = 0 embeddings = np.random.normal(size=(_num_embeddings, embedding_dim)) # LBANN implementation # Note: Embedding layer gradients are not exact if a padding index # is set. Avoid gradient checking by not using an optimizer. embedding_weights = lbann.Weights( optimizer=None, initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(embeddings)))) x = x_lbann y = lbann.Embedding(x, weights=embedding_weights, num_embeddings=_num_embeddings, embedding_dim=embedding_dim, padding_idx=padding_idx) z = lbann.L2Norm2(y) metrics.append(lbann.Metric(z, name='padding index = 0')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i) y = np.where((x == padding_idx).reshape((-1, 1)), 0, embeddings[x, :]) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ # Construct model num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with weights layers so that gradient checking will # verify that error signals are correct. slice_size = _samples.shape[-1] x0_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input0_weights') x1_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input1_weights') x_slice = lbann.Slice(lbann.Input(), slice_points=tools.str_list([0, slice_size, 2*slice_size])) x0 = lbann.Sum(x_slice, lbann.WeightsLayer(weights=x0_weights, dims=str(slice_size))) x1 = lbann.Sum(x_slice, lbann.WeightsLayer(weights=x1_weights, dims=str(slice_size))) x0_lbann = x0 x1_lbann = x1 # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Data-parallel layout # ------------------------------------------ # LBANN implementation x0 = x0_lbann x1 = x1_lbann y = lbann.CrossEntropy(x0, x1, data_layout='data_parallel') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='data-parallel layout')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) x0 = x[:slice_size] x1 = x[slice_size:] y = -np.inner(x1, np.log(x0)) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Model-parallel layout # ------------------------------------------ # LBANN implementation x0 = x0_lbann x1 = x1_lbann y = lbann.CrossEntropy(x0, x1, data_layout='model_parallel') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='model-parallel layout')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) x0 = x[:slice_size] x1 = x[slice_size:] y = -np.inner(x1, np.log(x0)) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append(lbann.CallbackCheckMetric( metric=metrics[-1].name, lower_bound=val-tol, upper_bound=val+tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x0_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum( lbann.Reshape(lbann.Input(), dims=tools.str_list(_sample_dims)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Basic 3x3 convolution # ------------------------------------------ # 3x3 conv, stride=1, pad=1, dilation=1, bias # Convolution settings kernel_dims = (5, _sample_dims[0], 3, 3) strides = (1, 1) pads = (1, 1) dilations = (1, 1) kernel = make_random_array(kernel_dims, 11) bias = make_random_array([kernel_dims[0]], 123) # Apply convolution kernel_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(kernel))), name='kernel1') bias_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(bias))), name='bias1') x = x_lbann y = lbann.Convolution(x, weights=(kernel_weights, bias_weights), num_dims=3, num_output_channels=kernel_dims[0], has_vectors=True, conv_dims=tools.str_list(kernel_dims[2:]), conv_strides=tools.str_list(strides), conv_pads=tools.str_list(pads), conv_dilations=tools.str_list(dilations), has_bias=True) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='basic 3x3 convolution')) # PyTorch implementation try: x = _samples y = pytorch_convolution(x, kernel, bias=bias, stride=strides, padding=pads, dilation=dilations) z = tools.numpy_l2norm2(y) / _num_samples val = z except: # Precomputed value val = 153.84937996554953 tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # 2x4 strided convolution # ------------------------------------------ # Convolution settings kernel_dims = (3, _sample_dims[0], 2, 4) strides = (3, 1) pads = (3, 0) dilations = (1, 1) num_groups = 1 kernel = make_random_array(kernel_dims, 19) # Apply convolution kernel_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(kernel))), name='kernel2') x = x_lbann y = lbann.Convolution(x, weights=(kernel_weights), num_dims=3, num_output_channels=kernel_dims[0], has_vectors=True, conv_dims=tools.str_list(kernel_dims[2:]), conv_strides=tools.str_list(strides), conv_pads=tools.str_list(pads), conv_dilations=tools.str_list(dilations), num_groups=num_groups, has_bias=False) z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='2x4 convolution')) # PyTorch implementation try: x = _samples y = pytorch_convolution(x, kernel, bias=None, stride=strides, padding=pads, dilation=dilations, groups=num_groups) z = tools.numpy_l2norm2(y) / _num_samples val = z except: # Precomputed value val = 19.24587403346207 tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum( lbann.Reshape(lbann.Input(data_field='samples'), dims=tools.str_list(_sample_size)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_size))) x_lbann = x # Objects for LBANN model obj = [] metrics = [] callbacks = [] # ------------------------------------------ # Data-parallel layout # ------------------------------------------ # LBANN implementation x = x_lbann y = lbann.Elu(x, alpha=1, data_layout='data_parallel') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='data-parallel layout')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) y = np.where(x < 0, np.expm1(x), x) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Model-parallel layout # ------------------------------------------ # LBANN implementation x = x_lbann y = lbann.Elu(x, alpha=0.5, data_layout='model_parallel') z = lbann.L2Norm2(y) obj.append(z) metrics.append(lbann.Metric(z, name='model-parallel layout')) # NumPy implementation vals = [] for i in range(num_samples()): x = get_sample(i).astype(np.float64) y = np.where(x < 0, 0.5 * np.expm1(x), x) z = tools.numpy_l2norm2(y) vals.append(z) val = np.mean(vals) tol = 8 * val * np.finfo(np.float32).eps callbacks.append( lbann.CallbackCheckMetric(metric=metrics[-1].name, lower_bound=val - tol, upper_bound=val + tol, error_on_failure=True, execution_modes='test')) # ------------------------------------------ # Gradient checking # ------------------------------------------ callbacks.append(lbann.CallbackCheckGradients(error_on_failure=True)) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 0 return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x_lbann), objective_function=obj, metrics=metrics, callbacks=callbacks)
def construct_model(lbann): """Construct LBANN model. Args: lbann (module): Module for LBANN Python frontend """ gpus_per_node = tools.gpus_per_node(lbann) if gpus_per_node == 0: e = 'this test requires GPUs.' print('Skip - ' + e) pytest.skip(e) # Input data # Note: Sum with a weights layer so that gradient checking will # verify that error signals are correct. x_weights = lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ConstantInitializer(value=0.0), name='input_weights') x = lbann.Sum( lbann.Reshape(lbann.Input(data_field='samples'), dims=tools.str_list(_sample_dims)), lbann.WeightsLayer(weights=x_weights, dims=tools.str_list(_sample_dims))) # Objects for LBANN model obj = [] metrics = [] callbacks = [] # Convolution/FC settings kernel_dims = (5, _sample_dims[0], 3, 3) strides = (1, 1) pads = (1, 1) dilations = (1, 1) kernel = make_random_array(kernel_dims, 11) fc_input_size = kernel_dims[0] * np.prod(_sample_dims[1:]) linearity1 = make_random_array((_output_size, fc_input_size), 13) linearity2 = make_random_array((_output_size, _output_size), 17) linearity_no_opt = make_random_array((_output_size, _output_size), 19) biases = make_random_array((_output_size, ), 19) # Weight values kernel_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(kernel))), name='kernel1') linearity1_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(linearity1, order='F')))) linearity2_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(linearity2, order='F')))) linearity_no_opt_weights = lbann.Weights( optimizer=lbann.NoOptimizer(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(linearity_no_opt, order='F')))) biases_weights = lbann.Weights( optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(biases, order='F')))) def create_bn_weights(layer_name, num_channels): weights_ary = [] for i in range(4): val = make_random_array((num_channels, ), 15 + i) weights_ary.append( lbann.Weights(optimizer=lbann.SGD(), initializer=lbann.ValueInitializer( values=tools.str_list(np.nditer(val))), name='{}_{}'.format(layer_name, i))) return weights_ary y = lbann.Convolution(x, weights=(kernel_weights, ), num_dims=3, num_output_channels=kernel_dims[0], has_vectors=True, conv_dims=tools.str_list(kernel_dims[2:]), conv_strides=tools.str_list(strides), conv_pads=tools.str_list(pads), conv_dilations=tools.str_list(dilations), has_bias=False) # y = lbann.BatchNormalization( # y, # weights=create_bn_weights("bn1", kernel_dims[0])) y = lbann.Relu(y) y = lbann.FullyConnected(y, weights=(linearity1_weights, ), data_layout='data_parallel', num_neurons=_output_size, has_bias=False) # y = lbann.BatchNormalization( # y, # weights=create_bn_weights("bn2", _output_size)) y = lbann.Relu(y) y = lbann.FullyConnected(y, weights=(linearity2_weights, biases_weights), data_layout='data_parallel', num_neurons=_output_size, has_bias=True) # y = lbann.BatchNormalization( # y, # weights=create_bn_weights("bn3", _output_size)) y = lbann.Relu(y) y = lbann.FullyConnected(y, weights=(linearity_no_opt_weights), data_layout='data_parallel', num_neurons=_output_size, has_bias=False) z = lbann.L2Norm2(y) obj.append(z) # ------------------------------------------ # Construct model # ------------------------------------------ num_epochs = 1 ### @todo Remove return lbann.Model(num_epochs, layers=lbann.traverse_layer_graph(x), objective_function=obj, metrics=metrics, callbacks=callbacks)