def generate_matrix_data(ctx, matrix_size, n_splits):
matrix_size = int(matrix_size)
n_splits = int(n_splits)
data_dir = get_matrix_dir(matrix_size, n_splits)
conf = MatrixConf(matrix_size, n_splits)
params_path = join(data_dir, MATRIX_CONF_STATE_KEY)
# Write params to file
print("Generating {}x{} matrix with {} splits".format(
matrix_size, matrix_size, n_splits))
params = np.array((matrix_size, n_splits), dtype=int32)
with open(params_path, "wb") as fh:
fh.write(params.tobytes())
mat_a = random_matrix(matrix_size)
mat_b = random_matrix(matrix_size)
subdivide_matrix_into_files(conf, mat_a, data_dir, SUBMATRICES_KEY_A)
subdivide_matrix_into_files(conf, mat_b, data_dir, SUBMATRICES_KEY_B)
def test_distributed_multiplication(self, split_level):
self.set_up_conf(split_level)
# Set up the problem
mat_a = random_matrix(self.conf.matrix_size)
mat_b = random_matrix(self.conf.matrix_size)
subdivide_matrix_into_state(self.conf, mat_a, SUBMATRICES_KEY_A)
subdivide_matrix_into_state(self.conf, mat_b, SUBMATRICES_KEY_B)
expected = np.dot(mat_a, mat_b)
# Invoke the divide and conquer
divide_and_conquer()
# Load the result
actual_bytes = get_state(RESULT_MATRIX_KEY, self.conf.bytes_per_matrix)
actual = np.frombuffer(actual_bytes, dtype=np.float32).reshape(
self.conf.matrix_size, self.conf.matrix_size)
# Note that the floating point errors can creep up so we have a relatively high tolerance here
np.testing.assert_array_almost_equal_nulp(actual, expected, nulp=20)
def test_reading_submatrix_from_state(self, split_level):
self.set_up_conf(split_level)
mat_a = random_matrix(self.conf.matrix_size)
mat_b = random_matrix(self.conf.matrix_size)
subdivide_matrix_into_state(self.conf, mat_a, self.key_a)
subdivide_matrix_into_state(self.conf, mat_b, self.key_b)
if split_level == 0:
row_idx = 0
col_idx = 0
elif split_level == 1:
row_idx = 1
col_idx = 1
elif split_level == 2:
row_idx = 3
col_idx = 2
else:
row_idx = 5
col_idx = 6
sm_size = self.conf.get_submatrix_size(split_level)
# Get the submatrix directly from the original
row_start = row_idx * sm_size
col_start = col_idx * sm_size
expected_a = mat_a[row_start:row_start + sm_size,
col_start:col_start + sm_size, ]
expected_b = mat_b[row_start:row_start + sm_size,
col_start:col_start + sm_size, ]
actual_a = read_input_submatrix(self.conf, self.key_a, row_idx,
col_idx)
actual_b = read_input_submatrix(self.conf, self.key_b, row_idx,
col_idx)
np.testing.assert_array_equal(expected_a, actual_a)
np.testing.assert_array_equal(expected_b, actual_b)
def test_matrix_file_round_trip(self, split_level):
self.set_up_conf(split_level)
mat_a = random_matrix(self.conf.matrix_size)
mat_b = random_matrix(self.conf.matrix_size)
file_dir = "/tmp/mat_test"
file_prefix_a = "mat_a"
file_prefix_b = "mat_b"
if exists(file_dir):
rmtree(file_dir)
makedirs(file_dir)
subdivide_matrix_into_files(self.conf, mat_a, file_dir, file_prefix_a)
subdivide_matrix_into_files(self.conf, mat_b, file_dir, file_prefix_b)
actual_a = reconstruct_matrix_from_files(self.conf, file_dir,
file_prefix_a)
actual_b = reconstruct_matrix_from_files(self.conf, file_dir,
file_prefix_b)
np.testing.assert_array_equal(mat_a, actual_a)
np.testing.assert_array_equal(mat_b, actual_b)
def test_matrix_round_trip(self):
mat_a = random_matrix(self.conf.matrix_size)
subdivide_matrix_into_state(self.conf, mat_a, self.key_a)
actual = reconstruct_matrix_from_submatrices(self.conf, self.key_a)
np.testing.assert_array_equal(mat_a, actual)