def impl_stats_const_file(self, shape, dtype, buffer_size):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
file_format = 'raw'
filename_format = "stats_const_{shape}_{dtype}.{format}"
filename = filename_format.format(shape=shape, dtype=dtype, format=file_format)
dmin = 3
for d in shape:
assert d >= dmin, "d={} < {}".format(d, dmin)
cube_max_index = tuple(0 for i in shape)
cube_min_index = tuple(1 for i in shape)
cube_zero_index = tuple(2 for i in shape)
cube_value = 1.0
cube_max = 10.0
cube_min = -23.0
cube_zero = 0.0
cube_sum = cube_max + cube_min + cube_value * (shape.count() - dmin)
cube_ave = cube_sum / float(shape.count())
cube = cb.const_cube(shape=shape, dtype=dtype, value=cube_value)
cube[cube_max_index] = cube_max
cube[cube_min_index] = cube_min
cube[cube_zero_index] = cube_zero
cube_count_zero = 1
cube_count_nonzero = shape.count() - cube_count_zero
cube_count_nan = 0
cube_count_inf = 0
stats_info_cube = cb.stats_info(cube)
self.assertEqual(stats_info_cube.cube_sum, cube_sum)
self.assertEqual(stats_info_cube.cube_ave, cube_ave)
self.assertEqual(stats_info_cube.cube_max, cube_max)
self.assertEqual(stats_info_cube.cube_min, cube_min)
self.assertEqual(stats_info_cube.cube_max_index, cube_max_index)
self.assertEqual(stats_info_cube.cube_min_index, cube_min_index)
self.assertEqual(stats_info_cube.cube_count_zero, cube_count_zero)
self.assertEqual(stats_info_cube.cube_count_nonzero, cube_count_nonzero)
self.assertEqual(stats_info_cube.cube_count_nan, cube_count_nan)
self.assertEqual(stats_info_cube.cube_count_inf, cube_count_inf)
cube.tofile(filename)
self.assertFileExistsAndHasShape(filename, shape=shape, dtype=dtype)
stats_info_oc = cb.stats_file(filename, shape=shape, dtype=dtype, file_format=file_format,
out_of_core=False)
self.assertEqual(stats_info_oc, stats_info_cube)
stats_info_ooc = cb.stats_file(filename, shape=shape, dtype=dtype, file_format=file_format,
out_of_core=True, progress_frequency=-1.0, buffer_size=buffer_size)
self.assertEqual(stats_info_ooc, stats_info_cube)
def write_linear_cube_out_of_core_4x4_int64(self):
dtype = np.int64
shape = Shape("4x4")
self.impl_write_cube_out_of_core(kind='linear',
shape=shape,
dtype=dtype,
buffer_size=shape.count() *
dtype().itemsize / 2)
def assertFileExistsAndHasShape(self, filename, shape, dtype=None):
self.assertFileExists(filename)
dtype = cb.get_dtype(dtype)
if not isinstance(shape, Shape):
shape = Shape(shape)
size = shape.count() * dtype().itemsize
filesize = os.stat(filename).st_size
if filesize != size:
raise AssertionError("file {!r} has size {} != {} [shape={}, dtype={}]".format(filename, filesize, size, shape, dtype.__name__))
def impl_data_type(self, keys, actual_data_type=None, actual_file_format=None, const_value=None):
shape = Shape("4x5")
data_type = keys.get('data_type', 'float32')
options = keys.get('options', None)
dtype = cb.get_dtype(data_type)
file_format = keys.get('file_format', 'raw')
if actual_data_type is None:
actual_data_type = data_type
actual_dtype = cb.get_dtype(actual_data_type)
if actual_file_format is None:
actual_file_format = file_format
rubik_dir = os.getcwd()
rubik_config = config.get_config()
rubik_config_data_type = os.path.join(rubik_dir, "rubik.config.{data_type}.{file_format}")
rubik_config.default_data_type = data_type
rubik_config.default_file_format = file_format
if options is not None:
rubik_config.default_options = options
with open(rubik_config_data_type, "w") as f:
rubik_config.write_config_file(f)
current_rubik_dir = os.environ.get("RUBIK_DIR", None)
current_rubik_config = os.environ.get("RUBIK_CONFIG", None)
out_filename_format = 'ctmp_{shape}_{dtype}.{format}'
out_filename = out_filename_format.format(shape=shape, dtype=actual_data_type, format=file_format)
try:
os.environ["RUBIK_DIR"] = rubik_dir
os.environ["RUBIK_CONFIG"] = rubik_config_data_type
environment.load_rubik_environment()
returncode, output, error = self.run_program(
"""-e 'cb.const_cube("{s}", value={c})' -o '{o}'""".format(
s=shape,
c=const_value,
o=out_filename_format))
self.assertEqual(returncode, 0)
if actual_file_format == 'raw':
self.assertFileExistsAndHasShape(out_filename, shape=shape, dtype=actual_data_type)
else:
self.assertFileExists(out_filename)
returncode, output, error = self.run_program(
"""-i '{o}' -s {s} -e 'i0.sum()' --print""".format(
s=shape,
o=out_filename_format),
join_stderr_stdout=False)
value = actual_dtype(output.rstrip('\n'))
self.assertEqual(value, shape.count() * const_value)
finally:
if current_rubik_dir is None:
del os.environ["RUBIK_DIR"]
else:
os.environ["RUBIK_DIR"] = current_rubik_dir
if current_rubik_config is None:
del os.environ["RUBIK_CONFIG"]
else:
os.environ["RUBIK_CONFIG"] = rubik_config_data_type
environment.load_rubik_environment()
def impl_data_type(self,
keys,
actual_data_type=None,
actual_file_format=None,
const_value=None):
shape = Shape("4x5")
data_type = keys.get('data_type', 'float32')
options = keys.get('options', None)
dtype = cb.get_dtype(data_type)
file_format = keys.get('file_format', 'raw')
if actual_data_type is None:
actual_data_type = data_type
actual_dtype = cb.get_dtype(actual_data_type)
if actual_file_format is None:
actual_file_format = file_format
rubik_dir = os.getcwd()
rubik_config = config.get_config()
rubik_config_data_type = os.path.join(
rubik_dir, "rubik.config.{data_type}.{file_format}")
rubik_config.default_data_type = data_type
rubik_config.default_file_format = file_format
if options is not None:
rubik_config.default_options = options
with open(rubik_config_data_type, "w") as f:
rubik_config.write_config_file(f)
current_rubik_dir = os.environ.get("RUBIK_DIR", None)
current_rubik_config = os.environ.get("RUBIK_CONFIG", None)
out_filename_format = 'ctmp_{shape}_{dtype}.{format}'
out_filename = out_filename_format.format(shape=shape,
dtype=actual_data_type,
format=file_format)
try:
os.environ["RUBIK_DIR"] = rubik_dir
os.environ["RUBIK_CONFIG"] = rubik_config_data_type
environment.load_rubik_environment()
returncode, output, error = self.run_program(
"""-e 'cb.const_cube("{s}", value={c})' -o '{o}'""".format(
s=shape, c=const_value, o=out_filename_format))
self.assertEqual(returncode, 0)
if actual_file_format == 'raw':
self.assertFileExistsAndHasShape(out_filename,
shape=shape,
dtype=actual_data_type)
else:
self.assertFileExists(out_filename)
returncode, output, error = self.run_program(
"""-i '{o}' -s {s} -e 'i0.sum()' --print""".format(
s=shape, o=out_filename_format),
join_stderr_stdout=False)
value = actual_dtype(output.rstrip('\n'))
self.assertEqual(value, shape.count() * const_value)
finally:
if current_rubik_dir is None:
del os.environ["RUBIK_DIR"]
else:
os.environ["RUBIK_DIR"] = current_rubik_dir
if current_rubik_config is None:
del os.environ["RUBIK_CONFIG"]
else:
os.environ["RUBIK_CONFIG"] = rubik_config_data_type
environment.load_rubik_environment()
def count(self):
for shape_t, shape_s, count in self._shapes:
shape_ft = Shape(shape_t)
shape_fs = Shape(shape_s)
self.assertEqual(shape_ft.count(), shape_fs.count())
self.assertEqual(shape_ft.count(), count)
def impl_stats_const_file(self, shape, dtype, buffer_size):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
file_format = 'raw'
filename_format = "stats_const_{shape}_{dtype}.{format}"
filename = filename_format.format(shape=shape,
dtype=dtype,
format=file_format)
dmin = 3
for d in shape:
assert d >= dmin, "d={} < {}".format(d, dmin)
cube_max_index = tuple(0 for i in shape)
cube_min_index = tuple(1 for i in shape)
cube_zero_index = tuple(2 for i in shape)
cube_value = 1.0
cube_max = 10.0
cube_min = -23.0
cube_zero = 0.0
cube_sum = cube_max + cube_min + cube_value * (shape.count() - dmin)
cube_ave = cube_sum / float(shape.count())
cube = cb.const_cube(shape=shape, dtype=dtype, value=cube_value)
cube[cube_max_index] = cube_max
cube[cube_min_index] = cube_min
cube[cube_zero_index] = cube_zero
cube_count_zero = 1
cube_count_nonzero = shape.count() - cube_count_zero
cube_count_nan = 0
cube_count_inf = 0
stats_info_cube = cb.stats_info(cube)
self.assertEqual(stats_info_cube.cube_sum, cube_sum)
self.assertEqual(stats_info_cube.cube_ave, cube_ave)
self.assertEqual(stats_info_cube.cube_max, cube_max)
self.assertEqual(stats_info_cube.cube_min, cube_min)
self.assertEqual(stats_info_cube.cube_max_index, cube_max_index)
self.assertEqual(stats_info_cube.cube_min_index, cube_min_index)
self.assertEqual(stats_info_cube.cube_count_zero, cube_count_zero)
self.assertEqual(stats_info_cube.cube_count_nonzero,
cube_count_nonzero)
self.assertEqual(stats_info_cube.cube_count_nan, cube_count_nan)
self.assertEqual(stats_info_cube.cube_count_inf, cube_count_inf)
cube.tofile(filename)
self.assertFileExistsAndHasShape(filename, shape=shape, dtype=dtype)
stats_info_oc = cb.stats_file(filename,
shape=shape,
dtype=dtype,
file_format=file_format,
out_of_core=False)
self.assertEqual(stats_info_oc, stats_info_cube)
stats_info_ooc = cb.stats_file(filename,
shape=shape,
dtype=dtype,
file_format=file_format,
out_of_core=True,
progress_frequency=-1.0,
buffer_size=buffer_size)
self.assertEqual(stats_info_ooc, stats_info_cube)
def write_linear_cube_out_of_core_4x4_float32(self):
dtype = np.float32
shape = Shape("4x4")
self.impl_write_cube_out_of_core(
kind="linear", shape=shape, dtype=dtype, buffer_size=shape.count() * dtype().itemsize / 2
)
def impl_write_cube_out_of_core(self, kind, shape, dtype, buffer_size=None):
dtype = utilities.get_dtype(dtype)
shape = Shape(shape)
filename_format_a = "a_{kind}_{{shape}}_{{dtype}}.{{format}}".format(kind=kind)
filename_format_b = "b_{kind}_{{shape}}_{{dtype}}.{{format}}".format(kind=kind)
filename_format_c = "c_{kind}_{{shape}}_{{dtype}}.{{format}}".format(kind=kind)
if issubclass(dtype, np.integer):
l_start = dtype(3)
l_increment = dtype(2)
c_value = dtype(7)
else:
l_start = dtype(0.5)
l_increment = dtype(1.5)
c_value = dtype(7.5)
p_args = []
n_args = {}
if kind == "random":
ooc_function = cb.write_random_cube
oc_function = cb.random_cube
elif kind == "linear":
ooc_function = cb.write_linear_cube
oc_function = cb.linear_cube
n_args["start"] = l_start
n_args["increment"] = l_increment
elif kind == "const":
ooc_function = cb.write_const_cube
oc_function = cb.const_cube
n_args["value"] = c_value
else:
assert False, kind
cb.set_random_seed(10)
ooc_function(filename_format_a, shape=shape, dtype=dtype, *p_args, **n_args)
filename_a = filename_format_a.format(shape=shape, dtype=dtype.__name__, format="raw")
self.assertFileExistsAndHasShape(filename_a, shape=shape, dtype=dtype)
ooc_cube = cb.read_cube_raw(filename_format_a, shape=shape, dtype=dtype)
self.assertTrue(ooc_cube.dtype == dtype)
self.assertEqual(ooc_cube.size, shape.count())
cb.set_random_seed(10)
oc_cube = oc_function(shape=shape, dtype=dtype, *p_args, **n_args)
self.assertCubesAreEqual(ooc_cube, oc_cube)
if kind == "random":
pass
elif kind == "linear":
# 0) start + 0 * increment
# 1) start + 1 * increment
# 2) start + 2 * increment
# 3) start + 3 * increment
# ...
# count) start + count * increment
# == (start * count) + (((count - 1) * count) // 2) * increment
count = shape.count()
sum_count = ((count - 1) * count) // 2
l_sum = (l_start * count) + (l_increment * sum_count)
self.assertEqual(cb.precise_sum(ooc_cube), l_sum)
elif kind == "const":
self.assertEqual(cb.precise_sum(ooc_cube), shape.count() * c_value)
filename_b = filename_format_b.format(shape=shape, dtype=dtype.__name__, format="raw")
ooc_cube.tofile(filename_b)
self.assertFilesAreEqual(filename_a, filename_b)
class RubikTestComparison(RubikTestCase):
METHOD_NAMES = []
def setUp(self):
self.shape = Shape("3x4x5")
self.l0 = cb.linear_cube(self.shape, start=0.5, increment=1.5)
self.l1 = cb.linear_cube(self.shape, start=0.5, increment=0.5)
self.l2 = cb.linear_cube(self.shape, start=-1.0, increment=0.5)
self.epsilon = 1.0e-8
e = self.epsilon
self.c0 = np.array([0.0, 0.0, e, 1.0, 1.0 + e, 1.0], )
self.c1 = np.array([0.0, e, 0.0, 1.0, 1.0, 1.0 + e])
# tolerance 0.0
@testmethod
def not_equals_cube_t0(self):
cube = cb.not_equals_cube(self.l0, self.l1, tolerance=0.0)
self.assertFalse(cube[0, 0, 0])
self.assertEqual(cube.sum(), self.shape.count() - 1)
@testmethod
def equals_cube_t0(self):
cube = cb.equals_cube(self.l0, self.l1, tolerance=0.0)
self.assertTrue(cube[0, 0, 0])
self.assertEqual(cube.sum(), 1)
@testmethod
def not_equals_num_t0(self):
num = cb.not_equals_num(self.l0, self.l1, tolerance=0.0)
self.assertEqual(num, self.shape.count() - 1)
@testmethod
def equals_num_t0(self):
num = cb.equals_num(self.l0, self.l1, tolerance=0.0)
self.assertEqual(num, 1)
@testmethod
def not_equals_t0(self):
self.assertTrue(cb.not_equals(self.l0, self.l1, tolerance=0.0))
@testmethod
def equals_t0(self):
self.assertFalse(cb.equals(self.l0, self.l1, tolerance=0.0))
# tolerance 2.0
@testmethod
def not_equals_cube_t2(self):
cube = cb.not_equals_cube(self.l0, self.l1, tolerance=2.0)
self.assertFalse(cube[0, 0, 0])
self.assertFalse(cube[0, 0, 1])
self.assertFalse(cube[0, 0, 2])
self.assertEqual(cube.sum(), self.shape.count() - 3)
@testmethod
def equals_cube_t2(self):
cube = cb.equals_cube(self.l0, self.l1, tolerance=2.0)
self.assertTrue(cube[0, 0, 0])
self.assertTrue(cube[0, 0, 1])
self.assertTrue(cube[0, 0, 2])
self.assertEqual(cube.sum(), 3)
@testmethod
def not_equals_num_t2(self):
num = cb.not_equals_num(self.l0, self.l1, tolerance=2.0)
self.assertEqual(num, self.shape.count() - 3)
@testmethod
def equals_num_t2(self):
num = cb.equals_num(self.l0, self.l1, tolerance=2.0)
self.assertEqual(num, 3)
@testmethod
def not_equals_t2(self):
self.assertTrue(cb.not_equals(self.l0, self.l1, tolerance=2.0))
@testmethod
def equals_t2(self):
self.assertFalse(cb.equals(self.l0, self.l1, tolerance=2.0))
# tolerance 10000.0
@testmethod
def not_equals_t10000(self):
self.assertFalse(cb.not_equals(self.l0, self.l1, tolerance=10000.0))
@testmethod
def equals_t10000(self):
self.assertTrue(cb.equals(self.l0, self.l1, tolerance=10000.0))
# threshold
@testmethod
def threshold_cube_3(self):
cube = cb.threshold_cube(self.l0, 3.0, value=-9.0)
self.assertEqual(cube[0, 0, 0], -9.0) # 0.5
self.assertEqual(cube[0, 0, 1], -9.0) # 2.0
self.assertEqual(cube[0, 0, 2], self.l0[0, 0, 2]) # 3.5
@testmethod
def threshold_cube_1(self):
cube = cb.threshold_cube(self.l0, 1.0, value=-9.0)
self.assertEqual(cube[0, 0, 0], -9.0) # 0.5
self.assertEqual(cube[0, 0, 1], self.l0[0, 0, 1]) # 2.0
@testmethod
def threshold_cube_0(self):
cube = cb.threshold_cube(self.l2, value=-9.0)
self.assertEqual(cube[0, 0, 0], -9.0) # -1.0
self.assertEqual(cube[0, 0, 1], -9.0) # -0.5
self.assertEqual(cube[0, 0, 2], -9.0) # 0.0
self.assertEqual(cube[0, 0, 3], self.l2[0, 0, 3]) # 0.5
# threshold
@testmethod
def abs_threshold_cube_0p6(self):
cube = cb.abs_threshold_cube(self.l2, 0.6, value=-9.0)
self.assertEqual(cube[0, 0, 0], self.l2[0, 0, 0]) # -1.0
self.assertEqual(cube[0, 0, 1], -9.0) # -0.5
self.assertEqual(cube[0, 0, 2], -9.0) # 0.0
self.assertEqual(cube[0, 0, 3], -9.0) # 0.5
self.assertEqual(cube[0, 0, 4], self.l2[0, 0, 4]) # 1.0
# abs_diff_cube
# self.c0:
# [0.0, 0.0, e, 1.0, 1.0 + e, 1.0],
# self.c1:
# [0.0, e, 0.0, 1.0, 1.0, 1.0 + e]
@testmethod
def abs_diff_cube_t0(self):
e = self.epsilon
cube = cb.abs_diff_cube(self.c0, self.c1)
cube_cmp = np.array([0.0, e, e, 0.0, e, e])
for v0, v1 in zip(self.c0, self.c1):
self.assertAlmostEqual(v0, v1)
@testmethod
def abs_diff_cube_inTn6(self):
e = self.epsilon
cube = cb.abs_diff_cube(self.c0, self.c1, in_threshold=1e-6)
cube_cmp = np.array([0.0, 0.0, 0.0, 0.0, e, e])
for v0, v1 in zip(self.c0, self.c1):
self.assertAlmostEqual(v0, v1)
@testmethod
def abs_diff_cube_outTn6(self):
e = self.epsilon
cube = cb.abs_diff_cube(self.c0, self.c1, out_threshold=1e-6)
cube_cmp = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0])
for v0, v1 in zip(self.c0, self.c1):
self.assertAlmostEqual(v0, v1)
@testmethod
def abs_diff_cube_inTn6_outTn6(self):
e = self.epsilon
cube = cb.abs_diff_cube(self.c0,
self.c1,
in_threshold=1e-6,
out_threshold=1e-6)
cube_cmp = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0])
for v0, v1 in zip(self.c0, self.c1):
self.assertAlmostEqual(v0, v1)
# rel_diff_cube
# self.c0:
# [0.0, 0.0, e, 1.0, 1.0 + e, 1.0],
# self.c1:
# [0.0, e, 0.0, 1.0, 1.0, 1.0 + e]
@testmethod
def rel_diff_cube_t0(self):
e = self.epsilon
cube = cb.rel_diff_cube(self.c0, self.c1)
self.assertEqual(cube[0], 0.0)
self.assertGreater(cube[1], 1.0e30)
self.assertGreater(cube[2], 0.0)
self.assertAlmostEqual(cube[3], e / (1.0 + e))
self.assertAlmostEqual(cube[4], e)
@testmethod
def rel_diff_cube_inTn6(self):
e = self.epsilon
cube = cb.rel_diff_cube(self.c0, self.c1, in_threshold=1e-6)
self.assertEqual(cube[0], 0.0)
self.assertEqual(cube[1], 0.0)
self.assertEqual(cube[2], 0.0)
self.assertAlmostEqual(cube[3], e / (1.0 + e))
self.assertAlmostEqual(cube[4], e)
@testmethod
def rel_diff_cube_outTn6(self):
e = self.epsilon
cube = cb.rel_diff_cube(self.c0, self.c1, out_threshold=1e-6)
self.assertEqual(cube[0], 0.0)
self.assertGreater(cube[1], 1.0e30)
self.assertGreater(cube[2], 0.0)
self.assertEqual(cube[3], 0.0)
self.assertEqual(cube[4], 0.0)
@testmethod
def rel_diff_cube_inTn6_outTn6(self):
e = self.epsilon
cube = cb.rel_diff_cube(self.c0,
self.c1,
in_threshold=1e-6,
out_threshold=1e-6)
self.assertEqual(cube[0], 0.0)
self.assertEqual(cube[1], 0.0)
self.assertEqual(cube[2], 0.0)
self.assertEqual(cube[3], 0.0)
self.assertEqual(cube[4], 0.0)
@testmethod
def rel_diff_cube_fraction(self):
c0 = np.array([2.0])
c1 = np.array([1.0])
cube = cb.rel_diff_cube(c0, c1)
self.assertAlmostEqual(cube[0], 0.5)
@testmethod
def rel_diff_cube_percentage(self):
c0 = np.array([2.0])
c1 = np.array([1.0])
cube = cb.rel_diff_cube(c0, c1, percentage=True)
self.assertAlmostEqual(cube[0], 50.0)
@testmethod
def zero_cube(self):
#c0: [0.0, 0.0, e, 1.0, 1.0 + e, 1.0],
cube = cb.zero_cube(self.c0)
cube_cmp = np.array([True, True, False, False, False, False])
for v0, v1 in zip(cube, cube_cmp):
self.assertEqual(v0, v1)
@testmethod
def zero_cube_Tn6(self):
#c0: [0.0, 0.0, e, 1.0, 1.0 + e, 1.0],
cube = cb.zero_cube(self.c0, tolerance=1e-6)
cube_cmp = np.array([True, True, True, False, False, False])
for v0, v1 in zip(cube, cube_cmp):
self.assertEqual(v0, v1)
@testmethod
def nonzero_cube(self):
#c0: [0.0, 0.0, e, 1.0, 1.0 + e, 1.0],
cube = cb.nonzero_cube(self.c0)
cube_cmp = np.array([True, True, False, False, False, False])
for v0, v1 in zip(cube, cube_cmp):
self.assertEqual(v0, not v1)
@testmethod
def nonzero_cube_Tn6(self):
#c0: [0.0, 0.0, e, 1.0, 1.0 + e, 1.0],
cube = cb.nonzero_cube(self.c0, tolerance=1e-6)
cube_cmp = np.array([True, True, True, False, False, False])
for v0, v1 in zip(cube, cube_cmp):
self.assertEqual(v0, not v1)
@testmethod
def where_indices(self):
a = np.eye(3)
a[1, 2] = -2.0
a[2, 1] = 2.0
cube = cb.where_indices(a)
self.assertEqual(cube.shape, (5, 3))
self.assertCubesAreEqual(cube[0], np.array([0.0, 0.0, 1.0]))
self.assertCubesAreEqual(cube[1], np.array([1.0, 1.0, 1.0]))
self.assertCubesAreEqual(cube[2], np.array([1.0, 2.0, -2.0]))
self.assertCubesAreEqual(cube[3], np.array([2.0, 1.0, 2.0]))
self.assertCubesAreEqual(cube[4], np.array([2.0, 2.0, 1.0]))
@testmethod
def where_indices_condition(self):
a = np.eye(3)
a[1, 2] = -2.0
a[2, 1] = 2.0
cube = cb.where_indices(a, np.abs(a) > 1.0)
self.assertEqual(cube.shape, (2, 3))
self.assertCubesAreEqual(cube[0], np.array([1.0, 2.0, -2.0]))
self.assertCubesAreEqual(cube[1], np.array([2.0, 1.0, 2.0]))
def impl_write_cube_out_of_core(self,
kind,
shape,
dtype,
buffer_size=None):
dtype = utilities.get_dtype(dtype)
shape = Shape(shape)
filename_format_a = "a_{kind}_{{shape}}_{{dtype}}.{{format}}".format(
kind=kind)
filename_format_b = "b_{kind}_{{shape}}_{{dtype}}.{{format}}".format(
kind=kind)
filename_format_c = "c_{kind}_{{shape}}_{{dtype}}.{{format}}".format(
kind=kind)
if issubclass(dtype, np.integer):
l_start = dtype(3)
l_increment = dtype(2)
c_value = dtype(7)
else:
l_start = dtype(0.5)
l_increment = dtype(1.5)
c_value = dtype(7.5)
p_args = []
n_args = {}
if kind == 'random':
ooc_function = cb.write_random_cube
oc_function = cb.random_cube
elif kind == 'linear':
ooc_function = cb.write_linear_cube
oc_function = cb.linear_cube
n_args['start'] = l_start
n_args['increment'] = l_increment
elif kind == 'const':
ooc_function = cb.write_const_cube
oc_function = cb.const_cube
n_args['value'] = c_value
else:
assert False, kind
cb.set_random_seed(10)
ooc_function(filename_format_a,
shape=shape,
dtype=dtype,
*p_args,
**n_args)
filename_a = filename_format_a.format(
shape=shape,
dtype=dtype.__name__,
format='raw',
)
self.assertFileExistsAndHasShape(filename_a, shape=shape, dtype=dtype)
ooc_cube = cb.read_cube_raw(filename_format_a,
shape=shape,
dtype=dtype)
self.assertTrue(ooc_cube.dtype == dtype)
self.assertEqual(ooc_cube.size, shape.count())
cb.set_random_seed(10)
oc_cube = oc_function(shape=shape, dtype=dtype, *p_args, **n_args)
self.assertCubesAreEqual(ooc_cube, oc_cube)
if kind == 'random':
pass
elif kind == 'linear':
# 0) start + 0 * increment
# 1) start + 1 * increment
# 2) start + 2 * increment
# 3) start + 3 * increment
# ...
# count) start + count * increment
# == (start * count) + (((count - 1) * count) // 2) * increment
count = shape.count()
sum_count = ((count - 1) * count) // 2
l_sum = (l_start * count) + (l_increment * sum_count)
self.assertEqual(cb.precise_sum(ooc_cube), l_sum)
elif kind == 'const':
self.assertEqual(cb.precise_sum(ooc_cube), shape.count() * c_value)
filename_b = filename_format_b.format(
shape=shape,
dtype=dtype.__name__,
format='raw',
)
ooc_cube.tofile(filename_b)
self.assertFilesAreEqual(filename_a, filename_b)
def count(self):
for shape_t, shape_s, count in self._shapes:
shape_ft = Shape(shape_t)
shape_fs = Shape(shape_s)
self.assertEqual(shape_ft.count(), shape_fs.count())
self.assertEqual(shape_ft.count(), count)
class RubikTestComparison(RubikTestCase):
METHOD_NAMES = []
def setUp(self):
self.shape = Shape("3x4x5")
self.l0 = cb.linear_cube(self.shape, start=0.5, increment=1.5)
self.l1 = cb.linear_cube(self.shape, start=0.5, increment=0.5)
self.l2 = cb.linear_cube(self.shape, start=-1.0, increment=0.5)
self.epsilon = 1.0e-8
e = self.epsilon
self.c0 = np.array([0.0, 0.0, e, 1.0, 1.0 + e, 1.0])
self.c1 = np.array([0.0, e, 0.0, 1.0, 1.0, 1.0 + e])
# tolerance 0.0
@testmethod
def not_equals_cube_t0(self):
cube = cb.not_equals_cube(self.l0, self.l1, tolerance=0.0)
self.assertFalse(cube[0, 0, 0])
self.assertEqual(cube.sum(), self.shape.count() - 1)
@testmethod
def equals_cube_t0(self):
cube = cb.equals_cube(self.l0, self.l1, tolerance=0.0)
self.assertTrue(cube[0, 0, 0])
self.assertEqual(cube.sum(), 1)
@testmethod
def not_equals_num_t0(self):
num = cb.not_equals_num(self.l0, self.l1, tolerance=0.0)
self.assertEqual(num, self.shape.count() - 1)
@testmethod
def equals_num_t0(self):
num = cb.equals_num(self.l0, self.l1, tolerance=0.0)
self.assertEqual(num, 1)
@testmethod
def not_equals_t0(self):
self.assertTrue(cb.not_equals(self.l0, self.l1, tolerance=0.0))
@testmethod
def equals_t0(self):
self.assertFalse(cb.equals(self.l0, self.l1, tolerance=0.0))
# tolerance 2.0
@testmethod
def not_equals_cube_t2(self):
cube = cb.not_equals_cube(self.l0, self.l1, tolerance=2.0)
self.assertFalse(cube[0, 0, 0])
self.assertFalse(cube[0, 0, 1])
self.assertFalse(cube[0, 0, 2])
self.assertEqual(cube.sum(), self.shape.count() - 3)
@testmethod
def equals_cube_t2(self):
cube = cb.equals_cube(self.l0, self.l1, tolerance=2.0)
self.assertTrue(cube[0, 0, 0])
self.assertTrue(cube[0, 0, 1])
self.assertTrue(cube[0, 0, 2])
self.assertEqual(cube.sum(), 3)
@testmethod
def not_equals_num_t2(self):
num = cb.not_equals_num(self.l0, self.l1, tolerance=2.0)
self.assertEqual(num, self.shape.count() - 3)
@testmethod
def equals_num_t2(self):
num = cb.equals_num(self.l0, self.l1, tolerance=2.0)
self.assertEqual(num, 3)
@testmethod
def not_equals_t2(self):
self.assertTrue(cb.not_equals(self.l0, self.l1, tolerance=2.0))
@testmethod
def equals_t2(self):
self.assertFalse(cb.equals(self.l0, self.l1, tolerance=2.0))
# tolerance 10000.0
@testmethod
def not_equals_t10000(self):
self.assertFalse(cb.not_equals(self.l0, self.l1, tolerance=10000.0))
@testmethod
def equals_t10000(self):
self.assertTrue(cb.equals(self.l0, self.l1, tolerance=10000.0))
# threshold
@testmethod
def threshold_cube_3(self):
cube = cb.threshold_cube(self.l0, 3.0, value=-9.0)
self.assertEqual(cube[0, 0, 0], -9.0) # 0.5
self.assertEqual(cube[0, 0, 1], -9.0) # 2.0
self.assertEqual(cube[0, 0, 2], self.l0[0, 0, 2]) # 3.5
@testmethod
def threshold_cube_1(self):
cube = cb.threshold_cube(self.l0, 1.0, value=-9.0)
self.assertEqual(cube[0, 0, 0], -9.0) # 0.5
self.assertEqual(cube[0, 0, 1], self.l0[0, 0, 1]) # 2.0
@testmethod
def threshold_cube_0(self):
cube = cb.threshold_cube(self.l2, value=-9.0)
self.assertEqual(cube[0, 0, 0], -9.0) # -1.0
self.assertEqual(cube[0, 0, 1], -9.0) # -0.5
self.assertEqual(cube[0, 0, 2], -9.0) # 0.0
self.assertEqual(cube[0, 0, 3], self.l2[0, 0, 3]) # 0.5
# threshold
@testmethod
def abs_threshold_cube_0p6(self):
cube = cb.abs_threshold_cube(self.l2, 0.6, value=-9.0)
self.assertEqual(cube[0, 0, 0], self.l2[0, 0, 0]) # -1.0
self.assertEqual(cube[0, 0, 1], -9.0) # -0.5
self.assertEqual(cube[0, 0, 2], -9.0) # 0.0
self.assertEqual(cube[0, 0, 3], -9.0) # 0.5
self.assertEqual(cube[0, 0, 4], self.l2[0, 0, 4]) # 1.0
# abs_diff_cube
# self.c0:
# [0.0, 0.0, e, 1.0, 1.0 + e, 1.0],
# self.c1:
# [0.0, e, 0.0, 1.0, 1.0, 1.0 + e]
@testmethod
def abs_diff_cube_t0(self):
e = self.epsilon
cube = cb.abs_diff_cube(self.c0, self.c1)
cube_cmp = np.array([0.0, e, e, 0.0, e, e])
for v0, v1 in zip(self.c0, self.c1):
self.assertAlmostEqual(v0, v1)
@testmethod
def abs_diff_cube_inTn6(self):
e = self.epsilon
cube = cb.abs_diff_cube(self.c0, self.c1, in_threshold=1e-6)
cube_cmp = np.array([0.0, 0.0, 0.0, 0.0, e, e])
for v0, v1 in zip(self.c0, self.c1):
self.assertAlmostEqual(v0, v1)
@testmethod
def abs_diff_cube_outTn6(self):
e = self.epsilon
cube = cb.abs_diff_cube(self.c0, self.c1, out_threshold=1e-6)
cube_cmp = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0])
for v0, v1 in zip(self.c0, self.c1):
self.assertAlmostEqual(v0, v1)
@testmethod
def abs_diff_cube_inTn6_outTn6(self):
e = self.epsilon
cube = cb.abs_diff_cube(self.c0, self.c1, in_threshold=1e-6, out_threshold=1e-6)
cube_cmp = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0])
for v0, v1 in zip(self.c0, self.c1):
self.assertAlmostEqual(v0, v1)
# rel_diff_cube
# self.c0:
# [0.0, 0.0, e, 1.0, 1.0 + e, 1.0],
# self.c1:
# [0.0, e, 0.0, 1.0, 1.0, 1.0 + e]
@testmethod
def rel_diff_cube_t0(self):
e = self.epsilon
cube = cb.rel_diff_cube(self.c0, self.c1)
self.assertEqual(cube[0], 0.0)
self.assertGreater(cube[1], 1.0e30)
self.assertGreater(cube[2], 0.0)
self.assertAlmostEqual(cube[3], e / (1.0 + e))
self.assertAlmostEqual(cube[4], e)
@testmethod
def rel_diff_cube_inTn6(self):
e = self.epsilon
cube = cb.rel_diff_cube(self.c0, self.c1, in_threshold=1e-6)
self.assertEqual(cube[0], 0.0)
self.assertEqual(cube[1], 0.0)
self.assertEqual(cube[2], 0.0)
self.assertAlmostEqual(cube[3], e / (1.0 + e))
self.assertAlmostEqual(cube[4], e)
@testmethod
def rel_diff_cube_outTn6(self):
e = self.epsilon
cube = cb.rel_diff_cube(self.c0, self.c1, out_threshold=1e-6)
self.assertEqual(cube[0], 0.0)
self.assertGreater(cube[1], 1.0e30)
self.assertGreater(cube[2], 0.0)
self.assertEqual(cube[3], 0.0)
self.assertEqual(cube[4], 0.0)
@testmethod
def rel_diff_cube_inTn6_outTn6(self):
e = self.epsilon
cube = cb.rel_diff_cube(self.c0, self.c1, in_threshold=1e-6, out_threshold=1e-6)
self.assertEqual(cube[0], 0.0)
self.assertEqual(cube[1], 0.0)
self.assertEqual(cube[2], 0.0)
self.assertEqual(cube[3], 0.0)
self.assertEqual(cube[4], 0.0)
@testmethod
def rel_diff_cube_fraction(self):
c0 = np.array([2.0])
c1 = np.array([1.0])
cube = cb.rel_diff_cube(c0, c1)
self.assertAlmostEqual(cube[0], 0.5)
@testmethod
def rel_diff_cube_percentage(self):
c0 = np.array([2.0])
c1 = np.array([1.0])
cube = cb.rel_diff_cube(c0, c1, percentage=True)
self.assertAlmostEqual(cube[0], 50.0)
@testmethod
def zero_cube(self):
# c0: [0.0, 0.0, e, 1.0, 1.0 + e, 1.0],
cube = cb.zero_cube(self.c0)
cube_cmp = np.array([True, True, False, False, False, False])
for v0, v1 in zip(cube, cube_cmp):
self.assertEqual(v0, v1)
@testmethod
def zero_cube_Tn6(self):
# c0: [0.0, 0.0, e, 1.0, 1.0 + e, 1.0],
cube = cb.zero_cube(self.c0, tolerance=1e-6)
cube_cmp = np.array([True, True, True, False, False, False])
for v0, v1 in zip(cube, cube_cmp):
self.assertEqual(v0, v1)
@testmethod
def nonzero_cube(self):
# c0: [0.0, 0.0, e, 1.0, 1.0 + e, 1.0],
cube = cb.nonzero_cube(self.c0)
cube_cmp = np.array([True, True, False, False, False, False])
for v0, v1 in zip(cube, cube_cmp):
self.assertEqual(v0, not v1)
@testmethod
def nonzero_cube_Tn6(self):
# c0: [0.0, 0.0, e, 1.0, 1.0 + e, 1.0],
cube = cb.nonzero_cube(self.c0, tolerance=1e-6)
cube_cmp = np.array([True, True, True, False, False, False])
for v0, v1 in zip(cube, cube_cmp):
self.assertEqual(v0, not v1)
@testmethod
def where_indices(self):
a = np.eye(3)
a[1, 2] = -2.0
a[2, 1] = 2.0
cube = cb.where_indices(a)
self.assertEqual(cube.shape, (5, 3))
self.assertCubesAreEqual(cube[0], np.array([0.0, 0.0, 1.0]))
self.assertCubesAreEqual(cube[1], np.array([1.0, 1.0, 1.0]))
self.assertCubesAreEqual(cube[2], np.array([1.0, 2.0, -2.0]))
self.assertCubesAreEqual(cube[3], np.array([2.0, 1.0, 2.0]))
self.assertCubesAreEqual(cube[4], np.array([2.0, 2.0, 1.0]))
@testmethod
def where_indices_condition(self):
a = np.eye(3)
a[1, 2] = -2.0
a[2, 1] = 2.0
cube = cb.where_indices(a, np.abs(a) > 1.0)
self.assertEqual(cube.shape, (2, 3))
self.assertCubesAreEqual(cube[0], np.array([1.0, 2.0, -2.0]))
self.assertCubesAreEqual(cube[1], np.array([2.0, 1.0, 2.0]))
