def setUp(self):
super(RubikTestStd, self).setUp()
self.X = 8
self.Y = 10
self.Z = 30
self.XYZ = Shape((self.X, self.Y, self.Z))
self.XY = Shape((self.X, self.Y))
self.file_format = 'raw'
self.filename_2d = {}
self.filename_3d = {}
for names, shape, filename_dict in ((('a', 'b', 'l'), self.XY,
self.filename_2d),
(('l', ), self.XYZ,
self.filename_3d)):
for name in names:
out_filename_format = "{n}_{{rank}}d_{{shape}}.{{format}}".format(
n=name)
out_filename = out_filename_format.format(
n=name,
rank=shape.rank(),
shape=shape,
format=self.file_format)
filename_dict[name] = out_filename
returncode, output, error = self.run_program(
"""-e 'cb.linear_cube("{s}")' -o '{o}'""".format(
s=shape, o=out_filename_format))
self.assertEqual(returncode, 0)
def getitem(self):
for shape_t, shape_s, count in self._shapes:
shape_ft = Shape(shape_t)
shape_fs = Shape(shape_s)
for index, d in enumerate(shape_t):
self.assertEqual(shape_ft[index], d)
self.assertEqual(shape_fs[index], d)
def iter(self):
for shape_t, shape_s, count in self._shapes:
shape_ft = Shape(shape_t)
shape_fs = Shape(shape_s)
for d, dt, ds in zip(shape_t, shape_ft, shape_fs):
self.assertEqual(dt, d)
self.assertEqual(ds, d)
def setUp(self):
super(RubikTestWork, self).setUp()
self.X = 10
self.Y = 20
self.Z = 30
self.H = 501
self.XYZ = Shape((self.X, self.Y, self.Z))
self.HXYZ = Shape((self.H, self.X, self.Y, self.Z))
self.XYHZ = Shape((self.X, self.Y, self.H, self.Z))
self.A = 8
self.B = 10
self.AB = Shape((self.A, self.B))
self.CONST_VALUE = 7.0
self.file_format = 'raw'
self.im_shape = self.XYZ
self.og_shape = self.XYHZ
self.c_shape = self.XYZ
self.lin_shape = self.AB
self.im_filename_format = 'im_{shape}.{format}'
self.im_filename = self.im_filename_format.format(shape=self.im_shape, format=self.file_format)
self.og_filename_format = 'im_{shape}.{format}'
self.og_filename = self.og_filename_format.format(shape=self.og_shape, format=self.file_format)
self.c_filename_format = 'c_{shape}.{format}'
self.c_filename = self.c_filename_format.format(shape=self.c_shape, format=self.file_format)
self.lin_filename_format = 'lin_{shape}.{format}'
self.lin_filename = self.lin_filename_format.format(shape=self.lin_shape, format=self.file_format)
returncode, output, error = self.run_program(
"""-e 'cb.random_cube("{s}")' -o '{o}'""".format(
s=self.im_shape,
o=self.im_filename_format))
self.assertEqual(returncode, 0)
returncode, output, error = self.run_program(
"""-e 'cb.random_cube("{s}")' -o '{o}'""".format(
s=self.og_shape,
o=self.og_filename_format))
self.assertEqual(returncode, 0)
returncode, output, error = self.run_program(
"""-e 'cb.const_cube("{s}", value={c})' -o '{o}'""".format(
s=self.c_shape,
c=self.CONST_VALUE,
o=self.c_filename_format))
self.assertEqual(returncode, 0)
returncode, output, error = self.run_program(
"""-e 'cb.linear_cube("{s}")' -o '{o}'""".format(
s=self.lin_shape,
c=self.CONST_VALUE,
o=self.lin_filename_format))
self.assertEqual(returncode, 0)
def rank(self):
for shape_t, shape_s, count in self._shapes:
shape_ft = Shape(shape_t)
shape_fs = Shape(shape_s)
self.assertEqual(len(shape_fs), len(shape_t))
self.assertEqual(len(shape_ft), len(shape_t))
self.assertEqual(shape_fs.rank(), len(shape_t))
self.assertEqual(shape_ft.rank(), len(shape_t))
def write_linear_cube_out_of_core_4x4_int64(self):
dtype = np.int64
shape = Shape("4x4")
self.impl_write_cube_out_of_core(kind='linear',
shape=shape,
dtype=dtype,
buffer_size=shape.count() *
dtype().itemsize / 2)
def assertFileExistsAndHasShape(self, filename, shape, dtype=None):
self.assertFileExists(filename)
dtype = cb.get_dtype(dtype)
if not isinstance(shape, Shape):
shape = Shape(shape)
size = shape.count() * dtype().itemsize
filesize = os.stat(filename).st_size
if filesize != size:
raise AssertionError("file {!r} has size {} != {} [shape={}, dtype={}]".format(filename, filesize, size, shape, dtype.__name__))
def cube_6D(self):
shape = Shape("6x10x5x9x4x12")
out_filename_format = "ltmp_{rank}d_{shape}.{format}"
out_filename = out_filename_format.format(shape=shape,
format=self.file_format,
rank=shape.rank())
returncode, output, error = self.run_program(
"""-e 'cb.linear_cube("{s}")' -o '{o}'""".format(
s=shape, o=out_filename_format))
self.assertFileExistsAndHasShape(out_filename, shape)
def setUp(self):
super(RubikTestExtractor, self).setUp()
shape_20x30x40 = Shape("20x30x40")
shape_30x40x20 = Shape("30x40x20")
shape_40x20x30 = Shape("40x20x30")
self._exs = collections.OrderedDict((
('5:,:10,::5', (shape_20x30x40, 24000, 15 * 10 * 8)),
('5:,:10,::5', (shape_30x40x20, 24000, 25 * 10 * 4)),
('5:,:10,::5', (shape_40x20x30, 24000, 35 * 10 * 6)),
))
def setUp(self):
self.shape = Shape("3x4x5")
self.l0 = cb.linear_cube(self.shape, start=0.5, increment=1.5)
self.l1 = cb.linear_cube(self.shape, start=0.5, increment=0.5)
self.l2 = cb.linear_cube(self.shape, start=-1.0, increment=0.5)
self.epsilon = 1.0e-8
e = self.epsilon
self.c0 = np.array([0.0, 0.0, e, 1.0, 1.0 + e, 1.0], )
self.c1 = np.array([0.0, e, 0.0, 1.0, 1.0, 1.0 + e])
def impl_data_type(self, keys, actual_data_type=None, actual_file_format=None, const_value=None):
shape = Shape("4x5")
data_type = keys.get('data_type', 'float32')
options = keys.get('options', None)
dtype = cb.get_dtype(data_type)
file_format = keys.get('file_format', 'raw')
if actual_data_type is None:
actual_data_type = data_type
actual_dtype = cb.get_dtype(actual_data_type)
if actual_file_format is None:
actual_file_format = file_format
rubik_dir = os.getcwd()
rubik_config = config.get_config()
rubik_config_data_type = os.path.join(rubik_dir, "rubik.config.{data_type}.{file_format}")
rubik_config.default_data_type = data_type
rubik_config.default_file_format = file_format
if options is not None:
rubik_config.default_options = options
with open(rubik_config_data_type, "w") as f:
rubik_config.write_config_file(f)
current_rubik_dir = os.environ.get("RUBIK_DIR", None)
current_rubik_config = os.environ.get("RUBIK_CONFIG", None)
out_filename_format = 'ctmp_{shape}_{dtype}.{format}'
out_filename = out_filename_format.format(shape=shape, dtype=actual_data_type, format=file_format)
try:
os.environ["RUBIK_DIR"] = rubik_dir
os.environ["RUBIK_CONFIG"] = rubik_config_data_type
environment.load_rubik_environment()
returncode, output, error = self.run_program(
"""-e 'cb.const_cube("{s}", value={c})' -o '{o}'""".format(
s=shape,
c=const_value,
o=out_filename_format))
self.assertEqual(returncode, 0)
if actual_file_format == 'raw':
self.assertFileExistsAndHasShape(out_filename, shape=shape, dtype=actual_data_type)
else:
self.assertFileExists(out_filename)
returncode, output, error = self.run_program(
"""-i '{o}' -s {s} -e 'i0.sum()' --print""".format(
s=shape,
o=out_filename_format),
join_stderr_stdout=False)
value = actual_dtype(output.rstrip('\n'))
self.assertEqual(value, shape.count() * const_value)
finally:
if current_rubik_dir is None:
del os.environ["RUBIK_DIR"]
else:
os.environ["RUBIK_DIR"] = current_rubik_dir
if current_rubik_config is None:
del os.environ["RUBIK_CONFIG"]
else:
os.environ["RUBIK_CONFIG"] = rubik_config_data_type
environment.load_rubik_environment()
def impl_stats_const_file(self, shape, dtype, buffer_size):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
file_format = 'raw'
filename_format = "stats_const_{shape}_{dtype}.{format}"
filename = filename_format.format(shape=shape, dtype=dtype, format=file_format)
dmin = 3
for d in shape:
assert d >= dmin, "d={} < {}".format(d, dmin)
cube_max_index = tuple(0 for i in shape)
cube_min_index = tuple(1 for i in shape)
cube_zero_index = tuple(2 for i in shape)
cube_value = 1.0
cube_max = 10.0
cube_min = -23.0
cube_zero = 0.0
cube_sum = cube_max + cube_min + cube_value * (shape.count() - dmin)
cube_ave = cube_sum / float(shape.count())
cube = cb.const_cube(shape=shape, dtype=dtype, value=cube_value)
cube[cube_max_index] = cube_max
cube[cube_min_index] = cube_min
cube[cube_zero_index] = cube_zero
cube_count_zero = 1
cube_count_nonzero = shape.count() - cube_count_zero
cube_count_nan = 0
cube_count_inf = 0
stats_info_cube = cb.stats_info(cube)
self.assertEqual(stats_info_cube.cube_sum, cube_sum)
self.assertEqual(stats_info_cube.cube_ave, cube_ave)
self.assertEqual(stats_info_cube.cube_max, cube_max)
self.assertEqual(stats_info_cube.cube_min, cube_min)
self.assertEqual(stats_info_cube.cube_max_index, cube_max_index)
self.assertEqual(stats_info_cube.cube_min_index, cube_min_index)
self.assertEqual(stats_info_cube.cube_count_zero, cube_count_zero)
self.assertEqual(stats_info_cube.cube_count_nonzero, cube_count_nonzero)
self.assertEqual(stats_info_cube.cube_count_nan, cube_count_nan)
self.assertEqual(stats_info_cube.cube_count_inf, cube_count_inf)
cube.tofile(filename)
self.assertFileExistsAndHasShape(filename, shape=shape, dtype=dtype)
stats_info_oc = cb.stats_file(filename, shape=shape, dtype=dtype, file_format=file_format,
out_of_core=False)
self.assertEqual(stats_info_oc, stats_info_cube)
stats_info_ooc = cb.stats_file(filename, shape=shape, dtype=dtype, file_format=file_format,
out_of_core=True, progress_frequency=-1.0, buffer_size=buffer_size)
self.assertEqual(stats_info_ooc, stats_info_cube)
def ne(self):
for shape_t0, shape_s0, count0 in self._shapes:
for shape_t1, shape_s1, count1 in self._shapes:
s0 = Shape(shape_t0)
s1 = Shape(shape_t1)
if shape_t0 == shape_t1:
self.assertEqual(s0, s1)
self.assertFalse(s0 != s1)
self.assertTrue(s0 == s1)
else:
self.assertNotEqual(s0, s1)
self.assertTrue(s0 != s1)
self.assertFalse(s0 == s1)
def stats_const_file_12x8x19x5_float64_3chunks(self):
dtype = np.float64
shape = Shape("12x8x19x5")
self.impl_stats_const_file(shape=shape,
dtype=dtype,
buffer_size=self.get_buffer_size(
shape=shape, dtype=dtype, chunks=3))
def stats_const_file_4x4_float32_2chunks(self):
dtype = np.float32
shape = Shape("4x4")
self.impl_stats_const_file(shape=shape,
dtype=dtype,
buffer_size=self.get_buffer_size(
shape=shape, dtype=dtype, chunks=2))
def stats_random_file_12x8x19x5_float32_2chunks(self):
dtype = np.float32
shape = Shape("12x8x19x5")
self.impl_stats_random_file(shape=shape,
dtype=dtype,
buffer_size=self.get_buffer_size(
shape=shape, dtype=dtype, chunks=2))
def stats_random_file_4x4_float64_2chunks(self):
dtype = np.float64
shape = Shape("4x4")
self.impl_stats_random_file(shape=shape,
dtype=dtype,
buffer_size=self.get_buffer_size(
shape=shape, dtype=dtype, chunks=3))
def setUp(self):
super(RubikTestInfo, self).setUp()
self.X = 8
self.Y = 10
self.Z = 30
self.XYZ = Shape((self.X, self.Y, self.Z))
self.file_format = 'raw'
self.filename_3d = {}
for name_starts, shape, filename_dict in (((('l0', 0), ('l1', 1),
('l2', 2)), self.XYZ,
self.filename_3d), ):
for name, start in name_starts:
out_filename_format = "{n}_{{rank}}d_{{shape}}.{{format}}".format(
n=name)
out_filename = out_filename_format.format(
n=name,
rank=shape.rank(),
shape=shape,
format=self.file_format)
filename_dict[name] = out_filename
returncode, output, error = self.run_program(
"""-e 'cb.linear_cube("{s}", start={S})' -o '{o}'""".
format(s=shape, S=start, o=out_filename_format))
self.assertEqual(returncode, 0)
def setUp(self):
super(RubikTestUsage, self).setUp()
self.X = 8
self.Y = 10
self.Z = 20
self.XYZ = Shape((self.X, self.Y, self.Z))
self.shape = self.XYZ
self.file_format = 'raw'
# create random cube:
self.r_filename_format = 'r_{shape}.{format}'
self.r_filename = self.r_filename_format.format(
shape=self.shape, format=self.file_format)
returncode, output, error = self.run_program(
"""-e 'cb.random_cube("{s}")' -o '{o}'""".format(
s=self.shape, o=self.r_filename_format))
# create linear cube:
self.l_filename_format = 'l_{shape}.{format}'
self.l_filename = self.l_filename_format.format(
shape=self.shape, format=self.file_format)
returncode, output, error = self.run_program(
"""-e 'cb.linear_cube("{s}")' -o '{o}'""".format(
s=self.shape, o=self.l_filename_format))
def extractor(self):
for extractor, sub_shape in (
(":,2:-2,:", Shape((self.X, max(0, self.Y - 4), self.Z))),
(":,::2,::4", Shape((self.X, (self.Y // 2), (self.Z // 4)))),
(":,5,:", Shape((self.X, self.Z))),
(":,::2,:", Shape((self.X, (self.Y // 2), self.Z))),
):
out_filename_format = 'rtmp4_{shape}.{format}'
out_filename = out_filename_format.format(shape=sub_shape,
format=self.file_format)
returncode, output, error = self.run_program(
"""-i '{r}' -s '{s}' -o '{o}' -x '{x}'""".format(
s=self.shape,
x=extractor,
r=self.r_filename_format,
o=out_filename_format))
self.assertFileExistsAndHasShape(out_filename, sub_shape)
def cube_6D(self):
shape = Shape("6x10x5x9x4x12")
out_filename_format = "ltmp_{rank}d_{shape}.{format}"
out_filename = out_filename_format.format(shape=shape, format=self.file_format, rank=shape.rank())
returncode, output, error = self.run_program(
"""-e 'cb.linear_cube("{s}")' -o '{o}'""".format(
s=shape,
o=out_filename_format))
self.assertFileExistsAndHasShape(out_filename, shape)
def setUp(self):
self.shape = Shape("3x4x5")
self.l0 = cb.linear_cube(self.shape, start=0.5, increment=1.5)
self.l1 = cb.linear_cube(self.shape, start=0.5, increment=0.5)
self.l2 = cb.linear_cube(self.shape, start=-1.0, increment=0.5)
self.epsilon = 1.0e-8
e = self.epsilon
self.c0 = np.array([0.0, 0.0, e, 1.0, 1.0 + e, 1.0])
self.c1 = np.array([0.0, e, 0.0, 1.0, 1.0, 1.0 + e])
def impl_histogram(self, shape, dtype, hlength, bins, mode):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
cube = cb.random_cube(shape=shape, dtype=dtype)
output_lines = cb.histogram(cube,
hlength=hlength,
bins=bins,
mode=mode)
self.assertEqual(len(output_lines), bins)
for line in output_lines:
self.assertEqual(len(line), hlength)
def rank(self):
for shape_t, shape_s, count in self._shapes:
shape_ft = Shape(shape_t)
shape_fs = Shape(shape_s)
self.assertEqual(len(shape_fs), len(shape_t))
self.assertEqual(len(shape_ft), len(shape_t))
self.assertEqual(shape_fs.rank(), len(shape_t))
self.assertEqual(shape_ft.rank(), len(shape_t))
def impl_const_cube(self, shape, dtype, value):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
file_format = 'raw'
filename_format = "const_{{shape}}_{{dtype}}_{value}.{{format}}".format(
value=value)
filename = filename_format.format(shape=shape,
dtype=dtype,
format=file_format)
cube = cb.const_cube(shape=shape, dtype=dtype, value=value)
cube_min = cube.min()
cube_max = cube.max()
self.assertEqual(cube_min, value)
self.assertEqual(cube_max, value)
self.assertEqual(dtype, cube.dtype.type)
def split(self):
out_filename_format = 'rtmp5_y{d1}_{shape}.{format}'
yincr = 2
extractor = ":,::{},:".format(yincr)
returncode, output, error = self.run_program(
"""-i '{r}' -s '{s}' -o '{o}' -x '{x}' --split 1""".format(
s=self.shape,
x=extractor,
r=self.r_filename_format,
o=out_filename_format))
sub_shape = Shape((self.X, self.Z))
for d1, y in enumerate(range(0, self.Y, yincr)):
out_filename = out_filename_format.format(shape=sub_shape,
format=self.file_format,
d1=d1)
self.assertFileExistsAndHasShape(out_filename, sub_shape)
def impl_random_cube(self, shape, dtype, min, max):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
file_format = 'raw'
filename_format = "random_{{shape}}_{{dtype}}_{min}_{max}.{{format}}".format(
min=min, max=max)
filename = filename_format.format(shape=shape,
dtype=dtype,
format=file_format)
cube = cb.random_cube(shape=shape, dtype=dtype, min=min, max=max)
cube_min = cube.min()
cube_max = cube.max()
self.assertGreaterEqual(cube_min, min)
self.assertLessEqual(cube_max, max)
self.assertEqual(dtype, cube.dtype.type)
def impl_read_write(self, shape, dtype, file_format):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
out0_filename_format = "o0tmp_{shape}_{dtype}.{format}"
out0_filename = out0_filename_format.format(shape=shape,
dtype=dtype.__name__,
format='raw')
out1_filename_format = "o1tmp_{shape}_{dtype}.{format}"
out1_filename = out1_filename_format.format(shape=shape,
dtype=dtype.__name__,
format=file_format)
out2_filename_format = "o2tmp_{shape}_{dtype}.{format}"
out2_filename = out2_filename_format.format(shape=shape,
dtype=dtype.__name__,
format='raw')
cube = cb.linear_cube(shape=shape, dtype=dtype)
cb.write_cube_raw(cube=cube, file=out0_filename_format)
self.assertFileExistsAndHasShape(out0_filename, shape, dtype=dtype)
returncode, output, error = self.run_program(
"""-i '{i}' -It '{t}' -s '{s}' -e 'write_cube(cube=_r, filename="{o}", format="{f}", dtype="{t}")'"""
.format(i=out0_filename_format,
s=shape,
o=out1_filename_format,
f=file_format,
t=dtype.__name__))
self.assertFileExists(out1_filename)
returncode, output, error = self.run_program(
"""-e 'read_cube(shape="{s}", filename="{i}", format="{f}", dtype="{t}")' -e 'write_cube(cube=_r, filename="{o}", format="raw", dtype="{t}")'"""
.format(i=out1_filename_format,
s=shape,
o=out2_filename_format,
f=file_format,
t=dtype.__name__))
self.assertFileExistsAndHasShape(out2_filename, shape, dtype=dtype)
self.assertFilesAreEqual(out0_filename, out2_filename)
self.remove_files(out0_filename, out1_filename, out2_filename)
def impl_read(self, shape, extractor, dtype, threshold_size):
shape = Shape(shape)
extractor = Extractor(extractor)
threshold_size = Memory(threshold_size)
dtype = cb.get_dtype(dtype)
file_format = 'raw'
out_filename_format = 'xtmp_{shape}_{dtype}.{format}'
out_filename = out_filename_format.format(shape=shape,
dtype=dtype.__name__,
format=file_format)
returncode, output, error = self.run_program(
"""-e 'cb.random_cube("{s}")' -o '{o}'""".format(
s=shape, o=out_filename_format))
self.assertEqual(returncode, 0)
self.assertFileExistsAndHasShape(out_filename,
shape=shape,
dtype=dtype)
out1_filename_format = 'xtmp1_{shape}_{dtype}.{format}'
out1_filename = out1_filename_format.format(shape=shape,
dtype=dtype.__name__,
format=file_format)
returncode, output, error = self.run_program(
"""-i '{o}' -s {s} -x '{x}' --read-threshold-size=0 -o {o1}""".
format(s=shape,
x=extractor,
o=out_filename_format,
o1=out1_filename_format))
self.assertEqual(returncode, 0)
out2_filename_format = 'xtmp2_{shape}_{dtype}.{format}'
out2_filename = out2_filename_format.format(shape=shape,
dtype=dtype.__name__,
format=file_format)
returncode, output, error = self.run_program(
"""-i '{o}' -s {s} -x '{x}' --read-threshold-size={ms} -o {o2}""".
format(s=shape,
x=extractor,
o=out_filename_format,
o2=out2_filename_format,
ms=threshold_size))
self.assertEqual(returncode, 0)
def expression_1(self):
shape = Shape((self.X, self.Z))
out1_filename_format = 'rtmp8_{shape}.{format}'
out1_filename = out1_filename_format.format(shape=shape,
format=self.file_format)
returncode, output, error = self.run_program(
"""-e 'cb.random_cube("{s}")' -o {o}""".format(
s=shape, o=out1_filename_format))
self.assertFileExistsAndHasShape(out1_filename, shape)
out2_filename_format = 'rtmp9_{shape}.{format}'
out2_filename = out2_filename_format.format(shape=shape,
format=self.file_format)
returncode, output, error = self.run_program(
"""-i '{r}' -s '{sr}' -x i0='{x}' -i {o1} -s '{s1}' -o {o2}""".
format(r=self.r_filename_format,
sr=self.shape,
x=':,0,:',
o1=out1_filename_format,
s1=shape,
o2=out2_filename_format))
self.assertFileExistsAndHasShape(out2_filename, shape)
def impl_const_blocks_cube(self, shape, dtype, start, increment,
const_dims, cmp_cube):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
file_format = 'raw'
filename_format = "const_blocks_{{shape}}_{{dtype}}_{start}_{increment}_{const_dims}.{{format}}".format(
start=start,
increment=increment,
const_dims="+".join(repr(d) for d in const_dims),
)
filename = filename_format.format(shape=shape,
dtype=dtype,
format=file_format)
cube = cb.const_blocks_cube(shape=shape,
dtype=dtype,
start=start,
increment=increment,
const_dims=const_dims)
cube_min = cube.min()
cube_max = cube.max()
self.assertEqual(cube_min, start)
self.assertEqual(cb.not_equals_num(cube, cmp_cube), 0)
def impl_stats_random_file(self, shape, dtype, buffer_size):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
file_format = 'raw'
filename_format = "stats_random_{shape}_{dtype}.{format}"
filename = filename_format.format(shape=shape,
dtype=dtype,
format=file_format)
dmin = 3
for d in shape:
assert d >= dmin, "d={} < {}".format(d, dmin)
cube = cb.random_cube(shape=shape, dtype=dtype)
stats_info_cube = cb.stats_info(cube)
cube.tofile(filename)
self.assertFileExistsAndHasShape(filename, shape=shape, dtype=dtype)
stats_info_oc = cb.stats_file(filename,
shape=shape,
dtype=dtype,
file_format=file_format,
out_of_core=False)
self.assertAlmostEqualStatsInfo(stats_info_oc, stats_info_cube)
stats_info_ooc = cb.stats_file(filename,
shape=shape,
dtype=dtype,
file_format=file_format,
out_of_core=True,
progress_frequency=-1.0,
buffer_size=buffer_size)
self.assertAlmostEqualStatsInfo(stats_info_ooc, stats_info_cube)
self.assertEqual(stats_info_oc.report(), stats_info_ooc.report())
class RubikTestComparison(RubikTestCase):
METHOD_NAMES = []
def setUp(self):
self.shape = Shape("3x4x5")
self.l0 = cb.linear_cube(self.shape, start=0.5, increment=1.5)
self.l1 = cb.linear_cube(self.shape, start=0.5, increment=0.5)
self.l2 = cb.linear_cube(self.shape, start=-1.0, increment=0.5)
self.epsilon = 1.0e-8
e = self.epsilon
self.c0 = np.array([0.0, 0.0, e, 1.0, 1.0 + e, 1.0])
self.c1 = np.array([0.0, e, 0.0, 1.0, 1.0, 1.0 + e])
# tolerance 0.0
@testmethod
def not_equals_cube_t0(self):
cube = cb.not_equals_cube(self.l0, self.l1, tolerance=0.0)
self.assertFalse(cube[0, 0, 0])
self.assertEqual(cube.sum(), self.shape.count() - 1)
@testmethod
def equals_cube_t0(self):
cube = cb.equals_cube(self.l0, self.l1, tolerance=0.0)
self.assertTrue(cube[0, 0, 0])
self.assertEqual(cube.sum(), 1)
@testmethod
def not_equals_num_t0(self):
num = cb.not_equals_num(self.l0, self.l1, tolerance=0.0)
self.assertEqual(num, self.shape.count() - 1)
@testmethod
def equals_num_t0(self):
num = cb.equals_num(self.l0, self.l1, tolerance=0.0)
self.assertEqual(num, 1)
@testmethod
def not_equals_t0(self):
self.assertTrue(cb.not_equals(self.l0, self.l1, tolerance=0.0))
@testmethod
def equals_t0(self):
self.assertFalse(cb.equals(self.l0, self.l1, tolerance=0.0))
# tolerance 2.0
@testmethod
def not_equals_cube_t2(self):
cube = cb.not_equals_cube(self.l0, self.l1, tolerance=2.0)
self.assertFalse(cube[0, 0, 0])
self.assertFalse(cube[0, 0, 1])
self.assertFalse(cube[0, 0, 2])
self.assertEqual(cube.sum(), self.shape.count() - 3)
@testmethod
def equals_cube_t2(self):
cube = cb.equals_cube(self.l0, self.l1, tolerance=2.0)
self.assertTrue(cube[0, 0, 0])
self.assertTrue(cube[0, 0, 1])
self.assertTrue(cube[0, 0, 2])
self.assertEqual(cube.sum(), 3)
@testmethod
def not_equals_num_t2(self):
num = cb.not_equals_num(self.l0, self.l1, tolerance=2.0)
self.assertEqual(num, self.shape.count() - 3)
@testmethod
def equals_num_t2(self):
num = cb.equals_num(self.l0, self.l1, tolerance=2.0)
self.assertEqual(num, 3)
@testmethod
def not_equals_t2(self):
self.assertTrue(cb.not_equals(self.l0, self.l1, tolerance=2.0))
@testmethod
def equals_t2(self):
self.assertFalse(cb.equals(self.l0, self.l1, tolerance=2.0))
# tolerance 10000.0
@testmethod
def not_equals_t10000(self):
self.assertFalse(cb.not_equals(self.l0, self.l1, tolerance=10000.0))
@testmethod
def equals_t10000(self):
self.assertTrue(cb.equals(self.l0, self.l1, tolerance=10000.0))
# threshold
@testmethod
def threshold_cube_3(self):
cube = cb.threshold_cube(self.l0, 3.0, value=-9.0)
self.assertEqual(cube[0, 0, 0], -9.0) # 0.5
self.assertEqual(cube[0, 0, 1], -9.0) # 2.0
self.assertEqual(cube[0, 0, 2], self.l0[0, 0, 2]) # 3.5
@testmethod
def threshold_cube_1(self):
cube = cb.threshold_cube(self.l0, 1.0, value=-9.0)
self.assertEqual(cube[0, 0, 0], -9.0) # 0.5
self.assertEqual(cube[0, 0, 1], self.l0[0, 0, 1]) # 2.0
@testmethod
def threshold_cube_0(self):
cube = cb.threshold_cube(self.l2, value=-9.0)
self.assertEqual(cube[0, 0, 0], -9.0) # -1.0
self.assertEqual(cube[0, 0, 1], -9.0) # -0.5
self.assertEqual(cube[0, 0, 2], -9.0) # 0.0
self.assertEqual(cube[0, 0, 3], self.l2[0, 0, 3]) # 0.5
# threshold
@testmethod
def abs_threshold_cube_0p6(self):
cube = cb.abs_threshold_cube(self.l2, 0.6, value=-9.0)
self.assertEqual(cube[0, 0, 0], self.l2[0, 0, 0]) # -1.0
self.assertEqual(cube[0, 0, 1], -9.0) # -0.5
self.assertEqual(cube[0, 0, 2], -9.0) # 0.0
self.assertEqual(cube[0, 0, 3], -9.0) # 0.5
self.assertEqual(cube[0, 0, 4], self.l2[0, 0, 4]) # 1.0
# abs_diff_cube
# self.c0:
# [0.0, 0.0, e, 1.0, 1.0 + e, 1.0],
# self.c1:
# [0.0, e, 0.0, 1.0, 1.0, 1.0 + e]
@testmethod
def abs_diff_cube_t0(self):
e = self.epsilon
cube = cb.abs_diff_cube(self.c0, self.c1)
cube_cmp = np.array([0.0, e, e, 0.0, e, e])
for v0, v1 in zip(self.c0, self.c1):
self.assertAlmostEqual(v0, v1)
@testmethod
def abs_diff_cube_inTn6(self):
e = self.epsilon
cube = cb.abs_diff_cube(self.c0, self.c1, in_threshold=1e-6)
cube_cmp = np.array([0.0, 0.0, 0.0, 0.0, e, e])
for v0, v1 in zip(self.c0, self.c1):
self.assertAlmostEqual(v0, v1)
@testmethod
def abs_diff_cube_outTn6(self):
e = self.epsilon
cube = cb.abs_diff_cube(self.c0, self.c1, out_threshold=1e-6)
cube_cmp = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0])
for v0, v1 in zip(self.c0, self.c1):
self.assertAlmostEqual(v0, v1)
@testmethod
def abs_diff_cube_inTn6_outTn6(self):
e = self.epsilon
cube = cb.abs_diff_cube(self.c0, self.c1, in_threshold=1e-6, out_threshold=1e-6)
cube_cmp = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0])
for v0, v1 in zip(self.c0, self.c1):
self.assertAlmostEqual(v0, v1)
# rel_diff_cube
# self.c0:
# [0.0, 0.0, e, 1.0, 1.0 + e, 1.0],
# self.c1:
# [0.0, e, 0.0, 1.0, 1.0, 1.0 + e]
@testmethod
def rel_diff_cube_t0(self):
e = self.epsilon
cube = cb.rel_diff_cube(self.c0, self.c1)
self.assertEqual(cube[0], 0.0)
self.assertGreater(cube[1], 1.0e30)
self.assertGreater(cube[2], 0.0)
self.assertAlmostEqual(cube[3], e / (1.0 + e))
self.assertAlmostEqual(cube[4], e)
@testmethod
def rel_diff_cube_inTn6(self):
e = self.epsilon
cube = cb.rel_diff_cube(self.c0, self.c1, in_threshold=1e-6)
self.assertEqual(cube[0], 0.0)
self.assertEqual(cube[1], 0.0)
self.assertEqual(cube[2], 0.0)
self.assertAlmostEqual(cube[3], e / (1.0 + e))
self.assertAlmostEqual(cube[4], e)
@testmethod
def rel_diff_cube_outTn6(self):
e = self.epsilon
cube = cb.rel_diff_cube(self.c0, self.c1, out_threshold=1e-6)
self.assertEqual(cube[0], 0.0)
self.assertGreater(cube[1], 1.0e30)
self.assertGreater(cube[2], 0.0)
self.assertEqual(cube[3], 0.0)
self.assertEqual(cube[4], 0.0)
@testmethod
def rel_diff_cube_inTn6_outTn6(self):
e = self.epsilon
cube = cb.rel_diff_cube(self.c0, self.c1, in_threshold=1e-6, out_threshold=1e-6)
self.assertEqual(cube[0], 0.0)
self.assertEqual(cube[1], 0.0)
self.assertEqual(cube[2], 0.0)
self.assertEqual(cube[3], 0.0)
self.assertEqual(cube[4], 0.0)
@testmethod
def rel_diff_cube_fraction(self):
c0 = np.array([2.0])
c1 = np.array([1.0])
cube = cb.rel_diff_cube(c0, c1)
self.assertAlmostEqual(cube[0], 0.5)
@testmethod
def rel_diff_cube_percentage(self):
c0 = np.array([2.0])
c1 = np.array([1.0])
cube = cb.rel_diff_cube(c0, c1, percentage=True)
self.assertAlmostEqual(cube[0], 50.0)
@testmethod
def zero_cube(self):
# c0: [0.0, 0.0, e, 1.0, 1.0 + e, 1.0],
cube = cb.zero_cube(self.c0)
cube_cmp = np.array([True, True, False, False, False, False])
for v0, v1 in zip(cube, cube_cmp):
self.assertEqual(v0, v1)
@testmethod
def zero_cube_Tn6(self):
# c0: [0.0, 0.0, e, 1.0, 1.0 + e, 1.0],
cube = cb.zero_cube(self.c0, tolerance=1e-6)
cube_cmp = np.array([True, True, True, False, False, False])
for v0, v1 in zip(cube, cube_cmp):
self.assertEqual(v0, v1)
@testmethod
def nonzero_cube(self):
# c0: [0.0, 0.0, e, 1.0, 1.0 + e, 1.0],
cube = cb.nonzero_cube(self.c0)
cube_cmp = np.array([True, True, False, False, False, False])
for v0, v1 in zip(cube, cube_cmp):
self.assertEqual(v0, not v1)
@testmethod
def nonzero_cube_Tn6(self):
# c0: [0.0, 0.0, e, 1.0, 1.0 + e, 1.0],
cube = cb.nonzero_cube(self.c0, tolerance=1e-6)
cube_cmp = np.array([True, True, True, False, False, False])
for v0, v1 in zip(cube, cube_cmp):
self.assertEqual(v0, not v1)
@testmethod
def where_indices(self):
a = np.eye(3)
a[1, 2] = -2.0
a[2, 1] = 2.0
cube = cb.where_indices(a)
self.assertEqual(cube.shape, (5, 3))
self.assertCubesAreEqual(cube[0], np.array([0.0, 0.0, 1.0]))
self.assertCubesAreEqual(cube[1], np.array([1.0, 1.0, 1.0]))
self.assertCubesAreEqual(cube[2], np.array([1.0, 2.0, -2.0]))
self.assertCubesAreEqual(cube[3], np.array([2.0, 1.0, 2.0]))
self.assertCubesAreEqual(cube[4], np.array([2.0, 2.0, 1.0]))
@testmethod
def where_indices_condition(self):
a = np.eye(3)
a[1, 2] = -2.0
a[2, 1] = 2.0
cube = cb.where_indices(a, np.abs(a) > 1.0)
self.assertEqual(cube.shape, (2, 3))
self.assertCubesAreEqual(cube[0], np.array([1.0, 2.0, -2.0]))
self.assertCubesAreEqual(cube[1], np.array([2.0, 1.0, 2.0]))
def shape(self):
for shape_t, shape_s, count in self._shapes:
shape_ft = Shape(shape_t)
shape_fs = Shape(shape_s)
self.assertEqual(shape_fs.shape(), shape_t)
self.assertEqual(shape_ft.shape(), shape_t)
def diff_linear_files_12x8x19x5_3chunks(self):
dtype = np.float64
shape = Shape("12x8x19x5")
self.impl_diff_linear_files(shape=shape, dtype=dtype,
buffer_size=self.get_buffer_size(shape=shape, dtype=dtype, chunks=3))
def diff_linear_files_4x4_2chunks(self):
dtype = np.float32
shape = Shape("4x4")
self.impl_diff_linear_files(shape=shape, dtype=dtype,
buffer_size=self.get_buffer_size(shape=shape, dtype=dtype, chunks=2))
def diff_random_files_12x8x19x5_2chunks(self):
dtype = np.float32
shape = Shape("12x8x19x5")
self.impl_diff_random_files(shape=shape, dtype=dtype,
buffer_size=self.get_buffer_size(shape=shape, dtype=dtype, chunks=2))
def impl_write_cube_out_of_core(self, kind, shape, dtype, buffer_size=None):
dtype = utilities.get_dtype(dtype)
shape = Shape(shape)
filename_format_a = "a_{kind}_{{shape}}_{{dtype}}.{{format}}".format(kind=kind)
filename_format_b = "b_{kind}_{{shape}}_{{dtype}}.{{format}}".format(kind=kind)
filename_format_c = "c_{kind}_{{shape}}_{{dtype}}.{{format}}".format(kind=kind)
if issubclass(dtype, np.integer):
l_start = dtype(3)
l_increment = dtype(2)
c_value = dtype(7)
else:
l_start = dtype(0.5)
l_increment = dtype(1.5)
c_value = dtype(7.5)
p_args = []
n_args = {}
if kind == "random":
ooc_function = cb.write_random_cube
oc_function = cb.random_cube
elif kind == "linear":
ooc_function = cb.write_linear_cube
oc_function = cb.linear_cube
n_args["start"] = l_start
n_args["increment"] = l_increment
elif kind == "const":
ooc_function = cb.write_const_cube
oc_function = cb.const_cube
n_args["value"] = c_value
else:
assert False, kind
cb.set_random_seed(10)
ooc_function(filename_format_a, shape=shape, dtype=dtype, *p_args, **n_args)
filename_a = filename_format_a.format(shape=shape, dtype=dtype.__name__, format="raw")
self.assertFileExistsAndHasShape(filename_a, shape=shape, dtype=dtype)
ooc_cube = cb.read_cube_raw(filename_format_a, shape=shape, dtype=dtype)
self.assertTrue(ooc_cube.dtype == dtype)
self.assertEqual(ooc_cube.size, shape.count())
cb.set_random_seed(10)
oc_cube = oc_function(shape=shape, dtype=dtype, *p_args, **n_args)
self.assertCubesAreEqual(ooc_cube, oc_cube)
if kind == "random":
pass
elif kind == "linear":
# 0) start + 0 * increment
# 1) start + 1 * increment
# 2) start + 2 * increment
# 3) start + 3 * increment
# ...
# count) start + count * increment
# == (start * count) + (((count - 1) * count) // 2) * increment
count = shape.count()
sum_count = ((count - 1) * count) // 2
l_sum = (l_start * count) + (l_increment * sum_count)
self.assertEqual(cb.precise_sum(ooc_cube), l_sum)
elif kind == "const":
self.assertEqual(cb.precise_sum(ooc_cube), shape.count() * c_value)
filename_b = filename_format_b.format(shape=shape, dtype=dtype.__name__, format="raw")
ooc_cube.tofile(filename_b)
self.assertFilesAreEqual(filename_a, filename_b)
def write_linear_cube_out_of_core_4x4_float32(self):
dtype = np.float32
shape = Shape("4x4")
self.impl_write_cube_out_of_core(
kind="linear", shape=shape, dtype=dtype, buffer_size=shape.count() * dtype().itemsize / 2
)