def impl_data_type(self, keys, actual_data_type=None, actual_file_format=None, const_value=None):
shape = Shape("4x5")
data_type = keys.get('data_type', 'float32')
options = keys.get('options', None)
dtype = cb.get_dtype(data_type)
file_format = keys.get('file_format', 'raw')
if actual_data_type is None:
actual_data_type = data_type
actual_dtype = cb.get_dtype(actual_data_type)
if actual_file_format is None:
actual_file_format = file_format
rubik_dir = os.getcwd()
rubik_config = config.get_config()
rubik_config_data_type = os.path.join(rubik_dir, "rubik.config.{data_type}.{file_format}")
rubik_config.default_data_type = data_type
rubik_config.default_file_format = file_format
if options is not None:
rubik_config.default_options = options
with open(rubik_config_data_type, "w") as f:
rubik_config.write_config_file(f)
current_rubik_dir = os.environ.get("RUBIK_DIR", None)
current_rubik_config = os.environ.get("RUBIK_CONFIG", None)
out_filename_format = 'ctmp_{shape}_{dtype}.{format}'
out_filename = out_filename_format.format(shape=shape, dtype=actual_data_type, format=file_format)
try:
os.environ["RUBIK_DIR"] = rubik_dir
os.environ["RUBIK_CONFIG"] = rubik_config_data_type
environment.load_rubik_environment()
returncode, output, error = self.run_program(
"""-e 'cb.const_cube("{s}", value={c})' -o '{o}'""".format(
s=shape,
c=const_value,
o=out_filename_format))
self.assertEqual(returncode, 0)
if actual_file_format == 'raw':
self.assertFileExistsAndHasShape(out_filename, shape=shape, dtype=actual_data_type)
else:
self.assertFileExists(out_filename)
returncode, output, error = self.run_program(
"""-i '{o}' -s {s} -e 'i0.sum()' --print""".format(
s=shape,
o=out_filename_format),
join_stderr_stdout=False)
value = actual_dtype(output.rstrip('\n'))
self.assertEqual(value, shape.count() * const_value)
finally:
if current_rubik_dir is None:
del os.environ["RUBIK_DIR"]
else:
os.environ["RUBIK_DIR"] = current_rubik_dir
if current_rubik_config is None:
del os.environ["RUBIK_CONFIG"]
else:
os.environ["RUBIK_CONFIG"] = rubik_config_data_type
environment.load_rubik_environment()
def impl_read(self, shape, extractor, dtype, threshold_size):
shape = Shape(shape)
extractor = Extractor(extractor)
threshold_size = Memory(threshold_size)
dtype = cb.get_dtype(dtype)
file_format = 'raw'
out_filename_format = 'xtmp_{shape}_{dtype}.{format}'
out_filename = out_filename_format.format(shape=shape, dtype=dtype.__name__, format=file_format)
returncode, output, error = self.run_program(
"""-e 'cb.random_cube("{s}")' -o '{o}'""".format(
s=shape,
o=out_filename_format))
self.assertEqual(returncode, 0)
self.assertFileExistsAndHasShape(out_filename, shape=shape, dtype=dtype)
out1_filename_format = 'xtmp1_{shape}_{dtype}.{format}'
out1_filename = out1_filename_format.format(shape=shape, dtype=dtype.__name__, format=file_format)
returncode, output, error = self.run_program(
"""-i '{o}' -s {s} -x '{x}' --read-threshold-size=0 -o {o1}""".format(
s=shape,
x=extractor,
o=out_filename_format,
o1=out1_filename_format))
self.assertEqual(returncode, 0)
out2_filename_format = 'xtmp2_{shape}_{dtype}.{format}'
out2_filename = out2_filename_format.format(shape=shape, dtype=dtype.__name__, format=file_format)
returncode, output, error = self.run_program(
"""-i '{o}' -s {s} -x '{x}' --read-threshold-size={ms} -o {o2}""".format(
s=shape,
x=extractor,
o=out_filename_format,
o2=out2_filename_format,
ms=threshold_size))
self.assertEqual(returncode, 0)
def impl_stats_random_file(self, shape, dtype, buffer_size):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
file_format = 'raw'
filename_format = "stats_random_{shape}_{dtype}.{format}"
filename = filename_format.format(shape=shape, dtype=dtype, format=file_format)
dmin = 3
for d in shape:
assert d >= dmin, "d={} < {}".format(d, dmin)
cube = cb.random_cube(shape=shape, dtype=dtype)
stats_info_cube = cb.stats_info(cube)
cube.tofile(filename)
self.assertFileExistsAndHasShape(filename, shape=shape, dtype=dtype)
stats_info_oc = cb.stats_file(filename, shape=shape, dtype=dtype, file_format=file_format,
out_of_core=False)
self.assertAlmostEqualStatsInfo(stats_info_oc, stats_info_cube)
stats_info_ooc = cb.stats_file(filename, shape=shape, dtype=dtype, file_format=file_format,
out_of_core=True, progress_frequency=-1.0, buffer_size=buffer_size)
self.assertAlmostEqualStatsInfo(stats_info_ooc, stats_info_cube)
self.assertEqual(stats_info_oc.report(), stats_info_ooc.report())
def impl_histogram(self, shape, dtype, hlength, bins, mode):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
cube = cb.random_cube(shape=shape, dtype=dtype)
output_lines = cb.histogram(cube, hlength=hlength, bins=bins, mode=mode)
self.assertEqual(len(output_lines), bins)
for line in output_lines:
self.assertEqual(len(line), hlength)
def assertFileExistsAndHasShape(self, filename, shape, dtype=None):
self.assertFileExists(filename)
dtype = cb.get_dtype(dtype)
if not isinstance(shape, Shape):
shape = Shape(shape)
size = shape.count() * dtype().itemsize
filesize = os.stat(filename).st_size
if filesize != size:
raise AssertionError("file {!r} has size {} != {} [shape={}, dtype={}]".format(filename, filesize, size, shape, dtype.__name__))
def impl_histogram(self, shape, dtype, hlength, bins, mode):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
cube = cb.random_cube(shape=shape, dtype=dtype)
output_lines = cb.histogram(cube,
hlength=hlength,
bins=bins,
mode=mode)
self.assertEqual(len(output_lines), bins)
for line in output_lines:
self.assertEqual(len(line), hlength)
def impl_stats_const_file(self, shape, dtype, buffer_size):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
file_format = 'raw'
filename_format = "stats_const_{shape}_{dtype}.{format}"
filename = filename_format.format(shape=shape, dtype=dtype, format=file_format)
dmin = 3
for d in shape:
assert d >= dmin, "d={} < {}".format(d, dmin)
cube_max_index = tuple(0 for i in shape)
cube_min_index = tuple(1 for i in shape)
cube_zero_index = tuple(2 for i in shape)
cube_value = 1.0
cube_max = 10.0
cube_min = -23.0
cube_zero = 0.0
cube_sum = cube_max + cube_min + cube_value * (shape.count() - dmin)
cube_ave = cube_sum / float(shape.count())
cube = cb.const_cube(shape=shape, dtype=dtype, value=cube_value)
cube[cube_max_index] = cube_max
cube[cube_min_index] = cube_min
cube[cube_zero_index] = cube_zero
cube_count_zero = 1
cube_count_nonzero = shape.count() - cube_count_zero
cube_count_nan = 0
cube_count_inf = 0
stats_info_cube = cb.stats_info(cube)
self.assertEqual(stats_info_cube.cube_sum, cube_sum)
self.assertEqual(stats_info_cube.cube_ave, cube_ave)
self.assertEqual(stats_info_cube.cube_max, cube_max)
self.assertEqual(stats_info_cube.cube_min, cube_min)
self.assertEqual(stats_info_cube.cube_max_index, cube_max_index)
self.assertEqual(stats_info_cube.cube_min_index, cube_min_index)
self.assertEqual(stats_info_cube.cube_count_zero, cube_count_zero)
self.assertEqual(stats_info_cube.cube_count_nonzero, cube_count_nonzero)
self.assertEqual(stats_info_cube.cube_count_nan, cube_count_nan)
self.assertEqual(stats_info_cube.cube_count_inf, cube_count_inf)
cube.tofile(filename)
self.assertFileExistsAndHasShape(filename, shape=shape, dtype=dtype)
stats_info_oc = cb.stats_file(filename, shape=shape, dtype=dtype, file_format=file_format,
out_of_core=False)
self.assertEqual(stats_info_oc, stats_info_cube)
stats_info_ooc = cb.stats_file(filename, shape=shape, dtype=dtype, file_format=file_format,
out_of_core=True, progress_frequency=-1.0, buffer_size=buffer_size)
self.assertEqual(stats_info_ooc, stats_info_cube)
def impl_const_cube(self, shape, dtype, value):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
file_format = 'raw'
filename_format = "const_{{shape}}_{{dtype}}_{value}.{{format}}".format(value=value)
filename = filename_format.format(shape=shape, dtype=dtype, format=file_format)
cube = cb.const_cube(shape=shape, dtype=dtype, value=value)
cube_min = cube.min()
cube_max = cube.max()
self.assertEqual(cube_min, value)
self.assertEqual(cube_max, value)
self.assertEqual(dtype, cube.dtype.type)
def impl_random_cube(self, shape, dtype, min, max):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
file_format = 'raw'
filename_format = "random_{{shape}}_{{dtype}}_{min}_{max}.{{format}}".format(min=min, max=max)
filename = filename_format.format(shape=shape, dtype=dtype, format=file_format)
cube = cb.random_cube(shape=shape, dtype=dtype, min=min, max=max)
cube_min = cube.min()
cube_max = cube.max()
self.assertGreaterEqual(cube_min, min)
self.assertLessEqual(cube_max, max)
self.assertEqual(dtype, cube.dtype.type)
def impl_const_cube(self, shape, dtype, value):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
file_format = 'raw'
filename_format = "const_{{shape}}_{{dtype}}_{value}.{{format}}".format(
value=value)
filename = filename_format.format(shape=shape,
dtype=dtype,
format=file_format)
cube = cb.const_cube(shape=shape, dtype=dtype, value=value)
cube_min = cube.min()
cube_max = cube.max()
self.assertEqual(cube_min, value)
self.assertEqual(cube_max, value)
self.assertEqual(dtype, cube.dtype.type)
def impl_random_cube(self, shape, dtype, min, max):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
file_format = 'raw'
filename_format = "random_{{shape}}_{{dtype}}_{min}_{max}.{{format}}".format(
min=min, max=max)
filename = filename_format.format(shape=shape,
dtype=dtype,
format=file_format)
cube = cb.random_cube(shape=shape, dtype=dtype, min=min, max=max)
cube_min = cube.min()
cube_max = cube.max()
self.assertGreaterEqual(cube_min, min)
self.assertLessEqual(cube_max, max)
self.assertEqual(dtype, cube.dtype.type)
def impl_const_blocks_cube(self, shape, dtype, start, increment, const_dims, cmp_cube):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
file_format = 'raw'
filename_format = "const_blocks_{{shape}}_{{dtype}}_{start}_{increment}_{const_dims}.{{format}}".format(
start=start,
increment=increment,
const_dims="+".join(repr(d) for d in const_dims),
)
filename = filename_format.format(shape=shape, dtype=dtype, format=file_format)
cube = cb.const_blocks_cube(shape=shape, dtype=dtype,
start=start, increment=increment, const_dims=const_dims)
cube_min = cube.min()
cube_max = cube.max()
self.assertEqual(cube_min, start)
self.assertEqual(cb.not_equals_num(cube, cmp_cube), 0)
def impl_read_write(self, shape, dtype, file_format):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
out0_filename_format = "o0tmp_{shape}_{dtype}.{format}"
out0_filename = out0_filename_format.format(shape=shape,
dtype=dtype.__name__,
format='raw')
out1_filename_format = "o1tmp_{shape}_{dtype}.{format}"
out1_filename = out1_filename_format.format(shape=shape,
dtype=dtype.__name__,
format=file_format)
out2_filename_format = "o2tmp_{shape}_{dtype}.{format}"
out2_filename = out2_filename_format.format(shape=shape,
dtype=dtype.__name__,
format='raw')
cube = cb.linear_cube(shape=shape, dtype=dtype)
cb.write_cube_raw(cube=cube, file=out0_filename_format)
self.assertFileExistsAndHasShape(out0_filename, shape, dtype=dtype)
returncode, output, error = self.run_program(
"""-i '{i}' -It '{t}' -s '{s}' -e 'write_cube(cube=_r, filename="{o}", format="{f}", dtype="{t}")'"""
.format(i=out0_filename_format,
s=shape,
o=out1_filename_format,
f=file_format,
t=dtype.__name__))
self.assertFileExists(out1_filename)
returncode, output, error = self.run_program(
"""-e 'read_cube(shape="{s}", filename="{i}", format="{f}", dtype="{t}")' -e 'write_cube(cube=_r, filename="{o}", format="raw", dtype="{t}")'"""
.format(i=out1_filename_format,
s=shape,
o=out2_filename_format,
f=file_format,
t=dtype.__name__))
self.assertFileExistsAndHasShape(out2_filename, shape, dtype=dtype)
self.assertFilesAreEqual(out0_filename, out2_filename)
self.remove_files(out0_filename, out1_filename, out2_filename)
def impl_read(self, shape, extractor, dtype, threshold_size):
shape = Shape(shape)
extractor = Extractor(extractor)
threshold_size = Memory(threshold_size)
dtype = cb.get_dtype(dtype)
file_format = 'raw'
out_filename_format = 'xtmp_{shape}_{dtype}.{format}'
out_filename = out_filename_format.format(shape=shape,
dtype=dtype.__name__,
format=file_format)
returncode, output, error = self.run_program(
"""-e 'cb.random_cube("{s}")' -o '{o}'""".format(
s=shape, o=out_filename_format))
self.assertEqual(returncode, 0)
self.assertFileExistsAndHasShape(out_filename,
shape=shape,
dtype=dtype)
out1_filename_format = 'xtmp1_{shape}_{dtype}.{format}'
out1_filename = out1_filename_format.format(shape=shape,
dtype=dtype.__name__,
format=file_format)
returncode, output, error = self.run_program(
"""-i '{o}' -s {s} -x '{x}' --read-threshold-size=0 -o {o1}""".
format(s=shape,
x=extractor,
o=out_filename_format,
o1=out1_filename_format))
self.assertEqual(returncode, 0)
out2_filename_format = 'xtmp2_{shape}_{dtype}.{format}'
out2_filename = out2_filename_format.format(shape=shape,
dtype=dtype.__name__,
format=file_format)
returncode, output, error = self.run_program(
"""-i '{o}' -s {s} -x '{x}' --read-threshold-size={ms} -o {o2}""".
format(s=shape,
x=extractor,
o=out_filename_format,
o2=out2_filename_format,
ms=threshold_size))
self.assertEqual(returncode, 0)
def impl_expression_filename(self, shape, dtype, mode):
shape = Shape(shape)
dtype = cb.get_dtype(dtype)
file_format = 'raw'
out_filename_format = "outcube_{mode}_{{shape}}_{{dtype}}.{{format}}".format(mode=mode)
out_filename = out_filename_format.format(shape=shape, dtype=dtype.__name__, format=file_format)
expr_filename = "expr_{mode}.txt".format(mode=mode)
with open(expr_filename, "w") as f_out:
f_out.write("""\
cube = cb.linear_cube(shape="{s}", dtype="{d}")
cb.write_cube(file_format="{f}", cube=cube, file="{o}")
""".format(s=shape, d=dtype.__name__, o=out_filename_format, f=file_format))
if mode == "f_option":
command = "-f {e}".format(e=expr_filename)
else:
command = "-e '@{e}'".format(e=expr_filename)
returncode, output, error = self.run_program(command)
self.assertEqual(returncode, 0)
self.assertFileExistsAndHasShape(out_filename, shape=shape, dtype=dtype)
def impl_read_write(self, shape, dtype, file_format):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
out0_filename_format = "o0tmp_{shape}_{dtype}.{format}"
out0_filename = out0_filename_format.format(shape=shape, dtype=dtype.__name__, format='raw')
out1_filename_format = "o1tmp_{shape}_{dtype}.{format}"
out1_filename = out1_filename_format.format(shape=shape, dtype=dtype.__name__, format=file_format)
out2_filename_format = "o2tmp_{shape}_{dtype}.{format}"
out2_filename = out2_filename_format.format(shape=shape, dtype=dtype.__name__, format='raw')
cube = cb.linear_cube(shape=shape, dtype=dtype)
cb.write_cube_raw(cube=cube, file=out0_filename_format)
self.assertFileExistsAndHasShape(out0_filename, shape, dtype=dtype)
returncode, output, error = self.run_program(
"""-i '{i}' -It '{t}' -s '{s}' -e 'write_cube(cube=_r, filename="{o}", format="{f}", dtype="{t}")'""".format(
i=out0_filename_format,
s=shape,
o=out1_filename_format,
f=file_format,
t=dtype.__name__))
self.assertFileExists(out1_filename)
returncode, output, error = self.run_program(
"""-e 'read_cube(shape="{s}", filename="{i}", format="{f}", dtype="{t}")' -e 'write_cube(cube=_r, filename="{o}", format="raw", dtype="{t}")'""".format(
i=out1_filename_format,
s=shape,
o=out2_filename_format,
f=file_format,
t=dtype.__name__))
self.assertFileExistsAndHasShape(out2_filename, shape, dtype=dtype)
self.assertFilesAreEqual(out0_filename, out2_filename)
self.remove_files(out0_filename, out1_filename, out2_filename)
def impl_const_blocks_cube(self, shape, dtype, start, increment,
const_dims, cmp_cube):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
file_format = 'raw'
filename_format = "const_blocks_{{shape}}_{{dtype}}_{start}_{increment}_{const_dims}.{{format}}".format(
start=start,
increment=increment,
const_dims="+".join(repr(d) for d in const_dims),
)
filename = filename_format.format(shape=shape,
dtype=dtype,
format=file_format)
cube = cb.const_blocks_cube(shape=shape,
dtype=dtype,
start=start,
increment=increment,
const_dims=const_dims)
cube_min = cube.min()
cube_max = cube.max()
self.assertEqual(cube_min, start)
self.assertEqual(cb.not_equals_num(cube, cmp_cube), 0)
def impl_stats_random_file(self, shape, dtype, buffer_size):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
file_format = 'raw'
filename_format = "stats_random_{shape}_{dtype}.{format}"
filename = filename_format.format(shape=shape,
dtype=dtype,
format=file_format)
dmin = 3
for d in shape:
assert d >= dmin, "d={} < {}".format(d, dmin)
cube = cb.random_cube(shape=shape, dtype=dtype)
stats_info_cube = cb.stats_info(cube)
cube.tofile(filename)
self.assertFileExistsAndHasShape(filename, shape=shape, dtype=dtype)
stats_info_oc = cb.stats_file(filename,
shape=shape,
dtype=dtype,
file_format=file_format,
out_of_core=False)
self.assertAlmostEqualStatsInfo(stats_info_oc, stats_info_cube)
stats_info_ooc = cb.stats_file(filename,
shape=shape,
dtype=dtype,
file_format=file_format,
out_of_core=True,
progress_frequency=-1.0,
buffer_size=buffer_size)
self.assertAlmostEqualStatsInfo(stats_info_ooc, stats_info_cube)
self.assertEqual(stats_info_oc.report(), stats_info_ooc.report())
def impl_data_type(self,
keys,
actual_data_type=None,
actual_file_format=None,
const_value=None):
shape = Shape("4x5")
data_type = keys.get('data_type', 'float32')
options = keys.get('options', None)
dtype = cb.get_dtype(data_type)
file_format = keys.get('file_format', 'raw')
if actual_data_type is None:
actual_data_type = data_type
actual_dtype = cb.get_dtype(actual_data_type)
if actual_file_format is None:
actual_file_format = file_format
rubik_dir = os.getcwd()
rubik_config = config.get_config()
rubik_config_data_type = os.path.join(
rubik_dir, "rubik.config.{data_type}.{file_format}")
rubik_config.default_data_type = data_type
rubik_config.default_file_format = file_format
if options is not None:
rubik_config.default_options = options
with open(rubik_config_data_type, "w") as f:
rubik_config.write_config_file(f)
current_rubik_dir = os.environ.get("RUBIK_DIR", None)
current_rubik_config = os.environ.get("RUBIK_CONFIG", None)
out_filename_format = 'ctmp_{shape}_{dtype}.{format}'
out_filename = out_filename_format.format(shape=shape,
dtype=actual_data_type,
format=file_format)
try:
os.environ["RUBIK_DIR"] = rubik_dir
os.environ["RUBIK_CONFIG"] = rubik_config_data_type
environment.load_rubik_environment()
returncode, output, error = self.run_program(
"""-e 'cb.const_cube("{s}", value={c})' -o '{o}'""".format(
s=shape, c=const_value, o=out_filename_format))
self.assertEqual(returncode, 0)
if actual_file_format == 'raw':
self.assertFileExistsAndHasShape(out_filename,
shape=shape,
dtype=actual_data_type)
else:
self.assertFileExists(out_filename)
returncode, output, error = self.run_program(
"""-i '{o}' -s {s} -e 'i0.sum()' --print""".format(
s=shape, o=out_filename_format),
join_stderr_stdout=False)
value = actual_dtype(output.rstrip('\n'))
self.assertEqual(value, shape.count() * const_value)
finally:
if current_rubik_dir is None:
del os.environ["RUBIK_DIR"]
else:
os.environ["RUBIK_DIR"] = current_rubik_dir
if current_rubik_config is None:
del os.environ["RUBIK_CONFIG"]
else:
os.environ["RUBIK_CONFIG"] = rubik_config_data_type
environment.load_rubik_environment()
def test_get_dtype(self):
self.assertEqual(cb.get_dtype("float32"), cb.get_dtype(np.float32))
self.assertEqual(cb.get_dtype("float32"), cb.get_dtype(None))
self.assertEqual(cb.get_dtype("int64"), cb.get_dtype(np.int64))
a = np.array([1.5, 2.4], dtype=np.float64)
self.assertEqual(cb.get_dtype(a.dtype), cb.get_dtype(np.float64))
def test_get_dtype_name_int(self):
a = np.array([1.5, 2.4], dtype=np.float64)
self.assertEqual(
cb.get_dtype_name(cb.as_dtype(a, dtype="int32").dtype), "int32")
self.assertEqual(cb.get_dtype_name(cb.as_dtype(a).dtype),
cb.get_dtype_name(cb.get_dtype(None)))
def test_get_dtype(self):
self.assertEqual(cb.get_dtype("float32"), cb.get_dtype(np.float32))
self.assertEqual(cb.get_dtype("float32"), cb.get_dtype(None))
self.assertEqual(cb.get_dtype("int64"), cb.get_dtype(np.int64))
a = np.array([1.5, 2.4], dtype=np.float64)
self.assertEqual(cb.get_dtype(a.dtype), cb.get_dtype(np.float64))
def test_get_dtype_name_int(self):
a = np.array([1.5, 2.4], dtype=np.float64)
self.assertEqual(cb.get_dtype_name(cb.as_dtype(a, dtype="int32").dtype), "int32")
self.assertEqual(cb.get_dtype_name(cb.as_dtype(a).dtype), cb.get_dtype_name(cb.get_dtype(None)))
def impl_labeled_options(self, shape, dtype, i0_label=None, i1_label=None, i2_label=None, o0_label=None, o1_label=None):
shape = Shape(shape)
dtype = cb.get_dtype(dtype)
file_format = 'raw'
i0_label_definition = ''
i1_label_definition = ''
i2_label_definition = ''
o0_label_definition = ''
o1_label_definition = ''
if i0_label is None:
i0_label = 'i0'
else:
i0_label_definition = '{}='.format(i0_label)
if i1_label is None:
i1_label = 'i1'
else:
i1_label_definition = '{}='.format(i1_label)
if i2_label is None:
i2_label = 'i2'
else:
i2_label_definition = '{}='.format(i2_label)
if o0_label is None:
o0_label = 'o0'
else:
o0_label_definition = '{}='.format(o0_label)
if o1_label is None:
o1_label = 'o1'
else:
o1_label_definition = '{}='.format(o1_label)
lc_filename_format = "lcube_{shape}_{dtype}.{format}"
lc_filename = lc_filename_format.format(shape=shape, dtype=dtype.__name__, format=file_format)
returncode, output, error = self.run_program(
"""-e 'cb.linear_cube("{s}")' -o {lc}""".format(
s=shape,
lc=lc_filename_format,
)
)
self.assertEqual(returncode, 0)
self.assertFileExistsAndHasShape(lc_filename, shape=shape, dtype=dtype)
rc_shape = Shape("100x{}".format(shape))
rc_extractor = "3," + ','.join(':' for d in shape)
rc_filename_format = "rcube_{shape}_{dtype}.{format}"
rc_filename = rc_filename_format.format(shape=rc_shape, dtype=dtype.__name__, format=file_format)
returncode, output, error = self.run_program(
"""-e 'cb.random_cube("{s}")' -o {rc}""".format(
s=rc_shape,
rc=rc_filename_format,
)
)
self.assertEqual(returncode, 0)
cc_filename_format = "ccube_{shape}_{dtype}.{format}"
cc_filename = cc_filename_format.format(shape=shape, dtype=dtype.__name__, format=file_format)
returncode, output, error = self.run_program(
"""-e 'cb.const_cube("{s}", value=0.2)' -o {cc}""".format(
s=shape,
cc=cc_filename_format,
)
)
self.assertEqual(returncode, 0)
self.assertFileExistsAndHasShape(cc_filename, shape=shape, dtype=dtype)
o0_filename_format = "o0cube_{shape}_{dtype}.{format}"
o0_file_format = 'text'
o0_filename = o0_filename_format.format(shape=shape, dtype=dtype.__name__, format=o0_file_format)
o1_filename_format = "o1cube_{shape}_{dtype}.{format}"
o1_file_format = 'csv'
o1_filename = o1_filename_format.format(shape=shape, dtype=dtype.__name__, format=o1_file_format)
command = """-i '{i0ld}{lc}' -i '{i1ld}{rc}' -i '{i2ld}{cc}' -s '{s}' -s '{i1l}={rs}' -x '{i1l}={rcx}' -e '{i0l} + {i1l}' -o '{o0ld}{o0}' -e '{i0l} - {i1l}' -o '{o1ld}{o1}' -Of '{o0l}={o0f}' -Of '{o1l}={o1f}'""".format(
s=shape,
rs=rc_shape,
lc=lc_filename_format,
rc=rc_filename_format,
cc=cc_filename_format,
o0=o0_filename_format,
o0f=o0_file_format,
o1=o1_filename_format,
o1f=o1_file_format,
rcx=rc_extractor,
i0l=i0_label,
i1l=i1_label,
i2l=i2_label,
o0l=o0_label,
o1l=o1_label,
i0ld=i0_label_definition,
i1ld=i1_label_definition,
i2ld=i2_label_definition,
o0ld=o0_label_definition,
o1ld=o1_label_definition,
)
returncode, output, error = self.run_program(command)
self.assertEqual(returncode, 0)
self.assertFileExists(o0_filename)
self.assertFileExists(o1_filename)
self.remove_files(rc_filename, lc_filename, cc_filename, o0_filename, o1_filename)
def impl_stats_const_file(self, shape, dtype, buffer_size):
dtype = cb.get_dtype(dtype)
shape = Shape(shape)
file_format = 'raw'
filename_format = "stats_const_{shape}_{dtype}.{format}"
filename = filename_format.format(shape=shape,
dtype=dtype,
format=file_format)
dmin = 3
for d in shape:
assert d >= dmin, "d={} < {}".format(d, dmin)
cube_max_index = tuple(0 for i in shape)
cube_min_index = tuple(1 for i in shape)
cube_zero_index = tuple(2 for i in shape)
cube_value = 1.0
cube_max = 10.0
cube_min = -23.0
cube_zero = 0.0
cube_sum = cube_max + cube_min + cube_value * (shape.count() - dmin)
cube_ave = cube_sum / float(shape.count())
cube = cb.const_cube(shape=shape, dtype=dtype, value=cube_value)
cube[cube_max_index] = cube_max
cube[cube_min_index] = cube_min
cube[cube_zero_index] = cube_zero
cube_count_zero = 1
cube_count_nonzero = shape.count() - cube_count_zero
cube_count_nan = 0
cube_count_inf = 0
stats_info_cube = cb.stats_info(cube)
self.assertEqual(stats_info_cube.cube_sum, cube_sum)
self.assertEqual(stats_info_cube.cube_ave, cube_ave)
self.assertEqual(stats_info_cube.cube_max, cube_max)
self.assertEqual(stats_info_cube.cube_min, cube_min)
self.assertEqual(stats_info_cube.cube_max_index, cube_max_index)
self.assertEqual(stats_info_cube.cube_min_index, cube_min_index)
self.assertEqual(stats_info_cube.cube_count_zero, cube_count_zero)
self.assertEqual(stats_info_cube.cube_count_nonzero,
cube_count_nonzero)
self.assertEqual(stats_info_cube.cube_count_nan, cube_count_nan)
self.assertEqual(stats_info_cube.cube_count_inf, cube_count_inf)
cube.tofile(filename)
self.assertFileExistsAndHasShape(filename, shape=shape, dtype=dtype)
stats_info_oc = cb.stats_file(filename,
shape=shape,
dtype=dtype,
file_format=file_format,
out_of_core=False)
self.assertEqual(stats_info_oc, stats_info_cube)
stats_info_ooc = cb.stats_file(filename,
shape=shape,
dtype=dtype,
file_format=file_format,
out_of_core=True,
progress_frequency=-1.0,
buffer_size=buffer_size)
self.assertEqual(stats_info_ooc, stats_info_cube)
