def impl_data_type(self, keys, actual_data_type=None, actual_file_format=None, const_value=None): shape = Shape("4x5") data_type = keys.get('data_type', 'float32') options = keys.get('options', None) dtype = cb.get_dtype(data_type) file_format = keys.get('file_format', 'raw') if actual_data_type is None: actual_data_type = data_type actual_dtype = cb.get_dtype(actual_data_type) if actual_file_format is None: actual_file_format = file_format rubik_dir = os.getcwd() rubik_config = config.get_config() rubik_config_data_type = os.path.join(rubik_dir, "rubik.config.{data_type}.{file_format}") rubik_config.default_data_type = data_type rubik_config.default_file_format = file_format if options is not None: rubik_config.default_options = options with open(rubik_config_data_type, "w") as f: rubik_config.write_config_file(f) current_rubik_dir = os.environ.get("RUBIK_DIR", None) current_rubik_config = os.environ.get("RUBIK_CONFIG", None) out_filename_format = 'ctmp_{shape}_{dtype}.{format}' out_filename = out_filename_format.format(shape=shape, dtype=actual_data_type, format=file_format) try: os.environ["RUBIK_DIR"] = rubik_dir os.environ["RUBIK_CONFIG"] = rubik_config_data_type environment.load_rubik_environment() returncode, output, error = self.run_program( """-e 'cb.const_cube("{s}", value={c})' -o '{o}'""".format( s=shape, c=const_value, o=out_filename_format)) self.assertEqual(returncode, 0) if actual_file_format == 'raw': self.assertFileExistsAndHasShape(out_filename, shape=shape, dtype=actual_data_type) else: self.assertFileExists(out_filename) returncode, output, error = self.run_program( """-i '{o}' -s {s} -e 'i0.sum()' --print""".format( s=shape, o=out_filename_format), join_stderr_stdout=False) value = actual_dtype(output.rstrip('\n')) self.assertEqual(value, shape.count() * const_value) finally: if current_rubik_dir is None: del os.environ["RUBIK_DIR"] else: os.environ["RUBIK_DIR"] = current_rubik_dir if current_rubik_config is None: del os.environ["RUBIK_CONFIG"] else: os.environ["RUBIK_CONFIG"] = rubik_config_data_type environment.load_rubik_environment()
def impl_read(self, shape, extractor, dtype, threshold_size): shape = Shape(shape) extractor = Extractor(extractor) threshold_size = Memory(threshold_size) dtype = cb.get_dtype(dtype) file_format = 'raw' out_filename_format = 'xtmp_{shape}_{dtype}.{format}' out_filename = out_filename_format.format(shape=shape, dtype=dtype.__name__, format=file_format) returncode, output, error = self.run_program( """-e 'cb.random_cube("{s}")' -o '{o}'""".format( s=shape, o=out_filename_format)) self.assertEqual(returncode, 0) self.assertFileExistsAndHasShape(out_filename, shape=shape, dtype=dtype) out1_filename_format = 'xtmp1_{shape}_{dtype}.{format}' out1_filename = out1_filename_format.format(shape=shape, dtype=dtype.__name__, format=file_format) returncode, output, error = self.run_program( """-i '{o}' -s {s} -x '{x}' --read-threshold-size=0 -o {o1}""".format( s=shape, x=extractor, o=out_filename_format, o1=out1_filename_format)) self.assertEqual(returncode, 0) out2_filename_format = 'xtmp2_{shape}_{dtype}.{format}' out2_filename = out2_filename_format.format(shape=shape, dtype=dtype.__name__, format=file_format) returncode, output, error = self.run_program( """-i '{o}' -s {s} -x '{x}' --read-threshold-size={ms} -o {o2}""".format( s=shape, x=extractor, o=out_filename_format, o2=out2_filename_format, ms=threshold_size)) self.assertEqual(returncode, 0)
def impl_stats_random_file(self, shape, dtype, buffer_size): dtype = cb.get_dtype(dtype) shape = Shape(shape) file_format = 'raw' filename_format = "stats_random_{shape}_{dtype}.{format}" filename = filename_format.format(shape=shape, dtype=dtype, format=file_format) dmin = 3 for d in shape: assert d >= dmin, "d={} < {}".format(d, dmin) cube = cb.random_cube(shape=shape, dtype=dtype) stats_info_cube = cb.stats_info(cube) cube.tofile(filename) self.assertFileExistsAndHasShape(filename, shape=shape, dtype=dtype) stats_info_oc = cb.stats_file(filename, shape=shape, dtype=dtype, file_format=file_format, out_of_core=False) self.assertAlmostEqualStatsInfo(stats_info_oc, stats_info_cube) stats_info_ooc = cb.stats_file(filename, shape=shape, dtype=dtype, file_format=file_format, out_of_core=True, progress_frequency=-1.0, buffer_size=buffer_size) self.assertAlmostEqualStatsInfo(stats_info_ooc, stats_info_cube) self.assertEqual(stats_info_oc.report(), stats_info_ooc.report())
def impl_histogram(self, shape, dtype, hlength, bins, mode): dtype = cb.get_dtype(dtype) shape = Shape(shape) cube = cb.random_cube(shape=shape, dtype=dtype) output_lines = cb.histogram(cube, hlength=hlength, bins=bins, mode=mode) self.assertEqual(len(output_lines), bins) for line in output_lines: self.assertEqual(len(line), hlength)
def assertFileExistsAndHasShape(self, filename, shape, dtype=None): self.assertFileExists(filename) dtype = cb.get_dtype(dtype) if not isinstance(shape, Shape): shape = Shape(shape) size = shape.count() * dtype().itemsize filesize = os.stat(filename).st_size if filesize != size: raise AssertionError("file {!r} has size {} != {} [shape={}, dtype={}]".format(filename, filesize, size, shape, dtype.__name__))
def impl_stats_const_file(self, shape, dtype, buffer_size): dtype = cb.get_dtype(dtype) shape = Shape(shape) file_format = 'raw' filename_format = "stats_const_{shape}_{dtype}.{format}" filename = filename_format.format(shape=shape, dtype=dtype, format=file_format) dmin = 3 for d in shape: assert d >= dmin, "d={} < {}".format(d, dmin) cube_max_index = tuple(0 for i in shape) cube_min_index = tuple(1 for i in shape) cube_zero_index = tuple(2 for i in shape) cube_value = 1.0 cube_max = 10.0 cube_min = -23.0 cube_zero = 0.0 cube_sum = cube_max + cube_min + cube_value * (shape.count() - dmin) cube_ave = cube_sum / float(shape.count()) cube = cb.const_cube(shape=shape, dtype=dtype, value=cube_value) cube[cube_max_index] = cube_max cube[cube_min_index] = cube_min cube[cube_zero_index] = cube_zero cube_count_zero = 1 cube_count_nonzero = shape.count() - cube_count_zero cube_count_nan = 0 cube_count_inf = 0 stats_info_cube = cb.stats_info(cube) self.assertEqual(stats_info_cube.cube_sum, cube_sum) self.assertEqual(stats_info_cube.cube_ave, cube_ave) self.assertEqual(stats_info_cube.cube_max, cube_max) self.assertEqual(stats_info_cube.cube_min, cube_min) self.assertEqual(stats_info_cube.cube_max_index, cube_max_index) self.assertEqual(stats_info_cube.cube_min_index, cube_min_index) self.assertEqual(stats_info_cube.cube_count_zero, cube_count_zero) self.assertEqual(stats_info_cube.cube_count_nonzero, cube_count_nonzero) self.assertEqual(stats_info_cube.cube_count_nan, cube_count_nan) self.assertEqual(stats_info_cube.cube_count_inf, cube_count_inf) cube.tofile(filename) self.assertFileExistsAndHasShape(filename, shape=shape, dtype=dtype) stats_info_oc = cb.stats_file(filename, shape=shape, dtype=dtype, file_format=file_format, out_of_core=False) self.assertEqual(stats_info_oc, stats_info_cube) stats_info_ooc = cb.stats_file(filename, shape=shape, dtype=dtype, file_format=file_format, out_of_core=True, progress_frequency=-1.0, buffer_size=buffer_size) self.assertEqual(stats_info_ooc, stats_info_cube)
def impl_const_cube(self, shape, dtype, value): dtype = cb.get_dtype(dtype) shape = Shape(shape) file_format = 'raw' filename_format = "const_{{shape}}_{{dtype}}_{value}.{{format}}".format(value=value) filename = filename_format.format(shape=shape, dtype=dtype, format=file_format) cube = cb.const_cube(shape=shape, dtype=dtype, value=value) cube_min = cube.min() cube_max = cube.max() self.assertEqual(cube_min, value) self.assertEqual(cube_max, value) self.assertEqual(dtype, cube.dtype.type)
def impl_random_cube(self, shape, dtype, min, max): dtype = cb.get_dtype(dtype) shape = Shape(shape) file_format = 'raw' filename_format = "random_{{shape}}_{{dtype}}_{min}_{max}.{{format}}".format(min=min, max=max) filename = filename_format.format(shape=shape, dtype=dtype, format=file_format) cube = cb.random_cube(shape=shape, dtype=dtype, min=min, max=max) cube_min = cube.min() cube_max = cube.max() self.assertGreaterEqual(cube_min, min) self.assertLessEqual(cube_max, max) self.assertEqual(dtype, cube.dtype.type)
def impl_const_cube(self, shape, dtype, value): dtype = cb.get_dtype(dtype) shape = Shape(shape) file_format = 'raw' filename_format = "const_{{shape}}_{{dtype}}_{value}.{{format}}".format( value=value) filename = filename_format.format(shape=shape, dtype=dtype, format=file_format) cube = cb.const_cube(shape=shape, dtype=dtype, value=value) cube_min = cube.min() cube_max = cube.max() self.assertEqual(cube_min, value) self.assertEqual(cube_max, value) self.assertEqual(dtype, cube.dtype.type)
def impl_random_cube(self, shape, dtype, min, max): dtype = cb.get_dtype(dtype) shape = Shape(shape) file_format = 'raw' filename_format = "random_{{shape}}_{{dtype}}_{min}_{max}.{{format}}".format( min=min, max=max) filename = filename_format.format(shape=shape, dtype=dtype, format=file_format) cube = cb.random_cube(shape=shape, dtype=dtype, min=min, max=max) cube_min = cube.min() cube_max = cube.max() self.assertGreaterEqual(cube_min, min) self.assertLessEqual(cube_max, max) self.assertEqual(dtype, cube.dtype.type)
def impl_const_blocks_cube(self, shape, dtype, start, increment, const_dims, cmp_cube): dtype = cb.get_dtype(dtype) shape = Shape(shape) file_format = 'raw' filename_format = "const_blocks_{{shape}}_{{dtype}}_{start}_{increment}_{const_dims}.{{format}}".format( start=start, increment=increment, const_dims="+".join(repr(d) for d in const_dims), ) filename = filename_format.format(shape=shape, dtype=dtype, format=file_format) cube = cb.const_blocks_cube(shape=shape, dtype=dtype, start=start, increment=increment, const_dims=const_dims) cube_min = cube.min() cube_max = cube.max() self.assertEqual(cube_min, start) self.assertEqual(cb.not_equals_num(cube, cmp_cube), 0)
def impl_read_write(self, shape, dtype, file_format): dtype = cb.get_dtype(dtype) shape = Shape(shape) out0_filename_format = "o0tmp_{shape}_{dtype}.{format}" out0_filename = out0_filename_format.format(shape=shape, dtype=dtype.__name__, format='raw') out1_filename_format = "o1tmp_{shape}_{dtype}.{format}" out1_filename = out1_filename_format.format(shape=shape, dtype=dtype.__name__, format=file_format) out2_filename_format = "o2tmp_{shape}_{dtype}.{format}" out2_filename = out2_filename_format.format(shape=shape, dtype=dtype.__name__, format='raw') cube = cb.linear_cube(shape=shape, dtype=dtype) cb.write_cube_raw(cube=cube, file=out0_filename_format) self.assertFileExistsAndHasShape(out0_filename, shape, dtype=dtype) returncode, output, error = self.run_program( """-i '{i}' -It '{t}' -s '{s}' -e 'write_cube(cube=_r, filename="{o}", format="{f}", dtype="{t}")'""" .format(i=out0_filename_format, s=shape, o=out1_filename_format, f=file_format, t=dtype.__name__)) self.assertFileExists(out1_filename) returncode, output, error = self.run_program( """-e 'read_cube(shape="{s}", filename="{i}", format="{f}", dtype="{t}")' -e 'write_cube(cube=_r, filename="{o}", format="raw", dtype="{t}")'""" .format(i=out1_filename_format, s=shape, o=out2_filename_format, f=file_format, t=dtype.__name__)) self.assertFileExistsAndHasShape(out2_filename, shape, dtype=dtype) self.assertFilesAreEqual(out0_filename, out2_filename) self.remove_files(out0_filename, out1_filename, out2_filename)
def impl_read(self, shape, extractor, dtype, threshold_size): shape = Shape(shape) extractor = Extractor(extractor) threshold_size = Memory(threshold_size) dtype = cb.get_dtype(dtype) file_format = 'raw' out_filename_format = 'xtmp_{shape}_{dtype}.{format}' out_filename = out_filename_format.format(shape=shape, dtype=dtype.__name__, format=file_format) returncode, output, error = self.run_program( """-e 'cb.random_cube("{s}")' -o '{o}'""".format( s=shape, o=out_filename_format)) self.assertEqual(returncode, 0) self.assertFileExistsAndHasShape(out_filename, shape=shape, dtype=dtype) out1_filename_format = 'xtmp1_{shape}_{dtype}.{format}' out1_filename = out1_filename_format.format(shape=shape, dtype=dtype.__name__, format=file_format) returncode, output, error = self.run_program( """-i '{o}' -s {s} -x '{x}' --read-threshold-size=0 -o {o1}""". format(s=shape, x=extractor, o=out_filename_format, o1=out1_filename_format)) self.assertEqual(returncode, 0) out2_filename_format = 'xtmp2_{shape}_{dtype}.{format}' out2_filename = out2_filename_format.format(shape=shape, dtype=dtype.__name__, format=file_format) returncode, output, error = self.run_program( """-i '{o}' -s {s} -x '{x}' --read-threshold-size={ms} -o {o2}""". format(s=shape, x=extractor, o=out_filename_format, o2=out2_filename_format, ms=threshold_size)) self.assertEqual(returncode, 0)
def impl_expression_filename(self, shape, dtype, mode): shape = Shape(shape) dtype = cb.get_dtype(dtype) file_format = 'raw' out_filename_format = "outcube_{mode}_{{shape}}_{{dtype}}.{{format}}".format(mode=mode) out_filename = out_filename_format.format(shape=shape, dtype=dtype.__name__, format=file_format) expr_filename = "expr_{mode}.txt".format(mode=mode) with open(expr_filename, "w") as f_out: f_out.write("""\ cube = cb.linear_cube(shape="{s}", dtype="{d}") cb.write_cube(file_format="{f}", cube=cube, file="{o}") """.format(s=shape, d=dtype.__name__, o=out_filename_format, f=file_format)) if mode == "f_option": command = "-f {e}".format(e=expr_filename) else: command = "-e '@{e}'".format(e=expr_filename) returncode, output, error = self.run_program(command) self.assertEqual(returncode, 0) self.assertFileExistsAndHasShape(out_filename, shape=shape, dtype=dtype)
def impl_read_write(self, shape, dtype, file_format): dtype = cb.get_dtype(dtype) shape = Shape(shape) out0_filename_format = "o0tmp_{shape}_{dtype}.{format}" out0_filename = out0_filename_format.format(shape=shape, dtype=dtype.__name__, format='raw') out1_filename_format = "o1tmp_{shape}_{dtype}.{format}" out1_filename = out1_filename_format.format(shape=shape, dtype=dtype.__name__, format=file_format) out2_filename_format = "o2tmp_{shape}_{dtype}.{format}" out2_filename = out2_filename_format.format(shape=shape, dtype=dtype.__name__, format='raw') cube = cb.linear_cube(shape=shape, dtype=dtype) cb.write_cube_raw(cube=cube, file=out0_filename_format) self.assertFileExistsAndHasShape(out0_filename, shape, dtype=dtype) returncode, output, error = self.run_program( """-i '{i}' -It '{t}' -s '{s}' -e 'write_cube(cube=_r, filename="{o}", format="{f}", dtype="{t}")'""".format( i=out0_filename_format, s=shape, o=out1_filename_format, f=file_format, t=dtype.__name__)) self.assertFileExists(out1_filename) returncode, output, error = self.run_program( """-e 'read_cube(shape="{s}", filename="{i}", format="{f}", dtype="{t}")' -e 'write_cube(cube=_r, filename="{o}", format="raw", dtype="{t}")'""".format( i=out1_filename_format, s=shape, o=out2_filename_format, f=file_format, t=dtype.__name__)) self.assertFileExistsAndHasShape(out2_filename, shape, dtype=dtype) self.assertFilesAreEqual(out0_filename, out2_filename) self.remove_files(out0_filename, out1_filename, out2_filename)
def impl_data_type(self, keys, actual_data_type=None, actual_file_format=None, const_value=None): shape = Shape("4x5") data_type = keys.get('data_type', 'float32') options = keys.get('options', None) dtype = cb.get_dtype(data_type) file_format = keys.get('file_format', 'raw') if actual_data_type is None: actual_data_type = data_type actual_dtype = cb.get_dtype(actual_data_type) if actual_file_format is None: actual_file_format = file_format rubik_dir = os.getcwd() rubik_config = config.get_config() rubik_config_data_type = os.path.join( rubik_dir, "rubik.config.{data_type}.{file_format}") rubik_config.default_data_type = data_type rubik_config.default_file_format = file_format if options is not None: rubik_config.default_options = options with open(rubik_config_data_type, "w") as f: rubik_config.write_config_file(f) current_rubik_dir = os.environ.get("RUBIK_DIR", None) current_rubik_config = os.environ.get("RUBIK_CONFIG", None) out_filename_format = 'ctmp_{shape}_{dtype}.{format}' out_filename = out_filename_format.format(shape=shape, dtype=actual_data_type, format=file_format) try: os.environ["RUBIK_DIR"] = rubik_dir os.environ["RUBIK_CONFIG"] = rubik_config_data_type environment.load_rubik_environment() returncode, output, error = self.run_program( """-e 'cb.const_cube("{s}", value={c})' -o '{o}'""".format( s=shape, c=const_value, o=out_filename_format)) self.assertEqual(returncode, 0) if actual_file_format == 'raw': self.assertFileExistsAndHasShape(out_filename, shape=shape, dtype=actual_data_type) else: self.assertFileExists(out_filename) returncode, output, error = self.run_program( """-i '{o}' -s {s} -e 'i0.sum()' --print""".format( s=shape, o=out_filename_format), join_stderr_stdout=False) value = actual_dtype(output.rstrip('\n')) self.assertEqual(value, shape.count() * const_value) finally: if current_rubik_dir is None: del os.environ["RUBIK_DIR"] else: os.environ["RUBIK_DIR"] = current_rubik_dir if current_rubik_config is None: del os.environ["RUBIK_CONFIG"] else: os.environ["RUBIK_CONFIG"] = rubik_config_data_type environment.load_rubik_environment()
def test_get_dtype(self): self.assertEqual(cb.get_dtype("float32"), cb.get_dtype(np.float32)) self.assertEqual(cb.get_dtype("float32"), cb.get_dtype(None)) self.assertEqual(cb.get_dtype("int64"), cb.get_dtype(np.int64)) a = np.array([1.5, 2.4], dtype=np.float64) self.assertEqual(cb.get_dtype(a.dtype), cb.get_dtype(np.float64))
def test_get_dtype_name_int(self): a = np.array([1.5, 2.4], dtype=np.float64) self.assertEqual( cb.get_dtype_name(cb.as_dtype(a, dtype="int32").dtype), "int32") self.assertEqual(cb.get_dtype_name(cb.as_dtype(a).dtype), cb.get_dtype_name(cb.get_dtype(None)))
def test_get_dtype_name_int(self): a = np.array([1.5, 2.4], dtype=np.float64) self.assertEqual(cb.get_dtype_name(cb.as_dtype(a, dtype="int32").dtype), "int32") self.assertEqual(cb.get_dtype_name(cb.as_dtype(a).dtype), cb.get_dtype_name(cb.get_dtype(None)))
def impl_labeled_options(self, shape, dtype, i0_label=None, i1_label=None, i2_label=None, o0_label=None, o1_label=None): shape = Shape(shape) dtype = cb.get_dtype(dtype) file_format = 'raw' i0_label_definition = '' i1_label_definition = '' i2_label_definition = '' o0_label_definition = '' o1_label_definition = '' if i0_label is None: i0_label = 'i0' else: i0_label_definition = '{}='.format(i0_label) if i1_label is None: i1_label = 'i1' else: i1_label_definition = '{}='.format(i1_label) if i2_label is None: i2_label = 'i2' else: i2_label_definition = '{}='.format(i2_label) if o0_label is None: o0_label = 'o0' else: o0_label_definition = '{}='.format(o0_label) if o1_label is None: o1_label = 'o1' else: o1_label_definition = '{}='.format(o1_label) lc_filename_format = "lcube_{shape}_{dtype}.{format}" lc_filename = lc_filename_format.format(shape=shape, dtype=dtype.__name__, format=file_format) returncode, output, error = self.run_program( """-e 'cb.linear_cube("{s}")' -o {lc}""".format( s=shape, lc=lc_filename_format, ) ) self.assertEqual(returncode, 0) self.assertFileExistsAndHasShape(lc_filename, shape=shape, dtype=dtype) rc_shape = Shape("100x{}".format(shape)) rc_extractor = "3," + ','.join(':' for d in shape) rc_filename_format = "rcube_{shape}_{dtype}.{format}" rc_filename = rc_filename_format.format(shape=rc_shape, dtype=dtype.__name__, format=file_format) returncode, output, error = self.run_program( """-e 'cb.random_cube("{s}")' -o {rc}""".format( s=rc_shape, rc=rc_filename_format, ) ) self.assertEqual(returncode, 0) cc_filename_format = "ccube_{shape}_{dtype}.{format}" cc_filename = cc_filename_format.format(shape=shape, dtype=dtype.__name__, format=file_format) returncode, output, error = self.run_program( """-e 'cb.const_cube("{s}", value=0.2)' -o {cc}""".format( s=shape, cc=cc_filename_format, ) ) self.assertEqual(returncode, 0) self.assertFileExistsAndHasShape(cc_filename, shape=shape, dtype=dtype) o0_filename_format = "o0cube_{shape}_{dtype}.{format}" o0_file_format = 'text' o0_filename = o0_filename_format.format(shape=shape, dtype=dtype.__name__, format=o0_file_format) o1_filename_format = "o1cube_{shape}_{dtype}.{format}" o1_file_format = 'csv' o1_filename = o1_filename_format.format(shape=shape, dtype=dtype.__name__, format=o1_file_format) command = """-i '{i0ld}{lc}' -i '{i1ld}{rc}' -i '{i2ld}{cc}' -s '{s}' -s '{i1l}={rs}' -x '{i1l}={rcx}' -e '{i0l} + {i1l}' -o '{o0ld}{o0}' -e '{i0l} - {i1l}' -o '{o1ld}{o1}' -Of '{o0l}={o0f}' -Of '{o1l}={o1f}'""".format( s=shape, rs=rc_shape, lc=lc_filename_format, rc=rc_filename_format, cc=cc_filename_format, o0=o0_filename_format, o0f=o0_file_format, o1=o1_filename_format, o1f=o1_file_format, rcx=rc_extractor, i0l=i0_label, i1l=i1_label, i2l=i2_label, o0l=o0_label, o1l=o1_label, i0ld=i0_label_definition, i1ld=i1_label_definition, i2ld=i2_label_definition, o0ld=o0_label_definition, o1ld=o1_label_definition, ) returncode, output, error = self.run_program(command) self.assertEqual(returncode, 0) self.assertFileExists(o0_filename) self.assertFileExists(o1_filename) self.remove_files(rc_filename, lc_filename, cc_filename, o0_filename, o1_filename)