def _test_slice_dynamic( test_case, input, slice_args, outputs, static_shape=None, dtype=flow.float32, device_tag=DEFAULT_DEVICE_TAG, ): input = input.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype)) outputs = [ output.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype)) for output in outputs ] if static_shape is None: static_shape = input.shape flow.clear_default_session() func_cfg = flow.FunctionConfig() func_cfg.default_data_type(dtype) func_cfg.default_placement_scope(flow.scope.placement(device_tag, "0:0")) func_cfg.default_logical_view(flow.scope.mirrored_view()) slice_func = _make_slice_dynamic_func(slice_args, static_shape, dtype, func_cfg) of_outputs = slice_func([input]) for (out, of_out) in zip(outputs, of_outputs): test_case.assertTrue(np.array_equal(out, of_out[0]))
def _test_slice_update( test_case, input, update, slice_args, output, dtype=flow.float32, device_tag=DEFAULT_DEVICE_TAG, verbose=False, ): input = input.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype)) update = update.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype)) output = output.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype)) flow.clear_default_session() func_cfg = flow.FunctionConfig() func_cfg.default_data_type(dtype) func_cfg.default_placement_scope(flow.scope.placement(device_tag, "0:0")) slice_func = _make_slice_update_func( slice_args, input.shape, update.shape, dtype, func_cfg ) of_output = slice_func(input, update) if verbose: print("input:\n{}".format(input)) print("update:\n{}".format(update)) print("slice_args:", slice_args) print("output:\n{}".format(output)) print("dtype:", dtype) print("device_tag:", device_tag) print("of_output:\n{}".format(of_output)) test_case.assertTrue(np.array_equal(output, of_output))
def _test_model_io(test_case, shape, dtype, lr, num_iters): flow.clear_default_session() flow.config.enable_legacy_model_io(True) gen_var = _make_gen_var_func(shape, dtype, lr) model_save_root_dir = "./log/snapshot/" if not os.path.exists(model_save_root_dir): os.makedirs(model_save_root_dir) snapshot_path = model_save_root_dir + "snapshot-{}".format( time.strftime("%Y%m%d-%H:%M:%S")) checkpoint = flow.train.CheckPoint() checkpoint.init() variables = [] for i in range(num_iters): var = gen_var( np.random.rand(*shape).astype( flow.convert_oneflow_dtype_to_numpy_dtype(dtype))) if i > 0: test_case.assertTrue( np.allclose(var, variables[-1] - lr / var.size)) variables.append(var) checkpoint.save("{}-{}".format(snapshot_path, i)) flow.clear_default_session() get_var = _make_get_var_func(shape, dtype) final_snapshot_path = "{}-{}".format(snapshot_path, num_iters - 1) checkpoint = flow.train.CheckPoint() checkpoint.load(final_snapshot_path) final_var = get_var() var_from_file = _load_snapshot_manually(final_snapshot_path, shape, dtype) test_case.assertTrue(np.allclose(final_var, var_from_file))
def _test_random_body(self, shape, dtype=flow.float32, target_dtype=flow.float32): np_dtype = flow.convert_oneflow_dtype_to_numpy_dtype(dtype) x = (1000 * np.random.random(shape)).astype(np_dtype) self._test_body(x, dtype=dtype, target_dtype=target_dtype)
def _test_ones_body(self, shape, dtype=flow.float32, target_dtype=flow.float32): np_dtype = flow.convert_oneflow_dtype_to_numpy_dtype(dtype) x = np.ones(shape, dtype=np_dtype) self._test_body(x, dtype=dtype, target_dtype=target_dtype)
def read_images_by_cv(image_files, dtype, channels=3): np_dtype = flow.convert_oneflow_dtype_to_numpy_dtype(dtype) images = [ cv2.imread(image_file).astype(np_dtype) for image_file in image_files ] assert all((isinstance(image, np.ndarray) for image in images)) assert all((image.ndim == 3 for image in images)) assert all((image.shape[2] == channels for image in images)) return images
def _test_slice_update_grad( test_case, input, update, slice_args, output, input_diff, update_diff, dtype=flow.float32, device_tag=DEFAULT_DEVICE_TAG, verbose=False, ): input = input.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype)) update = update.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype)) output = output.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype)) input_diff = input_diff.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype)) update_diff = update_diff.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype)) if verbose: print("dtype: {}".format(dtype)) print("device_tag: {}".format(device_tag)) print("input: {}\n{}\n".format(input.shape, input)) print("output: {}\n{}\n".format(output.shape, output)) def _make_diff_watcher(shape): def _watch_diff(diff: otp.Numpy): if shape == input_diff.shape: test_case.assertTrue(np.array_equal(diff, input_diff)) elif shape == update_diff.shape: test_case.assertTrue(np.array_equal(diff, update_diff)) return _watch_diff flow.clear_default_session() func_cfg = flow.FunctionConfig() func_cfg.default_data_type(dtype) func_cfg.default_placement_scope(flow.scope.placement(device_tag, "0:0")) slice_func = _make_slice_update_grad_func( slice_args, input.shape, update.shape, _make_diff_watcher, dtype, func_cfg ) ret = slice_func(input, update) test_case.assertTrue(np.array_equal(ret, output))
def _CopyToNdarray(self): method_name = oneflow._oneflow_internal.Dtype_GetOfBlobCopyToBufferFuncName( oneflow._oneflow_internal.deprecated.GetProtoDtype4OfDtype( self.dtype)) copy_method = getattr(oneflow._oneflow_internal, method_name) shape_tensor = np.zeros(self.num_axes, dtype=np.int64) oneflow._oneflow_internal.OfBlob_CopyShapeTo(self.of_blob_ptr_, shape_tensor) shape = tuple(shape_tensor.tolist()) tensor = np.zeros(shape, dtype=flow.convert_oneflow_dtype_to_numpy_dtype( self.dtype)) copy_method(self.of_blob_ptr_, tensor) return tensor
def _test_reverse(test_case, input, axis, dtype, verbose=False): assert isinstance(input, np.ndarray) input = input.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype)) slice_list = [slice(None)] * input.ndim for a in axis: if a < 0: a += input.ndim assert a >= 0 and a < input.ndim slice_list[a] = slice(None, None, -1) output = input[tuple(slice_list)] of_output = _of_reverse(input, axis, dtype) if verbose: print("input: {}\n{}\n".format(input.shape, input)) print("comparing output:\n{}\nvs.\n{}".format(output, of_output)) test_case.assertTrue(np.array_equal(output, of_output))
def _test_slice_with_grad( test_case, input, slice_args, output, diff, dtype=flow.float32, device_tag=DEFAULT_DEVICE_TAG, verbose=False, ): input = input.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype)) output = output.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype)) diff = diff.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype)) if verbose: print("dtype: {}".format(dtype)) print("device_tag: {}".format(device_tag)) print("input: {}\n{}\n".format(input.shape, input)) print("output: {}\n{}\n".format(output.shape, output)) print("diff: {}\n{}\n".format(diff.shape, diff)) def WatchDiff(of_diff: otp.Numpy): if verbose: print("of_diff: {}\n{}\n".format(of_diff.shape, of_diff)) test_case.assertTrue(np.array_equal(of_diff, diff)) flow.clear_default_session() func_cfg = flow.FunctionConfig() func_cfg.default_data_type(dtype) func_cfg.default_placement_scope(flow.scope.placement(device_tag, "0:0")) slice_func = _make_slice_with_grad_func( slice_args, input.shape, WatchDiff, dtype, func_cfg ) of_output = slice_func(input) if verbose: print("of_output: {}\n{}\n".format(of_output.shape, of_output)) test_case.assertTrue(np.array_equal(output, of_output))
def _test_categorical_ordinal_encoder(test_case, device_tag, dtype, size, capacity, num_tokens, num_iters): flow.clear_default_session() func_config = flow.FunctionConfig() func_config.default_data_type(flow.float) func_config.default_logical_view(flow.scope.consistent_view()) @flow.global_function(function_config=func_config) def test_job(x: oft.Numpy.Placeholder( shape=(size, ), dtype=dtype)) -> typing.Tuple[oft.Numpy, oft.Numpy]: with flow.scope.placement(device_tag, "0:0"): y = flow.layers.categorical_ordinal_encoder(x, capacity=capacity) z = flow.layers.categorical_ordinal_encoder(x, capacity=capacity, name="encode1") return (y, z) tokens = np.random.randint(-sys.maxsize, sys.maxsize, size=[ num_tokens ]).astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype)) k_set = set() v_set = set() kv_set = set() vk_set = set() for i in range(num_iters): x = tokens[np.random.randint(0, num_tokens, (size, ))] (y, z) = test_job(x) test_case.assertEqual(x.shape, y.shape) if device_tag == "cpu": test_case.assertTrue( np.array_equal(y, z), "\ny: {}\n{}\nz: {}\n{}".format(y.shape, y, z.shape, z), ) for (k, v) in zip(x, y): k_set.add(k) v_set.add(v) kv_set.add((k, v)) vk_set.add((v, k)) unique_size = len(k_set) test_case.assertEqual(len(v_set), unique_size) test_case.assertEqual(len(kv_set), unique_size) test_case.assertEqual(len(vk_set), unique_size)
def read_images_by_pil(image_files, dtype, channels=3): image_objs = [PIL.Image.open(image_file) for image_file in image_files] images = [] np_dtype = flow.convert_oneflow_dtype_to_numpy_dtype(dtype) for im in image_objs: bands = im.getbands() band = "".join(bands) if band == "RGB": images.append(np.asarray(im).astype(np_dtype)[:, :, ::-1]) elif band == "L": gs_image = np.asarray(im).astype(np_dtype) gs_image_shape = gs_image.shape assert len(gs_image_shape) == 2 gs_image = gs_image.reshape(gs_image_shape + (1, )) gs_image = np.broadcast_to(gs_image, shape=gs_image_shape + (3, )) images.append(gs_image) elif band == "BGR": images.append(np.asarray(im).astype(np_dtype)) else: raise NotImplementedError assert all((isinstance(image, np.ndarray) for image in images)) assert all((image.ndim == 3 for image in images)) assert all((image.shape[2] == channels for image in images)) return images
def _test_random_body(self, shape, permute, dtype=flow.float32): np_dtype = flow.convert_oneflow_dtype_to_numpy_dtype(dtype) x = np.random.random(shape).astype(np_dtype) self._test_body(x, permute, dtype=dtype)
def _test_ones_body(self, shape, indices, axis, dtype=flow.float32): np_dtype = flow.convert_oneflow_dtype_to_numpy_dtype(dtype) x = np.ones(shape, dtype=np_dtype) self._test_body(x, indices, axis, dtype=dtype)
def _load_snapshot_manually(path, shape, dtype): var_path = os.path.join(path, "var", "out") return np.fromfile( var_path, dtype=flow.convert_oneflow_dtype_to_numpy_dtype(dtype)).reshape(*shape)
def _test_image_resize_with_cv( test_case, image_files, target_size, min_size=None, max_size=None, keep_aspect_ratio=True, resize_side="shorter", dtype=flow.float32, origin_dtype=None, print_debug_info=False, ): if origin_dtype is None: origin_dtype = dtype image_list = image_test_util.read_images_by_cv(image_files, origin_dtype) if print_debug_info: print("origin images shapes: {}".format([image.shape for image in image_list])) print( "target_size: {}, min_size: {}, max_size: {}, keep_aspect_ratio: {}, \nresize_side: {}, dtype: {}, origin_dtype: {}".format( target_size, min_size, max_size, keep_aspect_ratio, resize_side, dtype, origin_dtype, ) ) (of_res_images, of_scales, of_new_sizes) = _of_image_resize( image_list=image_list, dtype=dtype, origin_dtype=origin_dtype, keep_aspect_ratio=keep_aspect_ratio, target_size=target_size, min_size=min_size, max_size=max_size, resize_side=resize_side, print_debug_info=print_debug_info, ) (cv_res_images, cv_scales, cv_new_sizes) = _cv_image_resize( image_list=image_list, target_size=target_size, keep_aspect_ratio=keep_aspect_ratio, min_size=min_size, max_size=max_size, resize_side=resize_side, dtype=flow.convert_oneflow_dtype_to_numpy_dtype(dtype), ) if print_debug_info: print("comparing resized image between of and cv") for (i, (of_image, cv_image)) in enumerate(zip(of_res_images, cv_res_images)): print(" origin image shape: {}".format(image_list[i].shape)) print( " resized image shape: {} vs. {}".format( of_image.shape, cv_image.shape ) ) print("comparing resized image scale between of and cv") for (of_scale, cv_scale) in zip(of_scales, cv_scales): print(" scale: {} vs. {}:".format(of_scale, cv_scale)) print("comparing resized image new size between of and cv") for (of_new_size, cv_new_size) in zip(of_new_sizes, cv_new_sizes): print(" new_size: {} vs. {}:".format(of_new_size, cv_new_size)) for ( of_res_image, cv_res_image, of_scale, cv_scale, of_new_size, cv_new_size, ) in zip( of_res_images, cv_res_images, of_scales, cv_scales, of_new_sizes, cv_new_sizes ): test_case.assertTrue(np.allclose(of_res_image, cv_res_image)) test_case.assertTrue(np.allclose(of_scale, cv_scale)) test_case.assertTrue(np.allclose(of_new_size, cv_new_size))