def _test_slice_dynamic(
test_case,
input,
slice_args,
outputs,
static_shape=None,
dtype=flow.float32,
device_tag=DEFAULT_DEVICE_TAG,
):
input = input.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype))
outputs = [
output.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype))
for output in outputs
]
if static_shape is None:
static_shape = input.shape
flow.clear_default_session()
func_cfg = flow.FunctionConfig()
func_cfg.default_data_type(dtype)
func_cfg.default_placement_scope(flow.scope.placement(device_tag, "0:0"))
func_cfg.default_logical_view(flow.scope.mirrored_view())
slice_func = _make_slice_dynamic_func(slice_args, static_shape, dtype, func_cfg)
of_outputs = slice_func([input])
for (out, of_out) in zip(outputs, of_outputs):
test_case.assertTrue(np.array_equal(out, of_out[0]))
def _test_slice_update(
test_case,
input,
update,
slice_args,
output,
dtype=flow.float32,
device_tag=DEFAULT_DEVICE_TAG,
verbose=False,
):
input = input.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype))
update = update.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype))
output = output.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype))
flow.clear_default_session()
func_cfg = flow.FunctionConfig()
func_cfg.default_data_type(dtype)
func_cfg.default_placement_scope(flow.scope.placement(device_tag, "0:0"))
slice_func = _make_slice_update_func(
slice_args, input.shape, update.shape, dtype, func_cfg
)
of_output = slice_func(input, update)
if verbose:
print("input:\n{}".format(input))
print("update:\n{}".format(update))
print("slice_args:", slice_args)
print("output:\n{}".format(output))
print("dtype:", dtype)
print("device_tag:", device_tag)
print("of_output:\n{}".format(of_output))
test_case.assertTrue(np.array_equal(output, of_output))
def _test_model_io(test_case, shape, dtype, lr, num_iters):
flow.clear_default_session()
flow.config.enable_legacy_model_io(True)
gen_var = _make_gen_var_func(shape, dtype, lr)
model_save_root_dir = "./log/snapshot/"
if not os.path.exists(model_save_root_dir):
os.makedirs(model_save_root_dir)
snapshot_path = model_save_root_dir + "snapshot-{}".format(
time.strftime("%Y%m%d-%H:%M:%S"))
checkpoint = flow.train.CheckPoint()
checkpoint.init()
variables = []
for i in range(num_iters):
var = gen_var(
np.random.rand(*shape).astype(
flow.convert_oneflow_dtype_to_numpy_dtype(dtype)))
if i > 0:
test_case.assertTrue(
np.allclose(var, variables[-1] - lr / var.size))
variables.append(var)
checkpoint.save("{}-{}".format(snapshot_path, i))
flow.clear_default_session()
get_var = _make_get_var_func(shape, dtype)
final_snapshot_path = "{}-{}".format(snapshot_path, num_iters - 1)
checkpoint = flow.train.CheckPoint()
checkpoint.load(final_snapshot_path)
final_var = get_var()
var_from_file = _load_snapshot_manually(final_snapshot_path, shape, dtype)
test_case.assertTrue(np.allclose(final_var, var_from_file))
def _test_random_body(self,
shape,
dtype=flow.float32,
target_dtype=flow.float32):
np_dtype = flow.convert_oneflow_dtype_to_numpy_dtype(dtype)
x = (1000 * np.random.random(shape)).astype(np_dtype)
self._test_body(x, dtype=dtype, target_dtype=target_dtype)
def _test_ones_body(self,
shape,
dtype=flow.float32,
target_dtype=flow.float32):
np_dtype = flow.convert_oneflow_dtype_to_numpy_dtype(dtype)
x = np.ones(shape, dtype=np_dtype)
self._test_body(x, dtype=dtype, target_dtype=target_dtype)
def read_images_by_cv(image_files, dtype, channels=3):
np_dtype = flow.convert_oneflow_dtype_to_numpy_dtype(dtype)
images = [
cv2.imread(image_file).astype(np_dtype) for image_file in image_files
]
assert all((isinstance(image, np.ndarray) for image in images))
assert all((image.ndim == 3 for image in images))
assert all((image.shape[2] == channels for image in images))
return images
def _test_slice_update_grad(
test_case,
input,
update,
slice_args,
output,
input_diff,
update_diff,
dtype=flow.float32,
device_tag=DEFAULT_DEVICE_TAG,
verbose=False,
):
input = input.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype))
update = update.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype))
output = output.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype))
input_diff = input_diff.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype))
update_diff = update_diff.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype))
if verbose:
print("dtype: {}".format(dtype))
print("device_tag: {}".format(device_tag))
print("input: {}\n{}\n".format(input.shape, input))
print("output: {}\n{}\n".format(output.shape, output))
def _make_diff_watcher(shape):
def _watch_diff(diff: otp.Numpy):
if shape == input_diff.shape:
test_case.assertTrue(np.array_equal(diff, input_diff))
elif shape == update_diff.shape:
test_case.assertTrue(np.array_equal(diff, update_diff))
return _watch_diff
flow.clear_default_session()
func_cfg = flow.FunctionConfig()
func_cfg.default_data_type(dtype)
func_cfg.default_placement_scope(flow.scope.placement(device_tag, "0:0"))
slice_func = _make_slice_update_grad_func(
slice_args, input.shape, update.shape, _make_diff_watcher, dtype, func_cfg
)
ret = slice_func(input, update)
test_case.assertTrue(np.array_equal(ret, output))
def _CopyToNdarray(self):
method_name = oneflow._oneflow_internal.Dtype_GetOfBlobCopyToBufferFuncName(
oneflow._oneflow_internal.deprecated.GetProtoDtype4OfDtype(
self.dtype))
copy_method = getattr(oneflow._oneflow_internal, method_name)
shape_tensor = np.zeros(self.num_axes, dtype=np.int64)
oneflow._oneflow_internal.OfBlob_CopyShapeTo(self.of_blob_ptr_,
shape_tensor)
shape = tuple(shape_tensor.tolist())
tensor = np.zeros(shape,
dtype=flow.convert_oneflow_dtype_to_numpy_dtype(
self.dtype))
copy_method(self.of_blob_ptr_, tensor)
return tensor
def _test_reverse(test_case, input, axis, dtype, verbose=False):
assert isinstance(input, np.ndarray)
input = input.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype))
slice_list = [slice(None)] * input.ndim
for a in axis:
if a < 0:
a += input.ndim
assert a >= 0 and a < input.ndim
slice_list[a] = slice(None, None, -1)
output = input[tuple(slice_list)]
of_output = _of_reverse(input, axis, dtype)
if verbose:
print("input: {}\n{}\n".format(input.shape, input))
print("comparing output:\n{}\nvs.\n{}".format(output, of_output))
test_case.assertTrue(np.array_equal(output, of_output))
def _test_slice_with_grad(
test_case,
input,
slice_args,
output,
diff,
dtype=flow.float32,
device_tag=DEFAULT_DEVICE_TAG,
verbose=False,
):
input = input.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype))
output = output.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype))
diff = diff.astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype))
if verbose:
print("dtype: {}".format(dtype))
print("device_tag: {}".format(device_tag))
print("input: {}\n{}\n".format(input.shape, input))
print("output: {}\n{}\n".format(output.shape, output))
print("diff: {}\n{}\n".format(diff.shape, diff))
def WatchDiff(of_diff: otp.Numpy):
if verbose:
print("of_diff: {}\n{}\n".format(of_diff.shape, of_diff))
test_case.assertTrue(np.array_equal(of_diff, diff))
flow.clear_default_session()
func_cfg = flow.FunctionConfig()
func_cfg.default_data_type(dtype)
func_cfg.default_placement_scope(flow.scope.placement(device_tag, "0:0"))
slice_func = _make_slice_with_grad_func(
slice_args, input.shape, WatchDiff, dtype, func_cfg
)
of_output = slice_func(input)
if verbose:
print("of_output: {}\n{}\n".format(of_output.shape, of_output))
test_case.assertTrue(np.array_equal(output, of_output))
def _test_categorical_ordinal_encoder(test_case, device_tag, dtype, size,
capacity, num_tokens, num_iters):
flow.clear_default_session()
func_config = flow.FunctionConfig()
func_config.default_data_type(flow.float)
func_config.default_logical_view(flow.scope.consistent_view())
@flow.global_function(function_config=func_config)
def test_job(x: oft.Numpy.Placeholder(
shape=(size, ), dtype=dtype)) -> typing.Tuple[oft.Numpy, oft.Numpy]:
with flow.scope.placement(device_tag, "0:0"):
y = flow.layers.categorical_ordinal_encoder(x, capacity=capacity)
z = flow.layers.categorical_ordinal_encoder(x,
capacity=capacity,
name="encode1")
return (y, z)
tokens = np.random.randint(-sys.maxsize, sys.maxsize, size=[
num_tokens
]).astype(flow.convert_oneflow_dtype_to_numpy_dtype(dtype))
k_set = set()
v_set = set()
kv_set = set()
vk_set = set()
for i in range(num_iters):
x = tokens[np.random.randint(0, num_tokens, (size, ))]
(y, z) = test_job(x)
test_case.assertEqual(x.shape, y.shape)
if device_tag == "cpu":
test_case.assertTrue(
np.array_equal(y, z),
"\ny: {}\n{}\nz: {}\n{}".format(y.shape, y, z.shape, z),
)
for (k, v) in zip(x, y):
k_set.add(k)
v_set.add(v)
kv_set.add((k, v))
vk_set.add((v, k))
unique_size = len(k_set)
test_case.assertEqual(len(v_set), unique_size)
test_case.assertEqual(len(kv_set), unique_size)
test_case.assertEqual(len(vk_set), unique_size)
def read_images_by_pil(image_files, dtype, channels=3):
image_objs = [PIL.Image.open(image_file) for image_file in image_files]
images = []
np_dtype = flow.convert_oneflow_dtype_to_numpy_dtype(dtype)
for im in image_objs:
bands = im.getbands()
band = "".join(bands)
if band == "RGB":
images.append(np.asarray(im).astype(np_dtype)[:, :, ::-1])
elif band == "L":
gs_image = np.asarray(im).astype(np_dtype)
gs_image_shape = gs_image.shape
assert len(gs_image_shape) == 2
gs_image = gs_image.reshape(gs_image_shape + (1, ))
gs_image = np.broadcast_to(gs_image, shape=gs_image_shape + (3, ))
images.append(gs_image)
elif band == "BGR":
images.append(np.asarray(im).astype(np_dtype))
else:
raise NotImplementedError
assert all((isinstance(image, np.ndarray) for image in images))
assert all((image.ndim == 3 for image in images))
assert all((image.shape[2] == channels for image in images))
return images
def _test_random_body(self, shape, permute, dtype=flow.float32):
np_dtype = flow.convert_oneflow_dtype_to_numpy_dtype(dtype)
x = np.random.random(shape).astype(np_dtype)
self._test_body(x, permute, dtype=dtype)
def _test_ones_body(self, shape, indices, axis, dtype=flow.float32):
np_dtype = flow.convert_oneflow_dtype_to_numpy_dtype(dtype)
x = np.ones(shape, dtype=np_dtype)
self._test_body(x, indices, axis, dtype=dtype)
def _load_snapshot_manually(path, shape, dtype):
var_path = os.path.join(path, "var", "out")
return np.fromfile(
var_path,
dtype=flow.convert_oneflow_dtype_to_numpy_dtype(dtype)).reshape(*shape)
def _test_image_resize_with_cv(
test_case,
image_files,
target_size,
min_size=None,
max_size=None,
keep_aspect_ratio=True,
resize_side="shorter",
dtype=flow.float32,
origin_dtype=None,
print_debug_info=False,
):
if origin_dtype is None:
origin_dtype = dtype
image_list = image_test_util.read_images_by_cv(image_files, origin_dtype)
if print_debug_info:
print("origin images shapes: {}".format([image.shape for image in image_list]))
print(
"target_size: {}, min_size: {}, max_size: {}, keep_aspect_ratio: {}, \nresize_side: {}, dtype: {}, origin_dtype: {}".format(
target_size,
min_size,
max_size,
keep_aspect_ratio,
resize_side,
dtype,
origin_dtype,
)
)
(of_res_images, of_scales, of_new_sizes) = _of_image_resize(
image_list=image_list,
dtype=dtype,
origin_dtype=origin_dtype,
keep_aspect_ratio=keep_aspect_ratio,
target_size=target_size,
min_size=min_size,
max_size=max_size,
resize_side=resize_side,
print_debug_info=print_debug_info,
)
(cv_res_images, cv_scales, cv_new_sizes) = _cv_image_resize(
image_list=image_list,
target_size=target_size,
keep_aspect_ratio=keep_aspect_ratio,
min_size=min_size,
max_size=max_size,
resize_side=resize_side,
dtype=flow.convert_oneflow_dtype_to_numpy_dtype(dtype),
)
if print_debug_info:
print("comparing resized image between of and cv")
for (i, (of_image, cv_image)) in enumerate(zip(of_res_images, cv_res_images)):
print(" origin image shape: {}".format(image_list[i].shape))
print(
" resized image shape: {} vs. {}".format(
of_image.shape, cv_image.shape
)
)
print("comparing resized image scale between of and cv")
for (of_scale, cv_scale) in zip(of_scales, cv_scales):
print(" scale: {} vs. {}:".format(of_scale, cv_scale))
print("comparing resized image new size between of and cv")
for (of_new_size, cv_new_size) in zip(of_new_sizes, cv_new_sizes):
print(" new_size: {} vs. {}:".format(of_new_size, cv_new_size))
for (
of_res_image,
cv_res_image,
of_scale,
cv_scale,
of_new_size,
cv_new_size,
) in zip(
of_res_images, cv_res_images, of_scales, cv_scales, of_new_sizes, cv_new_sizes
):
test_case.assertTrue(np.allclose(of_res_image, cv_res_image))
test_case.assertTrue(np.allclose(of_scale, cv_scale))
test_case.assertTrue(np.allclose(of_new_size, cv_new_size))
