def random_data_to_disk(size, miu=None, sigma=None, seed=None, random_data_disk_path=None):
"""
Generate local disk data
:param size: Generate disk data size
:param miu: Average value
:param sigma: Standard deviation
:param seed: Seed of random number
:param random_data_disk_path: Specify the disk data save path
:return:
"""
if miu is None or sigma is None:
miu_sigma_list = [[1, 0.1]]
else:
miu_sigma_list = []
for i in miu:
for j in sigma:
miu_sigma_list.append([i, j])
for miu_sigma in miu_sigma_list:
random_data = size // 8
random_data = random_gaussian(tuple([random_data]), miu=miu_sigma[0], sigma=miu_sigma[1], seed=seed)
if random_data_disk_path is None:
random_data_disk_path = os.environ.get("RANDOM_DATA_DISK_PATH")
if random_data_disk_path is None:
raise ValueError("Environment variable is missing from the current environment RANDOM_DATA_DISK_PATH "
": {0}".format(random_data_disk_path))
data_path = random_data_disk_path + "/random_data_%s_%s.bin" % (str(miu_sigma[0]), str(miu_sigma[1]))
with open(data_path, "w+") as file:
random_data.tofile(file)
file.close()
def gen_data(shape1, dtype1, shape2, dtype2):
params = random_gaussian(shape1).astype(dtype1)
out_dim1 = 1
for i in range(len(shape2) - 1):
out_dim1 = out_dim1 * shape2[i]
indices = gen_indices_gather_nd(shape1, shape2, dtype2)
expect = gather_nd_np(params, indices)
return params, indices, expect
def gen_data_ascend(dtype, ids_shape, num_segments, shape):
support_list = {"float16": np.float16, "float32": np.float32}
if not (dtype.lower() in support_list):
raise RuntimeError("Auto-tensor only support %s while dtype is %s" %
(",".join(support_list.keys()), dtype))
segment_ids = gen_segment_ids(ids_shape, num_segments)
# Generate data for testing the op
data_input = random_gaussian(shape, miu=1,
sigma=0.1).astype(support_list[dtype])
output_shape = (num_segments, ) + tuple(shape[len(ids_shape):])
expect = cal_outputs(data_input, support_list[dtype], segment_ids,
output_shape)
output = np.full(output_shape, np.nan, dtype)
return expect, data_input, output, segment_ids
def _gen_input_data(desc, infos, input_for_mod, commands):
"""Generate input data."""
idx = 0
csr_idx_pair = {}
input_mean_value = precheck(desc)
for input_desc in desc["input_desc"] if desc.get("input_desc") is not None else []:
tensor_name = input_desc[0]["tensor_name"]
infos["input_order"][tensor_name] = idx
commands.append("%s = np.array(input_dict.get('%s'))" % (tensor_name, tensor_name))
if not infos["gen_data"] and idx < len(input_for_mod):
infos["input_dict"][tensor_name] = input_for_mod[idx]
idx += 1
continue
shape = [1] if not input_desc[0]["shape"] else input_desc[0]["shape"]
dtype = input_desc[0]["data_type"]
if tensor_name in infos["clean_input"]:
item = np.zeros(shape).astype(dtype)
elif tensor_name in infos["csr_indptr"]:
if tensor_name in csr_idx_pair:
item = csr_idx_pair[tensor_name]
else:
indptr, indices = gen_csr_indices(infos["csr_indptr"][tensor_name])
item = indptr
csr_idx_pair[infos["csr_indptr"][tensor_name].name] = indices
elif tensor_name in infos["csr_indices"]:
if tensor_name in csr_idx_pair:
item = csr_idx_pair[tensor_name]
else:
indptr, indices = gen_csr_indices(infos["csr_indices"][tensor_name])
item = indices
csr_idx_pair[infos["csr_indices"][tensor_name].name] = indptr
elif tensor_name in infos["indices_input"].keys():
item = gen_indices(infos["indices_input"][tensor_name])
else:
item = random_gaussian(shape, miu=input_mean_value, sigma=0.1).astype(dtype)
input_for_mod.append(item)
infos["input_dict"][tensor_name] = item
idx += 1
def gen_data(shape1, dtype1, shape2, dtype2, axis):
params = random_gaussian(shape1).astype(dtype1)
indices = gen_indices_gather(shape1, shape2, dtype2, axis)
expect = gather_np(params, indices, axis)
return params, indices, expect
def gen_data(shape1, dtype1, shape2, dtype2, num):
input1 = random_gaussian(shape1).astype(dtype1)
input2 = gen_indices_unsorted_segment_sum(shape1, shape2, dtype2, num)
expect = np.zeros((num, ) + shape1[len(shape2):]).astype(dtype1)
np.add.at(expect, input2, input1)
return input1, input2, expect
def random_gaussian(size, miu=0, sigma=8, epsilon=0, seed=None):
"""Generate random array with absolution value obeys gaussian distribution."""
from akg.utils import gen_random as random
return random.random_gaussian(size, miu, sigma, epsilon, seed)