def test_pcc_distancesGPU():
hic_dist_tao = np.load("Utils/test_data/test_hic_dist_pcc.npy",
allow_pickle=True).item()
for k in hic_dist_tao.keys():
hic_dist_tao[k] = cp.array(hic_dist_tao[k])
struc_t = cp.load("Utils/test_data/test_struc_pcc.npy")
row_tau = np.load("Utils/test_data/test_row_pcc.npy",
allow_pickle=True).item()
col_tau = np.load("Utils/test_data/test_col_pcc.npy",
allow_pickle=True).item()
for i in row_tau.keys():
row_tau[i] = cp.array(row_tau[i])
col_tau[i] = cp.array(col_tau[i])
ts = cp.load("Utils/test_data/test_ts_pcc.npy")
real_val = np.load("Utils/test_data/test_vals_pcc.npy",
allow_pickle=True).item()
#for k in real_val.keys():
# real_val[k] = cp.array(real_val[k])
start_time = time.time()
passed_val = ut.pcc_distancesGPU(hic_dist_tao, struc_t, row_tau, col_tau,
ts)
print("pcc_distancesGPU", str(time.time() - start_time))
for k in passed_val.keys():
assert (((np.array(passed_val[k]) - np.array(real_val[k])) <
1e-9).all())
def load_test(self):
load_inputs = np.load(self.test_images_dat,
mmap_mode='r',
allow_pickle=True)
load_targets = np.load(self.test_labels_dat,
mmap_mode='r',
allow_pickle=True)
return load_inputs[:self.SAMPLES_TEST,:], load_targets[:self.SAMPLES_TEST,:]
def test_getWishDistGPU():
struc = cp.load("Utils/test_data/test_struc.npy")
row = cp.load("Utils/test_data/test_row.npy")
col = cp.load("Utils/test_data/test_col.npy")
real_wish = cp.load("Utils/test_data/test_wish.npy")
start_t = time.time()
wish = ut.getWishDistGPU(struc, row, col)
print("\ngetWishDistGPU", str(time.time() - start_t), "\n")
assert (((wish - real_wish) < 1e-10).all())
def load(self):
load_inputs = np.load(self.train_images_dat,
mmap_mode='r',
allow_pickle=True)
load_targets = np.load(self.train_labels_dat,
mmap_mode='r',
allow_pickle=True)
# copy_inputs, copy_targets = self.shuffle_sets(load_inputs,
# load_targets)
return load_inputs, load_targets
def __init__(self, n_layers, rg, measurement, f, beta=1, variant="dunlop"):
self.n_layers = n_layers
self.beta = beta
self.betaZ = cp.sqrt(1 - beta**2)
self.random_gen = rg
self.measurement = measurement
self.fourier = f
self.variant = variant
matrix_folder = pathlib.Path.cwd() / 'matrices'
if not matrix_folder.exists():
matrix_folder.mkdir()
if isinstance(self.measurement, Sinogram):
measurement_matrix_file_name = 'radon_matrix_{0}x{1}x{2}-{3}.npz'.format(
str(self.fourier.basis_number), str(self.measurement.dim),
str(self.measurement.n_theta), ORDER)
print('using Tomography Measurement')
elif isinstance(self.measurement, TwoDMeasurement):
measurement_matrix_file_name = 'plain_2D_measurement_matrix_{0}x{1}-{2}.npz'.format(
str(self.fourier.basis_number), str(self.measurement.dim),
ORDER)
print('using TwoDMeasurement')
self.measurement_matrix_file = matrix_folder / measurement_matrix_file_name
if not (self.measurement_matrix_file).exists():
self.H = measurement.get_measurement_matrix(
self.fourier.ix.ravel(ORDER), self.fourier.iy.ravel(ORDER))
temp2 = self.H.conj().T @ self.H
self.H_t_H = 0.5 * (temp2 + temp2.conj().T).real
cp.savez_compressed(self.measurement_matrix_file,
H=self.H.astype(cp.complex64),
H_t_H=self.H_t_H.astype(cp.float32))
else:
self.H = cp.load(self.measurement_matrix_file)['H']
self.H_t_H = cp.load(self.measurement_matrix_file)['H_t_H']
self.I = cp.eye(self.measurement.num_sample)
self.y = self.measurement.y
self.yBar = cp.concatenate(
(self.y, cp.zeros(2 * self.fourier.basis_number_2D_ravel - 1)))
self.meas_var = self.measurement.stdev**2
self.aggresiveness = 0.2
self.target_acceptance_rate = 0.234
self.beta_feedback_gain = 2.1
self.record_skip = 1
self.record_count = 0
self.max_record_history = 10000
self.Layers_sqrtBetas = cp.zeros(self.n_layers, dtype=cp.float32)
def main(npzout: str, npzin: str = None):
collection = []
with open("./resources/Collection.txt") as fileinput:
for row in fileinput:
collection.append(Counter(row.split()))
if npzin is None:
p_wt = normalize(cp.random.rand(LEXICON_SIZE, TOPIC_SIZE), axis=0)
p_td = normalize(cp.random.rand(COLLECTION_SIZE, TOPIC_SIZE), axis=0)
else:
npzfile = cp.load(npzin)
assert filecheck(npzfile)
p_wt = npzfile["p_wt"]
p_td = npzfile["p_td"]
baseline_likelihood = log_likelihood(collection, p_wt, p_td)
print("likelihood(baseline): %f" % (baseline_likelihood))
prev_likelihood = baseline_likelihood
for index in range(1000):
p_wt, p_td = em(collection, p_wt, p_td)
cp.savez(npzout, p_wt=p_wt, p_td=p_td)
likelihood = log_likelihood(collection, p_wt, p_td)
print("likelihood(%d): %f" % (index + 1, likelihood))
if (likelihood -
prev_likelihood) / abs(prev_likelihood) < UPDATE_THRESHOLD:
break
prev_likelihood = likelihood
def __init__(self, train, learningrate):
"""train = Nombre de shot pour l'apprentissage
learningrate = coeff important
"""
print("Calcul avec cupy ... cupy ... cupy ... cupy ... ")
self.train = train
self.learningrate = learningrate
# Réseau de neurones: colonne 1600 en entrée, 2 nodes de 100, sortie de 27 caractères
self.layers = [1600, 100, 100, 27]
# Fonction d'activation: imite l'activation d'un neuronne
self.activations = [relu, relu, sigmoid]
self.weight_list = None
fichier = cp.load('./semaphore.npz')
a = cp.array(fichier['x_train'])
b = cp.array(fichier['y_train'])
c = cp.array(fichier['x_test'])
d = cp.array(fichier['y_test'])
print(type(a))
self.x_train = a
self.y_train = b
self.x_test = c
self.y_test = d
print(type(self.x_train))
a = "Training: Shot {} Lettre {}; Testing: Shot {} Lettre {}"
print(
a.format(len(self.x_train), len(self.y_train), len(self.x_test),
len(self.y_test)))
def load_image(path_dir, mode):
# -----------------------------------------------------------------------------
if mode: # 学習用データセットの準備
fnames = sorted(glob.glob(path_dir + "/*"))
# file = [(cp.load(f)/255.0).astype(cp.float32).transpose([2, 0, 1]) for f in fnames]
file = []
for index, item in enumerate(fnames):
item = cp.load(item).astype(cp.float32).transpose([2, 0, 1]) / 255.0
file.append(item)
# label = [os.path.basename(f) for f in fnames]
labels = []
for num in range(len(fnames) / 2):
labels.append(cp.int32(num))
labels.append(cp.int32(num))
# List = list(zip(file, labels))
# data = chainer.datasets.TupleDataset(List)
data = chainer.datasets.TupleDataset(file, labels)
return data
# -------------------------------------------------------------------------------
if not mode: # 識別用データセットの準備
data = []
labels = []
filelist = sorted(glob.glob(path_dir + "/*"))
for index, item in enumerate(filelist):
item = np.load(item).astype(np.float32).transpose([2, 0, 1]) / 255.0
item = item.reshape((1, item.shape[0], item.shape[1], item.shape[2]))
data.append(item)
labels.append(int(index))
return data, labels
def load(self, path: str):
data0 = xp.load(path)
data = {int(n[1:]): d for n, d in data0.items()}
for i, p in enumerate(self.params):
d = data[i]
assert d.shape == p.shape
p.data = d
def zerosInit(self):
#Initialize the memmap with just zeros
if self.CPUPinn == True:
cupy.cuda.set_allocator(my_pinned_allocator)
self._preInit()
self.CUPYmemmap = cupy.load(self.fileName + '.cpy.npy', mmap_mode='r+')
if self.CPUPinn == True:
cupy.cuda.set_allocator(None)
def gadgetInit(self):
if self.CPUPinn == True:
cupy.cuda.set_allocator(my_pinned_allocator)
self.CUPYmemmap = cupy.load(self.fileName, mmap_mode='r+')
if self.CPUPinn == True:
cupy.cuda.set_allocator(None)
def test_load_pickle(self):
a = testing.shaped_arange((2, 3, 4), dtype=cupy.float32)
sio = io.BytesIO()
a.dump(sio)
s = sio.getvalue()
sio.close()
sio = io.BytesIO(s)
b = cupy.load(sio, allow_pickle=True)
testing.assert_array_equal(a, b)
sio.close()
sio = io.BytesIO(s)
with self.assertRaises(ValueError):
cupy.load(sio, allow_pickle=False)
sio.close()
def gadgetInit(self):
if self.CPUPinn == True:
cupy.cuda.set_allocator(my_pinned_allocator)
self.CUPYcorpus = cupy.load(self.fileName)
if self.CPUPinn == True:
cupy.cuda.set_allocator(None)
def comInit(self, CPUPinn=False):
if self.CPUPinn == True:
cupy.cuda.set_allocator(my_pinned_allocator)
self._preInit()
self.CUPYmemmap = cupy.load(fileName + '.cpy.npy', mmap_mode='r+')
if self.CPUPinn == True:
cupy.cuda.set_allocator(None)
def load(filename, tpb):
data = cp.load(filename)
keys = data['keys']
values = data['values']
default = data['default']
bpg = ceil(len(keys) / tpb)
cd = CudaDict(default=default, tpb=tpb, bpg=bpg)
cd[keys] = values
return cd
def get_resized_data(subset: str,
PATH: str,
resize: tuple,
fn_type='.txt',
sep=' ',
header=None):
try:
print('Loading the resized', subset, 'images ...', end=' ')
X = np.load(PATH + 'X_' + subset[:2] + '_' + str(resize[0]) + '.npy',
allow_pickle=True)
y = np.load(PATH + 'y_' + subset[:2] + '.npy', allow_pickle=True)
except:
print('Failed! QAQ')
print('Loading the original', subset, 'images ...', end=' ')
img_list, fn_list, y = load_images(subset, PATH, fn_type, sep, header)
X = img_resize(img_list, resize)
np.save(PATH + 'X_' + subset[:2] + '_' + str(resize[0]) + '.npy', X)
np.save(PATH + 'y_' + subset[:2] + '.npy', y)
print('Done!')
return X, y
def test_getWishDistancesGPU():
struc = cp.load("Utils/test_data/test_struc_getWishDistances.npy")
row_tau = np.load("Utils/test_data/test_row_tau_getWishDistances.npy",
allow_pickle=True).item()
col_tau = np.load("Utils/test_data/test_col_tau_getWishDistances.npy",
allow_pickle=True).item()
for i in row_tau.keys():
row_tau[i] = cp.array(row_tau[i])
col_tau[i] = cp.array(col_tau[i])
ts = cp.load("Utils/test_data/test_ts_getWishDistances.npy")
real_wishes = np.load("Utils/test_data/test_getWishDistances.npy",
allow_pickle=True).item()
for i in real_wishes.keys():
real_wishes[i] = cp.array(real_wishes[i])
start_time = time.time()
passed_wishes = ut.getWishDistancesGPU(struc, row_tau, col_tau, ts)
print("getWishDistancesGPU", time.time() - start_time)
for k in list(passed_wishes.keys()):
assert (((real_wishes[k] - passed_wishes[k]) < 1e-9).all())
def check_dev(weight_path, state_dict):
dev_sum = 0
for name, param in state_dict.items():
if "weight" in name:
weight = cp.load(weight_path + str(name) + ".npy")
weight_np = cp.asnumpy(weight)
weight_tensor = torch.from_numpy(weight_np).to("cuda")
dev = abs(weight_tensor - param)
dev_flat = dev.view(-1)
dev_sum += torch.sum(dev_flat, 0).item()
return dev_sum
def get_norm(self, weights):
loc = 0
norm = np.zeros(self.num_cols)
for i in range(len(self.files)):
with cp.cuda.Device(self.gpu_list[i % self.num_gpus]):
coef = cp.asarray(weights)
matrix_part = cp.load(self.files[i])
n = matrix_part.shape[1]
matrix_part = matrix_part * coef
norm_part = cp.sum(matrix_part**2, axis=0)
norm = norm + norm_part.get()
return norm
def dot_product(self, coef_vector):
loc = 0
product = np.zeros(self.num_rows)
for i in range(len(self.files)):
with cp.cuda.Device(self.gpu_list[i % self.num_gpus]):
coef = cp.asarray(coef_vector)
matrix_part = cp.load(self.files[i])
n = matrix_part.shape[0]
prod = cp.matmul(matrix_part, coef)
product[loc:loc + n] = prod.get()
loc = loc + n
return product
def optInit(self):
if self.CPUPinn == True:
cupy.cuda.set_allocator(my_pinned_allocator)
self._preInit()
self.CUPYmemmap = []
for optVar in self.optVarList:
self.CUPYmemmap.append(
cupy.load(self.fileName + optVar + '.cpy.npy', mmap_mode='r+'))
if self.CPUPinn == True:
cupy.cuda.set_allocator(None)
def test_save_load(self, dtype):
a = testing.shaped_arange((2, 3, 4), dtype=dtype)
sio = six.BytesIO()
cupy.save(sio, a)
s = sio.getvalue()
sio.close()
sio = six.BytesIO(s)
b = cupy.load(sio)
sio.close()
testing.assert_array_equal(a, b)
def test_save_load(self, dtype):
a = testing.shaped_arange((2, 3, 4), dtype=dtype)
sio = io.BytesIO()
cupy.save(sio, a)
s = sio.getvalue()
sio.close()
sio = io.BytesIO(s)
b = cupy.load(sio)
sio.close()
testing.assert_array_equal(a, b)
def method0(state_dict, weight_path, error_rate, num_bits=32):
for name, param in state_dict.items():
if "weight" not in name:
continue
else:
weight = cp.load(weight_path + str(name) + ".npy")
mask0, mask1 = create_mask(param, error_rate)
param_error = inject_error(weight, mask0, mask1, num_bits)
param_error_np = cp.asnumpy(param_error)
param_error_torch = torch.from_numpy(param_error_np)
param.copy_(param_error_torch)
return state_dict
def load_H_matrix(self):
if self.hybrid_mode:
#load H and H_t_H in host memory instead
with np.load(self.measurement_matrix_file) as data:
self.H = data['H']
self.H_t_H = data['H_t_H']
else:
with cp.load(self.measurement_matrix_file) as data:
self.H = data['H']
if 'H_t_H' in data.npz_file.files:
self.H_t_H = data['H_t_H']
else:
self.H_t_H = self.H.conj()[email protected]
def test_dump(self, dtype):
a = testing.shaped_arange((2, 3, 4), dtype=dtype)
sio = io.BytesIO()
a.dump(sio)
s = sio.getvalue()
sio.close()
sio = io.BytesIO(s)
b = cupy.load(sio, allow_pickle=True)
sio.close()
testing.assert_array_equal(a, b)
def load(file,
mmap_mode=None,
allow_pickle=False,
fix_imports=True,
encoding='ASCII'):
'''Try first to run as CuPy array, if not run as numpy'''
try:
z = cp.load(file, mmap_mode, allow_pickle)
except:
z = np.load(file, mmap_mode, allow_pickle, fix_imports, encoding)
'''Synchronize all GPU cores as preventative measure against race condition'''
cp.cuda.Stream.null.synchronize()
return z
def multiply(self, factor, new_files):
loc = 0
norm = np.zeros(self.num_rows)
for i in range(len(self.files)):
with cp.cuda.Device(self.gpu_list[i % self.num_gpus]):
coef = cp.asarray(factor)
matrix_part = cp.load(self.files[i])
n = matrix_part.shape[0]
matrix_part = matrix_part * coef
cp.save(new_files[i], matrix_part)
norm_part = cp.sum(matrix_part, axis=1)
norm[loc:loc + n] = norm_part.get()
loc = loc + n
return norm
def ECP_method(state_dict, weight_path, error_rate, set_map, num_bits=32):
for name, param in tqdm(state_dict.items(),
desc="Executing ECP: ",
leave=False):
if "weight" in name:
weight = cp.load(weight_path + str(name) + ".npy")
orig_weight = cp.copy(weight)
mask0, mask1 = create_mask(param, error_rate)
param_error = inject_error(weight, mask0, mask1, num_bits)
correct_param = ECP(param_error, orig_weight, set_map)
param_error_np = cp.asnumpy(correct_param)
param_error_torch = torch.from_numpy(param_error_np)
param.copy_(param_error_torch)
return state_dict
def dot_product_normalized_trans(self, coef_vector, norms):
product = np.zeros(self.num_cols)
loc = 0
for i in range(len(self.files)):
with cp.cuda.Device(self.gpu_list[i % self.num_gpus]):
norm_gpu = cp.asarray(norms)
coef = cp.asarray(coef_vector)
matrix_part = cp.load(self.files[i])
matrix_part = matrix_part / norm_gpu
matrix_part = cp.swapaxes(matrix_part, 0, 1)
n = matrix_part.shape[1]
prod = cp.matmul(matrix_part, coef[loc:loc + n])
loc = loc + n
product = product + prod.get()
return product
def __init__(self, G, D, fitting_batchsize=1000, data=None):
self.clf = LogisticRegression()
datapath = 'training_data/{}.npy'.format(data)
X_train = (xp.load(datapath).astype(xp.float32)) * 2 - 1
x_0 = cuda.to_cpu(
D(G.sampling(fitting_batchsize)).reshape(fitting_batchsize,
1).data)
y_0 = np.zeros(len(x_0))
x_1 = cuda.to_cpu(
D(X_train[:fitting_batchsize]).reshape(fitting_batchsize, 1).data)
y_1 = np.ones(len(x_1))
X = np.concatenate([x_0, x_1])
Y = np.concatenate([y_0, y_1])
self.clf.fit(X, Y)
self.Zvalue = Z(self, G, D, X_train[10000:10000 + 1000])
assert (self.Zvalue < 2 or self.Zvalue > -2)
def check_savez(self, savez, dtype):
a1 = testing.shaped_arange((2, 3, 4), dtype=dtype)
a2 = testing.shaped_arange((3, 4, 5), dtype=dtype)
sio = six.BytesIO()
savez(sio, a1, a2)
s = sio.getvalue()
sio.close()
sio = six.BytesIO(s)
with cupy.load(sio) as d:
b1 = d['arr_0']
b2 = d['arr_1']
sio.close()
testing.assert_array_equal(a1, b1)
testing.assert_array_equal(a2, b2)