Ejemplo n.º 1
0
def test_pcc_distancesGPU():
    hic_dist_tao = np.load("Utils/test_data/test_hic_dist_pcc.npy",
                           allow_pickle=True).item()
    for k in hic_dist_tao.keys():
        hic_dist_tao[k] = cp.array(hic_dist_tao[k])
    struc_t = cp.load("Utils/test_data/test_struc_pcc.npy")
    row_tau = np.load("Utils/test_data/test_row_pcc.npy",
                      allow_pickle=True).item()
    col_tau = np.load("Utils/test_data/test_col_pcc.npy",
                      allow_pickle=True).item()
    for i in row_tau.keys():
        row_tau[i] = cp.array(row_tau[i])
        col_tau[i] = cp.array(col_tau[i])

    ts = cp.load("Utils/test_data/test_ts_pcc.npy")
    real_val = np.load("Utils/test_data/test_vals_pcc.npy",
                       allow_pickle=True).item()
    #for k in real_val.keys():
    #	real_val[k] = cp.array(real_val[k])
    start_time = time.time()
    passed_val = ut.pcc_distancesGPU(hic_dist_tao, struc_t, row_tau, col_tau,
                                     ts)
    print("pcc_distancesGPU", str(time.time() - start_time))
    for k in passed_val.keys():
        assert (((np.array(passed_val[k]) - np.array(real_val[k])) <
                 1e-9).all())
Ejemplo n.º 2
0
 def load_test(self):
     load_inputs  = np.load(self.test_images_dat,
                              mmap_mode='r',
                              allow_pickle=True)
     load_targets = np.load(self.test_labels_dat,
                              mmap_mode='r',
                              allow_pickle=True)
     return load_inputs[:self.SAMPLES_TEST,:], load_targets[:self.SAMPLES_TEST,:]
Ejemplo n.º 3
0
def test_getWishDistGPU():
    struc = cp.load("Utils/test_data/test_struc.npy")
    row = cp.load("Utils/test_data/test_row.npy")
    col = cp.load("Utils/test_data/test_col.npy")
    real_wish = cp.load("Utils/test_data/test_wish.npy")
    start_t = time.time()
    wish = ut.getWishDistGPU(struc, row, col)
    print("\ngetWishDistGPU", str(time.time() - start_t), "\n")
    assert (((wish - real_wish) < 1e-10).all())
Ejemplo n.º 4
0
 def load(self):
     load_inputs  = np.load(self.train_images_dat,
                              mmap_mode='r',
                              allow_pickle=True)
     load_targets = np.load(self.train_labels_dat,
                              mmap_mode='r',
                              allow_pickle=True)
     # copy_inputs, copy_targets = self.shuffle_sets(load_inputs,
     #                                               load_targets)
     return load_inputs, load_targets
Ejemplo n.º 5
0
    def __init__(self, n_layers, rg, measurement, f, beta=1, variant="dunlop"):
        self.n_layers = n_layers
        self.beta = beta
        self.betaZ = cp.sqrt(1 - beta**2)
        self.random_gen = rg
        self.measurement = measurement
        self.fourier = f
        self.variant = variant

        matrix_folder = pathlib.Path.cwd() / 'matrices'
        if not matrix_folder.exists():
            matrix_folder.mkdir()

        if isinstance(self.measurement, Sinogram):
            measurement_matrix_file_name = 'radon_matrix_{0}x{1}x{2}-{3}.npz'.format(
                str(self.fourier.basis_number), str(self.measurement.dim),
                str(self.measurement.n_theta), ORDER)
            print('using Tomography Measurement')
        elif isinstance(self.measurement, TwoDMeasurement):
            measurement_matrix_file_name = 'plain_2D_measurement_matrix_{0}x{1}-{2}.npz'.format(
                str(self.fourier.basis_number), str(self.measurement.dim),
                ORDER)
            print('using TwoDMeasurement')

        self.measurement_matrix_file = matrix_folder / measurement_matrix_file_name
        if not (self.measurement_matrix_file).exists():
            self.H = measurement.get_measurement_matrix(
                self.fourier.ix.ravel(ORDER), self.fourier.iy.ravel(ORDER))
            temp2 = self.H.conj().T @ self.H
            self.H_t_H = 0.5 * (temp2 + temp2.conj().T).real
            cp.savez_compressed(self.measurement_matrix_file,
                                H=self.H.astype(cp.complex64),
                                H_t_H=self.H_t_H.astype(cp.float32))
        else:
            self.H = cp.load(self.measurement_matrix_file)['H']
            self.H_t_H = cp.load(self.measurement_matrix_file)['H_t_H']

        self.I = cp.eye(self.measurement.num_sample)
        self.y = self.measurement.y
        self.yBar = cp.concatenate(
            (self.y, cp.zeros(2 * self.fourier.basis_number_2D_ravel - 1)))
        self.meas_var = self.measurement.stdev**2

        self.aggresiveness = 0.2
        self.target_acceptance_rate = 0.234
        self.beta_feedback_gain = 2.1
        self.record_skip = 1
        self.record_count = 0
        self.max_record_history = 10000

        self.Layers_sqrtBetas = cp.zeros(self.n_layers, dtype=cp.float32)
Ejemplo n.º 6
0
def main(npzout: str, npzin: str = None):
    collection = []
    with open("./resources/Collection.txt") as fileinput:
        for row in fileinput:
            collection.append(Counter(row.split()))

    if npzin is None:
        p_wt = normalize(cp.random.rand(LEXICON_SIZE, TOPIC_SIZE), axis=0)
        p_td = normalize(cp.random.rand(COLLECTION_SIZE, TOPIC_SIZE), axis=0)
    else:
        npzfile = cp.load(npzin)
        assert filecheck(npzfile)
        p_wt = npzfile["p_wt"]
        p_td = npzfile["p_td"]

    baseline_likelihood = log_likelihood(collection, p_wt, p_td)
    print("likelihood(baseline): %f" % (baseline_likelihood))
    prev_likelihood = baseline_likelihood
    for index in range(1000):
        p_wt, p_td = em(collection, p_wt, p_td)
        cp.savez(npzout, p_wt=p_wt, p_td=p_td)
        likelihood = log_likelihood(collection, p_wt, p_td)
        print("likelihood(%d): %f" % (index + 1, likelihood))
        if (likelihood -
                prev_likelihood) / abs(prev_likelihood) < UPDATE_THRESHOLD:
            break
        prev_likelihood = likelihood
Ejemplo n.º 7
0
    def __init__(self, train, learningrate):
        """train = Nombre de shot pour l'apprentissage
        learningrate = coeff important
        """

        print("Calcul avec cupy ... cupy ... cupy ... cupy ... ")

        self.train = train
        self.learningrate = learningrate

        # Réseau de neurones: colonne 1600 en entrée, 2 nodes de 100, sortie de 27 caractères
        self.layers = [1600, 100, 100, 27]
        # Fonction d'activation: imite l'activation d'un neuronne
        self.activations = [relu, relu, sigmoid]
        self.weight_list = None

        fichier = cp.load('./semaphore.npz')

        a = cp.array(fichier['x_train'])
        b = cp.array(fichier['y_train'])
        c = cp.array(fichier['x_test'])
        d = cp.array(fichier['y_test'])
        print(type(a))

        self.x_train = a
        self.y_train = b
        self.x_test = c
        self.y_test = d

        print(type(self.x_train))
        a = "Training: Shot {} Lettre {}; Testing: Shot {} Lettre {}"
        print(
            a.format(len(self.x_train), len(self.y_train), len(self.x_test),
                     len(self.y_test)))
Ejemplo n.º 8
0
def load_image(path_dir, mode):

    # -----------------------------------------------------------------------------
    if mode:  # 学習用データセットの準備
        fnames = sorted(glob.glob(path_dir + "/*"))
        #  file = [(cp.load(f)/255.0).astype(cp.float32).transpose([2, 0, 1]) for f in fnames]
        file = []
        for index, item in enumerate(fnames):
            item = cp.load(item).astype(cp.float32).transpose([2, 0, 1]) / 255.0
            file.append(item)
        # label = [os.path.basename(f) for f in fnames]
        labels = []
        for num in range(len(fnames) / 2):
            labels.append(cp.int32(num))
            labels.append(cp.int32(num))

        # List = list(zip(file, labels))
        # data = chainer.datasets.TupleDataset(List)
        data = chainer.datasets.TupleDataset(file, labels)
        return data
        # -------------------------------------------------------------------------------
    if not mode:  # 識別用データセットの準備
        data = []
        labels = []
        filelist = sorted(glob.glob(path_dir + "/*"))
        for index, item in enumerate(filelist):
            item = np.load(item).astype(np.float32).transpose([2, 0, 1]) / 255.0
            item = item.reshape((1, item.shape[0], item.shape[1], item.shape[2]))
            data.append(item)
            labels.append(int(index))

        return data, labels
Ejemplo n.º 9
0
 def load(self, path: str):
     data0 = xp.load(path)
     data = {int(n[1:]): d for n, d in data0.items()}
     for i, p in enumerate(self.params):
         d = data[i]
         assert d.shape == p.shape
         p.data = d
Ejemplo n.º 10
0
 def zerosInit(self):
     #Initialize the memmap with just zeros
     if self.CPUPinn == True:
         cupy.cuda.set_allocator(my_pinned_allocator)
     self._preInit()
     self.CUPYmemmap = cupy.load(self.fileName + '.cpy.npy', mmap_mode='r+')
     if self.CPUPinn == True:
         cupy.cuda.set_allocator(None)
Ejemplo n.º 11
0
    def gadgetInit(self):
        if self.CPUPinn == True:
            cupy.cuda.set_allocator(my_pinned_allocator)

        self.CUPYmemmap = cupy.load(self.fileName, mmap_mode='r+')

        if self.CPUPinn == True:
            cupy.cuda.set_allocator(None)
Ejemplo n.º 12
0
    def test_load_pickle(self):
        a = testing.shaped_arange((2, 3, 4), dtype=cupy.float32)

        sio = io.BytesIO()
        a.dump(sio)
        s = sio.getvalue()
        sio.close()

        sio = io.BytesIO(s)
        b = cupy.load(sio, allow_pickle=True)
        testing.assert_array_equal(a, b)
        sio.close()

        sio = io.BytesIO(s)
        with self.assertRaises(ValueError):
            cupy.load(sio, allow_pickle=False)
        sio.close()
Ejemplo n.º 13
0
    def gadgetInit(self):
        if self.CPUPinn == True:
            cupy.cuda.set_allocator(my_pinned_allocator)

        self.CUPYcorpus = cupy.load(self.fileName)

        if self.CPUPinn == True:
            cupy.cuda.set_allocator(None)
Ejemplo n.º 14
0
    def comInit(self, CPUPinn=False):
        if self.CPUPinn == True:
            cupy.cuda.set_allocator(my_pinned_allocator)

        self._preInit()
        self.CUPYmemmap = cupy.load(fileName + '.cpy.npy', mmap_mode='r+')

        if self.CPUPinn == True:
            cupy.cuda.set_allocator(None)
Ejemplo n.º 15
0
 def load(filename, tpb):
     data = cp.load(filename)
     keys = data['keys']
     values = data['values']
     default = data['default']
     bpg = ceil(len(keys) / tpb)
     cd = CudaDict(default=default, tpb=tpb, bpg=bpg)
     cd[keys] = values
     return cd
Ejemplo n.º 16
0
def get_resized_data(subset: str,
                     PATH: str,
                     resize: tuple,
                     fn_type='.txt',
                     sep=' ',
                     header=None):
    try:
        print('Loading the resized', subset, 'images ...', end=' ')
        X = np.load(PATH + 'X_' + subset[:2] + '_' + str(resize[0]) + '.npy',
                    allow_pickle=True)
        y = np.load(PATH + 'y_' + subset[:2] + '.npy', allow_pickle=True)
    except:
        print('Failed! QAQ')
        print('Loading the original', subset, 'images ...', end=' ')
        img_list, fn_list, y = load_images(subset, PATH, fn_type, sep, header)
        X = img_resize(img_list, resize)
        np.save(PATH + 'X_' + subset[:2] + '_' + str(resize[0]) + '.npy', X)
        np.save(PATH + 'y_' + subset[:2] + '.npy', y)
    print('Done!')
    return X, y
Ejemplo n.º 17
0
def test_getWishDistancesGPU():
    struc = cp.load("Utils/test_data/test_struc_getWishDistances.npy")
    row_tau = np.load("Utils/test_data/test_row_tau_getWishDistances.npy",
                      allow_pickle=True).item()
    col_tau = np.load("Utils/test_data/test_col_tau_getWishDistances.npy",
                      allow_pickle=True).item()
    for i in row_tau.keys():
        row_tau[i] = cp.array(row_tau[i])
        col_tau[i] = cp.array(col_tau[i])

    ts = cp.load("Utils/test_data/test_ts_getWishDistances.npy")
    real_wishes = np.load("Utils/test_data/test_getWishDistances.npy",
                          allow_pickle=True).item()
    for i in real_wishes.keys():
        real_wishes[i] = cp.array(real_wishes[i])
    start_time = time.time()
    passed_wishes = ut.getWishDistancesGPU(struc, row_tau, col_tau, ts)
    print("getWishDistancesGPU", time.time() - start_time)
    for k in list(passed_wishes.keys()):
        assert (((real_wishes[k] - passed_wishes[k]) < 1e-9).all())
Ejemplo n.º 18
0
def check_dev(weight_path, state_dict):
    dev_sum = 0
    for name, param in state_dict.items():
        if "weight" in name:
            weight = cp.load(weight_path + str(name) + ".npy")
            weight_np = cp.asnumpy(weight)
            weight_tensor = torch.from_numpy(weight_np).to("cuda")
            dev = abs(weight_tensor - param)
            dev_flat = dev.view(-1)
            dev_sum += torch.sum(dev_flat, 0).item()
    return dev_sum
Ejemplo n.º 19
0
 def get_norm(self, weights):
     loc = 0
     norm = np.zeros(self.num_cols)
     for i in range(len(self.files)):
         with cp.cuda.Device(self.gpu_list[i % self.num_gpus]):
             coef = cp.asarray(weights)
             matrix_part = cp.load(self.files[i])
             n = matrix_part.shape[1]
             matrix_part = matrix_part * coef
             norm_part = cp.sum(matrix_part**2, axis=0)
             norm = norm + norm_part.get()
     return norm
Ejemplo n.º 20
0
 def dot_product(self, coef_vector):
     loc = 0
     product = np.zeros(self.num_rows)
     for i in range(len(self.files)):
         with cp.cuda.Device(self.gpu_list[i % self.num_gpus]):
             coef = cp.asarray(coef_vector)
             matrix_part = cp.load(self.files[i])
             n = matrix_part.shape[0]
             prod = cp.matmul(matrix_part, coef)
             product[loc:loc + n] = prod.get()
             loc = loc + n
     return product
Ejemplo n.º 21
0
    def optInit(self):
        if self.CPUPinn == True:
            cupy.cuda.set_allocator(my_pinned_allocator)

        self._preInit()
        self.CUPYmemmap = []
        for optVar in self.optVarList:
            self.CUPYmemmap.append(
                cupy.load(self.fileName + optVar + '.cpy.npy', mmap_mode='r+'))

        if self.CPUPinn == True:
            cupy.cuda.set_allocator(None)
Ejemplo n.º 22
0
    def test_save_load(self, dtype):
        a = testing.shaped_arange((2, 3, 4), dtype=dtype)
        sio = six.BytesIO()
        cupy.save(sio, a)
        s = sio.getvalue()
        sio.close()

        sio = six.BytesIO(s)
        b = cupy.load(sio)
        sio.close()

        testing.assert_array_equal(a, b)
Ejemplo n.º 23
0
    def test_save_load(self, dtype):
        a = testing.shaped_arange((2, 3, 4), dtype=dtype)
        sio = io.BytesIO()
        cupy.save(sio, a)
        s = sio.getvalue()
        sio.close()

        sio = io.BytesIO(s)
        b = cupy.load(sio)
        sio.close()

        testing.assert_array_equal(a, b)
Ejemplo n.º 24
0
def method0(state_dict, weight_path, error_rate, num_bits=32):
    for name, param in state_dict.items():
        if "weight" not in name:
            continue
        else:
            weight = cp.load(weight_path + str(name) + ".npy")
            mask0, mask1 = create_mask(param, error_rate)
            param_error = inject_error(weight, mask0, mask1, num_bits)
            param_error_np = cp.asnumpy(param_error)
            param_error_torch = torch.from_numpy(param_error_np)
            param.copy_(param_error_torch)
    return state_dict
Ejemplo n.º 25
0
 def load_H_matrix(self):
     if self.hybrid_mode:
         #load H and H_t_H in host memory instead
         with np.load(self.measurement_matrix_file) as data:
             self.H = data['H']
             self.H_t_H = data['H_t_H']
     else:
         with cp.load(self.measurement_matrix_file) as data:
             self.H = data['H']
             if 'H_t_H' in data.npz_file.files:
                 self.H_t_H = data['H_t_H']
             else:
                 self.H_t_H = self.H.conj()[email protected]
Ejemplo n.º 26
0
    def test_dump(self, dtype):
        a = testing.shaped_arange((2, 3, 4), dtype=dtype)

        sio = io.BytesIO()
        a.dump(sio)
        s = sio.getvalue()
        sio.close()

        sio = io.BytesIO(s)
        b = cupy.load(sio, allow_pickle=True)
        sio.close()

        testing.assert_array_equal(a, b)
Ejemplo n.º 27
0
    def load(file,
             mmap_mode=None,
             allow_pickle=False,
             fix_imports=True,
             encoding='ASCII'):
        '''Try first to run as CuPy array, if not run as numpy'''
        try:
            z = cp.load(file, mmap_mode, allow_pickle)
        except:
            z = np.load(file, mmap_mode, allow_pickle, fix_imports, encoding)
        '''Synchronize all GPU cores as preventative measure against race condition'''
        cp.cuda.Stream.null.synchronize()

        return z
Ejemplo n.º 28
0
 def multiply(self, factor, new_files):
     loc = 0
     norm = np.zeros(self.num_rows)
     for i in range(len(self.files)):
         with cp.cuda.Device(self.gpu_list[i % self.num_gpus]):
             coef = cp.asarray(factor)
             matrix_part = cp.load(self.files[i])
             n = matrix_part.shape[0]
             matrix_part = matrix_part * coef
             cp.save(new_files[i], matrix_part)
             norm_part = cp.sum(matrix_part, axis=1)
             norm[loc:loc + n] = norm_part.get()
             loc = loc + n
     return norm
Ejemplo n.º 29
0
def ECP_method(state_dict, weight_path, error_rate, set_map, num_bits=32):
    for name, param in tqdm(state_dict.items(),
                            desc="Executing ECP: ",
                            leave=False):
        if "weight" in name:
            weight = cp.load(weight_path + str(name) + ".npy")
            orig_weight = cp.copy(weight)
            mask0, mask1 = create_mask(param, error_rate)
            param_error = inject_error(weight, mask0, mask1, num_bits)
            correct_param = ECP(param_error, orig_weight, set_map)
            param_error_np = cp.asnumpy(correct_param)
            param_error_torch = torch.from_numpy(param_error_np)
            param.copy_(param_error_torch)
    return state_dict
Ejemplo n.º 30
0
 def dot_product_normalized_trans(self, coef_vector, norms):
     product = np.zeros(self.num_cols)
     loc = 0
     for i in range(len(self.files)):
         with cp.cuda.Device(self.gpu_list[i % self.num_gpus]):
             norm_gpu = cp.asarray(norms)
             coef = cp.asarray(coef_vector)
             matrix_part = cp.load(self.files[i])
             matrix_part = matrix_part / norm_gpu
             matrix_part = cp.swapaxes(matrix_part, 0, 1)
             n = matrix_part.shape[1]
             prod = cp.matmul(matrix_part, coef[loc:loc + n])
             loc = loc + n
             product = product + prod.get()
     return product
Ejemplo n.º 31
0
 def __init__(self, G, D, fitting_batchsize=1000, data=None):
     self.clf = LogisticRegression()
     datapath = 'training_data/{}.npy'.format(data)
     X_train = (xp.load(datapath).astype(xp.float32)) * 2 - 1
     x_0 = cuda.to_cpu(
         D(G.sampling(fitting_batchsize)).reshape(fitting_batchsize,
                                                  1).data)
     y_0 = np.zeros(len(x_0))
     x_1 = cuda.to_cpu(
         D(X_train[:fitting_batchsize]).reshape(fitting_batchsize, 1).data)
     y_1 = np.ones(len(x_1))
     X = np.concatenate([x_0, x_1])
     Y = np.concatenate([y_0, y_1])
     self.clf.fit(X, Y)
     self.Zvalue = Z(self, G, D, X_train[10000:10000 + 1000])
     assert (self.Zvalue < 2 or self.Zvalue > -2)
Ejemplo n.º 32
0
    def check_savez(self, savez, dtype):
        a1 = testing.shaped_arange((2, 3, 4), dtype=dtype)
        a2 = testing.shaped_arange((3, 4, 5), dtype=dtype)

        sio = six.BytesIO()
        savez(sio, a1, a2)
        s = sio.getvalue()
        sio.close()

        sio = six.BytesIO(s)
        with cupy.load(sio) as d:
            b1 = d['arr_0']
            b2 = d['arr_1']
        sio.close()

        testing.assert_array_equal(a1, b1)
        testing.assert_array_equal(a2, b2)