def time_packbits_axis1(self, dtype):
np.packbits(self.d2, axis=1)
def time_packbits(self, dtype):
np.packbits(self.d)
def compress_by_chunk(self, cupy_bool_tensor, num_chunks):
packed_sign = cupy.packbits(cupy_bool_tensor)
sign_list_packed = cupy.split(packed_sign, num_chunks)
cupy.cuda.get_current_stream().synchronize()
return sign_list_packed
raw_data = np.concatenate(data)
else:
raw_data = np.fromfile(raw_data_dir, dtype=np.uint8)
# load seed
seed = np.fromfile(seed_dir, dtype=np.uint8)
seed = np.unpackbits(seed[::-1], bitorder='little').reshape(-1)
seed = seed[:L]
toeplitz = linalg.toeplitz(seed, seed)[:k, :n]
toeplitz = cp.array(toeplitz)
data_size = len(raw_data)
batch_size = 1024 * 1024
chunk_size = batch_size * (n // 8)
output_datas = []
for i in range(data_size // chunk_size):
print(i)
input_data = raw_data[i * chunk_size:(i + 1) * chunk_size]
input_data = np.unpackbits(input_data).reshape(batch_size, n).T
input_data = cp.array(input_data)
output_data = cp.matmul(toeplitz, input_data).T
output_data = (output_data % 2).reshape(batch_size, k // 8, 8)
output_data = cp.packbits(output_data)
output_data = cp.asnumpy(output_data)
output_datas.append(output_data.reshape(-1))
output_datas = np.array(output_datas).reshape(-1)
output_datas.tofile(final_data_dir)
def pack2bits(first, second):
data = torch.cat((first, second.type(torch.bool)), 0)
return cupy2torch(cupy.packbits(torch2cupy(data))), first.numel()
def packbits(array):
return cupy2torch(cupy.packbits(torch2cupy(array))), array.numel()