def conver_data_to_bf(data):
com_to_len = data.shape[1]
# print(data.shape)
#
compressed_data = data[:, :]
#
# ------------Put into compressed and BF data---------
length = 10000
b = 5
num_hash = 10
dis = float(0.05)
g = (2 * b + 1) * com_to_len
false_positive = math.pow(1 - math.exp(-(num_hash * g) / length), num_hash)
# print ('lenth:',length,'b:',b,'num_hash:',num_hash,'dis:',dis ,'false_positive: ', false_positive)
false_positive = math.pow(1 - math.exp(-(float)(num_hash * g) / length), num_hash)
print ('lenth:', length, 'num_hash:', num_hash, 'false_positive: ', false_positive)
## # generate the npy with the bf and data
bf_ts = BF_TS(length, num_hash, b, dis / (2 * b), dis)
##
# ---------------------
print ('BF filter')
# print 'start'
start_time = time.time()
bf_train = bf_ts.convert_set_to_bf(compressed_data) # the result it a list and hard to convert to np array
print ('BF filter done')
output = bf_ts.convert_bitarray_to_train_data(bf_train, len(bf_train), length)
print('bf done using time: {} mins'.format((time.time() - start_time) / 60))
# cifar_bfed = np.stack([cifar_batch, cifar_batch2, cifar_batch3], axis=2)
return output
# # print(cifar_data['x'][0])
length = 50000
b=5
num_hash = 4
dis = float(0.03)
com_to_len = 1024
# g=(2*b+1)*com_to_len
false_positive= math.pow( 1-math.exp(-float(num_hash*2*b*com_to_len)/length) , num_hash )
print ('lenth:',length,'num_hash:',num_hash,'total distance:',dis,\
'step dis:',dis/(2*b),'neighbours:',b, 'false_positive: ', false_positive)
bf_ts=BF_TS(length,num_hash,b,dis/(2*b),-100)
timenow = time.time()
print(datetime.now())
# expected width distance = 0.003
batch = 4
print('batch ', batch)
filename = '/Users/wanli/Desktop/tmp/aruna/raw_cifar_preprocess/preprocess_batch_'+str(batch)+'.npy'
cifar_data = np.load(filename)
print(cifar_data.shape)
for i in range(3):