def trial(fd):
params = search_params()
for multiple in MULTIPLE_BLK:
for blk_size in BLK_SIZE:
for bound in B:
true_false_positives = int(multiple * blk_size)
mempool_size = true_false_positives + blk_size
print(
'Running %d trials for parameter combination: multiple of blk %d blk size %d CB bound %f'
% (NUM_TRIAL, multiple, blk_size, bound))
for x in range(NUM_TRIAL):
blk, receiver_mempool = create_mempools(
mempool_size, blk_size)
# Sender creates BF of blk
a, fpr_sender, iblt_rows_first = params.CB_solve_a(
mempool_size, blk_size, blk_size, 0, bound)
bloom_sender = BloomFilter(blk_size, fpr_sender)
# Sender creates IBLT of blk
iblt_sender_first = PYBLT(a, TXN_SHORT_BYTES)
# Add to BF and IBLT
for txn in blk:
bloom_sender.add(txn)
iblt_sender_first.insert(txn, 0x0)
# Receiver computes how many items pass through BF of sender and creates IBLT
iblt_receiver_first = PYBLT(a, TXN_SHORT_BYTES)
passed = []
for txn in receiver_mempool:
if txn in bloom_sender:
passed.append(txn)
iblt_receiver_first.insert(txn,
0x0) #(id and content)
observed_false_positives = len(passed) - blk_size
# Eppstein subtraction
T = iblt_receiver_first.subtract(iblt_sender_first)
boolean, result = T.list_entries()
# Check whether decoding successful
flag, in_blk = decode_blk(result, passed, blk)
# Each component of graphene blk size
first_IBLT = (iblt_rows_first * TAU)
first_BF = (bloom_sender.num_bits / 8.0)
# Compute size of Graphene block
graphene = first_IBLT + first_BF
# Size of Compact block (inv + getdata)
# getdata = (1-fraction) * BLK_SIZE * TXN_SHORT_BYTES
getdata = 0
inv = blk_size * TXN_SHORT_BYTES_CB
compact = inv + getdata
#print('getdata', getdata)
#print('Pinar', (len(in_blk) * TXN_SHORT_BYTES))
#print((boolean and flag))
#assert getdata == (len(in_blk) * TXN_SHORT_BYTES)
fd.write(
str(mempool_size) + '\t' + str(blk_size) + '\t' +
str(bound) + '\t' + str(fpr_sender) + '\t' + str(a) +
'\t' + str(iblt_rows_first) + '\t' +
str(true_false_positives) + '\t' +
str(observed_false_positives) + '\t' + str(compact) +
'\t' + str(boolean and flag) + '\t' + str(graphene) +
'\t' + str(first_IBLT) + '\t' + str(first_BF) + '\t' +
str(multiple) + '\n')
fd.flush()
import sys
import random
import numpy as np
from pybloom_live import BloomFilter
sys.path.append('.')
from search_params import search_params
PYBLT_PATH = '/home/pinar/IBLT-optimization/'
sys.path.append(PYBLT_PATH)
from pyblt import PYBLT
PYBLT.set_parameter_filename(PYBLT_PATH +
'param.export.0.995833.2018-07-17.csv')
# Constants
PATH = '.'
TXN_SHORT_BYTES_CB = 6
TXN_SHORT_BYTES = 8
BITSIZE = TXN_SHORT_BYTES * 8 # number of bytes per transaction, converted to bits per transaction
TAU = 12 # bytes per IBLT cell
BLK_SIZE = [200, 2000, 10000] # num of txns in block
B = [1 - (239 / 240)]
MULTIPLE_BLK = np.arange(0, 5.5, 0.5)
NUM_TRIAL = 1000
def create_mempools(mempool_size, blk_size):
# create a random set of transactions in a mempool
blk = [random.getrandbits(BITSIZE) for x in range(blk_size)]
in_blk = blk.copy()
def trial(fd):
params = search_params()
for blk_size in BLK_SIZE:
true_false_positives = blk_size
for bound in B:
for fraction in FRACTION:
# True_positives is the number of txns in the blk the receiver has
true_positives = int(blk_size * fraction)
mempool_size = true_false_positives + true_positives
print(
'Running %d trials for parameter combination: extra txns in mempool %d blk size %d fraction %f'
% (NUM_TRIAL, true_false_positives, blk_size, fraction))
# Size of Compact block (inv + getdata)
getdata = (1 - fraction) * blk_size * TXN_SHORT_BYTES_CB
inv = blk_size * TXN_SHORT_BYTES_CB
compact = inv + getdata
for i in range(NUM_TRIAL):
blk, receiver_mempool = create_mempools(
mempool_size, fraction, blk_size, true_false_positives)
# Sender creates BF of blk
a, fpr_sender, iblt_rows_first = params.solve_a(
mempool_size, blk_size, blk_size, 0)
bloom_sender = BloomFilter(blk_size, fpr_sender)
tmp = blk_size + 0.5
exponent = (-bloom_sender.num_slices *
tmp) / (bloom_sender.num_bits - 1)
real_fpr_sender = (1 -
exp(exponent))**bloom_sender.num_slices
#exponent = (-bloom_sender.num_slices*blk_size) / bloom_sender.num_bits
#tmp = (1-exp(exponent)) ** bloom_sender.num_slices
#real_fpr_sender = max(tmp, fpr_sender)
#assert real_fpr_sender >= fpr_sender
# Sender creates IBLT of blk
iblt_sender_first = PYBLT(a, TXN_SHORT_BYTES)
# Add to BF and IBLT
for txn in blk:
bloom_sender.add(txn)
iblt_sender_first.insert(txn, 0x0)
# Receiver computes how many items pass through BF of sender and creates IBLT
iblt_receiver_first = PYBLT(a, TXN_SHORT_BYTES)
Z = []
for txn in receiver_mempool:
if txn in bloom_sender:
Z.append(txn)
iblt_receiver_first.insert(txn,
0x0) #(id and content)
z = len(Z)
observed_false_positives = z - true_positives
# Eppstein subtraction
T = iblt_receiver_first.subtract(iblt_sender_first)
boolean, result = T.list_entries()
#assert boolean == False
# Check whether decoding successful
if boolean == True:
flag, in_blk = decode_blk(result, Z, blk)
# Each component of graphene blk size
first_IBLT = (iblt_rows_first * TAU)
first_BF = (bloom_sender.num_bits / 8.0)
extra = (len(in_blk) * TXN_SHORT_BYTES)
# Compute size of Graphene block
graphene = first_IBLT + first_BF + extra
fd.write(
str(true_false_positives) + '\t' + str(blk_size) +
'\t' + str(bound) + '\t' + str(fraction) + '\t' +
str(mempool_size) + '\t' + str(fpr_sender) + '\t' +
str(real_fpr_sender) + '\t' + str(0) + '\t' +
str(a) + '\t' + str(0) + '\t' + str(0) + '\t' +
str(0) + '\t' + str(z) + '\t' + str(0) + '\t' +
str(observed_false_positives) + '\t' +
str(boolean and flag) + '\t' + str(False) + '\t' +
str(graphene) + '\t' + str(first_IBLT) + '\t' +
str(first_BF) + '\t' + str(0) + '\t' + str(0) +
'\t' + str(extra) + '\t' + str(iblt_rows_first) +
'\t' + str(0) + '\t' + str(compact) + '\n')
else:
# Receiver creates BF of txns that passed through sender's BF
# print('z', z)
# print('bound', bound)
x_star = params.search_x_star(z, mempool_size,
real_fpr_sender, bound,
blk_size)
temp = (mempool_size - x_star) * real_fpr_sender
y_star = params.CB_bound(temp, real_fpr_sender, bound)
#print('y_star', y_star)
y_star = ceil(y_star)
b, fpr_receiver, iblt_rows_second = params.solve_a(
blk_size, z, x_star, y_star)
bloom_receiver = BloomFilter(z, fpr_receiver)
for txn in Z:
bloom_receiver.add(txn)
# Receiver determines IBLT size
iblt_sender_second = PYBLT(b + y_star, TXN_SHORT_BYTES)
# Sender creates IBLT of blk again and sends txns that do not pass through BF of receiver
count = 0
for txn in blk:
iblt_sender_second.insert(txn, 0x0)
if txn not in bloom_receiver:
T.insert(
txn, 0x0
) # add txns just received to subtracted IBLT
Z = Z + [txn] # sends the txn to the receiver
count = count + 1
iblt_receiver_second = PYBLT(b + y_star,
TXN_SHORT_BYTES)
for txn in Z:
iblt_receiver_second.insert(txn, 0x0)
# Eppstein subtraction
T_second = iblt_receiver_second.subtract(
iblt_sender_second)
boolean, result = T_second.list_entries()
#print(boolean)
#print('Z', z)
# Check whether blk was reconstructed properly
flag, in_blk = decode_blk(result, Z, blk)
final = False
if boolean == False or flag == False:
final, in_blk, not_in_blk = try_ping_pong(
T, T_second, set(), set())
#print('Ping pong result', final)
if final == True:
possibly_in_blk = set(Z)
possibly_in_blk.difference_update(not_in_blk)
reconstructed_blk = list(
in_blk.union(possibly_in_blk))
assert set(reconstructed_blk) == set(blk)
# Each component of graphene blk size
first_IBLT = (iblt_rows_first * TAU)
first_BF = (bloom_sender.num_bits / 8.0)
second_IBLT = (iblt_rows_second * TAU)
second_BF = (bloom_receiver.num_bits / 8.0)
extra = (len(in_blk) * TXN_SHORT_BYTES)
# Compute size of Graphene block
graphene = first_IBLT + first_BF + second_IBLT + second_BF + extra
fd.write(
str(true_false_positives) + '\t' + str(blk_size) +
'\t' + str(bound) + '\t' + str(fraction) + '\t' +
str(mempool_size) + '\t' + str(fpr_sender) + '\t' +
str(real_fpr_sender) + '\t' + str(fpr_receiver) +
'\t' + str(a) + '\t' + str(b) + '\t' +
str(x_star) + '\t' + str(y_star) + '\t' + str(z) +
'\t' + str(count) + '\t' +
str(observed_false_positives) + '\t' +
str(boolean and flag) + '\t' + str(final) + '\t' +
str(graphene) + '\t' + str(first_IBLT) + '\t' +
str(first_BF) + '\t' + str(second_IBLT) + '\t' +
str(second_BF) + '\t' + str(extra) + '\t' +
str(iblt_rows_first) + '\t' +
str(iblt_rows_second) + '\t' + str(compact) + '\n')
fd.flush()
def trial(fd):
params = search_params()
for blk_size in BLK_SIZE:
for fpr_r in FPR_RECEIVER:
for fraction in FRACTION:
# True_positives is the number of txns in the blk the receiver has
true_positives = int(blk_size * fraction)
true_false_positives = blk_size - true_positives
mempool_size = true_false_positives + true_positives
assert mempool_size == blk_size
print(
'Running %d trials for parameter combination: extra txns in mempool %d blk size %d CB bound %f fraction %f'
% (NUM_TRIAL, true_false_positives, blk_size, bound,
fraction))
# Size of Compact block (inv + getdata)
getdata = true_false_positives * TXN_SHORT_BYTES_CB
inv = blk_size * TXN_SHORT_BYTES_CB
compact = inv + getdata
for i in range(NUM_TRIAL):
blk, receiver_mempool = create_mempools(
mempool_size, fraction, blk_size, true_false_positives)
# Sender creates BF of blk
a, fpr_sender, iblt_rows_first = params.CB_solve_a(
mempool_size, blk_size, blk_size, 0, bound)
bloom_sender = BloomFilter(blk_size, fpr_sender)
tmp = blk_size + 0.5
exponent = (-bloom_sender.num_slices *
tmp) / (bloom_sender.num_bits - 1)
real_fpr_sender = (1 -
exp(exponent))**bloom_sender.num_slices
#exponent = (-bloom_sender.num_slices*blk_size) / bloom_sender.num_bits
#tmp = (1-exp(exponent)) ** bloom_sender.num_slices
#real_fpr_sender = max(tmp, fpr_sender)
#assert real_fpr_sender >= fpr_sender
# Sender creates IBLT of blk
iblt_sender_first = PYBLT(a, TXN_SHORT_BYTES)
# Add to BF and IBLT
for txn in blk:
bloom_sender.add(txn)
iblt_sender_first.insert(txn, 0x0)
# Receiver computes how many items pass through BF of sender and creates IBLT
iblt_receiver_first = PYBLT(a, TXN_SHORT_BYTES)
Z = []
for txn in receiver_mempool:
if txn in bloom_sender:
Z.append(txn)
iblt_receiver_first.insert(txn,
0x0) #(id and content)
z = len(Z)
observed_false_positives = z - true_positives
# Eppstein subtraction
T = iblt_receiver_first.subtract(iblt_sender_first)
boolean, result = T.list_entries()
#assert boolean == False
# Check whether decoding successful
if boolean == True:
flag, in_blk = decode_blk(result, Z, blk)
# Each component of graphene blk size
first_IBLT = (iblt_rows_first * TAU)
first_BF = (bloom_sender.num_bits / 8.0)
extra = (len(in_blk) * TXN_SHORT_BYTES)
# Compute size of Graphene block
graphene = first_IBLT + first_BF + extra
fd.write(
str(true_false_positives) + '\t' + str(blk_size) +
'\t' + str(bound) + '\t' + str(fraction) + '\t' +
str(mempool_size) + '\t' + str(fpr_sender) + '\t' +
str(real_fpr_sender) + '\t' + str(0) + '\t' +
str(a) + '\t' + str(0) + '\t' + str(0) + '\t' +
str(z) + '\t' + str(0) + '\t' +
str(observed_false_positives) + '\t' +
str(boolean and flag) + '\t' + str(False) + '\t' +
str(graphene) + '\t' + str(first_IBLT) + '\t' +
str(first_BF) + '\t' + str(0) + '\t' + str(0) +
'\t' + str(extra) + '\t' + str(iblt_rows_first) +
'\t' + str(0) + '\t' + str(compact) + '\t' +
str(0) + '\t' + str(0) + '\n')
else:
fpr_receiver = fpr_r
bloom_receiver = BloomFilter(z, fpr_receiver)
for txn in Z:
bloom_receiver.add(txn)
# Sender determines IBLT size
from_sender = []
for txn in blk:
if txn not in bloom_receiver:
from_sender.append(txn)
T.insert(txn, 0x0)
h = len(
from_sender) # sender sends these over to receiver
#z is the count of txns that pass through bloom filter S
x_star = params.search_x_star(z=blk_size - h,
mempool_size=blk_size,
fpr=fpr_receiver,
bound,
blk_size)
temp = (blk_size - x_star) * fpr_receiver
y_star = params.CB_bound(temp, fpr_receiver, bound)
y_star = ceil(y_star)
b, fpr_sender_second, iblt_rows_second = params.solve_a(
m=blk_size, n=x_star, x=x_star, y=y_star)
bloom_sender_second = BloomFilter(
blk_size - h, fpr_sender_second)
iblt_sender_second = PYBLT(b + y_star, TXN_SHORT_BYTES)
for txn in blk:
iblt_sender_second.insert(txn, 0x0)
if txn not in from_sender:
bloom_sender_second.add(txn)
# Receiver determines IBLT size
count = 0
for txn in Z:
if txn in bloom_sender_second:
from_sender.append(txn)
T.insert(txn, 0x0)
count = count + 1
iblt_receiver_second = PYBLT(b + y_star,
TXN_SHORT_BYTES)
# Size of IBLT
# if b+(blk_size-h-x_star)-1 >= len(params.params): # difference too much
# tmp = b+(blk_size-h-x_star) * 1.362549
# rows = ceil(tmp)
# iblt_rows_second = rows * 12
# else:
# rows = params.params[b+(blk_size-h-x_star)-1][3]
# iblt_rows_second = rows * 12
for txn in from_sender:
iblt_receiver_second.insert(txn, 0x0)
# Eppstein subtraction
T_second = iblt_receiver_second.subtract(
iblt_sender_second)
boolean, result = T_second.list_entries()
#print(boolean)
#print('Z', z)
# Check whether blk was reconstructed properly
flag, in_blk = decode_blk(result, from_sender, blk)
final = False
if boolean == False or flag == False:
final, in_blk, not_in_blk = try_ping_pong(
T, T_second, set(), set())
#print('Ping pong result', final)
if final == True:
possibly_in_blk = set(from_sender)
possibly_in_blk.difference_update(not_in_blk)
reconstructed_blk = list(
in_blk.union(possibly_in_blk))
assert set(reconstructed_blk) == set(blk)
# Each component of graphene blk size
first_IBLT = (iblt_rows_first * TAU)
first_BF = (bloom_sender.num_bits / 8.0)
second_IBLT = (iblt_rows_second * TAU)
second_BF = (bloom_receiver.num_bits / 8.0)
third_BF = (bloom_sender_second.num_bits / 8.0)
extra = (len(in_blk) * TXN_SHORT_BYTES)
# Compute size of Graphene block
graphene = first_IBLT + first_BF + second_IBLT + second_BF + third_BF + extra
fd.write(
str(true_false_positives) + '\t' + str(blk_size) +
'\t' + str(bound) + '\t' + str(fraction) + '\t' +
str(mempool_size) + '\t' + str(fpr_sender) + '\t' +
str(real_fpr_sender) + '\t' + str(fpr_receiver) +
'\t' + str(a) + '\t' + str(b) + '\t' +
str(x_star) + '\t' + str(z) + '\t' + str(count) +
'\t' + str(observed_false_positives) + '\t' +
str(boolean and flag) + '\t' + str(final) + '\t' +
str(graphene) + '\t' + str(first_IBLT) + '\t' +
str(first_BF) + '\t' + str(second_IBLT) + '\t' +
str(second_BF) + '\t' + str(extra) + '\t' +
str(iblt_rows_first) + '\t' +
str(iblt_rows_second) + '\t' + str(compact) +
'\t' + str(third_BF) + '\t' +
str(fpr_sender_second) + '\n')
fd.flush()
def basic_test():
PYBLT.set_parameter_filename(
'param.export.0.995833333333333.2018-07-12.csv')
f = PYBLT(10, 9)
f.insert(8, "txn_10008")
f.insert(2, "txn_1002")
# print(f.dump_table())
print("F Entries: ")
success, entries = f.list_entries()
print(success)
for x in entries:
print(x, entries[x])
g = PYBLT(10, 9)
g.insert(8, "txn_10008")
g.insert(7, "txn_10010")
print("G Entries: ")
success, entries = g.list_entries()
print(success)
for x in entries:
print(x, entries[x])
h = g.subtract(f)
print("\nh=g-f; should have 7, and then 2:")
success, entries = h.list_entries()
print(success)
for x in entries:
print(x, entries[x])
del (h)
h = f.subtract(g)
print("\nh=f-g; should have 2, and then 7:")
success, entries = h.list_entries()
print(success)
for x in entries:
print(x, entries[x])
del (h)