def test_wellformed():
resources = [hexlify(urandom(16)) for _ in range(1000)]
# def packageTx(data, deps, num_out)
transactions = []
for x in range(100):
deps = sample(resources, 2)
data = json.dumps({"ID": x})
tx = packageTx(data, deps, 2)
transactions.append((tx, data))
# [(hexlify(urandom(16)), sample(resources,2), []) for x in range(300)]
n = Node(resources, 1)
n.quiet = True
shuffle(transactions)
# tx_list = sample(transactions, 100)
with Timer() as t:
for tx, data in transactions:
idx, deps, out, txdata = tx
## First perform the Tx checks
assert packageTx(data, deps, 2) == tx
## Now process this transaction
n.process(tx)
print "Time taken: %2.2f sec" % (t.interval)
def test_wellformed():
resources = [hexlify(urandom(16)) for _ in range(1000)]
# def packageTx(data, deps, num_out)
transactions = []
for x in range(100):
deps = sample(resources,2)
data = json.dumps({"ID":x})
tx = packageTx(data, deps, 2)
transactions.append((tx, data))
# [(hexlify(urandom(16)), sample(resources,2), []) for x in range(300)]
n = Node(resources, 1)
n.quiet = True
shuffle(transactions)
# tx_list = sample(transactions, 100)
with Timer() as t:
for tx, data in transactions:
idx, deps, out, txdata = tx
## First perform the Tx checks
assert packageTx(data, deps, 2) == tx
## Now process this transaction
n.process(tx)
print "Time taken: %2.2f sec" % (t.interval)
def test_redis_shard_many():
shard_map = make_shard_map(100)
_, _, [A, B, C], txdata = packageTx(data="data1", deps=[], num_out=3)
pre = [A, B, C]
nodes = [RedisNode(pre, 1, name="n%s" % i, shard=i, shard_map=shard_map) for i in shard_map]
T1 = packageTx("333", [A, B], 2)
T2 = packageTx("bbb", [A, C], 2)
# Relevent Nodes
r1nodes = [n for n in nodes if n._within_TX(T1)]
r1nodes[0].process(T1)
# Relevent Nodes
r2nodes = [n for n in nodes if n._within_TX(T2)]
r2nodes[0].process(T2)
def test_condition():
Good = True
Good &= T1[0] in r1nodes[-1].commit_yes
Good &= T1[0] in r1nodes[0].commit_yes
Good &= T2[0] in r2nodes[-1].commit_no
Good &= T2[0] in r2nodes[0].commit_no
assert Good
print "All: %s" % Good
t = xTimer(3.0, test_condition)
t.start()
def test_mock_shard_many():
limits = sorted([hexlify(urandom(32)) for _ in range(100)])
limits = ["0" * 64] + limits + ["f" * 64]
_, _, [A, B, C], txdata = packageTx(data="data1", deps=[], num_out=3)
pre = [A, B, C]
nodes = [
MockNode(pre, 1, name="n%s" % i, shard=[b0, b1])
for i, (b0, b1) in enumerate(zip(limits[:-1], limits[1:]))
]
def send(msg):
# print "Send: " + str(msg)
tx = msg["Tx"]
ns = [n for n in nodes if n._within_TX(tx)]
for n in ns:
n.receive(msg)
for n in nodes:
n.set_send(send)
T1 = packageTx("333", [A, B], 2)
T2 = packageTx("bbb", [A, C], 2)
n1 = [n for n in nodes if n._within_TX(T1)]
n2 = [n for n in nodes if n._within_TX(T2)]
# assert len(n1) == 3 and len(n2) == 3
for n in n1:
n.process(T1)
for n in n2:
n.process(T2)
def test_mock_shard_many():
limits = sorted([hexlify(urandom(32)) for _ in range(100)])
limits = ["0" * 64] + limits + ["f" * 64]
_, _, [A, B, C], txdata = packageTx(data="data1", deps=[], num_out=3)
pre = [A, B, C]
nodes = [MockNode(pre, 1, name="n%s" % i, shard=[b0,b1]) for i, (b0, b1) in enumerate(zip(limits[:-1],limits[1:]))]
def send(msg):
# print "Send: " + str(msg)
tx = msg["Tx"]
ns = [n for n in nodes if n._within_TX(tx)]
for n in ns:
n.receive(msg)
for n in nodes:
n.set_send(send)
T1 = packageTx("333", [A, B], 2)
T2 = packageTx("bbb", [A, C], 2)
n1 = [n for n in nodes if n._within_TX(T1)]
n2 = [n for n in nodes if n._within_TX(T2)]
# assert len(n1) == 3 and len(n2) == 3
for n in n1:
n.process(T1)
for n in n2:
n.process(T2)
def test_redis_shard_many():
shard_map = make_shard_map(100)
_, _, [A, B, C], txdata = packageTx(data="data1", deps=[], num_out=3)
pre = [A, B, C]
nodes = [
RedisNode(pre, 1, name="n%s" % i, shard=i, shard_map=shard_map)
for i in shard_map
]
T1 = packageTx("333", [A, B], 2)
T2 = packageTx("bbb", [A, C], 2)
# Relevent Nodes
r1nodes = [n for n in nodes if n._within_TX(T1)]
r1nodes[0].process(T1)
# Relevent Nodes
r2nodes = [n for n in nodes if n._within_TX(T2)]
r2nodes[0].process(T2)
def test_condition():
Good = True
Good &= T1[0] in r1nodes[-1].commit_yes
Good &= T1[0] in r1nodes[0].commit_yes
Good &= T2[0] in r2nodes[-1].commit_no
Good &= T2[0] in r2nodes[0].commit_no
assert Good
print "All: %s" % Good
t = xTimer(3.0, test_condition)
t.start()
def test_redis_shard_reflect():
logging.getLogger().setLevel(logging.DEBUG)
shard_map = make_shard_map(100)
_, _, [A, B, C], txdata = packageTx(data="data1", deps=[], num_out=3)
pre = [A, B, C]
nodes = [RedisNode(pre, 1, name="n%s" % i, shard=i, shard_map=shard_map) for i in shard_map]
T1 = packageTx("333", [A, B], 2)
T2 = packageTx("bbb", [A, C], 2)
T1_json = dumps({ "action":"process", "from":"ext", "Tx":T1 })
T2_json = dumps({ "action":"process", "from":"ext", "Tx":T2 })
nodes[0].r.publish('votes:%s' % i , T1_json)
nodes[0].r.publish('votes:%s' % i , T2_json)
# Relevent Nodes
r1nodes = [n for n in nodes if n._within_TX(T1)]
r2nodes = [n for n in nodes if n._within_TX(T2)]
def test_condition():
Good = True
Good &= T1[0] in r1nodes[-1].commit_yes
Good &= T1[0] in r1nodes[0].commit_yes
Good &= T2[0] in r2nodes[-1].commit_no
Good &= T2[0] in r2nodes[0].commit_no
assert Good
print "All: %s" % Good
t = xTimer(3.0, test_condition)
t.start()
def test_redis_consensus():
# logging.getLogger().setLevel(logging.DEBUG)
_, _, [A, B, C], txdata = packageTx(data="data1", deps=[], num_out=3)
pre = [A, B, C]
node1 = RedisNode(pre, 2, name="n0")
node2 = RedisNode(pre, 2, name="n1")
node3 = RedisNode(pre, 2, name="n2")
T1 = packageTx("333", [A, B], 2)
T2 = packageTx("bbb", [A, C], 2)
node1.process(T1)
node1.process(T2)
def test_redis_consensus():
# logging.getLogger().setLevel(logging.DEBUG)
_, _, [A, B, C], txdata = packageTx(data="data1", deps=[], num_out=3)
pre = [A, B, C]
node1 = RedisNode(pre, 2, name="n0")
node2 = RedisNode(pre, 2, name="n1")
node3 = RedisNode(pre, 2, name="n2")
T1 = packageTx("333", [A, B], 2)
T2 = packageTx("bbb", [A, C], 2)
node1.process(T1)
node1.process(T2)
def receive(self, message):
""" How to process incoming messages. """
# Ignore messages we sent
if self.name == message["from"]:
return
tx = message['Tx']
idx, deps, new_obj, data = tx
if not tx == packageTx(data, deps, len(new_obj)):
raise Exception("Invalid transaction.")
if not self._within_TX(tx):
raise Exception("Transaction not of interest.")
if message['action'] == "vote":
vote = message['vote']
if vote not in self.pending_vote[tx[0]]:
self.pending_vote[tx[0]].add(vote)
self.process(tx)
if message['action'] == "commit":
idx = tx[0]
# if not (idx in self.commit_yes or idx in self.commit_no):
# self.process(tx)
if message["yesno"] == False:
self.commit_no.add(idx)
else:
self.do_commit_yes(tx)
def receive(self, message):
""" How to process incoming messages. """
# Ignore messages we sent
if self.name == message["from"]:
return
tx = message['Tx']
idx, deps, new_obj, data = tx
if not tx == packageTx(data, deps, len(new_obj)):
raise Exception("Invalid transaction.")
if not self._within_TX(tx):
raise Exception("Transaction not of interest.")
if message['action'] == "vote":
vote = message['vote']
if vote not in self.pending_vote[tx[0]]:
self.pending_vote[tx[0]].add( vote )
self.process(tx)
if message['action'] == "commit":
idx = tx[0]
# if not (idx in self.commit_yes or idx in self.commit_no):
# self.process(tx)
if message["yesno"] == False:
self.commit_no.add(idx)
else:
self.do_commit_yes(tx)
def receive(self, message):
""" How to process incoming messages. """
try:
# Ensure the messae decodes
message = loads(message)
# Make sure some basic stuctures are here
originator = message["from"]
tx, action = message['Tx'], message['action']
tx = json2Tx(tx)
idx, deps, new_obj, data = tx
except Exception as e:
raise Exception("Badly formatted messages: %s" % str(e))
# Ignore messages we sent
if self.name == originator:
return
if not idx == packageTx(data, deps, len(new_obj))[0]:
# TODO: Checker goes here.
raise Exception("Invalid transaction.")
if not self._within_TX(tx):
if action == "process":
# We are going to re-route the message on a correct channel
msg = dumps({ "action":"process", "from":self.name, "Tx":Tx2json(tx) })
self.send(tx, msg)
return
else:
raise Exception("Transaction not of interest.")
if action == "vote":
# We process an incoming vote.
n, l, v = message['vote']
vote = (n, tuple(l), v)
self.RLogger.info("Receive vote (%s) for %s (%s)" % (v, idx[:6], self.name))
if vote not in self.pending_vote[idx]:
self.pending_vote[idx].add( vote )
self.process(tx)
if action == "commit":
# We process an incoming commit.
yesno = message['accept']
self.RLogger.info("Receive commit (%s) for %s (%s)" % (yesno ,idx[:6], self.name))
## TODO: call chainer.
if yesno:
self.do_commit_yes(tx)
else:
self.commit_no.add(idx)
if action == "process":
# We process a request
self.process(tx)
def receive(self, message):
""" How to process incoming messages. """
try:
# Ensure the messae decodes
message = loads(message)
# Make sure some basic stuctures are here
originator = message["from"]
tx, action = message['Tx'], message['action']
idx, deps, new_obj, data = tx
except Exception as e:
raise Exception("Badly formatted messages: %s" % str(e))
# Ignore messages we sent
if self.name == originator:
return
if not idx == packageTx(data, deps, len(new_obj))[0]:
raise Exception("Invalid transaction.")
if not self._within_TX(tx):
if action == "process":
# We are going to re-route the message on a correct channel
msg = dumps({ "action":"process", "from":self.name, "Tx":tx })
self.send(tx, msg)
return
else:
raise Exception("Transaction not of interest.")
if action == "vote":
# We process an incoming vote.
n, l, v = message['vote']
vote = (n, tuple(l), v)
self.RLogger.info("Receive vote (%s) for %s (%s)" % (v, idx[:6], self.name))
if vote not in self.pending_vote[idx]:
self.pending_vote[idx].add( vote )
self.process(tx)
if action == "commit":
# We process an incoming commit.
yesno = message['yesno']
self.RLogger.info("Receive commit (%s) for %s (%s)" % (yesno ,idx[:6], self.name))
if yesno:
self.do_commit_yes(tx)
else:
self.commit_no.add(idx)
if action == "process":
# We process a request
self.process(tx)
def test_single():
kn = KafkaNode(name="node", host=host, port=port)
rnd = hexlify(urandom(16))
T0 = packageTx(data="data1,%s" % rnd, deps=[], num_out=3)
_, _, [A, B, C], txdata = T0
T0_json = dumps({"action": "process", "from": "ext", "Tx": Tx2json(T0)})
kn.send(T0, T0_json)
T1 = packageTx("333,%s" % rnd, [A, B], 2)
T2 = packageTx("bbb,%s" % rnd, [A, C], 2)
T1_json = dumps({"action": "process", "from": "ext", "Tx": Tx2json(T1)})
T2_json = dumps({"action": "process", "from": "ext", "Tx": Tx2json(T2)})
kn.send(T1, T1_json)
kn.send(T2, T2_json)
try:
while True:
time.sleep(1.0)
except KeyboardInterrupt:
del kn
print "Clean exit ..."
def test_redis_shard_reflect():
logging.getLogger().setLevel(logging.DEBUG)
shard_map = make_shard_map(100)
_, _, [A, B, C], txdata = packageTx(data="data1", deps=[], num_out=3)
pre = [A, B, C]
nodes = [
RedisNode(pre, 1, name="n%s" % i, shard=i, shard_map=shard_map)
for i in shard_map
]
T1 = packageTx("333", [A, B], 2)
T2 = packageTx("bbb", [A, C], 2)
T1_json = dumps({"action": "process", "from": "ext", "Tx": T1})
T2_json = dumps({"action": "process", "from": "ext", "Tx": T2})
nodes[0].r.publish('votes:%s' % i, T1_json)
nodes[0].r.publish('votes:%s' % i, T2_json)
# Relevent Nodes
r1nodes = [n for n in nodes if n._within_TX(T1)]
r2nodes = [n for n in nodes if n._within_TX(T2)]
def test_condition():
Good = True
Good &= T1[0] in r1nodes[-1].commit_yes
Good &= T1[0] in r1nodes[0].commit_yes
Good &= T2[0] in r2nodes[-1].commit_no
Good &= T2[0] in r2nodes[0].commit_no
assert Good
print "All: %s" % Good
t = xTimer(3.0, test_condition)
t.start()
def test_single():
kn = KafkaNode(name="node", host=host, port=port)
rnd = hexlify(urandom(16))
T0 = packageTx(data="data1,%s" % rnd, deps=[], num_out=3)
_, _, [A, B, C], txdata = T0
T0_json = dumps({ "action":"process", "from":"ext", "Tx":Tx2json(T0) })
kn.send(T0, T0_json)
T1 = packageTx("333,%s" % rnd, [A, B], 2)
T2 = packageTx("bbb,%s" % rnd, [A, C], 2)
T1_json = dumps({ "action":"process", "from":"ext", "Tx":Tx2json(T1) })
T2_json = dumps({ "action":"process", "from":"ext", "Tx":Tx2json(T2) })
kn.send(T1, T1_json)
kn.send(T2, T2_json)
try:
while True:
time.sleep(1.0)
except KeyboardInterrupt:
del kn
print "Clean exit ..."
def test_distribution():
shard_map = make_shard_map(100)
from collections import defaultdict
d = defaultdict(int)
for x in range(1000):
T1 = packageTx("333%s" % x, [], 2)
for i in shard_map:
b0, b1 = shard_map[i]
if within_TX(T1, b0, b1):
d[i] += 1
for i in sorted(d):
print "%3d | %s" % (i, "=" * d[i])
def test_distribution():
shard_map = make_shard_map(100)
from collections import defaultdict
d = defaultdict(int)
for x in range(1000):
T1 = packageTx("333%s" % x, [], 2)
for i in shard_map:
b0, b1 = shard_map[i]
if within_TX(T1, b0, b1):
d[i] += 1
for i in sorted(d):
print "%3d | %s" % (i, "=" * d[i])
host=host,
port=port) for i in shard_map
]
def concerned(tx):
l = []
for i in shard_map:
(b0, b1) = shard_map[i]
if within_TX(tx, b0, b1):
l.append(i)
return l
T0s = []
for _ in range(100):
rnd = hexlify(urandom(16))
T0 = packageTx(data="data1-%s" % rnd, deps=[], num_out=3)
T0s.append(T0)
# _, _, [A, B, C], txdata = T0
# T1 = packageTx("333-%s" % rnd, [A, B], 2)
# T2 = packageTx("bbb-%s" % rnd, [A, C], 2)
P = lambda T: dumps({
"action": "process",
"from": "ext",
"Tx": Tx2json(T)
})
print "T0 depends on: %s" % str(concerned(T0))
nodes[0].receive(P(T0))
nodes = [ KafkaNode(quorum=1, name="n%s" % i,
shard=i, shard_map=shard_map, host=host, port=port)
for i in shard_map ]
def concerned(tx):
l = []
for i in shard_map:
(b0, b1) = shard_map[i]
if within_TX(tx, b0, b1):
l.append(i)
return l
T0s = []
for _ in range(100):
rnd = hexlify(urandom(16))
T0 = packageTx(data="data1-%s" % rnd, deps=[], num_out=3)
T0s.append(T0)
# _, _, [A, B, C], txdata = T0
# T1 = packageTx("333-%s" % rnd, [A, B], 2)
# T2 = packageTx("bbb-%s" % rnd, [A, C], 2)
P = lambda T: dumps({ "action":"process", "from":"ext", "Tx":Tx2json(T) })
print "T0 depends on: %s" % str(concerned(T0))
nodes[0].receive( P(T0) )
# Relevent Nodes
# r1nodes = [n for n in nodes if n._within_TX(T1)]
# r1nodes[0].process(T1)
