def test_wellformed(): resources = [hexlify(urandom(16)) for _ in range(1000)] # def packageTx(data, deps, num_out) transactions = [] for x in range(100): deps = sample(resources, 2) data = json.dumps({"ID": x}) tx = packageTx(data, deps, 2) transactions.append((tx, data)) # [(hexlify(urandom(16)), sample(resources,2), []) for x in range(300)] n = Node(resources, 1) n.quiet = True shuffle(transactions) # tx_list = sample(transactions, 100) with Timer() as t: for tx, data in transactions: idx, deps, out, txdata = tx ## First perform the Tx checks assert packageTx(data, deps, 2) == tx ## Now process this transaction n.process(tx) print "Time taken: %2.2f sec" % (t.interval)
def test_wellformed(): resources = [hexlify(urandom(16)) for _ in range(1000)] # def packageTx(data, deps, num_out) transactions = [] for x in range(100): deps = sample(resources,2) data = json.dumps({"ID":x}) tx = packageTx(data, deps, 2) transactions.append((tx, data)) # [(hexlify(urandom(16)), sample(resources,2), []) for x in range(300)] n = Node(resources, 1) n.quiet = True shuffle(transactions) # tx_list = sample(transactions, 100) with Timer() as t: for tx, data in transactions: idx, deps, out, txdata = tx ## First perform the Tx checks assert packageTx(data, deps, 2) == tx ## Now process this transaction n.process(tx) print "Time taken: %2.2f sec" % (t.interval)
def test_redis_shard_many(): shard_map = make_shard_map(100) _, _, [A, B, C], txdata = packageTx(data="data1", deps=[], num_out=3) pre = [A, B, C] nodes = [RedisNode(pre, 1, name="n%s" % i, shard=i, shard_map=shard_map) for i in shard_map] T1 = packageTx("333", [A, B], 2) T2 = packageTx("bbb", [A, C], 2) # Relevent Nodes r1nodes = [n for n in nodes if n._within_TX(T1)] r1nodes[0].process(T1) # Relevent Nodes r2nodes = [n for n in nodes if n._within_TX(T2)] r2nodes[0].process(T2) def test_condition(): Good = True Good &= T1[0] in r1nodes[-1].commit_yes Good &= T1[0] in r1nodes[0].commit_yes Good &= T2[0] in r2nodes[-1].commit_no Good &= T2[0] in r2nodes[0].commit_no assert Good print "All: %s" % Good t = xTimer(3.0, test_condition) t.start()
def test_mock_shard_many(): limits = sorted([hexlify(urandom(32)) for _ in range(100)]) limits = ["0" * 64] + limits + ["f" * 64] _, _, [A, B, C], txdata = packageTx(data="data1", deps=[], num_out=3) pre = [A, B, C] nodes = [ MockNode(pre, 1, name="n%s" % i, shard=[b0, b1]) for i, (b0, b1) in enumerate(zip(limits[:-1], limits[1:])) ] def send(msg): # print "Send: " + str(msg) tx = msg["Tx"] ns = [n for n in nodes if n._within_TX(tx)] for n in ns: n.receive(msg) for n in nodes: n.set_send(send) T1 = packageTx("333", [A, B], 2) T2 = packageTx("bbb", [A, C], 2) n1 = [n for n in nodes if n._within_TX(T1)] n2 = [n for n in nodes if n._within_TX(T2)] # assert len(n1) == 3 and len(n2) == 3 for n in n1: n.process(T1) for n in n2: n.process(T2)
def test_mock_shard_many(): limits = sorted([hexlify(urandom(32)) for _ in range(100)]) limits = ["0" * 64] + limits + ["f" * 64] _, _, [A, B, C], txdata = packageTx(data="data1", deps=[], num_out=3) pre = [A, B, C] nodes = [MockNode(pre, 1, name="n%s" % i, shard=[b0,b1]) for i, (b0, b1) in enumerate(zip(limits[:-1],limits[1:]))] def send(msg): # print "Send: " + str(msg) tx = msg["Tx"] ns = [n for n in nodes if n._within_TX(tx)] for n in ns: n.receive(msg) for n in nodes: n.set_send(send) T1 = packageTx("333", [A, B], 2) T2 = packageTx("bbb", [A, C], 2) n1 = [n for n in nodes if n._within_TX(T1)] n2 = [n for n in nodes if n._within_TX(T2)] # assert len(n1) == 3 and len(n2) == 3 for n in n1: n.process(T1) for n in n2: n.process(T2)
def test_redis_shard_many(): shard_map = make_shard_map(100) _, _, [A, B, C], txdata = packageTx(data="data1", deps=[], num_out=3) pre = [A, B, C] nodes = [ RedisNode(pre, 1, name="n%s" % i, shard=i, shard_map=shard_map) for i in shard_map ] T1 = packageTx("333", [A, B], 2) T2 = packageTx("bbb", [A, C], 2) # Relevent Nodes r1nodes = [n for n in nodes if n._within_TX(T1)] r1nodes[0].process(T1) # Relevent Nodes r2nodes = [n for n in nodes if n._within_TX(T2)] r2nodes[0].process(T2) def test_condition(): Good = True Good &= T1[0] in r1nodes[-1].commit_yes Good &= T1[0] in r1nodes[0].commit_yes Good &= T2[0] in r2nodes[-1].commit_no Good &= T2[0] in r2nodes[0].commit_no assert Good print "All: %s" % Good t = xTimer(3.0, test_condition) t.start()
def test_redis_shard_reflect(): logging.getLogger().setLevel(logging.DEBUG) shard_map = make_shard_map(100) _, _, [A, B, C], txdata = packageTx(data="data1", deps=[], num_out=3) pre = [A, B, C] nodes = [RedisNode(pre, 1, name="n%s" % i, shard=i, shard_map=shard_map) for i in shard_map] T1 = packageTx("333", [A, B], 2) T2 = packageTx("bbb", [A, C], 2) T1_json = dumps({ "action":"process", "from":"ext", "Tx":T1 }) T2_json = dumps({ "action":"process", "from":"ext", "Tx":T2 }) nodes[0].r.publish('votes:%s' % i , T1_json) nodes[0].r.publish('votes:%s' % i , T2_json) # Relevent Nodes r1nodes = [n for n in nodes if n._within_TX(T1)] r2nodes = [n for n in nodes if n._within_TX(T2)] def test_condition(): Good = True Good &= T1[0] in r1nodes[-1].commit_yes Good &= T1[0] in r1nodes[0].commit_yes Good &= T2[0] in r2nodes[-1].commit_no Good &= T2[0] in r2nodes[0].commit_no assert Good print "All: %s" % Good t = xTimer(3.0, test_condition) t.start()
def test_redis_consensus(): # logging.getLogger().setLevel(logging.DEBUG) _, _, [A, B, C], txdata = packageTx(data="data1", deps=[], num_out=3) pre = [A, B, C] node1 = RedisNode(pre, 2, name="n0") node2 = RedisNode(pre, 2, name="n1") node3 = RedisNode(pre, 2, name="n2") T1 = packageTx("333", [A, B], 2) T2 = packageTx("bbb", [A, C], 2) node1.process(T1) node1.process(T2)
def receive(self, message): """ How to process incoming messages. """ # Ignore messages we sent if self.name == message["from"]: return tx = message['Tx'] idx, deps, new_obj, data = tx if not tx == packageTx(data, deps, len(new_obj)): raise Exception("Invalid transaction.") if not self._within_TX(tx): raise Exception("Transaction not of interest.") if message['action'] == "vote": vote = message['vote'] if vote not in self.pending_vote[tx[0]]: self.pending_vote[tx[0]].add(vote) self.process(tx) if message['action'] == "commit": idx = tx[0] # if not (idx in self.commit_yes or idx in self.commit_no): # self.process(tx) if message["yesno"] == False: self.commit_no.add(idx) else: self.do_commit_yes(tx)
def receive(self, message): """ How to process incoming messages. """ # Ignore messages we sent if self.name == message["from"]: return tx = message['Tx'] idx, deps, new_obj, data = tx if not tx == packageTx(data, deps, len(new_obj)): raise Exception("Invalid transaction.") if not self._within_TX(tx): raise Exception("Transaction not of interest.") if message['action'] == "vote": vote = message['vote'] if vote not in self.pending_vote[tx[0]]: self.pending_vote[tx[0]].add( vote ) self.process(tx) if message['action'] == "commit": idx = tx[0] # if not (idx in self.commit_yes or idx in self.commit_no): # self.process(tx) if message["yesno"] == False: self.commit_no.add(idx) else: self.do_commit_yes(tx)
def receive(self, message): """ How to process incoming messages. """ try: # Ensure the messae decodes message = loads(message) # Make sure some basic stuctures are here originator = message["from"] tx, action = message['Tx'], message['action'] tx = json2Tx(tx) idx, deps, new_obj, data = tx except Exception as e: raise Exception("Badly formatted messages: %s" % str(e)) # Ignore messages we sent if self.name == originator: return if not idx == packageTx(data, deps, len(new_obj))[0]: # TODO: Checker goes here. raise Exception("Invalid transaction.") if not self._within_TX(tx): if action == "process": # We are going to re-route the message on a correct channel msg = dumps({ "action":"process", "from":self.name, "Tx":Tx2json(tx) }) self.send(tx, msg) return else: raise Exception("Transaction not of interest.") if action == "vote": # We process an incoming vote. n, l, v = message['vote'] vote = (n, tuple(l), v) self.RLogger.info("Receive vote (%s) for %s (%s)" % (v, idx[:6], self.name)) if vote not in self.pending_vote[idx]: self.pending_vote[idx].add( vote ) self.process(tx) if action == "commit": # We process an incoming commit. yesno = message['accept'] self.RLogger.info("Receive commit (%s) for %s (%s)" % (yesno ,idx[:6], self.name)) ## TODO: call chainer. if yesno: self.do_commit_yes(tx) else: self.commit_no.add(idx) if action == "process": # We process a request self.process(tx)
def receive(self, message): """ How to process incoming messages. """ try: # Ensure the messae decodes message = loads(message) # Make sure some basic stuctures are here originator = message["from"] tx, action = message['Tx'], message['action'] idx, deps, new_obj, data = tx except Exception as e: raise Exception("Badly formatted messages: %s" % str(e)) # Ignore messages we sent if self.name == originator: return if not idx == packageTx(data, deps, len(new_obj))[0]: raise Exception("Invalid transaction.") if not self._within_TX(tx): if action == "process": # We are going to re-route the message on a correct channel msg = dumps({ "action":"process", "from":self.name, "Tx":tx }) self.send(tx, msg) return else: raise Exception("Transaction not of interest.") if action == "vote": # We process an incoming vote. n, l, v = message['vote'] vote = (n, tuple(l), v) self.RLogger.info("Receive vote (%s) for %s (%s)" % (v, idx[:6], self.name)) if vote not in self.pending_vote[idx]: self.pending_vote[idx].add( vote ) self.process(tx) if action == "commit": # We process an incoming commit. yesno = message['yesno'] self.RLogger.info("Receive commit (%s) for %s (%s)" % (yesno ,idx[:6], self.name)) if yesno: self.do_commit_yes(tx) else: self.commit_no.add(idx) if action == "process": # We process a request self.process(tx)
def test_single(): kn = KafkaNode(name="node", host=host, port=port) rnd = hexlify(urandom(16)) T0 = packageTx(data="data1,%s" % rnd, deps=[], num_out=3) _, _, [A, B, C], txdata = T0 T0_json = dumps({"action": "process", "from": "ext", "Tx": Tx2json(T0)}) kn.send(T0, T0_json) T1 = packageTx("333,%s" % rnd, [A, B], 2) T2 = packageTx("bbb,%s" % rnd, [A, C], 2) T1_json = dumps({"action": "process", "from": "ext", "Tx": Tx2json(T1)}) T2_json = dumps({"action": "process", "from": "ext", "Tx": Tx2json(T2)}) kn.send(T1, T1_json) kn.send(T2, T2_json) try: while True: time.sleep(1.0) except KeyboardInterrupt: del kn print "Clean exit ..."
def test_redis_shard_reflect(): logging.getLogger().setLevel(logging.DEBUG) shard_map = make_shard_map(100) _, _, [A, B, C], txdata = packageTx(data="data1", deps=[], num_out=3) pre = [A, B, C] nodes = [ RedisNode(pre, 1, name="n%s" % i, shard=i, shard_map=shard_map) for i in shard_map ] T1 = packageTx("333", [A, B], 2) T2 = packageTx("bbb", [A, C], 2) T1_json = dumps({"action": "process", "from": "ext", "Tx": T1}) T2_json = dumps({"action": "process", "from": "ext", "Tx": T2}) nodes[0].r.publish('votes:%s' % i, T1_json) nodes[0].r.publish('votes:%s' % i, T2_json) # Relevent Nodes r1nodes = [n for n in nodes if n._within_TX(T1)] r2nodes = [n for n in nodes if n._within_TX(T2)] def test_condition(): Good = True Good &= T1[0] in r1nodes[-1].commit_yes Good &= T1[0] in r1nodes[0].commit_yes Good &= T2[0] in r2nodes[-1].commit_no Good &= T2[0] in r2nodes[0].commit_no assert Good print "All: %s" % Good t = xTimer(3.0, test_condition) t.start()
def test_single(): kn = KafkaNode(name="node", host=host, port=port) rnd = hexlify(urandom(16)) T0 = packageTx(data="data1,%s" % rnd, deps=[], num_out=3) _, _, [A, B, C], txdata = T0 T0_json = dumps({ "action":"process", "from":"ext", "Tx":Tx2json(T0) }) kn.send(T0, T0_json) T1 = packageTx("333,%s" % rnd, [A, B], 2) T2 = packageTx("bbb,%s" % rnd, [A, C], 2) T1_json = dumps({ "action":"process", "from":"ext", "Tx":Tx2json(T1) }) T2_json = dumps({ "action":"process", "from":"ext", "Tx":Tx2json(T2) }) kn.send(T1, T1_json) kn.send(T2, T2_json) try: while True: time.sleep(1.0) except KeyboardInterrupt: del kn print "Clean exit ..."
def test_distribution(): shard_map = make_shard_map(100) from collections import defaultdict d = defaultdict(int) for x in range(1000): T1 = packageTx("333%s" % x, [], 2) for i in shard_map: b0, b1 = shard_map[i] if within_TX(T1, b0, b1): d[i] += 1 for i in sorted(d): print "%3d | %s" % (i, "=" * d[i])
host=host, port=port) for i in shard_map ] def concerned(tx): l = [] for i in shard_map: (b0, b1) = shard_map[i] if within_TX(tx, b0, b1): l.append(i) return l T0s = [] for _ in range(100): rnd = hexlify(urandom(16)) T0 = packageTx(data="data1-%s" % rnd, deps=[], num_out=3) T0s.append(T0) # _, _, [A, B, C], txdata = T0 # T1 = packageTx("333-%s" % rnd, [A, B], 2) # T2 = packageTx("bbb-%s" % rnd, [A, C], 2) P = lambda T: dumps({ "action": "process", "from": "ext", "Tx": Tx2json(T) }) print "T0 depends on: %s" % str(concerned(T0)) nodes[0].receive(P(T0))
nodes = [ KafkaNode(quorum=1, name="n%s" % i, shard=i, shard_map=shard_map, host=host, port=port) for i in shard_map ] def concerned(tx): l = [] for i in shard_map: (b0, b1) = shard_map[i] if within_TX(tx, b0, b1): l.append(i) return l T0s = [] for _ in range(100): rnd = hexlify(urandom(16)) T0 = packageTx(data="data1-%s" % rnd, deps=[], num_out=3) T0s.append(T0) # _, _, [A, B, C], txdata = T0 # T1 = packageTx("333-%s" % rnd, [A, B], 2) # T2 = packageTx("bbb-%s" % rnd, [A, C], 2) P = lambda T: dumps({ "action":"process", "from":"ext", "Tx":Tx2json(T) }) print "T0 depends on: %s" % str(concerned(T0)) nodes[0].receive( P(T0) ) # Relevent Nodes # r1nodes = [n for n in nodes if n._within_TX(T1)] # r1nodes[0].process(T1)