def make_block(self,
shard_ID,
mempools,
drain_amount,
genesis_blocks,
TTL=TTL_CONSTANT):
genesis_blocks = self.genesis_blocks()
# RUN FORK CHOICE RULE ON SELF
# will only have fork choices for parent and children
my_fork_choice = self.make_fork_choice(shard_ID)
# --------------------------------------------------------------------#
# GET PREVBLOCK POINTER FROM FORK CHOICE
prevblock = my_fork_choice
# --------------------------------------------------------------------#
# EXTEND THE TRANSACTION LOG FROM THE MEMPOOL
prev_txn_log = prevblock.txn_log
new_txn_log = copy.copy(prev_txn_log)
data = []
num_prev_txs = len(prev_txn_log)
neighbor_shard_IDs = []
if my_fork_choice.parent_ID is not None:
neighbor_shard_IDs.append(my_fork_choice.parent_ID)
for IDs in my_fork_choice.child_IDs:
neighbor_shard_IDs.append(IDs)
# BUILD SOURCES
sources = {ID: genesis_blocks[ID] for ID in SHARD_IDS}
for ID in neighbor_shard_IDs:
if ID == shard_ID:
continue
#if len(prevblock.received_log.log[ID]):
# assert prevblock.received_log.log[ID][-1].base.shard_ID == ID
# sources[ID] = prevblock.received_log.log[ID][-1].base
neighbor_fork_choice = self.make_fork_choice(ID)
# SOURCES = FORK CHOICE (except for self)
sources[ID] = neighbor_fork_choice
assert sources[ID].is_in_chain(
prevblock.sources[ID]
), "Sources inconsistent, new block shard id: %s, source from shard: %s, heights: %s over %s, new_source_parent_ID: %s, old_source_parent_ID: %s, first_children: %s, three_parent: %s" % (
shard_ID, ID, sources[ID].height, prevblock.sources[ID].height,
sources[ID].parent_ID, prevblock.sources[ID].parent_ID,
self.make_fork_choice(1).child_IDs,
self.make_fork_choice(3).parent_ID)
#for ID in SHARD_IDS:
# if ID not in neighbor_shard_IDs and ID != shard_ID:
# sources[ID] = prevblock.sources[ID]
# --------------------------------------------------------------------#
if num_prev_txs < len(mempools[shard_ID]) and 'opcode' in mempools[
shard_ID][num_prev_txs]:
child_to_become_parent = mempools[shard_ID][num_prev_txs][
'child_to_become_parent']
child_to_move_down = mempools[shard_ID][num_prev_txs][
'child_to_move_down']
# TODO: for swapping with root only one message will be needed
sources = copy.copy(prevblock.sources)
msg1 = SwitchMessage_BecomeAParent(sources[child_to_become_parent],
1, child_to_become_parent,
child_to_move_down,
sources[child_to_move_down])
msg2 = SwitchMessage_ChangeParent(sources[child_to_move_down], 1,
child_to_move_down,
child_to_become_parent,
sources[child_to_become_parent])
mempools[shard_ID] = mempools[shard_ID][:num_prev_txs] + mempools[
shard_ID][num_prev_txs + 1:]
sent_log = copy.copy(prevblock.sent_log)
received_log = copy.copy(prevblock.received_log)
sent_log.add_message(msg1.target_shard_ID, msg1)
sent_log.add_message(msg2.target_shard_ID, msg2)
ret = Block(shard_ID, prevblock, new_txn_log, sent_log,
received_log, sources, prevblock.vm_state)
ret.child_IDs.remove(child_to_move_down)
ret.routing_table[child_to_move_down] = child_to_become_parent
print("IMPORTANT: %s" % ret.routing_table)
return ret
for i in range(drain_amount):
if num_prev_txs + i < len(mempools[shard_ID]):
new_tx = mempools[shard_ID][num_prev_txs + i]
if 'opcode' in new_tx:
# this is a switch message, stop processing messages
break
new_txn_log.append(new_tx)
data.append(new_tx)
# --------------------------------------------------------------------#
# BUILD RECEIVED LOG WITH:
received_log = MessagesLog()
for ID in SHARD_IDS:
if ID == shard_ID:
continue
if ID in neighbor_shard_IDs:
neighbor_fork_choice = self.make_fork_choice(ID)
# RECEIVED = SENT MESSAGES FROM FORK CHOICE
received_log.log[ID] = copy.copy(
neighbor_fork_choice.sent_log.log[shard_ID])
else:
received_log.log[ID] = copy.copy(
prevblock.received_log.log[ID])
# --------------------------------------------------------------------#
# PREP NEWLY RECEIVED PMESSAGES IN A RECEIVEDLOG FOR EVM:
newly_received_messages = {}
new_sent_messages = MessagesLog()
for ID in SHARD_IDS:
previous_received_log_size = len(prevblock.received_log.log[ID])
newly_received_messages[ID] = received_log.log[ID][
previous_received_log_size:]
become_a_parent_of = None
change_parent_to = None
newly_received_payloads = MessagesLog()
for ID in SHARD_IDS:
for m in newly_received_messages[ID]:
if m.target_shard_ID == shard_ID:
if isinstance(m, SwitchMessage_BecomeAParent):
become_a_parent_of = (m.new_child_ID,
m.new_child_source)
elif isinstance(m, SwitchMessage_ChangeParent):
change_parent_to = (m.new_parent_ID,
m.new_parent_source)
routing_table = copy.copy(prevblock.routing_table)
new_parent_ID = prevblock.parent_ID
if become_a_parent_of is not None:
routing_table[become_a_parent_of[0]] = become_a_parent_of[0]
neighbor_fork_choice = self.make_fork_choice(become_a_parent_of[0])
sources[become_a_parent_of[0]] = neighbor_fork_choice
ID = become_a_parent_of[0]
received_log.log[ID] = copy.copy(
neighbor_fork_choice.sent_log.log[shard_ID])
previous_received_log_size = len(prevblock.received_log.log[ID])
newly_received_messages[ID] = received_log.log[ID][
previous_received_log_size:]
if change_parent_to is not None:
new_parent_ID = change_parent_to[0]
neighbor_fork_choice = self.make_fork_choice(change_parent_to[0])
sources[change_parent_to[0]] = neighbor_fork_choice
ID = change_parent_to[0]
received_log.log[ID] = copy.copy(
neighbor_fork_choice.sent_log.log[shard_ID])
previous_received_log_size = len(prevblock.received_log.log[ID])
newly_received_messages[ID] = received_log.log[ID][
previous_received_log_size:]
for ID in SHARD_IDS:
for m in newly_received_messages[ID]:
if m.target_shard_ID == shard_ID:
if isinstance(m, SwitchMessage_BecomeAParent):
become_a_parent_of = (m.new_child_ID,
m.new_child_source)
elif isinstance(m, SwitchMessage_ChangeParent):
change_parent_to = (m.new_parent_ID,
m.new_parent_source)
else:
newly_received_payloads.add_message(ID, m)
else:
next_hop_ID = self.next_hop(routing_table,
m.target_shard_ID)
if next_hop_ID is not None:
assert next_hop_ID in prevblock.child_IDs, "shard_ID: %s, destination: %s, next_hop: %s, children: %s" % (
shard_ID, ID, next_hop_ID, prevblock.child_IDs)
else:
next_hop_ID = new_parent_ID
assert next_hop_ID is not None
new_sent_messages.log[next_hop_ID].append(
Message(sources[next_hop_ID], m.TTL, m.target_shard_ID,
m.payload))
# --------------------------------------------------------------------#
# KEY EVM INTEGRATION HERE
# this is where we have this function that produces the new vm state and the new outgoing payloads
# new_vm_state, new_outgoing_payloads = apply_to_state(prevblock.vm_state, data, newly_received_payloads)
new_vm_state, new_outgoing_payloads = apply_to_state(
prevblock.vm_state, data, newly_received_payloads, genesis_blocks)
# --------------------------------------------------------------------#
# BUILD SENT LOG FROM NEW OUTGOING PAYLOADS
# by this time new_sent_messages might already have some messages from rerouting above
for ID in SHARD_IDS:
if ID != shard_ID:
for m in new_outgoing_payloads.log[ID]:
# if TTL == 0, then we'll make an invalid block
# one that sends a message that must be included by the base
# which already exists and therefore cannot include this message
if TTL > 0:
first_hop_ID = self.next_hop(routing_table, ID)
if first_hop_ID is not None:
assert first_hop_ID in prevblock.child_IDs, "shard_ID: %s, target: %s, first_hop_ID: %s, parent: %s, children: %s, rtable: %s" % (
shard_ID, ID, first_hop_ID,
prevblock.parent_ID, prevblock.child_IDs,
prevblock.routing_table)
else:
first_hop_ID = new_parent_ID
assert first_hop_ID is not None
new_sent_messages.log[first_hop_ID].append(
Message(sources[first_hop_ID], TTL, ID, m.payload))
else:
print("Warning: Not sending message because TTL == 0")
sent_log = prevblock.sent_log.append_MessagesLog(new_sent_messages)
# --------------------------------------------------------------------#
ret = Block(shard_ID,
prevblock,
new_txn_log,
sent_log,
received_log,
sources,
new_vm_state,
postpone_validation=True)
if become_a_parent_of is not None:
assert become_a_parent_of[
0] not in ret.child_IDs, "shard_ID: %s, become_of: %s, child_IDs: %s" % (
shard_ID, become_a_parent_of[0], ret.child_IDs)
ret.child_IDs.append(become_a_parent_of[0])
ret.routing_table[become_a_parent_of[0]] = become_a_parent_of[0]
if change_parent_to is not None:
assert change_parent_to[0] not in ret.child_IDs
assert change_parent_to[0] != ret.parent_ID
ret.parent_ID = change_parent_to[0]
check = ret.is_valid()
assert check[0], check[1]
return ret
def make_block(self, shard_ID, mempools, drain_amount, TTL=TTL_CONSTANT):
# RUN FORK CHOICE RULE
fork_choice = self.fork_choice()
# --------------------------------------------------------------------#
# GET PREVBLOCK POINTER FROM FORK CHOICE
prevblock = fork_choice[shard_ID]
# --------------------------------------------------------------------#
# EXTEND THE TRANSACTION LOG FROM THE MEMPOOL
prev_txn_log = prevblock.txn_log
new_txn_log = copy.copy(prev_txn_log)
data = []
num_prev_txs = len(prev_txn_log)
for i in range(drain_amount):
if num_prev_txs + i < len(mempools[shard_ID]):
new_tx = mempools[shard_ID][num_prev_txs + i]
new_txn_log.append(new_tx)
data.append(new_tx)
# --------------------------------------------------------------------#
# BUILD RECEIVED LOG WITH:
received_log = ReceivedLog()
for ID in SHARD_IDS:
if ID == shard_ID:
continue
# SOURCES = FORK CHOICE (except for self)
received_log.sources[ID] = fork_choice[ID]
# RECEIVED = SENT MESSAGES FROM FORK CHOICE
received_log.log[ID] = fork_choice[ID].sent_log.log[shard_ID]
# --------------------------------------------------------------------#
# PREP NEWLY RECEIVED PMESSAGES IN A RECEIVEDLOG FOR EVM:
newly_received_messages = {}
for ID in SHARD_IDS:
previous_received_log_size = len(prevblock.received_log.log[ID])
current_received_log_size = len(received_log.log[ID])
newly_received_messages[ID] = received_log.log[ID][
previous_received_log_size:]
newly_received_payloads = ReceivedLog()
for ID in SHARD_IDS:
for m in newly_received_messages[ID]:
newly_received_payloads.add_received_message(ID, m)
# --------------------------------------------------------------------#
# KEY EVM INTEGRATION HERE
# this is where we have this function that produces the new vm state and the new outgoing payloads
# new_vm_state, new_outgoing_payloads = apply_to_state(prevblock.vm_state, data, newly_received_payloads)
new_vm_state, new_outgoing_payloads = apply_to_state(
prevblock.vm_state, data, newly_received_payloads)
if shard_ID == 1:
new_outgoing_payloads.log[0], new_outgoing_payloads.log[
1] = new_outgoing_payloads.log[1], new_outgoing_payloads.log[0]
# --------------------------------------------------------------------#
# BUILD SENT LOG FROM NEW OUTGOING PAYLOADS
new_sent_messages = SentLog()
for ID in SHARD_IDS:
if ID != shard_ID:
for m in new_outgoing_payloads.log[ID]:
# if TTL == 0, then we'll make an invalid block
# one that sends a message that must be included by the base
# which already exists and therefore cannot include this message
if TTL > 0:
new_sent_messages.log[ID].append(
Message(fork_choice[ID], TTL, m.payload))
else:
print("Warning: Not sending message because TTL == 0")
sent_log = prevblock.sent_log.append_SentLog(new_sent_messages)
# --------------------------------------------------------------------#
return Block(shard_ID, prevblock, new_txn_log, sent_log, received_log,
new_vm_state)
def make_block(self, shard_ID, mempools, drain_amount, TTL=TTL_CONSTANT):
# first we execute the fork choice rule
fork_choice = self.fork_choice()
prevblock = fork_choice[shard_ID]
prev_txn_log = prevblock.txn_log
new_txn_log = copy.copy(prev_txn_log)
data = []
num_prev_txs = len(prev_txn_log)
print("num_prev_txs", num_prev_txs)
print("mempools[shard_ID] == prev_txn_log",
mempools[shard_ID] == prev_txn_log)
for i in range(drain_amount):
if num_prev_txs + i < len(mempools[shard_ID]):
new_tx = mempools[shard_ID][num_prev_txs + i]
new_txn_log.append(new_tx)
data.append(new_tx)
# then put together the new received log
received_log = ReceivedLog()
for ID in SHARD_IDS:
if ID == shard_ID:
continue
# we're just going to receive every send that we see from the fork choice (which filtered blocks who don't recieve things before their TTLs)
received_log.sources[ID] = fork_choice[ID]
received_log.log[ID] = fork_choice[ID].sent_log.log[shard_ID]
# which has the following newly received messages:
newly_received_messages = {}
for ID in SHARD_IDS:
previous_received_log_size = len(prevblock.received_log.log[ID])
current_received_log_size = len(received_log.log[ID])
assert current_received_log_size >= previous_received_log_size, "did not expect log size to shrink"
if current_received_log_size > previous_received_log_size:
print("RECEIVED LOG IS GROWING!!")
newly_received_messages[ID] = received_log.log[ID][
previous_received_log_size:]
# which have the following newly received payloads:
newly_received_payloads = ReceivedLog()
for ID in SHARD_IDS:
for m in newly_received_messages[ID]:
newly_received_payloads.add_received_message(ID, m)
'''
KEY INTEGRATION HERE
'''
# this is where we have this function that produces the new vm state and the new outgoing payloads
# new_vm_state, new_outgoing_payloads = INTEGRATE_HERE(prevblock.vm_state, data, newly_received_payloads)
old_state = copy.copy(prevblock.vm_state)
# TEST
newly_received_payloads = ReceivedLog()
new_vm_state, new_outgoing_payloads = apply_to_state(
prevblock.vm_state, data, newly_received_payloads)
print("new_outgoing_payloads", new_outgoing_payloads)
#if data != []:
# print("data", data)
# print("--------------------------------------------------------------------")
for ID in SHARD_IDS:
if new_outgoing_payloads.log[ID] != []:
print("NEW OUTGOING PAYLOADS[", ID, "]",
new_outgoing_payloads.log[ID])
# print("--------------------------------------------------------------------")
# print("new_vm_state", new_vm_state)
# print("old_state == new_vm_state", old_state == new_vm_state)
if old_state != new_vm_state:
print("data:", data)
# print("\n\n\n\n")
# we now package the sent_log with new messages that deliver these payloads
new_sent_messages = SentLog()
for ID in SHARD_IDS:
if ID != shard_ID:
for m in new_outgoing_payloads.log[ID]:
print("NEW OUTGOING PAYLOAD", m)
new_sent_messages.log[ID].append(
Message(fork_choice[ID], TTL, m.message_payload))
sent_log = prevblock.sent_log.append_SentLog(new_sent_messages)
return Block(shard_ID, prevblock, new_txn_log, sent_log, received_log,
new_vm_state)
def make_block(self,
shard_ID,
mempools,
drain_amount,
genesis_blocks,
TTL=TTL_CONSTANT):
genesis_blocks = self.genesis_blocks()
# RUN FORK CHOICE RULE
# will only have fork choices for parent and children
fork_choice = self.fork_choice(shard_ID)
# --------------------------------------------------------------------#
# GET PREVBLOCK POINTER FROM FORK CHOICE
prevblock = fork_choice[shard_ID]
# --------------------------------------------------------------------#
# EXTEND THE TRANSACTION LOG FROM THE MEMPOOL
prev_txn_log = prevblock.txn_log
new_txn_log = copy.copy(prev_txn_log)
data = []
num_prev_txs = len(prev_txn_log)
for i in range(drain_amount):
if num_prev_txs + i < len(mempools[shard_ID]):
new_tx = mempools[shard_ID][num_prev_txs + i]
new_txn_log.append(new_tx)
data.append(new_tx)
# --------------------------------------------------------------------#
# BUILD RECEIVED LOG WITH:
received_log = MessagesLog()
sources = {ID: genesis_blocks[ID] for ID in SHARD_IDS}
for ID in fork_choice.keys():
if ID == shard_ID:
continue
# SOURCES = FORK CHOICE (except for self)
sources[ID] = fork_choice[ID]
# RECEIVED = SENT MESSAGES FROM FORK CHOICE
received_log.log[ID] = fork_choice[ID].sent_log.log[shard_ID]
# --------------------------------------------------------------------#
# PREP NEWLY RECEIVED PMESSAGES IN A RECEIVEDLOG FOR EVM:
newly_received_messages = {}
new_sent_messages = MessagesLog()
for ID in fork_choice.keys():
previous_received_log_size = len(prevblock.received_log.log[ID])
current_received_log_size = len(received_log.log[ID])
newly_received_messages[ID] = received_log.log[ID][
previous_received_log_size:]
newly_received_payloads = MessagesLog()
for ID in fork_choice.keys():
for m in newly_received_messages[ID]:
if m.target_shard_ID == shard_ID:
newly_received_payloads.add_message(ID, m)
else:
next_hop_ID = self.next_hop(prevblock, m.target_shard_ID)
if next_hop_ID is not None:
assert next_hop_ID in prevblock.child_IDs
else:
next_hop_ID = prevblock.parent_ID
new_sent_messages.log[next_hop_ID].append(
Message(fork_choice[next_hop_ID], m.TTL,
m.target_shard_ID, m.payload))
# --------------------------------------------------------------------#
# KEY EVM INTEGRATION HERE
# this is where we have this function that produces the new vm state and the new outgoing payloads
# new_vm_state, new_outgoing_payloads = apply_to_state(prevblock.vm_state, data, newly_received_payloads)
new_vm_state, new_outgoing_payloads = apply_to_state(
prevblock.vm_state, data, newly_received_payloads, genesis_blocks)
# --------------------------------------------------------------------#
# BUILD SENT LOG FROM NEW OUTGOING PAYLOADS
# by this time new_sent_messages might already have some messages from rerouting above
for ID in SHARD_IDS:
if ID != shard_ID:
for m in new_outgoing_payloads.log[ID]:
# if TTL == 0, then we'll make an invalid block
# one that sends a message that must be included by the base
# which already exists and therefore cannot include this message
if TTL > 0:
first_hop_ID = self.next_hop(prevblock, ID)
if first_hop_ID is not None:
assert first_hop_ID in [prevblock.parent_ID
] + prevblock.child_IDs
else:
first_hop_ID = prevblock.parent_ID
new_sent_messages.log[first_hop_ID].append(
Message(fork_choice[first_hop_ID], TTL, ID,
m.payload))
else:
print("Warning: Not sending message because TTL == 0")
sent_log = prevblock.sent_log.append_MessagesLog(new_sent_messages)
# --------------------------------------------------------------------#
return Block(shard_ID, prevblock, new_txn_log, sent_log, received_log,
sources, new_vm_state)
def make_block(self,
shard_ID,
mempools,
drain_amount,
genesis_blocks,
TTL=TTL_CONSTANT):
global BLOCKS
# First, the previous block pointer:
prevblock = self.make_fork_choice(shard_ID, genesis_blocks)
assert prevblock.shard_ID == shard_ID, "expected consistent IDs"
new_received_log = {}
new_sources = {}
new_sent_log = {}
new_routing_table = copy.deepcopy(prevblock.routing_table)
new_parent_ID = copy.copy(prevblock.parent_ID)
new_child_IDs = copy.copy(prevblock.child_IDs)
# --------------------------------------------------------------------#
# This part determines whether our block is a switch block:
# --------------------------------------------------------------------#
# Assume not, and look for switch messages as the next pending messages in tx and message queues:
switch_block = False
switch_tx = None
switch_message = None
# look in the mempool
num_prev_txs = len(prevblock.txn_log)
# look at sent messages of prevblock's neighbors
neighbor_shard_IDs = prevblock.get_neighbors()
print("prevblock.get_neighbors()")
print(prevblock.get_neighbors())
print("prevblock.parent_ID")
print(prevblock.parent_ID)
print("prevblock.child_IDs")
print(prevblock.child_IDs)
temp_new_sources = {}
for ID in SHARD_IDS:
if ID not in neighbor_shard_IDs:
assert ID not in temp_new_sources.keys()
temp_new_sources[ID] = copy.copy(prevblock.sources[ID])
for ID in neighbor_shard_IDs:
assert ID not in temp_new_sources.keys()
temp_new_sources[ID] = copy.copy(
self.make_fork_choice(ID, genesis_blocks))
print("ID in new_child_IDs")
print(ID in new_child_IDs)
print("ID == new_parent_IDs")
print(ID == new_parent_ID)
assert temp_new_sources[ID].is_in_chain(
prevblock.sources[ID]
), "expected monotonic sources - error 0, shard_ID: %s, ID: %s" % (
shard_ID, ID)
last_receive_log_length = len(prevblock.received_log[ID])
if len(temp_new_sources[ID].sent_log[shard_ID]
) > last_receive_log_length:
for next_message in temp_new_sources[ID].sent_log[shard_ID][
last_receive_log_length:]:
if isinstance(
next_message,
SwitchMessage_BecomeAParent) or isinstance(
next_message,
SwitchMessage_ChangeParent) or isinstance(
next_message, SwitchMessage_Orbit):
assert not switch_block
switch_source_ID = ID
switch_source = temp_new_sources[ID]
switch_message = next_message
switch_block = True
# if we received a switch_message, do not initiate switch ourselves
while switch_message and num_prev_txs < len(
mempools[shard_ID]
) and 'opcode' in mempools[shard_ID][num_prev_txs]:
num_prev_txs += 1
# skip orbits if the order of shards is wrong
while num_prev_txs < len(mempools[shard_ID]) and 'opcode' in mempools[
shard_ID][num_prev_txs] and mempools[shard_ID][num_prev_txs][
'opcode'] == 'orbit' and mempools[shard_ID][num_prev_txs][
'child_to_become_parent'] == prevblock.parent_ID:
num_prev_txs += 1
if num_prev_txs < len(mempools[shard_ID]):
if 'opcode' in mempools[shard_ID][num_prev_txs]:
switch_tx = mempools[shard_ID][num_prev_txs]
print(
"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
)
print(
"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!",
shard_ID)
print(
"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!",
prevblock.height)
switch_block = True
if switch_block:
assert switch_message is not None or switch_tx is not None
assert switch_message is None or switch_tx is None
if prevblock.switch_block:
print("DEBUG: switch block follows another switch block")
# --------------------------------------------------------------------#
# If our block is a switch block, then we won't process anything
# against the EVM, nor receiving or sending messages that are not switch messages
# --------------------------------------------------------------------#
# We will first process switch blocks:
# BUILD SOURCES FOR PREVBLOCK NEIGHBORS
neighbor_shard_IDs = prevblock.get_neighbors()
new_sources = {}
for ID in SHARD_IDS:
if ID not in neighbor_shard_IDs:
assert ID not in new_sources.keys()
new_sources[ID] = (prevblock.sources[ID])
for ID in neighbor_shard_IDs:
if ID == shard_ID:
continue
assert ID not in new_sources.keys()
new_sources[ID] = self.make_fork_choice(ID, genesis_blocks)
assert new_sources[ID].shard_ID == ID, "expected consistent IDs"
print(str(new_sources[ID]))
print(str(prevblock.sources[ID]))
print("ID in new_child_IDs")
print(ID in new_child_IDs)
print("ID in new_parent_ID")
print(ID == new_parent_ID)
# fork choice should be orphaning any blocks that disagree with the prevblock's sources
# the prevblock is the fork choice for this shard
# which means that it is not filtered on this shard, meaning that it agrees with the fork choice of the parent
# in the child, blocks that disagree with the fork choice are orphaned
if ID == prevblock.parent_ID:
assert new_sources[ID].is_in_chain(
prevblock.sources[ID]
), "expected monotonic consistent sources - error 1.1"
elif ID in prevblock.child_IDs:
assert new_sources[ID].is_in_chain(
prevblock.sources[ID]
), "expected monotonic consistent sources - error 1.2"
else:
assert False, "expected neighbor ID to be either parent or child ID"
# check that fork choices have consistent sources
# try to make sure that we don't make a block with a source that isn't in fork_choice's
assert prevblock.shard_ID == shard_ID
print("ID in new_child_IDs")
print(ID in new_child_IDs)
print("ID in new_parent_ID")
print(ID == new_parent_ID)
try:
assert prevblock.is_in_chain(
new_sources[ID].sources[shard_ID]
), "expected - error 1, shard_ID: %s, ID: %s, parent_ID: %s, prev_parent_ID: %s, switch_block: %s" % (
shard_ID, ID, new_parent_ID, prevblock.parent_ID,
switch_block)
except:
prevblock.trace_history(ID)
new_sources[ID].trace_history(shard_ID)
raise
if switch_block:
# HACK: setting a source to something from which we didn't receive messages breaks
# thigs during rotation.
for ID in SHARD_IDS:
while len(new_sources[ID].sent_log[shard_ID]) > len(
prevblock.received_log[ID]):
assert new_sources[ID] != prevblock.sources[ID]
new_sources[ID] = new_sources[ID].prevblock
if switch_tx is not None:
if switch_tx['opcode'] == 'orbit':
new_txn_log = prevblock.txn_log + [switch_tx]
child_to_become_parent = mempools[shard_ID][num_prev_txs][
'child_to_become_parent']
shard_to_move_down = mempools[shard_ID][num_prev_txs][
'shard_to_move_down']
root_fork_choice = prevblock
child_source = self.make_fork_choice(
child_to_become_parent, genesis_blocks)
new_sources[child_to_become_parent] = child_source
new_sources[shard_to_move_down] = prevblock
assert shard_to_move_down == prevblock.shard_ID
msg1 = SwitchMessage_Orbit(child_source,
TTL_SWITCH_CONSTANT,
child_to_become_parent,
shard_to_move_down, None)
new_sent_log[child_to_become_parent] = prevblock.sent_log[
child_to_become_parent] + [msg1]
for ID in SHARD_IDS:
if ID != child_to_become_parent:
new_sent_log[ID] = prevblock.sent_log[ID]
new_received_log = prevblock.received_log
print(
"My shardID: %s, my children: %s, my old parent: %s, child_to_become_parent: %s"
% (shard_ID, new_child_IDs, prevblock.parent_ID,
child_to_become_parent))
new_child_IDs.remove(child_to_become_parent)
new_parent_ID = child_to_become_parent
for k, v in child_source.routing_table.items():
assert k in new_routing_table
del new_routing_table[k]
else:
assert switch_tx['opcode'] == 'switch'
new_txn_log = prevblock.txn_log + [switch_tx]
child_to_become_parent = mempools[shard_ID][num_prev_txs][
'child_to_become_parent']
child_to_move_down = mempools[shard_ID][num_prev_txs][
'child_to_move_down']
# this could be a more conservative choice, using fork choice is a bit high risk bc we might have more switch blocks in here
fork_choice_of_child_to_become_parent = self.make_fork_choice(
child_to_become_parent,
genesis_blocks) # new_sources[child_to_become_parent]
fork_choice_of_child_to_move_down = self.make_fork_choice(
child_to_move_down,
genesis_blocks) # new_sources[child_to_move_down]
msg1 = SwitchMessage_BecomeAParent(
fork_choice_of_child_to_become_parent,
TTL_SWITCH_CONSTANT, child_to_become_parent,
child_to_move_down)
msg2 = SwitchMessage_ChangeParent(
fork_choice_of_child_to_move_down, TTL_SWITCH_CONSTANT,
child_to_move_down, child_to_become_parent)
# they have the switch messages in the sent message queues
new_sent_log[child_to_become_parent] = prevblock.sent_log[
child_to_become_parent] + [msg1]
new_sent_log[child_to_move_down] = prevblock.sent_log[
child_to_move_down] + [msg2]
for ID in SHARD_IDS:
if ID != child_to_move_down and ID != child_to_become_parent:
new_sent_log[ID] = prevblock.sent_log[ID]
new_received_log = prevblock.received_log
# removing child from the switch block
new_child_IDs.remove(child_to_move_down)
# now the routing table
for ID in new_routing_table.keys():
if new_routing_table[ID] == child_to_move_down:
new_routing_table[ID] = child_to_become_parent
# may be redundant, but won't hurt anyone:
new_routing_table[
child_to_move_down] = child_to_become_parent
# parent_ID unchanged
# received_log unchanged
# sources unchanged
elif switch_message is not None:
#new_received_log[switch_source_ID] = prevblock.received_log[switch_source_ID] + [switch_message]
new_sources[
switch_source_ID] = switch_source.first_block_with_message_in_sent_log(
shard_ID, switch_message)
# COPYPASTE=====
newly_received_messages = {}
for ID in SHARD_IDS:
newly_received_messages[ID] = []
for ID in SHARD_IDS:
if ID == shard_ID:
continue
prev_received_log_length = len(prevblock.received_log[ID])
while (len(newly_received_messages[ID]) <
len(new_sources[ID].sent_log[shard_ID]) -
prev_received_log_length):
log_length = len(newly_received_messages[ID])
new_message = new_sources[ID].sent_log[shard_ID][
log_length + prev_received_log_length]
newly_received_messages[ID].append(new_message)
new_received_log = {}
for ID in SHARD_IDS:
new_received_log[ID] = prevblock.received_log[
ID] + newly_received_messages[ID]
new_sent_messages = {
} # for now we're going to fill this with routed messages
for ID in SHARD_IDS:
new_sent_messages[ID] = []
for ID in neighbor_shard_IDs:
for m in newly_received_messages[ID]:
if m.target_shard_ID == shard_ID:
pass # TODO: we are not processing payloads here, is it important?
else:
next_hop_ID = self.next_hop(
new_routing_table, m.target_shard_ID)
if next_hop_ID is not None:
assert next_hop_ID in prevblock.child_IDs, "shard_ID: %s, destination: %s, next_hop: %s, children: %s" % (
shard_ID, ID, next_hop_ID,
prevblock.child_IDs)
else:
next_hop_ID = new_parent_ID
assert next_hop_ID is not None
new_sent_messages[next_hop_ID].append(
Message(new_sources[next_hop_ID], m.TTL,
m.target_shard_ID, m.payload))
new_sent_log = {}
for ID in SHARD_IDS:
new_sent_log[
ID] = prevblock.sent_log[ID] + new_sent_messages[ID]
# //COPYPASTE=====
print("switch_source", switch_source)
print("switch_message", switch_message)
print("switch_source_ID", switch_source_ID)
print("new_sources[switch_source_ID]",
new_sources[switch_source_ID])
print(
"switch message in new_sources[switch_source_ID].sent_log[shard_ID]",
switch_message
in new_sources[switch_source_ID].sent_log[shard_ID])
print(
"switch message in new_sources[switch_source_ID].prevblock.sent_log[shard_ID]",
switch_message in
new_sources[switch_source_ID].prevblock.sent_log[shard_ID])
print("new_sources[switch_source_ID].switch_block",
new_sources[switch_source_ID].switch_block)
# assert new_sources[switch_source_ID].switch_block
if isinstance(
switch_message,
(SwitchMessage_BecomeAParent, SwitchMessage_Orbit)):
if switch_message.new_child_ID not in new_child_IDs:
new_child_IDs.append(switch_message.new_child_ID)
for ID in new_sources[
switch_message.new_child_ID].routing_table.keys():
new_routing_table[ID] = switch_message.new_child_ID
if isinstance(
switch_message,
(SwitchMessage_ChangeParent, SwitchMessage_Orbit)):
new_parent_ID = switch_message.new_parent_ID
new_txn_log = prevblock.txn_log
else:
assert False
new_block = Block(shard_ID, prevblock, True, new_txn_log,
new_sent_log, new_received_log, new_sources,
new_parent_ID, new_child_IDs, new_routing_table,
prevblock.vm_state)
assert new_block.switch_block
print("new_block", new_block)
print("new_block.switch_block", new_block.switch_block)
check = new_block.is_valid()
if not check[0]:
print(
"---------------------------------------------------------"
)
print(
"---------------------------------------------------------"
)
print("shard_ID", prevblock.shard_ID)
print(
"---------------------------------------------------------"
)
print(
"---------------------------------------------------------"
)
print("txn_log", new_txn_log)
print(
"---------------------------------------------------------"
)
print(
"---------------------------------------------------------"
)
print("self.sent_log", new_sent_log)
print(
"---------------------------------------------------------"
)
print(
"---------------------------------------------------------"
)
print("self.received_log", new_received_log)
print(
"---------------------------------------------------------"
)
print(
"---------------------------------------------------------"
)
print("shard_ID", shard_ID)
print(
"---------------------------------------------------------"
)
print(
"---------------------------------------------------------"
)
print("txn_log", new_txn_log)
print(
"---------------------------------------------------------"
)
print(
"---------------------------------------------------------"
)
print("self.sent_log", new_sent_log)
print(
"---------------------------------------------------------"
)
print(
"---------------------------------------------------------"
)
print("self.received_log", new_received_log)
print(
"---------------------------------------------------------"
)
print(
"---------------------------------------------------------"
)
print("receiving_opcode: ", switch_block)
print(
"---------------------------------------------------------"
)
print(
"---------------------------------------------------------"
)
try:
assert check[0], "Invalid Block: " + check[1]
except:
print(check[1])
time.sleep(1000)
#sources_hashes = {}
#for ID in SHARD_IDS:
# sources_hashes[ID] = new_block.sources[ID].hash
#BLOCKS[new_block.hash] = sources_hashes
return new_block
# --------------------------------------------------------------------#
# --------------------------------------------------------------------#
# --------------------------------------------------------------------#
# --------------------------------------------------------------------#
# --------------------------------------------------------------------#
# And now for the rest of the blocks, the ones that don't change the routing table
# But which do routing and execution of state against the EVM
newly_received_txns = []
for i in range(drain_amount):
if num_prev_txs + i < len(mempools[shard_ID]):
new_tx = mempools[shard_ID][num_prev_txs + i]
if 'opcode' in new_tx:
# Don't add switch transaction to tx log
break
newly_received_txns.append(new_tx)
# Construct new txn log
new_txn_log = prevblock.txn_log + newly_received_txns
# print("NEW TXN LEN: ", len(new_txn_log))
# print("PRE NEW RECEIPTS DATA LEN: ", len(newly_received_txns))
receiving_opcode = False
# --------------------------------------------------------------------#
# BUILD RECEIVED LOG WITH:
newly_received_messages = {}
for ID in SHARD_IDS:
newly_received_messages[ID] = []
for ID in SHARD_IDS:
if ID == shard_ID:
continue
prev_received_log_length = len(prevblock.received_log[ID])
while (len(newly_received_messages[ID]) <
len(new_sources[ID].sent_log[shard_ID]) -
prev_received_log_length):
log_length = len(newly_received_messages[ID])
new_message = new_sources[ID].sent_log[shard_ID][
log_length + prev_received_log_length]
if isinstance(new_message,
SwitchMessage_BecomeAParent) or isinstance(
new_message,
SwitchMessage_ChangeParent) or isinstance(
new_message, SwitchMessage_Orbit):
break #but only receive messages up to the first switch opcod
newly_received_messages[ID].append(new_message)
new_received_log = {}
for ID in SHARD_IDS:
new_received_log[
ID] = prevblock.received_log[ID] + newly_received_messages[ID]
# --------------------------------------------------------------------#
# BUILD NEW SENT MESSAGES
new_sent_messages = {
} # for now we're going to fill this with routed messages
for ID in SHARD_IDS:
new_sent_messages[ID] = []
newly_received_payloads = {} # destined for this shard's evm
for ID in SHARD_IDS:
newly_received_payloads[ID] = []
# ROUTING
for ID in neighbor_shard_IDs:
for m in newly_received_messages[ID]:
if m.target_shard_ID == shard_ID:
if isinstance(m, SwitchMessage_BecomeAParent):
continue
elif isinstance(m, SwitchMessage_ChangeParent):
continue
elif isinstance(m, SwitchMessage_Orbit):
continue
else:
newly_received_payloads[ID].append(m)
else:
next_hop_ID = self.next_hop(new_routing_table,
m.target_shard_ID)
if next_hop_ID is not None:
assert next_hop_ID in prevblock.child_IDs, "shard_ID: %s, destination: %s, next_hop: %s, children: %s" % (
shard_ID, ID, next_hop_ID, prevblock.child_IDs)
else:
next_hop_ID = new_parent_ID
assert next_hop_ID is not None
new_sent_messages[next_hop_ID].append(
Message(new_sources[next_hop_ID], m.TTL,
m.target_shard_ID, m.payload))
# --------------------------------------------------------------------#
# EVM integration here
# this is where we have this function that produces the new vm state and the new outgoing payloads
# new_vm_state, new_outgoing_payloads = apply_to_state(prevblock.vm_state, newly_received_txns, newly_received_payloads)
# 'newly_received_txns' is the new txn list
new_vm_state, new_outgoing_payloads = apply_to_state(
prevblock.vm_state, newly_received_txns, newly_received_payloads,
genesis_blocks)
# --------------------------------------------------------------------#
# print("OUTGOING PAYLOAD LENGTH", len(new_outgoing_payloads.values()))
# BUILD SENT LOG FROM NEW OUTGOING PAYLOADS
# by this time new_sent_messages might already have some messages from rerouting above
for ID in SHARD_IDS:
if ID != shard_ID:
for m in new_outgoing_payloads[ID]:
# print("HERE !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
first_hop_ID = self.next_hop(new_routing_table, ID)
if first_hop_ID is not None:
assert first_hop_ID in prevblock.child_IDs, "shard_ID: %s, target: %s, first_hop_ID: %s, parent: %s, children: %s, rtable: %s" % (
shard_ID, ID, first_hop_ID, prevblock.parent_ID,
prevblock.child_IDs, prevblock.routing_table)
else:
first_hop_ID = new_parent_ID
assert first_hop_ID is not None
new_sent_messages[first_hop_ID].append(
Message(new_sources[first_hop_ID], TTL, ID, m.payload))
SUM = 0
for k in new_sent_messages.keys():
SUM += len(new_sent_messages[k])
# print("NUM NEW SENT: ", SUM)
new_sent_log = {}
for ID in SHARD_IDS:
new_sent_log[ID] = prevblock.sent_log[ID] + new_sent_messages[ID]
# MAKE BLOCK AND CHECK VALIDITY
# Block(ID, prevblock=None, txn_log=[], sent_log=None, received_log=None, sources=None, parent_ID=None, child_IDs=None, routing_table=None, vm_state=genesis_state):
print(("LOL", new_sources))
ret = Block(shard_ID, prevblock, False, new_txn_log, new_sent_log,
new_received_log, new_sources, new_parent_ID,
new_child_IDs, new_routing_table, new_vm_state)
assert not ret.switch_block
check = ret.is_valid()
if not check[0]:
print("---------------------------------------------------------")
print("---------------------------------------------------------")
print("shard_ID", prevblock.shard_ID)
print("---------------------------------------------------------")
print("---------------------------------------------------------")
print("txn_log", new_txn_log)
print("---------------------------------------------------------")
print("---------------------------------------------------------")
print("self.sent_log", new_sent_log)
print("---------------------------------------------------------")
print("---------------------------------------------------------")
print("self.received_log", newly_received_messages)
print("---------------------------------------------------------")
print("---------------------------------------------------------")
print("---------------------------------------------------------")
print("---------------------------------------------------------")
print("shard_ID", shard_ID)
print("---------------------------------------------------------")
print("---------------------------------------------------------")
print("txn_log", new_txn_log)
print("---------------------------------------------------------")
print("---------------------------------------------------------")
print("self.sent_log", new_sent_log)
print("---------------------------------------------------------")
print("---------------------------------------------------------")
print("self.received_log", newly_received_messages)
print("---------------------------------------------------------")
print("---------------------------------------------------------")
print("receiving_opcode: ", receiving_opcode)
print("---------------------------------------------------------")
print("---------------------------------------------------------")
assert check[0], "Invalid Block: " + check[1]
sources_hashes = {}
for ID in SHARD_IDS:
sources_hashes[ID] = ret.sources[ID].hash
BLOCKS[ret.hash] = sources_hashes
return ret