class test_SimpleContentChunkifyer(unittest.TestCase):
def setUp(self):
self.chunkifyer = SimpleContentChunkifyer()
def tearDown(self):
pass
def test_generate_metadata_no_next(self):
"""Test generating a simple metadata object"""
name = Name("/test/data")
res = self.chunkifyer.generate_meta_data(2, 4, 0, 0, name)
self.assertEqual(res.name.to_string(), "/test/data")
self.assertEqual(res.content, "mdo:/test/data/c2;/test/data/c3:")
def test_generate_metadata_one_next(self):
"""Test generating a simple metadata object with one following"""
name = Name("/test/data")
res = self.chunkifyer.generate_meta_data(2, 4, 0, 1, name)
self.assertEqual(res.name.to_string(), "/test/data")
self.assertEqual(res.content,
"mdo:/test/data/c2;/test/data/c3:/test/data/m1")
def test_generate_metadata_two_next(self):
"""Test generating a simple metadata object with two following"""
name = Name("/test/data")
res = self.chunkifyer.generate_meta_data(2, 4, 1, 2, name)
self.assertEqual(res.name.to_string(), "/test/data/m1")
self.assertEqual(res.content,
"mdo:/test/data/c2;/test/data/c3:/test/data/m2")
def test_chunk_single_metadata(self):
name = Name("/test/data")
string = "A" * 4096 + "B" * 4096 + "C" * 4096
content = Content(name, string)
md, content = self.chunkifyer.chunk_data(content)
md_name_comp = ['/test/data']
md_data_comp = ['mdo:/test/data/c0;/test/data/c1;/test/data/c2:']
content_name_comp = ['/test/data/c0', '/test/data/c1', '/test/data/c2']
content_data_comp = ["A" * 4096, "B" * 4096, "C" * 4096]
for i in range(0, len(md)):
self.assertEqual(md[i].name.to_string(), md_name_comp[i])
self.assertEqual(md[i].content, md_data_comp[i])
for i in range(0, len(content)):
self.assertEqual(content[i].name.to_string(), content_name_comp[i])
self.assertEqual(content[i].content, content_data_comp[i])
def test_chunk_multiple_metadata(self):
"""Test chunking metadata with three metadata objects and 10 chunks"""
name = Name("/test/data")
string = "A"*4096 + "B"*4096 + "C"*4096 + "D"*4096 + "E"*4096 + "F"*4096 + "G"*4096 + "H"*4096 \
+ "I"*4096 + "J"*4000
content = Content(name, string)
md, chunked_content = self.chunkifyer.chunk_data(content)
md_name_comp = ['/test/data', '/test/data/m1', '/test/data/m2']
md_data_comp = [
'mdo:/test/data/c0;/test/data/c1;/test/data/c2;/test/data/c3:/test/data/m1',
'mdo:/test/data/c4;/test/data/c5;/test/data/c6;/test/data/c7:/test/data/m2',
'mdo:/test/data/c8;/test/data/c9:'
]
content_name_comp = [
'/test/data/c0', '/test/data/c1', '/test/data/c2', '/test/data/c3',
'/test/data/c4', '/test/data/c5', '/test/data/c6', '/test/data/c7',
'/test/data/c8', '/test/data/c9'
]
content_data_comp = [
"A" * 4096, "B" * 4096, "C" * 4096, "D" * 4096, "E" * 4096,
"F" * 4096, "G" * 4096, "H" * 4096, "I" * 4096, "J" * 4000
]
for i in range(0, len(md)):
self.assertEqual(md[i].name.to_string(), md_name_comp[i])
self.assertEqual(md[i].content, md_data_comp[i])
for i in range(0, len(chunked_content)):
self.assertEqual(chunked_content[i].name.to_string(),
content_name_comp[i])
self.assertEqual(chunked_content[i].content, content_data_comp[i])
def test_chunk_multiple_metadata_reassemble(self):
"""Test chunking metadata with three metadata objects and 10 chunks and reassemble"""
name = Name("/test/data")
string = "A" * 4096 + "B" * 4096 + "C" * 4096 + "D" * 4096 + "E" * 4096 + "F" * 4096 + "G" * 4096 + "H" * 4096 \
+ "I" * 4096 + "J" * 4000
content = Content(name, string)
md, chunked_content = self.chunkifyer.chunk_data(content)
md_name_comp = ['/test/data', '/test/data/m1', '/test/data/m2']
md_data_comp = [
'mdo:/test/data/c0;/test/data/c1;/test/data/c2;/test/data/c3:/test/data/m1',
'mdo:/test/data/c4;/test/data/c5;/test/data/c6;/test/data/c7:/test/data/m2',
'mdo:/test/data/c8;/test/data/c9:'
]
content_name_comp = [
'/test/data/c0', '/test/data/c1', '/test/data/c2', '/test/data/c3',
'/test/data/c4', '/test/data/c5', '/test/data/c6', '/test/data/c7',
'/test/data/c8', '/test/data/c9'
]
content_data_comp = [
"A" * 4096, "B" * 4096, "C" * 4096, "D" * 4096, "E" * 4096,
"F" * 4096, "G" * 4096, "H" * 4096, "I" * 4096, "J" * 4000
]
for i in range(0, len(md)):
self.assertEqual(md[i].name.to_string(), md_name_comp[i])
self.assertEqual(md[i].content, md_data_comp[i])
for i in range(0, len(chunked_content)):
self.assertEqual(chunked_content[i].name.to_string(),
content_name_comp[i])
self.assertEqual(chunked_content[i].content, content_data_comp[i])
reassembled_content = self.chunkifyer.reassamble_data(
md[0].name, chunked_content)
self.assertEqual(content, reassembled_content)
def test_parse_metadata_next(self):
"""Test parse metadata with next metadata"""
md, names = self.chunkifyer.parse_meta_data(
"mdo:/test/data/c0;/test/data/c1;/test/data/c2;/test/data/c3:/test/data/m1"
)
self.assertEqual(Name("/test/data/m1"), md)
names_comp = [
Name("/test/data/c0"),
Name("/test/data/c1"),
Name("/test/data/c2"),
Name("/test/data/c3")
]
self.assertEqual(names, names_comp)
def test_parse_metadata(self):
"""Test parse metadata"""
md, names = self.chunkifyer.parse_meta_data(
"mdo:/test/data/c0;/test/data/c1;/test/data/c2;/test/data/c3:")
self.assertEqual(None, md)
names_comp = [
Name("/test/data/c0"),
Name("/test/data/c1"),
Name("/test/data/c2"),
Name("/test/data/c3")
]
self.assertEqual(names, names_comp)
class BasicChunkLayer(LayerProcess):
""""Basic Chunking Layer for PICN"""
def __init__(self,
chunkifyer: BaseChunkifyer = None,
chunk_size: int = 4096,
manager: multiprocessing.Manager = None,
log_level=255):
super().__init__("ChunkLayer", log_level=log_level)
self.chunk_size = chunk_size
if chunkifyer == None:
self.chunkifyer = SimpleContentChunkifyer(chunk_size)
else:
self.chunkifyer: BaseChunkifyer = chunkifyer
if manager is None:
manager = multiprocessing.Manager()
self._chunk_table: Dict[Name, (Content, float)] = manager.dict()
self._request_table: List[RequestTableEntry] = manager.list()
def data_from_higher(self, to_lower: multiprocessing.Queue,
to_higher: multiprocessing.Queue, data):
self.logger.info("Got Data from higher")
faceid = data[0]
packet = data[1]
if isinstance(packet, Interest):
self.logger.info("Packet is Interest " + str(packet.name))
requestentry = self.get_request_table_entry(packet.name)
if requestentry is None:
self._request_table.append(RequestTableEntry(packet.name))
to_lower.put([faceid, packet])
return
if isinstance(packet, Content):
self.logger.info("Packet is Content (name=%s, %d bytes)" % \
(str(packet.name), len(packet.content)))
if len(packet.content) < self.chunk_size:
to_lower.put([faceid, packet])
else:
self.logger.info("Chunking Packet")
metadata, chunks = self.chunkifyer.chunk_data(
packet) #create metadata and chunks
self.logger.info("Metadata: " + metadata[0].content)
to_lower.put(
[faceid, metadata[0]]
) #return first name TODO HANDLE THE CASE, WHERE CHUNKS CAN TIMEOUT AND MUST BE REPRODUCED
for md in metadata: #add metadata to chunktable
if md.name not in self._chunk_table:
self._chunk_table[md.name] = (md, time.time())
for c in chunks: #add chunks to chunktable
if c.name not in self._chunk_table:
self._chunk_table[c.name] = (c, time.time())
if isinstance(packet, Nack):
requestentry = self.get_request_table_entry(packet.name)
if requestentry is not None:
self._request_table.remove(requestentry)
to_lower.put([faceid, packet])
def data_from_lower(self, to_lower: multiprocessing.Queue,
to_higher: multiprocessing.Queue, data):
self.logger.info("Got Data from lower")
faceid = data[0]
packet = data[1]
if isinstance(packet, Interest):
self.logger.info("Packet is Interest")
if packet.name in self._chunk_table: #Check if Interest is in chunktable
matching_content = self._chunk_table.get(packet.name)[0]
to_lower.put([faceid, matching_content])
else:
to_higher.put([faceid, packet])
return
if isinstance(packet, Content):
self.logger.info("Packet is Content")
request_table_entry = self.get_request_table_entry(packet.name)
if request_table_entry is None:
return
self._request_table.remove(request_table_entry)
if request_table_entry.chunked is False: #not chunked content
if not packet.get_bytes().startswith(b'mdo:'):
to_higher.put([faceid, packet])
return
else: # Received metadata data --> chunked content
request_table_entry.chunked = True
if packet.get_bytes().startswith(
b'mdo:'): # request all frames from metadata
request_table_entry = self.handle_received_meta_data(
faceid, packet, request_table_entry, to_lower)
else:
request_table_entry = self.handle_received_chunk_data(
faceid, packet, request_table_entry, to_higher)
if request_table_entry is None:
return #deletes entry if data was completed
self._request_table.append(request_table_entry)
if isinstance(packet, Nack):
requestentry = self.get_request_table_entry(packet.name)
if requestentry is not None:
self._request_table.remove(requestentry)
to_higher.put([faceid, packet])
def handle_received_meta_data(
self, faceid: int, packet: Content,
request_table_entry: RequestTableEntry,
to_lower: multiprocessing.Queue) -> RequestTableEntry:
"""Handle the case, where metadata are received from the network"""
md_entry = self.metadata_name_in_request_table(packet.name)
if md_entry is None:
return request_table_entry
request_table_entry = self.remove_metadata_name_from_request_table(
request_table_entry, packet.name)
md, chunks, size = self.chunkifyer.parse_meta_data(packet.content)
if md is not None: # there is another md file
request_table_entry.requested_md.append(md)
to_lower.put([faceid, Interest(md)])
else:
request_table_entry.lastchunk = chunks[-1]
for chunk in chunks: # request all chunks from the metadata file
request_table_entry.requested_chunks.append(chunk)
to_lower.put([faceid, Interest(chunk)])
self._chunk_table[packet.name] = (packet, time.time())
return request_table_entry
def handle_received_chunk_data(
self, faceid: int, packet: Content,
request_table_entry: RequestTableEntry,
to_higher: multiprocessing.Queue) -> RequestTableEntry:
"""Handle the case wehere chunk data are received """
chunk_entry = self.chunk_name_in_request_table(packet.name)
if chunk_entry is None:
return request_table_entry
request_table_entry.chunks.append(packet)
request_table_entry = self.remove_chunk_name_from_request_table_entry(
request_table_entry, packet.name)
self._chunk_table[packet.name] = (packet, time.time())
if request_table_entry.chunked and len(request_table_entry.requested_chunks) == 0 \
and len(request_table_entry.requested_md) == 0: # all chunks are available
data = request_table_entry.chunks
#data = sorted(data, key=lambda content: content.name.to_string()) #broken with more than 10 chunks
data = sorted(
data,
key=lambda content: int(''.join(
filter(str.isdigit, content.name.string_components[-1]))))
cont = self.chunkifyer.reassamble_data(request_table_entry.name,
data)
to_higher.put([faceid, cont])
return None
else:
return request_table_entry
def get_chunk_list_from_chunk_table(self,
data_names: Name) -> List[Content]:
"""get a list of content objects from a list of names"""
res = []
for name in data_names:
if name in self._chunk_table:
res.append(self._chunk_table[name][0])
return res
def get_request_table_entry(self, name: Name) -> RequestTableEntry:
"""check if a name is in the chunktable"""
for entry in self._request_table:
if entry.name == name or name in entry.requested_chunks or name in entry.requested_md:
return entry
return None
def chunk_name_in_request_table(self, name):
"""check if a received chunk is expected by the requesttable"""
for entry in self._request_table:
if name in entry.requested_chunks:
return True
return False
def remove_chunk_name_from_request_table_entry(self, request_table_entry: RequestTableEntry, name: Name)\
-> RequestTableEntry:
"""remove chunk from chunktable"""
if name not in request_table_entry.requested_chunks:
return request_table_entry
request_table_entry.requested_chunks.remove(name)
return request_table_entry
def metadata_name_in_request_table(self, name):
"""check if a received metadata is expected by the chunktable"""
for entry in self._request_table:
if name in entry.requested_md:
return True
return False
def remove_metadata_name_from_request_table(self, request_table_entry: RequestTableEntry, name: Name) \
-> RequestTableEntry:
"""remove metadata from chunktable"""
if name not in request_table_entry.requested_md:
return request_table_entry
request_table_entry.requested_md.remove(name)
return request_table_entry
class DataOffloadingChunklayer(LayerProcess):
"""This Chunklayer handles interrupted data uploads by asking neighbouring nodes for available content."""
def __init__(self,
cs: BaseContentStore,
pit: BasePendingInterestTable,
fib: BaseForwardingInformationBase,
chunkifyer: BaseChunkifyer = None,
chunk_size: int = 4096,
num_of_forwards: int = 1,
prefix: str = "car",
log_level: int = 255):
super().__init__("ChunkLayer", log_level=log_level)
self.cs = cs
self.pit = pit
self.fib = fib
self.chunk_size = chunk_size
if chunkifyer is None:
self.chunkifyer = SimpleContentChunkifyer(chunk_size)
else:
self.chunkifyer: BaseChunkifyer = chunkifyer
self.num_of_forwards = num_of_forwards
self.prefix = prefix
manager = multiprocessing.Manager()
self._chunk_table: Dict[Name, (Content, float)] = manager.dict()
self._request_table: List[RequestTableEntry] = manager.list()
self._ca_table: Dict[Name, CaEntry] = manager.dict()
self.recipient_cl: Dict[Name, Name] = manager.dict()
self.pass_through = False
def data_from_higher(self, to_lower: multiprocessing.Queue,
to_higher: multiprocessing.Queue, data):
self.logger.info("Got Data from higher")
faceid = data[0]
packet = data[1]
if isinstance(packet, Interest):
self.logger.info("Packet is Interest " + str(packet.name))
request_entry = self.get_request_entry(packet.name)
if request_entry is None:
self._request_table.append(RequestTableEntry(packet.name))
self._ca_table[packet.name] = CaEntry()
# If the interest starts with /car and not ends with NFN, request metadata and available chunks from neighbours
components = packet.name.string_components
if self.prefix in components[
0] and components[-1] != "NFN" and not request_entry:
self.pass_through = False
ca_entry = self._ca_table.get(packet.name)
name1 = Name("/nL") + packet.name + f"CA{self.num_of_forwards}"
name2 = Name("/nR") + packet.name + f"CA{self.num_of_forwards}"
name3 = Name("/nL") + packet.name + f"CL{self.num_of_forwards}"
name4 = Name("/nR") + packet.name + f"CL{self.num_of_forwards}"
if not self.pit.find_pit_entry(
name1) and self.fib.find_fib_entry(name1):
ca_entry.answer_L = False
ca_entry.received_all = False
to_lower.put([faceid, Interest(name1)])
if not self.pit.find_pit_entry(
name2) and self.fib.find_fib_entry(name2):
ca_entry.answer_R = False
ca_entry.received_all = False
to_lower.put([faceid, Interest(name2)])
if not self.pit.find_pit_entry(
name3) and self.fib.find_fib_entry(name3):
to_lower.put([faceid, Interest(name3)])
if not self.pit.find_pit_entry(
name4) and self.fib.find_fib_entry(name4):
to_lower.put([faceid, Interest(name4)])
self._ca_table[packet.name] = ca_entry
to_lower.put(data)
return
elif isinstance(packet, Content):
self.logger.info("Packet is Content (name=%s, %d bytes)" %
(str(packet.name), len(packet.content)))
if len(packet.content) < self.chunk_size:
to_lower.put(data)
else:
self.logger.info("Chunking Packet")
metadata, chunks = self.chunkifyer.chunk_data(
packet) # Create metadata and chunks
self.logger.info("Metadata: " + metadata[0].content)
to_lower.put([faceid, metadata[0]
]) # Return name of first metadata object
for md in metadata: # Add metadata to chunktable
if md.name not in self._chunk_table:
self._chunk_table[md.name] = (md, time.time())
for c in chunks: # Add chunks to chunktable
if c.name not in self._chunk_table:
self._chunk_table[c.name] = (c, time.time())
elif isinstance(packet, Nack):
request_entry = self.get_request_entry(packet.name)
if request_entry is not None:
self._request_table.remove(request_entry)
to_lower.put(data)
def data_from_lower(self, to_lower: multiprocessing.Queue,
to_higher: multiprocessing.Queue, data):
self.logger.info("Got Data from lower")
faceid = data[0]
packet = data[1]
components = packet.name.components
string_components = packet.name.string_components
last = components[-1]
if isinstance(packet, Interest):
self.logger.info("Packet is Interest")
# Interest for available chunks
if "CA" in string_components[-1]:
self.pass_through = True # This node doesn't handle CA content, only forwards to neighbours
chunks_available = self.get_chunks_available(packet)
if last == b"CA0" or isinstance(
chunks_available,
Content): # If there are chunks available, return them
to_lower.put([faceid, chunks_available])
else: # Otherwise try to pass on interest to neighbour
to_lower.put(
[faceid,
Interest(self.decrease_name(packet.name))])
# General Interest passed on to chunklayer
elif "CL" in string_components[-1]:
matching_content = self.get_matching_content_from_packed_name(
packet)
if last == b"CL0" or isinstance(
matching_content,
Content): # If there is matching content, return it
to_lower.put([faceid, matching_content])
else: # Otherwise try to pass on to neighbour
to_lower.put(
[faceid,
Interest(self.decrease_name(packet.name))])
elif packet.name in self._chunk_table:
matching_content = self._chunk_table.get(packet.name)[0]
to_lower.put([faceid, matching_content])
else:
to_higher.put(data)
elif isinstance(packet, Content):
self.logger.info("Packet is Content")
ca_content = False
cl_content = False
# Metadata or string containing available chunks from neighbour
if "CA" in string_components[-1]:
self.cs.remove_content_object(
packet.name
) # Remove from cs, so next interest comes through to chunklayer
if string_components[
-1] == f"CA{self.num_of_forwards}": # This is the requesting node --> unpack
ca_content = True
ca_entry = self._ca_table.get(self.unpack(packet.name))
# In order to be able to request the available chunks from the sender of this packet,
# we need to safe the first component.
# This is then used in pack_ca() to send packet to the correct recipient.
ca_entry.recipient = Name(components[:1])
# Only replace existing list of available chunks if it contains more chunks
self.save_if_longest(packet, ca_entry)
if components[0] == b"nL":
ca_entry.answer_L = True
else:
ca_entry.answer_R = True
if ca_entry.answer_L and ca_entry.answer_R: # We have a response from both neighbours and can proceed
packet = ca_entry.ca
self._request_table.append(
RequestTableEntry(packet.name))
self._ca_table[self.unpack(packet.name)] = ca_entry
else: # This is not the requesting node --> pass on to neighbour
to_lower.put([
faceid,
Content(self.increase_name(packet.name),
packet.content)
])
return
# Content from the chunklayer of a neighbouring node
elif "CL" in string_components[-1]:
if string_components[
-1] == f"CL{self.num_of_forwards}": # This is the requesting node --> unpack
cl_content = True
packet.name = self.unpack(packet.name)
# Save the sender of this packet as the recipient for further interests. Used in pack_cl()
self.recipient_cl[packet.name] = Name(components[:1])
else: # This is not the requesting node --> pass on to neighbour
to_lower.put([
faceid,
Content(self.increase_name(packet.name),
packet.content)
])
return
request_entry = self.get_request_entry(packet.name)
if request_entry is None:
return
self._request_table.remove(request_entry)
if "CA" in components[-2].decode("utf-8"):
if self.pass_through:
return
ca_content = True
request_entry.chunked = True
self.handle_content(faceid, packet, request_entry, ca_content,
cl_content, to_lower, to_higher)
elif isinstance(packet, Nack):
if self.prefix not in string_components[0]:
request_entry = self.get_request_entry(packet.name)
if request_entry:
self._request_table.remove(request_entry)
if "CA" in string_components[-1]:
if string_components[
-1] == f"CA{self.num_of_forwards}": # This is the requesting node
unpacked = self.unpack(packet.name)
ca_entry = self._ca_table.get(unpacked)
if components[0] == b"nL":
ca_entry.answer_L = True
else:
ca_entry.answer_R = True
if ca_entry.answer_L and ca_entry.answer_R: # We have an answer from both neighbour
if ca_entry.ca: # We have chunks available from one of the neighbours
packet = ca_entry.ca
self._ca_table[unpacked] = ca_entry
self.handle_content(
faceid, packet,
RequestTableEntry(packet.name), True,
False, to_lower, to_higher)
return
else:
ca_entry.received_all = True
request_entry = self.get_request_entry(
unpacked)
if request_entry and not request_entry.requested_md and not self.pass_through:
self.create_chunk_interests(
faceid, request_entry, ca_entry,
to_lower)
self._ca_table[unpacked] = ca_entry
else: # This is not the requesting node --> pass on to neighbour
name = self.increase_name(packet.name)
nack = Nack(name, NackReason.NO_CONTENT,
Interest(name))
to_lower.put([faceid, nack])
elif "CL" in string_components[-1]:
if string_components[
-1] != f"CL{self.num_of_forwards}": # This is not the requesting node --> pass on to neighbour
name = self.increase_name(packet.name)
nack = Nack(name, NackReason.NO_CONTENT,
Interest(name))
to_lower.put([faceid, nack])
else:
to_higher.put([faceid, packet])
self.pit.remove_pit_entry(packet.name)
else:
if "c" in string_components[-1]:
packet.name.components = components[:-1]
to_higher.put([
faceid,
Nack(packet.name, NackReason.NO_CONTENT,
Interest(packet.name))
])
else: #FIXME What to do here?
# to_higher.put([faceid, packet])
pass
def handle_content(self, faceid: int, packet: Content,
request_entry: RequestTableEntry, ca_content: bool,
cl_content: bool, to_lower: multiprocessing.Queue,
to_higher: multiprocessing.Queue):
"""Handle incoming content"""
if request_entry.chunked is False: # Not chunked content
if not packet.get_bytes().startswith(b'mdo:'):
if ca_content:
self.handle_ca(faceid, packet, to_lower)
else:
to_higher.put([faceid, packet])
return
else: # Received metadata data --> chunked content
request_entry.chunked = True
if packet.get_bytes().startswith(
b'mdo:'): # Request all frames from meta data
self.handle_received_meta_data(faceid, packet, request_entry,
to_lower, ca_content, cl_content)
else:
self.handle_received_chunk_data(faceid, packet, request_entry,
to_lower, to_higher, ca_content)
def handle_received_meta_data(self, faceid: int, packet: Content,
request_entry: RequestTableEntry,
to_lower: multiprocessing.Queue,
ca_content: bool, cl_content: bool):
"""Handle meta data"""
if not request_entry.md_complete:
if packet.name in request_entry.requested_md:
request_entry.requested_md.remove(packet.name)
md, chunks, size = self.chunkifyer.parse_meta_data(packet.content)
for chunk in chunks: # Request all chunks from the meta data file if not already received or requested
if chunk not in request_entry.requested_chunks and chunk not in [
i.name for i in request_entry.chunks
]:
request_entry.requested_chunks.append(chunk)
if md is not None: # There is another md file
if md not in request_entry.requested_md:
request_entry.requested_md.append(md)
if cl_content:
md = self.pack_cl(md)
to_lower.put([faceid, Interest(md)])
else:
# Only create interests if it is the requesting node handling this metadata and
# either the packet is CA content or there is no CA content available
request_entry.md_complete = True
if not self.pass_through:
if ca_content:
self.create_chunk_interests(faceid, request_entry,
CaEntry(), to_lower)
elif self._ca_table.get(
request_entry.name
).received_all: # We have an answer from both neighbours
self.create_chunk_interests(
faceid, request_entry,
self._ca_table.get(request_entry.name), to_lower)
self._chunk_table[packet.name] = (packet, time.time())
self._request_table.append(request_entry)
def handle_received_chunk_data(self, faceid: int, packet: Content,
request_entry: RequestTableEntry,
to_lower: multiprocessing.Queue,
to_higher: multiprocessing.Queue,
ca_content: bool):
"""Handle chunk data"""
if packet.name in request_entry.requested_chunks:
request_entry.requested_chunks.remove(packet.name)
request_entry.chunks.append(packet)
self._chunk_table[packet.name] = (packet, time.time())
if not request_entry.requested_chunks:
if not request_entry.requested_md: # All chunks are available
data = request_entry.chunks
data = sorted(data,
key=lambda content: int(''.join(
filter(str.isdigit, content.name.
string_components[-1]))))
cont = self.chunkifyer.reassamble_data(request_entry.name,
data)
if ca_content:
self.handle_ca(faceid, cont, to_lower)
else:
del self._ca_table[request_entry.name]
to_higher.put([faceid, cont])
return
self._request_table.append(request_entry)
def handle_ca(self, faceid: int, content: Content,
to_lower: multiprocessing.Queue):
"""Unpack the received ca message and create interests for all available chunks."""
content.name = self.unpack(content.name)
ca_entry = self._ca_table.get(content.name)
ca_entry.received_all = True
chunks_str = content.content.split(";")
request_entry = self.get_request_entry(content.name)
if request_entry and not self.pass_through:
if chunks_str[0] == "complete": # Neighbour has complete data
ca_entry.completely_available = True
if request_entry.md_complete:
self.create_chunk_interests(faceid, request_entry,
ca_entry, to_lower)
else:
ca_entry.chunks = [Name(chunk) for chunk in chunks_str]
# Create interests for all chunks that are available from neighbour
for chunk in ca_entry.chunks:
self.logger.info("TO NEIGHBOUR:",
self.pack_ca(chunk, ca_entry))
if chunk not in request_entry.requested_chunks and chunk not in [
i.name for i in request_entry.chunks
]:
request_entry.requested_chunks.append(chunk)
to_lower.put(
[faceid,
Interest(self.pack_ca(chunk, ca_entry))])
self._request_table.append(request_entry)
# If there is no name in requested_md try to request the remaining chunks.
# This is only necessary to get a NACK, so the simulation continues.
if not request_entry.requested_md:
for chunk in [
i for i in request_entry.requested_chunks
if i not in ca_entry.chunks
]:
self.logger.info("TO ORIGINAL SOURCE:", chunk)
to_lower.put([faceid, Interest(chunk)])
self._request_table.remove(request_entry)
self._ca_table[content.name] = ca_entry
def get_request_entry(self, name: Name):
"""
Check if a name is in the request table.
Return entry or None.
"""
for entry in self._request_table:
if entry.name == name or name in entry.requested_chunks or name in entry.requested_md:
return entry
return None
def get_chunks_available(self, packet: Packet):
"""
Check if chunks are available for a given name.
Return a content object containing the names of the available chunks, or NACK
"""
chunks_available = []
name = self.unpack(packet.name)
request_entry = self.get_request_entry(name)
cs_entry = self.cs.find_content_object(name)
if request_entry is not None:
chunks_available = [
str(chunk.name) for chunk in request_entry.chunks
]
elif cs_entry:
chunks_available.append("complete;")
if chunks_available:
chunks_available = Content(packet.name, ";".join(chunks_available))
if len(chunks_available.content) > self.chunk_size:
meta_data, chunks = self.chunkifyer.chunk_data(
chunks_available)
meta_data.extend(chunks)
for data in meta_data:
self.cs.remove_content_object(data.name)
self.cs.add_content_object(data)
return meta_data[0]
else:
return chunks_available
return Nack(packet.name, NackReason.NO_CONTENT, packet)
def get_matching_content_from_packed_name(self, packet: Packet):
"""Return either the content matching the unpacked name or NACK"""
name_in = self.unpack(packet.name)
if name_in in self._chunk_table:
matching_content = self._chunk_table.get(name_in)[0]
matching_content.name = packet.name
return matching_content
else:
return Nack(packet.name, NackReason.NO_CONTENT, packet)
def create_chunk_interests(self, faceid: int,
request_entry: RequestTableEntry,
ca_entry: CaEntry,
to_lower: multiprocessing.Queue):
"""Create interests for all the chunks in requested_md of the specified request table entry."""
if ca_entry.completely_available:
for chunk in [i for i in request_entry.requested_chunks]:
self.logger.info("TO NEIGHBOUR:",
self.pack_ca(chunk, ca_entry))
if chunk not in request_entry.requested_chunks and chunk not in [
i.name for i in request_entry.chunks
]:
request_entry.requested_chunks.append(chunk)
to_lower.put([faceid, Interest(self.pack_ca(chunk, ca_entry))])
else:
for chunk in [
i for i in request_entry.requested_chunks
if i not in ca_entry.chunks
]:
self.logger.info("TO ORIGINAL SOURCE:", chunk)
to_lower.put([faceid, Interest(chunk)])
def save_if_longest(self, packet: Content, ca_entry: CaEntry):
"""
Check if the received ca content is longer than the existing one and if so, replace it.
In the case where both neighbours have chunks available, we want to send the interests only to the one
which has more.
"""
if packet.get_bytes().startswith(
b'mdo:'): # Content is metadata, read size from metadata
_, _, content_size = self.chunkifyer.parse_meta_data(
packet.content)
else: # Content is string, size equals length of the string
content_size = (len(packet.content))
content_size = int(content_size)
if content_size > ca_entry.size:
ca_entry.ca = packet
ca_entry.size = content_size
def pack_ca(self, name: Name, ca_entry: CaEntry) -> Name:
"""Prepend the recipient, append "CL" and the number of forwards."""
return ca_entry.recipient + name + f"CL{self.num_of_forwards}"
def pack_cl(self, name: Name) -> Name:
"""Prepend the recipient, append "CA" and the number of forwards."""
lookup_name = Name(name.components[:-1])
return self.recipient_cl.get(
lookup_name) + name + f"CL{self.num_of_forwards}"
def unpack(self, name: Name) -> Name:
return Name(name.components[1:-1])
def increase_name(self, name: Name) -> Name:
"""
Increase the number at the end of the name.
The number is used to determine whether or not a packet gets forwarded to the next neighbour.
"""
components = name.components
last = components[-1].decode("utf-8")
i = int(''.join(filter(str.isdigit, last)))
if "CA" in last:
return Name(components[:-1]) + f"CA{i+1}"
elif "CL" in last:
return Name(components[:-1]) + f"CL{i+1}"
return name
def decrease_name(self, name: Name) -> Name:
"""
Decrease the number at the end of the name.
The number is used to determine whether or not a packet gets forwarded to the next neighbour.
"""
components = name.components
last = components[-1].decode("utf-8")
i = int(''.join(filter(str.isdigit, last)))
if "CA" in last:
return Name(components[:-1]) + f"CA{i-1}"
elif "CL" in last:
return Name(components[:-1]) + f"CL{i-1}"
return name
def set_number_of_forwards(self, number_of_forwards: int):
self.num_of_forwards = number_of_forwards
class DataOffloadingChunklayerSimple(LayerProcess):
"""This Chunklayer handles interrupted data uploads by asking neighbouring nodes for available content."""
def __init__(self,
cs: BaseContentStore,
pit: BasePendingInterestTable,
fib: BaseForwardingInformationBase,
chunkifyer: BaseChunkifyer = None,
chunk_size: int = 4096,
num_of_forwards: int = 1,
prefix: str = "car",
log_level: int = 255):
super().__init__("ChunkLayer", log_level=log_level)
self.cs = cs
self.pit = pit
self.fib = fib
self.chunk_size = chunk_size
if chunkifyer is None:
self.chunkifyer = SimpleContentChunkifyer(chunk_size)
else:
self.chunkifyer: BaseChunkifyer = chunkifyer
self.num_of_forwards = num_of_forwards
self.prefix = prefix
manager = multiprocessing.Manager()
self._chunk_table: Dict[Name, (Content, float)] = manager.dict()
self._request_table: List[RequestTableEntry] = manager.list()
self._cl_table: Dict[Name, ClEntry] = manager.dict()
self.pass_through = False
self.cl_sent = False
def data_from_higher(self, to_lower: multiprocessing.Queue,
to_higher: multiprocessing.Queue, data):
self.logger.info("Got Data from higher")
faceid = data[0]
packet = data[1]
if isinstance(packet, Interest):
self.logger.info("Packet is Interest " + str(packet.name))
request_entry = self.get_request_entry(packet.name)
if request_entry is None:
self._request_table.append(RequestTableEntry(packet.name))
# If the interest starts with "/car" and not ends with "NFN", request metadata and available chunks from neighbours
components = packet.name.string_components
if self.prefix in components[
0] and components[-1] != "NFN" and not request_entry:
self.pass_through = False
self._cl_table[packet.name] = ClEntry(data)
cl_entry = self._cl_table.get(packet.name)
name1 = Name("/nL") + packet.name + f"CL{self.num_of_forwards}"
name2 = Name("/nR") + packet.name + f"CL{self.num_of_forwards}"
if not self.pit.find_pit_entry(
name1) and self.fib.find_fib_entry(name1):
cl_entry.nack_L = False
self.cl_sent = True
to_lower.put([faceid, Interest(name1)])
if not self.pit.find_pit_entry(
name2) and self.fib.find_fib_entry(name2):
cl_entry.nack_R = False
self.cl_sent = True
to_lower.put([faceid, Interest(name2)])
if self.cl_sent:
self._cl_table[packet.name] = cl_entry
return
to_lower.put(data)
return
elif isinstance(packet, Content):
self.logger.info("Packet is Content (name=%s, %d bytes)" %
(str(packet.name), len(packet.content)))
if len(packet.content) < self.chunk_size:
to_lower.put(data)
else:
self.logger.info("Chunking Packet")
metadata, chunks = self.chunkifyer.chunk_data(
packet) # Create metadata and chunks
self.logger.info("Metadata: " + metadata[0].content)
to_lower.put([faceid, metadata[0]
]) # Return name of first metadata object
for md in metadata: # Add metadata to chunktable
if md.name not in self._chunk_table:
self._chunk_table[md.name] = (md, time.time())
for c in chunks: # Add chunks to chunktable
if c.name not in self._chunk_table:
self._chunk_table[c.name] = (c, time.time())
elif isinstance(packet, Nack):
request_entry = self.get_request_entry(packet.name)
if request_entry is not None:
self._request_table.remove(request_entry)
to_lower.put(data)
def data_from_lower(self, to_lower: multiprocessing.Queue,
to_higher: multiprocessing.Queue, data):
self.logger.info("Got Data from lower")
faceid = data[0]
packet = data[1]
components = packet.name.components
string_components = packet.name.string_components
last = components[-1]
if isinstance(packet, Interest):
self.logger.info("Packet is Interest")
# General Interest passed on to chunklayer
if "CL" in string_components[-1]:
matching_content = self.get_matching_content(packet)
if last == b"CL0" or isinstance(
matching_content,
Content): # If there is matching content, return it
to_lower.put([faceid, matching_content])
else: # Otherwise try to pass on to neighbour
to_lower.put(
[faceid,
Interest(self.decrease_name(packet.name))])
elif packet.name in self._chunk_table:
matching_content = self._chunk_table.get(packet.name)[0]
to_lower.put([faceid, matching_content])
else:
to_higher.put(data)
elif isinstance(packet, Content):
self.logger.info("Packet is Content")
cl_content = False
# Content from the chunklayer of a neighbouring node
if "CL" in string_components[-1]:
if string_components[
-1] == f"CL{self.num_of_forwards}": # This is the requesting node --> unpack
cl_content = True
packet.name = self.unpack(packet.name)
# Save the sender of this packet as the recipient for further interests. Used in pack_cl()
request_entry = self.get_request_entry(packet.name)
if request_entry:
cl_entry = self._cl_table.get(request_entry.name)
if cl_entry.interest_requested: # If we already resent requests to source, don't consider it
return
cl_entry.recipient = Name(components[:1])
self._cl_table[request_entry.name] = cl_entry
else: # This is not the requesting node --> pass on to neighbour
to_lower.put([
faceid,
Content(self.increase_name(packet.name),
packet.content)
])
return
request_entry = self.get_request_entry(packet.name)
if request_entry is None:
return
self.handle_content(faceid, packet, request_entry, cl_content,
to_lower, to_higher)
elif isinstance(packet, Nack):
if self.prefix not in string_components[0]:
request_entry = self.get_request_entry(packet.name)
if request_entry:
self._request_table.remove(request_entry)
if "CL" in string_components[-1]:
if string_components[
-1] == f"CL{self.num_of_forwards}": # This is the requesting node --> unpack
name_unpacked = self.unpack(packet.name)
request_entry = self.get_request_entry(name_unpacked)
if request_entry:
cl_entry = self._cl_table.get(request_entry.name)
if components[0] == b"nL":
cl_entry.nack_L = True
else:
cl_entry.nack_R = True
if cl_entry.nack_L and cl_entry.nack_R and not cl_entry.interest_requested:
# No more data available from neighbours, get it from car
self.get_missing_data_from_original_source(
faceid, request_entry, cl_entry, to_lower)
self._cl_table[request_entry.name] = cl_entry
else:
name = self.increase_name(packet.name)
nack = Nack(name, NackReason.NO_CONTENT,
Interest(name))
to_lower.put([faceid, nack])
else:
to_higher.put([faceid, packet])
self.pit.remove_pit_entry(packet.name)
else:
if "c" in string_components[-1] or "m" in string_components[-1]:
packet.name.components = components[:-1]
to_higher.put([
faceid,
Nack(packet.name, NackReason.NO_CONTENT,
Interest(packet.name))
])
else:
pass
# to_higher.put([faceid, packet])
def handle_content(self, faceid: int, packet: Content,
request_entry: RequestTableEntry, cl_content: bool,
to_lower: multiprocessing.Queue,
to_higher: multiprocessing.Queue):
"""Handle incoming content"""
self._request_table.remove(request_entry)
if request_entry.chunked is False: # Not chunked content
if not packet.get_bytes().startswith(b'mdo:'):
to_higher.put([faceid, packet])
return
else: # Received metadata data --> chunked content
request_entry.chunked = True
if packet.get_bytes().startswith(
b'mdo:'): # Request all frames from metadata
self.handle_received_meta_data(faceid, packet, request_entry,
to_lower, cl_content)
else:
self.handle_received_chunk_data(faceid, packet, request_entry,
to_higher)
def handle_received_meta_data(self, faceid: int, packet: Content,
request_entry: RequestTableEntry,
to_lower: multiprocessing.Queue,
cl_content: bool):
"""Handle meta data"""
if packet.name in request_entry.requested_md:
request_entry.requested_md.remove(packet.name)
md, chunks, size = self.chunkifyer.parse_meta_data(packet.content)
for chunk in chunks: # Request all chunks from the metadata file if not already received or requested
# if chunk not in request_entry.requested_chunks and chunk not in [i.name for i in request_entry.chunks]:
if chunk not in [i.name for i in request_entry.chunks]:
request_entry.requested_chunks.append(chunk)
if not self.pass_through:
if cl_content:
cl_entry = self._cl_table.get(request_entry.name)
if cl_entry.nack_L and cl_entry.nack_R:
break
chunk = self.pack_cl(chunk)
to_lower.put([faceid, Interest(chunk)])
if md is not None: # There is another md file
if md not in request_entry.requested_md:
request_entry.requested_md.append(md)
if not self.pass_through:
if cl_content:
cl_entry = self._cl_table.get(request_entry.name)
if not (cl_entry.nack_L and cl_entry.nack_R):
cl_entry.interest = [faceid, Interest(packet.name)]
self._cl_table[request_entry.name] = cl_entry
md = self.pack_cl(md)
to_lower.put([faceid, Interest(md)])
else:
to_lower.put([faceid, Interest(md)])
else:
request_entry.last_chunk = chunks[-1]
self._chunk_table[packet.name] = (packet, time.time())
self._request_table.append(request_entry)
def handle_received_chunk_data(self, faceid: int, packet: Content,
request_entry: RequestTableEntry,
to_higher: multiprocessing.Queue):
"""Handle chunk data"""
if packet.name in request_entry.requested_chunks:
request_entry.requested_chunks.remove(packet.name)
request_entry.chunks.append(packet)
self._chunk_table[packet.name] = (packet, time.time())
if not request_entry.requested_chunks and not request_entry.requested_md:
if not request_entry.requested_md: # All chunks are available
data = request_entry.chunks
data = sorted(data,
key=lambda content: int(''.join(
filter(str.isdigit, content.name.
string_components[-1]))))
cont = self.chunkifyer.reassamble_data(request_entry.name,
data)
to_higher.put([faceid, cont])
return
self._request_table.append(request_entry)
def get_request_entry(self, name: Name):
"""
Check if a name is in the request table.
Return entry or None.
"""
for entry in self._request_table:
if entry.name == name or name in entry.requested_chunks or name in entry.requested_md:
return entry
return None
def get_matching_content(self, packet: Packet):
"""Return either the content matching the packet name or NACK"""
name_in = self.unpack(packet.name)
cs_entry = self.cs.find_content_object(name_in)
if name_in in self._chunk_table:
matching_content = self._chunk_table.get(name_in)[0]
matching_content.name = packet.name
return matching_content
elif cs_entry:
matching_content = cs_entry.content
matching_content.name = packet.name
return matching_content
else:
return Nack(packet.name, NackReason.NO_CONTENT, packet)
def pack_cl(self, name: Name) -> Name:
"""Prepend the recipient, append "CL" and the number of forwards."""
lookup_name = Name(name.components[:-1])
return self._cl_table.get(
lookup_name).recipient + name + f"CL{self.num_of_forwards}"
def unpack(self, name: Name) -> Name:
return Name(name.components[1:-1])
def increase_name(self, name: Name) -> Name:
"""
Increase the number at the end of the name.
The number is used to determine whether or not a packet gets forwarded to the next neighbour.
"""
components = name.components
last = components[-1].decode("utf-8")
i = int(''.join(filter(str.isdigit, last)))
if "CL" in last:
return Name(components[:-1]) + f"CL{i+1}"
return name
def decrease_name(self, name: Name) -> Name:
"""
Decrease the number at the end of the name.
The number is used to determine whether or not a packet gets forwarded to the next neighbour.
"""
components = name.components
last = components[-1].decode("utf-8")
i = int(''.join(filter(str.isdigit, last)))
if "CL" in last:
return Name(components[:-1]) + f"CL{i-1}"
return name
def get_missing_data_from_original_source(self, faceid: int,
request_entry: RequestTableEntry,
cl_entry: ClEntry,
to_lower: multiprocessing.Queue):
"""
Start requesting the missing files from the original source.
"""
if not cl_entry.interest_requested:
if request_entry.requested_chunks:
# Request again all chunks that have been requested but not satisfied yet
for chunk in request_entry.requested_chunks:
to_lower.put([faceid, Interest(chunk)])
# If requested_md is not empty, request them again from source
if request_entry.requested_md:
for md in request_entry.requested_md:
to_lower.put([faceid, Interest(md)])
else: # if empty, request orginal interest from source
to_lower.put(cl_entry.interest)
cl_entry.interest_requested = True
def set_number_of_forwards(self, number_of_forwards: int):
self.num_of_forwards = number_of_forwards