class test_SimpleContentChunkifyer(unittest.TestCase): def setUp(self): self.chunkifyer = SimpleContentChunkifyer() def tearDown(self): pass def test_generate_metadata_no_next(self): """Test generating a simple metadata object""" name = Name("/test/data") res = self.chunkifyer.generate_meta_data(2, 4, 0, 0, name) self.assertEqual(res.name.to_string(), "/test/data") self.assertEqual(res.content, "mdo:/test/data/c2;/test/data/c3:") def test_generate_metadata_one_next(self): """Test generating a simple metadata object with one following""" name = Name("/test/data") res = self.chunkifyer.generate_meta_data(2, 4, 0, 1, name) self.assertEqual(res.name.to_string(), "/test/data") self.assertEqual(res.content, "mdo:/test/data/c2;/test/data/c3:/test/data/m1") def test_generate_metadata_two_next(self): """Test generating a simple metadata object with two following""" name = Name("/test/data") res = self.chunkifyer.generate_meta_data(2, 4, 1, 2, name) self.assertEqual(res.name.to_string(), "/test/data/m1") self.assertEqual(res.content, "mdo:/test/data/c2;/test/data/c3:/test/data/m2") def test_chunk_single_metadata(self): name = Name("/test/data") string = "A" * 4096 + "B" * 4096 + "C" * 4096 content = Content(name, string) md, content = self.chunkifyer.chunk_data(content) md_name_comp = ['/test/data'] md_data_comp = ['mdo:/test/data/c0;/test/data/c1;/test/data/c2:'] content_name_comp = ['/test/data/c0', '/test/data/c1', '/test/data/c2'] content_data_comp = ["A" * 4096, "B" * 4096, "C" * 4096] for i in range(0, len(md)): self.assertEqual(md[i].name.to_string(), md_name_comp[i]) self.assertEqual(md[i].content, md_data_comp[i]) for i in range(0, len(content)): self.assertEqual(content[i].name.to_string(), content_name_comp[i]) self.assertEqual(content[i].content, content_data_comp[i]) def test_chunk_multiple_metadata(self): """Test chunking metadata with three metadata objects and 10 chunks""" name = Name("/test/data") string = "A"*4096 + "B"*4096 + "C"*4096 + "D"*4096 + "E"*4096 + "F"*4096 + "G"*4096 + "H"*4096 \ + "I"*4096 + "J"*4000 content = Content(name, string) md, chunked_content = self.chunkifyer.chunk_data(content) md_name_comp = ['/test/data', '/test/data/m1', '/test/data/m2'] md_data_comp = [ 'mdo:/test/data/c0;/test/data/c1;/test/data/c2;/test/data/c3:/test/data/m1', 'mdo:/test/data/c4;/test/data/c5;/test/data/c6;/test/data/c7:/test/data/m2', 'mdo:/test/data/c8;/test/data/c9:' ] content_name_comp = [ '/test/data/c0', '/test/data/c1', '/test/data/c2', '/test/data/c3', '/test/data/c4', '/test/data/c5', '/test/data/c6', '/test/data/c7', '/test/data/c8', '/test/data/c9' ] content_data_comp = [ "A" * 4096, "B" * 4096, "C" * 4096, "D" * 4096, "E" * 4096, "F" * 4096, "G" * 4096, "H" * 4096, "I" * 4096, "J" * 4000 ] for i in range(0, len(md)): self.assertEqual(md[i].name.to_string(), md_name_comp[i]) self.assertEqual(md[i].content, md_data_comp[i]) for i in range(0, len(chunked_content)): self.assertEqual(chunked_content[i].name.to_string(), content_name_comp[i]) self.assertEqual(chunked_content[i].content, content_data_comp[i]) def test_chunk_multiple_metadata_reassemble(self): """Test chunking metadata with three metadata objects and 10 chunks and reassemble""" name = Name("/test/data") string = "A" * 4096 + "B" * 4096 + "C" * 4096 + "D" * 4096 + "E" * 4096 + "F" * 4096 + "G" * 4096 + "H" * 4096 \ + "I" * 4096 + "J" * 4000 content = Content(name, string) md, chunked_content = self.chunkifyer.chunk_data(content) md_name_comp = ['/test/data', '/test/data/m1', '/test/data/m2'] md_data_comp = [ 'mdo:/test/data/c0;/test/data/c1;/test/data/c2;/test/data/c3:/test/data/m1', 'mdo:/test/data/c4;/test/data/c5;/test/data/c6;/test/data/c7:/test/data/m2', 'mdo:/test/data/c8;/test/data/c9:' ] content_name_comp = [ '/test/data/c0', '/test/data/c1', '/test/data/c2', '/test/data/c3', '/test/data/c4', '/test/data/c5', '/test/data/c6', '/test/data/c7', '/test/data/c8', '/test/data/c9' ] content_data_comp = [ "A" * 4096, "B" * 4096, "C" * 4096, "D" * 4096, "E" * 4096, "F" * 4096, "G" * 4096, "H" * 4096, "I" * 4096, "J" * 4000 ] for i in range(0, len(md)): self.assertEqual(md[i].name.to_string(), md_name_comp[i]) self.assertEqual(md[i].content, md_data_comp[i]) for i in range(0, len(chunked_content)): self.assertEqual(chunked_content[i].name.to_string(), content_name_comp[i]) self.assertEqual(chunked_content[i].content, content_data_comp[i]) reassembled_content = self.chunkifyer.reassamble_data( md[0].name, chunked_content) self.assertEqual(content, reassembled_content) def test_parse_metadata_next(self): """Test parse metadata with next metadata""" md, names = self.chunkifyer.parse_meta_data( "mdo:/test/data/c0;/test/data/c1;/test/data/c2;/test/data/c3:/test/data/m1" ) self.assertEqual(Name("/test/data/m1"), md) names_comp = [ Name("/test/data/c0"), Name("/test/data/c1"), Name("/test/data/c2"), Name("/test/data/c3") ] self.assertEqual(names, names_comp) def test_parse_metadata(self): """Test parse metadata""" md, names = self.chunkifyer.parse_meta_data( "mdo:/test/data/c0;/test/data/c1;/test/data/c2;/test/data/c3:") self.assertEqual(None, md) names_comp = [ Name("/test/data/c0"), Name("/test/data/c1"), Name("/test/data/c2"), Name("/test/data/c3") ] self.assertEqual(names, names_comp)
class BasicChunkLayer(LayerProcess): """"Basic Chunking Layer for PICN""" def __init__(self, chunkifyer: BaseChunkifyer = None, chunk_size: int = 4096, manager: multiprocessing.Manager = None, log_level=255): super().__init__("ChunkLayer", log_level=log_level) self.chunk_size = chunk_size if chunkifyer == None: self.chunkifyer = SimpleContentChunkifyer(chunk_size) else: self.chunkifyer: BaseChunkifyer = chunkifyer if manager is None: manager = multiprocessing.Manager() self._chunk_table: Dict[Name, (Content, float)] = manager.dict() self._request_table: List[RequestTableEntry] = manager.list() def data_from_higher(self, to_lower: multiprocessing.Queue, to_higher: multiprocessing.Queue, data): self.logger.info("Got Data from higher") faceid = data[0] packet = data[1] if isinstance(packet, Interest): self.logger.info("Packet is Interest " + str(packet.name)) requestentry = self.get_request_table_entry(packet.name) if requestentry is None: self._request_table.append(RequestTableEntry(packet.name)) to_lower.put([faceid, packet]) return if isinstance(packet, Content): self.logger.info("Packet is Content (name=%s, %d bytes)" % \ (str(packet.name), len(packet.content))) if len(packet.content) < self.chunk_size: to_lower.put([faceid, packet]) else: self.logger.info("Chunking Packet") metadata, chunks = self.chunkifyer.chunk_data( packet) #create metadata and chunks self.logger.info("Metadata: " + metadata[0].content) to_lower.put( [faceid, metadata[0]] ) #return first name TODO HANDLE THE CASE, WHERE CHUNKS CAN TIMEOUT AND MUST BE REPRODUCED for md in metadata: #add metadata to chunktable if md.name not in self._chunk_table: self._chunk_table[md.name] = (md, time.time()) for c in chunks: #add chunks to chunktable if c.name not in self._chunk_table: self._chunk_table[c.name] = (c, time.time()) if isinstance(packet, Nack): requestentry = self.get_request_table_entry(packet.name) if requestentry is not None: self._request_table.remove(requestentry) to_lower.put([faceid, packet]) def data_from_lower(self, to_lower: multiprocessing.Queue, to_higher: multiprocessing.Queue, data): self.logger.info("Got Data from lower") faceid = data[0] packet = data[1] if isinstance(packet, Interest): self.logger.info("Packet is Interest") if packet.name in self._chunk_table: #Check if Interest is in chunktable matching_content = self._chunk_table.get(packet.name)[0] to_lower.put([faceid, matching_content]) else: to_higher.put([faceid, packet]) return if isinstance(packet, Content): self.logger.info("Packet is Content") request_table_entry = self.get_request_table_entry(packet.name) if request_table_entry is None: return self._request_table.remove(request_table_entry) if request_table_entry.chunked is False: #not chunked content if not packet.get_bytes().startswith(b'mdo:'): to_higher.put([faceid, packet]) return else: # Received metadata data --> chunked content request_table_entry.chunked = True if packet.get_bytes().startswith( b'mdo:'): # request all frames from metadata request_table_entry = self.handle_received_meta_data( faceid, packet, request_table_entry, to_lower) else: request_table_entry = self.handle_received_chunk_data( faceid, packet, request_table_entry, to_higher) if request_table_entry is None: return #deletes entry if data was completed self._request_table.append(request_table_entry) if isinstance(packet, Nack): requestentry = self.get_request_table_entry(packet.name) if requestentry is not None: self._request_table.remove(requestentry) to_higher.put([faceid, packet]) def handle_received_meta_data( self, faceid: int, packet: Content, request_table_entry: RequestTableEntry, to_lower: multiprocessing.Queue) -> RequestTableEntry: """Handle the case, where metadata are received from the network""" md_entry = self.metadata_name_in_request_table(packet.name) if md_entry is None: return request_table_entry request_table_entry = self.remove_metadata_name_from_request_table( request_table_entry, packet.name) md, chunks, size = self.chunkifyer.parse_meta_data(packet.content) if md is not None: # there is another md file request_table_entry.requested_md.append(md) to_lower.put([faceid, Interest(md)]) else: request_table_entry.lastchunk = chunks[-1] for chunk in chunks: # request all chunks from the metadata file request_table_entry.requested_chunks.append(chunk) to_lower.put([faceid, Interest(chunk)]) self._chunk_table[packet.name] = (packet, time.time()) return request_table_entry def handle_received_chunk_data( self, faceid: int, packet: Content, request_table_entry: RequestTableEntry, to_higher: multiprocessing.Queue) -> RequestTableEntry: """Handle the case wehere chunk data are received """ chunk_entry = self.chunk_name_in_request_table(packet.name) if chunk_entry is None: return request_table_entry request_table_entry.chunks.append(packet) request_table_entry = self.remove_chunk_name_from_request_table_entry( request_table_entry, packet.name) self._chunk_table[packet.name] = (packet, time.time()) if request_table_entry.chunked and len(request_table_entry.requested_chunks) == 0 \ and len(request_table_entry.requested_md) == 0: # all chunks are available data = request_table_entry.chunks #data = sorted(data, key=lambda content: content.name.to_string()) #broken with more than 10 chunks data = sorted( data, key=lambda content: int(''.join( filter(str.isdigit, content.name.string_components[-1])))) cont = self.chunkifyer.reassamble_data(request_table_entry.name, data) to_higher.put([faceid, cont]) return None else: return request_table_entry def get_chunk_list_from_chunk_table(self, data_names: Name) -> List[Content]: """get a list of content objects from a list of names""" res = [] for name in data_names: if name in self._chunk_table: res.append(self._chunk_table[name][0]) return res def get_request_table_entry(self, name: Name) -> RequestTableEntry: """check if a name is in the chunktable""" for entry in self._request_table: if entry.name == name or name in entry.requested_chunks or name in entry.requested_md: return entry return None def chunk_name_in_request_table(self, name): """check if a received chunk is expected by the requesttable""" for entry in self._request_table: if name in entry.requested_chunks: return True return False def remove_chunk_name_from_request_table_entry(self, request_table_entry: RequestTableEntry, name: Name)\ -> RequestTableEntry: """remove chunk from chunktable""" if name not in request_table_entry.requested_chunks: return request_table_entry request_table_entry.requested_chunks.remove(name) return request_table_entry def metadata_name_in_request_table(self, name): """check if a received metadata is expected by the chunktable""" for entry in self._request_table: if name in entry.requested_md: return True return False def remove_metadata_name_from_request_table(self, request_table_entry: RequestTableEntry, name: Name) \ -> RequestTableEntry: """remove metadata from chunktable""" if name not in request_table_entry.requested_md: return request_table_entry request_table_entry.requested_md.remove(name) return request_table_entry
class DataOffloadingChunklayer(LayerProcess): """This Chunklayer handles interrupted data uploads by asking neighbouring nodes for available content.""" def __init__(self, cs: BaseContentStore, pit: BasePendingInterestTable, fib: BaseForwardingInformationBase, chunkifyer: BaseChunkifyer = None, chunk_size: int = 4096, num_of_forwards: int = 1, prefix: str = "car", log_level: int = 255): super().__init__("ChunkLayer", log_level=log_level) self.cs = cs self.pit = pit self.fib = fib self.chunk_size = chunk_size if chunkifyer is None: self.chunkifyer = SimpleContentChunkifyer(chunk_size) else: self.chunkifyer: BaseChunkifyer = chunkifyer self.num_of_forwards = num_of_forwards self.prefix = prefix manager = multiprocessing.Manager() self._chunk_table: Dict[Name, (Content, float)] = manager.dict() self._request_table: List[RequestTableEntry] = manager.list() self._ca_table: Dict[Name, CaEntry] = manager.dict() self.recipient_cl: Dict[Name, Name] = manager.dict() self.pass_through = False def data_from_higher(self, to_lower: multiprocessing.Queue, to_higher: multiprocessing.Queue, data): self.logger.info("Got Data from higher") faceid = data[0] packet = data[1] if isinstance(packet, Interest): self.logger.info("Packet is Interest " + str(packet.name)) request_entry = self.get_request_entry(packet.name) if request_entry is None: self._request_table.append(RequestTableEntry(packet.name)) self._ca_table[packet.name] = CaEntry() # If the interest starts with /car and not ends with NFN, request metadata and available chunks from neighbours components = packet.name.string_components if self.prefix in components[ 0] and components[-1] != "NFN" and not request_entry: self.pass_through = False ca_entry = self._ca_table.get(packet.name) name1 = Name("/nL") + packet.name + f"CA{self.num_of_forwards}" name2 = Name("/nR") + packet.name + f"CA{self.num_of_forwards}" name3 = Name("/nL") + packet.name + f"CL{self.num_of_forwards}" name4 = Name("/nR") + packet.name + f"CL{self.num_of_forwards}" if not self.pit.find_pit_entry( name1) and self.fib.find_fib_entry(name1): ca_entry.answer_L = False ca_entry.received_all = False to_lower.put([faceid, Interest(name1)]) if not self.pit.find_pit_entry( name2) and self.fib.find_fib_entry(name2): ca_entry.answer_R = False ca_entry.received_all = False to_lower.put([faceid, Interest(name2)]) if not self.pit.find_pit_entry( name3) and self.fib.find_fib_entry(name3): to_lower.put([faceid, Interest(name3)]) if not self.pit.find_pit_entry( name4) and self.fib.find_fib_entry(name4): to_lower.put([faceid, Interest(name4)]) self._ca_table[packet.name] = ca_entry to_lower.put(data) return elif isinstance(packet, Content): self.logger.info("Packet is Content (name=%s, %d bytes)" % (str(packet.name), len(packet.content))) if len(packet.content) < self.chunk_size: to_lower.put(data) else: self.logger.info("Chunking Packet") metadata, chunks = self.chunkifyer.chunk_data( packet) # Create metadata and chunks self.logger.info("Metadata: " + metadata[0].content) to_lower.put([faceid, metadata[0] ]) # Return name of first metadata object for md in metadata: # Add metadata to chunktable if md.name not in self._chunk_table: self._chunk_table[md.name] = (md, time.time()) for c in chunks: # Add chunks to chunktable if c.name not in self._chunk_table: self._chunk_table[c.name] = (c, time.time()) elif isinstance(packet, Nack): request_entry = self.get_request_entry(packet.name) if request_entry is not None: self._request_table.remove(request_entry) to_lower.put(data) def data_from_lower(self, to_lower: multiprocessing.Queue, to_higher: multiprocessing.Queue, data): self.logger.info("Got Data from lower") faceid = data[0] packet = data[1] components = packet.name.components string_components = packet.name.string_components last = components[-1] if isinstance(packet, Interest): self.logger.info("Packet is Interest") # Interest for available chunks if "CA" in string_components[-1]: self.pass_through = True # This node doesn't handle CA content, only forwards to neighbours chunks_available = self.get_chunks_available(packet) if last == b"CA0" or isinstance( chunks_available, Content): # If there are chunks available, return them to_lower.put([faceid, chunks_available]) else: # Otherwise try to pass on interest to neighbour to_lower.put( [faceid, Interest(self.decrease_name(packet.name))]) # General Interest passed on to chunklayer elif "CL" in string_components[-1]: matching_content = self.get_matching_content_from_packed_name( packet) if last == b"CL0" or isinstance( matching_content, Content): # If there is matching content, return it to_lower.put([faceid, matching_content]) else: # Otherwise try to pass on to neighbour to_lower.put( [faceid, Interest(self.decrease_name(packet.name))]) elif packet.name in self._chunk_table: matching_content = self._chunk_table.get(packet.name)[0] to_lower.put([faceid, matching_content]) else: to_higher.put(data) elif isinstance(packet, Content): self.logger.info("Packet is Content") ca_content = False cl_content = False # Metadata or string containing available chunks from neighbour if "CA" in string_components[-1]: self.cs.remove_content_object( packet.name ) # Remove from cs, so next interest comes through to chunklayer if string_components[ -1] == f"CA{self.num_of_forwards}": # This is the requesting node --> unpack ca_content = True ca_entry = self._ca_table.get(self.unpack(packet.name)) # In order to be able to request the available chunks from the sender of this packet, # we need to safe the first component. # This is then used in pack_ca() to send packet to the correct recipient. ca_entry.recipient = Name(components[:1]) # Only replace existing list of available chunks if it contains more chunks self.save_if_longest(packet, ca_entry) if components[0] == b"nL": ca_entry.answer_L = True else: ca_entry.answer_R = True if ca_entry.answer_L and ca_entry.answer_R: # We have a response from both neighbours and can proceed packet = ca_entry.ca self._request_table.append( RequestTableEntry(packet.name)) self._ca_table[self.unpack(packet.name)] = ca_entry else: # This is not the requesting node --> pass on to neighbour to_lower.put([ faceid, Content(self.increase_name(packet.name), packet.content) ]) return # Content from the chunklayer of a neighbouring node elif "CL" in string_components[-1]: if string_components[ -1] == f"CL{self.num_of_forwards}": # This is the requesting node --> unpack cl_content = True packet.name = self.unpack(packet.name) # Save the sender of this packet as the recipient for further interests. Used in pack_cl() self.recipient_cl[packet.name] = Name(components[:1]) else: # This is not the requesting node --> pass on to neighbour to_lower.put([ faceid, Content(self.increase_name(packet.name), packet.content) ]) return request_entry = self.get_request_entry(packet.name) if request_entry is None: return self._request_table.remove(request_entry) if "CA" in components[-2].decode("utf-8"): if self.pass_through: return ca_content = True request_entry.chunked = True self.handle_content(faceid, packet, request_entry, ca_content, cl_content, to_lower, to_higher) elif isinstance(packet, Nack): if self.prefix not in string_components[0]: request_entry = self.get_request_entry(packet.name) if request_entry: self._request_table.remove(request_entry) if "CA" in string_components[-1]: if string_components[ -1] == f"CA{self.num_of_forwards}": # This is the requesting node unpacked = self.unpack(packet.name) ca_entry = self._ca_table.get(unpacked) if components[0] == b"nL": ca_entry.answer_L = True else: ca_entry.answer_R = True if ca_entry.answer_L and ca_entry.answer_R: # We have an answer from both neighbour if ca_entry.ca: # We have chunks available from one of the neighbours packet = ca_entry.ca self._ca_table[unpacked] = ca_entry self.handle_content( faceid, packet, RequestTableEntry(packet.name), True, False, to_lower, to_higher) return else: ca_entry.received_all = True request_entry = self.get_request_entry( unpacked) if request_entry and not request_entry.requested_md and not self.pass_through: self.create_chunk_interests( faceid, request_entry, ca_entry, to_lower) self._ca_table[unpacked] = ca_entry else: # This is not the requesting node --> pass on to neighbour name = self.increase_name(packet.name) nack = Nack(name, NackReason.NO_CONTENT, Interest(name)) to_lower.put([faceid, nack]) elif "CL" in string_components[-1]: if string_components[ -1] != f"CL{self.num_of_forwards}": # This is not the requesting node --> pass on to neighbour name = self.increase_name(packet.name) nack = Nack(name, NackReason.NO_CONTENT, Interest(name)) to_lower.put([faceid, nack]) else: to_higher.put([faceid, packet]) self.pit.remove_pit_entry(packet.name) else: if "c" in string_components[-1]: packet.name.components = components[:-1] to_higher.put([ faceid, Nack(packet.name, NackReason.NO_CONTENT, Interest(packet.name)) ]) else: #FIXME What to do here? # to_higher.put([faceid, packet]) pass def handle_content(self, faceid: int, packet: Content, request_entry: RequestTableEntry, ca_content: bool, cl_content: bool, to_lower: multiprocessing.Queue, to_higher: multiprocessing.Queue): """Handle incoming content""" if request_entry.chunked is False: # Not chunked content if not packet.get_bytes().startswith(b'mdo:'): if ca_content: self.handle_ca(faceid, packet, to_lower) else: to_higher.put([faceid, packet]) return else: # Received metadata data --> chunked content request_entry.chunked = True if packet.get_bytes().startswith( b'mdo:'): # Request all frames from meta data self.handle_received_meta_data(faceid, packet, request_entry, to_lower, ca_content, cl_content) else: self.handle_received_chunk_data(faceid, packet, request_entry, to_lower, to_higher, ca_content) def handle_received_meta_data(self, faceid: int, packet: Content, request_entry: RequestTableEntry, to_lower: multiprocessing.Queue, ca_content: bool, cl_content: bool): """Handle meta data""" if not request_entry.md_complete: if packet.name in request_entry.requested_md: request_entry.requested_md.remove(packet.name) md, chunks, size = self.chunkifyer.parse_meta_data(packet.content) for chunk in chunks: # Request all chunks from the meta data file if not already received or requested if chunk not in request_entry.requested_chunks and chunk not in [ i.name for i in request_entry.chunks ]: request_entry.requested_chunks.append(chunk) if md is not None: # There is another md file if md not in request_entry.requested_md: request_entry.requested_md.append(md) if cl_content: md = self.pack_cl(md) to_lower.put([faceid, Interest(md)]) else: # Only create interests if it is the requesting node handling this metadata and # either the packet is CA content or there is no CA content available request_entry.md_complete = True if not self.pass_through: if ca_content: self.create_chunk_interests(faceid, request_entry, CaEntry(), to_lower) elif self._ca_table.get( request_entry.name ).received_all: # We have an answer from both neighbours self.create_chunk_interests( faceid, request_entry, self._ca_table.get(request_entry.name), to_lower) self._chunk_table[packet.name] = (packet, time.time()) self._request_table.append(request_entry) def handle_received_chunk_data(self, faceid: int, packet: Content, request_entry: RequestTableEntry, to_lower: multiprocessing.Queue, to_higher: multiprocessing.Queue, ca_content: bool): """Handle chunk data""" if packet.name in request_entry.requested_chunks: request_entry.requested_chunks.remove(packet.name) request_entry.chunks.append(packet) self._chunk_table[packet.name] = (packet, time.time()) if not request_entry.requested_chunks: if not request_entry.requested_md: # All chunks are available data = request_entry.chunks data = sorted(data, key=lambda content: int(''.join( filter(str.isdigit, content.name. string_components[-1])))) cont = self.chunkifyer.reassamble_data(request_entry.name, data) if ca_content: self.handle_ca(faceid, cont, to_lower) else: del self._ca_table[request_entry.name] to_higher.put([faceid, cont]) return self._request_table.append(request_entry) def handle_ca(self, faceid: int, content: Content, to_lower: multiprocessing.Queue): """Unpack the received ca message and create interests for all available chunks.""" content.name = self.unpack(content.name) ca_entry = self._ca_table.get(content.name) ca_entry.received_all = True chunks_str = content.content.split(";") request_entry = self.get_request_entry(content.name) if request_entry and not self.pass_through: if chunks_str[0] == "complete": # Neighbour has complete data ca_entry.completely_available = True if request_entry.md_complete: self.create_chunk_interests(faceid, request_entry, ca_entry, to_lower) else: ca_entry.chunks = [Name(chunk) for chunk in chunks_str] # Create interests for all chunks that are available from neighbour for chunk in ca_entry.chunks: self.logger.info("TO NEIGHBOUR:", self.pack_ca(chunk, ca_entry)) if chunk not in request_entry.requested_chunks and chunk not in [ i.name for i in request_entry.chunks ]: request_entry.requested_chunks.append(chunk) to_lower.put( [faceid, Interest(self.pack_ca(chunk, ca_entry))]) self._request_table.append(request_entry) # If there is no name in requested_md try to request the remaining chunks. # This is only necessary to get a NACK, so the simulation continues. if not request_entry.requested_md: for chunk in [ i for i in request_entry.requested_chunks if i not in ca_entry.chunks ]: self.logger.info("TO ORIGINAL SOURCE:", chunk) to_lower.put([faceid, Interest(chunk)]) self._request_table.remove(request_entry) self._ca_table[content.name] = ca_entry def get_request_entry(self, name: Name): """ Check if a name is in the request table. Return entry or None. """ for entry in self._request_table: if entry.name == name or name in entry.requested_chunks or name in entry.requested_md: return entry return None def get_chunks_available(self, packet: Packet): """ Check if chunks are available for a given name. Return a content object containing the names of the available chunks, or NACK """ chunks_available = [] name = self.unpack(packet.name) request_entry = self.get_request_entry(name) cs_entry = self.cs.find_content_object(name) if request_entry is not None: chunks_available = [ str(chunk.name) for chunk in request_entry.chunks ] elif cs_entry: chunks_available.append("complete;") if chunks_available: chunks_available = Content(packet.name, ";".join(chunks_available)) if len(chunks_available.content) > self.chunk_size: meta_data, chunks = self.chunkifyer.chunk_data( chunks_available) meta_data.extend(chunks) for data in meta_data: self.cs.remove_content_object(data.name) self.cs.add_content_object(data) return meta_data[0] else: return chunks_available return Nack(packet.name, NackReason.NO_CONTENT, packet) def get_matching_content_from_packed_name(self, packet: Packet): """Return either the content matching the unpacked name or NACK""" name_in = self.unpack(packet.name) if name_in in self._chunk_table: matching_content = self._chunk_table.get(name_in)[0] matching_content.name = packet.name return matching_content else: return Nack(packet.name, NackReason.NO_CONTENT, packet) def create_chunk_interests(self, faceid: int, request_entry: RequestTableEntry, ca_entry: CaEntry, to_lower: multiprocessing.Queue): """Create interests for all the chunks in requested_md of the specified request table entry.""" if ca_entry.completely_available: for chunk in [i for i in request_entry.requested_chunks]: self.logger.info("TO NEIGHBOUR:", self.pack_ca(chunk, ca_entry)) if chunk not in request_entry.requested_chunks and chunk not in [ i.name for i in request_entry.chunks ]: request_entry.requested_chunks.append(chunk) to_lower.put([faceid, Interest(self.pack_ca(chunk, ca_entry))]) else: for chunk in [ i for i in request_entry.requested_chunks if i not in ca_entry.chunks ]: self.logger.info("TO ORIGINAL SOURCE:", chunk) to_lower.put([faceid, Interest(chunk)]) def save_if_longest(self, packet: Content, ca_entry: CaEntry): """ Check if the received ca content is longer than the existing one and if so, replace it. In the case where both neighbours have chunks available, we want to send the interests only to the one which has more. """ if packet.get_bytes().startswith( b'mdo:'): # Content is metadata, read size from metadata _, _, content_size = self.chunkifyer.parse_meta_data( packet.content) else: # Content is string, size equals length of the string content_size = (len(packet.content)) content_size = int(content_size) if content_size > ca_entry.size: ca_entry.ca = packet ca_entry.size = content_size def pack_ca(self, name: Name, ca_entry: CaEntry) -> Name: """Prepend the recipient, append "CL" and the number of forwards.""" return ca_entry.recipient + name + f"CL{self.num_of_forwards}" def pack_cl(self, name: Name) -> Name: """Prepend the recipient, append "CA" and the number of forwards.""" lookup_name = Name(name.components[:-1]) return self.recipient_cl.get( lookup_name) + name + f"CL{self.num_of_forwards}" def unpack(self, name: Name) -> Name: return Name(name.components[1:-1]) def increase_name(self, name: Name) -> Name: """ Increase the number at the end of the name. The number is used to determine whether or not a packet gets forwarded to the next neighbour. """ components = name.components last = components[-1].decode("utf-8") i = int(''.join(filter(str.isdigit, last))) if "CA" in last: return Name(components[:-1]) + f"CA{i+1}" elif "CL" in last: return Name(components[:-1]) + f"CL{i+1}" return name def decrease_name(self, name: Name) -> Name: """ Decrease the number at the end of the name. The number is used to determine whether or not a packet gets forwarded to the next neighbour. """ components = name.components last = components[-1].decode("utf-8") i = int(''.join(filter(str.isdigit, last))) if "CA" in last: return Name(components[:-1]) + f"CA{i-1}" elif "CL" in last: return Name(components[:-1]) + f"CL{i-1}" return name def set_number_of_forwards(self, number_of_forwards: int): self.num_of_forwards = number_of_forwards
class DataOffloadingChunklayerSimple(LayerProcess): """This Chunklayer handles interrupted data uploads by asking neighbouring nodes for available content.""" def __init__(self, cs: BaseContentStore, pit: BasePendingInterestTable, fib: BaseForwardingInformationBase, chunkifyer: BaseChunkifyer = None, chunk_size: int = 4096, num_of_forwards: int = 1, prefix: str = "car", log_level: int = 255): super().__init__("ChunkLayer", log_level=log_level) self.cs = cs self.pit = pit self.fib = fib self.chunk_size = chunk_size if chunkifyer is None: self.chunkifyer = SimpleContentChunkifyer(chunk_size) else: self.chunkifyer: BaseChunkifyer = chunkifyer self.num_of_forwards = num_of_forwards self.prefix = prefix manager = multiprocessing.Manager() self._chunk_table: Dict[Name, (Content, float)] = manager.dict() self._request_table: List[RequestTableEntry] = manager.list() self._cl_table: Dict[Name, ClEntry] = manager.dict() self.pass_through = False self.cl_sent = False def data_from_higher(self, to_lower: multiprocessing.Queue, to_higher: multiprocessing.Queue, data): self.logger.info("Got Data from higher") faceid = data[0] packet = data[1] if isinstance(packet, Interest): self.logger.info("Packet is Interest " + str(packet.name)) request_entry = self.get_request_entry(packet.name) if request_entry is None: self._request_table.append(RequestTableEntry(packet.name)) # If the interest starts with "/car" and not ends with "NFN", request metadata and available chunks from neighbours components = packet.name.string_components if self.prefix in components[ 0] and components[-1] != "NFN" and not request_entry: self.pass_through = False self._cl_table[packet.name] = ClEntry(data) cl_entry = self._cl_table.get(packet.name) name1 = Name("/nL") + packet.name + f"CL{self.num_of_forwards}" name2 = Name("/nR") + packet.name + f"CL{self.num_of_forwards}" if not self.pit.find_pit_entry( name1) and self.fib.find_fib_entry(name1): cl_entry.nack_L = False self.cl_sent = True to_lower.put([faceid, Interest(name1)]) if not self.pit.find_pit_entry( name2) and self.fib.find_fib_entry(name2): cl_entry.nack_R = False self.cl_sent = True to_lower.put([faceid, Interest(name2)]) if self.cl_sent: self._cl_table[packet.name] = cl_entry return to_lower.put(data) return elif isinstance(packet, Content): self.logger.info("Packet is Content (name=%s, %d bytes)" % (str(packet.name), len(packet.content))) if len(packet.content) < self.chunk_size: to_lower.put(data) else: self.logger.info("Chunking Packet") metadata, chunks = self.chunkifyer.chunk_data( packet) # Create metadata and chunks self.logger.info("Metadata: " + metadata[0].content) to_lower.put([faceid, metadata[0] ]) # Return name of first metadata object for md in metadata: # Add metadata to chunktable if md.name not in self._chunk_table: self._chunk_table[md.name] = (md, time.time()) for c in chunks: # Add chunks to chunktable if c.name not in self._chunk_table: self._chunk_table[c.name] = (c, time.time()) elif isinstance(packet, Nack): request_entry = self.get_request_entry(packet.name) if request_entry is not None: self._request_table.remove(request_entry) to_lower.put(data) def data_from_lower(self, to_lower: multiprocessing.Queue, to_higher: multiprocessing.Queue, data): self.logger.info("Got Data from lower") faceid = data[0] packet = data[1] components = packet.name.components string_components = packet.name.string_components last = components[-1] if isinstance(packet, Interest): self.logger.info("Packet is Interest") # General Interest passed on to chunklayer if "CL" in string_components[-1]: matching_content = self.get_matching_content(packet) if last == b"CL0" or isinstance( matching_content, Content): # If there is matching content, return it to_lower.put([faceid, matching_content]) else: # Otherwise try to pass on to neighbour to_lower.put( [faceid, Interest(self.decrease_name(packet.name))]) elif packet.name in self._chunk_table: matching_content = self._chunk_table.get(packet.name)[0] to_lower.put([faceid, matching_content]) else: to_higher.put(data) elif isinstance(packet, Content): self.logger.info("Packet is Content") cl_content = False # Content from the chunklayer of a neighbouring node if "CL" in string_components[-1]: if string_components[ -1] == f"CL{self.num_of_forwards}": # This is the requesting node --> unpack cl_content = True packet.name = self.unpack(packet.name) # Save the sender of this packet as the recipient for further interests. Used in pack_cl() request_entry = self.get_request_entry(packet.name) if request_entry: cl_entry = self._cl_table.get(request_entry.name) if cl_entry.interest_requested: # If we already resent requests to source, don't consider it return cl_entry.recipient = Name(components[:1]) self._cl_table[request_entry.name] = cl_entry else: # This is not the requesting node --> pass on to neighbour to_lower.put([ faceid, Content(self.increase_name(packet.name), packet.content) ]) return request_entry = self.get_request_entry(packet.name) if request_entry is None: return self.handle_content(faceid, packet, request_entry, cl_content, to_lower, to_higher) elif isinstance(packet, Nack): if self.prefix not in string_components[0]: request_entry = self.get_request_entry(packet.name) if request_entry: self._request_table.remove(request_entry) if "CL" in string_components[-1]: if string_components[ -1] == f"CL{self.num_of_forwards}": # This is the requesting node --> unpack name_unpacked = self.unpack(packet.name) request_entry = self.get_request_entry(name_unpacked) if request_entry: cl_entry = self._cl_table.get(request_entry.name) if components[0] == b"nL": cl_entry.nack_L = True else: cl_entry.nack_R = True if cl_entry.nack_L and cl_entry.nack_R and not cl_entry.interest_requested: # No more data available from neighbours, get it from car self.get_missing_data_from_original_source( faceid, request_entry, cl_entry, to_lower) self._cl_table[request_entry.name] = cl_entry else: name = self.increase_name(packet.name) nack = Nack(name, NackReason.NO_CONTENT, Interest(name)) to_lower.put([faceid, nack]) else: to_higher.put([faceid, packet]) self.pit.remove_pit_entry(packet.name) else: if "c" in string_components[-1] or "m" in string_components[-1]: packet.name.components = components[:-1] to_higher.put([ faceid, Nack(packet.name, NackReason.NO_CONTENT, Interest(packet.name)) ]) else: pass # to_higher.put([faceid, packet]) def handle_content(self, faceid: int, packet: Content, request_entry: RequestTableEntry, cl_content: bool, to_lower: multiprocessing.Queue, to_higher: multiprocessing.Queue): """Handle incoming content""" self._request_table.remove(request_entry) if request_entry.chunked is False: # Not chunked content if not packet.get_bytes().startswith(b'mdo:'): to_higher.put([faceid, packet]) return else: # Received metadata data --> chunked content request_entry.chunked = True if packet.get_bytes().startswith( b'mdo:'): # Request all frames from metadata self.handle_received_meta_data(faceid, packet, request_entry, to_lower, cl_content) else: self.handle_received_chunk_data(faceid, packet, request_entry, to_higher) def handle_received_meta_data(self, faceid: int, packet: Content, request_entry: RequestTableEntry, to_lower: multiprocessing.Queue, cl_content: bool): """Handle meta data""" if packet.name in request_entry.requested_md: request_entry.requested_md.remove(packet.name) md, chunks, size = self.chunkifyer.parse_meta_data(packet.content) for chunk in chunks: # Request all chunks from the metadata file if not already received or requested # if chunk not in request_entry.requested_chunks and chunk not in [i.name for i in request_entry.chunks]: if chunk not in [i.name for i in request_entry.chunks]: request_entry.requested_chunks.append(chunk) if not self.pass_through: if cl_content: cl_entry = self._cl_table.get(request_entry.name) if cl_entry.nack_L and cl_entry.nack_R: break chunk = self.pack_cl(chunk) to_lower.put([faceid, Interest(chunk)]) if md is not None: # There is another md file if md not in request_entry.requested_md: request_entry.requested_md.append(md) if not self.pass_through: if cl_content: cl_entry = self._cl_table.get(request_entry.name) if not (cl_entry.nack_L and cl_entry.nack_R): cl_entry.interest = [faceid, Interest(packet.name)] self._cl_table[request_entry.name] = cl_entry md = self.pack_cl(md) to_lower.put([faceid, Interest(md)]) else: to_lower.put([faceid, Interest(md)]) else: request_entry.last_chunk = chunks[-1] self._chunk_table[packet.name] = (packet, time.time()) self._request_table.append(request_entry) def handle_received_chunk_data(self, faceid: int, packet: Content, request_entry: RequestTableEntry, to_higher: multiprocessing.Queue): """Handle chunk data""" if packet.name in request_entry.requested_chunks: request_entry.requested_chunks.remove(packet.name) request_entry.chunks.append(packet) self._chunk_table[packet.name] = (packet, time.time()) if not request_entry.requested_chunks and not request_entry.requested_md: if not request_entry.requested_md: # All chunks are available data = request_entry.chunks data = sorted(data, key=lambda content: int(''.join( filter(str.isdigit, content.name. string_components[-1])))) cont = self.chunkifyer.reassamble_data(request_entry.name, data) to_higher.put([faceid, cont]) return self._request_table.append(request_entry) def get_request_entry(self, name: Name): """ Check if a name is in the request table. Return entry or None. """ for entry in self._request_table: if entry.name == name or name in entry.requested_chunks or name in entry.requested_md: return entry return None def get_matching_content(self, packet: Packet): """Return either the content matching the packet name or NACK""" name_in = self.unpack(packet.name) cs_entry = self.cs.find_content_object(name_in) if name_in in self._chunk_table: matching_content = self._chunk_table.get(name_in)[0] matching_content.name = packet.name return matching_content elif cs_entry: matching_content = cs_entry.content matching_content.name = packet.name return matching_content else: return Nack(packet.name, NackReason.NO_CONTENT, packet) def pack_cl(self, name: Name) -> Name: """Prepend the recipient, append "CL" and the number of forwards.""" lookup_name = Name(name.components[:-1]) return self._cl_table.get( lookup_name).recipient + name + f"CL{self.num_of_forwards}" def unpack(self, name: Name) -> Name: return Name(name.components[1:-1]) def increase_name(self, name: Name) -> Name: """ Increase the number at the end of the name. The number is used to determine whether or not a packet gets forwarded to the next neighbour. """ components = name.components last = components[-1].decode("utf-8") i = int(''.join(filter(str.isdigit, last))) if "CL" in last: return Name(components[:-1]) + f"CL{i+1}" return name def decrease_name(self, name: Name) -> Name: """ Decrease the number at the end of the name. The number is used to determine whether or not a packet gets forwarded to the next neighbour. """ components = name.components last = components[-1].decode("utf-8") i = int(''.join(filter(str.isdigit, last))) if "CL" in last: return Name(components[:-1]) + f"CL{i-1}" return name def get_missing_data_from_original_source(self, faceid: int, request_entry: RequestTableEntry, cl_entry: ClEntry, to_lower: multiprocessing.Queue): """ Start requesting the missing files from the original source. """ if not cl_entry.interest_requested: if request_entry.requested_chunks: # Request again all chunks that have been requested but not satisfied yet for chunk in request_entry.requested_chunks: to_lower.put([faceid, Interest(chunk)]) # If requested_md is not empty, request them again from source if request_entry.requested_md: for md in request_entry.requested_md: to_lower.put([faceid, Interest(md)]) else: # if empty, request orginal interest from source to_lower.put(cl_entry.interest) cl_entry.interest_requested = True def set_number_of_forwards(self, number_of_forwards: int): self.num_of_forwards = number_of_forwards