Exemplo n.º 1
0
class test_SimpleContentChunkifyer(unittest.TestCase):
    def setUp(self):
        self.chunkifyer = SimpleContentChunkifyer()

    def tearDown(self):
        pass

    def test_generate_metadata_no_next(self):
        """Test generating a simple metadata object"""
        name = Name("/test/data")

        res = self.chunkifyer.generate_meta_data(2, 4, 0, 0, name)

        self.assertEqual(res.name.to_string(), "/test/data")
        self.assertEqual(res.content, "mdo:/test/data/c2;/test/data/c3:")

    def test_generate_metadata_one_next(self):
        """Test generating a simple metadata object with one following"""
        name = Name("/test/data")

        res = self.chunkifyer.generate_meta_data(2, 4, 0, 1, name)

        self.assertEqual(res.name.to_string(), "/test/data")
        self.assertEqual(res.content,
                         "mdo:/test/data/c2;/test/data/c3:/test/data/m1")

    def test_generate_metadata_two_next(self):
        """Test generating a simple metadata object with two following"""
        name = Name("/test/data")

        res = self.chunkifyer.generate_meta_data(2, 4, 1, 2, name)

        self.assertEqual(res.name.to_string(), "/test/data/m1")
        self.assertEqual(res.content,
                         "mdo:/test/data/c2;/test/data/c3:/test/data/m2")

    def test_chunk_single_metadata(self):
        name = Name("/test/data")
        string = "A" * 4096 + "B" * 4096 + "C" * 4096
        content = Content(name, string)

        md, content = self.chunkifyer.chunk_data(content)

        md_name_comp = ['/test/data']
        md_data_comp = ['mdo:/test/data/c0;/test/data/c1;/test/data/c2:']

        content_name_comp = ['/test/data/c0', '/test/data/c1', '/test/data/c2']

        content_data_comp = ["A" * 4096, "B" * 4096, "C" * 4096]

        for i in range(0, len(md)):
            self.assertEqual(md[i].name.to_string(), md_name_comp[i])
            self.assertEqual(md[i].content, md_data_comp[i])

        for i in range(0, len(content)):
            self.assertEqual(content[i].name.to_string(), content_name_comp[i])
            self.assertEqual(content[i].content, content_data_comp[i])

    def test_chunk_multiple_metadata(self):
        """Test chunking metadata with three metadata objects and 10 chunks"""
        name = Name("/test/data")
        string = "A"*4096 + "B"*4096 + "C"*4096 + "D"*4096 + "E"*4096 + "F"*4096 + "G"*4096 + "H"*4096 \
                 + "I"*4096 + "J"*4000
        content = Content(name, string)

        md, chunked_content = self.chunkifyer.chunk_data(content)

        md_name_comp = ['/test/data', '/test/data/m1', '/test/data/m2']
        md_data_comp = [
            'mdo:/test/data/c0;/test/data/c1;/test/data/c2;/test/data/c3:/test/data/m1',
            'mdo:/test/data/c4;/test/data/c5;/test/data/c6;/test/data/c7:/test/data/m2',
            'mdo:/test/data/c8;/test/data/c9:'
        ]

        content_name_comp = [
            '/test/data/c0', '/test/data/c1', '/test/data/c2', '/test/data/c3',
            '/test/data/c4', '/test/data/c5', '/test/data/c6', '/test/data/c7',
            '/test/data/c8', '/test/data/c9'
        ]

        content_data_comp = [
            "A" * 4096, "B" * 4096, "C" * 4096, "D" * 4096, "E" * 4096,
            "F" * 4096, "G" * 4096, "H" * 4096, "I" * 4096, "J" * 4000
        ]

        for i in range(0, len(md)):
            self.assertEqual(md[i].name.to_string(), md_name_comp[i])
            self.assertEqual(md[i].content, md_data_comp[i])

        for i in range(0, len(chunked_content)):
            self.assertEqual(chunked_content[i].name.to_string(),
                             content_name_comp[i])
            self.assertEqual(chunked_content[i].content, content_data_comp[i])

    def test_chunk_multiple_metadata_reassemble(self):
        """Test chunking metadata with three metadata objects and 10 chunks and reassemble"""
        name = Name("/test/data")
        string = "A" * 4096 + "B" * 4096 + "C" * 4096 + "D" * 4096 + "E" * 4096 + "F" * 4096 + "G" * 4096 + "H" * 4096 \
                 + "I" * 4096 + "J" * 4000
        content = Content(name, string)

        md, chunked_content = self.chunkifyer.chunk_data(content)

        md_name_comp = ['/test/data', '/test/data/m1', '/test/data/m2']
        md_data_comp = [
            'mdo:/test/data/c0;/test/data/c1;/test/data/c2;/test/data/c3:/test/data/m1',
            'mdo:/test/data/c4;/test/data/c5;/test/data/c6;/test/data/c7:/test/data/m2',
            'mdo:/test/data/c8;/test/data/c9:'
        ]

        content_name_comp = [
            '/test/data/c0', '/test/data/c1', '/test/data/c2', '/test/data/c3',
            '/test/data/c4', '/test/data/c5', '/test/data/c6', '/test/data/c7',
            '/test/data/c8', '/test/data/c9'
        ]

        content_data_comp = [
            "A" * 4096, "B" * 4096, "C" * 4096, "D" * 4096, "E" * 4096,
            "F" * 4096, "G" * 4096, "H" * 4096, "I" * 4096, "J" * 4000
        ]

        for i in range(0, len(md)):
            self.assertEqual(md[i].name.to_string(), md_name_comp[i])
            self.assertEqual(md[i].content, md_data_comp[i])

        for i in range(0, len(chunked_content)):
            self.assertEqual(chunked_content[i].name.to_string(),
                             content_name_comp[i])
            self.assertEqual(chunked_content[i].content, content_data_comp[i])

        reassembled_content = self.chunkifyer.reassamble_data(
            md[0].name, chunked_content)
        self.assertEqual(content, reassembled_content)

    def test_parse_metadata_next(self):
        """Test parse metadata with next metadata"""
        md, names = self.chunkifyer.parse_meta_data(
            "mdo:/test/data/c0;/test/data/c1;/test/data/c2;/test/data/c3:/test/data/m1"
        )

        self.assertEqual(Name("/test/data/m1"), md)
        names_comp = [
            Name("/test/data/c0"),
            Name("/test/data/c1"),
            Name("/test/data/c2"),
            Name("/test/data/c3")
        ]
        self.assertEqual(names, names_comp)

    def test_parse_metadata(self):
        """Test parse metadata"""
        md, names = self.chunkifyer.parse_meta_data(
            "mdo:/test/data/c0;/test/data/c1;/test/data/c2;/test/data/c3:")

        self.assertEqual(None, md)
        names_comp = [
            Name("/test/data/c0"),
            Name("/test/data/c1"),
            Name("/test/data/c2"),
            Name("/test/data/c3")
        ]
        self.assertEqual(names, names_comp)
Exemplo n.º 2
0
class BasicChunkLayer(LayerProcess):
    """"Basic Chunking Layer for PICN"""
    def __init__(self,
                 chunkifyer: BaseChunkifyer = None,
                 chunk_size: int = 4096,
                 manager: multiprocessing.Manager = None,
                 log_level=255):
        super().__init__("ChunkLayer", log_level=log_level)
        self.chunk_size = chunk_size
        if chunkifyer == None:
            self.chunkifyer = SimpleContentChunkifyer(chunk_size)
        else:
            self.chunkifyer: BaseChunkifyer = chunkifyer
        if manager is None:
            manager = multiprocessing.Manager()
        self._chunk_table: Dict[Name, (Content, float)] = manager.dict()
        self._request_table: List[RequestTableEntry] = manager.list()

    def data_from_higher(self, to_lower: multiprocessing.Queue,
                         to_higher: multiprocessing.Queue, data):
        self.logger.info("Got Data from higher")
        faceid = data[0]
        packet = data[1]
        if isinstance(packet, Interest):
            self.logger.info("Packet is Interest " + str(packet.name))
            requestentry = self.get_request_table_entry(packet.name)
            if requestentry is None:
                self._request_table.append(RequestTableEntry(packet.name))
            to_lower.put([faceid, packet])
            return
        if isinstance(packet, Content):
            self.logger.info("Packet is Content (name=%s, %d bytes)" % \
                                      (str(packet.name), len(packet.content)))
            if len(packet.content) < self.chunk_size:
                to_lower.put([faceid, packet])
            else:
                self.logger.info("Chunking Packet")
                metadata, chunks = self.chunkifyer.chunk_data(
                    packet)  #create metadata and chunks
                self.logger.info("Metadata: " + metadata[0].content)
                to_lower.put(
                    [faceid, metadata[0]]
                )  #return first name TODO HANDLE THE CASE, WHERE CHUNKS CAN TIMEOUT AND MUST BE REPRODUCED
                for md in metadata:  #add metadata to chunktable
                    if md.name not in self._chunk_table:
                        self._chunk_table[md.name] = (md, time.time())
                for c in chunks:  #add chunks to chunktable
                    if c.name not in self._chunk_table:
                        self._chunk_table[c.name] = (c, time.time())
        if isinstance(packet, Nack):
            requestentry = self.get_request_table_entry(packet.name)
            if requestentry is not None:
                self._request_table.remove(requestentry)
            to_lower.put([faceid, packet])

    def data_from_lower(self, to_lower: multiprocessing.Queue,
                        to_higher: multiprocessing.Queue, data):
        self.logger.info("Got Data from lower")
        faceid = data[0]
        packet = data[1]
        if isinstance(packet, Interest):
            self.logger.info("Packet is Interest")
            if packet.name in self._chunk_table:  #Check if Interest is in chunktable
                matching_content = self._chunk_table.get(packet.name)[0]
                to_lower.put([faceid, matching_content])
            else:
                to_higher.put([faceid, packet])
            return
        if isinstance(packet, Content):
            self.logger.info("Packet is Content")
            request_table_entry = self.get_request_table_entry(packet.name)
            if request_table_entry is None:
                return
            self._request_table.remove(request_table_entry)
            if request_table_entry.chunked is False:  #not chunked content
                if not packet.get_bytes().startswith(b'mdo:'):
                    to_higher.put([faceid, packet])
                    return
                else:  # Received metadata data --> chunked content
                    request_table_entry.chunked = True
            if packet.get_bytes().startswith(
                    b'mdo:'):  # request all frames from metadata
                request_table_entry = self.handle_received_meta_data(
                    faceid, packet, request_table_entry, to_lower)
            else:
                request_table_entry = self.handle_received_chunk_data(
                    faceid, packet, request_table_entry, to_higher)
                if request_table_entry is None:
                    return  #deletes entry if data was completed
            self._request_table.append(request_table_entry)
        if isinstance(packet, Nack):
            requestentry = self.get_request_table_entry(packet.name)
            if requestentry is not None:
                self._request_table.remove(requestentry)
            to_higher.put([faceid, packet])

    def handle_received_meta_data(
            self, faceid: int, packet: Content,
            request_table_entry: RequestTableEntry,
            to_lower: multiprocessing.Queue) -> RequestTableEntry:
        """Handle the case, where metadata are received from the network"""
        md_entry = self.metadata_name_in_request_table(packet.name)
        if md_entry is None:
            return request_table_entry
        request_table_entry = self.remove_metadata_name_from_request_table(
            request_table_entry, packet.name)
        md, chunks, size = self.chunkifyer.parse_meta_data(packet.content)
        if md is not None:  # there is another md file
            request_table_entry.requested_md.append(md)
            to_lower.put([faceid, Interest(md)])
        else:
            request_table_entry.lastchunk = chunks[-1]
        for chunk in chunks:  # request all chunks from the metadata file
            request_table_entry.requested_chunks.append(chunk)
            to_lower.put([faceid, Interest(chunk)])
        self._chunk_table[packet.name] = (packet, time.time())
        return request_table_entry

    def handle_received_chunk_data(
            self, faceid: int, packet: Content,
            request_table_entry: RequestTableEntry,
            to_higher: multiprocessing.Queue) -> RequestTableEntry:
        """Handle the case wehere chunk data are received """
        chunk_entry = self.chunk_name_in_request_table(packet.name)
        if chunk_entry is None:
            return request_table_entry
        request_table_entry.chunks.append(packet)
        request_table_entry = self.remove_chunk_name_from_request_table_entry(
            request_table_entry, packet.name)
        self._chunk_table[packet.name] = (packet, time.time())
        if request_table_entry.chunked and len(request_table_entry.requested_chunks) == 0 \
                and len(request_table_entry.requested_md) == 0:  # all chunks are available
            data = request_table_entry.chunks
            #data = sorted(data, key=lambda content: content.name.to_string()) #broken with more than 10 chunks
            data = sorted(
                data,
                key=lambda content: int(''.join(
                    filter(str.isdigit, content.name.string_components[-1]))))
            cont = self.chunkifyer.reassamble_data(request_table_entry.name,
                                                   data)
            to_higher.put([faceid, cont])
            return None
        else:
            return request_table_entry

    def get_chunk_list_from_chunk_table(self,
                                        data_names: Name) -> List[Content]:
        """get a list of content objects from a list of names"""
        res = []
        for name in data_names:
            if name in self._chunk_table:
                res.append(self._chunk_table[name][0])
        return res

    def get_request_table_entry(self, name: Name) -> RequestTableEntry:
        """check if a name is in the chunktable"""
        for entry in self._request_table:
            if entry.name == name or name in entry.requested_chunks or name in entry.requested_md:
                return entry

        return None

    def chunk_name_in_request_table(self, name):
        """check if a received chunk is expected by the requesttable"""
        for entry in self._request_table:
            if name in entry.requested_chunks:
                return True
        return False

    def remove_chunk_name_from_request_table_entry(self, request_table_entry: RequestTableEntry, name: Name)\
            -> RequestTableEntry:
        """remove chunk from chunktable"""
        if name not in request_table_entry.requested_chunks:
            return request_table_entry
        request_table_entry.requested_chunks.remove(name)
        return request_table_entry

    def metadata_name_in_request_table(self, name):
        """check if a received metadata is expected by the chunktable"""

        for entry in self._request_table:
            if name in entry.requested_md:
                return True
        return False

    def remove_metadata_name_from_request_table(self, request_table_entry: RequestTableEntry, name: Name) \
            -> RequestTableEntry:
        """remove metadata from chunktable"""
        if name not in request_table_entry.requested_md:
            return request_table_entry
        request_table_entry.requested_md.remove(name)
        return request_table_entry
Exemplo n.º 3
0
class DataOffloadingChunklayer(LayerProcess):
    """This Chunklayer handles interrupted data uploads by asking neighbouring nodes for available content."""
    def __init__(self,
                 cs: BaseContentStore,
                 pit: BasePendingInterestTable,
                 fib: BaseForwardingInformationBase,
                 chunkifyer: BaseChunkifyer = None,
                 chunk_size: int = 4096,
                 num_of_forwards: int = 1,
                 prefix: str = "car",
                 log_level: int = 255):
        super().__init__("ChunkLayer", log_level=log_level)
        self.cs = cs
        self.pit = pit
        self.fib = fib
        self.chunk_size = chunk_size
        if chunkifyer is None:
            self.chunkifyer = SimpleContentChunkifyer(chunk_size)
        else:
            self.chunkifyer: BaseChunkifyer = chunkifyer
        self.num_of_forwards = num_of_forwards
        self.prefix = prefix

        manager = multiprocessing.Manager()
        self._chunk_table: Dict[Name, (Content, float)] = manager.dict()
        self._request_table: List[RequestTableEntry] = manager.list()
        self._ca_table: Dict[Name, CaEntry] = manager.dict()
        self.recipient_cl: Dict[Name, Name] = manager.dict()
        self.pass_through = False

    def data_from_higher(self, to_lower: multiprocessing.Queue,
                         to_higher: multiprocessing.Queue, data):
        self.logger.info("Got Data from higher")
        faceid = data[0]
        packet = data[1]

        if isinstance(packet, Interest):
            self.logger.info("Packet is Interest " + str(packet.name))
            request_entry = self.get_request_entry(packet.name)
            if request_entry is None:
                self._request_table.append(RequestTableEntry(packet.name))
                self._ca_table[packet.name] = CaEntry()

            # If the interest starts with /car and not ends with NFN, request metadata and available chunks from neighbours
            components = packet.name.string_components
            if self.prefix in components[
                    0] and components[-1] != "NFN" and not request_entry:
                self.pass_through = False
                ca_entry = self._ca_table.get(packet.name)
                name1 = Name("/nL") + packet.name + f"CA{self.num_of_forwards}"
                name2 = Name("/nR") + packet.name + f"CA{self.num_of_forwards}"
                name3 = Name("/nL") + packet.name + f"CL{self.num_of_forwards}"
                name4 = Name("/nR") + packet.name + f"CL{self.num_of_forwards}"
                if not self.pit.find_pit_entry(
                        name1) and self.fib.find_fib_entry(name1):
                    ca_entry.answer_L = False
                    ca_entry.received_all = False
                    to_lower.put([faceid, Interest(name1)])
                if not self.pit.find_pit_entry(
                        name2) and self.fib.find_fib_entry(name2):
                    ca_entry.answer_R = False
                    ca_entry.received_all = False
                    to_lower.put([faceid, Interest(name2)])
                if not self.pit.find_pit_entry(
                        name3) and self.fib.find_fib_entry(name3):
                    to_lower.put([faceid, Interest(name3)])
                if not self.pit.find_pit_entry(
                        name4) and self.fib.find_fib_entry(name4):
                    to_lower.put([faceid, Interest(name4)])
                self._ca_table[packet.name] = ca_entry

            to_lower.put(data)
            return

        elif isinstance(packet, Content):
            self.logger.info("Packet is Content (name=%s, %d bytes)" %
                             (str(packet.name), len(packet.content)))
            if len(packet.content) < self.chunk_size:
                to_lower.put(data)
            else:
                self.logger.info("Chunking Packet")
                metadata, chunks = self.chunkifyer.chunk_data(
                    packet)  # Create metadata and chunks
                self.logger.info("Metadata: " + metadata[0].content)
                to_lower.put([faceid, metadata[0]
                              ])  # Return name of first metadata object
                for md in metadata:  # Add metadata to chunktable
                    if md.name not in self._chunk_table:
                        self._chunk_table[md.name] = (md, time.time())
                for c in chunks:  # Add chunks to chunktable
                    if c.name not in self._chunk_table:
                        self._chunk_table[c.name] = (c, time.time())

        elif isinstance(packet, Nack):
            request_entry = self.get_request_entry(packet.name)
            if request_entry is not None:
                self._request_table.remove(request_entry)
            to_lower.put(data)

    def data_from_lower(self, to_lower: multiprocessing.Queue,
                        to_higher: multiprocessing.Queue, data):
        self.logger.info("Got Data from lower")
        faceid = data[0]
        packet = data[1]
        components = packet.name.components
        string_components = packet.name.string_components
        last = components[-1]

        if isinstance(packet, Interest):
            self.logger.info("Packet is Interest")

            # Interest for available chunks
            if "CA" in string_components[-1]:
                self.pass_through = True  # This node doesn't handle CA content, only forwards to neighbours
                chunks_available = self.get_chunks_available(packet)
                if last == b"CA0" or isinstance(
                        chunks_available,
                        Content):  # If there are chunks available, return them
                    to_lower.put([faceid, chunks_available])
                else:  # Otherwise try to pass on interest to neighbour
                    to_lower.put(
                        [faceid,
                         Interest(self.decrease_name(packet.name))])

            # General Interest passed on to chunklayer
            elif "CL" in string_components[-1]:
                matching_content = self.get_matching_content_from_packed_name(
                    packet)
                if last == b"CL0" or isinstance(
                        matching_content,
                        Content):  # If there is matching content, return it
                    to_lower.put([faceid, matching_content])
                else:  # Otherwise try to pass on to neighbour
                    to_lower.put(
                        [faceid,
                         Interest(self.decrease_name(packet.name))])

            elif packet.name in self._chunk_table:
                matching_content = self._chunk_table.get(packet.name)[0]
                to_lower.put([faceid, matching_content])
            else:
                to_higher.put(data)

        elif isinstance(packet, Content):
            self.logger.info("Packet is Content")
            ca_content = False
            cl_content = False

            # Metadata or string containing available chunks from neighbour
            if "CA" in string_components[-1]:
                self.cs.remove_content_object(
                    packet.name
                )  # Remove from cs, so next interest comes through to chunklayer
                if string_components[
                        -1] == f"CA{self.num_of_forwards}":  # This is the requesting node --> unpack
                    ca_content = True
                    ca_entry = self._ca_table.get(self.unpack(packet.name))
                    # In order to be able to request the available chunks from the sender of this packet,
                    # we need to safe the first component.
                    # This is then used in pack_ca() to send packet to the correct recipient.
                    ca_entry.recipient = Name(components[:1])
                    # Only replace existing list of available chunks if it contains more chunks
                    self.save_if_longest(packet, ca_entry)
                    if components[0] == b"nL":
                        ca_entry.answer_L = True
                    else:
                        ca_entry.answer_R = True
                    if ca_entry.answer_L and ca_entry.answer_R:  # We have a response from both neighbours and can proceed
                        packet = ca_entry.ca
                        self._request_table.append(
                            RequestTableEntry(packet.name))
                    self._ca_table[self.unpack(packet.name)] = ca_entry
                else:  # This is not the requesting node --> pass on to neighbour
                    to_lower.put([
                        faceid,
                        Content(self.increase_name(packet.name),
                                packet.content)
                    ])
                    return

            # Content from the chunklayer of a neighbouring node
            elif "CL" in string_components[-1]:
                if string_components[
                        -1] == f"CL{self.num_of_forwards}":  # This is the requesting node --> unpack
                    cl_content = True
                    packet.name = self.unpack(packet.name)
                    # Save the sender of this packet as the recipient for further interests. Used in pack_cl()
                    self.recipient_cl[packet.name] = Name(components[:1])
                else:  # This is not the requesting node --> pass on to neighbour
                    to_lower.put([
                        faceid,
                        Content(self.increase_name(packet.name),
                                packet.content)
                    ])
                    return

            request_entry = self.get_request_entry(packet.name)
            if request_entry is None:
                return
            self._request_table.remove(request_entry)
            if "CA" in components[-2].decode("utf-8"):
                if self.pass_through:
                    return
                ca_content = True
                request_entry.chunked = True
            self.handle_content(faceid, packet, request_entry, ca_content,
                                cl_content, to_lower, to_higher)

        elif isinstance(packet, Nack):
            if self.prefix not in string_components[0]:
                request_entry = self.get_request_entry(packet.name)
                if request_entry:
                    self._request_table.remove(request_entry)
                if "CA" in string_components[-1]:
                    if string_components[
                            -1] == f"CA{self.num_of_forwards}":  # This is the requesting node
                        unpacked = self.unpack(packet.name)
                        ca_entry = self._ca_table.get(unpacked)
                        if components[0] == b"nL":
                            ca_entry.answer_L = True
                        else:
                            ca_entry.answer_R = True
                        if ca_entry.answer_L and ca_entry.answer_R:  # We have an answer from both neighbour
                            if ca_entry.ca:  # We have chunks available from one of the neighbours
                                packet = ca_entry.ca
                                self._ca_table[unpacked] = ca_entry
                                self.handle_content(
                                    faceid, packet,
                                    RequestTableEntry(packet.name), True,
                                    False, to_lower, to_higher)
                                return
                            else:
                                ca_entry.received_all = True
                                request_entry = self.get_request_entry(
                                    unpacked)
                                if request_entry and not request_entry.requested_md and not self.pass_through:
                                    self.create_chunk_interests(
                                        faceid, request_entry, ca_entry,
                                        to_lower)

                        self._ca_table[unpacked] = ca_entry
                    else:  # This is not the requesting node --> pass on to neighbour
                        name = self.increase_name(packet.name)
                        nack = Nack(name, NackReason.NO_CONTENT,
                                    Interest(name))
                        to_lower.put([faceid, nack])
                elif "CL" in string_components[-1]:
                    if string_components[
                            -1] != f"CL{self.num_of_forwards}":  # This is not the requesting node --> pass on to neighbour
                        name = self.increase_name(packet.name)
                        nack = Nack(name, NackReason.NO_CONTENT,
                                    Interest(name))
                        to_lower.put([faceid, nack])
                else:
                    to_higher.put([faceid, packet])
                self.pit.remove_pit_entry(packet.name)
            else:
                if "c" in string_components[-1]:
                    packet.name.components = components[:-1]
                    to_higher.put([
                        faceid,
                        Nack(packet.name, NackReason.NO_CONTENT,
                             Interest(packet.name))
                    ])
                else:  #FIXME What to do here?
                    # to_higher.put([faceid, packet])
                    pass

    def handle_content(self, faceid: int, packet: Content,
                       request_entry: RequestTableEntry, ca_content: bool,
                       cl_content: bool, to_lower: multiprocessing.Queue,
                       to_higher: multiprocessing.Queue):
        """Handle incoming content"""
        if request_entry.chunked is False:  # Not chunked content
            if not packet.get_bytes().startswith(b'mdo:'):
                if ca_content:
                    self.handle_ca(faceid, packet, to_lower)
                else:
                    to_higher.put([faceid, packet])
                return
            else:  # Received metadata data --> chunked content
                request_entry.chunked = True
        if packet.get_bytes().startswith(
                b'mdo:'):  # Request all frames from meta data
            self.handle_received_meta_data(faceid, packet, request_entry,
                                           to_lower, ca_content, cl_content)
        else:
            self.handle_received_chunk_data(faceid, packet, request_entry,
                                            to_lower, to_higher, ca_content)

    def handle_received_meta_data(self, faceid: int, packet: Content,
                                  request_entry: RequestTableEntry,
                                  to_lower: multiprocessing.Queue,
                                  ca_content: bool, cl_content: bool):
        """Handle meta data"""
        if not request_entry.md_complete:
            if packet.name in request_entry.requested_md:
                request_entry.requested_md.remove(packet.name)
            md, chunks, size = self.chunkifyer.parse_meta_data(packet.content)
            for chunk in chunks:  # Request all chunks from the meta data file if not already received or requested
                if chunk not in request_entry.requested_chunks and chunk not in [
                        i.name for i in request_entry.chunks
                ]:
                    request_entry.requested_chunks.append(chunk)
            if md is not None:  # There is another md file
                if md not in request_entry.requested_md:
                    request_entry.requested_md.append(md)
                    if cl_content:
                        md = self.pack_cl(md)
                    to_lower.put([faceid, Interest(md)])
            else:
                # Only create interests if it is the requesting node handling this metadata and
                # either the packet is CA content or there is no CA content available
                request_entry.md_complete = True
                if not self.pass_through:
                    if ca_content:
                        self.create_chunk_interests(faceid, request_entry,
                                                    CaEntry(), to_lower)
                    elif self._ca_table.get(
                            request_entry.name
                    ).received_all:  # We have an answer from both neighbours
                        self.create_chunk_interests(
                            faceid, request_entry,
                            self._ca_table.get(request_entry.name), to_lower)

        self._chunk_table[packet.name] = (packet, time.time())
        self._request_table.append(request_entry)

    def handle_received_chunk_data(self, faceid: int, packet: Content,
                                   request_entry: RequestTableEntry,
                                   to_lower: multiprocessing.Queue,
                                   to_higher: multiprocessing.Queue,
                                   ca_content: bool):
        """Handle chunk data"""
        if packet.name in request_entry.requested_chunks:
            request_entry.requested_chunks.remove(packet.name)
            request_entry.chunks.append(packet)
        self._chunk_table[packet.name] = (packet, time.time())
        if not request_entry.requested_chunks:
            if not request_entry.requested_md:  # All chunks are available
                data = request_entry.chunks
                data = sorted(data,
                              key=lambda content: int(''.join(
                                  filter(str.isdigit, content.name.
                                         string_components[-1]))))
                cont = self.chunkifyer.reassamble_data(request_entry.name,
                                                       data)
                if ca_content:
                    self.handle_ca(faceid, cont, to_lower)
                else:
                    del self._ca_table[request_entry.name]
                    to_higher.put([faceid, cont])
                return

        self._request_table.append(request_entry)

    def handle_ca(self, faceid: int, content: Content,
                  to_lower: multiprocessing.Queue):
        """Unpack the received ca message and create interests for all available chunks."""
        content.name = self.unpack(content.name)
        ca_entry = self._ca_table.get(content.name)
        ca_entry.received_all = True

        chunks_str = content.content.split(";")

        request_entry = self.get_request_entry(content.name)
        if request_entry and not self.pass_through:
            if chunks_str[0] == "complete":  # Neighbour has complete data
                ca_entry.completely_available = True
                if request_entry.md_complete:
                    self.create_chunk_interests(faceid, request_entry,
                                                ca_entry, to_lower)
            else:
                ca_entry.chunks = [Name(chunk) for chunk in chunks_str]
                # Create interests for all chunks that are available from neighbour
                for chunk in ca_entry.chunks:
                    self.logger.info("TO NEIGHBOUR:",
                                     self.pack_ca(chunk, ca_entry))
                    if chunk not in request_entry.requested_chunks and chunk not in [
                            i.name for i in request_entry.chunks
                    ]:
                        request_entry.requested_chunks.append(chunk)
                    to_lower.put(
                        [faceid,
                         Interest(self.pack_ca(chunk, ca_entry))])
                self._request_table.append(request_entry)
                # If there is no name in requested_md try to request the remaining chunks.
                # This is only necessary to get a NACK, so the simulation continues.
                if not request_entry.requested_md:
                    for chunk in [
                            i for i in request_entry.requested_chunks
                            if i not in ca_entry.chunks
                    ]:
                        self.logger.info("TO ORIGINAL SOURCE:", chunk)
                        to_lower.put([faceid, Interest(chunk)])
                self._request_table.remove(request_entry)
            self._ca_table[content.name] = ca_entry

    def get_request_entry(self, name: Name):
        """
        Check if a name is in the request table.
        Return entry or None.
        """
        for entry in self._request_table:
            if entry.name == name or name in entry.requested_chunks or name in entry.requested_md:
                return entry
        return None

    def get_chunks_available(self, packet: Packet):
        """
        Check if chunks are available for a given name.
        Return a content object containing the names of the available chunks, or NACK
        """
        chunks_available = []
        name = self.unpack(packet.name)
        request_entry = self.get_request_entry(name)
        cs_entry = self.cs.find_content_object(name)

        if request_entry is not None:
            chunks_available = [
                str(chunk.name) for chunk in request_entry.chunks
            ]

        elif cs_entry:
            chunks_available.append("complete;")

        if chunks_available:
            chunks_available = Content(packet.name, ";".join(chunks_available))
            if len(chunks_available.content) > self.chunk_size:
                meta_data, chunks = self.chunkifyer.chunk_data(
                    chunks_available)
                meta_data.extend(chunks)
                for data in meta_data:
                    self.cs.remove_content_object(data.name)
                    self.cs.add_content_object(data)
                return meta_data[0]
            else:
                return chunks_available

        return Nack(packet.name, NackReason.NO_CONTENT, packet)

    def get_matching_content_from_packed_name(self, packet: Packet):
        """Return either the content matching the unpacked name or NACK"""
        name_in = self.unpack(packet.name)
        if name_in in self._chunk_table:
            matching_content = self._chunk_table.get(name_in)[0]
            matching_content.name = packet.name
            return matching_content
        else:
            return Nack(packet.name, NackReason.NO_CONTENT, packet)

    def create_chunk_interests(self, faceid: int,
                               request_entry: RequestTableEntry,
                               ca_entry: CaEntry,
                               to_lower: multiprocessing.Queue):
        """Create interests for all the chunks in requested_md of the specified request table entry."""

        if ca_entry.completely_available:
            for chunk in [i for i in request_entry.requested_chunks]:
                self.logger.info("TO NEIGHBOUR:",
                                 self.pack_ca(chunk, ca_entry))
                if chunk not in request_entry.requested_chunks and chunk not in [
                        i.name for i in request_entry.chunks
                ]:
                    request_entry.requested_chunks.append(chunk)
                to_lower.put([faceid, Interest(self.pack_ca(chunk, ca_entry))])
        else:
            for chunk in [
                    i for i in request_entry.requested_chunks
                    if i not in ca_entry.chunks
            ]:
                self.logger.info("TO ORIGINAL SOURCE:", chunk)
                to_lower.put([faceid, Interest(chunk)])

    def save_if_longest(self, packet: Content, ca_entry: CaEntry):
        """
        Check if the received ca content is longer than the existing one and if so, replace it.
        In the case where both neighbours have chunks available, we want to send the interests only to the one
        which has more.
        """
        if packet.get_bytes().startswith(
                b'mdo:'):  # Content is metadata, read size from metadata
            _, _, content_size = self.chunkifyer.parse_meta_data(
                packet.content)
        else:  # Content is string, size equals length of the string
            content_size = (len(packet.content))
        content_size = int(content_size)
        if content_size > ca_entry.size:
            ca_entry.ca = packet
            ca_entry.size = content_size

    def pack_ca(self, name: Name, ca_entry: CaEntry) -> Name:
        """Prepend the recipient, append "CL" and the number of forwards."""
        return ca_entry.recipient + name + f"CL{self.num_of_forwards}"

    def pack_cl(self, name: Name) -> Name:
        """Prepend the recipient, append "CA" and the number of forwards."""
        lookup_name = Name(name.components[:-1])
        return self.recipient_cl.get(
            lookup_name) + name + f"CL{self.num_of_forwards}"

    def unpack(self, name: Name) -> Name:
        return Name(name.components[1:-1])

    def increase_name(self, name: Name) -> Name:
        """
        Increase the number at the end of the name.
        The number is used to determine whether or not a packet gets forwarded to the next neighbour.
        """
        components = name.components
        last = components[-1].decode("utf-8")
        i = int(''.join(filter(str.isdigit, last)))
        if "CA" in last:
            return Name(components[:-1]) + f"CA{i+1}"
        elif "CL" in last:
            return Name(components[:-1]) + f"CL{i+1}"
        return name

    def decrease_name(self, name: Name) -> Name:
        """
        Decrease the number at the end of the name.
        The number is used to determine whether or not a packet gets forwarded to the next neighbour.
        """
        components = name.components
        last = components[-1].decode("utf-8")
        i = int(''.join(filter(str.isdigit, last)))
        if "CA" in last:
            return Name(components[:-1]) + f"CA{i-1}"
        elif "CL" in last:
            return Name(components[:-1]) + f"CL{i-1}"
        return name

    def set_number_of_forwards(self, number_of_forwards: int):
        self.num_of_forwards = number_of_forwards
Exemplo n.º 4
0
class DataOffloadingChunklayerSimple(LayerProcess):
    """This Chunklayer handles interrupted data uploads by asking neighbouring nodes for available content."""
    def __init__(self,
                 cs: BaseContentStore,
                 pit: BasePendingInterestTable,
                 fib: BaseForwardingInformationBase,
                 chunkifyer: BaseChunkifyer = None,
                 chunk_size: int = 4096,
                 num_of_forwards: int = 1,
                 prefix: str = "car",
                 log_level: int = 255):
        super().__init__("ChunkLayer", log_level=log_level)
        self.cs = cs
        self.pit = pit
        self.fib = fib
        self.chunk_size = chunk_size
        if chunkifyer is None:
            self.chunkifyer = SimpleContentChunkifyer(chunk_size)
        else:
            self.chunkifyer: BaseChunkifyer = chunkifyer
        self.num_of_forwards = num_of_forwards
        self.prefix = prefix

        manager = multiprocessing.Manager()
        self._chunk_table: Dict[Name, (Content, float)] = manager.dict()
        self._request_table: List[RequestTableEntry] = manager.list()
        self._cl_table: Dict[Name, ClEntry] = manager.dict()

        self.pass_through = False
        self.cl_sent = False

    def data_from_higher(self, to_lower: multiprocessing.Queue,
                         to_higher: multiprocessing.Queue, data):
        self.logger.info("Got Data from higher")
        faceid = data[0]
        packet = data[1]

        if isinstance(packet, Interest):
            self.logger.info("Packet is Interest " + str(packet.name))
            request_entry = self.get_request_entry(packet.name)
            if request_entry is None:
                self._request_table.append(RequestTableEntry(packet.name))

            # If the interest starts with "/car" and not ends with "NFN", request metadata and available chunks from neighbours
            components = packet.name.string_components
            if self.prefix in components[
                    0] and components[-1] != "NFN" and not request_entry:
                self.pass_through = False
                self._cl_table[packet.name] = ClEntry(data)
                cl_entry = self._cl_table.get(packet.name)

                name1 = Name("/nL") + packet.name + f"CL{self.num_of_forwards}"
                name2 = Name("/nR") + packet.name + f"CL{self.num_of_forwards}"
                if not self.pit.find_pit_entry(
                        name1) and self.fib.find_fib_entry(name1):
                    cl_entry.nack_L = False
                    self.cl_sent = True
                    to_lower.put([faceid, Interest(name1)])
                if not self.pit.find_pit_entry(
                        name2) and self.fib.find_fib_entry(name2):
                    cl_entry.nack_R = False
                    self.cl_sent = True
                    to_lower.put([faceid, Interest(name2)])
                if self.cl_sent:
                    self._cl_table[packet.name] = cl_entry
                    return

            to_lower.put(data)
            return

        elif isinstance(packet, Content):
            self.logger.info("Packet is Content (name=%s, %d bytes)" %
                             (str(packet.name), len(packet.content)))
            if len(packet.content) < self.chunk_size:
                to_lower.put(data)
            else:
                self.logger.info("Chunking Packet")
                metadata, chunks = self.chunkifyer.chunk_data(
                    packet)  # Create metadata and chunks
                self.logger.info("Metadata: " + metadata[0].content)
                to_lower.put([faceid, metadata[0]
                              ])  # Return name of first metadata object
                for md in metadata:  # Add metadata to chunktable
                    if md.name not in self._chunk_table:
                        self._chunk_table[md.name] = (md, time.time())
                for c in chunks:  # Add chunks to chunktable
                    if c.name not in self._chunk_table:
                        self._chunk_table[c.name] = (c, time.time())

        elif isinstance(packet, Nack):
            request_entry = self.get_request_entry(packet.name)
            if request_entry is not None:
                self._request_table.remove(request_entry)
            to_lower.put(data)

    def data_from_lower(self, to_lower: multiprocessing.Queue,
                        to_higher: multiprocessing.Queue, data):
        self.logger.info("Got Data from lower")
        faceid = data[0]
        packet = data[1]
        components = packet.name.components
        string_components = packet.name.string_components
        last = components[-1]

        if isinstance(packet, Interest):
            self.logger.info("Packet is Interest")

            # General Interest passed on to chunklayer
            if "CL" in string_components[-1]:
                matching_content = self.get_matching_content(packet)
                if last == b"CL0" or isinstance(
                        matching_content,
                        Content):  # If there is matching content, return it
                    to_lower.put([faceid, matching_content])
                else:  # Otherwise try to pass on to neighbour
                    to_lower.put(
                        [faceid,
                         Interest(self.decrease_name(packet.name))])

            elif packet.name in self._chunk_table:
                matching_content = self._chunk_table.get(packet.name)[0]
                to_lower.put([faceid, matching_content])
            else:
                to_higher.put(data)

        elif isinstance(packet, Content):
            self.logger.info("Packet is Content")
            cl_content = False

            # Content from the chunklayer of a neighbouring node
            if "CL" in string_components[-1]:
                if string_components[
                        -1] == f"CL{self.num_of_forwards}":  # This is the requesting node --> unpack
                    cl_content = True
                    packet.name = self.unpack(packet.name)
                    # Save the sender of this packet as the recipient for further interests. Used in pack_cl()
                    request_entry = self.get_request_entry(packet.name)
                    if request_entry:
                        cl_entry = self._cl_table.get(request_entry.name)
                        if cl_entry.interest_requested:  # If we already resent requests to source, don't consider it
                            return
                        cl_entry.recipient = Name(components[:1])
                        self._cl_table[request_entry.name] = cl_entry

                else:  # This is not the requesting node --> pass on to neighbour
                    to_lower.put([
                        faceid,
                        Content(self.increase_name(packet.name),
                                packet.content)
                    ])
                    return

            request_entry = self.get_request_entry(packet.name)
            if request_entry is None:
                return
            self.handle_content(faceid, packet, request_entry, cl_content,
                                to_lower, to_higher)

        elif isinstance(packet, Nack):
            if self.prefix not in string_components[0]:
                request_entry = self.get_request_entry(packet.name)
                if request_entry:
                    self._request_table.remove(request_entry)

                if "CL" in string_components[-1]:
                    if string_components[
                            -1] == f"CL{self.num_of_forwards}":  # This is the requesting node --> unpack
                        name_unpacked = self.unpack(packet.name)
                        request_entry = self.get_request_entry(name_unpacked)
                        if request_entry:
                            cl_entry = self._cl_table.get(request_entry.name)
                            if components[0] == b"nL":
                                cl_entry.nack_L = True
                            else:
                                cl_entry.nack_R = True
                            if cl_entry.nack_L and cl_entry.nack_R and not cl_entry.interest_requested:
                                # No more data available from neighbours, get it from car
                                self.get_missing_data_from_original_source(
                                    faceid, request_entry, cl_entry, to_lower)

                            self._cl_table[request_entry.name] = cl_entry
                    else:
                        name = self.increase_name(packet.name)
                        nack = Nack(name, NackReason.NO_CONTENT,
                                    Interest(name))
                        to_lower.put([faceid, nack])
                else:
                    to_higher.put([faceid, packet])
                self.pit.remove_pit_entry(packet.name)
            else:
                if "c" in string_components[-1] or "m" in string_components[-1]:
                    packet.name.components = components[:-1]
                    to_higher.put([
                        faceid,
                        Nack(packet.name, NackReason.NO_CONTENT,
                             Interest(packet.name))
                    ])
                else:
                    pass
                    # to_higher.put([faceid, packet])

    def handle_content(self, faceid: int, packet: Content,
                       request_entry: RequestTableEntry, cl_content: bool,
                       to_lower: multiprocessing.Queue,
                       to_higher: multiprocessing.Queue):
        """Handle incoming content"""
        self._request_table.remove(request_entry)
        if request_entry.chunked is False:  # Not chunked content
            if not packet.get_bytes().startswith(b'mdo:'):
                to_higher.put([faceid, packet])
                return
            else:  # Received metadata data --> chunked content
                request_entry.chunked = True
        if packet.get_bytes().startswith(
                b'mdo:'):  # Request all frames from metadata
            self.handle_received_meta_data(faceid, packet, request_entry,
                                           to_lower, cl_content)
        else:
            self.handle_received_chunk_data(faceid, packet, request_entry,
                                            to_higher)

    def handle_received_meta_data(self, faceid: int, packet: Content,
                                  request_entry: RequestTableEntry,
                                  to_lower: multiprocessing.Queue,
                                  cl_content: bool):
        """Handle meta data"""
        if packet.name in request_entry.requested_md:
            request_entry.requested_md.remove(packet.name)
        md, chunks, size = self.chunkifyer.parse_meta_data(packet.content)
        for chunk in chunks:  # Request all chunks from the metadata file if not already received or requested
            # if chunk not in request_entry.requested_chunks and chunk not in [i.name for i in request_entry.chunks]:
            if chunk not in [i.name for i in request_entry.chunks]:
                request_entry.requested_chunks.append(chunk)
                if not self.pass_through:
                    if cl_content:
                        cl_entry = self._cl_table.get(request_entry.name)
                        if cl_entry.nack_L and cl_entry.nack_R:
                            break
                        chunk = self.pack_cl(chunk)
                    to_lower.put([faceid, Interest(chunk)])
        if md is not None:  # There is another md file
            if md not in request_entry.requested_md:
                request_entry.requested_md.append(md)
            if not self.pass_through:
                if cl_content:
                    cl_entry = self._cl_table.get(request_entry.name)
                    if not (cl_entry.nack_L and cl_entry.nack_R):
                        cl_entry.interest = [faceid, Interest(packet.name)]
                        self._cl_table[request_entry.name] = cl_entry
                        md = self.pack_cl(md)
                        to_lower.put([faceid, Interest(md)])
                else:
                    to_lower.put([faceid, Interest(md)])
        else:
            request_entry.last_chunk = chunks[-1]
        self._chunk_table[packet.name] = (packet, time.time())
        self._request_table.append(request_entry)

    def handle_received_chunk_data(self, faceid: int, packet: Content,
                                   request_entry: RequestTableEntry,
                                   to_higher: multiprocessing.Queue):
        """Handle chunk data"""
        if packet.name in request_entry.requested_chunks:
            request_entry.requested_chunks.remove(packet.name)
            request_entry.chunks.append(packet)
        self._chunk_table[packet.name] = (packet, time.time())
        if not request_entry.requested_chunks and not request_entry.requested_md:
            if not request_entry.requested_md:  # All chunks are available
                data = request_entry.chunks
                data = sorted(data,
                              key=lambda content: int(''.join(
                                  filter(str.isdigit, content.name.
                                         string_components[-1]))))
                cont = self.chunkifyer.reassamble_data(request_entry.name,
                                                       data)
                to_higher.put([faceid, cont])
                return

        self._request_table.append(request_entry)

    def get_request_entry(self, name: Name):
        """
        Check if a name is in the request table.
        Return entry or None.
        """
        for entry in self._request_table:
            if entry.name == name or name in entry.requested_chunks or name in entry.requested_md:
                return entry
        return None

    def get_matching_content(self, packet: Packet):
        """Return either the content matching the packet name or NACK"""
        name_in = self.unpack(packet.name)
        cs_entry = self.cs.find_content_object(name_in)
        if name_in in self._chunk_table:
            matching_content = self._chunk_table.get(name_in)[0]
            matching_content.name = packet.name
            return matching_content
        elif cs_entry:
            matching_content = cs_entry.content
            matching_content.name = packet.name
            return matching_content
        else:
            return Nack(packet.name, NackReason.NO_CONTENT, packet)

    def pack_cl(self, name: Name) -> Name:
        """Prepend the recipient, append "CL" and the number of forwards."""
        lookup_name = Name(name.components[:-1])
        return self._cl_table.get(
            lookup_name).recipient + name + f"CL{self.num_of_forwards}"

    def unpack(self, name: Name) -> Name:
        return Name(name.components[1:-1])

    def increase_name(self, name: Name) -> Name:
        """
        Increase the number at the end of the name.
        The number is used to determine whether or not a packet gets forwarded to the next neighbour.
        """
        components = name.components
        last = components[-1].decode("utf-8")
        i = int(''.join(filter(str.isdigit, last)))
        if "CL" in last:
            return Name(components[:-1]) + f"CL{i+1}"
        return name

    def decrease_name(self, name: Name) -> Name:
        """
        Decrease the number at the end of the name.
        The number is used to determine whether or not a packet gets forwarded to the next neighbour.
        """
        components = name.components
        last = components[-1].decode("utf-8")
        i = int(''.join(filter(str.isdigit, last)))
        if "CL" in last:
            return Name(components[:-1]) + f"CL{i-1}"
        return name

    def get_missing_data_from_original_source(self, faceid: int,
                                              request_entry: RequestTableEntry,
                                              cl_entry: ClEntry,
                                              to_lower: multiprocessing.Queue):
        """
        Start requesting the missing files from the original source.
        """
        if not cl_entry.interest_requested:
            if request_entry.requested_chunks:
                # Request again all chunks that have been requested but not satisfied yet
                for chunk in request_entry.requested_chunks:
                    to_lower.put([faceid, Interest(chunk)])

            # If requested_md is not empty, request them again from source
            if request_entry.requested_md:
                for md in request_entry.requested_md:
                    to_lower.put([faceid, Interest(md)])
            else:  # if empty, request orginal interest from source
                to_lower.put(cl_entry.interest)
            cl_entry.interest_requested = True

    def set_number_of_forwards(self, number_of_forwards: int):
        self.num_of_forwards = number_of_forwards