def _onSyncData(self, encodedContent, interest):
"""
Process the sync data after the content is assembled by the
SegmentFetcher. Call _deletePendingInterests to delete any pending sync
Interest with the Interest name, which would have been satisfied by the
forwarder once it got the data. For each name in the data content, check
that we don't already have the name, and call _canAddReceivedName (which
may process the name as a prefix/sequenceNo). Call _onUpdate to notify
the application about the updates. Call _sendSyncInterest because the
last one was satisfied by the incoming data.
:param Blob encodedContent: The encoded sync data content that was
assembled by the SegmentFetcher.
:param Interest interest: The Interest for which we got the data.
"""
self._deletePendingInterests(interest.getName())
state = PSyncState(encodedContent)
names = []
logging.getLogger(__name__).debug("Sync Data Received: " +
state.toString())
for contentName in state.getContent():
if not (contentName in self._nameToHash):
logging.getLogger(__name__).debug("Checking whether to add " +
contentName.toUri())
if (self._canAddReceivedName == None
or self._canAddReceivedName(contentName)):
logging.getLogger(__name__).debug("Adding name " +
contentName.toUri())
# The Name is freshly created by PSyncState decode, so don't copy.
names.append(contentName)
self.insertIntoIblt(contentName)
# We should not call _satisfyPendingSyncInterests here because we
# just got data and deleted pending interests by calling
# _deletePendingInterests. But we might have interests which don't
# match this interest that might not have been deleted from the
# pending sync interests.
# We just got the data, so send a new sync Interest.
if len(names) > 0:
try:
self._onNamesUpdate(names)
except:
logging.exception("Error in onUpdate")
logging.getLogger(__name__).info(
"onSyncData: Renewing sync interest")
self._sendSyncInterest()
else:
logging.getLogger(
__name__).info("No new update, interest nonce: " +
interest.getNonce().toHex() + " , hash: " +
str(abs(hash(interest.getName()))))
def _onSyncData(self, encodedContent, interest):
"""
Process the sync data after the content is assembled by the
SegmentFetcher. Call _deletePendingInterests to delete any pending sync
Interest with the Interest name, which would have been satisfied by the
forwarder once it got the data. For each name in the data content, check
that we don't already have the name, and call _canAddReceivedName (which
may process the name as a prefix/sequenceNo). Call _onUpdate to notify
the application about the updates. Call _sendSyncInterest because the
last one was satisfied by the incoming data.
:param Blob encodedContent: The encoded sync data content that was
assembled by the SegmentFetcher.
:param Interest interest: The Interest for which we got the data.
"""
self._deletePendingInterests(interest.getName())
state = PSyncState(encodedContent)
names = []
logging.getLogger(__name__).debug("Sync Data Received: " + state.toString())
for contentName in state.getContent():
if not (contentName in self._nameToHash):
logging.getLogger(__name__).debug("Checking whether to add " +
contentName.toUri())
if (self._canAddReceivedName == None or
self._canAddReceivedName(contentName)):
logging.getLogger(__name__).debug("Adding name " +
contentName.toUri())
# The Name is freshly created by PSyncState decode, so don't copy.
names.append(contentName)
self.insertIntoIblt(contentName)
# We should not call _satisfyPendingSyncInterests here because we
# just got data and deleted pending interests by calling
# _deletePendingInterests. But we might have interests which don't
# match this interest that might not have been deleted from the
# pending sync interests.
# We just got the data, so send a new sync Interest.
if len(names) > 0:
try:
self._onNamesUpdate(names)
except:
logging.exception("Error in onUpdate")
logging.getLogger(__name__).info("onSyncData: Renewing sync interest")
self._sendSyncInterest()
else:
logging.getLogger(__name__).info("No new update, interest nonce: " +
interest.getNonce().toHex() + " , hash: " +
str(abs(hash(interest.getName()))))
def testEmptyContent(self):
state = PSyncState()
# Simulate getting a buffer of content from a segment fetcher.
data = Data()
data.setContent(state.wireEncode())
state2 = PSyncState(data.getContent())
self.assertEqual(0, len(state2.getContent()))
def _satisfyPendingInterests(self):
"""
Satisfy pending sync Interests. For a pending sync interests, if the
IBLT of the sync Interest has any difference from our own IBLT, then
send a Data back. If we can't decode the difference from the stored IBLT,
then delete it.
"""
logging.getLogger(__name__).debug("Satisfying full sync Interest: " +
str(len(self._pendingEntries)))
# Copy the keys before iterating se we can erase entries.
for keyName in list(self._pendingEntries.keys()):
pendingEntry = self._pendingEntries[keyName]
entryIblt = pendingEntry._iblt
difference = self._iblt.difference(entryIblt)
positive = set()
negative = set()
if not difference.listEntries(positive, negative):
logging.getLogger(__name__).info(
"Decode failed for pending interest")
if (len(positive) + len(negative) >= self._threshold
or (len(positive) == 0 and len(negative) == 0)):
logging.getLogger(__name__).info(
"positive + negative > threshold or no difference can be found. Erase pending interest."
)
# Prevent delayedRemovePendingEntry from removing a new entry
# with the same Name.
pendingEntry._isRemoved = True
del self._pendingEntries[keyName]
continue
state = PSyncState()
for hashValue in positive:
name = self._hashToName[hashValue]
if name in self._nameToHash:
state.addContent(name)
if len(state.getContent()) > 0:
logging.getLogger(__name__).debug("Satisfying sync content: " +
state.toString())
self._sendSyncData(keyName, state.wireEncode())
# Prevent _delayedRemovePendingEntry from removing a new entry
# with the same Name.
pendingEntry._isRemoved = True
del self._pendingEntries[keyName]
def _satisfyPendingInterests(self):
"""
Satisfy pending sync Interests. For a pending sync interests, if the
IBLT of the sync Interest has any difference from our own IBLT, then
send a Data back. If we can't decode the difference from the stored IBLT,
then delete it.
"""
logging.getLogger(__name__).debug("Satisfying full sync Interest: " +
str(len(self._pendingEntries)))
# Copy the keys before iterating se we can erase entries.
for keyName in list(self._pendingEntries.keys()):
pendingEntry = self._pendingEntries[keyName]
entryIblt = pendingEntry._iblt
difference = self._iblt.difference(entryIblt)
positive = set()
negative = set()
if not difference.listEntries(positive, negative):
logging.getLogger(__name__).info(
"Decode failed for pending interest")
if (len(positive) + len(negative) >= self._threshold or
(len(positive) == 0 and len(negative) == 0)):
logging.getLogger(__name__).info(
"positive + negative > threshold or no difference can be found. Erase pending interest.")
# Prevent delayedRemovePendingEntry from removing a new entry
# with the same Name.
pendingEntry._isRemoved = True
del self._pendingEntries[keyName]
continue
state = PSyncState()
for hashValue in positive:
name = self._hashToName[hashValue]
if name in self._nameToHash:
state.addContent(name)
if len(state.getContent()) > 0:
logging.getLogger(__name__).debug("Satisfying sync content: " +
state.toString())
self._sendSyncData(keyName, state.wireEncode())
# Prevent _delayedRemovePendingEntry from removing a new entry
# with the same Name.
pendingEntry._isRemoved = True
del self._pendingEntries[keyName]
def _onSyncInterest(self, prefixName, interest, face, interestFilterId, filter):
"""
Process a sync interest received from another party.
This gets the difference between our IBLT and the IBLT in the other sync
interest. If we cannot get the difference successfully, then send an
application Nack. If we have some things in our IBLT that the other side
does not have, then reply with the content. Or, if the number of
different items is greater than threshold or equals zero, then send a
Nack. Otherwise add the sync interest into the pendingEntries_ map with
the interest name as the key and a PendingEntryInfoFull as the value.
:param Name prefixName: The prefix Name for the sync group which we
registered.
:param Interest interest: The the received Interest.
"""
if self._segmentPublisher.replyFromStore(interest.getName()):
return
nameWithoutSyncPrefix = interest.getName().getSubName(prefixName.size())
if nameWithoutSyncPrefix.size() == 1:
# Get /<prefix>/IBLT from /<prefix>/IBLT
interestName = interest.getName()
elif nameWithoutSyncPrefix.size() == 3:
# Get /<prefix>/IBLT from /<prefix>/IBLT/<version>/<segment-no>
interestName = interest.getName().getPrefix(-2)
else:
return
ibltName = interestName.get(-1)
logging.getLogger(__name__).debug("Full Sync Interest received, nonce: " +
interest.getNonce().toHex() + ", hash:" + str(abs(hash(interestName))))
iblt = InvertibleBloomLookupTable(self._expectedNEntries)
try:
iblt.initialize(ibltName.getValue())
except Exception as ex:
logging.getLogger(__name__).error(
"Error in iblt.initialize: %s", str(ex))
return
difference = self._iblt.difference(iblt)
positive = set()
negative = set()
if not difference.listEntries(positive, negative):
logging.getLogger(__name__).info("Cannot decode differences, positive: " +
str(len(positive)) + " negative: " + str(len(negative)) +
" _threshold: " + str(self._threshold))
# Send all data if greater then the threshold, or if there are
# neither positive nor negative differences. Otherwise, continue
# below and send the positive as usual.
if (len(positive) + len(negative) >= self._threshold or
(len(positive) == 0 and len(negative) == 0)):
state1 = PSyncState()
for name in self._nameToHash.keys():
state1.addContent(name)
if len(state1.getContent()) > 0:
self._segmentPublisher.publish(
interest.getName(), interest.getName(), state1.wireEncode(),
self._syncReplyFreshnessPeriod, self._signingInfo)
return
state = PSyncState()
for hashValue in positive:
name = self._hashToName[hashValue]
if name in self._nameToHash:
if (self._canAddToSyncData == None or
self._canAddToSyncData(name, negative)):
state.addContent(name)
if len(state.getContent()) > 0:
logging.getLogger(__name__).debug("Sending sync content: " +
state.toString())
self._sendSyncData(interestName, state.wireEncode())
return
entry = FullPSync2017._PendingEntryInfoFull(iblt)
self._pendingEntries[interestName] = entry
self._face.callLater(
interest.getInterestLifetimeMilliseconds(),
lambda: self._delayedRemovePendingEntry
(interest.getName(), entry, interest.getNonce()))
def _onSyncInterest(self, prefixName, interest, face, interestFilterId,
filter):
"""
Process a sync interest received from another party.
This gets the difference between our IBLT and the IBLT in the other sync
interest. If we cannot get the difference successfully, then send an
application Nack. If we have some things in our IBLT that the other side
does not have, then reply with the content. Or, if the number of
different items is greater than threshold or equals zero, then send a
Nack. Otherwise add the sync interest into the pendingEntries_ map with
the interest name as the key and a PendingEntryInfoFull as the value.
:param Name prefixName: The prefix Name for the sync group which we
registered.
:param Interest interest: The the received Interest.
"""
if self._segmentPublisher.replyFromStore(interest.getName()):
return
nameWithoutSyncPrefix = interest.getName().getSubName(
prefixName.size())
if nameWithoutSyncPrefix.size() == 1:
# Get /<prefix>/IBLT from /<prefix>/IBLT
interestName = interest.getName()
elif nameWithoutSyncPrefix.size() == 3:
# Get /<prefix>/IBLT from /<prefix>/IBLT/<version>/<segment-no>
interestName = interest.getName().getPrefix(-2)
else:
return
ibltName = interestName.get(-1)
logging.getLogger(
__name__).debug("Full Sync Interest received, nonce: " +
interest.getNonce().toHex() + ", hash:" +
str(abs(hash(interestName))))
iblt = InvertibleBloomLookupTable(self._expectedNEntries)
try:
iblt.initialize(ibltName.getValue())
except Exception as ex:
logging.getLogger(__name__).error("Error in iblt.initialize: %s",
str(ex))
return
difference = self._iblt.difference(iblt)
positive = set()
negative = set()
if not difference.listEntries(positive, negative):
logging.getLogger(__name__).info(
"Cannot decode differences, positive: " + str(len(positive)) +
" negative: " + str(len(negative)) + " _threshold: " +
str(self._threshold))
# Send all data if greater then the threshold, or if there are
# neither positive nor negative differences. Otherwise, continue
# below and send the positive as usual.
if (len(positive) + len(negative) >= self._threshold
or (len(positive) == 0 and len(negative) == 0)):
state1 = PSyncState()
for name in self._nameToHash.keys():
state1.addContent(name)
if len(state1.getContent()) > 0:
self._segmentPublisher.publish(
interest.getName(), interest.getName(),
state1.wireEncode(), self._syncReplyFreshnessPeriod,
self._signingInfo)
return
state = PSyncState()
for hashValue in positive:
name = self._hashToName[hashValue]
if name in self._nameToHash:
if (self._canAddToSyncData == None
or self._canAddToSyncData(name, negative)):
state.addContent(name)
if len(state.getContent()) > 0:
logging.getLogger(__name__).debug("Sending sync content: " +
state.toString())
self._sendSyncData(interestName, state.wireEncode())
return
entry = FullPSync2017._PendingEntryInfoFull(iblt)
self._pendingEntries[interestName] = entry
self._face.callLater(
interest.getInterestLifetimeMilliseconds(),
lambda: self._delayedRemovePendingEntry(interest.getName(), entry,
interest.getNonce()))
def testEncodeDecode(self):
state = PSyncState()
state.addContent(Name("test1"))
state.addContent(Name("test2"))
# Simulate getting a buffer of content from a segment fetcher.
data = Data()
encoding = state.wireEncode()
expectedEncoding = [
0x80, 0x12, # PSyncContent
0x07, 0x07, 0x08, 0x05, 0x74, 0x65, 0x73, 0x74, 0x31, # Name = "/test1"
0x07, 0x07, 0x08, 0x05, 0x74, 0x65, 0x73, 0x74, 0x32 # Name = "/test2"
]
self.assertTrue(encoding.equals(Blob(expectedEncoding)))
data.setContent(encoding)
receivedState = PSyncState()
receivedState.wireDecode(data.getContent())
self.assertTrue(state.getContent() == receivedState.getContent())