def __init__(self, *args):
super(DiscoveryCommunity, self).__init__(*args)
self._explicitly_hot_text = []
self._implicitly_hot_text = []
self._top_squares = []
self._top_text = []
self._hots = CacheDict(max_caches=1024)
self._pending_callbacks.append(self._dispersy.callback.register(self._select_and_announce_hot))
self._pending_callbacks.append(self._dispersy.callback.register(self._periodically_collect_top_hots))
self._pending_callbacks.append(self._dispersy.callback.register(self._hot_cleanup))
class DiscoveryCommunity(Community):
def __init__(self, *args):
super(DiscoveryCommunity, self).__init__(*args)
self._explicitly_hot_text = []
self._implicitly_hot_text = []
self._top_squares = []
self._top_text = []
self._hots = CacheDict(max_caches=1024)
self._pending_callbacks.append(self._dispersy.callback.register(self._select_and_announce_hot))
self._pending_callbacks.append(self._dispersy.callback.register(self._periodically_collect_top_hots))
self._pending_callbacks.append(self._dispersy.callback.register(self._hot_cleanup))
def initiate_meta_messages(self):
return [Message(self, u"hots", NoAuthentication(), PublicResolution(), DirectDistribution(), CommunityDestination(node_count=5), HotsPayload(), self._dispersy._generic_timeline_check, self.on_hots),
Message(self, u"search", NoAuthentication(), PublicResolution(), DirectDistribution(), CommunityDestination(node_count=20), SearchPayload(), self._dispersy._generic_timeline_check, self.on_search)]
def initiate_conversions(self):
return [DefaultConversion(self), Conversion(self)]
@property
def top_squares(self):
return self._top_squares
@property
def top_text(self):
return self._top_text
def add_explicitly_hot_text(self, message):
# TODO all messages should be unique
self._explicitly_hot_text.append(message)
del self._explicitly_hot_text[20:]
def add_implicitly_hot_text(self, messages):
# TODO all messages should be unique
self._implicitly_hot_text.extend(messages)
del self._implicitly_hot_text[20:]
def _hot_cleanup(self):
while True:
yield 300.0
for _ in self._hots.cleanup():
pass
def _select_and_announce_hot(self):
meta = self._meta_messages[u"hots"]
while True:
# TODO yield 60.0, lowered for debugging
yield 10.0
# what is hot?
# explicit: a message the user marked as 'hot'
# implicit: a newly received message
messages = sample(self._explicitly_hot_text, min(15, len(self._explicitly_hot_text)))
messages.extend(sample(self._implicitly_hot_text, min(20-len(messages), len(self._implicitly_hot_text))))
# TODO all messages should be unique
if messages:
if __debug__: dprint(len(messages), "x text")
hots = [Hot(message.community.cid, message.authentication.member.mid, message.distribution.global_time) for message in messages]
message = meta.impl(distribution=(self.global_time,), payload=(hots,))
self._dispersy.store_update_forward([message], False, False, True)
def _collect_top_hots(self):
now = time()
self._top_squares = []
self._top_text = []
for index, key in enumerate(self._hots):
hot = self._hots[key]
assert isinstance(hot, HotCache), hot
if not hot.square:
try:
hot.square = self._dispersy.get_community(hot.cid, load=True)
except KeyError:
master = DummyMember(hot.cid)
hot.square = PreviewCommunity.join_community(master, self._my_member, self)
if index < 10:
if not hot.message and hot.last_requested < now - 10.0 and hot.sources:
hot.message = hot.square.fetch_hot_text(hot)
if hot.message:
self._top_text.append(hot.message)
if not hot.square in self._top_squares:
self._top_squares.append(hot.square)
if len(self._top_squares) == 10:
break
if __debug__:
for index, square in enumerate(self._top_squares):
dprint(index, "] SQUARE ", square.cid.encode("HEX"), " - ", square.title)
for index, message in enumerate(self._top_text):
dprint(index, "] TEXT ", message.community.cid.encode("HEX"), " - ", message.payload.text)
def _periodically_collect_top_hots(self):
while True:
# TODO yield 30.0, lowered for debugging
yield 10.0
self._collect_top_hots()
def on_hots(self, messages):
for message in messages:
for hot in message.payload.hots:
key = hot.key
# 'upgrade' Hot to HotCache, also modifies importance counters in CacheDict
if key in self._hots:
hot = self._hots[key]
else:
self._hots[key] = hot = HotCache(hot)
assert isinstance(hot, HotCache), hot
assert isinstance(self._hots[key], HotCache), self._hots[key]
hot.add_source(message.candidate)
if len(self._top_squares) + len(self._top_text) < 10:
self._collect_top_hots()
def keyword_search(self, keywords):
return self.expression_search(u"|".join(keywords))
def expression_search(self, expression):
meta = self._meta_messages[u"search"]
message = meta.impl(distribution=(self.global_time,), payload=(expression,))
self._dispersy.store_update_forward([message], False, False, True)
return message
def on_search(self, messages):
for message in messages:
if __debug__: dprint("searching for \\", message.payload.expression, "\\")
