def get_suggestions(request, count, mini=False):
authinfo = request.getSession(IAuthInfo)
myId = authinfo.username
SUGGESTIONS_UPDATE_FREQUENCY = 3 * 86400 # 5days
MAX_INVALID_SUGGESTIONS = 3
now = time.time()
suggestions = []
relation = Relation(myId, [])
yield defer.DeferredList([relation.initSubscriptionsList(),
relation.initFollowersList()])
@defer.inlineCallbacks
def _get_suggestions(myId, relation):
validSuggestions = []
invalidCount = 0
FORCE_UPDATE = False
cols = yield db.get_slice(myId, "suggestions", reverse=True)
for col in cols:
if now - col.column.timestamp/1e6 > SUGGESTIONS_UPDATE_FREQUENCY:
FORCE_UPDATE = True
for userId in col.column.value.split():
if isValidSuggestion(myId, userId, relation):
validSuggestions.append(userId)
else:
invalidCount += 1
defer.returnValue((validSuggestions, invalidCount, FORCE_UPDATE))
validSuggestions, invalidCount, FORCE_UPDATE = yield _get_suggestions(myId, relation)
if not validSuggestions:
yield _update_suggestions(request, relation)
validSuggestions, invalidCount, FORCE_UPDATE = yield _get_suggestions(myId, relation)
no_of_samples = count*2
population = count*5
if mini and len(validSuggestions) >= no_of_samples:
suggestions = random.sample(validSuggestions[:population], no_of_samples)
else:
suggestions = validSuggestions[:]
if FORCE_UPDATE or invalidCount > MAX_INVALID_SUGGESTIONS:
_update_suggestions(request, relation)
entities = base.EntitySet(suggestions[:count])
if suggestions:
suggestions = suggestions[:count]
yield entities.fetchData()
defer.returnValue((suggestions, entities))
def _search(self, request):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
landing = not self._ajax
myOrgId = args['orgId']
filter_map = {'people':'itemType'}
term = utils.getRequestArg(request, "q")
start = utils.getRequestArg(request, "start") or 0
filters = utils.getRequestArg(request, 'filter', multiValued=True) or []
filters = dict([(filter_map[x], x) for x in filters if x in filter_map])
args["term"] = term
nextPageStart = ''
if not term:
errors.MissingParams()
try:
start = int(start)
if start < 0:
raise ValueError
except ValueError:
errors.InvalidParamValue()
if script and landing:
t.render(request, "search.mako", **args)
if script and appchange:
t.renderScriptBlock(request, "search.mako", "layout",
landing, "#mainbar", "set", **args)
count = SEARCH_RESULTS_PER_PAGE
items = {}
convs = set()
highlighting = {}
toFetchItems = []
toFetchStart = start
toFetchEntities = set()
people = []
relation = Relation(myId, [])
yield defer.DeferredList([relation.initGroupsList(),
relation.initSubscriptionsList(),
relation.initFollowersList()])
regex = re.compile("(.*?)([^\s]*\s*[^\s]*\s*[^\s]*\s*)(<em class='highlight'>.*<\/em>)(\s*[^\s]*\s*[^\s]*\s*[^\s]*)(.*)")
res = yield solr.search(term, args['orgId'], count, toFetchStart, filters={'itemType': 'people'})
docs = res.data.get('response', {}).get('docs', [])
for item in docs:
entityId = item['id']
people.append(entityId)
toFetchEntities.add(entityId)
while 1:
res = yield solr.search(term, args['orgId'], count, toFetchStart)
messages = []
convItems = []
numMatched = res.data.get('response', {}).get('numFound', 0)
docs = res.data.get('response', {}).get('docs', [])
highlighting.update(res.data.get('highlighting', {}))
for index, item in enumerate(docs):
itemId = item['id']
parent = item.get('parent', None)
position = toFetchStart + index
if item.get('itemType', '') == "message":
if (item.get('id'), parent) not in messages:
messages.append((item.get('id'), parent))
elif item.get('itemType', '') == 'people':
entityId = item.get('id')
if entityId not in people:
people.append(entityId)
toFetchEntities.add(entityId)
elif parent:
convItems.append((itemId, parent, position))
convs.add(parent)
else:
convItems.append((itemId, itemId, position))
convs.add(item.get('id'))
if convs:
filteredConvs, deleted = yield utils.fetchAndFilterConvs(convs, relation, items, myId, myOrgId)
for itemId, convId, position in convItems:
if convId in filteredConvs and itemId not in toFetchItems:
toFetchItems.append(itemId)
if len(toFetchItems) == count:
if position +1 < numMatched:
nextPageStart = position + 1
break
if len(toFetchItems) == count or len(docs) < count:
break
toFetchStart = toFetchStart + count
_items = yield db.multiget_slice(toFetchItems, "items",
['meta', 'attachments', 'tags'])
items.update(utils.multiSuperColumnsToDict(_items))
for itemId, item in items.iteritems():
toFetchEntities.add(item['meta']['owner'])
if 'target' in item['meta']:
toFetchEntities.update(item['meta']['target'].split(','))
if itemId in highlighting and 'comment' in highlighting[itemId]:
match = re.match(regex, unquote(highlighting[itemId]['comment'][0]))
if match:
comment = "".join(match.groups()[1:4])
comment = comment + " …" if match.group(5) else comment
items[itemId]['meta']['comment'] = comment
entities = yield db.multiget_slice(toFetchEntities, "entities", ['basic'])
entities = utils.multiSuperColumnsToDict(entities)
for userId in people:
if userId in highlighting and userId in entities:
entities[userId]['basic']['reason'] = {}
for key in highlighting[userId]:
if key in entities[userId]['basic']:
entities[userId]['basic'][key] = " ".join(highlighting[userId][key])
else:
entities[userId]['basic']['reason'][key] = highlighting[userId][key]
fromFetchMore = True if start else False
args['term'] = term
args['items'] = items
args['people'] = people
args['entities'] = entities
args['relations'] = relation
args["conversations"] = toFetchItems
args["nextPageStart"] = nextPageStart
args['fromFetchMore'] = fromFetchMore
args['fromSidebar'] = 'people' in filters.values()
if script:
onload = "(function(obj){$$.convs.load(obj);})(this);"
if fromFetchMore:
t.renderScriptBlock(request, "search.mako", "results",
landing, "#next-load-wrapper", "replace",
True, handlers={"onload": onload}, **args)
else:
t.renderScriptBlock(request, "search.mako", "results",
landing, "#search-results", "set", True,
handlers={"onload": onload}, **args)
if 'people' not in filters.values() and people:
t.renderScriptBlock(request, "search.mako", "_displayUsersMini",
landing, "#people-block", "set", True, **args)
if script and landing:
request.write("</body></html>")
if not script:
t.render(request, "search.mako", **args)
def _update_suggestions(request, relation=None):
authinfo = request.getSession(IAuthInfo)
myId = authinfo.username
orgId = authinfo.organization
weights = {'group': {'follower': 15, 'subscription': 40, 'group': 30},
'follower': {'follower': 9, 'subscription': 24, 'group': 15},
'subscription': {'follower': 24, 'subscription': 64, 'group': 40}}
defaultWeight = 1
people = {}
@defer.inlineCallbacks
def _compute_weights(userIds, myGroups, type):
followers = yield db.multiget_slice(userIds, "followers", count=50)
subscriptions = yield db.multiget_slice(userIds, "subscriptions", count=50)
groups = yield db.multiget_slice(userIds, "entityGroupsMap")
followers = utils.multiColumnsToDict(followers)
subscriptions = utils.multiColumnsToDict(subscriptions)
groups = utils.multiColumnsToDict(groups)
for userId in followers:
for follower in followers[userId]:
people[follower] = people.setdefault(follower, defaultWeight) + weights[type]['follower']
for userId in subscriptions:
for subscription in subscriptions[userId]:
people[subscription] = people.setdefault(subscription, defaultWeight) + weights[type]['subscription']
for userId in groups:
groupIds = [x.split(':', 1)[1] for x in groups[userId]]
for groupId in groupIds:
if groupId in myGroups:
people[userId] = people.setdefault(userId, defaultWeight) + weights[type]['group']
if not relation:
relation = Relation(myId, [])
yield defer.DeferredList([relation.initSubscriptionsList(),
relation.initFollowersList(),
relation.initGroupsList()])
if relation.followers:
yield _compute_weights(relation.followers, relation.groups, 'follower')
if relation.subscriptions:
yield _compute_weights(relation.subscriptions, relation.groups, 'subscription')
if relation.groups:
groupMembers = yield db.multiget_slice(relation.groups, "groupMembers", count=20)
groupMembers = utils.multiColumnsToDict(groupMembers)
for groupId in groupMembers:
yield _compute_weights(groupMembers[groupId], relation.groups, 'group')
cols = yield db.get_slice(orgId, "orgUsers", count=100)
for col in cols:
userId = col.column.name
if userId not in people:
people[userId] = people.setdefault(userId, 0) + defaultWeight
suggestions = {}
for userId in people:
if isValidSuggestion(myId, userId, relation):
suggestions.setdefault(people[userId], []).append(userId)
yield db.remove(myId, "suggestions")
weights_userIds_map = {}
format = '>l'
for weight in suggestions:
key = struct.pack(format, weight)
weights_userIds_map[key] = ' '.join(suggestions[weight])
if weights_userIds_map:
yield db.batch_insert(myId, "suggestions", weights_userIds_map)
