def get_suggestions(request, count, mini=False): authinfo = request.getSession(IAuthInfo) myId = authinfo.username SUGGESTIONS_UPDATE_FREQUENCY = 3 * 86400 # 5days MAX_INVALID_SUGGESTIONS = 3 now = time.time() suggestions = [] relation = Relation(myId, []) yield defer.DeferredList([relation.initSubscriptionsList(), relation.initFollowersList()]) @defer.inlineCallbacks def _get_suggestions(myId, relation): validSuggestions = [] invalidCount = 0 FORCE_UPDATE = False cols = yield db.get_slice(myId, "suggestions", reverse=True) for col in cols: if now - col.column.timestamp/1e6 > SUGGESTIONS_UPDATE_FREQUENCY: FORCE_UPDATE = True for userId in col.column.value.split(): if isValidSuggestion(myId, userId, relation): validSuggestions.append(userId) else: invalidCount += 1 defer.returnValue((validSuggestions, invalidCount, FORCE_UPDATE)) validSuggestions, invalidCount, FORCE_UPDATE = yield _get_suggestions(myId, relation) if not validSuggestions: yield _update_suggestions(request, relation) validSuggestions, invalidCount, FORCE_UPDATE = yield _get_suggestions(myId, relation) no_of_samples = count*2 population = count*5 if mini and len(validSuggestions) >= no_of_samples: suggestions = random.sample(validSuggestions[:population], no_of_samples) else: suggestions = validSuggestions[:] if FORCE_UPDATE or invalidCount > MAX_INVALID_SUGGESTIONS: _update_suggestions(request, relation) entities = base.EntitySet(suggestions[:count]) if suggestions: suggestions = suggestions[:count] yield entities.fetchData() defer.returnValue((suggestions, entities))
def _search(self, request): (appchange, script, args, myId) = yield self._getBasicArgs(request) landing = not self._ajax myOrgId = args['orgId'] filter_map = {'people':'itemType'} term = utils.getRequestArg(request, "q") start = utils.getRequestArg(request, "start") or 0 filters = utils.getRequestArg(request, 'filter', multiValued=True) or [] filters = dict([(filter_map[x], x) for x in filters if x in filter_map]) args["term"] = term nextPageStart = '' if not term: errors.MissingParams() try: start = int(start) if start < 0: raise ValueError except ValueError: errors.InvalidParamValue() if script and landing: t.render(request, "search.mako", **args) if script and appchange: t.renderScriptBlock(request, "search.mako", "layout", landing, "#mainbar", "set", **args) count = SEARCH_RESULTS_PER_PAGE items = {} convs = set() highlighting = {} toFetchItems = [] toFetchStart = start toFetchEntities = set() people = [] relation = Relation(myId, []) yield defer.DeferredList([relation.initGroupsList(), relation.initSubscriptionsList(), relation.initFollowersList()]) regex = re.compile("(.*?)([^\s]*\s*[^\s]*\s*[^\s]*\s*)(<em class='highlight'>.*<\/em>)(\s*[^\s]*\s*[^\s]*\s*[^\s]*)(.*)") res = yield solr.search(term, args['orgId'], count, toFetchStart, filters={'itemType': 'people'}) docs = res.data.get('response', {}).get('docs', []) for item in docs: entityId = item['id'] people.append(entityId) toFetchEntities.add(entityId) while 1: res = yield solr.search(term, args['orgId'], count, toFetchStart) messages = [] convItems = [] numMatched = res.data.get('response', {}).get('numFound', 0) docs = res.data.get('response', {}).get('docs', []) highlighting.update(res.data.get('highlighting', {})) for index, item in enumerate(docs): itemId = item['id'] parent = item.get('parent', None) position = toFetchStart + index if item.get('itemType', '') == "message": if (item.get('id'), parent) not in messages: messages.append((item.get('id'), parent)) elif item.get('itemType', '') == 'people': entityId = item.get('id') if entityId not in people: people.append(entityId) toFetchEntities.add(entityId) elif parent: convItems.append((itemId, parent, position)) convs.add(parent) else: convItems.append((itemId, itemId, position)) convs.add(item.get('id')) if convs: filteredConvs, deleted = yield utils.fetchAndFilterConvs(convs, relation, items, myId, myOrgId) for itemId, convId, position in convItems: if convId in filteredConvs and itemId not in toFetchItems: toFetchItems.append(itemId) if len(toFetchItems) == count: if position +1 < numMatched: nextPageStart = position + 1 break if len(toFetchItems) == count or len(docs) < count: break toFetchStart = toFetchStart + count _items = yield db.multiget_slice(toFetchItems, "items", ['meta', 'attachments', 'tags']) items.update(utils.multiSuperColumnsToDict(_items)) for itemId, item in items.iteritems(): toFetchEntities.add(item['meta']['owner']) if 'target' in item['meta']: toFetchEntities.update(item['meta']['target'].split(',')) if itemId in highlighting and 'comment' in highlighting[itemId]: match = re.match(regex, unquote(highlighting[itemId]['comment'][0])) if match: comment = "".join(match.groups()[1:4]) comment = comment + " …" if match.group(5) else comment items[itemId]['meta']['comment'] = comment entities = yield db.multiget_slice(toFetchEntities, "entities", ['basic']) entities = utils.multiSuperColumnsToDict(entities) for userId in people: if userId in highlighting and userId in entities: entities[userId]['basic']['reason'] = {} for key in highlighting[userId]: if key in entities[userId]['basic']: entities[userId]['basic'][key] = " ".join(highlighting[userId][key]) else: entities[userId]['basic']['reason'][key] = highlighting[userId][key] fromFetchMore = True if start else False args['term'] = term args['items'] = items args['people'] = people args['entities'] = entities args['relations'] = relation args["conversations"] = toFetchItems args["nextPageStart"] = nextPageStart args['fromFetchMore'] = fromFetchMore args['fromSidebar'] = 'people' in filters.values() if script: onload = "(function(obj){$$.convs.load(obj);})(this);" if fromFetchMore: t.renderScriptBlock(request, "search.mako", "results", landing, "#next-load-wrapper", "replace", True, handlers={"onload": onload}, **args) else: t.renderScriptBlock(request, "search.mako", "results", landing, "#search-results", "set", True, handlers={"onload": onload}, **args) if 'people' not in filters.values() and people: t.renderScriptBlock(request, "search.mako", "_displayUsersMini", landing, "#people-block", "set", True, **args) if script and landing: request.write("</body></html>") if not script: t.render(request, "search.mako", **args)
def _update_suggestions(request, relation=None): authinfo = request.getSession(IAuthInfo) myId = authinfo.username orgId = authinfo.organization weights = {'group': {'follower': 15, 'subscription': 40, 'group': 30}, 'follower': {'follower': 9, 'subscription': 24, 'group': 15}, 'subscription': {'follower': 24, 'subscription': 64, 'group': 40}} defaultWeight = 1 people = {} @defer.inlineCallbacks def _compute_weights(userIds, myGroups, type): followers = yield db.multiget_slice(userIds, "followers", count=50) subscriptions = yield db.multiget_slice(userIds, "subscriptions", count=50) groups = yield db.multiget_slice(userIds, "entityGroupsMap") followers = utils.multiColumnsToDict(followers) subscriptions = utils.multiColumnsToDict(subscriptions) groups = utils.multiColumnsToDict(groups) for userId in followers: for follower in followers[userId]: people[follower] = people.setdefault(follower, defaultWeight) + weights[type]['follower'] for userId in subscriptions: for subscription in subscriptions[userId]: people[subscription] = people.setdefault(subscription, defaultWeight) + weights[type]['subscription'] for userId in groups: groupIds = [x.split(':', 1)[1] for x in groups[userId]] for groupId in groupIds: if groupId in myGroups: people[userId] = people.setdefault(userId, defaultWeight) + weights[type]['group'] if not relation: relation = Relation(myId, []) yield defer.DeferredList([relation.initSubscriptionsList(), relation.initFollowersList(), relation.initGroupsList()]) if relation.followers: yield _compute_weights(relation.followers, relation.groups, 'follower') if relation.subscriptions: yield _compute_weights(relation.subscriptions, relation.groups, 'subscription') if relation.groups: groupMembers = yield db.multiget_slice(relation.groups, "groupMembers", count=20) groupMembers = utils.multiColumnsToDict(groupMembers) for groupId in groupMembers: yield _compute_weights(groupMembers[groupId], relation.groups, 'group') cols = yield db.get_slice(orgId, "orgUsers", count=100) for col in cols: userId = col.column.name if userId not in people: people[userId] = people.setdefault(userId, 0) + defaultWeight suggestions = {} for userId in people: if isValidSuggestion(myId, userId, relation): suggestions.setdefault(people[userId], []).append(userId) yield db.remove(myId, "suggestions") weights_userIds_map = {} format = '>l' for weight in suggestions: key = struct.pack(format, weight) weights_userIds_map[key] = ' '.join(suggestions[weight]) if weights_userIds_map: yield db.batch_insert(myId, "suggestions", weights_userIds_map)