def _new(self, request, data=None): (appchange, script, args, myId) = yield self._getBasicArgs(request) me = args['me'] landing = not self._ajax authInfo = request.getSession(IAuthInfo) convType = data['type'] convId, conv, keywords = yield Item.new(request, authInfo, convType) if keywords: block = t.getBlock('item.mako', 'requireReviewDlg', keywords=keywords) request.write('$$.convs.reviewRequired(%s);' % json.dumps(block)) return target = conv['meta'].get('target', None) toFetchEntities = set() if target: toFetchEntities.update(target.split(',')) convType = utils.getRequestArg(request, "type") plugin = plugins[convType] entityIds = yield plugin.fetchData(args, convId) toFetchEntities.update(entityIds) entities = base.EntitySet(toFetchEntities) yield entities.fetchData() entities.update(args['me']) relation = Relation(myId, []) yield relation.initGroupsList() data = {"items": {convId: conv}, "relations": relation, "entities": entities, "script": True} args.update(data) onload = "(function(obj){$$.convs.load(obj);$('#sharebar-attach-uploaded').empty();})(this);" t.renderScriptBlock(request, "item.mako", "item_layout", False, "#user-feed", "prepend", args=[convId, 'conv-item-created'], handlers={"onload": onload}, **args) defaultType = plugins.keys()[0] plugins[defaultType].renderShareBlock(request, True) if plugin and hasattr(plugin, 'renderFeedSideBlock'): entityId = myId referer = request.getHeader('referer') matchObj = feedPathObj.search(referer) if matchObj: matchedStr = matchObj.group('entityId') if matchedStr != "": entityId = matchedStr else: if target: entityId = target.split(',')[0] args["groupId"] = target.split(',')[0] request.write("$('#feed-side-block-container').empty();") yield plugin.renderFeedSideBlock(request, landing, entityId, args)
def createNewItem(request, itemType, owner, acl=None, subType=None, groupIds=None, richText=False): if not acl: acl = getRequestArg(request, "acl", sanitize=False) try: acl = json.loads(acl) orgs = acl.get("accept", {}).get("orgs", []) if len(orgs) > 1 or (len(orgs) == 1 and orgs[0] != owner.basic["org"]): msg = "Cannot grant access to other orgs on this item" raise errors.PermissionDenied(_(msg)) except: acl = {"accept": {"orgs": [owner.basic["org"]]}} accept_groups = acl.get("accept", {}).get("groups", []) deny_groups = acl.get("deny", {}).get("groups", []) groups = [x for x in accept_groups if x not in deny_groups] if groups: relation = Relation(owner.id, []) yield relation.initGroupsList() if not all([x in relation.groups for x in groups]): msg = "Only group members can post to a group" raise errors.PermissionDenied(_(msg)) acl = pickle.dumps(acl) item = { "meta": { "acl": acl, "org": owner.basic["org"], "type": itemType, "uuid": uuid.uuid1().bytes, "owner": owner.id, "timestamp": str(int(time.time())), "richText": str(richText), }, "followers": {owner.id: ""}, } if subType: item["meta"]["subType"] = subType if groups: item["meta"]["target"] = ",".join(groups) tmpFileIds = getRequestArg(request, "fId", False, True) attachments = {} if tmpFileIds: attachments = yield _upload_files(owner.id, tmpFileIds) if attachments: item["attachments"] = {} for attachmentId in attachments: fileId, name, size, ftype = attachments[attachmentId] item["attachments"][attachmentId] = "%s:%s:%s" % (name, size, ftype) defer.returnValue(item)
def getValidItemId(request, arg, type=None, columns=None, itemId=None, myOrgId=None, myId=None): if not itemId: itemId = getRequestArg(request, arg, sanitize=False) itemType = type if type else "item" if not itemId: raise errors.MissingParams([_("%s id") % _(itemType).capitalize()]) columns = [] if not columns else columns columns.extend(["meta", "attachments"]) item = yield db.get_slice(itemId, "items", columns) if not item: raise errors.InvalidItem(itemType, itemId) item = supercolumnsToDict(item) meta = item["meta"] if type and meta["type"] != type: raise errors.InvalidItem(itemType, itemId) parentId = meta.get("parent", None) if parentId: parent = yield db.get_slice(parentId, "items", ["meta"]) parent = supercolumnsToDict(parent) acl = parent["meta"]["acl"] owner = parent["meta"]["owner"] deleted = parent["meta"].get("state", None) == "deleted" else: parent = item acl = meta["acl"] owner = meta["owner"] deleted = parent["meta"].get("state", None) == "deleted" if deleted: raise errors.InvalidItem(itemType, itemId) if not myOrgId: myOrgId = request.getSession(IAuthInfo).organization if not myId: myId = request.getSession(IAuthInfo).username relation = Relation(myId, []) yield relation.initGroupsList() if not checkAcl(myId, myOrgId, False, relation, parent["meta"]): raise errors.ItemAccessDenied(itemType, itemId) defer.returnValue((itemId, item))
def _to_python(self, itemId, state): itemType = self.itemType if self.itemType else 'item' if not itemId or not itemId[0]: raise MissingParam('%s-id' % (itemType), itemId, state) itemId = itemId[0] columns = set(['meta']).update(self.columns) item = yield db.get_slice(itemId, "items", columns) if not item: raise InvalidItem(itemType, itemId, state) item = utils.supercolumnsToDict(item) meta = item["meta"] if self.itemType and meta["type"] != self.itemType: raise InvalidItem(itemType, itemId, state) parentId = meta.get("parent", None) if parentId: parent = yield db.get_slice(parentId, "items", ["meta"]) parent = utils.supercolumnsToDict(parent) else: parent = item acl = parent["meta"]["acl"] owner = parent["meta"]["owner"] #parent is deleted deleted = parent['meta'].get('state', None) == 'deleted' #item is deleted deleted = deleted or meta.get('state', None) == 'deleted' if deleted: raise InvalidItem(itemType, itemId, state) if self.source != 'api': request = state.request authInfo = request.getSession(IAuthInfo) myId = authInfo.username orgId = authInfo.organization isOrgAdmin = authInfo.isAdmin isOrgAdmin = self.checkAdmin and isOrgAdmin relation = Relation(myId, []) yield relation.initGroupsList() if not utils.checkAcl(myId, orgId, isOrgAdmin, relation, parent['meta']): raise ItemAccessDenied(itemType, itemId, state) defer.returnValue((itemId, item))
def _getUserItems(self, request, userId, start='', count=10): authinfo = request.getSession(IAuthInfo) myId = authinfo.username myOrgId = authinfo.organization toFetchItems = set() toFetchEntities = set() toFetchTags = set() toFetchResponses = set() toFetchCount = count + 1 toFetchStart = utils.decodeKey(start) if start else '' fetchedUserItem = [] responses = {} convs = [] userItemsRaw = [] userItems = [] reasonStr = {} timestamps = {} items = {} nextPageStart = None args = {'myId': myId} relation = Relation(myId, []) yield relation.initGroupsList() toFetchEntities.add(userId) while len(convs) < toFetchCount: cols = yield db.get_slice(userId, "userItems", start=toFetchStart, reverse=True, count=toFetchCount) tmpIds = [] for col in cols: convId = col.column.value.split(":")[2] if convId not in tmpIds and convId not in convs: tmpIds.append(convId) (filteredConvs, deletedConvs) = yield utils.fetchAndFilterConvs\ (tmpIds, relation, items, myId, myOrgId) for col in cols[0:count]: convId = col.column.value.split(":")[2] if len(convs) == count or len(fetchedUserItem) == count*2: nextPageStart = col.column.name break if convId not in filteredConvs and convId not in convs: continue fetchedUserItem.append(col) if convId not in convs: convs.append(convId) if len(cols) < toFetchCount or nextPageStart: break if cols: toFetchStart = cols[-1].column.name if nextPageStart: nextPageStart = utils.encodeKey(nextPageStart) for col in fetchedUserItem: value = tuple(col.column.value.split(":")) timestamps[value] = col.column.timestamp/1e6 rtype, itemId, convId, convType, convOwnerId, commentSnippet = value commentSnippet = """<span class="snippet">"%s"</span>""" %(_(commentSnippet)) toFetchEntities.add(convOwnerId) if rtype == 'I': toFetchItems.add(convId) toFetchResponses.add(convId) userItems.append(value) elif rtype == "L" and itemId == convId and convOwnerId != userId: reasonStr[value] = _("liked %s's %s") userItems.append(value) elif rtype == "L" and convOwnerId != userId: r = "answer" if convType == 'question' else 'comment' reasonStr[value] = _("liked") + " %s " %(commentSnippet) + _("%s "%r) + _("on %s's %s") userItems.append(value) elif rtype in ["C", 'Q'] and convOwnerId != userId: reasonStr[value] = "%s"%(commentSnippet) + _(" on %s's %s") userItems.append(value) itemResponses = yield db.multiget_slice(toFetchResponses, "itemResponses", count=2, reverse=True) for convId, comments in itemResponses.items(): responses[convId] = [] for comment in comments: userId_, itemKey = comment.column.value.split(':') if itemKey not in toFetchItems: responses[convId].insert(0,itemKey) toFetchItems.add(itemKey) toFetchEntities.add(userId_) items = yield db.multiget_slice(toFetchItems, "items", ["meta", "tags", "attachments"]) items = utils.multiSuperColumnsToDict(items) args["items"] = items extraDataDeferreds = [] for convId in convs: if convId not in items: continue meta = items[convId]["meta"] itemType = meta["type"] toFetchEntities.add(meta["owner"]) if "target" in meta: toFetchEntities.update(meta["target"].split(',')) toFetchTags.update(items[convId].get("tags", {}).keys()) if itemType in plugins: d = plugins[itemType].fetchData(args, convId) extraDataDeferreds.append(d) result = yield defer.DeferredList(extraDataDeferreds) for success, ret in result: if success: toFetchEntities.update(ret) entities = base.EntitySet(toFetchEntities) yield entities.fetchData() tags = {} if toFetchTags: userOrgId = entities[userId].basic["org"] fetchedTags = yield db.get_slice(userOrgId, "orgTags", toFetchTags) tags = utils.supercolumnsToDict(fetchedTags) fetchedLikes = yield db.multiget(toFetchItems, "itemLikes", myId) myLikes = utils.multiColumnsToDict(fetchedLikes) data = {"entities": entities, "reasonStr": reasonStr, "tags": tags, "myLikes": myLikes, "userItems": userItems, "responses": responses, "nextPageStart": nextPageStart, "timestamps": timestamps } del args['myId'] args.update(data) defer.returnValue(args)
def _render(self, request): (appchange, script, args, myId) = yield self._getBasicArgs(request) # We are setting an empty value to 'cu' here just to make sure that # any errors when looking validating the entity should not leave us # in a bad state. request.addCookie('cu', '', path="/ajax/profile") if request.args.get("id", None): userId, ign = yield utils.getValidEntityId(request, "id", "user") else: userId = myId # XXX: We should use getValidEntityId to fetch the entire user # info instead of another call to the database. request.addCookie('cu', userId, path="/ajax/profile") user = base.Entity(userId) yield user.fetchData([]) if user._data: args['user'] = user detail = utils.getRequestArg(request, "dt") or "activity" args["detail"] = detail args["userId"] = userId args["menuId"] = "people" args["entities"] = base.EntitySet({myId:args['me'], userId:user}) # When scripts are enabled, updates are sent to the page as # and when we get the required data from the database. # When we are the landing page, we also render the page header # and all updates are wrapped in <script> blocks. landing = not self._ajax # User entered the URL directly # Render the header. Other things will follow. if script and landing: t.render(request, "profile.mako", **args) # Start with displaying the template and navigation menu if script and appchange: t.renderScriptBlock(request, "profile.mako", "layout", landing, "#mainbar", "set", **args) # Prefetch some data about how I am related to the user. # This is required in order to reliably filter our profile details # that I don't have access to. relation = Relation(myId, [userId]) args["relations"] = relation yield defer.DeferredList([relation.initSubscriptionsList(), relation.initGroupsList()]) # Reload all user-depended blocks if the currently displayed user is # not the same as the user for which new data is being requested. if script: t.renderScriptBlock(request, "profile.mako", "summary", landing, "#profile-summary", "set", **args) t.renderScriptBlock(request, "profile.mako", "user_subactions", landing, "#user-subactions", "set", **args) fetchedEntities = set() start = utils.getRequestArg(request, "start") or '' fromFetchMore = ((not landing) and (not appchange) and start) if detail == "activity": userItems = yield self._getUserItems(request, userId, start=start) args.update(userItems) elif detail == 'files': end = utils.getRequestArg(request, "end") or '' end = utils.decodeKey(end) start = utils.decodeKey(start) userFiles = yield files.userFiles(myId, userId, args['orgId'], start, end, fromFeed=False) args['userfiles'] = userFiles args['fromProfile'] = True if script: t.renderScriptBlock(request, "profile.mako", "tabs", landing, "#profile-tabs", "set", **args) handlers = {} if detail != "activity" \ else {"onload": "(function(obj){$$.convs.load(obj);})(this);"} if fromFetchMore and detail == "activity": t.renderScriptBlock(request, "profile.mako", "content", landing, "#next-load-wrapper", "replace", True, handlers=handlers, **args) else: t.renderScriptBlock(request, "profile.mako", "content", landing, "#profile-content", "set", True, handlers=handlers, **args) # List the user's subscriptions cols = yield db.get_slice(userId, "subscriptions", count=11) subscriptions = set(utils.columnsToDict(cols).keys()) args["subscriptions"] = subscriptions # List the user's followers cols = yield db.get_slice(userId, "followers", count=11) followers = set(utils.columnsToDict(cols).keys()) args["followers"] = followers # Fetch item data (name and avatar) for subscriptions, followers, # user groups and common items. entitiesToFetch = followers.union(subscriptions)\ .difference(fetchedEntities) # List the user's groups (and look for groups common with me) cols = yield db.multiget_slice([myId, userId], "entityGroupsMap") myGroups = set([x.column.name.split(':', 1)[1] for x in cols[myId]]) userGroups = set([x.column.name.split(':', 1)[1] for x in cols[userId]]) commonGroups = myGroups.intersection(userGroups) if len(userGroups) > 10: userGroups = sample(userGroups, 10) args["userGroups"] = userGroups args["commonGroups"] = commonGroups groupsToFetch = commonGroups.union(userGroups) entitiesToFetch = entitiesToFetch.union(groupsToFetch) entities = base.EntitySet(entitiesToFetch) yield entities.fetchData() for entityId, entity in entities.items(): if not entity._data: del entities[entityId] args["entities"].update(entities) if script: t.renderScriptBlock(request, "profile.mako", "user_subscriptions", landing, "#user-subscriptions", "set", **args) t.renderScriptBlock(request, "profile.mako", "user_followers", landing, "#user-followers", "set", **args) t.renderScriptBlock(request, "profile.mako", "user_me", landing, "#user-me", "set", **args) t.renderScriptBlock(request, "profile.mako", "user_groups", landing, "#user-groups", "set", **args) if script and landing: request.write("</body></html>") if not script: t.render(request, "profile.mako", **args)
def get(auth, feedId=None, feedItemsId=None, convIds=None, getFn=None, cleanFn=None, start='', count=10, getReasons=True, forceCount=False, itemType=None): """Fetch data from feed represented by feedId. Returns a dictionary that has the items from feed, start of the next page and responses and likes that I know of. Keyword params: @auth - An instance of AuthInfo representing the authenticated user @feedId - Id of the feed from which the data is to be fetched @feedItemsId - Id of the feed from with feed items must be fetched @convIds - List of conversation id's to be used as feed @getFn - Function that must be used to fetch items @cleanFn - Function that must be used to clean items that don't exist @start - Id of the item where the fetching must start @count - Number of items to fetch @getReasons - Add reason strings to the returned dictionary @forceCount - Try hard to get atleast count items from the feed """ toFetchItems = set() # Items and entities that need to be fetched toFetchEntities = set() # toFetchTags = set() # items = {} # Fetched items, entities and tags entities = base.EntitySet([])# tags = {} # deleted = [] # List of items that were deleted deleteKeys = [] # List of keys that need to be deleted responses = {} # Cached data of item responses and likes likes = {} # myLikes = {} myId = auth.username orgId = auth.organization feedSource = "feed_%s" % itemType\ if itemType and itemType in plugins\ and plugins[itemType].hasIndex\ else "feed" feedItemsId = feedItemsId or myId feedItems_d = [] # List of deferred used to fetch feedItems # Used for checking ACL relation = Relation(myId, []) yield relation.initGroupsList() # The updates that will be used to build reason strings convReasonUpdates = {} # Data that is sent to various plugins and returned by this function # XXX: myKey is depricated - use myId data = {"myId": myId, "orgId": orgId, "responses": responses, "likes": likes, "myLikes": myLikes, "items": items, "entities": entities, "tags": tags, "myKey": myId, "relations": relation} @defer.inlineCallbacks def fetchFeedItems(ids): rawFeedItems = yield db.get_slice(feedItemsId, "feedItems", ids) \ if ids else defer.succeed([]) for conv in rawFeedItems: convId = conv.super_column.name convUpdates = conv.super_column.columns responses[convId] = [] likes[convId] = [] latest = None updatesByType = {} for update in convUpdates: parts = update.value.split(':') updatesByType.setdefault(parts[0], []).append(parts) if parts[1] != myId: # Ignore my own updates latest = parts # when displaying latest actors # Parse all notification to make sure we fetch any required # items, entities. and cache generic stuff that we display for tipe in updatesByType.keys(): updates = updatesByType[tipe] if tipe in _feedUpdatePlugins: (i,e) = _feedUpdatePlugins[tipe].parse(convId, updates) toFetchItems.update(i) toFetchEntities.update(e) if tipe == "L": for update in updates: if update[2] == convId: likes[convId].append(update[1]) # XXX: Adding this item may break the sorting # of responses on this conversation # Bug #493 #else: # responses[convId].append(update[2]) elif tipe in ["C", "Q"]: for update in updates: responses[convId].append(update[2]) # Store any information that can be used to render # the reason strings when we have the required data if getReasons and latest: convReasonUpdates[convId] = updatesByType[latest[0]] # Fetch the feed if required and at the same time make sure # we delete unwanted items from the feed (cleanup). # NOTE: We assume that there will be very few deletes. nextPageStart = None if not convIds: feedId = feedId or myId allFetchedConvIds = set() # Complete set of convIds fetched itemsFromFeed = {} # All the key-values retrieved from feed keysFromFeed = [] # Sorted list of keys (used for paging) convIds = [] # List of convIds that will be displayed fetchStart = utils.decodeKey(start) fetchCount = count + 1 while len(convIds) < count: fetchedConvIds = [] # Use the getFn function if given. # NOTE: Part of this code is duplicated just below this. if getFn: results = yield getFn(start=fetchStart, count=fetchCount) for name, value in results.items(): keysFromFeed.append(name) if value not in allFetchedConvIds: fetchedConvIds.append(value) allFetchedConvIds.add(value) itemsFromFeed[name] = value else: deleteKeys.append(name) # Fetch user's feed when getFn isn't given. # NOTE: Part of this code is from above else: results = yield db.get_slice(feedId, feedSource, count=fetchCount, start=fetchStart, reverse=True) for col in results: value = col.column.value keysFromFeed.append(col.column.name) if value not in allFetchedConvIds: fetchedConvIds.append(value) allFetchedConvIds.add(value) itemsFromFeed[col.column.name] = value else: deleteKeys.append(col.column.name) # Initiate fetching feed items for all the conversation Ids. # Meanwhile we check if the authenticated user has access to # all the fetched conversation ids. # NOTE: Conversations would rarely be filtered out. So, we # just go ahead with fetching data for all convs. feedItems_d.append(fetchFeedItems(fetchedConvIds)) (filteredConvIds, deletedIds) = yield utils.fetchAndFilterConvs\ (fetchedConvIds, relation, items, myId, orgId) convIds.extend(filteredConvIds) deleted.extend(deletedIds) # Unless we are forced to fetch count number of items, we only # iterate till we fetch atleast half of them if (not forceCount and len(convIds) > (count/2)) or\ len(results) < fetchCount: break # If we need more items, we start fetching from where we # left in the previous iteration. fetchStart = keysFromFeed[-1] # If DB fetch got as many items as I requested # there may be additional items present in the feed # So, we cut one item from what we return and start the # next page from there. if len(results) == fetchCount: lastConvId = convIds[-1] for key in reversed(keysFromFeed): if key in itemsFromFeed and itemsFromFeed[key] == lastConvId: nextPageStart = utils.encodeKey(key) convIds = convIds[:-1] else: (convIds, deletedIds) = yield utils.fetchAndFilterConvs(convIds, relation, items, myId, orgId) # NOTE: Unlike the above case where we fetch convIds from # database (where we set the nextPageStart to a key), # here we set nextPageStart to the convId. if len(convIds) > count: nextPageStart = utils.encodeKey(convIds[count]) convIds = convIds[0:count] # Since convIds were directly passed to us, we would also # return the list of convIds deleted back to the caller. if deletedIds: data["deleted"] = deletedIds # We don't have any conversations to display! if not convIds: defer.returnValue({"conversations": []}) # Delete any convs that were deleted from the feeds and # any duplicates that were marked for deletion cleanup_d = [] if deleted: for key, value in itemsFromFeed.items(): if value in deleted: deleteKeys.append(key) if cleanFn: d1 = cleanFn(list(deleteKeys)) else: d1 = db.batch_remove({feedSource: [feedId]}, names=deleteKeys) d2 = db.batch_remove({'feedItems': [feedId]}, names=list(deleted)) cleanup_d = [d1, d2] # We now have a filtered list of conversations that can be displayed # Let's wait till all the feed items have been fetched and processed yield defer.DeferredList(feedItems_d) # Fetch the remaining items (comments on the actual conversations) items_d = db.multiget_slice(toFetchItems, "items", ["meta" ,"attachments"]) # Fetch tags on all the conversations that will be displayed for convId in convIds: conv = items[convId] toFetchEntities.add(conv["meta"]["owner"]) if "target" in conv["meta"]: toFetchEntities.update(conv["meta"]["target"].split(',')) toFetchTags.update(conv.get("tags",{}).keys()) tags_d = db.get_slice(orgId, "orgTags", toFetchTags) \ if toFetchTags else defer.succeed([]) # Fetch the list of my likes. # XXX: Latency can be pretty high here becuase many nodes will have to # be contacted for the information. Alternative could be to cache # all likes by a user somewhere. myLikes_d = db.multiget(toFetchItems.union(convIds), "itemLikes", myId) # Fetch extra data that is required to render special items # We already fetched the conversation items, plugins merely # add more data to the already fetched items for convId in convIds[:]: itemType = items[convId]["meta"]["type"] if itemType in plugins: try: entityIds = yield plugins[itemType].fetchData(data, convId) toFetchEntities.update(entityIds) except Exception, e: log.err(e) convIds.remove(convId)
def fetchMatchingEvents(self, request, args, entityId, count=5, start=None): """Find matching events for the user, org or group for a given time range. Events are sorted by their start time and then by their end time. """ myId = args["myId"] convs = [] invitations = [] toFetchEntities = set() my_tz = timezone(args["me"].basic["timezone"]) if not start: # since we store times in UTC, find out the utc time for the user's # 00:00 hours instead of utc 00:00. utc_now = datetime.datetime.now(pytz.utc) mytz_now = utc_now.astimezone(my_tz) mytz_start = mytz_now else: mytz_start = start.replace(tzinfo=my_tz) args["start"] = mytz_start.strftime("%Y-%m-%d") timestamp = calendar.timegm(mytz_start.utctimetuple()) timeUUID = utils.uuid1(timestamp=timestamp) start = timeUUID.bytes page = args.get("page", 1) cols = yield db.get_slice(entityId, "userAgenda", start=start, count=(page*count)*2) matched_events = [col.column.value for col in cols] res = yield db.multiget_slice(matched_events, "items", ["meta"]) matched_events = utils.multiSuperColumnsToDict(res) to_sort_time_tuples = [(x, y["meta"]["event_startTime"], y["meta"]["event_endTime"]) \ for x, y in matched_events.iteritems()] sorted_time_tuples = sorted(to_sort_time_tuples, key=itemgetter(int(1), int(2))) sorted_event_ids = [x[0] for x in sorted_time_tuples] events_in_this_page = sorted_event_ids[(page-1)*count:page*count] if len(events_in_this_page) >= count: nextPage = page + 1 args.update({'nextPage': nextPage}) else: args.update({'nextPage': 0}) args["prevPage"] = page - 1 args["items"] = matched_events args["conversations"] = events_in_this_page #Now fetch all related entities, participants, owners, attendees, # invitees, groups etc for convId in events_in_this_page: entityIds = yield self.fetchData(args, convId) toFetchEntities.update(entityIds) relation = Relation(myId, []) yield relation.initGroupsList() for event, event_meta in matched_events.iteritems(): target = event_meta['meta'].get('target') if target: toFetchEntities.update(target.split(',')) entities = base.EntitySet(toFetchEntities) yield entities.fetchData() args["entities"] = entities args["relations"] = relation
def _invite(self, request, convId=None): (appchange, script, args, myId) = yield self._getBasicArgs(request) landing = not self._ajax myOrgId = args['orgId'] convId, conv = yield utils.getValidItemId(request, "id", columns=["invitees"]) # Parse invitees from the tag edit plugin values arg_keys = request.args.keys() invitees, new_invitees = [], [] for arg in arg_keys: if arg.startswith("invitee[") and arg.endswith("-a]"): rcpt = arg.replace("invitee[", "").replace("-a]", "") if rcpt != "": invitees.append(rcpt) if myId not in conv["invitees"].keys(): raise errors.invalidRequest(_("Only those who are invited can invite others")) res = base.EntitySet(invitees) yield res.fetchData() invitees = [x for x in res.keys() if res[x].basic["org"] == myOrgId] invitees = [x for x in invitees if x not in conv["invitees"].keys()] relation = Relation(myId, []) updateConv = {"meta":{}, "invitees":{}} if myId == conv["meta"]["owner"]: #If invited by owner, add the invitees to the ACL acl = conv["meta"]["acl"] acl = pickle.loads(acl) acl.setdefault("accept", {}).setdefault("users", []) acl["accept"]["users"].extend(invitees) updateConv["meta"]["acl"] = pickle.dumps(acl) new_invitees.extend(invitees) else: for invitee in invitees: relation = Relation(invitee, []) yield relation.initGroupsList() withinAcl = utils.checkAcl(invitee, myOrgId, False, relation, conv["meta"]) if withinAcl: new_invitees.append(invitee) if new_invitees: convMeta = conv["meta"] starttime = int(convMeta["event_startTime"]) starttimeUUID = utils.uuid1(timestamp=starttime) starttimeUUID = starttimeUUID.bytes endtime = int(convMeta["event_endTime"]) endtimeUUID = utils.uuid1(timestamp=endtime) endtimeUUID = endtimeUUID.bytes updateConv["invitees"] = dict([(x, myId) for x in new_invitees]) d = yield db.batch_insert(convId, "items", updateConv) yield event.inviteUsers(request, starttimeUUID, endtimeUUID, convId, conv["meta"], myId, myOrgId, new_invitees) request.write("""$$.alerts.info('%s');""" \ %("%d people invited to this event" %len(new_invitees))) else: if not invitees: request.write("""$$.alerts.info('%s');""" \ %("Invited persons are already on the invitation list")) else: request.write("""$$.alerts.info('%s');""" \ %("Invited persons do not have access to this event")) request.write("$('#item-subactions .tagedit-listelement-old').remove();")
def _search(self, request): (appchange, script, args, myId) = yield self._getBasicArgs(request) landing = not self._ajax myOrgId = args['orgId'] filter_map = {'people':'itemType'} term = utils.getRequestArg(request, "q") start = utils.getRequestArg(request, "start") or 0 filters = utils.getRequestArg(request, 'filter', multiValued=True) or [] filters = dict([(filter_map[x], x) for x in filters if x in filter_map]) args["term"] = term nextPageStart = '' if not term: errors.MissingParams() try: start = int(start) if start < 0: raise ValueError except ValueError: errors.InvalidParamValue() if script and landing: t.render(request, "search.mako", **args) if script and appchange: t.renderScriptBlock(request, "search.mako", "layout", landing, "#mainbar", "set", **args) count = SEARCH_RESULTS_PER_PAGE items = {} convs = set() highlighting = {} toFetchItems = [] toFetchStart = start toFetchEntities = set() people = [] relation = Relation(myId, []) yield defer.DeferredList([relation.initGroupsList(), relation.initSubscriptionsList(), relation.initFollowersList()]) regex = re.compile("(.*?)([^\s]*\s*[^\s]*\s*[^\s]*\s*)(<em class='highlight'>.*<\/em>)(\s*[^\s]*\s*[^\s]*\s*[^\s]*)(.*)") res = yield solr.search(term, args['orgId'], count, toFetchStart, filters={'itemType': 'people'}) docs = res.data.get('response', {}).get('docs', []) for item in docs: entityId = item['id'] people.append(entityId) toFetchEntities.add(entityId) while 1: res = yield solr.search(term, args['orgId'], count, toFetchStart) messages = [] convItems = [] numMatched = res.data.get('response', {}).get('numFound', 0) docs = res.data.get('response', {}).get('docs', []) highlighting.update(res.data.get('highlighting', {})) for index, item in enumerate(docs): itemId = item['id'] parent = item.get('parent', None) position = toFetchStart + index if item.get('itemType', '') == "message": if (item.get('id'), parent) not in messages: messages.append((item.get('id'), parent)) elif item.get('itemType', '') == 'people': entityId = item.get('id') if entityId not in people: people.append(entityId) toFetchEntities.add(entityId) elif parent: convItems.append((itemId, parent, position)) convs.add(parent) else: convItems.append((itemId, itemId, position)) convs.add(item.get('id')) if convs: filteredConvs, deleted = yield utils.fetchAndFilterConvs(convs, relation, items, myId, myOrgId) for itemId, convId, position in convItems: if convId in filteredConvs and itemId not in toFetchItems: toFetchItems.append(itemId) if len(toFetchItems) == count: if position +1 < numMatched: nextPageStart = position + 1 break if len(toFetchItems) == count or len(docs) < count: break toFetchStart = toFetchStart + count _items = yield db.multiget_slice(toFetchItems, "items", ['meta', 'attachments', 'tags']) items.update(utils.multiSuperColumnsToDict(_items)) for itemId, item in items.iteritems(): toFetchEntities.add(item['meta']['owner']) if 'target' in item['meta']: toFetchEntities.update(item['meta']['target'].split(',')) if itemId in highlighting and 'comment' in highlighting[itemId]: match = re.match(regex, unquote(highlighting[itemId]['comment'][0])) if match: comment = "".join(match.groups()[1:4]) comment = comment + " …" if match.group(5) else comment items[itemId]['meta']['comment'] = comment entities = yield db.multiget_slice(toFetchEntities, "entities", ['basic']) entities = utils.multiSuperColumnsToDict(entities) for userId in people: if userId in highlighting and userId in entities: entities[userId]['basic']['reason'] = {} for key in highlighting[userId]: if key in entities[userId]['basic']: entities[userId]['basic'][key] = " ".join(highlighting[userId][key]) else: entities[userId]['basic']['reason'][key] = highlighting[userId][key] fromFetchMore = True if start else False args['term'] = term args['items'] = items args['people'] = people args['entities'] = entities args['relations'] = relation args["conversations"] = toFetchItems args["nextPageStart"] = nextPageStart args['fromFetchMore'] = fromFetchMore args['fromSidebar'] = 'people' in filters.values() if script: onload = "(function(obj){$$.convs.load(obj);})(this);" if fromFetchMore: t.renderScriptBlock(request, "search.mako", "results", landing, "#next-load-wrapper", "replace", True, handlers={"onload": onload}, **args) else: t.renderScriptBlock(request, "search.mako", "results", landing, "#search-results", "set", True, handlers={"onload": onload}, **args) if 'people' not in filters.values() and people: t.renderScriptBlock(request, "search.mako", "_displayUsersMini", landing, "#people-block", "set", True, **args) if script and landing: request.write("</body></html>") if not script: t.render(request, "search.mako", **args)
def search(self, request): (appchange, script, args, myId) = yield self._getBasicArgs(request) landing = not self._ajax myOrgId = args['orgId'] term = utils.getRequestArg(request, "q") if not term: errors.MissingParams(['Search term']) args["term"] = term itemType = utils.getRequestArg(request, "it") or 0 try: itemType = int(itemType) if not 0 < itemType < 31: raise ValueError except ValueError: itemType = self.TYPE_ITEMS | self.TYPE_PEOPLE args["itemType"] = itemType start = utils.getRequestArg(request, "start") or 0 try: start = int(start) if start < 0: raise ValueError except ValueError: start = 0 args["start"] = start if script and landing: t.render(request, "search.mako", **args) if script and appchange: t.renderScriptBlock(request, "search.mako", "layout", landing, "#mainbar", "set", **args) toFetchEntities = set() toFetchItems = set() toFetchTags = set() deferreds = [] highlight = {} # If searching for more than one itemType then use half the count. count = SEARCH_RESULTS_PER_PAGE if itemType in [1,2,4,8,16] else SEARCH_RESULTS_PER_PAGE/2 relation = Relation(myId, []) relation_d = relation.initGroupsList() args['relations'] = relation people = {} args['matchedUsers'] = people if itemType & self.TYPE_PEOPLE: d = solr.search(term, myOrgId, count, start, filters={'_type':'people'}) def _gotPeople(results): docs = results.data.get('response', {}).get('docs', []) highlight.update(results.data.get('highlighting')) for item in docs: entityId = item['id'] people[entityId] = item args['matchedUserCount'] = results.data.get('response', {}).get('numFound', 0) d.addCallback(_gotPeople) deferreds.append(d) items = {} matchedItemIds = [] args['items'] = items args['matchedItemIds'] = matchedItemIds if itemType & self.TYPE_ITEMS: yield relation_d aclFilterEntities = relation.groups + [myOrgId, myId] filters = {'_type': 'item', '_acceptACL': '(%s)' % ' OR '.join(aclFilterEntities), '-_denyACL': '(%s)' % ' OR '.join(aclFilterEntities)} d = solr.search(term, myOrgId, count, start, filters=filters) @defer.inlineCallbacks def _gotConvs(results): docs = results.data.get('response', {}).get('docs', []) highlight.update(results.data.get('highlighting')) for index, item in enumerate(docs): itemId = item['id'] parentId = item.get('parent', None) if parentId: toFetchItems.add(itemId) toFetchItems.add(parentId) matchedItemIds.append(itemId) else: toFetchItems.add(itemId) matchedItemIds.append(itemId) if toFetchItems and matchedItemIds: fetchedItems = yield db.multiget_slice(toFetchItems, "items", ["meta", "tags", "attachments"]) fetchedItems = utils.multiSuperColumnsToDict(fetchedItems) for itemId, item in fetchedItems.items(): toFetchEntities.add(item['meta']['owner']) if 'target' in item['meta']: toFetchEntities.update(item['meta']['target'].split(',')) items.update(fetchedItems) extraDataDeferreds = [] for itemId in toFetchItems: if itemId in matchedItemIds and itemId in items: meta = items[itemId]['meta'] itemType = meta.get('type', 'status') if itemType in plugins: d = plugins[itemType].fetchData(args, itemId) extraDataDeferreds.append(d) result = yield defer.DeferredList(extraDataDeferreds) for success, ret in result: if success: toFetchEntities.update(ret) args['matchedItemCount'] = results.data.get('response', {}).get('numFound', 0) d.addCallback(_gotConvs) deferreds.append(d) yield defer.DeferredList(deferreds) entities = base.EntitySet(toFetchEntities) args['entities'] = entities if toFetchEntities: yield entities.fetchData() tags = {} args['tags'] = tags if toFetchTags: fetchedTags = yield db.get_slice(myOrgId, "orgTags", toFetchTags) tags.update(utils.supercolumnsToDict(fetchedTags)) args['highlight'] = highlight if script: t.renderScriptBlock(request, "search.mako", "results", landing, "#search-results", "set", **args) else: t.render(request, "search.mako", **args)
def userFiles(myId, entityId, myOrgId, start='', end='', fromFeed=True): allItems = {} hasPrevPage = False # Do we have another page before the current one. nextPageStart = '' # Start item for the next page accessibleFiles = [] accessibleItems = [] toFetchEntities = set() count = constants.FILES_PER_PAGE toFetchCount = count + 1 relation = Relation(myId, []) yield relation.initGroupsList() # Fetching files owned by entityId or files that were part of entityId's feed. cf = 'entityFeed_files' if fromFeed else 'user_files' # End is actually the start item of next page. # If @end is set, we have to display @count items before @end. For that # we fetch @count + 2 items before (and including) @end. Of the extra items # fetched, one item helps us determine if there is another page before this # and the other one is the start of next page. if end: start = end reverse = False toFetchCount += 1 else: reverse = True while 1: files = yield db.get_slice(entityId, cf, count=toFetchCount, start=start, reverse=reverse) files = utils.columnsToDict(files, True) toFetchItems = [] for tuuid in files: if len(files[tuuid].split(':')) == 4: fid, name, itemId, attachmentId = files[tuuid].split(':') toFetchItems.append(itemId) toFetchItems = [itemId for itemId in toFetchItems if itemId not in accessibleItems] if toFetchItems: items = yield db.multiget_slice(toFetchItems, "items", ["meta"]) items = utils.multiSuperColumnsToDict(items) toFetchConvIds = [items[itemId]['meta']['parent'] for itemId in items if 'parent' in items[itemId]['meta'] and items[itemId]['meta']['parent'] not in allItems] if toFetchConvIds: convs = yield db.multiget_slice(toFetchConvIds, "items", ["meta"]) convs = utils.multiSuperColumnsToDict(convs) allItems.update(convs) allItems.update(items) for itemId in items: if 'parent' in items[itemId]['meta']: convId = items[itemId]['meta']['parent'] acl = allItems[convId]['meta']['acl'] else: acl = items[itemId]['meta']['acl'] convId = itemId if utils.checkAcl(myId, myOrgId, False, relation, allItems[convId]['meta']): accessibleItems.append(itemId) for tuuid in files: if len(files[tuuid].split(':')) == 4: fid, name, itemId, ownerId = files[tuuid].split(':') if itemId in accessibleItems: accessibleFiles.append((tuuid, (fid, urlsafe_b64decode(name), itemId, ownerId, allItems[itemId]))) toFetchEntities.add(ownerId) if len(files) < toFetchCount or len(accessibleFiles) > count: break else: start = files.keys()[-1] if end: # We have enough items to have another page before this. if len(accessibleFiles) > count + 1: hasPrevPage = True accessibleFiles = accessibleFiles[:count + 1] # Revert the list to get most recent items first. accessibleFiles.reverse() # The last item is actually the first item of next page. nextPageStart = accessibleFiles[-1][0] accessibleFiles = accessibleFiles[:-1] elif start: hasPrevPage = True # XXX: may not always be true, but the edge case is OK if len(accessibleFiles) > count: nextPageStart = accessibleFiles[count][0] accessibleFiles = accessibleFiles[:count] defer.returnValue((accessibleFiles, hasPrevPage, nextPageStart, toFetchEntities))
def _update_suggestions(request, relation=None): authinfo = request.getSession(IAuthInfo) myId = authinfo.username orgId = authinfo.organization weights = {'group': {'follower': 15, 'subscription': 40, 'group': 30}, 'follower': {'follower': 9, 'subscription': 24, 'group': 15}, 'subscription': {'follower': 24, 'subscription': 64, 'group': 40}} defaultWeight = 1 people = {} @defer.inlineCallbacks def _compute_weights(userIds, myGroups, type): followers = yield db.multiget_slice(userIds, "followers", count=50) subscriptions = yield db.multiget_slice(userIds, "subscriptions", count=50) groups = yield db.multiget_slice(userIds, "entityGroupsMap") followers = utils.multiColumnsToDict(followers) subscriptions = utils.multiColumnsToDict(subscriptions) groups = utils.multiColumnsToDict(groups) for userId in followers: for follower in followers[userId]: people[follower] = people.setdefault(follower, defaultWeight) + weights[type]['follower'] for userId in subscriptions: for subscription in subscriptions[userId]: people[subscription] = people.setdefault(subscription, defaultWeight) + weights[type]['subscription'] for userId in groups: groupIds = [x.split(':', 1)[1] for x in groups[userId]] for groupId in groupIds: if groupId in myGroups: people[userId] = people.setdefault(userId, defaultWeight) + weights[type]['group'] if not relation: relation = Relation(myId, []) yield defer.DeferredList([relation.initSubscriptionsList(), relation.initFollowersList(), relation.initGroupsList()]) if relation.followers: yield _compute_weights(relation.followers, relation.groups, 'follower') if relation.subscriptions: yield _compute_weights(relation.subscriptions, relation.groups, 'subscription') if relation.groups: groupMembers = yield db.multiget_slice(relation.groups, "groupMembers", count=20) groupMembers = utils.multiColumnsToDict(groupMembers) for groupId in groupMembers: yield _compute_weights(groupMembers[groupId], relation.groups, 'group') cols = yield db.get_slice(orgId, "orgUsers", count=100) for col in cols: userId = col.column.name if userId not in people: people[userId] = people.setdefault(userId, 0) + defaultWeight suggestions = {} for userId in people: if isValidSuggestion(myId, userId, relation): suggestions.setdefault(people[userId], []).append(userId) yield db.remove(myId, "suggestions") weights_userIds_map = {} format = '>l' for weight in suggestions: key = struct.pack(format, weight) weights_userIds_map[key] = ' '.join(suggestions[weight]) if weights_userIds_map: yield db.batch_insert(myId, "suggestions", weights_userIds_map)
def renderItem(self, request, toFeed=False): (appchange, script, args, myId) = yield self._getBasicArgs(request) landing = not self._ajax myOrgId = args["orgId"] convId, conv = yield utils.getValidItemId(request, "id", columns=['tags']) itemType = conv["meta"].get("type", None) if 'parent' in conv['meta']: raise errors.InvalidItem('conversation', convId) start = utils.getRequestArg(request, "start") or '' start = utils.decodeKey(start) args['convId'] = convId args['isItemView'] = True args['items'] = {convId: conv} meta = conv["meta"] owner = meta["owner"] relation = Relation(myId, []) yield defer.DeferredList([relation.initGroupsList(), relation.initSubscriptionsList()]) args["relations"] = relation if script and landing: t.render(request, "item.mako", **args) if script and appchange: t.renderScriptBlock(request, "item.mako", "layout", landing, "#mainbar", "set", **args) args["entities"] = {} toFetchEntities = set() toFetchTags = set(conv.get("tags", {}).keys()) plugin = plugins[itemType] if itemType in plugins else None if plugin: entityIds = yield plugin.fetchData(args) toFetchEntities.update(entityIds) toFetchEntities.add(conv['meta']['owner']) if "target" in conv["meta"]: toFetchEntities.update(conv['meta']['target'].split(',')) if conv['meta']['owner'] not in toFetchEntities: toFetchEntities.add(conv['meta']['owner']) entities = base.EntitySet(toFetchEntities) yield entities.fetchData() args["entities"] = entities renderers = [] if script: t.renderScriptBlock(request, "item.mako", "conv_root", landing, "#conv-root-%s > .conv-summary" % (convId), "set", **args) convOwner = args["items"][convId]["meta"]["owner"] args["ownerId"] = convOwner if script: if itemType != "feedback": t.renderScriptBlock(request, "item.mako", "conv_owner", landing, "#conv-avatar-%s" % convId, "set", **args) else: feedbackType = conv['meta']['subType'] t.renderScriptBlock(request, "item.mako", "feedback_icon", landing, "#conv-avatar-%s" % convId, "set", args=[feedbackType]) # A copy of this code for fetching comments is present in _responses # Most changes here may need to be done there too. itemResponses = yield db.get_slice(convId, "itemResponses", start=start, reverse=True, count=constants.COMMENTS_PER_PAGE + 1) nextPageStart = itemResponses[-1].column.name\ if len(itemResponses) > constants.COMMENTS_PER_PAGE\ else None itemResponses = itemResponses[:-1] \ if len(itemResponses) > constants.COMMENTS_PER_PAGE\ else itemResponses responseKeys = [] for response in itemResponses: userKey, responseKey = response.column.value.split(":") responseKeys.append(responseKey) toFetchEntities.add(userKey) responseKeys.reverse() subscriptions = list(relation.subscriptions) likes = yield db.get_slice(convId, "itemLikes", subscriptions) \ if subscriptions else defer.succeed([]) toFetchEntities.update([x.column.name for x in likes]) entities = base.EntitySet(toFetchEntities) d1 = entities.fetchData() d2 = db.multiget_slice(responseKeys, "items", ["meta", "attachments"]) d3 = db.multiget_slice(responseKeys + [convId], "itemLikes", [myId]) d4 = db.get_slice(myOrgId, "orgTags", toFetchTags)\ if toFetchTags else defer.succeed([]) yield d1 fetchedItems = yield d2 myLikes = yield d3 fetchedTags = yield d4 fetchedItems = utils.multiSuperColumnsToDict(fetchedItems) myLikes = utils.multiColumnsToDict(myLikes) fetchedTags = utils.supercolumnsToDict(fetchedTags) # Do some error correction/consistency checking to ensure that the # response items actually exist. I don't know of any reason why these # items may not exist. missingIds = [x for x, y in fetchedItems.items() if not y] if missingIds: yield self._cleanupMissingComments(convId, missingIds, itemResponses) args["items"].update(fetchedItems) args["entities"].update(entities) args["myLikes"] = myLikes args["tags"] = fetchedTags args["responses"] = {convId: responseKeys} if nextPageStart: args["oldest"] = utils.encodeKey(nextPageStart) if script: t.renderScriptBlock(request, "item.mako", 'conv_footer', landing, '#item-footer-%s' % convId, 'set', **args) t.renderScriptBlock(request, "item.mako", 'conv_tags', landing, '#conv-tags-wrapper-%s' % convId, 'set', handlers={"onload": "$('#conv-meta-wrapper-%s').removeClass('no-tags')" % convId} if toFetchTags else None, **args) t.renderScriptBlock(request, "item.mako", 'conv_comments', landing, '#conv-comments-wrapper-%s' % convId, 'set', **args) t.renderScriptBlock(request, "item.mako", 'conv_comment_form', landing, '#comment-form-wrapper-%s' % convId, 'set', True, handlers={"onload": "(function(obj){$$.convs.load(obj);})(this);"}, **args) numLikes = int(conv["meta"].get("likesCount", "0")) if numLikes: numLikes = int(conv["meta"].get("likesCount", "0")) iLike = myId in args["myLikes"].get(convId, []) t.renderScriptBlock(request, "item.mako", 'conv_likes', landing, '#conv-likes-wrapper-%s' % convId, 'set', args=[convId, numLikes, iLike, [x.column.name for x in likes]], entities=args['entities']) if plugin and hasattr(plugin, 'renderItemSideBlock'): plugin.renderItemSideBlock(request, landing, args) if script and landing: request.write("</body></html>") if not script: t.render(request, "item.mako", **args)
def _renderReport(self, request, partial=False): (appchange, script, args, myId) = yield self._getBasicArgs(request) landing = not self._ajax convId, conv = yield utils.getValidItemId(request, "id", columns=["reports"]) if 'parent' in conv['meta']: raise errors.InvalidItem('conversation', convId) args["entities"] = {} toFetchEntities = set() args['convId'] = convId args['items'] = {convId: conv} convMeta = conv["meta"] convType = convMeta.get("type", None) relation = Relation(myId, []) yield relation.initGroupsList() args["relations"] = relation if script and landing: t.render(request, "item-report.mako", **args) if script and appchange: t.renderScriptBlock(request, "item-report.mako", "layout", landing, "#mainbar", "set", **args) plugin = plugins[convType] if convType in plugins else None if plugin: entityIds = yield plugin.fetchData(args) toFetchEntities.update(entityIds) convOwner = convMeta['owner'] toFetchEntities.add(convOwner) if "target" in convMeta: toFetchEntities.update(convMeta['target'].split(',')) if "reportId" in convMeta: toFetchEntities.add(convMeta['reportedBy']) entities = base.EntitySet(toFetchEntities) yield entities.fetchData() args["entities"] = entities args["ownerId"] = convOwner if script: t.renderScriptBlock(request, "item.mako", "conv_root", landing, "#conv-root-%s > .conv-summary" % (convId), "set", **args) t.renderScriptBlock(request, "item-report.mako", 'conv_footer', landing, '#item-footer-%s' % convId, 'set', **args) if convType != "feedback": t.renderScriptBlock(request, "item.mako", "conv_owner", landing, "#conv-avatar-%s" % convId, "set", **args) else: feedbackType = conv['meta']['subType'] t.renderScriptBlock(request, "item.mako", "feedback_icon", landing, "#conv-avatar-%s" % convId, "set", args=[feedbackType]) yield self._renderReportResponses(request, convId, convMeta, args) if not script: t.render(request, "item-report.mako", **args)