예제 #1
0
    def _getUserItems(self, request, userId, start='', count=10):
        authinfo = request.getSession(IAuthInfo)
        myId = authinfo.username
        myOrgId = authinfo.organization

        toFetchItems = set()
        toFetchEntities = set()
        toFetchTags = set()
        toFetchResponses = set()
        toFetchCount = count + 1
        toFetchStart = utils.decodeKey(start) if start else ''
        fetchedUserItem = []
        responses = {}
        convs = []
        userItemsRaw = []
        userItems = []
        reasonStr = {}
        timestamps = {}
        items = {}
        nextPageStart = None
        args = {'myId': myId}

        relation = Relation(myId, [])
        yield relation.initGroupsList()

        toFetchEntities.add(userId)

        while len(convs) < toFetchCount:
            cols = yield db.get_slice(userId, "userItems", start=toFetchStart,
                                      reverse=True, count=toFetchCount)
            tmpIds = []
            for col in cols:
                convId = col.column.value.split(":")[2]
                if convId not in tmpIds and convId not in convs:
                    tmpIds.append(convId)
            (filteredConvs, deletedConvs) = yield utils.fetchAndFilterConvs\
                                        (tmpIds, relation, items, myId, myOrgId)
            for col in cols[0:count]:
                convId = col.column.value.split(":")[2]
                if len(convs) == count or len(fetchedUserItem) == count*2:
                    nextPageStart = col.column.name
                    break
                if convId not in filteredConvs and convId not in convs:
                    continue
                fetchedUserItem.append(col)
                if convId not in convs:
                    convs.append(convId)
            if len(cols) < toFetchCount or nextPageStart:
                break
            if cols:
                toFetchStart = cols[-1].column.name
        if nextPageStart:
            nextPageStart = utils.encodeKey(nextPageStart)

        for col in fetchedUserItem:
            value = tuple(col.column.value.split(":"))
            timestamps[value] = col.column.timestamp/1e6
            rtype, itemId, convId, convType, convOwnerId, commentSnippet = value
            commentSnippet = """<span class="snippet">"%s"</span>""" %(_(commentSnippet))
            toFetchEntities.add(convOwnerId)
            if rtype == 'I':
                toFetchItems.add(convId)
                toFetchResponses.add(convId)
                userItems.append(value)
            elif rtype == "L" and itemId == convId and convOwnerId != userId:
                reasonStr[value] = _("liked %s's %s")
                userItems.append(value)
            elif rtype == "L"  and convOwnerId != userId:
                r = "answer" if convType == 'question' else 'comment'
                reasonStr[value] = _("liked") + " %s " %(commentSnippet) + _("%s "%r) + _("on %s's %s")
                userItems.append(value)
            elif rtype in ["C", 'Q'] and convOwnerId != userId:
                reasonStr[value] = "%s"%(commentSnippet) + _(" on %s's %s")
                userItems.append(value)

        itemResponses = yield db.multiget_slice(toFetchResponses, "itemResponses",
                                                count=2, reverse=True)
        for convId, comments in itemResponses.items():
            responses[convId] = []
            for comment in comments:
                userId_, itemKey = comment.column.value.split(':')
                if itemKey not in toFetchItems:
                    responses[convId].insert(0,itemKey)
                    toFetchItems.add(itemKey)
                    toFetchEntities.add(userId_)

        items = yield db.multiget_slice(toFetchItems, "items", ["meta", "tags", "attachments"])
        items = utils.multiSuperColumnsToDict(items)
        args["items"] = items
        extraDataDeferreds = []

        for convId in convs:
            if convId not in items:
                continue

            meta = items[convId]["meta"]
            itemType = meta["type"]
            toFetchEntities.add(meta["owner"])
            if "target" in meta:
                toFetchEntities.update(meta["target"].split(','))

            toFetchTags.update(items[convId].get("tags", {}).keys())

            if itemType in plugins:
                d =  plugins[itemType].fetchData(args, convId)
                extraDataDeferreds.append(d)

        result = yield defer.DeferredList(extraDataDeferreds)
        for success, ret in result:
            if success:
                toFetchEntities.update(ret)

        entities = base.EntitySet(toFetchEntities)
        yield entities.fetchData()

        tags = {}
        if toFetchTags:
            userOrgId = entities[userId].basic["org"]
            fetchedTags = yield db.get_slice(userOrgId, "orgTags", toFetchTags)
            tags = utils.supercolumnsToDict(fetchedTags)

        fetchedLikes = yield db.multiget(toFetchItems, "itemLikes", myId)
        myLikes = utils.multiColumnsToDict(fetchedLikes)

        data = {"entities": entities, "reasonStr": reasonStr,
                "tags": tags, "myLikes": myLikes, "userItems": userItems,
                "responses": responses, "nextPageStart": nextPageStart,
                "timestamps": timestamps }
        del args['myId']
        args.update(data)
        defer.returnValue(args)
예제 #2
0
파일: Feed.py 프로젝트: psunkari/flocked-in
def get(auth, feedId=None, feedItemsId=None, convIds=None,
        getFn=None, cleanFn=None, start='', count=10, getReasons=True,
        forceCount=False, itemType=None):
    """Fetch data from feed represented by feedId. Returns a dictionary
    that has the items from feed, start of the next page and responses and
    likes that I know of.

    Keyword params:
    @auth -  An instance of AuthInfo representing the authenticated user
    @feedId -  Id of the feed from which the data is to be fetched
    @feedItemsId -  Id of the feed from with feed items must be fetched
    @convIds -  List of conversation id's to be used as feed
    @getFn -  Function that must be used to fetch items
    @cleanFn -  Function that must be used to clean items that don't exist
    @start -  Id of the item where the fetching must start
    @count - Number of items to fetch
    @getReasons - Add reason strings to the returned dictionary
    @forceCount - Try hard to get atleast count items from the feed

    """
    toFetchItems = set()    # Items and entities that need to be fetched
    toFetchEntities = set() #
    toFetchTags = set()     #

    items = {}              # Fetched items, entities and tags
    entities = base.EntitySet([])#
    tags = {}               #

    deleted = []            # List of items that were deleted
    deleteKeys = []         # List of keys that need to be deleted

    responses = {}          # Cached data of item responses and likes
    likes = {}              #
    myLikes = {}

    myId = auth.username
    orgId = auth.organization

    feedSource = "feed_%s" % itemType\
                        if itemType and itemType in plugins\
                                    and plugins[itemType].hasIndex\
                        else "feed"
    feedItemsId = feedItemsId or myId
    feedItems_d = []            # List of deferred used to fetch feedItems

    # Used for checking ACL
    relation = Relation(myId, [])
    yield relation.initGroupsList()

    # The updates that will be used to build reason strings
    convReasonUpdates = {}

    # Data that is sent to various plugins and returned by this function
    # XXX: myKey is depricated - use myId
    data = {"myId": myId, "orgId": orgId, "responses": responses,
            "likes": likes, "myLikes": myLikes, "items": items,
            "entities": entities, "tags": tags, "myKey": myId,
            "relations": relation}

    @defer.inlineCallbacks
    def fetchFeedItems(ids):
        rawFeedItems = yield db.get_slice(feedItemsId, "feedItems", ids) \
                                            if ids else defer.succeed([])
        for conv in rawFeedItems:
            convId = conv.super_column.name
            convUpdates = conv.super_column.columns
            responses[convId] = []
            likes[convId] = []
            latest = None

            updatesByType = {}
            for update in convUpdates:
                parts = update.value.split(':')
                updatesByType.setdefault(parts[0], []).append(parts)
                if parts[1] != myId:    # Ignore my own updates
                    latest = parts      # when displaying latest actors

            # Parse all notification to make sure we fetch any required
            # items, entities. and cache generic stuff that we display
            for tipe in updatesByType.keys():
                updates = updatesByType[tipe]

                if tipe in _feedUpdatePlugins:
                    (i,e) = _feedUpdatePlugins[tipe].parse(convId, updates)
                    toFetchItems.update(i)
                    toFetchEntities.update(e)

                if tipe == "L":
                    for update in updates:
                        if update[2] == convId:
                            likes[convId].append(update[1])
                        # XXX: Adding this item may break the sorting
                        #      of responses on this conversation
                        #      Bug #493
                        #else:
                        #    responses[convId].append(update[2])
                elif tipe in ["C", "Q"]:
                    for update in updates:
                        responses[convId].append(update[2])

            # Store any information that can be used to render
            # the reason strings when we have the required data
            if getReasons and latest:
                convReasonUpdates[convId] = updatesByType[latest[0]]

    # Fetch the feed if required and at the same time make sure
    # we delete unwanted items from the feed (cleanup).
    # NOTE: We assume that there will be very few deletes.
    nextPageStart = None
    if not convIds:
        feedId = feedId or myId
        allFetchedConvIds = set()   # Complete set of convIds fetched
        itemsFromFeed = {}          # All the key-values retrieved from feed
        keysFromFeed = []           # Sorted list of keys (used for paging)
        convIds = []                # List of convIds that will be displayed

        fetchStart = utils.decodeKey(start)
        fetchCount = count + 1

        while len(convIds) < count:
            fetchedConvIds = []

            # Use the getFn function if given.
            # NOTE: Part of this code is duplicated just below this.
            if getFn:
                results = yield getFn(start=fetchStart, count=fetchCount)
                for name, value in results.items():
                    keysFromFeed.append(name)
                    if value not in allFetchedConvIds:
                        fetchedConvIds.append(value)
                        allFetchedConvIds.add(value)
                        itemsFromFeed[name] = value
                    else:
                        deleteKeys.append(name)

            # Fetch user's feed when getFn isn't given.
            # NOTE: Part of this code is from above
            else:
                results = yield db.get_slice(feedId, feedSource,
                                             count=fetchCount,
                                             start=fetchStart, reverse=True)
                for col in results:
                    value = col.column.value
                    keysFromFeed.append(col.column.name)
                    if value not in allFetchedConvIds:
                        fetchedConvIds.append(value)
                        allFetchedConvIds.add(value)
                        itemsFromFeed[col.column.name] = value
                    else:
                        deleteKeys.append(col.column.name)

            # Initiate fetching feed items for all the conversation Ids.
            # Meanwhile we check if the authenticated user has access to
            # all the fetched conversation ids.
            # NOTE: Conversations would rarely be filtered out. So, we
            #       just go ahead with fetching data for all convs.
            feedItems_d.append(fetchFeedItems(fetchedConvIds))
            (filteredConvIds, deletedIds) = yield utils.fetchAndFilterConvs\
                                (fetchedConvIds, relation, items, myId, orgId)
            convIds.extend(filteredConvIds)
            deleted.extend(deletedIds)

            # Unless we are forced to fetch count number of items, we only
            # iterate till we fetch atleast half of them
            if (not forceCount and len(convIds) > (count/2)) or\
                    len(results) < fetchCount:
                break

            # If we need more items, we start fetching from where we
            # left in the previous iteration.
            fetchStart = keysFromFeed[-1]

        # If DB fetch got as many items as I requested
        # there may be additional items present in the feed
        # So, we cut one item from what we return and start the
        # next page from there.
        if len(results) == fetchCount:
            lastConvId = convIds[-1]
            for key in reversed(keysFromFeed):
                if key in itemsFromFeed and itemsFromFeed[key] == lastConvId:
                    nextPageStart = utils.encodeKey(key)
            convIds = convIds[:-1]

    else:
        (convIds, deletedIds) = yield utils.fetchAndFilterConvs(convIds,
                                            relation, items, myId, orgId)
        # NOTE: Unlike the above case where we fetch convIds from
        #       database (where we set the nextPageStart to a key),
        #       here we set nextPageStart to the convId.
        if len(convIds) > count:
            nextPageStart = utils.encodeKey(convIds[count])
            convIds = convIds[0:count]

        # Since convIds were directly passed to us, we would also
        # return the list of convIds deleted back to the caller.
        if deletedIds:
            data["deleted"] = deletedIds

    # We don't have any conversations to display!
    if not convIds:
        defer.returnValue({"conversations": []})

    # Delete any convs that were deleted from the feeds and
    # any duplicates that were marked for deletion
    cleanup_d = []
    if deleted:
        for key, value in itemsFromFeed.items():
            if value in deleted:
                deleteKeys.append(key)

        if cleanFn:
            d1 = cleanFn(list(deleteKeys))
        else:
            d1 = db.batch_remove({feedSource: [feedId]}, names=deleteKeys)

        d2 = db.batch_remove({'feedItems': [feedId]}, names=list(deleted))
        cleanup_d = [d1, d2]

    # We now have a filtered list of conversations that can be displayed
    # Let's wait till all the feed items have been fetched and processed
    yield defer.DeferredList(feedItems_d)

    # Fetch the remaining items (comments on the actual conversations)
    items_d = db.multiget_slice(toFetchItems, "items", ["meta" ,"attachments"])

    # Fetch tags on all the conversations that will be displayed
    for convId in convIds:
        conv = items[convId]
        toFetchEntities.add(conv["meta"]["owner"])
        if "target" in conv["meta"]:
            toFetchEntities.update(conv["meta"]["target"].split(','))
        toFetchTags.update(conv.get("tags",{}).keys())

    tags_d = db.get_slice(orgId, "orgTags", toFetchTags) \
                                if toFetchTags else defer.succeed([])

    # Fetch the list of my likes.
    # XXX: Latency can be pretty high here becuase many nodes will have to
    #      be contacted for the information.  Alternative could be to cache
    #      all likes by a user somewhere.
    myLikes_d = db.multiget(toFetchItems.union(convIds), "itemLikes", myId)

    # Fetch extra data that is required to render special items
    # We already fetched the conversation items, plugins merely
    # add more data to the already fetched items
    for convId in convIds[:]:
        itemType = items[convId]["meta"]["type"]
        if itemType in plugins:
            try:
                entityIds = yield plugins[itemType].fetchData(data, convId)
                toFetchEntities.update(entityIds)
            except Exception, e:
                log.err(e)
                convIds.remove(convId)
예제 #3
0
파일: tags.py 프로젝트: psunkari/flocked-in
    def _listTags(self, request):
        (appchange, script, args, myId) = yield self._getBasicArgs(request)
        landing = not self._ajax
        myOrgId = args["orgId"]

        start = utils.getRequestArg(request, 'start') or ''
        nextPageStart = ''
        prevPageStart = ''
        count = constants.PEOPLE_PER_PAGE
        toFetchCount = count + 1
        start = utils.decodeKey(start)

        args["menuId"] = "tags"
        if script and landing:
            t.render(request, "tags.mako", **args)

        if script and appchange:
            t.renderScriptBlock(request, "tags.mako", "layout",
                                landing, "#mainbar", "set", **args)

        if script:
            t.renderScriptBlock(request, "tags.mako", "header",
                                landing, "#tags-header", "set", **args)

        tagsByName = yield db.get_slice(myOrgId, "orgTagsByName",
                                        start=start, count=toFetchCount)
        tagIds = [x.column.value for x in tagsByName]

        if len(tagsByName) > count:
            nextPageStart = utils.encodeKey(tagsByName[-1].column.name)
            tagIds = tagIds[:-1]

        if start:
            prevCols = yield db.get_slice(myOrgId, "orgTagsByName",
                                          start=start, reverse=True,
                                          count=toFetchCount)
            if len(prevCols) > 1:
                prevPageStart = utils.encodeKey(prevCols[-1].column.name)

        tags = {}
        if tagIds:
            tags = yield db.get_slice(myOrgId, "orgTags", tagIds)
            tags = utils.supercolumnsToDict(tags)

        # TODO: We need an index of all tags that the user is following
        #       Probably convert the 'subscriptions' column family to 'Super'
        #       and have people and tags in the same column family.
        tagsFollowing = []
        if tagIds:
            cols = yield db.multiget(tagIds, "tagFollowers", myId)
            tagsFollowing = [x for x in cols.keys() if cols[x]]

        args['tags'] = tags
        args['tagIds'] = tagIds
        args['tagsFollowing'] = tagsFollowing
        args['nextPageStart'] = nextPageStart
        args['prevPageStart'] = prevPageStart

        if script:
            if appchange:
                t.renderScriptBlock(request, "tags.mako", "tagsListLayout",
                                    landing, "#content", "set", **args)
            else:
                t.renderScriptBlock(request, "tags.mako", "listTags",
                                    landing, "#tags-wrapper", "set", **args)
                t.renderScriptBlock(request, "tags.mako", "paging",
                                    landing, "#tags-paging", "set", **args)

        if not script:
            t.render(request, "tags.mako", **args)
예제 #4
0
def _sendInvitations(myOrgUsers, otherOrgUsers, me, myId, myOrg):
    rootUrl = config.get('General', 'URL')
    brandName = config.get('Branding', 'Name')
    senderName = me.basic["name"]
    senderOrgName = myOrg.basic["name"]
    senderAvatarUrl = utils.userAvatar(myId, me, "medium")
    sentUsers = []
    blockedUsers = []
    existingUsers = []

    myOrgSubject = "%s invited you to %s" % (senderName, brandName)
    myOrgBody = "Hi,\n\n"\
                "%(senderName)s has invited you to %(senderOrgName)s network on %(brandName)s.\n"\
                "To activate your account please visit: %(activationUrl)s.\n\n"
    otherOrgSubject = "%s invited you to %s" % (senderName, brandName)
    otherOrgBody = "Hi,\n\n"\
                   "%(senderName)s has invited you to try %(brandName)s.\n"\
                   "To activate your account please visit: %(activationUrl)s.\n\n"

    signature = "Flocked.in Team.\n\n\n\n"\
                "--\n"\
                "To block invitations from %(senderName)s visit %(blockSenderUrl)s\n"\
                "To block all invitations from %(brandName)s visit %(blockAllUrl)s"

    blockSenderTmpl = "%(rootUrl)s/signup/blockSender?email=%(emailId)s&token=%(token)s"
    blockAllTmpl = "%(rootUrl)s/signup/blockAll?email=%(emailId)s&token=%(token)s"
    activationTmpl = "%(rootUrl)s/signup?email=%(emailId)s&token=%(token)s"

    # Combine all users.
    myOrgUsers.extend(otherOrgUsers)

    # Ensure that the users do not already exist and that the users are
    # not in the doNotSpam list (for this sender or globally)
    d1 = db.multiget(myOrgUsers, "userAuth", "user")
    d2 = db.multiget_slice(myOrgUsers, "doNotSpam", [myId, '*'])
    existing = yield d1
    existing = utils.multiColumnsToDict(existing)
    doNotSpam = yield d2
    doNotSpam = utils.multiColumnsToDict(doNotSpam)

    deferreds = []
    for emailId in myOrgUsers:
        if emailId in existing and existing[emailId]:
            existingUsers.append(emailId)
            continue

        token = utils.getRandomKey()

        # Add invitation to the database
        localpart, domainpart = emailId.split('@')
        deferreds.append(db.insert(domainpart, "invitations", myId, token, emailId))
        deferreds.append(db.insert(myId, "invitationsSent", '', emailId))

        # Mail the invitation if everything is ok.
        if emailId in doNotSpam and doNotSpam[emailId]:
            blockedUsers.append(emailId)
            continue

        activationUrl = activationTmpl % locals()
        blockAllUrl = blockAllTmpl % locals()
        blockSenderUrl = blockSenderTmpl % locals()
        sameOrg = False if emailId in otherOrgUsers else True
        if not sameOrg:
            subject = otherOrgSubject
            textBody = (otherOrgBody + signature) % locals()
        else:
            subject = myOrgSubject
            textBody = (myOrgBody + signature) % locals()

        # XXX: getBlock blocks the application for disk reads when reading template
        htmlBody = t.getBlock("emails.mako", "invite", **locals())
        deferreds.append(utils.sendmail(emailId, subject, textBody, htmlBody))
        sentUsers.append(emailId)

    yield defer.DeferredList(deferreds)
    defer.returnValue((sentUsers, blockedUsers, existingUsers))