def _getUserItems(self, request, userId, start='', count=10):
authinfo = request.getSession(IAuthInfo)
myId = authinfo.username
myOrgId = authinfo.organization
toFetchItems = set()
toFetchEntities = set()
toFetchTags = set()
toFetchResponses = set()
toFetchCount = count + 1
toFetchStart = utils.decodeKey(start) if start else ''
fetchedUserItem = []
responses = {}
convs = []
userItemsRaw = []
userItems = []
reasonStr = {}
timestamps = {}
items = {}
nextPageStart = None
args = {'myId': myId}
relation = Relation(myId, [])
yield relation.initGroupsList()
toFetchEntities.add(userId)
while len(convs) < toFetchCount:
cols = yield db.get_slice(userId, "userItems", start=toFetchStart,
reverse=True, count=toFetchCount)
tmpIds = []
for col in cols:
convId = col.column.value.split(":")[2]
if convId not in tmpIds and convId not in convs:
tmpIds.append(convId)
(filteredConvs, deletedConvs) = yield utils.fetchAndFilterConvs\
(tmpIds, relation, items, myId, myOrgId)
for col in cols[0:count]:
convId = col.column.value.split(":")[2]
if len(convs) == count or len(fetchedUserItem) == count*2:
nextPageStart = col.column.name
break
if convId not in filteredConvs and convId not in convs:
continue
fetchedUserItem.append(col)
if convId not in convs:
convs.append(convId)
if len(cols) < toFetchCount or nextPageStart:
break
if cols:
toFetchStart = cols[-1].column.name
if nextPageStart:
nextPageStart = utils.encodeKey(nextPageStart)
for col in fetchedUserItem:
value = tuple(col.column.value.split(":"))
timestamps[value] = col.column.timestamp/1e6
rtype, itemId, convId, convType, convOwnerId, commentSnippet = value
commentSnippet = """<span class="snippet">"%s"</span>""" %(_(commentSnippet))
toFetchEntities.add(convOwnerId)
if rtype == 'I':
toFetchItems.add(convId)
toFetchResponses.add(convId)
userItems.append(value)
elif rtype == "L" and itemId == convId and convOwnerId != userId:
reasonStr[value] = _("liked %s's %s")
userItems.append(value)
elif rtype == "L" and convOwnerId != userId:
r = "answer" if convType == 'question' else 'comment'
reasonStr[value] = _("liked") + " %s " %(commentSnippet) + _("%s "%r) + _("on %s's %s")
userItems.append(value)
elif rtype in ["C", 'Q'] and convOwnerId != userId:
reasonStr[value] = "%s"%(commentSnippet) + _(" on %s's %s")
userItems.append(value)
itemResponses = yield db.multiget_slice(toFetchResponses, "itemResponses",
count=2, reverse=True)
for convId, comments in itemResponses.items():
responses[convId] = []
for comment in comments:
userId_, itemKey = comment.column.value.split(':')
if itemKey not in toFetchItems:
responses[convId].insert(0,itemKey)
toFetchItems.add(itemKey)
toFetchEntities.add(userId_)
items = yield db.multiget_slice(toFetchItems, "items", ["meta", "tags", "attachments"])
items = utils.multiSuperColumnsToDict(items)
args["items"] = items
extraDataDeferreds = []
for convId in convs:
if convId not in items:
continue
meta = items[convId]["meta"]
itemType = meta["type"]
toFetchEntities.add(meta["owner"])
if "target" in meta:
toFetchEntities.update(meta["target"].split(','))
toFetchTags.update(items[convId].get("tags", {}).keys())
if itemType in plugins:
d = plugins[itemType].fetchData(args, convId)
extraDataDeferreds.append(d)
result = yield defer.DeferredList(extraDataDeferreds)
for success, ret in result:
if success:
toFetchEntities.update(ret)
entities = base.EntitySet(toFetchEntities)
yield entities.fetchData()
tags = {}
if toFetchTags:
userOrgId = entities[userId].basic["org"]
fetchedTags = yield db.get_slice(userOrgId, "orgTags", toFetchTags)
tags = utils.supercolumnsToDict(fetchedTags)
fetchedLikes = yield db.multiget(toFetchItems, "itemLikes", myId)
myLikes = utils.multiColumnsToDict(fetchedLikes)
data = {"entities": entities, "reasonStr": reasonStr,
"tags": tags, "myLikes": myLikes, "userItems": userItems,
"responses": responses, "nextPageStart": nextPageStart,
"timestamps": timestamps }
del args['myId']
args.update(data)
defer.returnValue(args)
def get(auth, feedId=None, feedItemsId=None, convIds=None,
getFn=None, cleanFn=None, start='', count=10, getReasons=True,
forceCount=False, itemType=None):
"""Fetch data from feed represented by feedId. Returns a dictionary
that has the items from feed, start of the next page and responses and
likes that I know of.
Keyword params:
@auth - An instance of AuthInfo representing the authenticated user
@feedId - Id of the feed from which the data is to be fetched
@feedItemsId - Id of the feed from with feed items must be fetched
@convIds - List of conversation id's to be used as feed
@getFn - Function that must be used to fetch items
@cleanFn - Function that must be used to clean items that don't exist
@start - Id of the item where the fetching must start
@count - Number of items to fetch
@getReasons - Add reason strings to the returned dictionary
@forceCount - Try hard to get atleast count items from the feed
"""
toFetchItems = set() # Items and entities that need to be fetched
toFetchEntities = set() #
toFetchTags = set() #
items = {} # Fetched items, entities and tags
entities = base.EntitySet([])#
tags = {} #
deleted = [] # List of items that were deleted
deleteKeys = [] # List of keys that need to be deleted
responses = {} # Cached data of item responses and likes
likes = {} #
myLikes = {}
myId = auth.username
orgId = auth.organization
feedSource = "feed_%s" % itemType\
if itemType and itemType in plugins\
and plugins[itemType].hasIndex\
else "feed"
feedItemsId = feedItemsId or myId
feedItems_d = [] # List of deferred used to fetch feedItems
# Used for checking ACL
relation = Relation(myId, [])
yield relation.initGroupsList()
# The updates that will be used to build reason strings
convReasonUpdates = {}
# Data that is sent to various plugins and returned by this function
# XXX: myKey is depricated - use myId
data = {"myId": myId, "orgId": orgId, "responses": responses,
"likes": likes, "myLikes": myLikes, "items": items,
"entities": entities, "tags": tags, "myKey": myId,
"relations": relation}
@defer.inlineCallbacks
def fetchFeedItems(ids):
rawFeedItems = yield db.get_slice(feedItemsId, "feedItems", ids) \
if ids else defer.succeed([])
for conv in rawFeedItems:
convId = conv.super_column.name
convUpdates = conv.super_column.columns
responses[convId] = []
likes[convId] = []
latest = None
updatesByType = {}
for update in convUpdates:
parts = update.value.split(':')
updatesByType.setdefault(parts[0], []).append(parts)
if parts[1] != myId: # Ignore my own updates
latest = parts # when displaying latest actors
# Parse all notification to make sure we fetch any required
# items, entities. and cache generic stuff that we display
for tipe in updatesByType.keys():
updates = updatesByType[tipe]
if tipe in _feedUpdatePlugins:
(i,e) = _feedUpdatePlugins[tipe].parse(convId, updates)
toFetchItems.update(i)
toFetchEntities.update(e)
if tipe == "L":
for update in updates:
if update[2] == convId:
likes[convId].append(update[1])
# XXX: Adding this item may break the sorting
# of responses on this conversation
# Bug #493
#else:
# responses[convId].append(update[2])
elif tipe in ["C", "Q"]:
for update in updates:
responses[convId].append(update[2])
# Store any information that can be used to render
# the reason strings when we have the required data
if getReasons and latest:
convReasonUpdates[convId] = updatesByType[latest[0]]
# Fetch the feed if required and at the same time make sure
# we delete unwanted items from the feed (cleanup).
# NOTE: We assume that there will be very few deletes.
nextPageStart = None
if not convIds:
feedId = feedId or myId
allFetchedConvIds = set() # Complete set of convIds fetched
itemsFromFeed = {} # All the key-values retrieved from feed
keysFromFeed = [] # Sorted list of keys (used for paging)
convIds = [] # List of convIds that will be displayed
fetchStart = utils.decodeKey(start)
fetchCount = count + 1
while len(convIds) < count:
fetchedConvIds = []
# Use the getFn function if given.
# NOTE: Part of this code is duplicated just below this.
if getFn:
results = yield getFn(start=fetchStart, count=fetchCount)
for name, value in results.items():
keysFromFeed.append(name)
if value not in allFetchedConvIds:
fetchedConvIds.append(value)
allFetchedConvIds.add(value)
itemsFromFeed[name] = value
else:
deleteKeys.append(name)
# Fetch user's feed when getFn isn't given.
# NOTE: Part of this code is from above
else:
results = yield db.get_slice(feedId, feedSource,
count=fetchCount,
start=fetchStart, reverse=True)
for col in results:
value = col.column.value
keysFromFeed.append(col.column.name)
if value not in allFetchedConvIds:
fetchedConvIds.append(value)
allFetchedConvIds.add(value)
itemsFromFeed[col.column.name] = value
else:
deleteKeys.append(col.column.name)
# Initiate fetching feed items for all the conversation Ids.
# Meanwhile we check if the authenticated user has access to
# all the fetched conversation ids.
# NOTE: Conversations would rarely be filtered out. So, we
# just go ahead with fetching data for all convs.
feedItems_d.append(fetchFeedItems(fetchedConvIds))
(filteredConvIds, deletedIds) = yield utils.fetchAndFilterConvs\
(fetchedConvIds, relation, items, myId, orgId)
convIds.extend(filteredConvIds)
deleted.extend(deletedIds)
# Unless we are forced to fetch count number of items, we only
# iterate till we fetch atleast half of them
if (not forceCount and len(convIds) > (count/2)) or\
len(results) < fetchCount:
break
# If we need more items, we start fetching from where we
# left in the previous iteration.
fetchStart = keysFromFeed[-1]
# If DB fetch got as many items as I requested
# there may be additional items present in the feed
# So, we cut one item from what we return and start the
# next page from there.
if len(results) == fetchCount:
lastConvId = convIds[-1]
for key in reversed(keysFromFeed):
if key in itemsFromFeed and itemsFromFeed[key] == lastConvId:
nextPageStart = utils.encodeKey(key)
convIds = convIds[:-1]
else:
(convIds, deletedIds) = yield utils.fetchAndFilterConvs(convIds,
relation, items, myId, orgId)
# NOTE: Unlike the above case where we fetch convIds from
# database (where we set the nextPageStart to a key),
# here we set nextPageStart to the convId.
if len(convIds) > count:
nextPageStart = utils.encodeKey(convIds[count])
convIds = convIds[0:count]
# Since convIds were directly passed to us, we would also
# return the list of convIds deleted back to the caller.
if deletedIds:
data["deleted"] = deletedIds
# We don't have any conversations to display!
if not convIds:
defer.returnValue({"conversations": []})
# Delete any convs that were deleted from the feeds and
# any duplicates that were marked for deletion
cleanup_d = []
if deleted:
for key, value in itemsFromFeed.items():
if value in deleted:
deleteKeys.append(key)
if cleanFn:
d1 = cleanFn(list(deleteKeys))
else:
d1 = db.batch_remove({feedSource: [feedId]}, names=deleteKeys)
d2 = db.batch_remove({'feedItems': [feedId]}, names=list(deleted))
cleanup_d = [d1, d2]
# We now have a filtered list of conversations that can be displayed
# Let's wait till all the feed items have been fetched and processed
yield defer.DeferredList(feedItems_d)
# Fetch the remaining items (comments on the actual conversations)
items_d = db.multiget_slice(toFetchItems, "items", ["meta" ,"attachments"])
# Fetch tags on all the conversations that will be displayed
for convId in convIds:
conv = items[convId]
toFetchEntities.add(conv["meta"]["owner"])
if "target" in conv["meta"]:
toFetchEntities.update(conv["meta"]["target"].split(','))
toFetchTags.update(conv.get("tags",{}).keys())
tags_d = db.get_slice(orgId, "orgTags", toFetchTags) \
if toFetchTags else defer.succeed([])
# Fetch the list of my likes.
# XXX: Latency can be pretty high here becuase many nodes will have to
# be contacted for the information. Alternative could be to cache
# all likes by a user somewhere.
myLikes_d = db.multiget(toFetchItems.union(convIds), "itemLikes", myId)
# Fetch extra data that is required to render special items
# We already fetched the conversation items, plugins merely
# add more data to the already fetched items
for convId in convIds[:]:
itemType = items[convId]["meta"]["type"]
if itemType in plugins:
try:
entityIds = yield plugins[itemType].fetchData(data, convId)
toFetchEntities.update(entityIds)
except Exception, e:
log.err(e)
convIds.remove(convId)
def _listTags(self, request):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
landing = not self._ajax
myOrgId = args["orgId"]
start = utils.getRequestArg(request, 'start') or ''
nextPageStart = ''
prevPageStart = ''
count = constants.PEOPLE_PER_PAGE
toFetchCount = count + 1
start = utils.decodeKey(start)
args["menuId"] = "tags"
if script and landing:
t.render(request, "tags.mako", **args)
if script and appchange:
t.renderScriptBlock(request, "tags.mako", "layout",
landing, "#mainbar", "set", **args)
if script:
t.renderScriptBlock(request, "tags.mako", "header",
landing, "#tags-header", "set", **args)
tagsByName = yield db.get_slice(myOrgId, "orgTagsByName",
start=start, count=toFetchCount)
tagIds = [x.column.value for x in tagsByName]
if len(tagsByName) > count:
nextPageStart = utils.encodeKey(tagsByName[-1].column.name)
tagIds = tagIds[:-1]
if start:
prevCols = yield db.get_slice(myOrgId, "orgTagsByName",
start=start, reverse=True,
count=toFetchCount)
if len(prevCols) > 1:
prevPageStart = utils.encodeKey(prevCols[-1].column.name)
tags = {}
if tagIds:
tags = yield db.get_slice(myOrgId, "orgTags", tagIds)
tags = utils.supercolumnsToDict(tags)
# TODO: We need an index of all tags that the user is following
# Probably convert the 'subscriptions' column family to 'Super'
# and have people and tags in the same column family.
tagsFollowing = []
if tagIds:
cols = yield db.multiget(tagIds, "tagFollowers", myId)
tagsFollowing = [x for x in cols.keys() if cols[x]]
args['tags'] = tags
args['tagIds'] = tagIds
args['tagsFollowing'] = tagsFollowing
args['nextPageStart'] = nextPageStart
args['prevPageStart'] = prevPageStart
if script:
if appchange:
t.renderScriptBlock(request, "tags.mako", "tagsListLayout",
landing, "#content", "set", **args)
else:
t.renderScriptBlock(request, "tags.mako", "listTags",
landing, "#tags-wrapper", "set", **args)
t.renderScriptBlock(request, "tags.mako", "paging",
landing, "#tags-paging", "set", **args)
if not script:
t.render(request, "tags.mako", **args)
def _sendInvitations(myOrgUsers, otherOrgUsers, me, myId, myOrg):
rootUrl = config.get('General', 'URL')
brandName = config.get('Branding', 'Name')
senderName = me.basic["name"]
senderOrgName = myOrg.basic["name"]
senderAvatarUrl = utils.userAvatar(myId, me, "medium")
sentUsers = []
blockedUsers = []
existingUsers = []
myOrgSubject = "%s invited you to %s" % (senderName, brandName)
myOrgBody = "Hi,\n\n"\
"%(senderName)s has invited you to %(senderOrgName)s network on %(brandName)s.\n"\
"To activate your account please visit: %(activationUrl)s.\n\n"
otherOrgSubject = "%s invited you to %s" % (senderName, brandName)
otherOrgBody = "Hi,\n\n"\
"%(senderName)s has invited you to try %(brandName)s.\n"\
"To activate your account please visit: %(activationUrl)s.\n\n"
signature = "Flocked.in Team.\n\n\n\n"\
"--\n"\
"To block invitations from %(senderName)s visit %(blockSenderUrl)s\n"\
"To block all invitations from %(brandName)s visit %(blockAllUrl)s"
blockSenderTmpl = "%(rootUrl)s/signup/blockSender?email=%(emailId)s&token=%(token)s"
blockAllTmpl = "%(rootUrl)s/signup/blockAll?email=%(emailId)s&token=%(token)s"
activationTmpl = "%(rootUrl)s/signup?email=%(emailId)s&token=%(token)s"
# Combine all users.
myOrgUsers.extend(otherOrgUsers)
# Ensure that the users do not already exist and that the users are
# not in the doNotSpam list (for this sender or globally)
d1 = db.multiget(myOrgUsers, "userAuth", "user")
d2 = db.multiget_slice(myOrgUsers, "doNotSpam", [myId, '*'])
existing = yield d1
existing = utils.multiColumnsToDict(existing)
doNotSpam = yield d2
doNotSpam = utils.multiColumnsToDict(doNotSpam)
deferreds = []
for emailId in myOrgUsers:
if emailId in existing and existing[emailId]:
existingUsers.append(emailId)
continue
token = utils.getRandomKey()
# Add invitation to the database
localpart, domainpart = emailId.split('@')
deferreds.append(db.insert(domainpart, "invitations", myId, token, emailId))
deferreds.append(db.insert(myId, "invitationsSent", '', emailId))
# Mail the invitation if everything is ok.
if emailId in doNotSpam and doNotSpam[emailId]:
blockedUsers.append(emailId)
continue
activationUrl = activationTmpl % locals()
blockAllUrl = blockAllTmpl % locals()
blockSenderUrl = blockSenderTmpl % locals()
sameOrg = False if emailId in otherOrgUsers else True
if not sameOrg:
subject = otherOrgSubject
textBody = (otherOrgBody + signature) % locals()
else:
subject = myOrgSubject
textBody = (myOrgBody + signature) % locals()
# XXX: getBlock blocks the application for disk reads when reading template
htmlBody = t.getBlock("emails.mako", "invite", **locals())
deferreds.append(utils.sendmail(emailId, subject, textBody, htmlBody))
sentUsers.append(emailId)
yield defer.DeferredList(deferreds)
defer.returnValue((sentUsers, blockedUsers, existingUsers))