def _unlike(self, request, data=None):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
orgId = args['orgId']
itemId, item = data['id']
item = yield Item.unlike(itemId, item, myId, orgId)
if not item:
return
args["items"] = {itemId: item}
args["myLikes"] = {itemId: []}
likesCount = int(item["meta"]["likesCount"])
convId = item["meta"].get('parent', itemId)
if itemId != convId:
t.renderScriptBlock(request, "item.mako", "item_footer", False,
"#item-footer-%s" % (itemId), "set",
args=[itemId], **args)
else:
relation = Relation(myId, [])
yield relation.initSubscriptionsList()
toFetchEntities = set()
likes = []
subscriptions = list(relation.subscriptions)
if subscriptions:
likes = yield db.get_slice(convId, "itemLikes", subscriptions)
likes = [x.column.name for x in likes]
toFetchEntities = set(likes)
feedItems = yield db.get_slice(myId, "feedItems", [convId])
feedItems = utils.supercolumnsToDict(feedItems)
isFeed = (utils.getRequestArg(request, "_pg") != "/item")
hasComments = False
if not isFeed:
hasComments = True
else:
feedItems = yield db.get_slice(myId, "feedItems", [convId])
feedItems = utils.supercolumnsToDict(feedItems)
for tuuid in feedItems.get(convId, {}):
val = feedItems[convId][tuuid]
rtype = val.split(":")[0]
if rtype == "C":
hasComments = True
entities = base.EntitySet(toFetchEntities)
if toFetchEntities:
yield entities.fetchData()
args["entities"] = entities
handler = {"onload": "(function(){$$.convs.showHideComponent('%s', 'likes', false)})();" % (convId)} if not likes else None
t.renderScriptBlock(request, "item.mako", "conv_footer", False,
"#item-footer-%s" % (itemId), "set",
args=[itemId, hasComments, likes], **args)
t.renderScriptBlock(request, "item.mako", 'conv_likes', False,
'#conv-likes-wrapper-%s' % convId, 'set', True,
args=[itemId, likesCount, False, likes], handlers=handler, **args)
def _new(self, request, data=None):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
me = args['me']
landing = not self._ajax
authInfo = request.getSession(IAuthInfo)
convType = data['type']
convId, conv, keywords = yield Item.new(request, authInfo, convType)
if keywords:
block = t.getBlock('item.mako', 'requireReviewDlg', keywords=keywords)
request.write('$$.convs.reviewRequired(%s);' % json.dumps(block))
return
target = conv['meta'].get('target', None)
toFetchEntities = set()
if target:
toFetchEntities.update(target.split(','))
convType = utils.getRequestArg(request, "type")
plugin = plugins[convType]
entityIds = yield plugin.fetchData(args, convId)
toFetchEntities.update(entityIds)
entities = base.EntitySet(toFetchEntities)
yield entities.fetchData()
entities.update(args['me'])
relation = Relation(myId, [])
yield relation.initGroupsList()
data = {"items": {convId: conv}, "relations": relation,
"entities": entities, "script": True}
args.update(data)
onload = "(function(obj){$$.convs.load(obj);$('#sharebar-attach-uploaded').empty();})(this);"
t.renderScriptBlock(request, "item.mako", "item_layout",
False, "#user-feed", "prepend", args=[convId, 'conv-item-created'],
handlers={"onload": onload}, **args)
defaultType = plugins.keys()[0]
plugins[defaultType].renderShareBlock(request, True)
if plugin and hasattr(plugin, 'renderFeedSideBlock'):
entityId = myId
referer = request.getHeader('referer')
matchObj = feedPathObj.search(referer)
if matchObj:
matchedStr = matchObj.group('entityId')
if matchedStr != "":
entityId = matchedStr
else:
if target:
entityId = target.split(',')[0]
args["groupId"] = target.split(',')[0]
request.write("$('#feed-side-block-container').empty();")
yield plugin.renderFeedSideBlock(request, landing, entityId, args)
def createNewItem(request, itemType, owner, acl=None, subType=None, groupIds=None, richText=False):
if not acl:
acl = getRequestArg(request, "acl", sanitize=False)
try:
acl = json.loads(acl)
orgs = acl.get("accept", {}).get("orgs", [])
if len(orgs) > 1 or (len(orgs) == 1 and orgs[0] != owner.basic["org"]):
msg = "Cannot grant access to other orgs on this item"
raise errors.PermissionDenied(_(msg))
except:
acl = {"accept": {"orgs": [owner.basic["org"]]}}
accept_groups = acl.get("accept", {}).get("groups", [])
deny_groups = acl.get("deny", {}).get("groups", [])
groups = [x for x in accept_groups if x not in deny_groups]
if groups:
relation = Relation(owner.id, [])
yield relation.initGroupsList()
if not all([x in relation.groups for x in groups]):
msg = "Only group members can post to a group"
raise errors.PermissionDenied(_(msg))
acl = pickle.dumps(acl)
item = {
"meta": {
"acl": acl,
"org": owner.basic["org"],
"type": itemType,
"uuid": uuid.uuid1().bytes,
"owner": owner.id,
"timestamp": str(int(time.time())),
"richText": str(richText),
},
"followers": {owner.id: ""},
}
if subType:
item["meta"]["subType"] = subType
if groups:
item["meta"]["target"] = ",".join(groups)
tmpFileIds = getRequestArg(request, "fId", False, True)
attachments = {}
if tmpFileIds:
attachments = yield _upload_files(owner.id, tmpFileIds)
if attachments:
item["attachments"] = {}
for attachmentId in attachments:
fileId, name, size, ftype = attachments[attachmentId]
item["attachments"][attachmentId] = "%s:%s:%s" % (name, size, ftype)
defer.returnValue(item)
def get_suggestions(request, count, mini=False):
authinfo = request.getSession(IAuthInfo)
myId = authinfo.username
SUGGESTIONS_UPDATE_FREQUENCY = 3 * 86400 # 5days
MAX_INVALID_SUGGESTIONS = 3
now = time.time()
suggestions = []
relation = Relation(myId, [])
yield defer.DeferredList([relation.initSubscriptionsList(),
relation.initFollowersList()])
@defer.inlineCallbacks
def _get_suggestions(myId, relation):
validSuggestions = []
invalidCount = 0
FORCE_UPDATE = False
cols = yield db.get_slice(myId, "suggestions", reverse=True)
for col in cols:
if now - col.column.timestamp/1e6 > SUGGESTIONS_UPDATE_FREQUENCY:
FORCE_UPDATE = True
for userId in col.column.value.split():
if isValidSuggestion(myId, userId, relation):
validSuggestions.append(userId)
else:
invalidCount += 1
defer.returnValue((validSuggestions, invalidCount, FORCE_UPDATE))
validSuggestions, invalidCount, FORCE_UPDATE = yield _get_suggestions(myId, relation)
if not validSuggestions:
yield _update_suggestions(request, relation)
validSuggestions, invalidCount, FORCE_UPDATE = yield _get_suggestions(myId, relation)
no_of_samples = count*2
population = count*5
if mini and len(validSuggestions) >= no_of_samples:
suggestions = random.sample(validSuggestions[:population], no_of_samples)
else:
suggestions = validSuggestions[:]
if FORCE_UPDATE or invalidCount > MAX_INVALID_SUGGESTIONS:
_update_suggestions(request, relation)
entities = base.EntitySet(suggestions[:count])
if suggestions:
suggestions = suggestions[:count]
yield entities.fetchData()
defer.returnValue((suggestions, entities))
def getValidItemId(request, arg, type=None, columns=None, itemId=None, myOrgId=None, myId=None):
if not itemId:
itemId = getRequestArg(request, arg, sanitize=False)
itemType = type if type else "item"
if not itemId:
raise errors.MissingParams([_("%s id") % _(itemType).capitalize()])
columns = [] if not columns else columns
columns.extend(["meta", "attachments"])
item = yield db.get_slice(itemId, "items", columns)
if not item:
raise errors.InvalidItem(itemType, itemId)
item = supercolumnsToDict(item)
meta = item["meta"]
if type and meta["type"] != type:
raise errors.InvalidItem(itemType, itemId)
parentId = meta.get("parent", None)
if parentId:
parent = yield db.get_slice(parentId, "items", ["meta"])
parent = supercolumnsToDict(parent)
acl = parent["meta"]["acl"]
owner = parent["meta"]["owner"]
deleted = parent["meta"].get("state", None) == "deleted"
else:
parent = item
acl = meta["acl"]
owner = meta["owner"]
deleted = parent["meta"].get("state", None) == "deleted"
if deleted:
raise errors.InvalidItem(itemType, itemId)
if not myOrgId:
myOrgId = request.getSession(IAuthInfo).organization
if not myId:
myId = request.getSession(IAuthInfo).username
relation = Relation(myId, [])
yield relation.initGroupsList()
if not checkAcl(myId, myOrgId, False, relation, parent["meta"]):
raise errors.ItemAccessDenied(itemType, itemId)
defer.returnValue((itemId, item))
def getPeople(myId, entityId, orgId, start='',
count=PEOPLE_PER_PAGE, fn=None, fetchBlocked=True):
blockedUsers = []
toFetchCount = count + 1
nextPageStart = None
prevPageStart = None
userIds = []
if fetchBlocked:
cols = yield db.get_slice(orgId, "blockedUsers")
blockedUsers = utils.columnsToDict(cols).keys()
if not fn:
d1 = db.get_slice(entityId, "displayNameIndex",
start=start, count=toFetchCount)
d2 = db.get_slice(entityId, "displayNameIndex",
start=start, count=toFetchCount,
reverse=True) if start else None
# Get the list of users (sorted by displayName)
cols = yield d1
userIds = [col.column.name.split(":")[1] for col in cols]
if len(userIds) > count:
nextPageStart = utils.encodeKey(cols[-1].column.name)
userIds = userIds[0:count]
toFetchUsers = userIds
# Start of previous page
if start and d2:
prevCols = yield d2
if len(prevCols) > 1:
prevPageStart = utils.encodeKey(prevCols[-1].column.name)
else:
userIds, nextPageStart, prevPageStart\
= yield fn(entityId, start, toFetchCount)
toFetchUsers = userIds
entities = base.EntitySet(toFetchUsers)
usersDeferred = entities.fetchData()
relation = Relation(myId, userIds)
results = yield defer.DeferredList([usersDeferred,
relation.initSubscriptionsList()])
defer.returnValue((entities, relation, userIds,\
blockedUsers, nextPageStart, prevPageStart))
def _to_python(self, itemId, state):
itemType = self.itemType if self.itemType else 'item'
if not itemId or not itemId[0]:
raise MissingParam('%s-id' % (itemType), itemId, state)
itemId = itemId[0]
columns = set(['meta']).update(self.columns)
item = yield db.get_slice(itemId, "items", columns)
if not item:
raise InvalidItem(itemType, itemId, state)
item = utils.supercolumnsToDict(item)
meta = item["meta"]
if self.itemType and meta["type"] != self.itemType:
raise InvalidItem(itemType, itemId, state)
parentId = meta.get("parent", None)
if parentId:
parent = yield db.get_slice(parentId, "items", ["meta"])
parent = utils.supercolumnsToDict(parent)
else:
parent = item
acl = parent["meta"]["acl"]
owner = parent["meta"]["owner"]
#parent is deleted
deleted = parent['meta'].get('state', None) == 'deleted'
#item is deleted
deleted = deleted or meta.get('state', None) == 'deleted'
if deleted:
raise InvalidItem(itemType, itemId, state)
if self.source != 'api':
request = state.request
authInfo = request.getSession(IAuthInfo)
myId = authInfo.username
orgId = authInfo.organization
isOrgAdmin = authInfo.isAdmin
isOrgAdmin = self.checkAdmin and isOrgAdmin
relation = Relation(myId, [])
yield relation.initGroupsList()
if not utils.checkAcl(myId, orgId, isOrgAdmin, relation, parent['meta']):
raise ItemAccessDenied(itemType, itemId, state)
defer.returnValue((itemId, item))
def _getUserItems(self, request, userId, start='', count=10):
authinfo = request.getSession(IAuthInfo)
myId = authinfo.username
myOrgId = authinfo.organization
toFetchItems = set()
toFetchEntities = set()
toFetchTags = set()
toFetchResponses = set()
toFetchCount = count + 1
toFetchStart = utils.decodeKey(start) if start else ''
fetchedUserItem = []
responses = {}
convs = []
userItemsRaw = []
userItems = []
reasonStr = {}
timestamps = {}
items = {}
nextPageStart = None
args = {'myId': myId}
relation = Relation(myId, [])
yield relation.initGroupsList()
toFetchEntities.add(userId)
while len(convs) < toFetchCount:
cols = yield db.get_slice(userId, "userItems", start=toFetchStart,
reverse=True, count=toFetchCount)
tmpIds = []
for col in cols:
convId = col.column.value.split(":")[2]
if convId not in tmpIds and convId not in convs:
tmpIds.append(convId)
(filteredConvs, deletedConvs) = yield utils.fetchAndFilterConvs\
(tmpIds, relation, items, myId, myOrgId)
for col in cols[0:count]:
convId = col.column.value.split(":")[2]
if len(convs) == count or len(fetchedUserItem) == count*2:
nextPageStart = col.column.name
break
if convId not in filteredConvs and convId not in convs:
continue
fetchedUserItem.append(col)
if convId not in convs:
convs.append(convId)
if len(cols) < toFetchCount or nextPageStart:
break
if cols:
toFetchStart = cols[-1].column.name
if nextPageStart:
nextPageStart = utils.encodeKey(nextPageStart)
for col in fetchedUserItem:
value = tuple(col.column.value.split(":"))
timestamps[value] = col.column.timestamp/1e6
rtype, itemId, convId, convType, convOwnerId, commentSnippet = value
commentSnippet = """<span class="snippet">"%s"</span>""" %(_(commentSnippet))
toFetchEntities.add(convOwnerId)
if rtype == 'I':
toFetchItems.add(convId)
toFetchResponses.add(convId)
userItems.append(value)
elif rtype == "L" and itemId == convId and convOwnerId != userId:
reasonStr[value] = _("liked %s's %s")
userItems.append(value)
elif rtype == "L" and convOwnerId != userId:
r = "answer" if convType == 'question' else 'comment'
reasonStr[value] = _("liked") + " %s " %(commentSnippet) + _("%s "%r) + _("on %s's %s")
userItems.append(value)
elif rtype in ["C", 'Q'] and convOwnerId != userId:
reasonStr[value] = "%s"%(commentSnippet) + _(" on %s's %s")
userItems.append(value)
itemResponses = yield db.multiget_slice(toFetchResponses, "itemResponses",
count=2, reverse=True)
for convId, comments in itemResponses.items():
responses[convId] = []
for comment in comments:
userId_, itemKey = comment.column.value.split(':')
if itemKey not in toFetchItems:
responses[convId].insert(0,itemKey)
toFetchItems.add(itemKey)
toFetchEntities.add(userId_)
items = yield db.multiget_slice(toFetchItems, "items", ["meta", "tags", "attachments"])
items = utils.multiSuperColumnsToDict(items)
args["items"] = items
extraDataDeferreds = []
for convId in convs:
if convId not in items:
continue
meta = items[convId]["meta"]
itemType = meta["type"]
toFetchEntities.add(meta["owner"])
if "target" in meta:
toFetchEntities.update(meta["target"].split(','))
toFetchTags.update(items[convId].get("tags", {}).keys())
if itemType in plugins:
d = plugins[itemType].fetchData(args, convId)
extraDataDeferreds.append(d)
result = yield defer.DeferredList(extraDataDeferreds)
for success, ret in result:
if success:
toFetchEntities.update(ret)
entities = base.EntitySet(toFetchEntities)
yield entities.fetchData()
tags = {}
if toFetchTags:
userOrgId = entities[userId].basic["org"]
fetchedTags = yield db.get_slice(userOrgId, "orgTags", toFetchTags)
tags = utils.supercolumnsToDict(fetchedTags)
fetchedLikes = yield db.multiget(toFetchItems, "itemLikes", myId)
myLikes = utils.multiColumnsToDict(fetchedLikes)
data = {"entities": entities, "reasonStr": reasonStr,
"tags": tags, "myLikes": myLikes, "userItems": userItems,
"responses": responses, "nextPageStart": nextPageStart,
"timestamps": timestamps }
del args['myId']
args.update(data)
defer.returnValue(args)
def _search(self, request):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
landing = not self._ajax
myOrgId = args['orgId']
filter_map = {'people':'itemType'}
term = utils.getRequestArg(request, "q")
start = utils.getRequestArg(request, "start") or 0
filters = utils.getRequestArg(request, 'filter', multiValued=True) or []
filters = dict([(filter_map[x], x) for x in filters if x in filter_map])
args["term"] = term
nextPageStart = ''
if not term:
errors.MissingParams()
try:
start = int(start)
if start < 0:
raise ValueError
except ValueError:
errors.InvalidParamValue()
if script and landing:
t.render(request, "search.mako", **args)
if script and appchange:
t.renderScriptBlock(request, "search.mako", "layout",
landing, "#mainbar", "set", **args)
count = SEARCH_RESULTS_PER_PAGE
items = {}
convs = set()
highlighting = {}
toFetchItems = []
toFetchStart = start
toFetchEntities = set()
people = []
relation = Relation(myId, [])
yield defer.DeferredList([relation.initGroupsList(),
relation.initSubscriptionsList(),
relation.initFollowersList()])
regex = re.compile("(.*?)([^\s]*\s*[^\s]*\s*[^\s]*\s*)(<em class='highlight'>.*<\/em>)(\s*[^\s]*\s*[^\s]*\s*[^\s]*)(.*)")
res = yield solr.search(term, args['orgId'], count, toFetchStart, filters={'itemType': 'people'})
docs = res.data.get('response', {}).get('docs', [])
for item in docs:
entityId = item['id']
people.append(entityId)
toFetchEntities.add(entityId)
while 1:
res = yield solr.search(term, args['orgId'], count, toFetchStart)
messages = []
convItems = []
numMatched = res.data.get('response', {}).get('numFound', 0)
docs = res.data.get('response', {}).get('docs', [])
highlighting.update(res.data.get('highlighting', {}))
for index, item in enumerate(docs):
itemId = item['id']
parent = item.get('parent', None)
position = toFetchStart + index
if item.get('itemType', '') == "message":
if (item.get('id'), parent) not in messages:
messages.append((item.get('id'), parent))
elif item.get('itemType', '') == 'people':
entityId = item.get('id')
if entityId not in people:
people.append(entityId)
toFetchEntities.add(entityId)
elif parent:
convItems.append((itemId, parent, position))
convs.add(parent)
else:
convItems.append((itemId, itemId, position))
convs.add(item.get('id'))
if convs:
filteredConvs, deleted = yield utils.fetchAndFilterConvs(convs, relation, items, myId, myOrgId)
for itemId, convId, position in convItems:
if convId in filteredConvs and itemId not in toFetchItems:
toFetchItems.append(itemId)
if len(toFetchItems) == count:
if position +1 < numMatched:
nextPageStart = position + 1
break
if len(toFetchItems) == count or len(docs) < count:
break
toFetchStart = toFetchStart + count
_items = yield db.multiget_slice(toFetchItems, "items",
['meta', 'attachments', 'tags'])
items.update(utils.multiSuperColumnsToDict(_items))
for itemId, item in items.iteritems():
toFetchEntities.add(item['meta']['owner'])
if 'target' in item['meta']:
toFetchEntities.update(item['meta']['target'].split(','))
if itemId in highlighting and 'comment' in highlighting[itemId]:
match = re.match(regex, unquote(highlighting[itemId]['comment'][0]))
if match:
comment = "".join(match.groups()[1:4])
comment = comment + " …" if match.group(5) else comment
items[itemId]['meta']['comment'] = comment
entities = yield db.multiget_slice(toFetchEntities, "entities", ['basic'])
entities = utils.multiSuperColumnsToDict(entities)
for userId in people:
if userId in highlighting and userId in entities:
entities[userId]['basic']['reason'] = {}
for key in highlighting[userId]:
if key in entities[userId]['basic']:
entities[userId]['basic'][key] = " ".join(highlighting[userId][key])
else:
entities[userId]['basic']['reason'][key] = highlighting[userId][key]
fromFetchMore = True if start else False
args['term'] = term
args['items'] = items
args['people'] = people
args['entities'] = entities
args['relations'] = relation
args["conversations"] = toFetchItems
args["nextPageStart"] = nextPageStart
args['fromFetchMore'] = fromFetchMore
args['fromSidebar'] = 'people' in filters.values()
if script:
onload = "(function(obj){$$.convs.load(obj);})(this);"
if fromFetchMore:
t.renderScriptBlock(request, "search.mako", "results",
landing, "#next-load-wrapper", "replace",
True, handlers={"onload": onload}, **args)
else:
t.renderScriptBlock(request, "search.mako", "results",
landing, "#search-results", "set", True,
handlers={"onload": onload}, **args)
if 'people' not in filters.values() and people:
t.renderScriptBlock(request, "search.mako", "_displayUsersMini",
landing, "#people-block", "set", True, **args)
if script and landing:
request.write("</body></html>")
if not script:
t.render(request, "search.mako", **args)
def get(auth, feedId=None, feedItemsId=None, convIds=None,
getFn=None, cleanFn=None, start='', count=10, getReasons=True,
forceCount=False, itemType=None):
"""Fetch data from feed represented by feedId. Returns a dictionary
that has the items from feed, start of the next page and responses and
likes that I know of.
Keyword params:
@auth - An instance of AuthInfo representing the authenticated user
@feedId - Id of the feed from which the data is to be fetched
@feedItemsId - Id of the feed from with feed items must be fetched
@convIds - List of conversation id's to be used as feed
@getFn - Function that must be used to fetch items
@cleanFn - Function that must be used to clean items that don't exist
@start - Id of the item where the fetching must start
@count - Number of items to fetch
@getReasons - Add reason strings to the returned dictionary
@forceCount - Try hard to get atleast count items from the feed
"""
toFetchItems = set() # Items and entities that need to be fetched
toFetchEntities = set() #
toFetchTags = set() #
items = {} # Fetched items, entities and tags
entities = base.EntitySet([])#
tags = {} #
deleted = [] # List of items that were deleted
deleteKeys = [] # List of keys that need to be deleted
responses = {} # Cached data of item responses and likes
likes = {} #
myLikes = {}
myId = auth.username
orgId = auth.organization
feedSource = "feed_%s" % itemType\
if itemType and itemType in plugins\
and plugins[itemType].hasIndex\
else "feed"
feedItemsId = feedItemsId or myId
feedItems_d = [] # List of deferred used to fetch feedItems
# Used for checking ACL
relation = Relation(myId, [])
yield relation.initGroupsList()
# The updates that will be used to build reason strings
convReasonUpdates = {}
# Data that is sent to various plugins and returned by this function
# XXX: myKey is depricated - use myId
data = {"myId": myId, "orgId": orgId, "responses": responses,
"likes": likes, "myLikes": myLikes, "items": items,
"entities": entities, "tags": tags, "myKey": myId,
"relations": relation}
@defer.inlineCallbacks
def fetchFeedItems(ids):
rawFeedItems = yield db.get_slice(feedItemsId, "feedItems", ids) \
if ids else defer.succeed([])
for conv in rawFeedItems:
convId = conv.super_column.name
convUpdates = conv.super_column.columns
responses[convId] = []
likes[convId] = []
latest = None
updatesByType = {}
for update in convUpdates:
parts = update.value.split(':')
updatesByType.setdefault(parts[0], []).append(parts)
if parts[1] != myId: # Ignore my own updates
latest = parts # when displaying latest actors
# Parse all notification to make sure we fetch any required
# items, entities. and cache generic stuff that we display
for tipe in updatesByType.keys():
updates = updatesByType[tipe]
if tipe in _feedUpdatePlugins:
(i,e) = _feedUpdatePlugins[tipe].parse(convId, updates)
toFetchItems.update(i)
toFetchEntities.update(e)
if tipe == "L":
for update in updates:
if update[2] == convId:
likes[convId].append(update[1])
# XXX: Adding this item may break the sorting
# of responses on this conversation
# Bug #493
#else:
# responses[convId].append(update[2])
elif tipe in ["C", "Q"]:
for update in updates:
responses[convId].append(update[2])
# Store any information that can be used to render
# the reason strings when we have the required data
if getReasons and latest:
convReasonUpdates[convId] = updatesByType[latest[0]]
# Fetch the feed if required and at the same time make sure
# we delete unwanted items from the feed (cleanup).
# NOTE: We assume that there will be very few deletes.
nextPageStart = None
if not convIds:
feedId = feedId or myId
allFetchedConvIds = set() # Complete set of convIds fetched
itemsFromFeed = {} # All the key-values retrieved from feed
keysFromFeed = [] # Sorted list of keys (used for paging)
convIds = [] # List of convIds that will be displayed
fetchStart = utils.decodeKey(start)
fetchCount = count + 1
while len(convIds) < count:
fetchedConvIds = []
# Use the getFn function if given.
# NOTE: Part of this code is duplicated just below this.
if getFn:
results = yield getFn(start=fetchStart, count=fetchCount)
for name, value in results.items():
keysFromFeed.append(name)
if value not in allFetchedConvIds:
fetchedConvIds.append(value)
allFetchedConvIds.add(value)
itemsFromFeed[name] = value
else:
deleteKeys.append(name)
# Fetch user's feed when getFn isn't given.
# NOTE: Part of this code is from above
else:
results = yield db.get_slice(feedId, feedSource,
count=fetchCount,
start=fetchStart, reverse=True)
for col in results:
value = col.column.value
keysFromFeed.append(col.column.name)
if value not in allFetchedConvIds:
fetchedConvIds.append(value)
allFetchedConvIds.add(value)
itemsFromFeed[col.column.name] = value
else:
deleteKeys.append(col.column.name)
# Initiate fetching feed items for all the conversation Ids.
# Meanwhile we check if the authenticated user has access to
# all the fetched conversation ids.
# NOTE: Conversations would rarely be filtered out. So, we
# just go ahead with fetching data for all convs.
feedItems_d.append(fetchFeedItems(fetchedConvIds))
(filteredConvIds, deletedIds) = yield utils.fetchAndFilterConvs\
(fetchedConvIds, relation, items, myId, orgId)
convIds.extend(filteredConvIds)
deleted.extend(deletedIds)
# Unless we are forced to fetch count number of items, we only
# iterate till we fetch atleast half of them
if (not forceCount and len(convIds) > (count/2)) or\
len(results) < fetchCount:
break
# If we need more items, we start fetching from where we
# left in the previous iteration.
fetchStart = keysFromFeed[-1]
# If DB fetch got as many items as I requested
# there may be additional items present in the feed
# So, we cut one item from what we return and start the
# next page from there.
if len(results) == fetchCount:
lastConvId = convIds[-1]
for key in reversed(keysFromFeed):
if key in itemsFromFeed and itemsFromFeed[key] == lastConvId:
nextPageStart = utils.encodeKey(key)
convIds = convIds[:-1]
else:
(convIds, deletedIds) = yield utils.fetchAndFilterConvs(convIds,
relation, items, myId, orgId)
# NOTE: Unlike the above case where we fetch convIds from
# database (where we set the nextPageStart to a key),
# here we set nextPageStart to the convId.
if len(convIds) > count:
nextPageStart = utils.encodeKey(convIds[count])
convIds = convIds[0:count]
# Since convIds were directly passed to us, we would also
# return the list of convIds deleted back to the caller.
if deletedIds:
data["deleted"] = deletedIds
# We don't have any conversations to display!
if not convIds:
defer.returnValue({"conversations": []})
# Delete any convs that were deleted from the feeds and
# any duplicates that were marked for deletion
cleanup_d = []
if deleted:
for key, value in itemsFromFeed.items():
if value in deleted:
deleteKeys.append(key)
if cleanFn:
d1 = cleanFn(list(deleteKeys))
else:
d1 = db.batch_remove({feedSource: [feedId]}, names=deleteKeys)
d2 = db.batch_remove({'feedItems': [feedId]}, names=list(deleted))
cleanup_d = [d1, d2]
# We now have a filtered list of conversations that can be displayed
# Let's wait till all the feed items have been fetched and processed
yield defer.DeferredList(feedItems_d)
# Fetch the remaining items (comments on the actual conversations)
items_d = db.multiget_slice(toFetchItems, "items", ["meta" ,"attachments"])
# Fetch tags on all the conversations that will be displayed
for convId in convIds:
conv = items[convId]
toFetchEntities.add(conv["meta"]["owner"])
if "target" in conv["meta"]:
toFetchEntities.update(conv["meta"]["target"].split(','))
toFetchTags.update(conv.get("tags",{}).keys())
tags_d = db.get_slice(orgId, "orgTags", toFetchTags) \
if toFetchTags else defer.succeed([])
# Fetch the list of my likes.
# XXX: Latency can be pretty high here becuase many nodes will have to
# be contacted for the information. Alternative could be to cache
# all likes by a user somewhere.
myLikes_d = db.multiget(toFetchItems.union(convIds), "itemLikes", myId)
# Fetch extra data that is required to render special items
# We already fetched the conversation items, plugins merely
# add more data to the already fetched items
for convId in convIds[:]:
itemType = items[convId]["meta"]["type"]
if itemType in plugins:
try:
entityIds = yield plugins[itemType].fetchData(data, convId)
toFetchEntities.update(entityIds)
except Exception, e:
log.err(e)
convIds.remove(convId)
def _render(self, request):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
# We are setting an empty value to 'cu' here just to make sure that
# any errors when looking validating the entity should not leave us
# in a bad state.
request.addCookie('cu', '', path="/ajax/profile")
if request.args.get("id", None):
userId, ign = yield utils.getValidEntityId(request, "id", "user")
else:
userId = myId
# XXX: We should use getValidEntityId to fetch the entire user
# info instead of another call to the database.
request.addCookie('cu', userId, path="/ajax/profile")
user = base.Entity(userId)
yield user.fetchData([])
if user._data:
args['user'] = user
detail = utils.getRequestArg(request, "dt") or "activity"
args["detail"] = detail
args["userId"] = userId
args["menuId"] = "people"
args["entities"] = base.EntitySet({myId:args['me'], userId:user})
# When scripts are enabled, updates are sent to the page as
# and when we get the required data from the database.
# When we are the landing page, we also render the page header
# and all updates are wrapped in <script> blocks.
landing = not self._ajax
# User entered the URL directly
# Render the header. Other things will follow.
if script and landing:
t.render(request, "profile.mako", **args)
# Start with displaying the template and navigation menu
if script and appchange:
t.renderScriptBlock(request, "profile.mako", "layout",
landing, "#mainbar", "set", **args)
# Prefetch some data about how I am related to the user.
# This is required in order to reliably filter our profile details
# that I don't have access to.
relation = Relation(myId, [userId])
args["relations"] = relation
yield defer.DeferredList([relation.initSubscriptionsList(),
relation.initGroupsList()])
# Reload all user-depended blocks if the currently displayed user is
# not the same as the user for which new data is being requested.
if script:
t.renderScriptBlock(request, "profile.mako", "summary",
landing, "#profile-summary", "set", **args)
t.renderScriptBlock(request, "profile.mako", "user_subactions",
landing, "#user-subactions", "set", **args)
fetchedEntities = set()
start = utils.getRequestArg(request, "start") or ''
fromFetchMore = ((not landing) and (not appchange) and start)
if detail == "activity":
userItems = yield self._getUserItems(request, userId, start=start)
args.update(userItems)
elif detail == 'files':
end = utils.getRequestArg(request, "end") or ''
end = utils.decodeKey(end)
start = utils.decodeKey(start)
userFiles = yield files.userFiles(myId, userId, args['orgId'], start, end, fromFeed=False)
args['userfiles'] = userFiles
args['fromProfile'] = True
if script:
t.renderScriptBlock(request, "profile.mako", "tabs", landing,
"#profile-tabs", "set", **args)
handlers = {} if detail != "activity" \
else {"onload": "(function(obj){$$.convs.load(obj);})(this);"}
if fromFetchMore and detail == "activity":
t.renderScriptBlock(request, "profile.mako", "content", landing,
"#next-load-wrapper", "replace", True,
handlers=handlers, **args)
else:
t.renderScriptBlock(request, "profile.mako", "content", landing,
"#profile-content", "set", True,
handlers=handlers, **args)
# List the user's subscriptions
cols = yield db.get_slice(userId, "subscriptions", count=11)
subscriptions = set(utils.columnsToDict(cols).keys())
args["subscriptions"] = subscriptions
# List the user's followers
cols = yield db.get_slice(userId, "followers", count=11)
followers = set(utils.columnsToDict(cols).keys())
args["followers"] = followers
# Fetch item data (name and avatar) for subscriptions, followers,
# user groups and common items.
entitiesToFetch = followers.union(subscriptions)\
.difference(fetchedEntities)
# List the user's groups (and look for groups common with me)
cols = yield db.multiget_slice([myId, userId], "entityGroupsMap")
myGroups = set([x.column.name.split(':', 1)[1] for x in cols[myId]])
userGroups = set([x.column.name.split(':', 1)[1] for x in cols[userId]])
commonGroups = myGroups.intersection(userGroups)
if len(userGroups) > 10:
userGroups = sample(userGroups, 10)
args["userGroups"] = userGroups
args["commonGroups"] = commonGroups
groupsToFetch = commonGroups.union(userGroups)
entitiesToFetch = entitiesToFetch.union(groupsToFetch)
entities = base.EntitySet(entitiesToFetch)
yield entities.fetchData()
for entityId, entity in entities.items():
if not entity._data:
del entities[entityId]
args["entities"].update(entities)
if script:
t.renderScriptBlock(request, "profile.mako", "user_subscriptions",
landing, "#user-subscriptions", "set", **args)
t.renderScriptBlock(request, "profile.mako", "user_followers",
landing, "#user-followers", "set", **args)
t.renderScriptBlock(request, "profile.mako", "user_me",
landing, "#user-me", "set", **args)
t.renderScriptBlock(request, "profile.mako", "user_groups",
landing, "#user-groups", "set", **args)
if script and landing:
request.write("</body></html>")
if not script:
t.render(request, "profile.mako", **args)
def _invite(self, request, convId=None):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
landing = not self._ajax
myOrgId = args['orgId']
convId, conv = yield utils.getValidItemId(request, "id", columns=["invitees"])
# Parse invitees from the tag edit plugin values
arg_keys = request.args.keys()
invitees, new_invitees = [], []
for arg in arg_keys:
if arg.startswith("invitee[") and arg.endswith("-a]"):
rcpt = arg.replace("invitee[", "").replace("-a]", "")
if rcpt != "":
invitees.append(rcpt)
if myId not in conv["invitees"].keys():
raise errors.invalidRequest(_("Only those who are invited can invite others"))
res = base.EntitySet(invitees)
yield res.fetchData()
invitees = [x for x in res.keys() if res[x].basic["org"] == myOrgId]
invitees = [x for x in invitees if x not in conv["invitees"].keys()]
relation = Relation(myId, [])
updateConv = {"meta":{}, "invitees":{}}
if myId == conv["meta"]["owner"]:
#If invited by owner, add the invitees to the ACL
acl = conv["meta"]["acl"]
acl = pickle.loads(acl)
acl.setdefault("accept", {}).setdefault("users", [])
acl["accept"]["users"].extend(invitees)
updateConv["meta"]["acl"] = pickle.dumps(acl)
new_invitees.extend(invitees)
else:
for invitee in invitees:
relation = Relation(invitee, [])
yield relation.initGroupsList()
withinAcl = utils.checkAcl(invitee, myOrgId, False,
relation, conv["meta"])
if withinAcl:
new_invitees.append(invitee)
if new_invitees:
convMeta = conv["meta"]
starttime = int(convMeta["event_startTime"])
starttimeUUID = utils.uuid1(timestamp=starttime)
starttimeUUID = starttimeUUID.bytes
endtime = int(convMeta["event_endTime"])
endtimeUUID = utils.uuid1(timestamp=endtime)
endtimeUUID = endtimeUUID.bytes
updateConv["invitees"] = dict([(x, myId) for x in new_invitees])
d = yield db.batch_insert(convId, "items", updateConv)
yield event.inviteUsers(request, starttimeUUID, endtimeUUID,
convId, conv["meta"], myId,
myOrgId, new_invitees)
request.write("""$$.alerts.info('%s');""" \
%("%d people invited to this event" %len(new_invitees)))
else:
if not invitees:
request.write("""$$.alerts.info('%s');""" \
%("Invited persons are already on the invitation list"))
else:
request.write("""$$.alerts.info('%s');""" \
%("Invited persons do not have access to this event"))
request.write("$('#item-subactions .tagedit-listelement-old').remove();")
def fetchMatchingEvents(self, request, args, entityId, count=5, start=None):
"""Find matching events for the user, org or group for a given time
range. Events are sorted by their start time and then by their end time.
"""
myId = args["myId"]
convs = []
invitations = []
toFetchEntities = set()
my_tz = timezone(args["me"].basic["timezone"])
if not start:
# since we store times in UTC, find out the utc time for the user's
# 00:00 hours instead of utc 00:00.
utc_now = datetime.datetime.now(pytz.utc)
mytz_now = utc_now.astimezone(my_tz)
mytz_start = mytz_now
else:
mytz_start = start.replace(tzinfo=my_tz)
args["start"] = mytz_start.strftime("%Y-%m-%d")
timestamp = calendar.timegm(mytz_start.utctimetuple())
timeUUID = utils.uuid1(timestamp=timestamp)
start = timeUUID.bytes
page = args.get("page", 1)
cols = yield db.get_slice(entityId, "userAgenda", start=start,
count=(page*count)*2)
matched_events = [col.column.value for col in cols]
res = yield db.multiget_slice(matched_events, "items", ["meta"])
matched_events = utils.multiSuperColumnsToDict(res)
to_sort_time_tuples = [(x, y["meta"]["event_startTime"],
y["meta"]["event_endTime"]) \
for x, y in matched_events.iteritems()]
sorted_time_tuples = sorted(to_sort_time_tuples,
key=itemgetter(int(1), int(2)))
sorted_event_ids = [x[0] for x in sorted_time_tuples]
events_in_this_page = sorted_event_ids[(page-1)*count:page*count]
if len(events_in_this_page) >= count:
nextPage = page + 1
args.update({'nextPage': nextPage})
else:
args.update({'nextPage': 0})
args["prevPage"] = page - 1
args["items"] = matched_events
args["conversations"] = events_in_this_page
#Now fetch all related entities, participants, owners, attendees,
# invitees, groups etc
for convId in events_in_this_page:
entityIds = yield self.fetchData(args, convId)
toFetchEntities.update(entityIds)
relation = Relation(myId, [])
yield relation.initGroupsList()
for event, event_meta in matched_events.iteritems():
target = event_meta['meta'].get('target')
if target:
toFetchEntities.update(target.split(','))
entities = base.EntitySet(toFetchEntities)
yield entities.fetchData()
args["entities"] = entities
args["relations"] = relation
def _renderReport(self, request, partial=False):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
landing = not self._ajax
convId, conv = yield utils.getValidItemId(request, "id", columns=["reports"])
if 'parent' in conv['meta']:
raise errors.InvalidItem('conversation', convId)
args["entities"] = {}
toFetchEntities = set()
args['convId'] = convId
args['items'] = {convId: conv}
convMeta = conv["meta"]
convType = convMeta.get("type", None)
relation = Relation(myId, [])
yield relation.initGroupsList()
args["relations"] = relation
if script and landing:
t.render(request, "item-report.mako", **args)
if script and appchange:
t.renderScriptBlock(request, "item-report.mako", "layout",
landing, "#mainbar", "set", **args)
plugin = plugins[convType] if convType in plugins else None
if plugin:
entityIds = yield plugin.fetchData(args)
toFetchEntities.update(entityIds)
convOwner = convMeta['owner']
toFetchEntities.add(convOwner)
if "target" in convMeta:
toFetchEntities.update(convMeta['target'].split(','))
if "reportId" in convMeta:
toFetchEntities.add(convMeta['reportedBy'])
entities = base.EntitySet(toFetchEntities)
yield entities.fetchData()
args["entities"] = entities
args["ownerId"] = convOwner
if script:
t.renderScriptBlock(request, "item.mako", "conv_root",
landing, "#conv-root-%s > .conv-summary" % (convId),
"set", **args)
t.renderScriptBlock(request, "item-report.mako", 'conv_footer',
landing, '#item-footer-%s' % convId,
'set', **args)
if convType != "feedback":
t.renderScriptBlock(request, "item.mako", "conv_owner",
landing, "#conv-avatar-%s" % convId,
"set", **args)
else:
feedbackType = conv['meta']['subType']
t.renderScriptBlock(request, "item.mako", "feedback_icon",
landing, "#conv-avatar-%s" % convId,
"set", args=[feedbackType])
yield self._renderReportResponses(request, convId, convMeta, args)
if not script:
t.render(request, "item-report.mako", **args)
def renderItem(self, request, toFeed=False):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
landing = not self._ajax
myOrgId = args["orgId"]
convId, conv = yield utils.getValidItemId(request, "id", columns=['tags'])
itemType = conv["meta"].get("type", None)
if 'parent' in conv['meta']:
raise errors.InvalidItem('conversation', convId)
start = utils.getRequestArg(request, "start") or ''
start = utils.decodeKey(start)
args['convId'] = convId
args['isItemView'] = True
args['items'] = {convId: conv}
meta = conv["meta"]
owner = meta["owner"]
relation = Relation(myId, [])
yield defer.DeferredList([relation.initGroupsList(),
relation.initSubscriptionsList()])
args["relations"] = relation
if script and landing:
t.render(request, "item.mako", **args)
if script and appchange:
t.renderScriptBlock(request, "item.mako", "layout",
landing, "#mainbar", "set", **args)
args["entities"] = {}
toFetchEntities = set()
toFetchTags = set(conv.get("tags", {}).keys())
plugin = plugins[itemType] if itemType in plugins else None
if plugin:
entityIds = yield plugin.fetchData(args)
toFetchEntities.update(entityIds)
toFetchEntities.add(conv['meta']['owner'])
if "target" in conv["meta"]:
toFetchEntities.update(conv['meta']['target'].split(','))
if conv['meta']['owner'] not in toFetchEntities:
toFetchEntities.add(conv['meta']['owner'])
entities = base.EntitySet(toFetchEntities)
yield entities.fetchData()
args["entities"] = entities
renderers = []
if script:
t.renderScriptBlock(request, "item.mako", "conv_root",
landing, "#conv-root-%s > .conv-summary" % (convId),
"set", **args)
convOwner = args["items"][convId]["meta"]["owner"]
args["ownerId"] = convOwner
if script:
if itemType != "feedback":
t.renderScriptBlock(request, "item.mako", "conv_owner",
landing, "#conv-avatar-%s" % convId,
"set", **args)
else:
feedbackType = conv['meta']['subType']
t.renderScriptBlock(request, "item.mako", "feedback_icon",
landing, "#conv-avatar-%s" % convId,
"set", args=[feedbackType])
# A copy of this code for fetching comments is present in _responses
# Most changes here may need to be done there too.
itemResponses = yield db.get_slice(convId, "itemResponses",
start=start, reverse=True,
count=constants.COMMENTS_PER_PAGE + 1)
nextPageStart = itemResponses[-1].column.name\
if len(itemResponses) > constants.COMMENTS_PER_PAGE\
else None
itemResponses = itemResponses[:-1] \
if len(itemResponses) > constants.COMMENTS_PER_PAGE\
else itemResponses
responseKeys = []
for response in itemResponses:
userKey, responseKey = response.column.value.split(":")
responseKeys.append(responseKey)
toFetchEntities.add(userKey)
responseKeys.reverse()
subscriptions = list(relation.subscriptions)
likes = yield db.get_slice(convId, "itemLikes", subscriptions) \
if subscriptions else defer.succeed([])
toFetchEntities.update([x.column.name for x in likes])
entities = base.EntitySet(toFetchEntities)
d1 = entities.fetchData()
d2 = db.multiget_slice(responseKeys, "items", ["meta", "attachments"])
d3 = db.multiget_slice(responseKeys + [convId], "itemLikes", [myId])
d4 = db.get_slice(myOrgId, "orgTags", toFetchTags)\
if toFetchTags else defer.succeed([])
yield d1
fetchedItems = yield d2
myLikes = yield d3
fetchedTags = yield d4
fetchedItems = utils.multiSuperColumnsToDict(fetchedItems)
myLikes = utils.multiColumnsToDict(myLikes)
fetchedTags = utils.supercolumnsToDict(fetchedTags)
# Do some error correction/consistency checking to ensure that the
# response items actually exist. I don't know of any reason why these
# items may not exist.
missingIds = [x for x, y in fetchedItems.items() if not y]
if missingIds:
yield self._cleanupMissingComments(convId, missingIds, itemResponses)
args["items"].update(fetchedItems)
args["entities"].update(entities)
args["myLikes"] = myLikes
args["tags"] = fetchedTags
args["responses"] = {convId: responseKeys}
if nextPageStart:
args["oldest"] = utils.encodeKey(nextPageStart)
if script:
t.renderScriptBlock(request, "item.mako", 'conv_footer',
landing, '#item-footer-%s' % convId,
'set', **args)
t.renderScriptBlock(request, "item.mako", 'conv_tags',
landing, '#conv-tags-wrapper-%s' % convId, 'set',
handlers={"onload": "$('#conv-meta-wrapper-%s').removeClass('no-tags')" % convId} if toFetchTags else None, **args)
t.renderScriptBlock(request, "item.mako", 'conv_comments',
landing, '#conv-comments-wrapper-%s' % convId, 'set', **args)
t.renderScriptBlock(request, "item.mako", 'conv_comment_form',
landing, '#comment-form-wrapper-%s' % convId, 'set',
True, handlers={"onload": "(function(obj){$$.convs.load(obj);})(this);"}, **args)
numLikes = int(conv["meta"].get("likesCount", "0"))
if numLikes:
numLikes = int(conv["meta"].get("likesCount", "0"))
iLike = myId in args["myLikes"].get(convId, [])
t.renderScriptBlock(request, "item.mako", 'conv_likes',
landing, '#conv-likes-wrapper-%s' % convId, 'set',
args=[convId, numLikes, iLike, [x.column.name for x in likes]],
entities=args['entities'])
if plugin and hasattr(plugin, 'renderItemSideBlock'):
plugin.renderItemSideBlock(request, landing, args)
if script and landing:
request.write("</body></html>")
if not script:
t.render(request, "item.mako", **args)
def _update_suggestions(request, relation=None):
authinfo = request.getSession(IAuthInfo)
myId = authinfo.username
orgId = authinfo.organization
weights = {'group': {'follower': 15, 'subscription': 40, 'group': 30},
'follower': {'follower': 9, 'subscription': 24, 'group': 15},
'subscription': {'follower': 24, 'subscription': 64, 'group': 40}}
defaultWeight = 1
people = {}
@defer.inlineCallbacks
def _compute_weights(userIds, myGroups, type):
followers = yield db.multiget_slice(userIds, "followers", count=50)
subscriptions = yield db.multiget_slice(userIds, "subscriptions", count=50)
groups = yield db.multiget_slice(userIds, "entityGroupsMap")
followers = utils.multiColumnsToDict(followers)
subscriptions = utils.multiColumnsToDict(subscriptions)
groups = utils.multiColumnsToDict(groups)
for userId in followers:
for follower in followers[userId]:
people[follower] = people.setdefault(follower, defaultWeight) + weights[type]['follower']
for userId in subscriptions:
for subscription in subscriptions[userId]:
people[subscription] = people.setdefault(subscription, defaultWeight) + weights[type]['subscription']
for userId in groups:
groupIds = [x.split(':', 1)[1] for x in groups[userId]]
for groupId in groupIds:
if groupId in myGroups:
people[userId] = people.setdefault(userId, defaultWeight) + weights[type]['group']
if not relation:
relation = Relation(myId, [])
yield defer.DeferredList([relation.initSubscriptionsList(),
relation.initFollowersList(),
relation.initGroupsList()])
if relation.followers:
yield _compute_weights(relation.followers, relation.groups, 'follower')
if relation.subscriptions:
yield _compute_weights(relation.subscriptions, relation.groups, 'subscription')
if relation.groups:
groupMembers = yield db.multiget_slice(relation.groups, "groupMembers", count=20)
groupMembers = utils.multiColumnsToDict(groupMembers)
for groupId in groupMembers:
yield _compute_weights(groupMembers[groupId], relation.groups, 'group')
cols = yield db.get_slice(orgId, "orgUsers", count=100)
for col in cols:
userId = col.column.name
if userId not in people:
people[userId] = people.setdefault(userId, 0) + defaultWeight
suggestions = {}
for userId in people:
if isValidSuggestion(myId, userId, relation):
suggestions.setdefault(people[userId], []).append(userId)
yield db.remove(myId, "suggestions")
weights_userIds_map = {}
format = '>l'
for weight in suggestions:
key = struct.pack(format, weight)
weights_userIds_map[key] = ' '.join(suggestions[weight])
if weights_userIds_map:
yield db.batch_insert(myId, "suggestions", weights_userIds_map)
def search(self, request):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
landing = not self._ajax
myOrgId = args['orgId']
term = utils.getRequestArg(request, "q")
if not term:
errors.MissingParams(['Search term'])
args["term"] = term
itemType = utils.getRequestArg(request, "it") or 0
try:
itemType = int(itemType)
if not 0 < itemType < 31:
raise ValueError
except ValueError:
itemType = self.TYPE_ITEMS | self.TYPE_PEOPLE
args["itemType"] = itemType
start = utils.getRequestArg(request, "start") or 0
try:
start = int(start)
if start < 0:
raise ValueError
except ValueError:
start = 0
args["start"] = start
if script and landing:
t.render(request, "search.mako", **args)
if script and appchange:
t.renderScriptBlock(request, "search.mako", "layout",
landing, "#mainbar", "set", **args)
toFetchEntities = set()
toFetchItems = set()
toFetchTags = set()
deferreds = []
highlight = {}
# If searching for more than one itemType then use half the count.
count = SEARCH_RESULTS_PER_PAGE if itemType in [1,2,4,8,16] else SEARCH_RESULTS_PER_PAGE/2
relation = Relation(myId, [])
relation_d = relation.initGroupsList()
args['relations'] = relation
people = {}
args['matchedUsers'] = people
if itemType & self.TYPE_PEOPLE:
d = solr.search(term, myOrgId, count, start, filters={'_type':'people'})
def _gotPeople(results):
docs = results.data.get('response', {}).get('docs', [])
highlight.update(results.data.get('highlighting'))
for item in docs:
entityId = item['id']
people[entityId] = item
args['matchedUserCount'] = results.data.get('response', {}).get('numFound', 0)
d.addCallback(_gotPeople)
deferreds.append(d)
items = {}
matchedItemIds = []
args['items'] = items
args['matchedItemIds'] = matchedItemIds
if itemType & self.TYPE_ITEMS:
yield relation_d
aclFilterEntities = relation.groups + [myOrgId, myId]
filters = {'_type': 'item',
'_acceptACL': '(%s)' % ' OR '.join(aclFilterEntities),
'-_denyACL': '(%s)' % ' OR '.join(aclFilterEntities)}
d = solr.search(term, myOrgId, count, start, filters=filters)
@defer.inlineCallbacks
def _gotConvs(results):
docs = results.data.get('response', {}).get('docs', [])
highlight.update(results.data.get('highlighting'))
for index, item in enumerate(docs):
itemId = item['id']
parentId = item.get('parent', None)
if parentId:
toFetchItems.add(itemId)
toFetchItems.add(parentId)
matchedItemIds.append(itemId)
else:
toFetchItems.add(itemId)
matchedItemIds.append(itemId)
if toFetchItems and matchedItemIds:
fetchedItems = yield db.multiget_slice(toFetchItems,
"items", ["meta", "tags", "attachments"])
fetchedItems = utils.multiSuperColumnsToDict(fetchedItems)
for itemId, item in fetchedItems.items():
toFetchEntities.add(item['meta']['owner'])
if 'target' in item['meta']:
toFetchEntities.update(item['meta']['target'].split(','))
items.update(fetchedItems)
extraDataDeferreds = []
for itemId in toFetchItems:
if itemId in matchedItemIds and itemId in items:
meta = items[itemId]['meta']
itemType = meta.get('type', 'status')
if itemType in plugins:
d = plugins[itemType].fetchData(args, itemId)
extraDataDeferreds.append(d)
result = yield defer.DeferredList(extraDataDeferreds)
for success, ret in result:
if success:
toFetchEntities.update(ret)
args['matchedItemCount'] = results.data.get('response', {}).get('numFound', 0)
d.addCallback(_gotConvs)
deferreds.append(d)
yield defer.DeferredList(deferreds)
entities = base.EntitySet(toFetchEntities)
args['entities'] = entities
if toFetchEntities:
yield entities.fetchData()
tags = {}
args['tags'] = tags
if toFetchTags:
fetchedTags = yield db.get_slice(myOrgId, "orgTags", toFetchTags)
tags.update(utils.supercolumnsToDict(fetchedTags))
args['highlight'] = highlight
if script:
t.renderScriptBlock(request, "search.mako", "results",
landing, "#search-results", "set", **args)
else:
t.render(request, "search.mako", **args)
def userFiles(myId, entityId, myOrgId, start='', end='', fromFeed=True):
allItems = {}
hasPrevPage = False # Do we have another page before the current one.
nextPageStart = '' # Start item for the next page
accessibleFiles = []
accessibleItems = []
toFetchEntities = set()
count = constants.FILES_PER_PAGE
toFetchCount = count + 1
relation = Relation(myId, [])
yield relation.initGroupsList()
# Fetching files owned by entityId or files that were part of entityId's feed.
cf = 'entityFeed_files' if fromFeed else 'user_files'
# End is actually the start item of next page.
# If @end is set, we have to display @count items before @end. For that
# we fetch @count + 2 items before (and including) @end. Of the extra items
# fetched, one item helps us determine if there is another page before this
# and the other one is the start of next page.
if end:
start = end
reverse = False
toFetchCount += 1
else:
reverse = True
while 1:
files = yield db.get_slice(entityId, cf, count=toFetchCount, start=start, reverse=reverse)
files = utils.columnsToDict(files, True)
toFetchItems = []
for tuuid in files:
if len(files[tuuid].split(':')) == 4:
fid, name, itemId, attachmentId = files[tuuid].split(':')
toFetchItems.append(itemId)
toFetchItems = [itemId for itemId in toFetchItems if itemId not in accessibleItems]
if toFetchItems:
items = yield db.multiget_slice(toFetchItems, "items", ["meta"])
items = utils.multiSuperColumnsToDict(items)
toFetchConvIds = [items[itemId]['meta']['parent'] for itemId in items if 'parent' in items[itemId]['meta'] and items[itemId]['meta']['parent'] not in allItems]
if toFetchConvIds:
convs = yield db.multiget_slice(toFetchConvIds, "items", ["meta"])
convs = utils.multiSuperColumnsToDict(convs)
allItems.update(convs)
allItems.update(items)
for itemId in items:
if 'parent' in items[itemId]['meta']:
convId = items[itemId]['meta']['parent']
acl = allItems[convId]['meta']['acl']
else:
acl = items[itemId]['meta']['acl']
convId = itemId
if utils.checkAcl(myId, myOrgId, False, relation, allItems[convId]['meta']):
accessibleItems.append(itemId)
for tuuid in files:
if len(files[tuuid].split(':')) == 4:
fid, name, itemId, ownerId = files[tuuid].split(':')
if itemId in accessibleItems:
accessibleFiles.append((tuuid,
(fid, urlsafe_b64decode(name),
itemId, ownerId, allItems[itemId])))
toFetchEntities.add(ownerId)
if len(files) < toFetchCount or len(accessibleFiles) > count:
break
else:
start = files.keys()[-1]
if end:
# We have enough items to have another page before this.
if len(accessibleFiles) > count + 1:
hasPrevPage = True
accessibleFiles = accessibleFiles[:count + 1]
# Revert the list to get most recent items first.
accessibleFiles.reverse()
# The last item is actually the first item of next page.
nextPageStart = accessibleFiles[-1][0]
accessibleFiles = accessibleFiles[:-1]
elif start:
hasPrevPage = True # XXX: may not always be true, but the edge case is OK
if len(accessibleFiles) > count:
nextPageStart = accessibleFiles[count][0]
accessibleFiles = accessibleFiles[:count]
defer.returnValue((accessibleFiles, hasPrevPage, nextPageStart, toFetchEntities))
