def _listBlockedUsers(self, request):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
orgId = args["orgId"]
landing = not self._ajax
start = utils.getRequestArg(request, 'start') or ''
start = utils.decodeKey(start)
count = PEOPLE_PER_PAGE
toFetchCount = count + 1
nextPageStart = ''
prevPageStart = ''
args["title"] = "Manage Users"
args["menuId"] = "users"
args["viewType"] = "blocked"
if script and landing:
t.render(request, "admin.mako", **args)
if script and appchange:
t.renderScriptBlock(request, "admin.mako", "layout",
landing, "#mainbar", "set", **args)
args["heading"] = "Admin Console - Blocked Users"
cols = yield db.get_slice(orgId, "blockedUsers",
start=start, count=toFetchCount)
blockedUsers = [col.column.name for col in cols]
if len(blockedUsers) > count:
nextPageStart = utils.encodeKey(blockedUsers[-1])
blockedUsers = blockedUsers[:count]
if start:
cols = yield db.get_slice(orgId, "blockedUsers", start=start,
count=toFetchCount, reverse=True)
if len(cols) > 1:
prevPageStart = utils.decodeKey(cols[-1].column.name)
entities = base.EntitySet(blockedUsers)
yield entities.fetchData()
args["entities"] = entities
args['nextPageStart'] = nextPageStart
args['prevPageStart'] = prevPageStart
if script:
t.renderScriptBlock(request, "admin.mako", "viewOptions",
landing, "#users-view", "set", **args)
t.renderScriptBlock(request, "admin.mako", "list_users",
landing, "#content", "set", **args)
if script and landing:
request.write("</body></html>")
if not script:
t.render(request, "admin.mako", **args)
def _updateExpertise(self, request, remove=False):
myId = request.getSession(IAuthInfo).username
orgId = request.getSession(IAuthInfo).organization
expertise = utils.getRequestArg(request, 'expertise', False)
if not expertise:
raise errors.MissingParams(['Expertise'])
if not remove:
decoded = expertise.decode('utf-8', 'replace')
if len(decoded) > 50 or not re.match('^[\w-]*$', decoded):
raise errors.InvalidRequest('Expertise can only be upto 50 characters long and can include numerals, alphabet and hyphens (-) only.')
yield db.insert(myId, "entities", '', expertise, "expertise")
else:
yield db.remove(myId, "entities", utils.decodeKey(expertise), "expertise")
me = base.Entity(myId)
yield me.fetchData([])
expertise = me.get('expertise')
onload = "$('#expertise-textbox').val('');"
yield t.renderScriptBlock(request, "settings.mako", "_expertise",
False, "#expertise-container", "set", True,
handlers={"onload": onload}, args=[expertise])
yield search.solr.updatePeopleIndex(myId, me, orgId)
def _editCompany(self, request):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
remove = utils.getRequestArg(request, 'action') == 'd'
encodedCompanyId = utils.getRequestArg(request, 'id', sanitize=False)
companyId = utils.decodeKey(encodedCompanyId) if encodedCompanyId else None
if companyId and remove:
db.remove(myId, "entities", companyId, "companies")
request.write('$("#%s").remove();' % encodedCompanyId)
return
today = datetime.date.today()
try:
startYear = int(utils.getRequestArg(request, 'startyear'))
startMonth = int(utils.getRequestArg(request, 'startmonth'))
startDay = datetime.date(startYear, startMonth, 1)
except (ValueError, TypeError):
raise errors.InvalidRequest('Please give a valid start month and year')
try:
endYear = utils.getRequestArg(request, 'endyear')
if not endYear:
endYear = 9999
endMonth = 12
else:
endYear = int(endYear)
endMonth = int(utils.getRequestArg(request, 'endmonth'))
endDay = datetime.date(endYear, endMonth, 1)
except (ValueError, TypeError):
raise errors.InvalidRequest('Please give a valid end month and year')
if startDay > today or startDay > endDay or (endDay > today and endYear != 9999):
raise errors.InvalidRequest('The start month/year and end month/year are invalid!')
name = utils.getRequestArg(request, 'company')
title = utils.getRequestArg(request, 'title')
if not remove and not name:
errors.MissingParams(['Name'])
if companyId:
db.remove(myId, "entities", companyId, "companies")
newCompanyId = "%s%s:%s%s:%s" % (endYear, endMonth, startYear, startMonth, name)
newCompanyVal = title
db.insert(myId, "entities", newCompanyVal, newCompanyId, "companies")
if companyId:
yield t.renderScriptBlock(request, "settings.mako", "companyItem",
False, "#"+encodedCompanyId, "replace",
args=[newCompanyId, newCompanyVal])
else:
onload = """$('#company-empty-msg').remove();"""+\
"""$('#addemp-wrap').replaceWith('<div id="addemp-wrap"><button class="button ajax" id="addedu-button" data-ref="/settings/company">Add Company</button></div>');"""
yield t.renderScriptBlock(request, "settings.mako", "companyItem",
False, "#companies-wrapper", "append", True,
handlers={'onload': onload},
args=[newCompanyId, newCompanyVal])
def _renderFileList(self, request):
appchange, script, args, myId = yield self._getBasicArgs(request)
landing = not self._ajax
myOrgId = args["orgId"]
start = utils.getRequestArg(request, "start") or ''
start = utils.decodeKey(start)
end = utils.getRequestArg(request, "end") or ''
end = utils.decodeKey(end)
viewType = utils.getRequestArg(request, "type")
viewType = viewType if viewType in ['myFiles', 'companyFiles', 'myFeedFiles'] else 'myFiles'
args['menuId'] = 'files'
if script and landing:
t.render(request, "files.mako", **args)
if script and appchange:
t.renderScriptBlock(request, "files.mako", 'layout',
landing, "#mainbar", "set", **args)
args['viewType'] = viewType
entityId = myId if viewType in ['myFiles', 'myFeedFiles'] else myOrgId
fromFeed = viewType != 'myFiles'
if script:
t.renderScriptBlock(request, "files.mako", "viewOptions",
landing, "#file-view", "set", args=[viewType])
files = yield userFiles(myId, entityId, args['orgId'], start, end, fromFeed)
toFetchEntities = files[3]
entities = base.EntitySet(toFetchEntities)
yield entities.fetchData()
args['entities'] = entities
args['userfiles'] = files
if script:
t.renderScriptBlock(request, "files.mako", "listFiles",
landing, "#files-content", "set", **args)
t.renderScriptBlock(request, "files.mako", "pagingBar",
landing, "#files-paging", "set", **args)
else:
t.render(request, "files.mako", **args)
def _getFileInfo(self, request):
"""Fetch the meta info on a file that is being requested to be
downloaded. Returns the meta info of the file in question.
Keyword Arguments:
itemId: id of the conversation on which this file is attached.
attachmentId: id of the file on the amazon S3 that is to be served.
version: version of the file on the amazon S3 that the user is
requesting.
"""
authinfo = request.getSession(IAuthInfo)
myId = authinfo.username
myOrgId = authinfo.organization
itemId = utils.getRequestArg(request, "id", sanitize=False)
attachmentId = utils.getRequestArg(request, "fid", sanitize=False)
version = utils.getRequestArg(request, "ver", sanitize=False) or ''
columns = ["meta", "attachments", "participants"]
if not (itemId and attachmentId):
raise errors.MissingParams([])
item = yield db.get_slice(itemId, "mConversations", columns)
item = utils.supercolumnsToDict(item)
if not item:
raise errors.InvalidMessage(itemId)
if myId not in item.get('participants', {}):
raise errors.MessageAccessDenied(itemId)
# Check if the attachmentId belong to item
if attachmentId not in item['attachments'].keys():
raise errors.InvalidAttachment(itemId, attachmentId, version)
fileId, filetype, name = None, 'text/plain', 'file'
if version:
version = utils.decodeKey(version)
try:
cols = yield db.get(attachmentId, "attachmentVersions", version)
except ttypes.NotFoundException:
raise errors.InvalidAttachment(itemId, attachmentId, version)
except ttypes.InvalidRequestException:
raise errors.InvalidAttachment(itemId, attachmentId, version)
cols = utils.columnsToDict([cols])
else:
cols = yield db.get_slice(attachmentId, "attachmentVersions", count=1, reverse=True)
cols = utils.columnsToDict(cols)
version = cols.keys()[0]
fileId, name, size, filetype = cols[version].split(':')
files = yield db.get_slice(fileId, "files", ["meta"])
files = utils.supercolumnsToDict(files)
url = files['meta']['uri']
owner = files["meta"]["owner"]
defer.returnValue([owner, url, filetype, size, name])
def _deleteKeyword(self, request):
orgId = request.getSession(IAuthInfo).organization
keyword = utils.getRequestArg(request, 'keyword') or ''
keyword = utils.decodeKey(keyword)
if not keyword:
return
yield db.remove(orgId, "keywords", keyword)
yield db.remove(orgId, "originalKeywords", keyword)
yield db.remove(orgId + ':' + keyword, "keywordItems")
request.write('$("#keyword-%s").remove()' % (utils.encodeKey(keyword)))
def render_GET(self, request):
segmentCount = len(request.postpath)
viewType = utils.getRequestArg(request, "type") or "all"
start = utils.getRequestArg(request, "start") or ""
start = utils.decodeKey(start)
d = None
if segmentCount == 0:
d = self._render(request, viewType, start)
elif segmentCount == 1 and self._ajax and request.postpath[0] == "invite":
d = self._renderInvitePeople(request)
elif segmentCount == 1 and request.postpath[0] == "suggestions":
d = self._renderSuggestions(request)
return self._epilogue(request, d)
def _editSchoolForm(self, request):
encodedSchoolId = utils.getRequestArg(request, 'id')
schoolId = utils.decodeKey(encodedSchoolId) if encodedSchoolId else None
if schoolId:
try:
myId = request.getSession(IAuthInfo).username
schoolVal = yield db.get(myId, 'entities', schoolId, "schools")
schoolVal = schoolVal.column.value
yield t.renderScriptBlock(request, "settings.mako", "schoolForm",
False, "#"+encodedSchoolId, "replace",
args=[schoolId, schoolVal])
return
except: pass
yield t.renderScriptBlock(request, "settings.mako", "schoolForm",
False, "#addedu-wrap", "set")
def _editCompanyForm(self, request):
encodedCompanyId = utils.getRequestArg(request, 'id')
companyId = utils.decodeKey(encodedCompanyId) if encodedCompanyId else None
if companyId:
try:
myId = request.getSession(IAuthInfo).username
companyVal = yield db.get(myId, 'entities', companyId, "companies")
companyVal = companyVal.column.value
yield t.renderScriptBlock(request, "settings.mako", "companyForm",
False, "#"+encodedCompanyId, "replace",
args=[companyId, companyVal])
return
except: pass
yield t.renderScriptBlock(request, "settings.mako", "companyForm",
False, "#addemp-wrap", "set")
def _editSchool(self, request):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
remove = utils.getRequestArg(request, 'action') == 'd'
encodedSchoolId = utils.getRequestArg(request, 'id', sanitize=False)
schoolId = utils.decodeKey(encodedSchoolId) if encodedSchoolId else None
if schoolId and remove:
db.remove(myId, "entities", schoolId, "schools")
request.write('$("#%s").remove();' % encodedSchoolId)
return
curYear = datetime.date.today().year
try:
year = int(utils.getRequestArg(request, 'year'))
if not 1920 < year <= curYear:
raise ValueError
except (ValueError,TypeError):
raise errors.InvalidRequest('Invalid graduation year')
name = utils.getRequestArg(request, 'school')
degree = utils.getRequestArg(request, 'degree')
if not remove and not name:
errors.MissingParams(['Name'])
if schoolId:
db.remove(myId, "entities", schoolId, "schools")
newSchoolId = "%s:%s" % (year, name)
newSchoolVal = degree
db.insert(myId, "entities", newSchoolVal, newSchoolId, "schools")
if schoolId:
yield t.renderScriptBlock(request, "settings.mako", "schoolItem",
False, "#"+encodedSchoolId, "replace",
args=[newSchoolId, newSchoolVal])
else:
onload = """$('#school-empty-msg').remove();"""+\
"""$('#addedu-wrap').replaceWith('<div id="addedu-wrap"><button class="button ajax" id="addedu-button" data-ref="/settings/school">Add School</button></div>');"""
yield t.renderScriptBlock(request, "settings.mako", "schoolItem",
False, "#schools-wrapper", "append", True,
handlers={'onload': onload},
args=[newSchoolId, newSchoolVal])
def _responses(self, request, data=None):
convId, conv = data['id']
start = data['start']
isFeed = data['_pg'] != '/item'
showing = data["nc"]
start = utils.decodeKey(start)
myId = request.getSession(IAuthInfo).username
landing = not self._ajax
me = base.Entity(myId)
responseCount = int(conv["meta"].get("responseCount", "0"))
if isFeed and responseCount > constants.MAX_COMMENTS_IN_FEED:
request.write("$$.fetchUri('/item?id=%s');" % convId)
return
ret = yield Item.responses(myId, convId, conv, start)
yield me.fetchData()
args = ret
args.update({"convId": convId, "isFeed": isFeed, "me": me})
if isFeed:
args["isItemView"] = False
showing = len(args['responses'][convId])
handler = {"onload": "(function(){$$.convs.showHideComponent('%s', 'comments', true); $('[name=\"nc\"]', '#comment-form-%s').val('%s');})();" % (convId, convId, showing)}
t.renderScriptBlock(request, "item.mako", 'conv_comments',
landing, '#conv-comments-wrapper-%s' % convId,
'set', True, handlers=handler, **args)
else:
showing = int(showing) + len(args['responses'][convId])
args["showing"] = showing
args["total"] = int(args["items"][convId]["meta"].get("responseCount", "0"))
args["isItemView"] = True
t.renderScriptBlock(request, "item.mako", 'conv_comments_head',
landing, '#comments-header-%s' % convId,
'set', **args)
t.renderScriptBlock(request, "item.mako", 'conv_comments_only',
landing, '#comments-%s' % convId, 'prepend', True,
handlers={"onload": "(function(){$('[name=\"nc\"]', '#comment-form-%s').val('%s');})();" % (convId, showing)},
**args)
def _listBannedUsers(self, request, data=None):
appchange, script, args, myId = yield self._getBasicArgs(request)
landing = not self._ajax
me = args['me']
group = data['id']
start = data['start']
start = utils.decodeKey(start)
entities = base.EntitySet(group)
args.update({"menuId": "banned", "groupId": group.id,
"entities": entities, "heading": group.basic['name']})
if me.id not in group.admins:
raise errors.InvalidRequest(_("Access Denied"))
if script and landing:
t.render(request, "group-settings.mako", **args)
if script and appchange:
t.renderScriptBlock(request, "group-settings.mako", "layout",
landing, "#mainbar", "set", **args)
data = yield Group.getBlockedMembers(group, me, start)
print data.keys()
args.update(data)
args['entities'].update(group)
args["tab"] = "banned"
if script:
t.renderScriptBlock(request, "group-settings.mako", "titlebar",
landing, "#titlebar", "set", **args)
t.renderScriptBlock(request, "group-settings.mako", "displayUsers",
landing, "#groups-wrapper", "set", **args)
t.renderScriptBlock(request, "group-settings.mako", "bannedUsersPaging",
landing, "#groups-paging", "set", **args)
def _listAllUsers(self, request):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
orgId = args["orgId"]
landing = not self._ajax
start = utils.getRequestArg(request, 'start') or ''
args["title"] = "Manage Users"
args["menuId"] = "users"
args["viewType"] = "all"
start = utils.decodeKey(start)
if script and landing:
t.render(request, "admin.mako", **args)
if script and appchange:
t.renderScriptBlock(request, "admin.mako", "layout",
landing, "#mainbar", "set", **args)
users, relations, userIds, blockedUsers, \
nextPageStart, prevPageStart = yield people.getPeople(myId, orgId,
orgId, start=start)
args["entities"] = users
args["relations"] = relations
args["people"] = userIds
args["nextPageStart"] = nextPageStart
args["prevPageStart"] = prevPageStart
args["blockedUsers"] = blockedUsers
if script:
t.renderScriptBlock(request, "admin.mako", "viewOptions",
landing, "#users-view", "set", **args)
t.renderScriptBlock(request, "admin.mako", "list_users",
landing, "#content", "set", **args)
else:
t.render(request, "admin.mako", **args)
def _getFileInfo(self, request):
authinfo = request.getSession(IAuthInfo)
myId = authinfo.username
myOrgId = authinfo.organization
itemId, item = yield utils.getValidItemId(request, 'id')
attachmentId = utils.getRequestArg(request, "fid", sanitize=False)
version = utils.getRequestArg(request, "ver", sanitize=False)
if not attachmentId:
raise errors.MissingParams()
# Check if the attachmentId belong to item
if attachmentId not in item['attachments'].keys():
raise errors.EntityAccessDenied("attachment", attachmentId)
if version:
version = utils.decodeKey(version)
cols = yield db.get(attachmentId, "attachmentVersions", version)
cols = utils.columnsToDict([cols])
else:
cols = yield db.get_slice(attachmentId, 'attachmentVersions', count=1, reverse=True)
cols = utils.columnsToDict(cols)
version = cols.keys()[0]
if not cols:
raise errors.InvalidRequest()
fileId, fileType, name = None, 'text/plain', 'file'
fileId, name, size, fileType = cols[version].split(':')
files = yield db.get_slice(fileId, "files", ["meta"])
files = utils.supercolumnsToDict(files)
url = files['meta']['uri']
owner = files["meta"]["owner"]
defer.returnValue([owner, url, fileType, size, name])
def _getNotifications(self, request, count=15):
authinfo = request.getSession(IAuthInfo)
myId = authinfo.username
myOrgId = authinfo.organization
nextPageStart = None
keysFromStore = [] # List of keys fetched
notifyIds = [] # convIds for which we have notifications
details_d = [] # Deferreds waiting of notification items
toFetchTags = set()
toFetchEntities = set()
tags = {}
entities = {}
timestamps = {}
notifyStrs = {}
notifyClasses = {}
notifyUsers = {}
fetchStart = utils.getRequestArg(request, 'start') or ''
if fetchStart:
fetchStart = utils.decodeKey(fetchStart)
fetchCount = count + 2
while len(notifyIds) < count:
fetchedNotifyIds = []
results = yield db.get_slice(myId, "notifications", count=fetchCount,
start=fetchStart, reverse=True)
for col in results:
value = col.column.value
if value not in notifyIds:
fetchedNotifyIds.append(value)
keysFromStore.append(col.column.name)
timestamps[value] = col.column.timestamp / 1e6
if not keysFromStore:
break
fetchStart = keysFromStore[-1]
notifyIds.extend(fetchedNotifyIds)
if len(results) < fetchCount:
break
if len(keysFromStore) > count:
nextPageStart = utils.encodeKey(keysFromStore[count])
notifyIds = notifyIds[0:count]
elif len(results) == fetchCount:
nextPageStart = utils.encodeKey(keysFromStore[-1])
notifyIds = notifyIds[0:-1]
# We don't have notifications on any conversations
if not notifyIds:
defer.returnValue({})
# We need the name of current user's organization
toFetchEntities.add(myOrgId)
# Fetch more data about the notifications
notifyItems = yield db.get_slice(myId, "notificationItems",
notifyIds, reverse=True)
notifyValues = {}
notifyParts = {}
notifyPlugins = {}
notifyPluginData = {}
for notify in notifyItems:
notifyId = notify.super_column.name
updates = notify.super_column.columns
updates.reverse()
notifyValues[notifyId] = []
parts = notifyId.split(':')
notifyType = parts[3] if parts[0] else parts[1]
plugin = _notificationPlugins.get(notifyType, None)
if not plugin:
continue
values = [update.value for update in updates]
userIds, entityIds, pluginData = \
yield plugin.fetchAggregationData(myId, myOrgId, parts, values)
notifyValues[notifyId] = utils.uniqify(values)
notifyParts[notifyId] = parts
notifyPlugins[notifyId] = plugin
notifyPluginData[notifyId] = pluginData
notifyUsers[notifyId] = utils.uniqify(userIds)
toFetchEntities.update(entityIds)
# Fetch the required entities
entities = base.EntitySet(toFetchEntities)
yield entities.fetchData()
myOrg = entities.get(myOrgId)
# Build strings
notifyStrs = {}
data = {'entities': entities, 'myId': myId, 'orgId': myOrgId}
for notifyId in notifyIds:
parts = notifyParts.get(notifyId, None)
if not parts:
continue
plugin = notifyPlugins[notifyId]
notifyStrs[notifyId] = plugin.aggregation(parts,
notifyValues[notifyId], data,
notifyPluginData[notifyId])
args = {"notifications": notifyIds, "notifyStr": notifyStrs,
"notifyClasses": notifyClasses, "notifyUsers": notifyUsers,
"entities": entities, "timestamps": timestamps,
"nextPageStart": nextPageStart}
defer.returnValue(args)
def _renderChat(self, request):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
orgId = args['orgId']
landing = not self._ajax
if script and landing:
t.render(request, "chat.mako", **args)
if appchange and script:
t.renderScriptBlock(request, "chat.mako", "layout",
landing, "#mainbar", "set", **args)
chatId = utils.getRequestArg(request, 'id')
start = utils.getRequestArg(request, 'start') or ''
start = utils.decodeKey(start)
count = 25
if not chatId:
raise errors.MissingParams(["Chat Id"])
chatParticipants = yield db.get_slice(chatId, "chatParticipants")
chatParticipants = utils.columnsToDict(chatParticipants).keys()
if myId not in chatParticipants:
raise errors.ChatAccessDenied(chatId)
entityIds = set()
chatLogs = []
nextPageStart = ''
cols = yield db.get_slice(chatId, "chatLogs", start=start, count=count + 1)
for col in cols:
timestamp = col.column.timestamp / 1e6
entityId, comment = col.column.value.split(':', 1)
entityIds.add(entityId)
chatLogs.append((entityId, comment, timestamp))
if len(cols) == count + 1:
nextPageStart = utils.encodeKey(cols[-1].column.name)
chatLogs = chatLogs[:count]
entities = base.EntitySet(chatParticipants)
yield entities.fetchData()
entities.update(args['me'])
title = "Chat with " + ",".join([entities[x].basic['name'] \
for x in chatParticipants if x != myId])
args.update({"chatLogs": chatLogs, "chatId": chatId,
"entities": entities, "nextPageStart": nextPageStart,
"menuId": "chats", "view": "log"})
if script:
if not start:
onload = """
$$.menu.selectItem('chats');
"""
t.renderScriptBlock(request, 'chat.mako', "chat_title", landing,
"#title", "set", True,
handlers={"onload": onload}, chatTitle=title)
t.renderScriptBlock(request, "chat.mako", "chat",
landing, ".center-contents", "set", **args)
else:
t.renderScriptBlock(request, "chat.mako", "chat",
landing, "#next-page-loader", "replace", **args)
else:
t.render(request, "chat.mako", **args)
def _renderChatArchives(self, request):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
orgId = args['orgId']
landing = not self._ajax
start = utils.getRequestArg(request, 'start') or ''
start = utils.decodeKey(start)
count = constants.CHATS_PER_PAGE
if script and landing:
t.render(request, "chat.mako", **args)
if appchange and script:
t.renderScriptBlock(request, "chat.mako", "layout",
landing, "#mainbar", "set", **args)
chats = {}
chatParticipants = {}
prevPageStart = ''
nextPageStart = ''
cols = yield db.get_slice(myId, "chatArchiveList", start=start,
count=count + 1, reverse=True)
chatIds = [col.column.value for col in cols]
if len(cols) == count + 1:
chatIds = chatIds[:count]
nextPageStart = utils.encodeKey(cols[-1].column.name)
cols = yield db.get_slice(myId, "chatArchiveList", start=start,
count=count + 1)
if len(cols) > 1 and start:
prevPageStart = utils.encodeKey(cols[-1].column.name)
if chatIds:
cols = yield db.multiget_slice(chatIds, "chatLogs", reverse=False)
chats = OrderedDict()
for chatId in cols:
chats[chatId] = []
for col in cols[chatId]:
entityId, comment = col.column.value.split(':', 1)
chats[chatId] = (entityId, comment, col.column.timestamp / 1e6)
participants = yield db.multiget_slice(chatIds, "chatParticipants")
participants = utils.multiColumnsToDict(participants)
entityIds = set([])
for chatId in participants:
entityIds.update(participants[chatId])
entities = base.EntitySet(entityIds)
yield entities.fetchData()
entities.update(args['me'])
args.update({'chatParticipants': participants,
'entities': entities,
'chats': chats,
'chatIds': chatIds,
'nextPageStart': nextPageStart,
'prevPageStart': prevPageStart,
'view': 'list',
'menuId': 'chats'})
if script:
onload = """
$$.menu.selectItem('chats');
"""
t.renderScriptBlock(request, "chat.mako", "chatList",
landing, '.center-contents', "set", True,
handlers={"onload": onload}, **args)
else:
t.render(request, "chat.mako", **args)
def _listGroups(self, request):
appchange, script, args, myId = yield self._getBasicArgs(request)
landing = not self._ajax
me = args['me']
viewType = utils.getRequestArg(request, 'type') or 'myGroups'
start = utils.getRequestArg(request, 'start') or ''
start = utils.decodeKey(start)
viewTypes = ['myGroups', 'allGroups', 'adminGroups', 'pendingRequests', 'invitations']
viewType = 'myGroups' if viewType not in viewTypes else viewType
args["menuId"] = "groups"
args['viewType'] = viewType
cols = yield db.get_slice(myId, "entities", super_column='adminOfGroups')
managedGroupIds = [col.column.name for col in cols]
##TODO: can we use getLatestCounts instead of fetching pendingConnections?
cols = yield db.multiget_slice(managedGroupIds, "pendingConnections", count=1)
cols = utils.multiColumnsToDict(cols)
showPendingRequestsTab = sum([len(cols[groupId]) for groupId in cols]) > 0
args["showPendingRequestsTab"] = showPendingRequestsTab
if viewType == 'pendingRequests' and not showPendingRequestsTab:
viewType = 'myGroups'
args["viewType"] = viewType
cols = yield db.get_slice(myId, "pendingConnections", start="GI:", count=1)
args["showInvitationsTab"] = bool(len([col for col in cols if col.column.name.startswith('GI:')]))
if viewType == 'invitations' and not args["showInvitationsTab"]:
viewType = 'myGroups'
args['viewType'] = viewType
counts = yield utils.getLatestCounts(request, False)
groupRequestCount = args["groupRequestCount"] = counts["groups"]
if script and landing:
t.render(request, "groups.mako", **args)
if script and appchange:
t.renderScriptBlock(request, "groups.mako", "layout",
landing, "#mainbar", "set", **args)
if viewType not in ['pendingRequests', 'invitations']:
if viewType == 'myGroups':
data = yield Group.getGroups(me, me, start)
elif viewType == 'allGroups':
data = yield Group.getGroups(me, args['org'], start)
else:
data = yield Group.getManagedGroups(me, start)
args.update(data)
elif viewType == 'pendingRequests':
data = yield Group.getGroupRequests(me, start)
args.update(data)
args['tab'] = 'pending'
elif viewType == 'invitations':
data = yield Group.getAllInvitations(me, start)
args.update(data)
if script:
t.renderScriptBlock(request, "groups.mako", "titlebar",
landing, "#titlebar", "set", **args)
t.renderScriptBlock(request, "groups.mako", "viewOptions",
landing, "#groups-view", "set", args=[viewType],
showPendingRequestsTab=showPendingRequestsTab,
showInvitationsTab=args['showInvitationsTab'],
groupRequestCount=groupRequestCount)
if viewType == "pendingRequests":
t.renderScriptBlock(request, "groups.mako", "allPendingRequests",
landing, "#groups-wrapper", "set", **args)
else:
t.renderScriptBlock(request, "groups.mako", "listGroups",
landing, "#groups-wrapper", "set", **args)
t.renderScriptBlock(request, "groups.mako", "paging",
landing, "#groups-paging", "set", **args)
if not script:
t.render(request, "groups.mako", **args)
def _listConversations(self, request):
"""Renders a time sorted list of coversations in a particular view.
Keyword Arguments:
filerType: The folder view which is to rendered. One of ['unread', 'all',
'archive', 'trash'].
start: The base64 encoded timeUUID of the starting conversation id of
the page that needs to be rendered.
"""
(appchange, script, args, myId) = yield self._getBasicArgs(request)
landing = not self._ajax
filterType = utils.getRequestArg(request, 'type')
folder = self._folders[filterType] if filterType in self._folders else\
self._folders['inbox']
start = utils.getRequestArg(request, "start") or ''
start = utils.decodeKey(start)
if script and landing:
t.render(request, "message.mako", **args)
if appchange and script:
t.renderScriptBlock(request, "message.mako", "layout",
landing, "#mainbar", "set", **args)
unread = []
convs = []
users = set()
count = 10
fetchCount = count + 1
nextPageStart = ''
prevPageStart = ''
cols = yield db.get_slice(myId, folder, reverse=True, start=start, count=fetchCount)
for col in cols:
x, convId = col.column.value.split(':')
convs.append(convId)
if x == 'u':
unread.append(convId)
if len(cols) == fetchCount:
nextPageStart = utils.encodeKey(col.column.name)
convs = convs[:count]
###XXX: try to avoid extra fetch
cols = yield db.get_slice(myId, folder, count=fetchCount, start=start)
if cols and len(cols) > 1 and start:
prevPageStart = utils.encodeKey(cols[-1].column.name)
cols = yield db.multiget_slice(convs, 'mConversations')
conversations = utils.multiSuperColumnsToDict(cols)
m = {}
for convId in conversations:
if not conversations[convId]:
continue
participants = conversations[convId]['participants'].keys()
users.update(participants)
conversations[convId]['people'] = participants
conversations[convId]['read'] = str(int(convId not in unread))
messageCount = yield db.get_count(convId, "mConvMessages")
conversations[convId]['count'] = messageCount
m[convId] = conversations[convId]
users = base.EntitySet(users)
yield users.fetchData()
args.update({"view": "messages"})
args.update({"messages": m})
args.update({"people": users})
args.update({"mids": convs})
args.update({"menuId": "messages"})
args.update({"filterType": filterType or "all"})
args['nextPageStart'] = nextPageStart
args['prevPageStart'] = prevPageStart
if script:
onload = """
$$.menu.selectItem('%s');
$('#mainbar .contents').removeClass("has-right");
""" % args["menuId"]
t.renderScriptBlock(request, "message.mako", "render_conversations", landing,
".center-contents", "set", True,
handlers={"onload": onload}, **args)
yield utils.render_LatestCounts(request, landing)
else:
t.render(request, "message.mako", **args)
def get(auth, feedId=None, feedItemsId=None, convIds=None,
getFn=None, cleanFn=None, start='', count=10, getReasons=True,
forceCount=False, itemType=None):
"""Fetch data from feed represented by feedId. Returns a dictionary
that has the items from feed, start of the next page and responses and
likes that I know of.
Keyword params:
@auth - An instance of AuthInfo representing the authenticated user
@feedId - Id of the feed from which the data is to be fetched
@feedItemsId - Id of the feed from with feed items must be fetched
@convIds - List of conversation id's to be used as feed
@getFn - Function that must be used to fetch items
@cleanFn - Function that must be used to clean items that don't exist
@start - Id of the item where the fetching must start
@count - Number of items to fetch
@getReasons - Add reason strings to the returned dictionary
@forceCount - Try hard to get atleast count items from the feed
"""
toFetchItems = set() # Items and entities that need to be fetched
toFetchEntities = set() #
toFetchTags = set() #
items = {} # Fetched items, entities and tags
entities = base.EntitySet([])#
tags = {} #
deleted = [] # List of items that were deleted
deleteKeys = [] # List of keys that need to be deleted
responses = {} # Cached data of item responses and likes
likes = {} #
myLikes = {}
myId = auth.username
orgId = auth.organization
feedSource = "feed_%s" % itemType\
if itemType and itemType in plugins\
and plugins[itemType].hasIndex\
else "feed"
feedItemsId = feedItemsId or myId
feedItems_d = [] # List of deferred used to fetch feedItems
# Used for checking ACL
relation = Relation(myId, [])
yield relation.initGroupsList()
# The updates that will be used to build reason strings
convReasonUpdates = {}
# Data that is sent to various plugins and returned by this function
# XXX: myKey is depricated - use myId
data = {"myId": myId, "orgId": orgId, "responses": responses,
"likes": likes, "myLikes": myLikes, "items": items,
"entities": entities, "tags": tags, "myKey": myId,
"relations": relation}
@defer.inlineCallbacks
def fetchFeedItems(ids):
rawFeedItems = yield db.get_slice(feedItemsId, "feedItems", ids) \
if ids else defer.succeed([])
for conv in rawFeedItems:
convId = conv.super_column.name
convUpdates = conv.super_column.columns
responses[convId] = []
likes[convId] = []
latest = None
updatesByType = {}
for update in convUpdates:
parts = update.value.split(':')
updatesByType.setdefault(parts[0], []).append(parts)
if parts[1] != myId: # Ignore my own updates
latest = parts # when displaying latest actors
# Parse all notification to make sure we fetch any required
# items, entities. and cache generic stuff that we display
for tipe in updatesByType.keys():
updates = updatesByType[tipe]
if tipe in _feedUpdatePlugins:
(i,e) = _feedUpdatePlugins[tipe].parse(convId, updates)
toFetchItems.update(i)
toFetchEntities.update(e)
if tipe == "L":
for update in updates:
if update[2] == convId:
likes[convId].append(update[1])
# XXX: Adding this item may break the sorting
# of responses on this conversation
# Bug #493
#else:
# responses[convId].append(update[2])
elif tipe in ["C", "Q"]:
for update in updates:
responses[convId].append(update[2])
# Store any information that can be used to render
# the reason strings when we have the required data
if getReasons and latest:
convReasonUpdates[convId] = updatesByType[latest[0]]
# Fetch the feed if required and at the same time make sure
# we delete unwanted items from the feed (cleanup).
# NOTE: We assume that there will be very few deletes.
nextPageStart = None
if not convIds:
feedId = feedId or myId
allFetchedConvIds = set() # Complete set of convIds fetched
itemsFromFeed = {} # All the key-values retrieved from feed
keysFromFeed = [] # Sorted list of keys (used for paging)
convIds = [] # List of convIds that will be displayed
fetchStart = utils.decodeKey(start)
fetchCount = count + 1
while len(convIds) < count:
fetchedConvIds = []
# Use the getFn function if given.
# NOTE: Part of this code is duplicated just below this.
if getFn:
results = yield getFn(start=fetchStart, count=fetchCount)
for name, value in results.items():
keysFromFeed.append(name)
if value not in allFetchedConvIds:
fetchedConvIds.append(value)
allFetchedConvIds.add(value)
itemsFromFeed[name] = value
else:
deleteKeys.append(name)
# Fetch user's feed when getFn isn't given.
# NOTE: Part of this code is from above
else:
results = yield db.get_slice(feedId, feedSource,
count=fetchCount,
start=fetchStart, reverse=True)
for col in results:
value = col.column.value
keysFromFeed.append(col.column.name)
if value not in allFetchedConvIds:
fetchedConvIds.append(value)
allFetchedConvIds.add(value)
itemsFromFeed[col.column.name] = value
else:
deleteKeys.append(col.column.name)
# Initiate fetching feed items for all the conversation Ids.
# Meanwhile we check if the authenticated user has access to
# all the fetched conversation ids.
# NOTE: Conversations would rarely be filtered out. So, we
# just go ahead with fetching data for all convs.
feedItems_d.append(fetchFeedItems(fetchedConvIds))
(filteredConvIds, deletedIds) = yield utils.fetchAndFilterConvs\
(fetchedConvIds, relation, items, myId, orgId)
convIds.extend(filteredConvIds)
deleted.extend(deletedIds)
# Unless we are forced to fetch count number of items, we only
# iterate till we fetch atleast half of them
if (not forceCount and len(convIds) > (count/2)) or\
len(results) < fetchCount:
break
# If we need more items, we start fetching from where we
# left in the previous iteration.
fetchStart = keysFromFeed[-1]
# If DB fetch got as many items as I requested
# there may be additional items present in the feed
# So, we cut one item from what we return and start the
# next page from there.
if len(results) == fetchCount:
lastConvId = convIds[-1]
for key in reversed(keysFromFeed):
if key in itemsFromFeed and itemsFromFeed[key] == lastConvId:
nextPageStart = utils.encodeKey(key)
convIds = convIds[:-1]
else:
(convIds, deletedIds) = yield utils.fetchAndFilterConvs(convIds,
relation, items, myId, orgId)
# NOTE: Unlike the above case where we fetch convIds from
# database (where we set the nextPageStart to a key),
# here we set nextPageStart to the convId.
if len(convIds) > count:
nextPageStart = utils.encodeKey(convIds[count])
convIds = convIds[0:count]
# Since convIds were directly passed to us, we would also
# return the list of convIds deleted back to the caller.
if deletedIds:
data["deleted"] = deletedIds
# We don't have any conversations to display!
if not convIds:
defer.returnValue({"conversations": []})
# Delete any convs that were deleted from the feeds and
# any duplicates that were marked for deletion
cleanup_d = []
if deleted:
for key, value in itemsFromFeed.items():
if value in deleted:
deleteKeys.append(key)
if cleanFn:
d1 = cleanFn(list(deleteKeys))
else:
d1 = db.batch_remove({feedSource: [feedId]}, names=deleteKeys)
d2 = db.batch_remove({'feedItems': [feedId]}, names=list(deleted))
cleanup_d = [d1, d2]
# We now have a filtered list of conversations that can be displayed
# Let's wait till all the feed items have been fetched and processed
yield defer.DeferredList(feedItems_d)
# Fetch the remaining items (comments on the actual conversations)
items_d = db.multiget_slice(toFetchItems, "items", ["meta" ,"attachments"])
# Fetch tags on all the conversations that will be displayed
for convId in convIds:
conv = items[convId]
toFetchEntities.add(conv["meta"]["owner"])
if "target" in conv["meta"]:
toFetchEntities.update(conv["meta"]["target"].split(','))
toFetchTags.update(conv.get("tags",{}).keys())
tags_d = db.get_slice(orgId, "orgTags", toFetchTags) \
if toFetchTags else defer.succeed([])
# Fetch the list of my likes.
# XXX: Latency can be pretty high here becuase many nodes will have to
# be contacted for the information. Alternative could be to cache
# all likes by a user somewhere.
myLikes_d = db.multiget(toFetchItems.union(convIds), "itemLikes", myId)
# Fetch extra data that is required to render special items
# We already fetched the conversation items, plugins merely
# add more data to the already fetched items
for convId in convIds[:]:
itemType = items[convId]["meta"]["type"]
if itemType in plugins:
try:
entityIds = yield plugins[itemType].fetchData(data, convId)
toFetchEntities.update(entityIds)
except Exception, e:
log.err(e)
convIds.remove(convId)
def _getKeywordMatches(self, request, keyword, start='', count=10):
args = {}
authinfo = request.getSession(IAuthInfo)
myId = authinfo.username
orgId = authinfo.organization
items = {}
itemIds = []
itemIdKeyMap = {}
allFetchedItems = set()
deleted = set()
fetchStart = utils.decodeKey(start)
fetchCount = count + 2
while len(itemIds) < count:
fetchedItemIds = []
toFetchItems = set()
results = yield db.get_slice(orgId + ":" + keyword, "keywordItems",
count=fetchCount, start=fetchStart,
reverse=True)
for col in results:
fetchStart = col.column.name
itemAndParentIds = col.column.value.split(':')
itemIdKeyMap[itemAndParentIds[0]] = fetchStart
fetchedItemIds.append(itemAndParentIds[0])
for itemId in itemAndParentIds:
if itemId not in allFetchedItems:
toFetchItems.add(itemId)
allFetchedItems.add(itemId)
if toFetchItems:
fetchedItems = yield db.multiget_slice(toFetchItems, "items",
["meta", "attachments"])
fetchedItems = utils.multiSuperColumnsToDict(fetchedItems)
items.update(fetchedItems)
for itemId in fetchedItemIds:
item = items[itemId]
if not 'meta' in item:
continue
state = item['meta'].get('state', 'published')
if state == 'deleted':
deleted.add(itemIdKeyMap[itemId])
elif utils.checkAcl(myId, orgId, True, None, item['meta']):
itemIds.append(itemId)
if len(results) < fetchCount:
break
if len(itemIds) > count:
nextPageStart = utils.encodeKey(itemIdKeyMap[itemIds[-1]])
itemIds = itemIds[:-1]
else:
nextPageStart = None
dd = db.batch_remove({'keywordItems': [orgId + ':' + keyword]},
names=deleted) if deleted else defer.succeed([])
args.update({'items': items, 'myId': myId})
toFetchEntities = set()
extraDataDeferreds = []
for itemId in itemIds:
item = items[itemId]
itemMeta = item['meta']
toFetchEntities.add(itemMeta['owner'])
if 'target' in itemMeta:
toFetchEntities.update(itemMeta['target'].split(','))
if 'parent' in itemMeta:
parentId = itemMeta['parent']
if parentId in items:
toFetchEntities.add(items[parentId]['meta']['owner'])
itemType = itemMeta.get('type', 'status')
if itemType in plugins:
d = plugins[itemType].fetchData(args, itemId)
extraDataDeferreds.append(d)
result = yield defer.DeferredList(extraDataDeferreds)
for success, ret in result:
if success:
toFetchEntities.update(ret)
fetchedEntities = {}
if toFetchEntities:
fetchedEntities = base.EntitySet(toFetchEntities)
yield fetchedEntities.fetchData()
yield dd
args.update({'entities': fetchedEntities,
'matches': itemIds, 'nextPageStart': nextPageStart})
defer.returnValue(args)
def _render(self, request):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
# We are setting an empty value to 'cu' here just to make sure that
# any errors when looking validating the entity should not leave us
# in a bad state.
request.addCookie('cu', '', path="/ajax/profile")
if request.args.get("id", None):
userId, ign = yield utils.getValidEntityId(request, "id", "user")
else:
userId = myId
# XXX: We should use getValidEntityId to fetch the entire user
# info instead of another call to the database.
request.addCookie('cu', userId, path="/ajax/profile")
user = base.Entity(userId)
yield user.fetchData([])
if user._data:
args['user'] = user
detail = utils.getRequestArg(request, "dt") or "activity"
args["detail"] = detail
args["userId"] = userId
args["menuId"] = "people"
args["entities"] = base.EntitySet({myId:args['me'], userId:user})
# When scripts are enabled, updates are sent to the page as
# and when we get the required data from the database.
# When we are the landing page, we also render the page header
# and all updates are wrapped in <script> blocks.
landing = not self._ajax
# User entered the URL directly
# Render the header. Other things will follow.
if script and landing:
t.render(request, "profile.mako", **args)
# Start with displaying the template and navigation menu
if script and appchange:
t.renderScriptBlock(request, "profile.mako", "layout",
landing, "#mainbar", "set", **args)
# Prefetch some data about how I am related to the user.
# This is required in order to reliably filter our profile details
# that I don't have access to.
relation = Relation(myId, [userId])
args["relations"] = relation
yield defer.DeferredList([relation.initSubscriptionsList(),
relation.initGroupsList()])
# Reload all user-depended blocks if the currently displayed user is
# not the same as the user for which new data is being requested.
if script:
t.renderScriptBlock(request, "profile.mako", "summary",
landing, "#profile-summary", "set", **args)
t.renderScriptBlock(request, "profile.mako", "user_subactions",
landing, "#user-subactions", "set", **args)
fetchedEntities = set()
start = utils.getRequestArg(request, "start") or ''
fromFetchMore = ((not landing) and (not appchange) and start)
if detail == "activity":
userItems = yield self._getUserItems(request, userId, start=start)
args.update(userItems)
elif detail == 'files':
end = utils.getRequestArg(request, "end") or ''
end = utils.decodeKey(end)
start = utils.decodeKey(start)
userFiles = yield files.userFiles(myId, userId, args['orgId'], start, end, fromFeed=False)
args['userfiles'] = userFiles
args['fromProfile'] = True
if script:
t.renderScriptBlock(request, "profile.mako", "tabs", landing,
"#profile-tabs", "set", **args)
handlers = {} if detail != "activity" \
else {"onload": "(function(obj){$$.convs.load(obj);})(this);"}
if fromFetchMore and detail == "activity":
t.renderScriptBlock(request, "profile.mako", "content", landing,
"#next-load-wrapper", "replace", True,
handlers=handlers, **args)
else:
t.renderScriptBlock(request, "profile.mako", "content", landing,
"#profile-content", "set", True,
handlers=handlers, **args)
# List the user's subscriptions
cols = yield db.get_slice(userId, "subscriptions", count=11)
subscriptions = set(utils.columnsToDict(cols).keys())
args["subscriptions"] = subscriptions
# List the user's followers
cols = yield db.get_slice(userId, "followers", count=11)
followers = set(utils.columnsToDict(cols).keys())
args["followers"] = followers
# Fetch item data (name and avatar) for subscriptions, followers,
# user groups and common items.
entitiesToFetch = followers.union(subscriptions)\
.difference(fetchedEntities)
# List the user's groups (and look for groups common with me)
cols = yield db.multiget_slice([myId, userId], "entityGroupsMap")
myGroups = set([x.column.name.split(':', 1)[1] for x in cols[myId]])
userGroups = set([x.column.name.split(':', 1)[1] for x in cols[userId]])
commonGroups = myGroups.intersection(userGroups)
if len(userGroups) > 10:
userGroups = sample(userGroups, 10)
args["userGroups"] = userGroups
args["commonGroups"] = commonGroups
groupsToFetch = commonGroups.union(userGroups)
entitiesToFetch = entitiesToFetch.union(groupsToFetch)
entities = base.EntitySet(entitiesToFetch)
yield entities.fetchData()
for entityId, entity in entities.items():
if not entity._data:
del entities[entityId]
args["entities"].update(entities)
if script:
t.renderScriptBlock(request, "profile.mako", "user_subscriptions",
landing, "#user-subscriptions", "set", **args)
t.renderScriptBlock(request, "profile.mako", "user_followers",
landing, "#user-followers", "set", **args)
t.renderScriptBlock(request, "profile.mako", "user_me",
landing, "#user-me", "set", **args)
t.renderScriptBlock(request, "profile.mako", "user_groups",
landing, "#user-groups", "set", **args)
if script and landing:
request.write("</body></html>")
if not script:
t.render(request, "profile.mako", **args)
def _getUserItems(self, request, userId, start='', count=10):
authinfo = request.getSession(IAuthInfo)
myId = authinfo.username
myOrgId = authinfo.organization
toFetchItems = set()
toFetchEntities = set()
toFetchTags = set()
toFetchResponses = set()
toFetchCount = count + 1
toFetchStart = utils.decodeKey(start) if start else ''
fetchedUserItem = []
responses = {}
convs = []
userItemsRaw = []
userItems = []
reasonStr = {}
timestamps = {}
items = {}
nextPageStart = None
args = {'myId': myId}
relation = Relation(myId, [])
yield relation.initGroupsList()
toFetchEntities.add(userId)
while len(convs) < toFetchCount:
cols = yield db.get_slice(userId, "userItems", start=toFetchStart,
reverse=True, count=toFetchCount)
tmpIds = []
for col in cols:
convId = col.column.value.split(":")[2]
if convId not in tmpIds and convId not in convs:
tmpIds.append(convId)
(filteredConvs, deletedConvs) = yield utils.fetchAndFilterConvs\
(tmpIds, relation, items, myId, myOrgId)
for col in cols[0:count]:
convId = col.column.value.split(":")[2]
if len(convs) == count or len(fetchedUserItem) == count*2:
nextPageStart = col.column.name
break
if convId not in filteredConvs and convId not in convs:
continue
fetchedUserItem.append(col)
if convId not in convs:
convs.append(convId)
if len(cols) < toFetchCount or nextPageStart:
break
if cols:
toFetchStart = cols[-1].column.name
if nextPageStart:
nextPageStart = utils.encodeKey(nextPageStart)
for col in fetchedUserItem:
value = tuple(col.column.value.split(":"))
timestamps[value] = col.column.timestamp/1e6
rtype, itemId, convId, convType, convOwnerId, commentSnippet = value
commentSnippet = """<span class="snippet">"%s"</span>""" %(_(commentSnippet))
toFetchEntities.add(convOwnerId)
if rtype == 'I':
toFetchItems.add(convId)
toFetchResponses.add(convId)
userItems.append(value)
elif rtype == "L" and itemId == convId and convOwnerId != userId:
reasonStr[value] = _("liked %s's %s")
userItems.append(value)
elif rtype == "L" and convOwnerId != userId:
r = "answer" if convType == 'question' else 'comment'
reasonStr[value] = _("liked") + " %s " %(commentSnippet) + _("%s "%r) + _("on %s's %s")
userItems.append(value)
elif rtype in ["C", 'Q'] and convOwnerId != userId:
reasonStr[value] = "%s"%(commentSnippet) + _(" on %s's %s")
userItems.append(value)
itemResponses = yield db.multiget_slice(toFetchResponses, "itemResponses",
count=2, reverse=True)
for convId, comments in itemResponses.items():
responses[convId] = []
for comment in comments:
userId_, itemKey = comment.column.value.split(':')
if itemKey not in toFetchItems:
responses[convId].insert(0,itemKey)
toFetchItems.add(itemKey)
toFetchEntities.add(userId_)
items = yield db.multiget_slice(toFetchItems, "items", ["meta", "tags", "attachments"])
items = utils.multiSuperColumnsToDict(items)
args["items"] = items
extraDataDeferreds = []
for convId in convs:
if convId not in items:
continue
meta = items[convId]["meta"]
itemType = meta["type"]
toFetchEntities.add(meta["owner"])
if "target" in meta:
toFetchEntities.update(meta["target"].split(','))
toFetchTags.update(items[convId].get("tags", {}).keys())
if itemType in plugins:
d = plugins[itemType].fetchData(args, convId)
extraDataDeferreds.append(d)
result = yield defer.DeferredList(extraDataDeferreds)
for success, ret in result:
if success:
toFetchEntities.update(ret)
entities = base.EntitySet(toFetchEntities)
yield entities.fetchData()
tags = {}
if toFetchTags:
userOrgId = entities[userId].basic["org"]
fetchedTags = yield db.get_slice(userOrgId, "orgTags", toFetchTags)
tags = utils.supercolumnsToDict(fetchedTags)
fetchedLikes = yield db.multiget(toFetchItems, "itemLikes", myId)
myLikes = utils.multiColumnsToDict(fetchedLikes)
data = {"entities": entities, "reasonStr": reasonStr,
"tags": tags, "myLikes": myLikes, "userItems": userItems,
"responses": responses, "nextPageStart": nextPageStart,
"timestamps": timestamps }
del args['myId']
args.update(data)
defer.returnValue(args)
def updateData():
yield db.truncate('user_files')
try:
yield db.get('asdf', 'entityFeed_files', uuid.uuid1().bytes)
except ttypes.InvalidRequestException as exception:
log.msg(exception)
raise Exception('entityFeed_files CF missing, create the CF')
except ttypes.NotFoundException:
pass
entities = {}
items = {}
rows = yield db.get_range_slice('items', count=10000, reverse=True)
for row in rows:
itemId = row.key
item = utils.supercolumnsToDict(row.columns)
items[itemId]=item
for itemId in items:
item = items[itemId]
log.msg(itemId)
if 'meta' not in item:
continue
# Add org to all items
try:
owner = item['meta']['owner']
col = yield db.get(owner, "entities", 'org', 'basic')
ownerOrgId = col.column.value
yield db.insert(itemId, 'items', ownerOrgId, 'org', 'meta')
except Exception as e:
if item['meta'].get('type', '') == 'feedback':
yield db.insert(itemId, 'items', owner, 'org', 'meta')
# Fix ACLs
if 'parent' not in item['meta']:
acl = item['meta']['acl']
convOwner = item['meta']['owner']
convId = itemId
if acl == 'company':
col = yield db.get(convOwner, "entities", "org", "basic")
ownerOrgId = col.column.value
acl = pickle.dumps({"accept":{"orgs":[ownerOrgId]}})
yield db.insert(convId, 'items', acl, 'acl', 'meta')
else:
try:
acl = pickle.loads(acl)
if 'accept' in acl and 'friends' in acl['accept'] and isinstance(acl['accept']['friends'], bool):
del acl['accept']['friends']
acl = pickle.dumps(acl)
yield db.insert(convId, 'items', acl, 'acl', 'meta')
except :
log.msg('cannot unpack acl', acl)
# Migrate files
# truncate user_files
# update user_files and entityFeed_files
if 'owner' in item['meta'] and 'attachments' in item:
ownerId = item['meta']['owner']
if ownerId not in entities:
cols = yield db.get_slice(ownerId, 'entities', ['basic'])
entities.update({ownerId: utils.supercolumnsToDict(cols)})
for attachmentId in item['attachments']:
orgId = entities[ownerId]['basic']['org']
timeuuid, name = item['attachments'][attachmentId].split(':')[:2]
timeuuid = utils.decodeKey(timeuuid)
val = '%s:%s:%s:%s' % (attachmentId, name, itemId, ownerId)
yield db.insert(ownerId, "user_files", val, timeuuid)
if 'parent' not in item['meta'] and item['meta'].get('acl', ''):
_entities = yield utils.expandAcl(ownerId, orgId, item['meta']['acl'],
itemId, ownerId, True)
for entityId in _entities:
yield db.insert(entityId, "entityFeed_files", val, timeuuid)
# Migrate items
# Meta fields in "link", "event" and "poll"
if item['meta'].get('type', None) in ['link', 'poll', 'event']:
itemMeta = item['meta']
itemType = itemMeta['type']
updated = {}
if itemType == "link":
if 'url' in itemMeta:
updated['link_url'] = itemMeta['url']
if 'title' in itemMeta:
updated['link_title'] = itemMeta['title']
if 'summary' in itemMeta:
updated['link_summary'] = itemMeta['summary']
if 'imgSrc' in itemMeta:
updated['link_imgSrc'] = itemMeta['imgSrc']
if 'embedType' in itemMeta:
updated['link_embedType'] = itemMeta['embedType']
if 'embedSrc' in itemMeta:
updated['link_embedSrc'] = itemMeta['embedSrc']
if 'embedHeight' in itemMeta:
updated['link_embedHeight'] = itemMeta['embedHeight']
if 'embedWidth' in itemMeta:
updated['link_embedWidth'] = itemMeta['embedWidth']
elif itemType == 'poll':
if 'question' in itemMeta:
updated['comment'] = itemMeta['question']
else:
print 'Found an event:', itemId
if updated:
yield db.batch_insert(itemId, 'items', {'meta': updated})
#
# Create poll indexes for feed and userItems
#
rows = yield db.get_range_slice('entities', count=10000, reverse=True)
mutations = {}
for row in rows:
entityId = row.key
entity = utils.supercolumnsToDict(row.columns)
if entity['basic']['type'] != 'user':
continue
d1 = db.get_slice(entityId, 'feed', count=10000)
d2 = db.get_slice(entityId, 'userItems', count=10000)
results = yield d1
for col in results:
value = col.column.value
if value in items:
if items.get(value, {}).get('meta', {}).get('type', '') == 'poll':
mutations.setdefault(entityId, {}).setdefault('feed_poll', {}).update({col.column.name: value})
results = yield d2
for col in results:
value = col.column.value
responseType, itemId, convId, convType, others = value.split(':', 4)
if convType == 'poll':
mutations.setdefault(entityId, {}).setdefault('userItems_poll', {}).update({col.column.name: value})
yield db.batch_mutate(mutations)
#Group type changed from public-private to open-closed.
rows = yield db.get_range_slice('entityGroupsMap', count=1000)
groupIds = set()
for row in rows:
for col in row.columns:
name_, groupId = col.column.name.split(':')
groupIds.add(groupId)
cols = yield db.multiget_slice(groupIds, "entities")
groups = utils.multiSuperColumnsToDict(cols)
for groupId in groups:
access = groups[groupId]['basic']['access'].lower()
if access == 'public':
yield db.insert(groupId, 'entities', 'open', 'access', 'basic')
elif access.lower() == 'private':
yield db.insert(groupId, 'entities', 'closed', 'access', 'basic')
#Fix entityGroupsMap
rows = yield db.get_range_slice('entityGroupsMap', count=1000)
for row in rows:
entityId = row.key
for col in row.columns:
name_, groupId = col.column.name.split(':')
if col.column.name != '%s:%s'%(groups[groupId]['basic']['name'].lower(), groupId):
yield db.remove(entityId, 'entityGroupsMap', col.column.name)
yield db.insert(entityId, 'entityGroupsMap', '', '%s:%s' %(groups[groupId]['basic']['name'].lower(), groupId))
def _listTags(self, request):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
landing = not self._ajax
myOrgId = args["orgId"]
start = utils.getRequestArg(request, 'start') or ''
nextPageStart = ''
prevPageStart = ''
count = constants.PEOPLE_PER_PAGE
toFetchCount = count + 1
start = utils.decodeKey(start)
args["menuId"] = "tags"
if script and landing:
t.render(request, "tags.mako", **args)
if script and appchange:
t.renderScriptBlock(request, "tags.mako", "layout",
landing, "#mainbar", "set", **args)
if script:
t.renderScriptBlock(request, "tags.mako", "header",
landing, "#tags-header", "set", **args)
tagsByName = yield db.get_slice(myOrgId, "orgTagsByName",
start=start, count=toFetchCount)
tagIds = [x.column.value for x in tagsByName]
if len(tagsByName) > count:
nextPageStart = utils.encodeKey(tagsByName[-1].column.name)
tagIds = tagIds[:-1]
if start:
prevCols = yield db.get_slice(myOrgId, "orgTagsByName",
start=start, reverse=True,
count=toFetchCount)
if len(prevCols) > 1:
prevPageStart = utils.encodeKey(prevCols[-1].column.name)
tags = {}
if tagIds:
tags = yield db.get_slice(myOrgId, "orgTags", tagIds)
tags = utils.supercolumnsToDict(tags)
# TODO: We need an index of all tags that the user is following
# Probably convert the 'subscriptions' column family to 'Super'
# and have people and tags in the same column family.
tagsFollowing = []
if tagIds:
cols = yield db.multiget(tagIds, "tagFollowers", myId)
tagsFollowing = [x for x in cols.keys() if cols[x]]
args['tags'] = tags
args['tagIds'] = tagIds
args['tagsFollowing'] = tagsFollowing
args['nextPageStart'] = nextPageStart
args['prevPageStart'] = prevPageStart
if script:
if appchange:
t.renderScriptBlock(request, "tags.mako", "tagsListLayout",
landing, "#content", "set", **args)
else:
t.renderScriptBlock(request, "tags.mako", "listTags",
landing, "#tags-wrapper", "set", **args)
t.renderScriptBlock(request, "tags.mako", "paging",
landing, "#tags-paging", "set", **args)
if not script:
t.render(request, "tags.mako", **args)
def renderItem(self, request, toFeed=False):
(appchange, script, args, myId) = yield self._getBasicArgs(request)
landing = not self._ajax
myOrgId = args["orgId"]
convId, conv = yield utils.getValidItemId(request, "id", columns=['tags'])
itemType = conv["meta"].get("type", None)
if 'parent' in conv['meta']:
raise errors.InvalidItem('conversation', convId)
start = utils.getRequestArg(request, "start") or ''
start = utils.decodeKey(start)
args['convId'] = convId
args['isItemView'] = True
args['items'] = {convId: conv}
meta = conv["meta"]
owner = meta["owner"]
relation = Relation(myId, [])
yield defer.DeferredList([relation.initGroupsList(),
relation.initSubscriptionsList()])
args["relations"] = relation
if script and landing:
t.render(request, "item.mako", **args)
if script and appchange:
t.renderScriptBlock(request, "item.mako", "layout",
landing, "#mainbar", "set", **args)
args["entities"] = {}
toFetchEntities = set()
toFetchTags = set(conv.get("tags", {}).keys())
plugin = plugins[itemType] if itemType in plugins else None
if plugin:
entityIds = yield plugin.fetchData(args)
toFetchEntities.update(entityIds)
toFetchEntities.add(conv['meta']['owner'])
if "target" in conv["meta"]:
toFetchEntities.update(conv['meta']['target'].split(','))
if conv['meta']['owner'] not in toFetchEntities:
toFetchEntities.add(conv['meta']['owner'])
entities = base.EntitySet(toFetchEntities)
yield entities.fetchData()
args["entities"] = entities
renderers = []
if script:
t.renderScriptBlock(request, "item.mako", "conv_root",
landing, "#conv-root-%s > .conv-summary" % (convId),
"set", **args)
convOwner = args["items"][convId]["meta"]["owner"]
args["ownerId"] = convOwner
if script:
if itemType != "feedback":
t.renderScriptBlock(request, "item.mako", "conv_owner",
landing, "#conv-avatar-%s" % convId,
"set", **args)
else:
feedbackType = conv['meta']['subType']
t.renderScriptBlock(request, "item.mako", "feedback_icon",
landing, "#conv-avatar-%s" % convId,
"set", args=[feedbackType])
# A copy of this code for fetching comments is present in _responses
# Most changes here may need to be done there too.
itemResponses = yield db.get_slice(convId, "itemResponses",
start=start, reverse=True,
count=constants.COMMENTS_PER_PAGE + 1)
nextPageStart = itemResponses[-1].column.name\
if len(itemResponses) > constants.COMMENTS_PER_PAGE\
else None
itemResponses = itemResponses[:-1] \
if len(itemResponses) > constants.COMMENTS_PER_PAGE\
else itemResponses
responseKeys = []
for response in itemResponses:
userKey, responseKey = response.column.value.split(":")
responseKeys.append(responseKey)
toFetchEntities.add(userKey)
responseKeys.reverse()
subscriptions = list(relation.subscriptions)
likes = yield db.get_slice(convId, "itemLikes", subscriptions) \
if subscriptions else defer.succeed([])
toFetchEntities.update([x.column.name for x in likes])
entities = base.EntitySet(toFetchEntities)
d1 = entities.fetchData()
d2 = db.multiget_slice(responseKeys, "items", ["meta", "attachments"])
d3 = db.multiget_slice(responseKeys + [convId], "itemLikes", [myId])
d4 = db.get_slice(myOrgId, "orgTags", toFetchTags)\
if toFetchTags else defer.succeed([])
yield d1
fetchedItems = yield d2
myLikes = yield d3
fetchedTags = yield d4
fetchedItems = utils.multiSuperColumnsToDict(fetchedItems)
myLikes = utils.multiColumnsToDict(myLikes)
fetchedTags = utils.supercolumnsToDict(fetchedTags)
# Do some error correction/consistency checking to ensure that the
# response items actually exist. I don't know of any reason why these
# items may not exist.
missingIds = [x for x, y in fetchedItems.items() if not y]
if missingIds:
yield self._cleanupMissingComments(convId, missingIds, itemResponses)
args["items"].update(fetchedItems)
args["entities"].update(entities)
args["myLikes"] = myLikes
args["tags"] = fetchedTags
args["responses"] = {convId: responseKeys}
if nextPageStart:
args["oldest"] = utils.encodeKey(nextPageStart)
if script:
t.renderScriptBlock(request, "item.mako", 'conv_footer',
landing, '#item-footer-%s' % convId,
'set', **args)
t.renderScriptBlock(request, "item.mako", 'conv_tags',
landing, '#conv-tags-wrapper-%s' % convId, 'set',
handlers={"onload": "$('#conv-meta-wrapper-%s').removeClass('no-tags')" % convId} if toFetchTags else None, **args)
t.renderScriptBlock(request, "item.mako", 'conv_comments',
landing, '#conv-comments-wrapper-%s' % convId, 'set', **args)
t.renderScriptBlock(request, "item.mako", 'conv_comment_form',
landing, '#comment-form-wrapper-%s' % convId, 'set',
True, handlers={"onload": "(function(obj){$$.convs.load(obj);})(this);"}, **args)
numLikes = int(conv["meta"].get("likesCount", "0"))
if numLikes:
numLikes = int(conv["meta"].get("likesCount", "0"))
iLike = myId in args["myLikes"].get(convId, [])
t.renderScriptBlock(request, "item.mako", 'conv_likes',
landing, '#conv-likes-wrapper-%s' % convId, 'set',
args=[convId, numLikes, iLike, [x.column.name for x in likes]],
entities=args['entities'])
if plugin and hasattr(plugin, 'renderItemSideBlock'):
plugin.renderItemSideBlock(request, landing, args)
if script and landing:
request.write("</body></html>")
if not script:
t.render(request, "item.mako", **args)
