def notify(userIds, notifyId, value, timeUUID=None, **kwargs):
if not userIds:
return defer.succeed([])
timeUUID = timeUUID or uuid.uuid1().bytes
notifyIdParts = notifyId.split(':')
notifyType = notifyIdParts[1] if not notifyIdParts[0] else notifyIdParts[3]
plugin = _notificationPlugins.get(notifyType, None)
if not plugin:
return defer.succeed([])
deferreds = []
# Delete existing notifications for the same item/activiy
if plugin.notifyOnWeb:
d1 = db.multiget_slice(userIds, "notificationItems",
super_column=notifyId, count=3, reverse=True)
def deleteOlderNotifications(results):
mutations = {}
timestamp = int(time.time() * 1e6)
for key, cols in results.iteritems():
names = [col.column.name for col in cols
if col.column.name != timeUUID]
if names:
colmap = dict([(x, None) for x in names])
deletion = ttypes.Deletion(timestamp, 'notifications',
ttypes.SlicePredicate(column_names=names))
mutations[key] = {'notifications': colmap,
'latest': [deletion]}
if mutations:
return db.batch_mutate(mutations)
else:
return defer.succeed([])
d1.addCallback(deleteOlderNotifications)
deferreds.append(d1)
# Create new notifications
mutations = {}
for userId in userIds:
colmap = {timeUUID: notifyId}
mutations[userId] = {'notifications': colmap,
'latest': {'notifications': colmap},
'notificationItems': {notifyId: {timeUUID: value}}}
deferreds.append(db.batch_mutate(mutations))
for handler in notificationHandlers:
d = handler.notify(userIds, notifyIdParts, value, **kwargs)
deferreds.append(d)
return defer.DeferredList(deferreds)
def edit(me, group, name, access, desc, displayPic):
"""update group meta info.
Only group-admin can edit group meta info.
Keyword params:
@me:
@group:
@name: name of the group.
@access: group access type (open/closed).
@desc: description of the group.
@displayPic: profile pic of the group.
"""
if me.id not in group.admins:
raise errors.PermissionDenied('Only administrator can edit group meta data')
if name:
start = name.lower() + ':'
cols = yield db.get_slice(me.basic['org'], "entityGroupsMap",
start=start, count=1)
for col in cols:
name_, groupId_ = col.column.name.split(':')
if name_ == name.lower() and groupId_ != group.id:
raise errors.InvalidGroupName(name)
meta = {'basic': {}}
if name and name != group.basic['name']:
meta['basic']['name'] = name
if desc and desc != group.basic.get('desc', ''):
meta['basic']['desc'] = desc
if access in ['closed', 'open'] and access != group.basic['access']:
meta['basic']['access'] = access
if displayPic:
avatar = yield saveAvatarItem(group.id, me.basic['org'], displayPic)
meta['basic']['avatar'] = avatar
if name and name != group.basic["name"]:
members = yield db.get_slice(group.id, "groupMembers")
members = utils.columnsToDict(members).keys()
entities = members + [me.basic['org']]
oldColName = "%s:%s" % (group.basic["name"].lower(), group.id)
colname = '%s:%s' % (name.lower(), group.id)
mutations = {}
for entity in entities:
mutations[entity] = {'entityGroupsMap': {colname: '',
oldColName: None}}
#XXX:notify group-members about the change in name
yield db.batch_mutate(mutations)
if meta['basic']:
yield db.batch_insert(group.id, 'entities', meta)
if not desc and group.basic.get('desc', ''):
yield db.remove(group.id, "entities", 'desc', 'basic')
if (not desc and group.basic.get('desc', '')) or meta['basic']:
defer.returnValue(True)
def updateNameIndex(userId, targetIds, newName, oldName):
muts = {}
for targetId in targetIds:
if oldName or newName:
muts[targetId] = {"nameIndex": {}}
if oldName:
for part in oldName.split():
colName = part.lower() + ":" + userId
muts[targetId]["nameIndex"][colName] = None
if newName:
for part in newName.split():
colName = part.lower() + ":" + userId
muts[targetId]["nameIndex"][colName] = ""
if muts:
yield db.batch_mutate(muts)
def updateDisplayNameIndex(userId, targetIds, newName, oldName):
calls = []
muts = {}
for targetId in targetIds:
if oldName or newName:
muts[targetId] = {"displayNameIndex": {}}
if oldName:
colName = oldName.lower() + ":" + userId
muts[targetId]["displayNameIndex"][colName] = None
if newName:
colName = newName.lower() + ":" + userId
muts[targetId]["displayNameIndex"][colName] = ""
if muts:
yield db.batch_mutate(muts)
def deleteOlderNotifications(results):
mutations = {}
timestamp = int(time.time() * 1e6)
for key, cols in results.iteritems():
names = [col.column.name for col in cols
if col.column.name != timeUUID]
if names:
colmap = dict([(x, None) for x in names])
deletion = ttypes.Deletion(timestamp, 'notifications',
ttypes.SlicePredicate(column_names=names))
mutations[key] = {'notifications': colmap,
'latest': [deletion]}
if mutations:
return db.batch_mutate(mutations)
else:
return defer.succeed([])
def updateData():
def _getAllFiles():
SKey = config.get('CloudFiles', 'SecretKey')
AKey = config.get('CloudFiles', 'AccessKey')
bucket = config.get('CloudFiles', 'Bucket')
conn = S3Connection(AKey, SKey)
bucket = conn.get_bucket(bucket)
files = []
for key in bucket.list():
if not key.name.endswith('/'):
files.append(key.name)
return files
files = yield threads.deferToThread(_getAllFiles)
S3fileIds = [x.split("/")[2] for x in files]
log.msg("Fetched %d files" %(len(S3fileIds)))
log.msg("Fetching info about all the files")
d = db.multiget_slice(S3fileIds, "files")
files_map = {}
for f in files:
org, owner, fileId = f.split("/")
files_map[fileId] = (owner, org)
res = yield d
ids = utils.multiSuperColumnsToDict(res)
fileIds = [x for x in ids.keys() if ids[x] and "owner" not in ids[x]["meta"]]
log.msg("Out of %d S3 files, found %d files in \'files\' in old format" %( len(S3fileIds), len(fileIds)))
updated_file_meta = {}
for fileId in fileIds:
owner, org = files_map[fileId]
updated_file_meta[fileId] = {"files":{"meta":{"owner": owner}}}
log.msg(updated_file_meta)
yield db.batch_mutate(updated_file_meta)
def _deliverMessage(self, convId, recipients, timeUUID, owner):
"""To each participant in a conversation, add the conversation
to the list of unread conversations.
CF Changes:
mConversations
mConvFolders
messages
mAllConversations
mDeletedConversations
mArchivedConversations
latest
"""
convFolderMap = {}
userFolderMap = {}
toNotify = {}
toRemove = {'latest': []}
conv = yield db.get_slice(convId, "mConversations", ['meta'])
conv = utils.supercolumnsToDict(conv)
oldTimeUUID = conv['meta']['uuid']
unread = "u:%s" % (convId)
read = "r:%s" % (convId)
cols = yield db.get_slice(convId, 'mConvFolders', recipients)
cols = utils.supercolumnsToDict(cols)
for recipient in recipients:
deliverToInbox = True
# for a new message, mConvFolders will be empty
# so recipient may not necessarily be present in cols
if recipient != owner:
toNotify[recipient] = {'latest': {'messages': {timeUUID: convId}}}
toRemove['latest'].append(recipient)
for folder in cols.get(recipient, []):
cf = self._folders[folder] if folder in self._folders else folder
yield db.remove(recipient, cf, oldTimeUUID)
if cf == 'mDeletedConversations':
#don't add to recipient's inbox if the conv is deleted.
deliverToInbox = False
else:
yield db.remove(convId, 'mConvFolders', folder, recipient)
if deliverToInbox:
val = unread if recipient != owner else read
convFolderMap[recipient] = {'mAllConversations': {timeUUID: val}}
userFolderMap[recipient] = {'mAllConversations': ''}
if recipient != owner:
convFolderMap[recipient]['mUnreadConversations'] = {timeUUID: unread}
userFolderMap[recipient]['mUnreadConversations'] = ''
else:
val = unread if recipient != owner else read
convFolderMap[recipient] = {'mDeletedConversations': {timeUUID: val}}
yield db.batch_mutate(convFolderMap)
yield db.batch_insert(convId, "mConvFolders", userFolderMap)
if toRemove['latest'] and oldTimeUUID != timeUUID:
yield db.batch_remove(toRemove, names=[oldTimeUUID], supercolumn="messages")
if toNotify:
yield db.batch_mutate(toNotify)
def updateData():
yield db.truncate('user_files')
try:
yield db.get('asdf', 'entityFeed_files', uuid.uuid1().bytes)
except ttypes.InvalidRequestException as exception:
log.msg(exception)
raise Exception('entityFeed_files CF missing, create the CF')
except ttypes.NotFoundException:
pass
entities = {}
items = {}
rows = yield db.get_range_slice('items', count=10000, reverse=True)
for row in rows:
itemId = row.key
item = utils.supercolumnsToDict(row.columns)
items[itemId]=item
for itemId in items:
item = items[itemId]
log.msg(itemId)
if 'meta' not in item:
continue
# Add org to all items
try:
owner = item['meta']['owner']
col = yield db.get(owner, "entities", 'org', 'basic')
ownerOrgId = col.column.value
yield db.insert(itemId, 'items', ownerOrgId, 'org', 'meta')
except Exception as e:
if item['meta'].get('type', '') == 'feedback':
yield db.insert(itemId, 'items', owner, 'org', 'meta')
# Fix ACLs
if 'parent' not in item['meta']:
acl = item['meta']['acl']
convOwner = item['meta']['owner']
convId = itemId
if acl == 'company':
col = yield db.get(convOwner, "entities", "org", "basic")
ownerOrgId = col.column.value
acl = pickle.dumps({"accept":{"orgs":[ownerOrgId]}})
yield db.insert(convId, 'items', acl, 'acl', 'meta')
else:
try:
acl = pickle.loads(acl)
if 'accept' in acl and 'friends' in acl['accept'] and isinstance(acl['accept']['friends'], bool):
del acl['accept']['friends']
acl = pickle.dumps(acl)
yield db.insert(convId, 'items', acl, 'acl', 'meta')
except :
log.msg('cannot unpack acl', acl)
# Migrate files
# truncate user_files
# update user_files and entityFeed_files
if 'owner' in item['meta'] and 'attachments' in item:
ownerId = item['meta']['owner']
if ownerId not in entities:
cols = yield db.get_slice(ownerId, 'entities', ['basic'])
entities.update({ownerId: utils.supercolumnsToDict(cols)})
for attachmentId in item['attachments']:
orgId = entities[ownerId]['basic']['org']
timeuuid, name = item['attachments'][attachmentId].split(':')[:2]
timeuuid = utils.decodeKey(timeuuid)
val = '%s:%s:%s:%s' % (attachmentId, name, itemId, ownerId)
yield db.insert(ownerId, "user_files", val, timeuuid)
if 'parent' not in item['meta'] and item['meta'].get('acl', ''):
_entities = yield utils.expandAcl(ownerId, orgId, item['meta']['acl'],
itemId, ownerId, True)
for entityId in _entities:
yield db.insert(entityId, "entityFeed_files", val, timeuuid)
# Migrate items
# Meta fields in "link", "event" and "poll"
if item['meta'].get('type', None) in ['link', 'poll', 'event']:
itemMeta = item['meta']
itemType = itemMeta['type']
updated = {}
if itemType == "link":
if 'url' in itemMeta:
updated['link_url'] = itemMeta['url']
if 'title' in itemMeta:
updated['link_title'] = itemMeta['title']
if 'summary' in itemMeta:
updated['link_summary'] = itemMeta['summary']
if 'imgSrc' in itemMeta:
updated['link_imgSrc'] = itemMeta['imgSrc']
if 'embedType' in itemMeta:
updated['link_embedType'] = itemMeta['embedType']
if 'embedSrc' in itemMeta:
updated['link_embedSrc'] = itemMeta['embedSrc']
if 'embedHeight' in itemMeta:
updated['link_embedHeight'] = itemMeta['embedHeight']
if 'embedWidth' in itemMeta:
updated['link_embedWidth'] = itemMeta['embedWidth']
elif itemType == 'poll':
if 'question' in itemMeta:
updated['comment'] = itemMeta['question']
else:
print 'Found an event:', itemId
if updated:
yield db.batch_insert(itemId, 'items', {'meta': updated})
#
# Create poll indexes for feed and userItems
#
rows = yield db.get_range_slice('entities', count=10000, reverse=True)
mutations = {}
for row in rows:
entityId = row.key
entity = utils.supercolumnsToDict(row.columns)
if entity['basic']['type'] != 'user':
continue
d1 = db.get_slice(entityId, 'feed', count=10000)
d2 = db.get_slice(entityId, 'userItems', count=10000)
results = yield d1
for col in results:
value = col.column.value
if value in items:
if items.get(value, {}).get('meta', {}).get('type', '') == 'poll':
mutations.setdefault(entityId, {}).setdefault('feed_poll', {}).update({col.column.name: value})
results = yield d2
for col in results:
value = col.column.value
responseType, itemId, convId, convType, others = value.split(':', 4)
if convType == 'poll':
mutations.setdefault(entityId, {}).setdefault('userItems_poll', {}).update({col.column.name: value})
yield db.batch_mutate(mutations)
#Group type changed from public-private to open-closed.
rows = yield db.get_range_slice('entityGroupsMap', count=1000)
groupIds = set()
for row in rows:
for col in row.columns:
name_, groupId = col.column.name.split(':')
groupIds.add(groupId)
cols = yield db.multiget_slice(groupIds, "entities")
groups = utils.multiSuperColumnsToDict(cols)
for groupId in groups:
access = groups[groupId]['basic']['access'].lower()
if access == 'public':
yield db.insert(groupId, 'entities', 'open', 'access', 'basic')
elif access.lower() == 'private':
yield db.insert(groupId, 'entities', 'closed', 'access', 'basic')
#Fix entityGroupsMap
rows = yield db.get_range_slice('entityGroupsMap', count=1000)
for row in rows:
entityId = row.key
for col in row.columns:
name_, groupId = col.column.name.split(':')
if col.column.name != '%s:%s'%(groups[groupId]['basic']['name'].lower(), groupId):
yield db.remove(entityId, 'entityGroupsMap', col.column.name)
yield db.insert(entityId, 'entityGroupsMap', '', '%s:%s' %(groups[groupId]['basic']['name'].lower(), groupId))
def push(userId, orgId, convId, conv, timeUUID, updateVal,
feeds=None, promote=True, promoteActor=False):
"""Push an item update to feeds. This function adds an update to
feedItems and if necessary promotes the conversation to the top of
the feed.
Keyword params:
@userId - Id of the actor
@orgId - orgId of the actor
@convId - Id of the conversation which got updated
@conv - The conversation that got updated
@timeUUID - UUID1 that represents this update
@updateVal - Value that is stored in feedItems
@feeds - List of feeds that must be updated
@promote - Promote the item up the conversation?
@promoteActor - When promote is True, should this update promote
the conversation even on the actor's feed.
"""
meta = conv['meta']
convType = meta['type']
if not feeds:
feeds = yield utils.expandAcl(userId, orgId, meta['acl'],
convId, meta['owner'])
feeds.add(userId)
userFeedItems = yield db.multiget_slice(feeds, "feedItems",
super_column=convId, reverse=True)
# XXX: Assumes that updateVal is of the following format
# <updateType>:<actorId>:<plugin specifics>
def _updateFeedItems(userId, promoteToUser):
userUpdatesOfType = [] # Items in feed of type that promote
userUpdateIds = [] # TimeUUIDs of all updates that promote
allUpdatesOfType = [] # All items in feed by type
oldest = None # Most relevant item for removal
oldFeedKeys = []
updateType = updateVal.split(':', 1)[0]
cur = userFeedItems.get(userId, [])
for x in cur:
uid = x.column.name
val = x.column.value.split(':')
if uid == timeUUID: # We already know this update!!!
defer.returnValue(None)
rtype = val[0]
if rtype not in ('!', 'I'):
if rtype == updateType:
allUpdatesOfType.append(uid)
if val[1] == userId:
userUpdateIds.append(uid)
userUpdatesOfType.append(uid)
oldest = uid
oldFeedKeys.append(uid)
curUpdateCount = len(cur)
curUpdateCountForType = len(allUpdatesOfType)
if curUpdateCountForType == MAXFEEDITEMSBYTYPE:
# If this update isn't promoting the conversation up the feed,
# we ought to make sure that we don't remove an item that is the
# reason for the current position of conversation in the feed.
if not promoteToUser and \
(len(userUpdatesOfType) == MAXFEEDITEMSBYTYPE or \
(allUpdatesOfType[-1] not in userUpdatesOfType and \
len(userUpdatesOfType) == MAXFEEDITEMSBYTYPE - 1)):
oldest = userUpdatesOfType[-2]
else:
oldest = allUpdatesOfType[-1]
if not promoteToUser and \
(len(userUpdateIds) == MAXFEEDITEMS - 1 or \
(oldest not in userUpdateIds and\
len(userUpdateIds) == MAXFEEDITEMS - 2)):
oldest = userUpdateIds[-2]
feedItemToRemove = oldest if curUpdateCountForType == MAXFEEDITEMSBYTYPE\
or curUpdateCount == MAXFEEDITEMS\
else None
insertConv = True if curUpdateCount == 0 and updateType != 'I' else False
return oldFeedKeys, feedItemToRemove, insertConv
# Fetch list of changes to feedItems
removeFeedKeys = {}
removeFeedItems = {}
insertDummyConvs = []
for feedId in feeds:
promoteToUser = promote and (feedId != userId or promoteActor)
(oldFeedKeys, feedItemToRemove, insertConv) = \
_updateFeedItems(feedId, promoteToUser)
if oldFeedKeys and promoteToUser:
removeFeedKeys[feedId] = oldFeedKeys
if feedItemToRemove:
removeFeedItems[feedId] = [feedItemToRemove]
if insertConv:
insertDummyConvs.append(feedId)
# Update feedItems
feedItemsMutations = {}
feedItemsRemovalMutations = {}
timestamp = int(time.time() * 1e6)
for feedId in feeds:
mutations = {}
keys = removeFeedKeys.get(feedId, None)
if keys:
predicate = ttypes.SlicePredicate(column_names=keys)
deletion = ttypes.Deletion(timestamp, predicate=predicate)
mutations['feed'] = [deletion]
if feedId in insertDummyConvs:
dummyValue = ':'.join(['!', meta['owner'], convId])
mutations['feedItems'] = {convId: {meta['uuid']: dummyValue,
timeUUID: updateVal}}
else:
mutations['feedItems'] = {convId: {timeUUID: updateVal}}
feedItemsMutations[feedId] = mutations
feedItemKey = removeFeedItems.get(feedId, None)
if feedItemKey:
predicate = ttypes.SlicePredicate(column_names=feedItemKey)
deletion = ttypes.Deletion(timestamp, convId, predicate=predicate)
feedItemsRemovalMutations[feedId] = {'feedItems': [deletion]}
# Promote items in all feeds
feedMutations = {}
for feedId in feeds:
promoteToUser = promote and (feedId != userId or promoteActor)
if promoteToUser:
mutations = {'feed': {timeUUID: convId}}
if convType in plugins and plugins[convType].hasIndex:
mutations['feed_%s' % convType] = {timeUUID: convId}
feedMutations[feedId] = mutations
yield db.batch_mutate(feedItemsMutations)
yield db.batch_mutate(feedItemsRemovalMutations)
yield db.batch_mutate(feedMutations)
def unpush(userId, orgId, convId, conv, updateVal, feeds=None):
meta = conv['meta']
convType = meta['type']
if not feeds:
feeds = yield utils.expandAcl(userId, orgId, meta['acl'],
convId, meta['owner'])
feeds.add(userId)
userFeedItems = yield db.multiget_slice(feeds, "feedItems",
super_column=convId, reverse=True)
updateType = updateVal.split(':', 1)[0]
if updateType == 'T':
updateValParts = updateVal.split(':')
hasIndex = False
if convType in plugins and plugins[convType].hasIndex:
hasIndex = True
indexColFamily = 'feed_' + convType
timestamp = int(time.time() * 1e6)
removeMutations = {}
insertMutations = {}
for feedId in feeds:
cols = userFeedItems[feedId]
if not cols:
continue
updatesCount = len(cols)
latest, second, pseudoFeedTime = None, None, None
for col in cols:
timeUUID = col.column.name
val = col.column.value
valUpdateType = val.split(":", 1)[0]
if valUpdateType != '!':
if latest and not second:
second = timeUUID
if not latest:
latest = timeUUID
elif updatesCount == 2 and valUpdateType == "!":
pseudoFeedTime = timeUUID
for col in cols:
timeUUID = col.column.name
val = col.column.value
valParts = val.split(':')
if (updateType == 'T' and val.startswith('T:') and
len(valParts) == 5 and updateValParts[2] == valParts[2] and
updateValParts[4] == valParts[4]) or (val == updateVal):
removals = {}
# Remove the update from feedItems. If this is the only
# update then remove the entire super column
if not pseudoFeedTime:
predicate = ttypes.SlicePredicate(column_names=[timeUUID])
superCol = convId
else:
predicate = ttypes.SlicePredicate(column_names=[convId])
superCol = None
feedItemsDeletion = ttypes.Deletion(timestamp, superCol, predicate)
removals['feedItems'] = [feedItemsDeletion]
# If this is the latest update, remove conv from it's
# current position in feed and feed indexes.
feedRemoveKeys = []
if latest == timeUUID:
feedRemoveKeys.append(timeUUID)
if pseudoFeedTime:
feedRemoveKeys.append(pseudoFeedTime)
if feedRemoveKeys:
feedPredicate = ttypes.SlicePredicate(column_names=feedRemoveKeys)
feedDeletion = ttypes.Deletion(timestamp, predicate=feedPredicate)
removals['feed'] = [feedDeletion]
if hasIndex:
removals[indexColFamily] = [feedDeletion]
# Reposition feed to the next most recent update
if latest == timeUUID and second:
insertMutations[feedId] = {'feed': {second: convId}}
removeMutations[feedId] = removals
break
yield db.batch_mutate(insertMutations)
yield db.batch_mutate(removeMutations)
def postFeedback(self, request):
"""creates a feedback item with feedback, feedback-{mood} tags.
Push item to feeback, feedback-{mood} tag followers.
Note: Item is owned by feedback-domain/synovel.com, not user sending
the feedback. Only users of feedback-domain/synovel.com can access
the item.
"""
comment = utils.getRequestArg(request, 'comment')
mood = utils.getRequestArg(request, 'mood')
if not mood or not comment:
raise errors.MissingParams([_("Feedback")])
authInfo = request.getSession(IAuthInfo)
myId = authInfo.username
orgId = authInfo.organization
tagName = 'feedback'
moodTagName = 'feedback-' + mood
feedbackDomain = config.get('Feedback', 'Domain') or 'synovel.com'
cols = yield db.get_slice(feedbackDomain, 'domainOrgMap')
if not cols:
raise errors.ConfigurationError("feedbackDomain is invalid!")
# Only one org exists per domain
synovelOrgId = cols[0].column.name
tagId, tag = yield tags.ensureTag(request, tagName, synovelOrgId)
moodTagId, moodTag = yield tags.ensureTag(request, moodTagName,
synovelOrgId)
# Anyone in synovel can receive feedback.
acl = {'accept': {'orgs': [synovelOrgId]}}
acl = json.dumps(acl)
synovelOrg = base.Entity(synovelOrgId)
yield synovelOrg.fetchData()
# createNewItem expects an entity object with has org in basic info.
# organizations wont have 'org' set.
synovelOrg.basic['org'] = synovelOrgId
item = yield utils.createNewItem(request, 'feedback', synovelOrg,
acl, subType=mood)
item['meta']['org'] = synovelOrgId
item['meta']['userId'] = myId
item['meta']['userOrgId'] = orgId
item['meta']['comment'] = comment
item['tags'] = {tagId: synovelOrgId, moodTagId: synovelOrgId}
itemId = utils.getUniqueKey()
tagItemCount = int(tag['itemsCount'])
moodTagItemCount = int(moodTag['itemsCount'])
if tagItemCount % 10 == 7:
tagItemCount = yield db.get_count(tagId, "tagItems")
if moodTagItemCount % 10 == 7:
moodTagItemCount = yield db.get_count(moodTagId, "tagItems")
tagItemCount += 1
moodTagItemCount += 1
# Finally save the feedback
yield db.batch_insert(itemId, "items", item)
yield db.insert(tagId, "tagItems", itemId, item["meta"]["uuid"])
yield db.insert(moodTagId, "tagItems", itemId, item["meta"]["uuid"])
yield db.insert(synovelOrgId, "orgTags", str(tagItemCount),
"itemsCount", tagId)
yield db.insert(synovelOrgId, "orgTags", str(moodTagItemCount),
"itemsCount", moodTagId)
cols = yield db.multiget_slice([tagId, moodTagId], "tagFollowers")
followers = utils.multiColumnsToDict(cols)
followers = set(followers[tagId].keys() + followers[moodTagId].keys())
value = {"feed": {item['meta']['uuid']: itemId}}
muts = dict([(x, value) for x in followers])
if muts:
yield db.batch_mutate(muts)
def migrateFriendsToFollowers():
# Migrate all friends to followers/subscriptions.
connectionRows = yield db.get_range_slice('connections', count=10000)
for connectionRow in connectionRows:
userId = connectionRow.key
friends = [x.super_column.name for x in connectionRow.columns]
yield db.batch_insert(userId, "followers", dict([(x, '') for x in friends]))
yield db.batch_mutate(dict([(x, {'subscriptions': {userId: ''}}) for x in friends]))
log.msg('>>>>>>>> Converted all connections to following.')
# Remove name indices of friends
entityRows = yield db.get_range_slice('entities', count=10000, names=['basic'])
entities = dict([(x.key, utils.supercolumnsToDict(x.columns)) for x in entityRows])
userIds = [x for x in entities.keys() if entities[x]['basic']['type'] == 'user']
for userId in userIds:
yield db.remove(userId, 'displayNameIndex')
yield db.remove(userId, 'nameIndex')
log.msg('>>>>>>>> Removed name indices for friends.')
# Convert all "connection" activity to "follow".
# We already have two separate items, so subtype conversion should be good.
itemRows = yield db.get_range_slice('items', count=10000, names=['meta'])
items = dict([(x.key, utils.supercolumnsToDict(x.columns)) for x in itemRows])
connectionItems = [x for x in items.keys()\
if items[x]['meta'].get('type', '') == 'activity'\
and items[x]['meta']['subType'] == 'connection']
yield db.batch_mutate(dict([(x, {'items':{'meta':{'subType':'following'}}}) for x in connectionItems]))
log.msg('>>>>>>>> All connection items converted to following.')
# Remove all friend requests from pendingConnections
pendingRows = yield db.get_range_slice('pendingConnections', count=10000)
for pendingRow in pendingRows:
userId = pendingRow.key
pendingFriendRequestIds = [x.column.name for x in pendingRow.columns \
if not x.column.name.startswith('G')]
if pendingFriendRequestIds:
yield db.batch_remove({'pendingConnections': [userId]}, names=pendingFriendRequestIds)
log.msg('>>>>>>>> Removed pending friend requests.')
# Remove all friend requests from latest
yield db.batch_remove({'latest': userIds}, names='people')
log.msg('>>>>>>>> Removed friend requests from latest.')
# Remove all friend-request-accepted notifications
notifyMutations = {}
for userId in userIds:
items = yield db.get_slice(userId, "notificationItems", super_column=':FA')
if items:
names = [col.column.name for col in items]
colmap = dict([(x, None) for x in names])
deletion = Deletion(time.time() * 1000000, 'notifications',
SlicePredicate(column_names=names))
notifyMutations[userId] = {'notifications': colmap, 'latest': [deletion]}
yield db.remove(userId, 'notificationItems', super_column=':FA')
if notifyMutations:
yield db.batch_mutate(notifyMutations)
log.msg('>>>>>>>> Removed friend notifications from notifications and latest.')
# Finally, remove the connections column family.
yield db.system_drop_column_family('connections')
yield db.system_drop_column_family('connectionsByTag')
log.msg('>>>>>>>> Removed the connections column family.')