class CachingTagValueAPI(object):
"""The public API to cached L{TagValue}-related logic in the model.
@param user: The L{User} to perform operations on behalf of.
"""
def __init__(self, user):
self._api = TagValueAPI(user, factory=CachingAPIFactory())
self._user = user
def get(self, objectIDs, paths=None):
"""See L{TagValueAPI.get}."""
return self._api.get(objectIDs, paths)
def set(self, values):
"""See L{TagValueAPI.set}."""
result = self._api.set(values)
RecentObjectActivityCache().clear(values.keys())
RecentUserActivityCache().clear([self._user.username])
return result
def delete(self, values):
"""See L{TagValueAPI.delete}."""
if isgenerator(values):
values = list(values)
result = self._api.delete(values)
objectIDs = [objectID for objectID, path in values]
RecentObjectActivityCache().clear(objectIDs)
RecentUserActivityCache().clear([self._user.username])
return result
def testCheckIntegrityChecksErrorInAllRows(self):
"""
L{checkIntegrity} should check for integrity errors in L{Namespace}s,
L{Tag}s, L{User}s, L{AboutTagValue}s and L{TagValue}s.
"""
createSystemData()
[(userObjectID, _)] = UserAPI().create([(u'user', u'pass', u'Name',
u'*****@*****.**')])
user = getUser(u'user')
result = NamespaceAPI(user).create([(u'user/namespace', u'description')
])
[(namespaceObjectID, _)] = result
[(tagObjectID, _)] = TagAPI(user).create([(u'user/tag', u'description')
])
objectID1 = uuid4()
objectID2 = uuid4()
createAboutTagValue(objectID1, u'Bad about tag')
TagValueAPI(user).set({objectID2: {u'fluiddb/about': 'about value'}})
TagValueAPI(user).delete([(userObjectID, u'fluiddb/users/username')])
TagValueAPI(user).delete([(namespaceObjectID,
u'fluiddb/namespaces/path')])
TagValueAPI(user).delete([(tagObjectID, u'fluiddb/tags/description')])
checkIntegrity()
self.assertEqual(
"Integrity Error in namespace u'user/namespace': "
'Path tag is missing.\n'
"Integrity Error in tag u'user/tag': Description tag is missing.\n"
"Integrity Error in user u'user': Username tag is missing.\n"
"Integrity Error in object %s: AboutTagValue doesn't have an "
'associated TagValue.\n'
"Integrity Error in object %s: fluiddb/about TagValue doesn't "
'have an associated AboutTagValue.\n' % (objectID1, objectID2),
self.log.getvalue())
def testGetFromObjectsWithBinaryValue(self):
"""
L{RecentActivityAPI.getForObjects} returns binary values using the
expected format.
"""
tagValues = TagValueAPI(self.user)
objectID = uuid4()
values = {
objectID: {
u'user/tag': {
u'mime-type': u'text/plain',
u'contents': 'Hello, world!'
}
}
}
tagValues.set(values)
self.store.commit()
value = {u'value-type': u'text/plain', u'size': 13}
expected = [(u'user/tag', objectID, None, value, u'user')]
result = self.recentActivity.getForObjects([objectID])
# Remove the creation times from the result, with the order is enough.
result = [(path, objectID, about, value, username)
for path, objectID, about, value, username, time in result]
self.assertEqual(expected, result)
def testRenderRecentUserActivity(self):
"""
L{RecentUserActivityResource.deferred_render_GET} renders a response
with recent activity data for the given user.
"""
objectID = ObjectAPI(self.user).create(u'object1')
TagValueAPI(self.user).set({objectID: {u'username/tag1': u'A'}})
self.store.commit()
TagValueAPI(self.user).set({objectID: {u'username/tag2': u'B'}})
self.store.commit()
request = FakeRequest()
with login(u'username', self.user.objectID, self.transact) as session:
resource = RecentUserActivityResource(self.facade, session,
u'username')
body = yield resource.deferred_render_GET(request)
body = json.loads(body)
expected = [{
u'username': u'username',
u'about': u'object1',
u'id': str(objectID),
u'tag': u'username/tag2',
u'value': u'B'
}, {
u'username': u'username',
u'about': u'object1',
u'id': str(objectID),
u'tag': u'username/tag1',
u'value': u'A'
}]
# Clean up timestamps.
for item in body:
del item['updated-at']
self.assertEqual(expected, body)
self.assertEqual(http.OK, request.code)
def setUp(self):
super(DataImportHandlerTest, self).setUp()
createSystemData()
UserAPI().create([(u'user', u'secret', u'User', u'*****@*****.**')])
self.user = getUser(u'user')
self.objects = ObjectAPI(self.user)
self.values = TagValueAPI(self.user)
self.index = ObjectIndex(self.client)
def extractTrendingHashtags(store, limit=10, duration=None):
"""Extract information about trending hashtags and store it in FluidDB.
@param store: The storm store to query and to save our result to.
@param limit: Optionally, the number of objects to retrieve.
@param duration: Optionally, the recent time period to look at when
determining which hashtags are trending. Default is 28 days.
The storm query below results in SQL like:
SELECT COUNT(DISTINCT comments.object_id) AS count,
about_tag_values.value,
array_agg(ROW(comments.username, comments.creation_time))
FROM about_tag_values, comment_object_link, comments
WHERE about_tag_values.value LIKE '#%' AND
about_tag_values.object_id = comment_object_link.object_id AND
comments.object_id = comment_object_link.comment_id AND
comments.creation_time >= '2012-11-09 07:42:40'::TIMESTAMP AND
CHAR_LENGTH(about_tag_values.value) >= 2
GROUP BY about_tag_values.value
ORDER BY count DESC
LIMIT 10
"""
duration = timedelta(days=28) if duration is None else duration
startTime = datetime.utcnow() - duration
count = Alias(Count(Comment.objectID, distinct=True))
result = store.find(
(count, AboutTagValue.value,
Func('array_agg', Row(Comment.username, Comment.creationTime))),
Like(AboutTagValue.value,
u'#%'), AboutTagValue.objectID == CommentObjectLink.objectID,
Comment.objectID == CommentObjectLink.commentID,
Comment.creationTime >= startTime,
Func('CHAR_LENGTH', AboutTagValue.value) >= 2)
result.group_by(AboutTagValue.value)
result.order_by(Desc(count))
result.config(limit=limit)
data = [{
'count': count,
'usernames': _sortUsernames(usernames),
'value': hashtag
} for count, hashtag, usernames in result]
user = getUser(u'fluidinfo.com')
tagValues = TagValueAPI(user)
objectID = ObjectAPI(user).create(u'fluidinfo.com')
tagValues.set(
{objectID: {
u'fluidinfo.com/trending-hashtags': json.dumps(data)
}})
store.commit()
def setVersionTag(version):
"""Updates the fluiddb/version tag.
@param version: The new version string.
"""
user = getUser(u'fluiddb')
objectID = ObjectAPI(user).create(u'fluidinfo')
releaseDate = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
values = {
objectID: {
u'fluiddb/api-version': {
'mime-type': 'text/plain',
'contents': version
},
u'fluiddb/release-date': {
'mime-type': 'text/plain',
'contents': releaseDate + '\n'
}
}
}
TagValueAPI(user).set(values)
PermissionAPI(user).set([
(u'fluiddb/api-version', Operation.READ_TAG_VALUE, Policy.OPEN, []),
(u'fluiddb/release-date', Operation.READ_TAG_VALUE, Policy.OPEN, [])
])
try:
transaction.commit()
except:
transaction.abort()
raise
def getTagValueAPI(self, user):
"""Get a L{TagValueAPI} instance for the specified user.
@param user: The L{User} to configure the L{TagValueAPI} instance.
@return: A L{TagValueAPI} instance.
"""
return TagValueAPI(user)
def testRenderRecentUsersActivity(self):
"""
L{RecentUsersActivityResource.deferred_render_GET} renders a response
with recent activity data by the users returned by the given query.
"""
objectID = ObjectAPI(self.user).create(u'object1')
self.store.commit()
TagValueAPI(self.user).set({objectID: {u'username/tag1': u'A'}})
runDataImportHandler(self.client.url)
request = FakeRequest(
args={'query': ['fluiddb/users/username = "******"']})
with login(u'username', self.user.objectID, self.transact) as session:
resource = RecentUsersActivityResource(self.facade, session)
body = yield resource.deferred_render_GET(request)
body = json.loads(body)
expected = [{
u'about': u'object1',
u'id': str(objectID),
u'tag': u'username/tag1',
u'username': u'username',
u'value': u'A'
}]
# Clean up timestamps.
for item in body:
del item['updated-at']
self.assertEqual(expected, body)
self.assertEqual(http.OK, request.code)
def testDeleteRemovesSystemTags(self):
"""
L{UserAPI.delete} removes the C{fluiddb/users/*} tag values stored for
deleted L{User}s.
"""
self.users.create([(u'user', u'pass', u'User', u'*****@*****.**')])
user = getUser(u'user')
NamespaceAPI(user).delete([u'user/private'])
[(objectID, _)] = self.users.delete([u'user'])
tagValues = TagValueAPI(self.system.users[u'fluiddb'])
result = tagValues.get(objectIDs=[objectID],
paths=[u'fluiddb/users/username',
u'fluiddb/users/name',
u'fluiddb/users/email',
u'fluiddb/users/role'])
self.assertEqual({}, result)
def testDeleteTagWithData(self):
"""
L{FacadeTagMixin.deleteTag} removes L{TagValue}s associated with the
deleted L{Tag}.
"""
objectID = uuid4()
TagAPI(self.user).create([(u'username/tag', u'A tag.')])
tagValues = TagValueAPI(self.user)
tagValues.set({objectID: {u'username/tag': 42}})
self.store.commit()
with login(u'username', uuid4(), self.transact) as session:
yield self.facade.deleteTag(session, u'username/tag')
self.store.rollback()
self.assertEqual({}, tagValues.get(objectIDs=[objectID],
paths=[u'username/tag']))
def setUp(self):
super(TagValueAPITest, self).setUp()
self.system = createSystemData()
UserAPI().create([(u'username', u'password', u'User',
u'*****@*****.**')])
self.user = getUser(u'username')
self.permissions = PermissionAPI(self.user)
self.tagValues = TagValueAPI(self.user)
def testDeleteRemovesSystemTags(self):
"""
L{UserAPI.delete} removes the C{fluiddb/users/*} tag values stored for
deleted L{User}s.
"""
self.users.create([(u'user', u'pass', u'User', u'*****@*****.**')])
user = getUser(u'user')
NamespaceAPI(user).delete([u'user/private'])
[(objectID, _)] = self.users.delete([u'user'])
tagValues = TagValueAPI(self.system.users[u'fluiddb'])
result = tagValues.get(objectIDs=[objectID],
paths=[
u'fluiddb/users/username',
u'fluiddb/users/name',
u'fluiddb/users/email',
u'fluiddb/users/role'
])
self.assertEqual({}, result)
def _getResult(self):
"""Get the result of the trending hashtag function.
return: The value stored by the trending hashtag function, converted
from JSON.
"""
result = TagValueAPI(self.user).get([self.fluidinfObjectID],
paths=[self.tag])
return loads(result[self.fluidinfObjectID][self.tag].value)
def testGetRecentUserActivityForQuery(self):
"""
L{FacadeRecentActivityMixin.getRecentUserActivityForQuery} returns a
C{dict} with information about the recent tag values by the users whose
objects are returned by the given query.
"""
UserAPI().create([(u'user2', u'password', u'User',
u'*****@*****.**')])
user2 = getUser(u'user2')
objectID1 = uuid4()
# Use commit() frequently to have different timestamps on each value.
TagValueAPI(self.user).set({objectID1: {u'username/test': u'A'}})
self.store.commit()
TagValueAPI(user2).set({objectID1: {u'user2/test': u'B'}})
self.store.commit()
runDataImportHandler(self.client.url)
expected = [{'about': None,
'id': str(objectID1),
'tag': u'user2/test',
'username': u'user2',
'value': u'B'},
{'about': None,
'id': str(objectID1),
'tag': u'username/test',
'username': u'username',
'value': u'A'}]
with login(self.user.username, uuid4(), self.transact) as session:
result = yield self.facade.getRecentUserActivityForQuery(
session,
u'fluiddb/users/username = "******" '
'OR fluiddb/users/username = "******"')
# Remove the creation times from the result.
for item in result:
del item['updated-at']
self.assertEqual(expected, result)
def testDeleteRemovesPath(self):
"""
L{TagAPI.delete} removes the C{fluiddb/tags/path} values stored
for deleted L{Tag}s.
"""
values = [(u'username/tag', u'A description')]
[(objectID, path)] = self.tags.create(values)
self.tags.delete([u'username/tag'])
result = TagValueAPI(self.user).get(objectIDs=[objectID],
paths=[u'fluiddb/tags/path'])
self.assertEqual({}, result)
def testSearchWithHasQueryDoesNotHitSolr(self):
"""
L{ObjectAPI.search} doesn't hit Solr to resolve C{has <path>} queries.
"""
# Use an invalid Solr URL to test that we're not hitting Solr.
self.config.set('index', 'url', 'http://none')
objectID = self.objects.create(u'TestObject')
TagValueAPI(self.user).set({objectID: {u'username/test': 'value'}})
query = parseQuery(u'has username/test')
result = yield self.objects.search([query]).get()
self.assertEqual({query: set([objectID])}, result)
def testDeleteRemovesDescription(self):
"""
L{NamespaceAPI.delete} removes the C{fluiddb/namespaces/description}
values stored for deleted L{Namespace}s.
"""
values = [(u'username/child', u'A description')]
[(objectID, path)] = self.namespaces.create(values)
self.namespaces.delete([u'username/child'])
result = TagValueAPI(self.user).get(
objectIDs=[objectID], paths=[u'fluiddb/namespaces/description'])
self.assertEqual({}, result)
def testDeleteUpdatesObjectsTagged(self):
"""
L{TagAPI.delete} updates the objects previously tagged with the given
paths.
"""
objectID = uuid4()
self.tags.create([(u'username/child', u'A description')])
TagValueAPI(self.user).set({objectID: {u'username/child': 64}})
getDirtyObjects([objectID]).remove()
self.tags.delete([u'username/child'])
self.assertIn(objectID, getDirtyObjects().values(DirtyObject.objectID))
def testSetSkipsUpdatingTagValuesWhenPasswordChanges(self):
"""
The C{fluiddb/users/name} and C{fluiddb/users/email} L{TagValue}s are
not updated when only the L{User.password} is changed by
L{UserAPI.set}.
"""
user = createUser(u'user', u'secret', u'User', u'*****@*****.**')
self.users.set([(u'user', u'pwd', None, None, None)])
result = TagValueAPI(user).get([user.objectID], [
u'fluiddb/users/name', u'fluiddb/users/email',
u'fluiddb/users/role'
])
self.assertEqual({}, result)
def testGetFromObjectsWithBinaryValue(self):
"""
L{RecentActivityAPI.getForObjects} returns binary values using the
expected format.
"""
tagValues = TagValueAPI(self.user)
objectID = uuid4()
values = {objectID: {u'user/tag': {u'mime-type': u'text/plain',
u'contents': 'Hello, world!'}}}
tagValues.set(values)
self.store.commit()
value = {u'value-type': u'text/plain',
u'size': 13}
expected = [(u'user/tag', objectID, None, value, u'user')]
result = self.recentActivity.getForObjects([objectID])
# Remove the creation times from the result, with the order is enough.
result = [(path, objectID, about, value, username)
for path, objectID, about, value, username, time in result]
self.assertEqual(expected, result)
def testRenderRecentObjectActivityWithNoQuery(self):
"""
L{RecentUsersActivityResource.deferred_render_GET} raises
L{MissingArgument} if the C{query} argument is not given.
"""
objectID = ObjectAPI(self.user).create(u'object1')
self.store.commit()
TagValueAPI(self.user).set({objectID: {u'username/tag1': u'A'}})
runDataImportHandler(self.client.url)
request = FakeRequest()
with login(u'username', self.user.objectID, self.transact) as session:
resource = RecentUsersActivityResource(self.facade, session)
deferred = resource.deferred_render_GET(request)
yield self.assertFailure(deferred, MissingArgument)
def testGetRecentActivityForQuery(self):
"""
L{FacadeRecentActivityMixin.getRecentActivityForQuery} returns a
C{dict} with information about the recent tag values on the objects
returned by the given query.
"""
tagValues = TagValueAPI(self.user)
objectID1 = ObjectAPI(self.user).create(u'object1')
# Use commit() frequently to have different timestamps on each value.
self.store.commit()
tagValues.set({objectID1: {u'user/following': u'A'}})
self.store.commit()
runDataImportHandler(self.client.url)
expected = [{'about': u'object1',
'id': str(objectID1),
'tag': u'user/following',
'username': u'username',
'value': u'A'},
{'about': u'object1',
'id': str(objectID1),
'tag': u'fluiddb/about',
'username': u'fluiddb',
'value': u'object1'}]
with login(self.user.username, uuid4(), self.transact) as session:
result = yield self.facade.getRecentActivityForQuery(
session, u'has user/following')
# Remove the creation times from the result.
for item in result:
del item['updated-at']
self.assertEqual(expected, result)
def testGetRecentUserActivityForQueryWithObjectsNotUsers(self):
"""
L{FacadeRecentActivityMixin.getRecentUserActivityForQuery} returns an
empty result if the objects returned by the query are not users.
"""
tagValues = TagValueAPI(self.user)
objectID1 = ObjectAPI(self.user).create(u'object1')
# Use commit() frequently to have different timestamps on each value.
self.store.commit()
tagValues.set({objectID1: {u'user/following': u'A'}})
self.store.commit()
runDataImportHandler(self.client.url)
with login(self.user.username, uuid4(), self.transact) as session:
result = yield self.facade.getRecentUserActivityForQuery(
session, u'has user/following')
# Remove the creation times from the result.
for item in result:
del item['updated-at']
self.assertEqual([], result)
def testSetUpdatesTagValues(self):
"""
The C{fluiddb/users/name} and C{fluiddb/users/email} L{TagValue}s are
updated when L{User} details are changed by L{UserAPI.set}.
"""
user = createUser(u'user', u'secret', u'User', u'*****@*****.**')
self.users.set([(u'user', u'pwd', u'name', u'*****@*****.**',
Role.USER_MANAGER)])
result = TagValueAPI(user).get([user.objectID], [
u'fluiddb/users/name', u'fluiddb/users/email',
u'fluiddb/users/role'
])
email = result[user.objectID][u'fluiddb/users/email'].value
name = result[user.objectID][u'fluiddb/users/name'].value
role = result[user.objectID][u'fluiddb/users/role'].value
self.assertEqual(u'*****@*****.**', email)
self.assertEqual(u'name', name)
self.assertEqual(u'USER_MANAGER', role)
def testCheckIntegrityGetsAllRowsUsingMultipleQueries(self):
"""
L{checkIntegrity} should check all the rows of a given object using
multiple queries if the C{maxRowsPerQuery} argument is smaller than the
total number of rows for a given object.
"""
createSystemData()
UserAPI().create([(u'user', u'pass', u'Name', u'*****@*****.**')])
user = getUser(u'user')
paths = [u'user/namespace%d' % i for i in xrange(10)]
values = [(path, u'description') for path in paths]
NamespaceAPI(user).create(values)
namespaces = getNamespaces(paths=paths)
values = [(namespace.objectID, u'fluiddb/namespaces/path')
for namespace in namespaces]
TagValueAPI(user).delete(values)
checkIntegrity(maxRowsPerQuery=2)
error = 'Integrity Error in namespace %r: Path tag is missing.'
expectedErrors = '\n'.join(error % path for path in paths) + '\n'
self.assertEqual(expectedErrors, self.log.getvalue())
"""Creates fluiddb/users/role for all the users."""
from fluiddb.application import setConfig, setupConfig
from fluiddb.scripts.commands import setupStore
from fluiddb.data.user import getUsers
from fluiddb.model.user import getUser
from fluiddb.model.value import TagValueAPI
if __name__ == '__main__':
store = setupStore('postgres:///fluidinfo', 'main')
setConfig(setupConfig(None))
print __doc__
tagValues = TagValueAPI(getUser(u'fluiddb'))
for user in list(getUsers()):
print 'Adding role for', user.username
values = {user.objectID: {u'fluiddb/users/role': unicode(user.role)}}
tagValues.set(values)
store.commit()
def testGetForUsersReturnsOnlyAllowedTags(self):
"""
L{SecureRecentActivityAPI.getForUser} returns all the tags for the
superuser.
"""
tagValues = TagValueAPI(self.user)
objectID1 = ObjectAPI(self.user).create(u'object1')
objectID2 = uuid4()
# Use commit() frequently to have different timestamps on each value.
self.store.commit()
tagValues.set({objectID1: {u'user/tag1': u'A'}})
self.store.commit()
tagValues.set({objectID1: {u'user/tag2': u'B'}})
self.store.commit()
UserAPI().create([(u'user2', u'secret', u'User', u'*****@*****.**')])
tagValues = TagValueAPI(getUser(u'user2'))
tagValues.set({objectID1: {u'user2/tag1': u'C'}})
self.store.commit()
tagValues.set({objectID2: {u'user2/tag2': u'D'}})
self.store.commit()
UserAPI().create([(u'user3', u'secret', u'User', u'*****@*****.**')])
tagValues = TagValueAPI(getUser(u'user3'))
tagValues.set({objectID1: {u'user3/tag1': u'C'}})
self.store.commit()
tagValues.set({objectID2: {u'user3/tag2': u'D'}})
self.store.commit()
self.permissions.set([(u'user/tag2', Operation.READ_TAG_VALUE,
Policy.OPEN, [u'user']),
(u'user2/tag2', Operation.READ_TAG_VALUE,
Policy.CLOSED, [])])
expected = [
(u'user2/tag2', objectID2, None, u'D', u'user2'),
(u'user2/tag1', objectID1, u'object1', u'C', u'user2'),
(u'user/tag2', objectID1, u'object1', u'B', u'user'),
(u'user/tag1', objectID1, u'object1', u'A', u'user')]
result = self.recentActivity.getForUsers([u'user', u'user2'])
# Remove the creation times from the result, with the order is enough.
result = [(path, objectID, about, value, username)
for path, objectID, about, value, username, time in result]
self.assertEqual(expected, result)
def testGetForObjectsReturnsOnlyAllowedTags(self):
"""
L{SecureRecentActivityAPI.getForObjects} will return all the tags for
which the user has C{Operation.READ_TAG_VALUE} permissions, but not
those for which the user doesn't have.
"""
tagValues = TagValueAPI(self.user)
objectID1 = ObjectAPI(self.user).create(u'object1')
objectID2 = uuid4()
# Use commit() frequently to have different timestamps on each value.
self.store.commit()
tagValues.set({objectID1: {u'user/tag1': u'A'}})
self.store.commit()
tagValues.set({objectID1: {u'user/tag2': u'B'}})
self.store.commit()
tagValues.set({objectID2: {u'user/tag1': u'C'}})
self.store.commit()
tagValues.set({objectID2: {u'user/tag2': u'D'}})
self.store.commit()
tagValues.set({uuid4(): {u'user/tag1': u'E'}})
self.store.commit()
tagValues.set({uuid4(): {u'user/tag2': u'F'}})
self.store.commit()
self.permissions.set([(u'user/tag2', Operation.READ_TAG_VALUE,
Policy.OPEN, [u'user'])])
expected = [
(u'user/tag1', objectID2, None, u'C', u'user'),
(u'user/tag1', objectID1, u'object1', u'A', u'user'),
(u'fluiddb/about', objectID1, u'object1', u'object1', u'fluiddb')]
result = self.recentActivity.getForObjects([objectID1, objectID2])
# Remove the creation times from the result, with the order is enough.
result = [(path, objectID, about, value, username)
for path, objectID, about, value, username, time in result]
self.assertEqual(expected, result)
store = setupStore('postgres:///fluidinfo', 'main')
setConfig(setupConfig(None))
print __doc__
usernames = store.find(Comment.username).config(distinct=True)
for username in list(usernames):
user = getUser(username)
if user is None:
print "Ignoring non existing user."
continue
print 'Following objects commented by', username
result = store.find(CommentObjectLink.objectID,
CommentObjectLink.commentID == Comment.objectID,
Comment.username == username)
allObjectIDs = list(result.config(distinct=True))
BATCH_SIZE = 100
while allObjectIDs:
targets = allObjectIDs[:BATCH_SIZE]
followValues = dict((objectID, {
username + u'/follows': None
}) for objectID in targets)
TagValueAPI(user).set(followValues)
print '\t Following', len(targets), 'objects.'
store.commit()
allObjectIDs = allObjectIDs[BATCH_SIZE:]
print 'Sleeping two seconds. Giving a breath to the DIH.'
time.sleep(2)
followTag = username + u'/follows'
result = store.find(TagValue.objectID, TagValue.tagID == Tag.id,
Tag.path == followTag,
TagValue.creationTime > OLDER_THAN)
allObjectIDs = list(result)
print '\t Found', len(allObjectIDs), 'followed objects.'
BATCH_SIZE = 100
while allObjectIDs:
objectIDs = allObjectIDs[:BATCH_SIZE]
allObjectIDs = allObjectIDs[BATCH_SIZE:]
result = getAboutTagValues(objectIDs=objectIDs)
followedURLsObjectIDs = [
about.objectID for about in result
if URL_REGEX.match(about.value)
]
if not followedURLsObjectIDs:
continue
print '\t Unfollowing', len(followedURLsObjectIDs), 'URLs'
count += len(followedURLsObjectIDs)
TagValueAPI(user).delete([(objectID, followTag)
for objectID in followedURLsObjectIDs])
store.commit()
if count > 50000:
print '** Sleeping one minute. Giving a breath to the DIH. **'
time.sleep(60)
count = 0
print 'Done.'
from fluiddb.model.user import getUser
from fluiddb.model.value import TagValueAPI
from fluiddb.scripts.commands import setupStore
if __name__ == '__main__':
store = setupStore('postgres:///fluidinfo', 'main')
setConfig(setupConfig(None))
BATCH_SIZE = 50
print __doc__
superUser = getUser(u'fluiddb')
tagValues = TagValueAPI(superUser)
print 'Getting tags to fix.'
tagsToFix = []
for tagID, path in getTags().values(Tag.id, Tag.path):
root = path.split('/', 1)[0]
if not root.islower():
tagsToFix.append(tagID)
print 'Getting namespaces to fix.'
namespacesToFix = []
for namespaceID, path in getNamespaces().values(Namespace.id,
Namespace.path):
root = path.split('/', 1)[0]
if not root.islower():
namespacesToFix.append(namespaceID)
def testGetForObjectsReturnsOnlyAllowedTags(self):
"""
L{SecureRecentActivityAPI.getForObjects} will return all the tags for
which the user has C{Operation.READ_TAG_VALUE} permissions, but not
those for which the user doesn't have.
"""
tagValues = TagValueAPI(self.user)
objectID1 = ObjectAPI(self.user).create(u'object1')
objectID2 = uuid4()
# Use commit() frequently to have different timestamps on each value.
self.store.commit()
tagValues.set({objectID1: {u'user/tag1': u'A'}})
self.store.commit()
tagValues.set({objectID1: {u'user/tag2': u'B'}})
self.store.commit()
tagValues.set({objectID2: {u'user/tag1': u'C'}})
self.store.commit()
tagValues.set({objectID2: {u'user/tag2': u'D'}})
self.store.commit()
tagValues.set({uuid4(): {u'user/tag1': u'E'}})
self.store.commit()
tagValues.set({uuid4(): {u'user/tag2': u'F'}})
self.store.commit()
self.permissions.set([(u'user/tag2', Operation.READ_TAG_VALUE,
Policy.OPEN, [u'anon'])])
expected = [(u'user/tag1', objectID2, None, u'C', u'user'),
(u'user/tag1', objectID1, u'object1', u'A', u'user'),
(u'fluiddb/about', objectID1, u'object1', u'object1',
u'fluiddb')]
result = self.recentActivity.getForObjects([objectID1, objectID2])
# Remove the creation times from the result, with the order is enough.
result = [(path, objectID, about, value, username)
for path, objectID, about, value, username, time in result]
self.assertEqual(expected, result)
def testGetForUsersReturnsOnlyAllowedTags(self):
"""
L{SecureRecentActivityAPI.getForUser} returns all the tags for the
superuser.
"""
tagValues = TagValueAPI(self.user)
objectID1 = ObjectAPI(self.user).create(u'object1')
objectID2 = uuid4()
# Use commit() frequently to have different timestamps on each value.
self.store.commit()
tagValues.set({objectID1: {u'user/tag1': u'A'}})
self.store.commit()
tagValues.set({objectID1: {u'user/tag2': u'B'}})
self.store.commit()
UserAPI().create([(u'user2', u'secret', u'User', u'*****@*****.**')])
tagValues = TagValueAPI(getUser(u'user2'))
tagValues.set({objectID1: {u'user2/tag1': u'C'}})
self.store.commit()
tagValues.set({objectID2: {u'user2/tag2': u'D'}})
self.store.commit()
UserAPI().create([(u'user3', u'secret', u'User', u'*****@*****.**')])
tagValues = TagValueAPI(getUser(u'user3'))
tagValues.set({objectID1: {u'user3/tag1': u'C'}})
self.store.commit()
tagValues.set({objectID2: {u'user3/tag2': u'D'}})
self.store.commit()
self.permissions.set([
(u'user/tag2', Operation.READ_TAG_VALUE, Policy.OPEN, [u'user']),
(u'user2/tag2', Operation.READ_TAG_VALUE, Policy.CLOSED, [])
])
expected = [(u'user2/tag2', objectID2, None, u'D', u'user2'),
(u'user2/tag1', objectID1, u'object1', u'C', u'user2'),
(u'user/tag2', objectID1, u'object1', u'B', u'user'),
(u'user/tag1', objectID1, u'object1', u'A', u'user')]
result = self.recentActivity.getForUsers([u'user', u'user2'])
# Remove the creation times from the result, with the order is enough.
result = [(path, objectID, about, value, username)
for path, objectID, about, value, username, time in result]
self.assertEqual(expected, result)
def testGetRecentAboutActivity(self):
"""
L{FacadeRecentActivityMixin.getRecentAboutActivity} returns a C{dict}
with information about the recent tag values on the given object.
"""
tagValues = TagValueAPI(self.user)
objectID1 = ObjectAPI(self.user).create(u'object1')
objectID2 = uuid4()
# Use commit() frequently to have different timestamps on each value.
self.store.commit()
tagValues.set({objectID1: {u'user/tag1': u'A'}})
self.store.commit()
tagValues.set({objectID1: {u'user/tag2': u'B'}})
self.store.commit()
tagValues.set({objectID2: {u'user/tag1': u'C'}})
self.store.commit()
tagValues.set({objectID2: {u'user/tag2': u'D'}})
self.store.commit()
tagValues.set({uuid4(): {u'user/tag1': u'E'}})
self.store.commit()
tagValues.set({uuid4(): {u'user/tag2': u'F'}})
self.store.commit()
expected = [
{'tag': u'user/tag2',
'id': str(objectID1),
'about': u'object1',
'value': u'B',
'username': u'username'},
{'tag': u'user/tag1',
'id': str(objectID1),
'about': u'object1',
'value': u'A',
'username': u'username'},
{'tag': u'fluiddb/about',
'id': str(objectID1),
'about': u'object1',
'value': u'object1',
'username': u'fluiddb'}]
with login(self.user.username, uuid4(), self.transact) as session:
result = yield self.facade.getRecentAboutActivity(
session, 'object1')
# Remove the creation times from the result.
for item in result:
del item['updated-at']
self.assertEqual(expected, result)
