def testMissingEmailValue(self):
"""
L{UserIntegrityChecker.check} logs an error if the given L{User}
doesn't have a C{fluiddb/users/email} value.
"""
user = self.createTestUser(u'user')
getTagValues([(user.objectID, self.emailTag.id)]).remove()
self.checker.check([user])
self.assertEqual(
"Integrity Error in user u'user': "
'Email tag is missing.\n', self.log.getvalue())
def testMissingEmailValue(self):
"""
L{UserIntegrityChecker.check} logs an error if the given L{User}
doesn't have a C{fluiddb/users/email} value.
"""
user = self.createTestUser(u'user')
getTagValues([(user.objectID, self.emailTag.id)]).remove()
self.checker.check([user])
self.assertEqual("Integrity Error in user u'user': "
'Email tag is missing.\n',
self.log.getvalue())
def testGetForUsersUpdatesTheCache(self):
"""
L{CachingRecentActivityAPI.getForUsers} adds cache misses pulled
from the database to the cache.
"""
tagValues = self.getTagValueAPI(self.user)
objectID = self.getObjectAPI(self.user).create(u'object')
tagValues.set({objectID: {u'user/tag': u'A'}})
databaseResult = self.recentActivity.getForUsers([u'user'])
# Remove all the tag values to ensure that the values for the next
# request are taken from the cache.
getTagValues().remove()
cachedResult = self.recentActivity.getForUsers([u'user'])
self.assertEqual(databaseResult, cachedResult)
def testGetForUsersUpdatesTheCache(self):
"""
L{CachingRecentActivityAPI.getForUsers} adds cache misses pulled
from the database to the cache.
"""
tagValues = self.getTagValueAPI(self.user)
objectID = self.getObjectAPI(self.user).create(u'object')
tagValues.set({objectID: {u'user/tag': u'A'}})
databaseResult = self.recentActivity.getForUsers([u'user'])
# Remove all the tag values to ensure that the values for the next
# request are taken from the cache.
getTagValues().remove()
cachedResult = self.recentActivity.getForUsers([u'user'])
self.assertEqual(databaseResult, cachedResult)
def testSet(self):
"""L{TagAPI.set} updates the description for the specified L{Tag}s."""
descriptionTag = self.system.tags[u'fluiddb/tags/description']
tag = createTag(self.user, self.user.namespace, u'tag')
createTagPermission(tag)
self.tags.set({u'username/tag': u'A fancy new description.'})
result = getTagValues([(tag.objectID, descriptionTag.id)])
description = result.one()
self.assertEqual(u'A fancy new description.', description.value)
def testCreateCreatesAboutTag(self):
"""
L{TagAPI.create} creates new C{fluiddb/about} L{TagValue}s when
creating new L{Tag}s.
"""
values = [(u'username/tag', u'A tag description')]
[(objectID, path)] = self.tags.create(values)
tag = getTags(paths=[u'fluiddb/about']).one()
value = getTagValues([(objectID, tag.id)]).one()
self.assertEqual(u'Object for the attribute username/tag', value.value)
def testBatchIndexTouchesAllObjects(self):
"""C{batchIndex} touches all objects in the given file."""
createTagValue(self.userID, self.tagID, uuid4(), 10)
createTagValue(self.userID, self.tagID, uuid4(), 20)
allObjects = set(getTagValues().values(TagValue.objectID))
self.createObjectsFile()
batchIndex(self.objectsFilename, 0, 10)
touchedObjects = set(getDirtyObjects().values(DirtyObject.objectID))
self.assertEqual(allObjects, touchedObjects)
def testCreateCreatesRoleTags(self):
"""
L{UserAPI.create} creates the C{fluiddb/users/role} tag value for
the created user.
"""
values = [(u'username', u'secret', u'Full Name', u'*****@*****.**')]
[(objectID, username)] = self.users.create(values)
tagID = self.system.tags[u'fluiddb/users/role'].id
value = getTagValues([(objectID, tagID)]).one()
self.assertEqual(u'USER', value.value)
def testCreateCreatesAboutTag(self):
"""
L{UserAPI.create} creates new C{fluiddb/about} L{TagValue}s when
creating new L{User}s.
"""
values = [(u'username', u'secret', u'User', u'*****@*****.**')]
[(objectID, username)] = self.users.create(values)
tag = getTags(paths=[u'fluiddb/about']).one()
value = getTagValues([(objectID, tag.id)]).one()
self.assertEqual(u'@username', value.value)
def testCreateCreatesAboutTag(self):
"""
L{UserAPI.create} creates new C{fluiddb/about} L{TagValue}s when
creating new L{User}s.
"""
values = [(u'username', u'secret', u'User', u'*****@*****.**')]
[(objectID, username)] = self.users.create(values)
tag = getTags(paths=[u'fluiddb/about']).one()
value = getTagValues([(objectID, tag.id)]).one()
self.assertEqual(u'@username', value.value)
def testCreateStoresTagPaths(self):
"""
L{TagAPI.create} creates new C{fluiddb/tags/path} L{TagValue}s to store
the specified L{Tag} paths.
"""
values = [(u'username/tag', u'A tag description')]
[(objectID, path)] = self.tags.create(values)
tag = getTags(paths=[u'fluiddb/tags/path']).one()
value = getTagValues([(objectID, tag.id)]).one()
self.assertEqual(u'username/tag', value.value)
def testCreateStoresNamespaceDescriptions(self):
"""
L{NamespaceAPI.create} creates new C{fluiddb/namespaces/description}
L{TagValue}s to store the specified L{Namespace} descriptions.
"""
values = [(u'username/namespace', u'A namespace description')]
[(objectID, path)] = self.namespaces.create(values)
tag = getTags(paths=[u'fluiddb/namespaces/description']).one()
value = getTagValues([(objectID, tag.id)]).one()
self.assertEqual(u'A namespace description', value.value)
def testCreateCreatesRoleTags(self):
"""
L{UserAPI.create} creates the C{fluiddb/users/role} tag value for
the created user.
"""
values = [(u'username', u'secret', u'Full Name', u'*****@*****.**')]
[(objectID, username)] = self.users.create(values)
tagID = self.system.tags[u'fluiddb/users/role'].id
value = getTagValues([(objectID, tagID)]).one()
self.assertEqual(u'USER', value.value)
def testCreate(self):
"""
L{ObjectAPI.create} creates new L{AboutTagValue} owned by the super
user and returns the UUID for the newly created object.
"""
objectID = self.objects.create(u'A fancy about tag value')
tag = self.system.tags[u'fluiddb/about']
about = getTagValues([(objectID, tag.id)]).one()
self.assertEqual(u'A fancy about tag value', about.value)
self.assertEqual(self.system.users[u'fluiddb'].id, about.creatorID)
def testCreate(self):
"""
L{ObjectAPI.create} creates new L{AboutTagValue} owned by the super
user and returns the UUID for the newly created object.
"""
objectID = self.objects.create(u'A fancy about tag value')
tag = self.system.tags[u'fluiddb/about']
about = getTagValues([(objectID, tag.id)]).one()
self.assertEqual(u'A fancy about tag value', about.value)
self.assertEqual(self.system.users[u'fluiddb'].id, about.creatorID)
def testCreateStoresNamespacePaths(self):
"""
L{NamespaceAPI.create} creates new C{fluiddb/namespaces/path}
L{TagValue}s to store the specified L{Namespace} paths.
"""
values = [(u'username/namespace', u'A namespace description')]
[(objectID, path)] = self.namespaces.create(values)
tag = getTags(paths=[u'fluiddb/namespaces/path']).one()
value = getTagValues([(objectID, tag.id)]).one()
self.assertEqual(u'username/namespace', value.value)
def testBatchIndexTouchesAllObjects(self):
"""C{batchIndex} touches all objects in the given file."""
createTagValue(self.userID, self.tagID, uuid4(), 10)
createTagValue(self.userID, self.tagID, uuid4(), 20)
allObjects = set(getTagValues().values(TagValue.objectID))
self.createObjectsFile()
batchIndex(self.objectsFilename, 0, 10)
touchedObjects = set(getDirtyObjects().values(DirtyObject.objectID))
self.assertEqual(allObjects, touchedObjects)
def testSet(self):
"""
L{NamespaceAPI.set} updates the description for the specified
L{Namespace}s.
"""
descriptionTag = self.system.tags[u'fluiddb/namespaces/description']
self.namespaces.set({u'username': u'A fancy new description.'})
result = getTagValues([(self.user.namespace.objectID,
descriptionTag.id)])
description = result.one()
self.assertEqual(u'A fancy new description.', description.value)
def _getValues(self, objects, path):
"""Get all L{TagValue}s for the given objects and the given path.
@param objects: A sequence of L{Tag}s, L{Namespace}s or L{User}s.
@param path: The path of the L{Tag} to get the vaues from.
@return: A C{dict} mapping object IDs to values.
"""
tag = self._getSystemTag(path)
result = getTagValues([(object.objectID, tag.id)
for object in objects])
return dict(result.values(TagValue.objectID, TagValue.value))
def testDeleteKeepsTheAboutTag(self):
"""
L{TagAPI.delete} keeps the C{fluiddb/about} tag value for the deleted
tag.
"""
values = [(u'username/tag', u'A tag description')]
[(objectID, path)] = self.tags.create(values)
self.tags.delete([u'username/tag'])
tag = getTags(paths=[u'fluiddb/about']).one()
value = getTagValues([(objectID, tag.id)]).one()
self.assertEqual(u'Object for the attribute username/tag', value.value)
def testSet(self):
"""
L{NamespaceAPI.set} updates the description for the specified
L{Namespace}s.
"""
descriptionTag = self.system.tags[u'fluiddb/namespaces/description']
self.namespaces.set({u'username': u'A fancy new description.'})
result = getTagValues(
[(self.user.namespace.objectID, descriptionTag.id)])
description = result.one()
self.assertEqual(u'A fancy new description.', description.value)
def _getValues(self, objects, path):
"""Get all L{TagValue}s for the given objects and the given path.
@param objects: A sequence of L{Tag}s, L{Namespace}s or L{User}s.
@param path: The path of the L{Tag} to get the vaues from.
@return: A C{dict} mapping object IDs to values.
"""
tag = self._getSystemTag(path)
result = getTagValues([(object.objectID, tag.id)
for object in objects])
return dict(result.values(TagValue.objectID, TagValue.value))
def testCreateCreatesAboutTag(self):
"""
L{NamespaceAPI.create} creates new C{fluiddb/about} L{TagValue}s when
creating new L{Namespace}s.
"""
values = [(u'username/namespace', u'A namespace description')]
[(objectID, path)] = self.namespaces.create(values)
tag = getTags(paths=[u'fluiddb/about']).one()
value = getTagValues([(objectID, tag.id)]).one()
self.assertEqual(u'Object for the namespace username/namespace',
value.value)
def testWrongNameValue(self):
"""
L{UserIntegrityChecker.check} logs an error if the given L{User}
has a wrong C{fluiddb/users/name} value.
"""
user = self.createTestUser(u'user')
name = getTagValues([(user.objectID, self.nameTag.id)]).one()
name.value = u'Wrong Name'
self.checker.check([user])
self.assertEqual("Integrity Error in user u'user': "
'Name tag is incorrect.\n',
self.log.getvalue())
def testWrongAboutValue(self):
"""
L{UserIntegrityChecker.check} logs an error if the given L{User}
has a wrong C{fluiddb/about} value.
"""
user = self.createTestUser(u'user')
about = getTagValues([(user.objectID, self.aboutTag.id)]).one()
about.value = u'Wrong about value.'
self.checker.check([user])
self.assertEqual(
"Integrity Error in user u'user': "
'About tag is incorrect.\n', self.log.getvalue())
def testWrongEmailValue(self):
"""
L{UserIntegrityChecker.check} logs an error if the given L{User}
has a wrong C{fluiddb/users/email} value.
"""
user = self.createTestUser(u'user')
email = getTagValues([(user.objectID, self.emailTag.id)]).one()
email.value = u'*****@*****.**'
self.checker.check([user])
self.assertEqual("Integrity Error in user u'user': "
'Email tag is incorrect.\n',
self.log.getvalue())
def testWrongNameValue(self):
"""
L{UserIntegrityChecker.check} logs an error if the given L{User}
has a wrong C{fluiddb/users/name} value.
"""
user = self.createTestUser(u'user')
name = getTagValues([(user.objectID, self.nameTag.id)]).one()
name.value = u'Wrong Name'
self.checker.check([user])
self.assertEqual(
"Integrity Error in user u'user': "
'Name tag is incorrect.\n', self.log.getvalue())
def testDeleteKeepsTheAboutTag(self):
"""
L{NamespaceAPI.delete} keeps the C{fluiddb/about} tag value for the
deleted namespace.
"""
values = [(u'username/namespace', u'A namespace description')]
[(objectID, path)] = self.namespaces.create(values)
self.namespaces.delete([u'username/namespace'])
tag = getTags(paths=[u'fluiddb/about']).one()
value = getTagValues([(objectID, tag.id)]).one()
self.assertEqual(u'Object for the namespace username/namespace',
value.value)
def testWrongEmailValue(self):
"""
L{UserIntegrityChecker.check} logs an error if the given L{User}
has a wrong C{fluiddb/users/email} value.
"""
user = self.createTestUser(u'user')
email = getTagValues([(user.objectID, self.emailTag.id)]).one()
email.value = u'*****@*****.**'
self.checker.check([user])
self.assertEqual(
"Integrity Error in user u'user': "
'Email tag is incorrect.\n', self.log.getvalue())
def testDeleteKeepsTheAboutTag(self):
"""
L{NamespaceAPI.delete} keeps the C{fluiddb/about} tag value for the
deleted namespace.
"""
values = [(u'username/namespace', u'A namespace description')]
[(objectID, path)] = self.namespaces.create(values)
self.namespaces.delete([u'username/namespace'])
tag = getTags(paths=[u'fluiddb/about']).one()
value = getTagValues([(objectID, tag.id)]).one()
self.assertEqual(u'Object for the namespace username/namespace',
value.value)
def testWrongAboutValue(self):
"""
L{UserIntegrityChecker.check} logs an error if the given L{User}
has a wrong C{fluiddb/about} value.
"""
user = self.createTestUser(u'user')
about = getTagValues([(user.objectID, self.aboutTag.id)]).one()
about.value = u'Wrong about value.'
self.checker.check([user])
self.assertEqual("Integrity Error in user u'user': "
'About tag is incorrect.\n',
self.log.getvalue())
def testSetUpdates(self):
"""L{TagValueAPI.set} updates an existing value with a new one."""
objectID = uuid4()
namespace = createNamespace(self.user, u'name')
createNamespacePermission(namespace)
tag = createTag(self.user, namespace, u'tag')
createTagPermission(tag)
self.tagValues.set({objectID: {u'name/tag': 5}})
self.tagValues.set({objectID: {u'name/tag': None}})
value = getTagValues([(objectID, tag.id)]).one()
self.assertEqual(tag.id, value.tagID)
self.assertEqual(objectID, value.objectID)
self.assertIn(objectID, getDirtyObjects().values(DirtyObject.objectID))
def testGetTagValues(self):
"""
L{getTagValues} returns all L{TagValue}s in the database, by default.
"""
objectID1 = uuid4()
objectID2 = uuid4()
user = createUser(u'username', u'password', u'User',
u'*****@*****.**')
user.namespaceID = createNamespace(user, user.username, None).id
tag = createTag(user, user.namespace, u'name')
value1 = self.store.add(TagValue(user.id, tag.id, objectID1, None))
value2 = self.store.add(TagValue(user.id, tag.id, objectID2, 42))
self.assertEqual(sorted([value1, value2]), sorted(getTagValues()))
def testSetUpdates(self):
"""L{TagValueAPI.set} updates an existing value with a new one."""
objectID = uuid4()
namespace = createNamespace(self.user, u'name')
createNamespacePermission(namespace)
tag = createTag(self.user, namespace, u'tag')
createTagPermission(tag)
self.tagValues.set({objectID: {u'name/tag': 5}})
self.tagValues.set({objectID: {u'name/tag': None}})
value = getTagValues([(objectID, tag.id)]).one()
self.assertEqual(tag.id, value.tagID)
self.assertEqual(objectID, value.objectID)
self.assertIn(objectID, getDirtyObjects().values(DirtyObject.objectID))
def testSet(self):
"""L{TagValueAPI.set} stores new L{TagValue}s."""
namespace = createNamespace(self.user, u'name')
createNamespacePermission(namespace)
tag = createTag(self.user, namespace, u'tag')
createTagPermission(tag)
objectID = uuid4()
values = {objectID: {u'name/tag': 42}}
self.tagValues.set(values)
value = getTagValues([(objectID, tag.id)]).one()
self.assertEqual(tag.id, value.tagID)
self.assertEqual(objectID, value.objectID)
self.assertIn(objectID, getDirtyObjects().values(DirtyObject.objectID))
def testSet(self):
"""L{TagValueAPI.set} stores new L{TagValue}s."""
namespace = createNamespace(self.user, u'name')
createNamespacePermission(namespace)
tag = createTag(self.user, namespace, u'tag')
createTagPermission(tag)
objectID = uuid4()
values = {objectID: {u'name/tag': 42}}
self.tagValues.set(values)
value = getTagValues([(objectID, tag.id)]).one()
self.assertEqual(tag.id, value.tagID)
self.assertEqual(objectID, value.objectID)
self.assertIn(objectID, getDirtyObjects().values(DirtyObject.objectID))
def testGetTagValuesWithTagIDsAndObjectIDs(self):
"""
When C{(Tag.id, object ID)} 2-tuples are provided L{getTagValues}
returns matching L{TagValue}s.
"""
objectID1 = uuid4()
objectID2 = uuid4()
user = createUser(u'username', u'password', u'User',
u'*****@*****.**')
user.namespaceID = createNamespace(user, user.username, None).id
tag = createTag(user, user.namespace, u'name')
value = self.store.add(TagValue(user.id, tag.id, objectID1, None))
self.store.add(TagValue(user.id, tag.id, objectID2, 42))
self.assertEqual(value, getTagValues([(objectID1, tag.id)]).one())
def testEmailValueWithoutUser(self):
"""
L{TagValueIntegrityChecker.check} logs an error if a given L{TagValue}
for fluiddb/users/email doesn't have an associated L{User} row.
"""
tag = self.system.tags[u'fluiddb/users/email']
tagValue1 = createTagValue(self.superuser.id, tag.id, uuid4(),
u'*****@*****.**')
tagValue2 = getTagValues([(self.superuser.objectID, tag.id)]).one()
self.checker.check([tagValue1, tagValue2])
self.assertEqual('Integrity Error in object %s: '
"fluiddb/users/email TagValue doesn't have "
'an associated User.\n' % tagValue1.objectID,
self.log.getvalue())
def testEmailValueWithoutUser(self):
"""
L{TagValueIntegrityChecker.check} logs an error if a given L{TagValue}
for fluiddb/users/email doesn't have an associated L{User} row.
"""
tag = self.system.tags[u'fluiddb/users/email']
tagValue1 = createTagValue(self.superuser.id, tag.id, uuid4(),
u'*****@*****.**')
tagValue2 = getTagValues([(self.superuser.objectID, tag.id)]).one()
self.checker.check([tagValue1, tagValue2])
self.assertEqual(
'Integrity Error in object %s: '
"fluiddb/users/email TagValue doesn't have "
'an associated User.\n' % tagValue1.objectID, self.log.getvalue())
def testBatchIndexLogsErrorIfObjectIDIsNotWellFormed(self):
"""
If C{batchIndex} encounters a malformed objectID in the file it will
continue the process after printing an error in the logs.
"""
createTagValue(self.userID, self.tagID, uuid4(), 10)
createTagValue(self.userID, self.tagID, uuid4(), 20)
allObjects = set(getTagValues().values(TagValue.objectID))
self.createObjectsFile()
with open(self.objectsFilename, 'a') as objectsFilename:
objectsFilename.write('wrong-id')
batchIndex(self.objectsFilename, 0, 10)
touchedObjects = set(getDirtyObjects().values(DirtyObject.objectID))
self.assertEqual(allObjects, touchedObjects)
self.assertIn("Invalid objectID: 'wrong-id'", self.log.getvalue())
def testBatchIndexLogsErrorIfObjectIDIsNotWellFormed(self):
"""
If C{batchIndex} encounters a malformed objectID in the file it will
continue the process after printing an error in the logs.
"""
createTagValue(self.userID, self.tagID, uuid4(), 10)
createTagValue(self.userID, self.tagID, uuid4(), 20)
allObjects = set(getTagValues().values(TagValue.objectID))
self.createObjectsFile()
with open(self.objectsFilename, 'a') as objectsFilename:
objectsFilename.write('wrong-id')
batchIndex(self.objectsFilename, 0, 10)
touchedObjects = set(getDirtyObjects().values(DirtyObject.objectID))
self.assertEqual(allObjects, touchedObjects)
self.assertIn("Invalid objectID: 'wrong-id'", self.log.getvalue())
def testCreateObjectWithAboutValue(self):
"""
L{FacadeObjectAPI.createObject} always returns a valid C{UUID} in a
C{str} for a new object ID if an about value is given and it doesn't
already exist.
"""
self.store.commit()
with login(self.user.username, uuid4(), self.transact) as session:
objectID = yield self.facade.createObject(session, about='foo')
objectID = UUID(objectID)
self.store.rollback()
value = getAboutTagValues([objectID], [u'foo']).one()
self.assertEqual(u'foo', value.value)
tag = self.system.tags[u'fluiddb/about']
value = getTagValues(values=[(objectID, tag.id)]).one()
self.assertIdentical(self.system.users[u'fluiddb'], value.creator)
def testCreateObjectWithAboutValue(self):
"""
L{FacadeObjectAPI.createObject} always returns a valid C{UUID} in a
C{str} for a new object ID if an about value is given and it doesn't
already exist.
"""
self.store.commit()
with login(self.user.username, uuid4(), self.transact) as session:
objectID = yield self.facade.createObject(session, about='foo')
objectID = UUID(objectID)
self.store.rollback()
value = getAboutTagValues([objectID], [u'foo']).one()
self.assertEqual(u'foo', value.value)
tag = self.system.tags[u'fluiddb/about']
value = getTagValues(values=[(objectID, tag.id)]).one()
self.assertIdentical(self.system.users[u'fluiddb'], value.creator)
def testSetBinaryValue(self):
"""
L{TagValueAPI.set} can store binary L{TagValue}s. The contents
included in the dict representing the value are written to a file and
added to the L{FileStore}.
"""
namespace = createNamespace(self.user, u'name')
createNamespacePermission(namespace)
tag = createTag(self.user, namespace, u'tag')
createTagPermission(tag)
objectID = uuid4()
self.tagValues.set(
{objectID: {u'name/tag': {'mime-type': 'text/plain',
'contents': 'Hello, world!'}}})
value = getTagValues([(objectID, tag.id)]).one()
self.assertEqual(tag.id, value.tagID)
self.assertEqual(objectID, value.objectID)
self.assertEqual({'mime-type': 'text/plain',
'size': 13},
value.value)
self.assertIn(objectID, getDirtyObjects().values(DirtyObject.objectID))
def delete(self, values):
"""Delete L{TagValue}s.
@param values: A sequence of C{(objectID, Tag.path)} 2-tuples to
delete values for.
@raise FeatureError: Raised if the given list of values is empty.
@return: The number of values deleted.
"""
if isgenerator(values):
values = list(values)
if not values:
raise FeatureError("Can't delete an empty list of tag values.")
paths = set([path for objectID, path in values])
objectIDs = set([objectID for objectID, path in values])
tagIDs = dict(getTags(paths).values(Tag.path, Tag.id))
values = [(objectID, tagIDs[path]) for objectID, path in values]
result = getTagValues(values).remove()
if result:
touchObjects(objectIDs)
return result
def delete(self, values):
"""Delete L{TagValue}s.
@param values: A sequence of C{(objectID, Tag.path)} 2-tuples to
delete values for.
@raise FeatureError: Raised if the given list of values is empty.
@return: The number of values deleted.
"""
if isgenerator(values):
values = list(values)
if not values:
raise FeatureError("Can't delete an empty list of tag values.")
paths = set([path for objectID, path in values])
objectIDs = set([objectID for objectID, path in values])
tagIDs = dict(getTags(paths).values(Tag.path, Tag.id))
values = [(objectID, tagIDs[path]) for objectID, path in values]
result = getTagValues(values).remove()
if result:
touchObjects(objectIDs)
return result
def checkIntegrity(maxRowsPerQuery=10000):
"""
Check the integrity of the database for cases which the database
engine can't detect.
@param maxRowsPerQuery: Limit the number of rows fetched by SQL queries to
avoid excessive use of memory.
"""
results = _splitResult(getNamespaces(), Namespace.id, maxRowsPerQuery)
for result in results:
namespaces = list(result)
NamespaceIntegrityChecker().check(namespaces)
results = _splitResult(getTags(), Tag.id, maxRowsPerQuery)
for result in results:
tags = list(result)
TagIntegrityChecker().check(tags)
results = _splitResult(getUsers(), User.id, maxRowsPerQuery)
for result in results:
users = list(result)
UserIntegrityChecker().check(users)
results = _splitResult(getAboutTagValues(), AboutTagValue.objectID,
maxRowsPerQuery)
for result in results:
aboutTagValues = list(result)
AboutTagValueIntegrityChecker().check(aboutTagValues)
# In the case of TagValues we limit the query to only tag paths starting
# with "fluiddb" because these are the only ones we're checking and we
# don't want a huge result.
store = getMainStore()
result = store.find(TagValue,
TagValue.tagID == Tag.id,
Tag.path.startswith(u'fluiddb/'))
results = _splitResult(getTagValues(), TagValue.id, maxRowsPerQuery)
for result in results:
tagValues = list(result)
TagValueIntegrityChecker().check(tagValues)
def checkIntegrity(maxRowsPerQuery=10000):
"""
Check the integrity of the database for cases which the database
engine can't detect.
@param maxRowsPerQuery: Limit the number of rows fetched by SQL queries to
avoid excessive use of memory.
"""
results = _splitResult(getNamespaces(), Namespace.id, maxRowsPerQuery)
for result in results:
namespaces = list(result)
NamespaceIntegrityChecker().check(namespaces)
results = _splitResult(getTags(), Tag.id, maxRowsPerQuery)
for result in results:
tags = list(result)
TagIntegrityChecker().check(tags)
results = _splitResult(getUsers(), User.id, maxRowsPerQuery)
for result in results:
users = list(result)
UserIntegrityChecker().check(users)
results = _splitResult(getAboutTagValues(), AboutTagValue.objectID,
maxRowsPerQuery)
for result in results:
aboutTagValues = list(result)
AboutTagValueIntegrityChecker().check(aboutTagValues)
# In the case of TagValues we limit the query to only tag paths starting
# with "fluiddb" because these are the only ones we're checking and we
# don't want a huge result.
store = getMainStore()
result = store.find(TagValue, TagValue.tagID == Tag.id,
Tag.path.startswith(u'fluiddb/'))
results = _splitResult(getTagValues(), TagValue.id, maxRowsPerQuery)
for result in results:
tagValues = list(result)
TagValueIntegrityChecker().check(tagValues)
def testSetBinaryValue(self):
"""
L{TagValueAPI.set} can store binary L{TagValue}s. The contents
included in the dict representing the value are written to a file and
added to the L{FileStore}.
"""
namespace = createNamespace(self.user, u'name')
createNamespacePermission(namespace)
tag = createTag(self.user, namespace, u'tag')
createTagPermission(tag)
objectID = uuid4()
self.tagValues.set({
objectID: {
u'name/tag': {
'mime-type': 'text/plain',
'contents': 'Hello, world!'
}
}
})
value = getTagValues([(objectID, tag.id)]).one()
self.assertEqual(tag.id, value.tagID)
self.assertEqual(objectID, value.objectID)
self.assertEqual({'mime-type': 'text/plain', 'size': 13}, value.value)
self.assertIn(objectID, getDirtyObjects().values(DirtyObject.objectID))
def createObjectsFile(self):
"""Helper function to create a file with a list of all object IDs."""
allObjects = set(getTagValues().values(TagValue.objectID))
with open(self.objectsFilename, 'w') as objectsFile:
for objectID in allObjects:
objectsFile.write(str(objectID) + '\n')
def set(self, values):
"""Set or update L{TagValue}s.
L{Tag}s that don't exist are created automatically before L{TagValue}s
are stored. Associated L{TagPermission}s are created automatically
with the system-wide default permissions.
@param values: A C{dict} mapping object IDs to tags and values,
matching the following format::
{<object-id>: {<path>: <value>,
<path>: {'mime-type': <mime-type>,
'contents': <contents>}}}
A binary L{TagValue} is represented using a different layout than
other values types, as shown for the second value.
@raise FeatureError: Raised if the given list of values is empty.
@raise MalformedPathError: Raised if one of the given paths for a
nonexistent tag is empty or has unacceptable characters.
"""
if not values:
raise FeatureError("Can't set an empty list of tag values.")
objectIDs = set(values.keys())
# Implicitly create missing tags, if there are any.
paths = set()
for tagValues in values.itervalues():
paths.update(tagValues.iterkeys())
tagIDs = dict(getTags(paths=paths).values(Tag.path, Tag.id))
existingPaths = set(tagIDs.iterkeys())
unknownPaths = paths - existingPaths
if unknownPaths:
tags = [(path, u'Object for the attribute %s' % path)
for path in unknownPaths]
self._factory.tags(self._user).create(tags)
tagIDs = dict(getTags(paths=paths).values(Tag.path, Tag.id))
# Delete all existing tag values for the specified object IDs and
# paths.
deleteValues = []
for objectID in values:
for path in values[objectID].iterkeys():
deleteValues.append((objectID, tagIDs[path]))
getTagValues(deleteValues).remove()
# Set new tag values for the specified object IDs and paths.
for objectID in values:
tagValues = values[objectID]
for path, value in tagValues.iteritems():
tagID = tagIDs[path]
if isinstance(value, dict):
content = value['contents']
value = createTagValue(self._user.id, tagID, objectID,
{'mime-type': value['mime-type'],
'size': len(content)})
# This is necessary to tell PostgreSQL that generates a
# `value.id` immediately.
value.id = AutoReload
createOpaqueValue(value.id, content)
else:
createTagValue(self._user.id, tagID, objectID, value)
touchObjects(objectIDs)
def set(self, values):
"""Set or update L{TagValue}s.
L{Tag}s that don't exist are created automatically before L{TagValue}s
are stored. Associated L{TagPermission}s are created automatically
with the system-wide default permissions.
@param values: A C{dict} mapping object IDs to tags and values,
matching the following format::
{<object-id>: {<path>: <value>,
<path>: {'mime-type': <mime-type>,
'contents': <contents>}}}
A binary L{TagValue} is represented using a different layout than
other values types, as shown for the second value.
@raise FeatureError: Raised if the given list of values is empty.
@raise MalformedPathError: Raised if one of the given paths for a
nonexistent tag is empty or has unacceptable characters.
"""
if not values:
raise FeatureError("Can't set an empty list of tag values.")
objectIDs = set(values.keys())
# Implicitly create missing tags, if there are any.
paths = set()
for tagValues in values.itervalues():
paths.update(tagValues.iterkeys())
tagIDs = dict(getTags(paths=paths).values(Tag.path, Tag.id))
existingPaths = set(tagIDs.iterkeys())
unknownPaths = paths - existingPaths
if unknownPaths:
tags = [(path, u'Object for the attribute %s' % path)
for path in unknownPaths]
self._factory.tags(self._user).create(tags)
tagIDs = dict(getTags(paths=paths).values(Tag.path, Tag.id))
# Delete all existing tag values for the specified object IDs and
# paths.
deleteValues = []
for objectID in values:
for path in values[objectID].iterkeys():
deleteValues.append((objectID, tagIDs[path]))
getTagValues(deleteValues).remove()
# Set new tag values for the specified object IDs and paths.
for objectID in values:
tagValues = values[objectID]
for path, value in tagValues.iteritems():
tagID = tagIDs[path]
if isinstance(value, dict):
content = value['contents']
value = createTagValue(self._user.id, tagID, objectID, {
'mime-type': value['mime-type'],
'size': len(content)
})
# This is necessary to tell PostgreSQL that generates a
# `value.id` immediately.
value.id = AutoReload
createOpaqueValue(value.id, content)
else:
createTagValue(self._user.id, tagID, objectID, value)
touchObjects(objectIDs)
def createObjectsFile(self):
"""Helper function to create a file with a list of all object IDs."""
allObjects = set(getTagValues().values(TagValue.objectID))
with open(self.objectsFilename, 'w') as objectsFile:
for objectID in allObjects:
objectsFile.write(str(objectID) + '\n')
superUser = getUsers(usernames=[u'fluiddb']).one()
result = store.find(AboutTagValue, AboutTagValue.value.like(u'@%'))
for aboutValue in result:
if aboutValue.value == aboutValue.value.lower():
continue
print 'Migrating mixed cased', aboutValue.value.encode('utf-8')
newAbout = u'@%s' % aboutValue.value.lower()
oldObjectID = aboutValue.objectID
newObjectID = ObjectAPI(superUser).create(newAbout)
result = store.find(TagValue,
TagValue.objectID == oldObjectID,
TagValue.tagID != aboutTag.id)
for tagValue in result:
existingValue = getTagValues([(newObjectID, tagValue.tagID)]).one()
if existingValue is not None:
error = ('ERROR: Cannot migrate value {path} on {about} '
'because the value already exist.')
print error.format(path=tagValue.tag.path,
about=aboutValue.value.encode('utf-8'))
else:
tagValue.objectID = newObjectID
store.commit()
for i, user in enumerate(getUsers()):
print 'Migrating', user.username.encode('utf-8'), 'object.'
newAbout = u'@%s' % user.username
aboutValue = getAboutTagValues(objectIDs=[user.objectID]).one()
