def testMissingEmailValue(self): """ L{UserIntegrityChecker.check} logs an error if the given L{User} doesn't have a C{fluiddb/users/email} value. """ user = self.createTestUser(u'user') getTagValues([(user.objectID, self.emailTag.id)]).remove() self.checker.check([user]) self.assertEqual( "Integrity Error in user u'user': " 'Email tag is missing.\n', self.log.getvalue())
def testMissingEmailValue(self): """ L{UserIntegrityChecker.check} logs an error if the given L{User} doesn't have a C{fluiddb/users/email} value. """ user = self.createTestUser(u'user') getTagValues([(user.objectID, self.emailTag.id)]).remove() self.checker.check([user]) self.assertEqual("Integrity Error in user u'user': " 'Email tag is missing.\n', self.log.getvalue())
def testGetForUsersUpdatesTheCache(self): """ L{CachingRecentActivityAPI.getForUsers} adds cache misses pulled from the database to the cache. """ tagValues = self.getTagValueAPI(self.user) objectID = self.getObjectAPI(self.user).create(u'object') tagValues.set({objectID: {u'user/tag': u'A'}}) databaseResult = self.recentActivity.getForUsers([u'user']) # Remove all the tag values to ensure that the values for the next # request are taken from the cache. getTagValues().remove() cachedResult = self.recentActivity.getForUsers([u'user']) self.assertEqual(databaseResult, cachedResult)
def testSet(self): """L{TagAPI.set} updates the description for the specified L{Tag}s.""" descriptionTag = self.system.tags[u'fluiddb/tags/description'] tag = createTag(self.user, self.user.namespace, u'tag') createTagPermission(tag) self.tags.set({u'username/tag': u'A fancy new description.'}) result = getTagValues([(tag.objectID, descriptionTag.id)]) description = result.one() self.assertEqual(u'A fancy new description.', description.value)
def testCreateCreatesAboutTag(self): """ L{TagAPI.create} creates new C{fluiddb/about} L{TagValue}s when creating new L{Tag}s. """ values = [(u'username/tag', u'A tag description')] [(objectID, path)] = self.tags.create(values) tag = getTags(paths=[u'fluiddb/about']).one() value = getTagValues([(objectID, tag.id)]).one() self.assertEqual(u'Object for the attribute username/tag', value.value)
def testBatchIndexTouchesAllObjects(self): """C{batchIndex} touches all objects in the given file.""" createTagValue(self.userID, self.tagID, uuid4(), 10) createTagValue(self.userID, self.tagID, uuid4(), 20) allObjects = set(getTagValues().values(TagValue.objectID)) self.createObjectsFile() batchIndex(self.objectsFilename, 0, 10) touchedObjects = set(getDirtyObjects().values(DirtyObject.objectID)) self.assertEqual(allObjects, touchedObjects)
def testCreateCreatesRoleTags(self): """ L{UserAPI.create} creates the C{fluiddb/users/role} tag value for the created user. """ values = [(u'username', u'secret', u'Full Name', u'*****@*****.**')] [(objectID, username)] = self.users.create(values) tagID = self.system.tags[u'fluiddb/users/role'].id value = getTagValues([(objectID, tagID)]).one() self.assertEqual(u'USER', value.value)
def testCreateCreatesAboutTag(self): """ L{UserAPI.create} creates new C{fluiddb/about} L{TagValue}s when creating new L{User}s. """ values = [(u'username', u'secret', u'User', u'*****@*****.**')] [(objectID, username)] = self.users.create(values) tag = getTags(paths=[u'fluiddb/about']).one() value = getTagValues([(objectID, tag.id)]).one() self.assertEqual(u'@username', value.value)
def testCreateStoresTagPaths(self): """ L{TagAPI.create} creates new C{fluiddb/tags/path} L{TagValue}s to store the specified L{Tag} paths. """ values = [(u'username/tag', u'A tag description')] [(objectID, path)] = self.tags.create(values) tag = getTags(paths=[u'fluiddb/tags/path']).one() value = getTagValues([(objectID, tag.id)]).one() self.assertEqual(u'username/tag', value.value)
def testCreateStoresNamespaceDescriptions(self): """ L{NamespaceAPI.create} creates new C{fluiddb/namespaces/description} L{TagValue}s to store the specified L{Namespace} descriptions. """ values = [(u'username/namespace', u'A namespace description')] [(objectID, path)] = self.namespaces.create(values) tag = getTags(paths=[u'fluiddb/namespaces/description']).one() value = getTagValues([(objectID, tag.id)]).one() self.assertEqual(u'A namespace description', value.value)
def testCreate(self): """ L{ObjectAPI.create} creates new L{AboutTagValue} owned by the super user and returns the UUID for the newly created object. """ objectID = self.objects.create(u'A fancy about tag value') tag = self.system.tags[u'fluiddb/about'] about = getTagValues([(objectID, tag.id)]).one() self.assertEqual(u'A fancy about tag value', about.value) self.assertEqual(self.system.users[u'fluiddb'].id, about.creatorID)
def testCreateStoresNamespacePaths(self): """ L{NamespaceAPI.create} creates new C{fluiddb/namespaces/path} L{TagValue}s to store the specified L{Namespace} paths. """ values = [(u'username/namespace', u'A namespace description')] [(objectID, path)] = self.namespaces.create(values) tag = getTags(paths=[u'fluiddb/namespaces/path']).one() value = getTagValues([(objectID, tag.id)]).one() self.assertEqual(u'username/namespace', value.value)
def testSet(self): """ L{NamespaceAPI.set} updates the description for the specified L{Namespace}s. """ descriptionTag = self.system.tags[u'fluiddb/namespaces/description'] self.namespaces.set({u'username': u'A fancy new description.'}) result = getTagValues([(self.user.namespace.objectID, descriptionTag.id)]) description = result.one() self.assertEqual(u'A fancy new description.', description.value)
def _getValues(self, objects, path): """Get all L{TagValue}s for the given objects and the given path. @param objects: A sequence of L{Tag}s, L{Namespace}s or L{User}s. @param path: The path of the L{Tag} to get the vaues from. @return: A C{dict} mapping object IDs to values. """ tag = self._getSystemTag(path) result = getTagValues([(object.objectID, tag.id) for object in objects]) return dict(result.values(TagValue.objectID, TagValue.value))
def testDeleteKeepsTheAboutTag(self): """ L{TagAPI.delete} keeps the C{fluiddb/about} tag value for the deleted tag. """ values = [(u'username/tag', u'A tag description')] [(objectID, path)] = self.tags.create(values) self.tags.delete([u'username/tag']) tag = getTags(paths=[u'fluiddb/about']).one() value = getTagValues([(objectID, tag.id)]).one() self.assertEqual(u'Object for the attribute username/tag', value.value)
def testSet(self): """ L{NamespaceAPI.set} updates the description for the specified L{Namespace}s. """ descriptionTag = self.system.tags[u'fluiddb/namespaces/description'] self.namespaces.set({u'username': u'A fancy new description.'}) result = getTagValues( [(self.user.namespace.objectID, descriptionTag.id)]) description = result.one() self.assertEqual(u'A fancy new description.', description.value)
def testCreateCreatesAboutTag(self): """ L{NamespaceAPI.create} creates new C{fluiddb/about} L{TagValue}s when creating new L{Namespace}s. """ values = [(u'username/namespace', u'A namespace description')] [(objectID, path)] = self.namespaces.create(values) tag = getTags(paths=[u'fluiddb/about']).one() value = getTagValues([(objectID, tag.id)]).one() self.assertEqual(u'Object for the namespace username/namespace', value.value)
def testWrongNameValue(self): """ L{UserIntegrityChecker.check} logs an error if the given L{User} has a wrong C{fluiddb/users/name} value. """ user = self.createTestUser(u'user') name = getTagValues([(user.objectID, self.nameTag.id)]).one() name.value = u'Wrong Name' self.checker.check([user]) self.assertEqual("Integrity Error in user u'user': " 'Name tag is incorrect.\n', self.log.getvalue())
def testWrongAboutValue(self): """ L{UserIntegrityChecker.check} logs an error if the given L{User} has a wrong C{fluiddb/about} value. """ user = self.createTestUser(u'user') about = getTagValues([(user.objectID, self.aboutTag.id)]).one() about.value = u'Wrong about value.' self.checker.check([user]) self.assertEqual( "Integrity Error in user u'user': " 'About tag is incorrect.\n', self.log.getvalue())
def testWrongEmailValue(self): """ L{UserIntegrityChecker.check} logs an error if the given L{User} has a wrong C{fluiddb/users/email} value. """ user = self.createTestUser(u'user') email = getTagValues([(user.objectID, self.emailTag.id)]).one() email.value = u'*****@*****.**' self.checker.check([user]) self.assertEqual("Integrity Error in user u'user': " 'Email tag is incorrect.\n', self.log.getvalue())
def testWrongNameValue(self): """ L{UserIntegrityChecker.check} logs an error if the given L{User} has a wrong C{fluiddb/users/name} value. """ user = self.createTestUser(u'user') name = getTagValues([(user.objectID, self.nameTag.id)]).one() name.value = u'Wrong Name' self.checker.check([user]) self.assertEqual( "Integrity Error in user u'user': " 'Name tag is incorrect.\n', self.log.getvalue())
def testDeleteKeepsTheAboutTag(self): """ L{NamespaceAPI.delete} keeps the C{fluiddb/about} tag value for the deleted namespace. """ values = [(u'username/namespace', u'A namespace description')] [(objectID, path)] = self.namespaces.create(values) self.namespaces.delete([u'username/namespace']) tag = getTags(paths=[u'fluiddb/about']).one() value = getTagValues([(objectID, tag.id)]).one() self.assertEqual(u'Object for the namespace username/namespace', value.value)
def testWrongEmailValue(self): """ L{UserIntegrityChecker.check} logs an error if the given L{User} has a wrong C{fluiddb/users/email} value. """ user = self.createTestUser(u'user') email = getTagValues([(user.objectID, self.emailTag.id)]).one() email.value = u'*****@*****.**' self.checker.check([user]) self.assertEqual( "Integrity Error in user u'user': " 'Email tag is incorrect.\n', self.log.getvalue())
def testWrongAboutValue(self): """ L{UserIntegrityChecker.check} logs an error if the given L{User} has a wrong C{fluiddb/about} value. """ user = self.createTestUser(u'user') about = getTagValues([(user.objectID, self.aboutTag.id)]).one() about.value = u'Wrong about value.' self.checker.check([user]) self.assertEqual("Integrity Error in user u'user': " 'About tag is incorrect.\n', self.log.getvalue())
def testSetUpdates(self): """L{TagValueAPI.set} updates an existing value with a new one.""" objectID = uuid4() namespace = createNamespace(self.user, u'name') createNamespacePermission(namespace) tag = createTag(self.user, namespace, u'tag') createTagPermission(tag) self.tagValues.set({objectID: {u'name/tag': 5}}) self.tagValues.set({objectID: {u'name/tag': None}}) value = getTagValues([(objectID, tag.id)]).one() self.assertEqual(tag.id, value.tagID) self.assertEqual(objectID, value.objectID) self.assertIn(objectID, getDirtyObjects().values(DirtyObject.objectID))
def testGetTagValues(self): """ L{getTagValues} returns all L{TagValue}s in the database, by default. """ objectID1 = uuid4() objectID2 = uuid4() user = createUser(u'username', u'password', u'User', u'*****@*****.**') user.namespaceID = createNamespace(user, user.username, None).id tag = createTag(user, user.namespace, u'name') value1 = self.store.add(TagValue(user.id, tag.id, objectID1, None)) value2 = self.store.add(TagValue(user.id, tag.id, objectID2, 42)) self.assertEqual(sorted([value1, value2]), sorted(getTagValues()))
def testSet(self): """L{TagValueAPI.set} stores new L{TagValue}s.""" namespace = createNamespace(self.user, u'name') createNamespacePermission(namespace) tag = createTag(self.user, namespace, u'tag') createTagPermission(tag) objectID = uuid4() values = {objectID: {u'name/tag': 42}} self.tagValues.set(values) value = getTagValues([(objectID, tag.id)]).one() self.assertEqual(tag.id, value.tagID) self.assertEqual(objectID, value.objectID) self.assertIn(objectID, getDirtyObjects().values(DirtyObject.objectID))
def testGetTagValuesWithTagIDsAndObjectIDs(self): """ When C{(Tag.id, object ID)} 2-tuples are provided L{getTagValues} returns matching L{TagValue}s. """ objectID1 = uuid4() objectID2 = uuid4() user = createUser(u'username', u'password', u'User', u'*****@*****.**') user.namespaceID = createNamespace(user, user.username, None).id tag = createTag(user, user.namespace, u'name') value = self.store.add(TagValue(user.id, tag.id, objectID1, None)) self.store.add(TagValue(user.id, tag.id, objectID2, 42)) self.assertEqual(value, getTagValues([(objectID1, tag.id)]).one())
def testEmailValueWithoutUser(self): """ L{TagValueIntegrityChecker.check} logs an error if a given L{TagValue} for fluiddb/users/email doesn't have an associated L{User} row. """ tag = self.system.tags[u'fluiddb/users/email'] tagValue1 = createTagValue(self.superuser.id, tag.id, uuid4(), u'*****@*****.**') tagValue2 = getTagValues([(self.superuser.objectID, tag.id)]).one() self.checker.check([tagValue1, tagValue2]) self.assertEqual('Integrity Error in object %s: ' "fluiddb/users/email TagValue doesn't have " 'an associated User.\n' % tagValue1.objectID, self.log.getvalue())
def testEmailValueWithoutUser(self): """ L{TagValueIntegrityChecker.check} logs an error if a given L{TagValue} for fluiddb/users/email doesn't have an associated L{User} row. """ tag = self.system.tags[u'fluiddb/users/email'] tagValue1 = createTagValue(self.superuser.id, tag.id, uuid4(), u'*****@*****.**') tagValue2 = getTagValues([(self.superuser.objectID, tag.id)]).one() self.checker.check([tagValue1, tagValue2]) self.assertEqual( 'Integrity Error in object %s: ' "fluiddb/users/email TagValue doesn't have " 'an associated User.\n' % tagValue1.objectID, self.log.getvalue())
def testBatchIndexLogsErrorIfObjectIDIsNotWellFormed(self): """ If C{batchIndex} encounters a malformed objectID in the file it will continue the process after printing an error in the logs. """ createTagValue(self.userID, self.tagID, uuid4(), 10) createTagValue(self.userID, self.tagID, uuid4(), 20) allObjects = set(getTagValues().values(TagValue.objectID)) self.createObjectsFile() with open(self.objectsFilename, 'a') as objectsFilename: objectsFilename.write('wrong-id') batchIndex(self.objectsFilename, 0, 10) touchedObjects = set(getDirtyObjects().values(DirtyObject.objectID)) self.assertEqual(allObjects, touchedObjects) self.assertIn("Invalid objectID: 'wrong-id'", self.log.getvalue())
def testCreateObjectWithAboutValue(self): """ L{FacadeObjectAPI.createObject} always returns a valid C{UUID} in a C{str} for a new object ID if an about value is given and it doesn't already exist. """ self.store.commit() with login(self.user.username, uuid4(), self.transact) as session: objectID = yield self.facade.createObject(session, about='foo') objectID = UUID(objectID) self.store.rollback() value = getAboutTagValues([objectID], [u'foo']).one() self.assertEqual(u'foo', value.value) tag = self.system.tags[u'fluiddb/about'] value = getTagValues(values=[(objectID, tag.id)]).one() self.assertIdentical(self.system.users[u'fluiddb'], value.creator)
def testSetBinaryValue(self): """ L{TagValueAPI.set} can store binary L{TagValue}s. The contents included in the dict representing the value are written to a file and added to the L{FileStore}. """ namespace = createNamespace(self.user, u'name') createNamespacePermission(namespace) tag = createTag(self.user, namespace, u'tag') createTagPermission(tag) objectID = uuid4() self.tagValues.set( {objectID: {u'name/tag': {'mime-type': 'text/plain', 'contents': 'Hello, world!'}}}) value = getTagValues([(objectID, tag.id)]).one() self.assertEqual(tag.id, value.tagID) self.assertEqual(objectID, value.objectID) self.assertEqual({'mime-type': 'text/plain', 'size': 13}, value.value) self.assertIn(objectID, getDirtyObjects().values(DirtyObject.objectID))
def delete(self, values): """Delete L{TagValue}s. @param values: A sequence of C{(objectID, Tag.path)} 2-tuples to delete values for. @raise FeatureError: Raised if the given list of values is empty. @return: The number of values deleted. """ if isgenerator(values): values = list(values) if not values: raise FeatureError("Can't delete an empty list of tag values.") paths = set([path for objectID, path in values]) objectIDs = set([objectID for objectID, path in values]) tagIDs = dict(getTags(paths).values(Tag.path, Tag.id)) values = [(objectID, tagIDs[path]) for objectID, path in values] result = getTagValues(values).remove() if result: touchObjects(objectIDs) return result
def checkIntegrity(maxRowsPerQuery=10000): """ Check the integrity of the database for cases which the database engine can't detect. @param maxRowsPerQuery: Limit the number of rows fetched by SQL queries to avoid excessive use of memory. """ results = _splitResult(getNamespaces(), Namespace.id, maxRowsPerQuery) for result in results: namespaces = list(result) NamespaceIntegrityChecker().check(namespaces) results = _splitResult(getTags(), Tag.id, maxRowsPerQuery) for result in results: tags = list(result) TagIntegrityChecker().check(tags) results = _splitResult(getUsers(), User.id, maxRowsPerQuery) for result in results: users = list(result) UserIntegrityChecker().check(users) results = _splitResult(getAboutTagValues(), AboutTagValue.objectID, maxRowsPerQuery) for result in results: aboutTagValues = list(result) AboutTagValueIntegrityChecker().check(aboutTagValues) # In the case of TagValues we limit the query to only tag paths starting # with "fluiddb" because these are the only ones we're checking and we # don't want a huge result. store = getMainStore() result = store.find(TagValue, TagValue.tagID == Tag.id, Tag.path.startswith(u'fluiddb/')) results = _splitResult(getTagValues(), TagValue.id, maxRowsPerQuery) for result in results: tagValues = list(result) TagValueIntegrityChecker().check(tagValues)
def testSetBinaryValue(self): """ L{TagValueAPI.set} can store binary L{TagValue}s. The contents included in the dict representing the value are written to a file and added to the L{FileStore}. """ namespace = createNamespace(self.user, u'name') createNamespacePermission(namespace) tag = createTag(self.user, namespace, u'tag') createTagPermission(tag) objectID = uuid4() self.tagValues.set({ objectID: { u'name/tag': { 'mime-type': 'text/plain', 'contents': 'Hello, world!' } } }) value = getTagValues([(objectID, tag.id)]).one() self.assertEqual(tag.id, value.tagID) self.assertEqual(objectID, value.objectID) self.assertEqual({'mime-type': 'text/plain', 'size': 13}, value.value) self.assertIn(objectID, getDirtyObjects().values(DirtyObject.objectID))
def createObjectsFile(self): """Helper function to create a file with a list of all object IDs.""" allObjects = set(getTagValues().values(TagValue.objectID)) with open(self.objectsFilename, 'w') as objectsFile: for objectID in allObjects: objectsFile.write(str(objectID) + '\n')
def set(self, values): """Set or update L{TagValue}s. L{Tag}s that don't exist are created automatically before L{TagValue}s are stored. Associated L{TagPermission}s are created automatically with the system-wide default permissions. @param values: A C{dict} mapping object IDs to tags and values, matching the following format:: {<object-id>: {<path>: <value>, <path>: {'mime-type': <mime-type>, 'contents': <contents>}}} A binary L{TagValue} is represented using a different layout than other values types, as shown for the second value. @raise FeatureError: Raised if the given list of values is empty. @raise MalformedPathError: Raised if one of the given paths for a nonexistent tag is empty or has unacceptable characters. """ if not values: raise FeatureError("Can't set an empty list of tag values.") objectIDs = set(values.keys()) # Implicitly create missing tags, if there are any. paths = set() for tagValues in values.itervalues(): paths.update(tagValues.iterkeys()) tagIDs = dict(getTags(paths=paths).values(Tag.path, Tag.id)) existingPaths = set(tagIDs.iterkeys()) unknownPaths = paths - existingPaths if unknownPaths: tags = [(path, u'Object for the attribute %s' % path) for path in unknownPaths] self._factory.tags(self._user).create(tags) tagIDs = dict(getTags(paths=paths).values(Tag.path, Tag.id)) # Delete all existing tag values for the specified object IDs and # paths. deleteValues = [] for objectID in values: for path in values[objectID].iterkeys(): deleteValues.append((objectID, tagIDs[path])) getTagValues(deleteValues).remove() # Set new tag values for the specified object IDs and paths. for objectID in values: tagValues = values[objectID] for path, value in tagValues.iteritems(): tagID = tagIDs[path] if isinstance(value, dict): content = value['contents'] value = createTagValue(self._user.id, tagID, objectID, {'mime-type': value['mime-type'], 'size': len(content)}) # This is necessary to tell PostgreSQL that generates a # `value.id` immediately. value.id = AutoReload createOpaqueValue(value.id, content) else: createTagValue(self._user.id, tagID, objectID, value) touchObjects(objectIDs)
def set(self, values): """Set or update L{TagValue}s. L{Tag}s that don't exist are created automatically before L{TagValue}s are stored. Associated L{TagPermission}s are created automatically with the system-wide default permissions. @param values: A C{dict} mapping object IDs to tags and values, matching the following format:: {<object-id>: {<path>: <value>, <path>: {'mime-type': <mime-type>, 'contents': <contents>}}} A binary L{TagValue} is represented using a different layout than other values types, as shown for the second value. @raise FeatureError: Raised if the given list of values is empty. @raise MalformedPathError: Raised if one of the given paths for a nonexistent tag is empty or has unacceptable characters. """ if not values: raise FeatureError("Can't set an empty list of tag values.") objectIDs = set(values.keys()) # Implicitly create missing tags, if there are any. paths = set() for tagValues in values.itervalues(): paths.update(tagValues.iterkeys()) tagIDs = dict(getTags(paths=paths).values(Tag.path, Tag.id)) existingPaths = set(tagIDs.iterkeys()) unknownPaths = paths - existingPaths if unknownPaths: tags = [(path, u'Object for the attribute %s' % path) for path in unknownPaths] self._factory.tags(self._user).create(tags) tagIDs = dict(getTags(paths=paths).values(Tag.path, Tag.id)) # Delete all existing tag values for the specified object IDs and # paths. deleteValues = [] for objectID in values: for path in values[objectID].iterkeys(): deleteValues.append((objectID, tagIDs[path])) getTagValues(deleteValues).remove() # Set new tag values for the specified object IDs and paths. for objectID in values: tagValues = values[objectID] for path, value in tagValues.iteritems(): tagID = tagIDs[path] if isinstance(value, dict): content = value['contents'] value = createTagValue(self._user.id, tagID, objectID, { 'mime-type': value['mime-type'], 'size': len(content) }) # This is necessary to tell PostgreSQL that generates a # `value.id` immediately. value.id = AutoReload createOpaqueValue(value.id, content) else: createTagValue(self._user.id, tagID, objectID, value) touchObjects(objectIDs)
superUser = getUsers(usernames=[u'fluiddb']).one() result = store.find(AboutTagValue, AboutTagValue.value.like(u'@%')) for aboutValue in result: if aboutValue.value == aboutValue.value.lower(): continue print 'Migrating mixed cased', aboutValue.value.encode('utf-8') newAbout = u'@%s' % aboutValue.value.lower() oldObjectID = aboutValue.objectID newObjectID = ObjectAPI(superUser).create(newAbout) result = store.find(TagValue, TagValue.objectID == oldObjectID, TagValue.tagID != aboutTag.id) for tagValue in result: existingValue = getTagValues([(newObjectID, tagValue.tagID)]).one() if existingValue is not None: error = ('ERROR: Cannot migrate value {path} on {about} ' 'because the value already exist.') print error.format(path=tagValue.tag.path, about=aboutValue.value.encode('utf-8')) else: tagValue.objectID = newObjectID store.commit() for i, user in enumerate(getUsers()): print 'Migrating', user.username.encode('utf-8'), 'object.' newAbout = u'@%s' % user.username aboutValue = getAboutTagValues(objectIDs=[user.objectID]).one()