def testDelete(self):
"""
L{UserAPI.delete} removes L{User}s, returning the objectIDs and
usernames of the deleted users.
"""
self.users.create([(u'username', u'password', u'User',
u'*****@*****.**')])
getNamespaces(paths=[u'username/private']).remove()
self.users.delete([u'username'])
self.assertIdentical(None, getUser(u'username'))
def testDelete(self):
"""
L{UserAPI.delete} removes L{User}s, returning the objectIDs and
usernames of the deleted users.
"""
self.users.create([(u'username', u'password', u'User',
u'*****@*****.**')])
getNamespaces(paths=[u'username/private']).remove()
self.users.delete([u'username'])
self.assertIdentical(None, getUser(u'username'))
def testCreateReusesPreviousObjectIDs(self):
"""
If a L{User} is deleted and created again, L{UserAPI.create}
uses the old object ID.
"""
values = [(u'username', u'secret', u'User', u'*****@*****.**')]
result1 = self.users.create(values)
getNamespaces(paths=[u'username/private']).remove()
self.users.delete([u'username'])
result2 = self.users.create(values)
self.assertEqual(result1, result2)
def testMissingNamespace(self):
"""
L{UserIntegrityChecker.check} logs an error if the given L{User}
doesn't have a L{Namespace} with the same name.
"""
user = self.createTestUser(u'user')
getNamespaces(paths=[u'user']).remove()
self.checker.check([user])
self.assertEqual("Integrity Error in user u'user': "
'Root namespace is missing.\n',
self.log.getvalue())
def testMissingNamespace(self):
"""
L{UserIntegrityChecker.check} logs an error if the given L{User}
doesn't have a L{Namespace} with the same name.
"""
user = self.createTestUser(u'user')
getNamespaces(paths=[u'user']).remove()
self.checker.check([user])
self.assertEqual(
"Integrity Error in user u'user': "
'Root namespace is missing.\n', self.log.getvalue())
def testCreateReusesPreviousObjectIDs(self):
"""
If a L{User} is deleted and created again, L{UserAPI.create}
uses the old object ID.
"""
values = [(u'username', u'secret', u'User', u'*****@*****.**')]
result1 = self.users.create(values)
getNamespaces(paths=[u'username/private']).remove()
self.users.delete([u'username'])
result2 = self.users.create(values)
self.assertEqual(result1, result2)
def testDeleteDoesNotDeleteOtherUsers(self):
"""
L{UserAPI.delete} removes the L{User}s it is asked to delete
but not other users.
"""
self.users.create([(u'username1', u'password', u'User',
u'*****@*****.**')])
self.users.create([(u'username2', u'password', u'User',
u'*****@*****.**')])
getNamespaces(paths=[u'username1/private']).remove()
self.users.delete([u'username1'])
user = getUser(u'username2')
self.assertEqual(u'username2', user.username)
def testDeleteDoesNotDeleteOtherUsers(self):
"""
L{UserAPI.delete} removes the L{User}s it is asked to delete
but not other users.
"""
self.users.create([(u'username1', u'password', u'User',
u'*****@*****.**')])
self.users.create([(u'username2', u'password', u'User',
u'*****@*****.**')])
getNamespaces(paths=[u'username1/private']).remove()
self.users.delete([u'username1'])
user = getUser(u'username2')
self.assertEqual(u'username2', user.username)
def testDeleteDoesNotDeleteOtherUsersWhenPassedAGenerator(self):
"""
L{UserAPI.delete} removes the L{User}s it is asked to delete but
not other users, when it is passed a generator (as opposed to a
C{list}).
"""
self.users.create([(u'username1', u'password', u'User',
u'*****@*****.**')])
self.users.create([(u'username2', u'password', u'User',
u'*****@*****.**')])
getNamespaces(paths=[u'username1/private']).remove()
self.users.delete(username for username in [u'username1'])
user = getUser(u'username2')
self.assertEqual(u'username2', user.username)
def testDeleteDoesNotDeleteOtherUsersWhenPassedAGenerator(self):
"""
L{UserAPI.delete} removes the L{User}s it is asked to delete but
not other users, when it is passed a generator (as opposed to a
C{list}).
"""
self.users.create([(u'username1', u'password', u'User',
u'*****@*****.**')])
self.users.create([(u'username2', u'password', u'User',
u'*****@*****.**')])
getNamespaces(paths=[u'username1/private']).remove()
self.users.delete(username for username in [u'username1'])
user = getUser(u'username2')
self.assertEqual(u'username2', user.username)
def check(self, tags):
"""Check given L{Tag}s for integrity errors.
@param tags: A sequence of L{Tag}s to be checked.
"""
aboutValues = self._getValues(tags, u'fluiddb/about')
pathValues = self._getValues(tags, u'fluiddb/tags/path')
descriptionValues = self._getValues(tags, u'fluiddb/tags/description')
tagPermissions = dict(getTagPermissions([tag.path for tag in tags]))
parentPaths = [getParentPath(tag.path) for tag in tags]
parentNamespaces = getNamespaces(parentPaths)
parentNamespaces = dict((namespace.path, namespace)
for namespace in parentNamespaces)
users = getUsers()
users = dict((user.id, user) for user in users)
for tag in tags:
expectedAbout = u'Object for the attribute %s' % tag.path
self.checkAboutValue(tag, aboutValues, expectedAbout)
self.checkPathValue(tag, pathValues)
self.checkDescriptionValue(tag, descriptionValues)
self.checkPermissions(tag, tagPermissions, users,
Operation.TAG_OPERATIONS)
self.checkParent(tag, parentNamespaces)
self.checkValidPath(tag)
def getUnknownParentPaths(self, unknownPaths):
"""Get a C{dict} mapping unknown paths to their closest L{Namespace}.
This function finds the closest L{Namespace} parent for the specified
unknown paths. It walks back down each path until it finds a parent or
determines that one doesn't exist.
@param unknownPaths: A C{set} of unknown L{Tag.path}s and
L{Namespace.path}s.
@return: A C{dict} that maps unknown paths to their closest
L{Namepace.path} parent.
"""
if not unknownPaths:
return {}
hierarchy = getPathHierarchy(unknownPaths)
existingPaths = getNamespaces(paths=hierarchy).values(Namespace.path)
existingPaths = set(existingPaths)
closestParents = {}
for path in unknownPaths:
parentPath = getParentPath(path)
while parentPath is not None:
if parentPath in existingPaths:
closestParents[path] = parentPath
break
parentPath = getParentPath(parentPath)
return closestParents
def check(self, namespaces):
""" Check given L{Namespace}s for integrity errors.
@param namespaces: A sequence of L{Namespace}s to be checked.
"""
aboutValues = self._getValues(namespaces, u'fluiddb/about')
pathValues = self._getValues(namespaces, u'fluiddb/namespaces/path')
descriptionValues = self._getValues(namespaces,
u'fluiddb/namespaces/description')
paths = [namespace.path for namespace in namespaces]
namespacePermissions = dict(getNamespacePermissions(paths))
parentPaths = [getParentPath(namespace.path)
for namespace in namespaces
if namespace.parentID is not None]
parentNamespaces = getNamespaces(paths=parentPaths)
parentNamespaces = dict((namespace.path, namespace)
for namespace in parentNamespaces)
users = getUsers()
users = dict((user.id, user) for user in users)
for namespace in namespaces:
expectedAbout = u'Object for the namespace %s' % namespace.path
self.checkAboutValue(namespace, aboutValues, expectedAbout)
self.checkPathValue(namespace, pathValues)
self.checkDescriptionValue(namespace, descriptionValues)
self.checkPermissions(namespace, namespacePermissions, users,
Operation.NAMESPACE_OPERATIONS)
self.checkParent(namespace, parentNamespaces)
self.checkValidPath(namespace)
def getUnknownParentPaths(self, unknownPaths):
"""Get a C{dict} mapping unknown paths to their closest L{Namespace}.
This function finds the closest L{Namespace} parent for the specified
unknown paths. It walks back down each path until it finds a parent or
determines that one doesn't exist.
@param unknownPaths: A C{set} of unknown L{Tag.path}s and
L{Namespace.path}s.
@return: A C{dict} that maps unknown paths to their closest
L{Namepace.path} parent.
"""
if not unknownPaths:
return {}
hierarchy = getPathHierarchy(unknownPaths)
existingPaths = getNamespaces(paths=hierarchy).values(Namespace.path)
existingPaths = set(existingPaths)
closestParents = {}
for path in unknownPaths:
parentPath = getParentPath(path)
while parentPath is not None:
if parentPath in existingPaths:
closestParents[path] = parentPath
break
parentPath = getParentPath(parentPath)
return closestParents
def check(self, namespaces):
""" Check given L{Namespace}s for integrity errors.
@param namespaces: A sequence of L{Namespace}s to be checked.
"""
aboutValues = self._getValues(namespaces, u'fluiddb/about')
pathValues = self._getValues(namespaces, u'fluiddb/namespaces/path')
descriptionValues = self._getValues(namespaces,
u'fluiddb/namespaces/description')
paths = [namespace.path for namespace in namespaces]
namespacePermissions = dict(getNamespacePermissions(paths))
parentPaths = [
getParentPath(namespace.path) for namespace in namespaces
if namespace.parentID is not None
]
parentNamespaces = getNamespaces(paths=parentPaths)
parentNamespaces = dict(
(namespace.path, namespace) for namespace in parentNamespaces)
users = getUsers()
users = dict((user.id, user) for user in users)
for namespace in namespaces:
expectedAbout = u'Object for the namespace %s' % namespace.path
self.checkAboutValue(namespace, aboutValues, expectedAbout)
self.checkPathValue(namespace, pathValues)
self.checkDescriptionValue(namespace, descriptionValues)
self.checkPermissions(namespace, namespacePermissions, users,
Operation.NAMESPACE_OPERATIONS)
self.checkParent(namespace, parentNamespaces)
self.checkValidPath(namespace)
def check(self, tags):
"""Check given L{Tag}s for integrity errors.
@param tags: A sequence of L{Tag}s to be checked.
"""
aboutValues = self._getValues(tags, u'fluiddb/about')
pathValues = self._getValues(tags, u'fluiddb/tags/path')
descriptionValues = self._getValues(tags, u'fluiddb/tags/description')
tagPermissions = dict(getTagPermissions([tag.path for tag in tags]))
parentPaths = [getParentPath(tag.path) for tag in tags]
parentNamespaces = getNamespaces(parentPaths)
parentNamespaces = dict(
(namespace.path, namespace) for namespace in parentNamespaces)
users = getUsers()
users = dict((user.id, user) for user in users)
for tag in tags:
expectedAbout = u'Object for the attribute %s' % tag.path
self.checkAboutValue(tag, aboutValues, expectedAbout)
self.checkPathValue(tag, pathValues)
self.checkDescriptionValue(tag, descriptionValues)
self.checkPermissions(tag, tagPermissions, users,
Operation.TAG_OPERATIONS)
self.checkParent(tag, parentNamespaces)
self.checkValidPath(tag)
def create(self, values):
"""Create new L{Namespace}s.
Missing parent L{Namespace}s are created automatically. For example,
if C{foo/bar/baz} is requested, and C{foo/bar} doesn't already exist,
it will be created before C{foo/bar/baz} is created. Associated
L{NamespacePermission}s are created automatically with the system-wide
default permissions.
@param values: A sequence of C{(path, description)} 2-tuples.
@raises DuplicatePathError: Raised if the path for a new L{Namespace}
collides with an existing one.
@raise MalformedPathError: Raised if one of the given paths is empty
or has unacceptable characters.
@return: A C{list} of C{(objectID, path)} 2-tuples for the new
L{Namespace}s.
"""
from fluiddb.model.user import getUser
if not values:
return []
paths = [path for path, description in values]
descriptions = dict(values)
self._checkForDuplicates(paths)
admin = getUser(u'fluiddb')
objects = self._factory.objects(admin)
systemValues = {}
paths = getPathHierarchy(paths)
existingNamespaces = dict((namespace.path, namespace)
for namespace in getNamespaces(paths=paths))
newNamespaces = []
for path in sorted(paths):
if path in existingNamespaces:
continue
parentPath = getParentPath(path)
parentID = (existingNamespaces[parentPath].id
if parentPath is not None
else None)
namespace = createNamespace(self._user, path, parentID)
aboutValue = u'Object for the namespace %s' % path
description = descriptions.get(path, aboutValue)
namespace.objectID = objects.create(aboutValue)
systemValues[namespace.objectID] = {
u'fluiddb/namespaces/description': description,
u'fluiddb/namespaces/path': path,
u'fluiddb/about': aboutValue}
existingNamespaces[path] = namespace
newNamespaces.append(namespace)
self._createPermissions(newNamespaces)
self._factory.tagValues(admin).set(systemValues)
values = [(namespace.objectID, namespace.path)
for namespace in newNamespaces]
return values
def testCreateWithoutPrivateNamespace(self):
"""
L{User.create} creates a new C{<username>/private} namespace, by
default. This behaviour is suppressed if a flag is passed.
"""
users = [(u'user', u'secret', u'User', u'*****@*****.**')]
self.users.create(users, createPrivateNamespace=False)
self.assertIdentical(None,
getNamespaces(paths=[u'user/private']).one())
def testCreateWithoutPrivateNamespace(self):
"""
L{User.create} creates a new C{<username>/private} namespace, by
default. This behaviour is suppressed if a flag is passed.
"""
users = [(u'user', u'secret', u'User', u'*****@*****.**')]
self.users.create(users, createPrivateNamespace=False)
self.assertIdentical(None,
getNamespaces(paths=[u'user/private']).one())
def testDeleteWithChildTag(self):
"""
L{NamespaceAPI.delete} raises a L{NotEmptyError} if a child L{Tag}
exists.
"""
self.namespaces.create([(u'username/child', u'A description')])
namespace = getNamespaces(paths=[u'username/child']).one()
createTag(self.user, namespace, u'tag')
self.assertRaises(NotEmptyError,
self.namespaces.delete, [u'username/child'])
def testGetNamespacesWithObjectIDs(self):
"""
When L{Namespace.objectIDs}s are provided L{getNamespaces} returns
matching L{Namespace}s.
"""
user = createUser(u'username', u'password', u'User',
u'*****@*****.**')
namespace = createNamespace(user, u'name')
result = getNamespaces(objectIDs=[namespace.objectID])
self.assertIdentical(namespace, result.one())
def testDeleteWithChildTag(self):
"""
L{NamespaceAPI.delete} raises a L{NotEmptyError} if a child L{Tag}
exists.
"""
self.namespaces.create([(u'username/child', u'A description')])
namespace = getNamespaces(paths=[u'username/child']).one()
createTag(self.user, namespace, u'tag')
self.assertRaises(NotEmptyError, self.namespaces.delete,
[u'username/child'])
def _getMissingNamespaces(self, paths):
"""Get a C{set} of missing L{Namespace.path}s.
@param paths: The L{Namespace.path}s to get.
@return: A C{set} of missing L{Namespace.path}s.
"""
if paths:
result = getNamespaces(paths=paths)
return paths - set(result.values(Namespace.path))
else:
return set()
def testWrongNamespace(self):
"""
L{UserIntegrityChecker.check} logs an error if the given L{User}
have a wrong associated L{Namespace}.
"""
user = self.createTestUser(u'user')
user.namespaceID = getNamespaces(paths=[u'fluiddb']).one().id
self.checker.check([user])
self.assertEqual(
"Integrity Error in user u'user': "
'Assigned namespace is incorrect.\n', self.log.getvalue())
def testWrongNamespace(self):
"""
L{UserIntegrityChecker.check} logs an error if the given L{User}
have a wrong associated L{Namespace}.
"""
user = self.createTestUser(u'user')
user.namespaceID = getNamespaces(paths=[u'fluiddb']).one().id
self.checker.check([user])
self.assertEqual("Integrity Error in user u'user': "
'Assigned namespace is incorrect.\n',
self.log.getvalue())
def testCreateCreatesNamespace(self):
"""
L{User.create} creates new L{User}s based on the provided data and
a L{Namespace} for the new L{User} to create their L{Tags}s.
"""
users = [(u'user', u'secret', u'User', u'*****@*****.**')]
self.users.create(users)
user = getUser(u'user')
namespace = getNamespaces(paths=[u'user']).one()
self.assertEqual(u'user', namespace.path)
self.assertIdentical(user, namespace.creator)
def testCreateCreatesNamespace(self):
"""
L{User.create} creates new L{User}s based on the provided data and
a L{Namespace} for the new L{User} to create their L{Tags}s.
"""
users = [(u'user', u'secret', u'User', u'*****@*****.**')]
self.users.create(users)
user = getUser(u'user')
namespace = getNamespaces(paths=[u'user']).one()
self.assertEqual(u'user', namespace.path)
self.assertIdentical(user, namespace.creator)
def testGetNamespaces(self):
"""
L{getNamespaces} returns all L{Namespace}s in the database, by
default.
"""
user = createUser(u'username', u'password', u'User',
u'*****@*****.**')
user.namespaceID = createNamespace(user, user.username, None).id
namespace = createNamespace(user, u'name')
result = getNamespaces()
self.assertEqual([namespace, user.namespace],
list(result.order_by(Namespace.path)))
def _checkForDuplicates(self, paths):
"""
Make sure L{Namespace} or L{Tag} paths don't exist before trying to
create new namespaces.
@param values: A sequence of C{(path, description)} 2-tuples.
@raises DuplicatePathError: Raised if the path for a new L{Namespace}
collides with an existing one.
"""
existingPaths = list(getNamespaces(paths=paths).values(Namespace.path))
if existingPaths:
raise DuplicatePathError(
'Paths already exist: %s' % ', '.join(existingPaths))
def testDeleteInvalidatesCachedNamespacePermissions(self):
"""
L{CachingNamespaceAPI.delete} invalidates L{NamespacePermission}s to
ensure the cache is always fresh.
"""
self.namespaces.create([(u'username/namespace', u'A namespace')])
namespace = getNamespaces(paths=[u'username/namespace']).one()
cache = PermissionCache()
cache.saveNamespacePermissions(
{u'username/namespace': namespace.permission})
self.namespaces.delete([u'username/namespace'])
cached = cache.getNamespacePermissions([u'username/namespace'])
self.assertEqual({}, cached.results)
self.assertEqual([u'username/namespace'], cached.uncachedValues)
def apply(store):
# Using model code in a patch isn't ideal, but writing this patch with
# pure SQL will be heinous.
for user in getUsers():
if user.username in ('fluiddb', 'anon'):
continue
namespaces = NamespaceAPI(user)
path = '%s/private' % user.username
if not namespaces.get([path]):
namespaces.create([
(path, u'Private namespace for user %s' % user.username)
])
namespace = getNamespaces(paths=[path]).one()
permission = namespace.permission
permission.set(Operation.LIST_NAMESPACE, Policy.CLOSED, [user.id])
def testCreateNamespace(self):
"""L{Facade.createNamespace} creates a new L{Namespace}."""
self.store.commit()
with login(u'username', self.user.objectID, self.transact) as session:
objectID = yield self.facade.createNamespace(
session, u'username', u'name', u'A namespace.')
self.store.rollback()
self.assertNotIdentical(None, objectID)
objectID = UUID(objectID)
namespace = getNamespaces(objectIDs=[objectID]).one()
self.assertIdentical(self.user, namespace.creator)
self.assertEqual(u'username/name', namespace.path)
self.assertEqual(u'name', namespace.name)
self.assertEqual(objectID, namespace.objectID)
def testCreateNamespace(self):
"""L{Facade.createNamespace} creates a new L{Namespace}."""
self.store.commit()
with login(u'username', self.user.objectID, self.transact) as session:
objectID = yield self.facade.createNamespace(
session, u'username', u'name', u'A namespace.')
self.store.rollback()
self.assertNotIdentical(None, objectID)
objectID = UUID(objectID)
namespace = getNamespaces(objectIDs=[objectID]).one()
self.assertIdentical(self.user, namespace.creator)
self.assertEqual(u'username/name', namespace.path)
self.assertEqual(u'name', namespace.name)
self.assertEqual(objectID, namespace.objectID)
def get(self, paths, withDescriptions=None, withNamespaces=None,
withTags=None):
"""Get information about L{Namespace}s matching C{paths}.
@param paths: A sequence of L{Namespace.path}s.
@param withDescriptions: Optionally, a C{bool} indicating whether or
not to include L{Namespace} descriptions in the result. Default
is C{False}.
@param withNamespaces: Optionally, a C{bool} indicating whether or not
to include the names of child L{Namespace}s in the result.
Default is C{False}.
@param withTags: Optionally, a C{bool} indicating whether or not to
include the names of child L{Tag}s in the result. Default is
C{False}.
@return: A C{dict} that maps L{Namespace.path}s to C{dict}s with
information about matching L{Namespace}s, matching the following
format::
{<path>: {'tagNames': [<tag-name>, ...],
'namespaceNames': [<namespace-name>, ...],
'id': <object-id>,
'description': <description>}}
"""
if not paths:
return {}
result = getNamespaces(paths=paths)
values = list(result.values(Namespace.id, Namespace.objectID,
Namespace.path))
descriptions = (
self._getDescriptions(objectID for id, objectID, path in values)
if withDescriptions else None)
childNamespaces = (self._getChildNamespaces(paths)
if withNamespaces else None)
childTags = self._getChildTags(paths) if withTags else None
namespaces = {}
for id, objectID, path in values:
value = {'id': objectID}
if withDescriptions:
value['description'] = descriptions[objectID]
if withNamespaces:
value['namespaceNames'] = childNamespaces.get(id, [])
if withTags:
value['tagNames'] = childTags.get(id, [])
namespaces[path] = value
return namespaces
def create(self, values):
"""Create new L{Tag}s.
L{Namespace}s that don't exist are created automatically before
L{Tag}s are created. Associated L{NamespacePermission} and
L{TagPermission}s are created automatically with the system-wide
default permissions.
@param values: A sequence of C{(Tag.path, description)} 2-tuples.
@raise DuplicatePathError: Raised if the path for a new L{Tag}
collides with an existing one.
@raise FeatureError: Raised if the given list of values is empty.
@raise UnknownParentPathError: Raised if the parent for a new L{Tag}
can't be found.
@raise MalformedPathError: Raised if one of the given paths is empty
or has unacceptable characters.
@return: A C{list} of C{(objectID, path)} 2-tuples for the new L{Tag}s.
"""
if not values:
raise FeatureError("Can't create an empty list of tags.")
# Make sure tag paths don't exist before trying to create new tags.
paths = [path for path, _ in values]
existingPaths = list(getTags(paths=paths).values(Tag.path))
if existingPaths:
raise DuplicatePathError(
'Paths already exist: %s' % ', '.join(existingPaths))
# Get intermediate namespaces. If they don't exist, create them
# automatically.
paths = [path for (path, _) in values]
parentPaths = getParentPaths(paths)
missingParentPaths = self._getMissingNamespaces(parentPaths)
self._factory.namespaces(self._user).create(
[(path, u'Object for the namespace %s' % path)
for path in missingParentPaths])
# Create the new tags.
result = getNamespaces(paths=parentPaths)
parentNamespaces = dict((namespace.path, namespace)
for namespace in result)
return self._createTags(values, parentNamespaces)
def check(self, tagValues):
"""Check a given L{TagValue}s for integrity errors.
@param tagValues: A sequence of L{TagValue}s to be checked.
"""
objectIDs = [tagValue.objectID for tagValue in tagValues]
aboutTagValues = getAboutTagValues(objectIDs=objectIDs)
aboutTagValues = dict((aboutTagValue.objectID, aboutTagValue)
for aboutTagValue in aboutTagValues)
namespaces = dict((namespace.objectID, namespace)
for namespace in getNamespaces(objectIDs=objectIDs))
users = dict((user.objectID, user)
for user in getUsers(objectIDs=objectIDs))
tags = dict((tag.objectID, tag)
for tag in getTags(objectIDs=objectIDs))
for tagValue in tagValues:
self.checkAboutTagValue(tagValue, aboutTagValues)
self.checkNamespaceValues(tagValue, namespaces)
self.checkTagValues(tagValue, tags)
self.checkUserValues(tagValue, users)
def check(self, tagValues):
"""Check a given L{TagValue}s for integrity errors.
@param tagValues: A sequence of L{TagValue}s to be checked.
"""
objectIDs = [tagValue.objectID for tagValue in tagValues]
aboutTagValues = getAboutTagValues(objectIDs=objectIDs)
aboutTagValues = dict((aboutTagValue.objectID, aboutTagValue)
for aboutTagValue in aboutTagValues)
namespaces = dict((namespace.objectID, namespace)
for namespace in getNamespaces(objectIDs=objectIDs))
users = dict(
(user.objectID, user) for user in getUsers(objectIDs=objectIDs))
tags = dict(
(tag.objectID, tag) for tag in getTags(objectIDs=objectIDs))
for tagValue in tagValues:
self.checkAboutTagValue(tagValue, aboutTagValues)
self.checkNamespaceValues(tagValue, namespaces)
self.checkTagValues(tagValue, tags)
self.checkUserValues(tagValue, users)
def checkIntegrity(maxRowsPerQuery=10000):
"""
Check the integrity of the database for cases which the database
engine can't detect.
@param maxRowsPerQuery: Limit the number of rows fetched by SQL queries to
avoid excessive use of memory.
"""
results = _splitResult(getNamespaces(), Namespace.id, maxRowsPerQuery)
for result in results:
namespaces = list(result)
NamespaceIntegrityChecker().check(namespaces)
results = _splitResult(getTags(), Tag.id, maxRowsPerQuery)
for result in results:
tags = list(result)
TagIntegrityChecker().check(tags)
results = _splitResult(getUsers(), User.id, maxRowsPerQuery)
for result in results:
users = list(result)
UserIntegrityChecker().check(users)
results = _splitResult(getAboutTagValues(), AboutTagValue.objectID,
maxRowsPerQuery)
for result in results:
aboutTagValues = list(result)
AboutTagValueIntegrityChecker().check(aboutTagValues)
# In the case of TagValues we limit the query to only tag paths starting
# with "fluiddb" because these are the only ones we're checking and we
# don't want a huge result.
store = getMainStore()
result = store.find(TagValue,
TagValue.tagID == Tag.id,
Tag.path.startswith(u'fluiddb/'))
results = _splitResult(getTagValues(), TagValue.id, maxRowsPerQuery)
for result in results:
tagValues = list(result)
TagValueIntegrityChecker().check(tagValues)
def check(self, users):
"""Check a given L{User}s for integrity errors.
@param users: A sequence of L{User}s to be checked.
"""
aboutValues = self._getValues(users, u'fluiddb/about')
usernameValues = self._getValues(users, u'fluiddb/users/username')
nameValues = self._getValues(users, u'fluiddb/users/name')
emailValues = self._getValues(users, u'fluiddb/users/email')
namespaces = getNamespaces(paths=[user.username for user in users])
namespaces = dict((namespace.path, namespace)
for namespace in namespaces)
for user in users:
expectedAbout = u'@%s' % user.username
self.checkAboutValue(user, aboutValues, expectedAbout)
self.checkUsernameValue(user, usernameValues)
self.checkNameValue(user, nameValues)
self.checkEmailValue(user, emailValues)
self.checkNamespace(user, namespaces)
self.checkUsername(user)
def check(self, users):
"""Check a given L{User}s for integrity errors.
@param users: A sequence of L{User}s to be checked.
"""
aboutValues = self._getValues(users, u'fluiddb/about')
usernameValues = self._getValues(users, u'fluiddb/users/username')
nameValues = self._getValues(users, u'fluiddb/users/name')
emailValues = self._getValues(users, u'fluiddb/users/email')
namespaces = getNamespaces(paths=[user.username for user in users])
namespaces = dict(
(namespace.path, namespace) for namespace in namespaces)
for user in users:
expectedAbout = u'@%s' % user.username
self.checkAboutValue(user, aboutValues, expectedAbout)
self.checkUsernameValue(user, usernameValues)
self.checkNameValue(user, nameValues)
self.checkEmailValue(user, emailValues)
self.checkNamespace(user, namespaces)
self.checkUsername(user)
def testCheckIntegrityGetsAllRowsUsingMultipleQueries(self):
"""
L{checkIntegrity} should check all the rows of a given object using
multiple queries if the C{maxRowsPerQuery} argument is smaller than the
total number of rows for a given object.
"""
createSystemData()
UserAPI().create([(u'user', u'pass', u'Name', u'*****@*****.**')])
user = getUser(u'user')
paths = [u'user/namespace%d' % i for i in xrange(10)]
values = [(path, u'description') for path in paths]
NamespaceAPI(user).create(values)
namespaces = getNamespaces(paths=paths)
values = [(namespace.objectID, u'fluiddb/namespaces/path')
for namespace in namespaces]
TagValueAPI(user).delete(values)
checkIntegrity(maxRowsPerQuery=2)
error = 'Integrity Error in namespace %r: Path tag is missing.'
expectedErrors = '\n'.join(error % path for path in paths) + '\n'
self.assertEqual(expectedErrors, self.log.getvalue())
def checkIntegrity(maxRowsPerQuery=10000):
"""
Check the integrity of the database for cases which the database
engine can't detect.
@param maxRowsPerQuery: Limit the number of rows fetched by SQL queries to
avoid excessive use of memory.
"""
results = _splitResult(getNamespaces(), Namespace.id, maxRowsPerQuery)
for result in results:
namespaces = list(result)
NamespaceIntegrityChecker().check(namespaces)
results = _splitResult(getTags(), Tag.id, maxRowsPerQuery)
for result in results:
tags = list(result)
TagIntegrityChecker().check(tags)
results = _splitResult(getUsers(), User.id, maxRowsPerQuery)
for result in results:
users = list(result)
UserIntegrityChecker().check(users)
results = _splitResult(getAboutTagValues(), AboutTagValue.objectID,
maxRowsPerQuery)
for result in results:
aboutTagValues = list(result)
AboutTagValueIntegrityChecker().check(aboutTagValues)
# In the case of TagValues we limit the query to only tag paths starting
# with "fluiddb" because these are the only ones we're checking and we
# don't want a huge result.
store = getMainStore()
result = store.find(TagValue, TagValue.tagID == Tag.id,
Tag.path.startswith(u'fluiddb/'))
results = _splitResult(getTagValues(), TagValue.id, maxRowsPerQuery)
for result in results:
tagValues = list(result)
TagValueIntegrityChecker().check(tagValues)
def set(self, values):
"""Set or update L{Namespace}s.
@param values: A C{dict} mapping L{Namespace.path}s to descriptions.
@return: A C{list} of C{(objectID, Namespace.path)} 2-tuples
representing the L{Namespace}s that were updated.
"""
from fluiddb.model.user import getUser
paths = set(values.iterkeys())
result = getNamespaces(paths=paths)
namespaces = dict(result.values(Namespace.path, Namespace.objectID))
descriptions = {}
updatedNamespaces = []
for path, description in values.iteritems():
objectID = namespaces[path]
descriptions[objectID] = {
u'fluiddb/namespaces/description': description}
updatedNamespaces.append((objectID, path))
admin = getUser(u'fluiddb')
self._factory.tagValues(admin).set(descriptions)
return updatedNamespaces
def testCheckIntegrityGetsAllRowsUsingMultipleQueries(self):
"""
L{checkIntegrity} should check all the rows of a given object using
multiple queries if the C{maxRowsPerQuery} argument is smaller than the
total number of rows for a given object.
"""
createSystemData()
UserAPI().create([(u'user', u'pass', u'Name',
u'*****@*****.**')])
user = getUser(u'user')
paths = [u'user/namespace%d' % i for i in xrange(10)]
values = [(path, u'description') for path in paths]
NamespaceAPI(user).create(values)
namespaces = getNamespaces(paths=paths)
values = [(namespace.objectID, u'fluiddb/namespaces/path')
for namespace in namespaces]
TagValueAPI(user).delete(values)
checkIntegrity(maxRowsPerQuery=2)
error = 'Integrity Error in namespace %r: Path tag is missing.'
expectedErrors = '\n'.join(error % path for path in paths) + '\n'
self.assertEqual(expectedErrors, self.log.getvalue())
def testCreateCreatesPrivateNamespace(self):
"""
L{User.create} creates a new C{<username>/private} namespace for all
L{User}s. The L{Namespace} has permissions set so that only the owner
can work with data in this namespace.
"""
users = [(u'user', u'secret', u'User', u'*****@*****.**')]
self.users.create(users)
user = getUser(u'user')
namespace = getNamespaces(paths=[u'user/private']).one()
self.assertEqual(u'user/private', namespace.path)
self.assertIdentical(user, namespace.creator)
permission = namespace.permission
self.assertEqual((Policy.CLOSED, [user.id]),
permission.get(Operation.CREATE_NAMESPACE))
self.assertEqual((Policy.CLOSED, [user.id]),
permission.get(Operation.UPDATE_NAMESPACE))
self.assertEqual((Policy.CLOSED, [user.id]),
permission.get(Operation.DELETE_NAMESPACE))
self.assertEqual((Policy.CLOSED, [user.id]),
permission.get(Operation.LIST_NAMESPACE))
self.assertEqual((Policy.CLOSED, [user.id]),
permission.get(Operation.CONTROL_NAMESPACE))
def testCreateCreatesPrivateNamespace(self):
"""
L{User.create} creates a new C{<username>/private} namespace for all
L{User}s. The L{Namespace} has permissions set so that only the owner
can work with data in this namespace.
"""
users = [(u'user', u'secret', u'User', u'*****@*****.**')]
self.users.create(users)
user = getUser(u'user')
namespace = getNamespaces(paths=[u'user/private']).one()
self.assertEqual(u'user/private', namespace.path)
self.assertIdentical(user, namespace.creator)
permission = namespace.permission
self.assertEqual((Policy.CLOSED, [user.id]),
permission.get(Operation.CREATE_NAMESPACE))
self.assertEqual((Policy.CLOSED, [user.id]),
permission.get(Operation.UPDATE_NAMESPACE))
self.assertEqual((Policy.CLOSED, [user.id]),
permission.get(Operation.DELETE_NAMESPACE))
self.assertEqual((Policy.CLOSED, [user.id]),
permission.get(Operation.LIST_NAMESPACE))
self.assertEqual((Policy.CLOSED, [user.id]),
permission.get(Operation.CONTROL_NAMESPACE))
def getUnknownPaths(self, values):
"""Check if the paths in a sequence of path-operation exist.
@param values: A sequence of C{(path, Operation)} 2-tuples.
@raise FeatureError: Raised if an invalid path or L{Operation} is
given.
@return: A C{set} with the unknown paths.
"""
tagPaths = set()
namespacePaths = set()
for path, operation in values:
if path is None:
raise FeatureError('A path must be provided.')
elif operation in Operation.TAG_OPERATIONS:
tagPaths.add(path)
elif operation in Operation.NAMESPACE_OPERATIONS:
namespacePaths.add(path)
else:
raise FeatureError('Invalid operation %s for the path %r' %
(operation, path))
if tagPaths:
existingTags = set(getTags(paths=tagPaths).values(Tag.path))
unknownTags = tagPaths - existingTags
unknownTags.discard(u'fluiddb/id')
else:
unknownTags = set()
if namespacePaths:
result = getNamespaces(paths=namespacePaths).values(Namespace.path)
existingNamespaces = set(result)
unknownNamespaces = namespacePaths - existingNamespaces
else:
unknownNamespaces = set()
return unknownTags.union(unknownNamespaces)
def getUnknownPaths(self, values):
"""Check if the paths in a sequence of path-operation exist.
@param values: A sequence of C{(path, Operation)} 2-tuples.
@raise FeatureError: Raised if an invalid path or L{Operation} is
given.
@return: A C{set} with the unknown paths.
"""
tagPaths = set()
namespacePaths = set()
for path, operation in values:
if path is None:
raise FeatureError('A path must be provided.')
elif operation in Operation.TAG_OPERATIONS:
tagPaths.add(path)
elif operation in Operation.NAMESPACE_OPERATIONS:
namespacePaths.add(path)
else:
raise FeatureError('Invalid operation %s for the path %r'
% (operation, path))
if tagPaths:
existingTags = set(getTags(paths=tagPaths).values(Tag.path))
unknownTags = tagPaths - existingTags
unknownTags.discard(u'fluiddb/id')
else:
unknownTags = set()
if namespacePaths:
result = getNamespaces(paths=namespacePaths).values(Namespace.path)
existingNamespaces = set(result)
unknownNamespaces = namespacePaths - existingNamespaces
else:
unknownNamespaces = set()
return unknownTags.union(unknownNamespaces)
def delete(self, paths):
"""Delete L{Namespace}s matching C{paths}.
@param paths: A sequence of L{Namespace.path}s.
@raises NotEmptyError: Raised if the L{Namespace} is not empty.
@return: A C{list} of C{(objectID, Namespace.path)} 2-tuples
representing the deleted L{Namespace}s.
"""
if isgenerator(paths):
paths = list(paths)
if getChildNamespaces(paths).any() or getChildTags(paths).any():
raise NotEmptyError("Can't delete non-empty namespaces.")
result = getNamespaces(paths=paths)
deletedNamespaces = list(result.values(Namespace.objectID,
Namespace.path))
values = [(objectID, systemTag)
for objectID, _ in deletedNamespaces
for systemTag in (u'fluiddb/namespaces/description',
u'fluiddb/namespaces/path')]
if values:
self._factory.tagValues(self._user).delete(values)
result.remove()
return deletedNamespaces
def create(self, values, createPrivateNamespace=None):
"""Create new L{User}s.
@param values: A sequence of C{(username, password, fullname, email)}
4-tuples.
@param createPrivateNamespace: Optionally, a flag to specify whether
or not the C{<username>/private} L{Namespace} should be created.
Default is C{True}.
@raise DuplicateUserError: Raised if the username for a new L{User}
collides with an existing one.
@raise FeatureError: Raised if C{values} is empty.
@return: A C{list} of C{(objectID, username)} 2-tuples for the new
L{User}s.
"""
if not values:
raise FeatureError('Information about at least one user must be '
'provided.')
# Make sure usernames don't exist before trying to create
# new users.
usernames = [username for username, _, _, _ in values]
result = getUsers(usernames=usernames)
existingUsernames = set(result.values(User.username))
if existingUsernames:
raise DuplicateUserError(existingUsernames)
# Create the users.
systemValues = {}
result = []
privateUpdateResults = []
admin = getUser(u'fluiddb')
objects = self._factory.objects(admin)
for username, password, fullname, email in values:
user = createUser(username, password, fullname, email)
about = u'@%s' % username
user.objectID = objects.create(about)
namespaces = self._factory.namespaces(user)
# Create the user's root namespace.
namespaces.create([(username,
u'Namespace for user %s' % username)])
namespace = getNamespaces(paths=[username]).one()
user.namespaceID = namespace.id
# Create the user's private namespace.
if createPrivateNamespace is None or createPrivateNamespace:
privateNamespaceName = '%s/private' % username
privateUpdateResults.append(
namespaces.create(
[(privateNamespaceName,
u'Private namespace for user %s' % username)]))
namespace = getNamespaces(paths=[privateNamespaceName]).one()
permission = namespace.permission
permission.set(Operation.LIST_NAMESPACE, Policy.CLOSED,
[user.id])
# Create system tags
systemValues[user.objectID] = {
u'fluiddb/users/username': username,
u'fluiddb/users/name': fullname,
u'fluiddb/users/email': email,
u'fluiddb/users/role': unicode(user.role)}
result.append((user.objectID, user.username))
self._factory.tagValues(admin).set(systemValues)
return result
print __doc__
superUser = getUser(u'fluiddb')
tagValues = TagValueAPI(superUser)
print 'Getting tags to fix.'
tagsToFix = []
for tagID, path in getTags().values(Tag.id, Tag.path):
root = path.split('/', 1)[0]
if not root.islower():
tagsToFix.append(tagID)
print 'Getting namespaces to fix.'
namespacesToFix = []
for namespaceID, path in getNamespaces().values(Namespace.id,
Namespace.path):
root = path.split('/', 1)[0]
if not root.islower():
namespacesToFix.append(namespaceID)
print 'Fixing tags.'
i = 0
currentIDs = tagsToFix[i:i + BATCH_SIZE]
while currentIDs:
systemValues = {}
for tag in store.find(Tag, Tag.id.is_in(currentIDs)):
root, rest = tag.path.split('/', 1)
newPath = u'/'.join([root.lower(), rest])
print 'Replacing tag', tag.path, 'with', newPath
tag.path = newPath
