def contains(self, query):
"""Determine if C{query} is contained within this query.
Note that this method only matches simple queries that represent a
single expression.
@param query: The L{Query} to match against this one.
@raises FeatureError: Raised if C{query} is too complex to match.
@return: C{True} if C{query} is present, otherwise C{False}.
"""
if query.rootNode.left is not None:
if (query.rootNode.left.left is not None
or query.rootNode.left.right is not None):
raise FeatureError("Query is too complex to match.")
if query.rootNode.right is not None:
if (query.rootNode.right.left is not None
or query.rootNode.right.right is not None):
raise FeatureError("Query is too complex to match.")
def traverse(node):
if node is None:
return False
elif (node == query.rootNode and node.left == query.rootNode.left
and node.right == query.rootNode.right):
return True
else:
return traverse(node.left) or traverse(node.right)
return traverse(self.rootNode)
def search(self, queries, implicitCreate=True):
"""Find object IDs matching specified L{Query}s.
@param queries: The sequence of L{Query}s to resolve.
@param implicitCreate: Optionally a flag indicating if nonexistent
objects should be created for special tags like C{fluiddb/about}.
Default is L{True}.
@return: A L{SearchResult} configured to resolve the specified
L{Query}s.
"""
if not queries:
raise FeatureError('Queries must be provided.')
idQueries = []
aboutQueries = []
hasQueries = []
solrQueries = []
for query in queries:
if isEqualsQuery(query, u'fluiddb/id'):
idQueries.append(query)
elif isEqualsQuery(query, u'fluiddb/about'):
aboutQueries.append(query)
elif isHasQuery(query):
hasQueries.append(query)
else:
solrQueries.append(query)
index = getObjectIndex()
specialResults = self._resolveAboutQueries(aboutQueries,
implicitCreate)
specialResults.update(self._resolveFluiddbIDQueries(idQueries))
specialResults.update(self._resolveHasQueries(hasQueries))
return SearchResult(index, solrQueries, specialResults)
def getQueryParser():
"""Get a L{QueryParser} to parse Fluidinfo queries.
The process of building a L{QueryParser} is quite expensive. PLY
generates parse tables and writes them to a file on disk. As a result, a
single parser instance is generated and cached in memory. The same
L{QueryParser} instance is returned each time this function is called.
As a result, you must be especially careful about thread-safety. The
L{QueryParser} must only be used to parse one input at a time and never
shared among threads.
@raise FeatureError: Raised if this function is invoked outside the main
thread.
@return: The global L{QueryParser} instance.
"""
if currentThread().getName() != 'MainThread':
raise FeatureError(
'A query parser may only be used in the main thread.')
global _parser
if _parser is None:
lexer = getQueryLexer()
parser = QueryParser(lexer.tokens)
parser.build(module=parser,
debug=False,
outputdir=getConfig().get('service', 'temp-path'))
_parser = parser
# Setup illegal queries to ensure we do it in a safe way and avoid
# races.
getIllegalQueries()
return _parser
def get(self, paths, withDescriptions=None):
"""Get information about L{Tag}s matching C{paths}.
@param paths: A sequence of L{Tag.path}s.
@param withDescriptions: Optionally, a C{bool} indicating whether or
not to include L{Tag} descriptions in the result. Default is
C{False}.
@return: A C{dict} that maps L{Tag.path}s to C{dict}s with information
about matching L{Tag}s, matching the following format::
{<path>: {'id': <object-id>,
'description': <description>}}
"""
if not paths:
raise FeatureError("Can't retrieve an empty list of tags.")
result = getTags(paths=paths)
values = list(result.values(Tag.path, Tag.objectID))
descriptions = (
self._getDescriptions(objectID for path, objectID in values)
if withDescriptions else None)
tags = {}
for path, objectID in values:
value = {'id': objectID}
if withDescriptions:
value['description'] = descriptions.get(objectID, u'')
tags[path] = value
return tags
def delete(self, usernames):
"""Delete L{User}s matching C{username}s.
@param usernames: A sequence of L{User.username}s.
@raise FeatureError: Raised if no L{User.username}s are provided.
@raise UnknownUserError: Raised if one or more usernames don't match
existing L{User}s.
@return: A C{list} of C{(objectID, User.username)} 2-tuples
representing the L{User}s that that were removed.
"""
if isgenerator(usernames):
usernames = list(usernames)
if not usernames:
raise FeatureError('At least one username must be provided.')
usernames = set(usernames)
result = getUsers(usernames=usernames)
existingUsernames = set(result.values(User.username))
unknownUsernames = usernames - existingUsernames
if unknownUsernames:
raise UnknownUserError(list(unknownUsernames))
admin = getUser(u'fluiddb')
deletedUsers = list(result.values(User.objectID, User.username))
# FIXME: Deleting a user will leave the permission exception lists
# containing the user in a corrupt state.
result.remove()
self._factory.tagValues(admin).delete([
(objectID, systemTag) for objectID, _ in deletedUsers
for systemTag in [
u'fluiddb/users/username', u'fluiddb/users/name',
u'fluiddb/users/email', u'fluiddb/users/role'
]
])
return deletedUsers
def get(self, objectIDs, paths=None):
"""Get L{TagValue}s matching filtering criteria.
@param objectIDs: A sequence of object IDs to retrieve values for.
@param paths: Optionally, a sequence of L{Tag.path}s to return. The
default is to return values for all available L{Tag.path}s.
@raise FeatureError: Raised if any of the arguments is empty or
C{None}.
@return: A C{dict} mapping object IDs to tags and values, matching the
following format::
{<object-id>: {<path>: <L{TagValue}>}}
"""
if not objectIDs:
raise FeatureError("Can't get tag values for an empty list of "
'object IDs.')
if not paths:
objects = self._factory.objects(self._user)
paths = objects.getTagsForObjects(objectIDs)
result = {}
if u'fluiddb/id' in paths:
for objectID in objectIDs:
tagValue = FluidinfoTagValue.fromObjectID(objectID)
result[objectID] = {u'fluiddb/id': tagValue}
# Avoid querying the database if only the 'fluiddb/id' tag has
# been requested, since we already have the results.
if [u'fluiddb/id'] == paths:
return result
collection = TagValueCollection(objectIDs=objectIDs, paths=paths)
for tag, tagValue in collection.values():
if tagValue.objectID not in result:
result[tagValue.objectID] = {}
tagValue = FluidinfoTagValue.fromTagValue(tagValue)
if isinstance(tagValue.value, dict):
# We have to make a copy of the value because we don't want
# storm to try to add the 'contents' binary value to the
# database.
tagValue.value = dict(tagValue.value)
opaque = getOpaqueValues([tagValue.id]).one()
if opaque is None:
raise RuntimeError('Opaque value not found.')
tagValue.value['contents'] = opaque.content
result[tagValue.objectID][tag.path] = tagValue
return result
def getUnknownPaths(self, values):
"""Check if the paths in a sequence of path-operation exist.
@param values: A sequence of C{(path, Operation)} 2-tuples.
@raise FeatureError: Raised if an invalid path or L{Operation} is
given.
@return: A C{set} with the unknown paths.
"""
tagPaths = set()
namespacePaths = set()
for path, operation in values:
if path is None:
raise FeatureError('A path must be provided.')
elif operation in Operation.TAG_OPERATIONS:
tagPaths.add(path)
elif operation in Operation.NAMESPACE_OPERATIONS:
namespacePaths.add(path)
else:
raise FeatureError('Invalid operation %s for the path %r' %
(operation, path))
if tagPaths:
existingTags = set(getTags(paths=tagPaths).values(Tag.path))
unknownTags = tagPaths - existingTags
unknownTags.discard(u'fluiddb/id')
else:
unknownTags = set()
if namespacePaths:
result = getNamespaces(paths=namespacePaths).values(Namespace.path)
existingNamespaces = set(result)
unknownNamespaces = namespacePaths - existingNamespaces
else:
unknownNamespaces = set()
return unknownTags.union(unknownNamespaces)
def set(self, values):
"""Update information about L{User}s.
If an incoming field is C{None} the appropriate instance field will not
be modified.
@param values: A sequence of C{(username, password, fullname, email,
role)} 5-tuples.
@raise FeatureError: Raised if C{values} is empty.
@raise UnknownUserError: Raised if a specified L{User} does not exist.
@return: A 2-tuples representing the L{User}s that were updated.
"""
if not values:
raise FeatureError('Information about at least one user must be '
'provided.')
usernames = set(username for username, _, _, _, _ in values)
users = dict(
(user.username, user) for user in getUsers(usernames=usernames))
existingUsernames = set(users.iterkeys())
unknownUsernames = usernames - existingUsernames
if unknownUsernames:
raise UnknownUserError(list(unknownUsernames))
result = []
systemValues = {}
for username, password, fullname, email, role in values:
user = users[username]
valuesToUpdate = {}
if password is not None:
user.passwordHash = hashPassword(password)
if fullname is not None:
user.fullname = fullname
valuesToUpdate[u'fluiddb/users/name'] = user.fullname
if email is not None:
user.email = email
valuesToUpdate[u'fluiddb/users/email'] = user.email
if role is not None:
user.role = role
valuesToUpdate[u'fluiddb/users/role'] = unicode(user.role)
if valuesToUpdate:
systemValues[user.objectID] = valuesToUpdate
result.append((user.objectID, user.username))
if systemValues:
admin = getUser(u'fluiddb')
self._factory.tagValues(admin).set(systemValues)
return result
def create(self, values):
"""Create new L{Tag}s.
L{Namespace}s that don't exist are created automatically before
L{Tag}s are created. Associated L{NamespacePermission} and
L{TagPermission}s are created automatically with the system-wide
default permissions.
@param values: A sequence of C{(Tag.path, description)} 2-tuples.
@raise DuplicatePathError: Raised if the path for a new L{Tag}
collides with an existing one.
@raise FeatureError: Raised if the given list of values is empty.
@raise UnknownParentPathError: Raised if the parent for a new L{Tag}
can't be found.
@raise MalformedPathError: Raised if one of the given paths is empty
or has unacceptable characters.
@return: A C{list} of C{(objectID, path)} 2-tuples for the new L{Tag}s.
"""
if not values:
raise FeatureError("Can't create an empty list of tags.")
# Make sure tag paths don't exist before trying to create new tags.
paths = [path for path, _ in values]
existingPaths = list(getTags(paths=paths).values(Tag.path))
if existingPaths:
raise DuplicatePathError(
'Paths already exist: %s' % ', '.join(existingPaths))
# Get intermediate namespaces. If they don't exist, create them
# automatically.
paths = [path for (path, _) in values]
parentPaths = getParentPaths(paths)
missingParentPaths = self._getMissingNamespaces(parentPaths)
self._factory.namespaces(self._user).create(
[(path, u'Object for the namespace %s' % path)
for path in missingParentPaths])
# Create the new tags.
result = getNamespaces(paths=parentPaths)
parentNamespaces = dict((namespace.path, namespace)
for namespace in result)
return self._createTags(values, parentNamespaces)
def delete(self, values):
"""Delete L{TagValue}s.
@param values: A sequence of C{(objectID, Tag.path)} 2-tuples to
delete values for.
@raise FeatureError: Raised if the given list of values is empty.
@return: The number of values deleted.
"""
if isgenerator(values):
values = list(values)
if not values:
raise FeatureError("Can't delete an empty list of tag values.")
paths = set([path for objectID, path in values])
objectIDs = set([objectID for objectID, path in values])
tagIDs = dict(getTags(paths).values(Tag.path, Tag.id))
values = [(objectID, tagIDs[path]) for objectID, path in values]
result = getTagValues(values).remove()
if result:
touchObjects(objectIDs)
return result
def get(self, usernames):
"""Get information about L{User}s matching C{usernames}.
@param usernames: A sequence of L{User.username}s.
@raise FeatureError: Raised if no L{User.username}s are provided.
@return: A C{dict} that maps L{User.username}s to C{dict}s with
information about matching L{User}s, matching the following
format::
{<username>: {'id': <object-id>,
'name': <full-name>,
'role': <role>}}
"""
if not usernames:
raise FeatureError('At least one username must be provided.')
result = getUsers(usernames=usernames)
result = result.values(User.objectID, User.username, User.fullname,
User.role)
users = {}
for objectID, username, name, role in result:
users[username] = {'id': objectID, 'name': name, 'role': role}
return users
def get(self, values):
"""Get permissions matching pairs of paths and L{Operation}s.
@param values: A sequence of C{(path, Operation)} tuples.
@raise FeatureError: Raised if the given list of values is empty.
@return: A C{dict} that maps C{(path, Operation)} tuples to C{(Policy,
exceptions)} tuples. Example::
{(<path>, <operation>): (<policy>, ['user1', 'user2', ...]), ...}
"""
if not values:
raise FeatureError("Can't get an empty list of permissions.")
# Get the requested permission data.
permissions = {}
namespaceValues = []
tagValues = []
for path, operation in values:
if operation in Operation.NAMESPACE_OPERATIONS:
namespaceValues.append((path, operation))
else:
tagValues.append((path, operation))
if namespaceValues:
permissions.update(self._getNamespacePermissions(namespaceValues))
if tagValues:
permissions.update(self._getTagPermissions(tagValues))
# Translate User.id's in the exception lists to User.username's.
userIDs = set()
for key, (policy, exceptions) in permissions.iteritems():
userIDs.update(exceptions)
usernames = dict(getUsers(ids=userIDs).values(User.id, User.username))
for key, (policy, exceptions) in permissions.items():
permissions[key] = (policy,
[usernames[userID] for userID in exceptions])
return permissions
def set(self, values):
"""Set or update L{TagValue}s.
L{Tag}s that don't exist are created automatically before L{TagValue}s
are stored. Associated L{TagPermission}s are created automatically
with the system-wide default permissions.
@param values: A C{dict} mapping object IDs to tags and values,
matching the following format::
{<object-id>: {<path>: <value>,
<path>: {'mime-type': <mime-type>,
'contents': <contents>}}}
A binary L{TagValue} is represented using a different layout than
other values types, as shown for the second value.
@raise FeatureError: Raised if the given list of values is empty.
@raise MalformedPathError: Raised if one of the given paths for a
nonexistent tag is empty or has unacceptable characters.
"""
if not values:
raise FeatureError("Can't set an empty list of tag values.")
objectIDs = set(values.keys())
# Implicitly create missing tags, if there are any.
paths = set()
for tagValues in values.itervalues():
paths.update(tagValues.iterkeys())
tagIDs = dict(getTags(paths=paths).values(Tag.path, Tag.id))
existingPaths = set(tagIDs.iterkeys())
unknownPaths = paths - existingPaths
if unknownPaths:
tags = [(path, u'Object for the attribute %s' % path)
for path in unknownPaths]
self._factory.tags(self._user).create(tags)
tagIDs = dict(getTags(paths=paths).values(Tag.path, Tag.id))
# Delete all existing tag values for the specified object IDs and
# paths.
deleteValues = []
for objectID in values:
for path in values[objectID].iterkeys():
deleteValues.append((objectID, tagIDs[path]))
getTagValues(deleteValues).remove()
# Set new tag values for the specified object IDs and paths.
for objectID in values:
tagValues = values[objectID]
for path, value in tagValues.iteritems():
tagID = tagIDs[path]
if isinstance(value, dict):
content = value['contents']
value = createTagValue(self._user.id, tagID, objectID, {
'mime-type': value['mime-type'],
'size': len(content)
})
# This is necessary to tell PostgreSQL that generates a
# `value.id` immediately.
value.id = AutoReload
createOpaqueValue(value.id, content)
else:
createTagValue(self._user.id, tagID, objectID, value)
touchObjects(objectIDs)
def getFollowedObjects(self,
username,
limit=20,
olderThan=None,
objectType=None):
"""Get the objects followed by the specified user.
@param username: The user to get the followed objects for.
@param limit: Optionally, The maximum number of objects to return.
@param olderThan: Optionally a C{datetime} indicating to return only
objects followed before the specified time.
@param objectType: Optionally, a C{str} representing the object type
to filter from the objects. The allowed values are C{url},
C{user} and C{hashtag}.
@return: A C{list} of objects followed by C{username} represented by
a C{dict} with the following format::
[
{
'about': '<about>',
'creationTime': <float_timestamp>,
'following': True
},
...
]
"""
store = getMainStore()
where = [
TagValue.tagID == Tag.id,
TagValue.objectID == AboutTagValue.objectID,
Tag.path == username + u'/follows'
]
if olderThan is not None:
where.append(TagValue.creationTime < olderThan)
if objectType is not None:
if objectType == 'user':
where.append(Like(AboutTagValue.value, u'@%'))
elif objectType == 'url':
where.append(Like(AboutTagValue.value, u'http%'))
elif objectType == 'hashtag':
where.append(Like(AboutTagValue.value, u'#%'))
else:
raise FeatureError('Unknown object type.')
result = store.find(
(TagValue.objectID, AboutTagValue.value, TagValue.creationTime),
where)
result = result.order_by(Desc(TagValue.creationTime))
result = list(result.config(limit=limit))
objectIDs = [objectID for objectID, _, _ in result]
if self._user.username != username:
callerObjectIDs = set(
store.find(TagValue.objectID, Tag.id == TagValue.tagID,
Tag.path == self._user.username + u'/follows',
TagValue.objectID.is_in(objectIDs)))
else:
callerObjectIDs = None
return [{
u'about':
about,
u'following': (objectID in callerObjectIDs
if callerObjectIDs is not None else True),
u'creationTime': (timegm(creationTime.utctimetuple()) +
float(creationTime.strftime('0.%f')))
} for objectID, about, creationTime in result]
def create(self,
text,
username,
about=None,
importer=None,
when=None,
url=None):
"""Create a new comment.
@param text: The C{unicode} comment text.
@param username: the C{unicode} username of the commenter.
@param about: Optionally, a C{list} of C{unicode} values the comment is
about.
@param importer: A C{unicode} string giving the name of the importer.
@param when: A C{datetime.datetime} instance or C{None} if the
current time should be used.
@param url: A C{str} URL or C{None} if there is no URL associated with
this comment.
@raise L{FeatureError}: if (1) the comment text is C{None} or is all
whitespace, or (2) if the importer name contains the separator
(space) that we use in the about value for comment objects.
@return: A C{dict} as follows:
{
fluidinfo.com/info/about: A C{list} of all the about values
(i.e., URLs and hashtags) in the comment text, including
the thing the comment was about (if anything). The hashtags
are in lowercase.
fluidinfo.com/info/timestamp: The C{int} UTC timestamp (seconds
since the epoch) the comment was created at.
fluidinfo.com/info/url: The C{url}, as received.
fluidinfo.com/info/username: The C{username}, as received.
}
"""
if not text or text.strip() == '':
raise FeatureError('Comment text non-existent or just whitespace.')
if importer:
if ' ' in importer:
raise FeatureError('Comment importer name contains a space.')
else:
importer = u'fluidinfo.com'
when = when or datetime.utcnow()
floatTime = timegm(when.utctimetuple()) + float(when.strftime('0.%f'))
isoTime = when.isoformat()
if not url:
url = 'https://fluidinfo.com/comment/%s/%s/%s' % (
importer, username, isoTime)
# Put all the explicit about values into a list called abouts. Items
# are stripped and those that are not URLs are lowercased.
abouts = []
if about:
for item in map(unicode.strip, about):
abouts.append(item if URL_REGEX.match(item) else item.lower())
abouts.extend(self._extractAbouts(text))
abouts = uniqueList(abouts)
commentObjectAbout = u'%s %s %s' % (importer, username, isoTime)
commentID = self._objects.create(commentObjectAbout)
values = {
u'fluidinfo.com/info/about': abouts,
u'fluidinfo.com/info/username': username,
u'fluidinfo.com/info/text': text,
u'fluidinfo.com/info/url': url,
u'fluidinfo.com/info/timestamp': floatTime
}
self._tagValues.set({commentID: values})
if abouts:
# Get all the object IDs of the target objects. If an object does
# not exist, create it.
result = getAboutTagValues(values=abouts)
existingObjects = dict(
result.values(AboutTagValue.value, AboutTagValue.objectID))
missingAbouts = set(abouts) - set(existingObjects.iterkeys())
for aboutValue in missingAbouts:
existingObjects[aboutValue] = self._objects.create(aboutValue)
createComment(commentID, existingObjects.values(), username, when)
return values
def create(self, values, createPrivateNamespace=None):
"""Create new L{User}s.
@param values: A sequence of C{(username, password, fullname, email)}
4-tuples.
@param createPrivateNamespace: Optionally, a flag to specify whether
or not the C{<username>/private} L{Namespace} should be created.
Default is C{True}.
@raise DuplicateUserError: Raised if the username for a new L{User}
collides with an existing one.
@raise FeatureError: Raised if C{values} is empty.
@return: A C{list} of C{(objectID, username)} 2-tuples for the new
L{User}s.
"""
if not values:
raise FeatureError('Information about at least one user must be '
'provided.')
# Make sure usernames don't exist before trying to create
# new users.
usernames = [username for username, _, _, _ in values]
result = getUsers(usernames=usernames)
existingUsernames = set(result.values(User.username))
if existingUsernames:
raise DuplicateUserError(existingUsernames)
# Create the users.
systemValues = {}
result = []
privateUpdateResults = []
admin = getUser(u'fluiddb')
objects = self._factory.objects(admin)
for username, password, fullname, email in values:
user = createUser(username, password, fullname, email)
about = u'@%s' % username
user.objectID = objects.create(about)
namespaces = self._factory.namespaces(user)
# Create the user's root namespace.
namespaces.create([(username, u'Namespace for user %s' % username)
])
namespace = getNamespaces(paths=[username]).one()
user.namespaceID = namespace.id
# Create the user's private namespace.
if createPrivateNamespace is None or createPrivateNamespace:
privateNamespaceName = '%s/private' % username
privateUpdateResults.append(
namespaces.create([
(privateNamespaceName,
u'Private namespace for user %s' % username)
]))
namespace = getNamespaces(paths=[privateNamespaceName]).one()
permission = namespace.permission
permission.set(Operation.LIST_NAMESPACE, Policy.CLOSED,
[user.id])
# Create system tags
systemValues[user.objectID] = {
u'fluiddb/users/username': username,
u'fluiddb/users/name': fullname,
u'fluiddb/users/email': email,
u'fluiddb/users/role': unicode(user.role)
}
result.append((user.objectID, user.username))
self._factory.tagValues(admin).set(systemValues)
return result