def removeTestingData():
"""
Delete L{User}s, L{Namespace}s and L{Tag}s used for testing purposes.
"""
admin = getUser(u'fluiddb')
logging.info('Deleting testing tags.')
result = TagAPI(admin).get(TESTING_DATA[u'tags'])
if result:
TagAPI(admin).delete(result.keys())
logging.info('Deleting testing namespaces.')
result = NamespaceAPI(admin).get(TESTING_DATA[u'namespaces'])
# we must delete namespaces one by one, otherwise we'll get NotEmptyError.
for path in sorted(result.keys(), reverse=True):
NamespaceAPI(admin).delete([path])
logging.info('Deleting testing users.')
result = UserAPI().get(TESTING_DATA[u'users'])
if result:
for username in result:
path = '%s/private' % username
try:
NamespaceAPI(admin).delete([path])
except FeatureError:
# FIXME This is a bit crap, but it's faster than checking to
# see if the namespace exists before attempting to delete it.
continue
if result:
UserAPI().delete(result.keys())
getMainStore().commit()
def prepareForTesting():
"""
Create a set of L{User}s, L{Namespace}s and L{Tag}s for testing purposes.
"""
admin = getUser(u'fluiddb')
logging.info('Creating testing users.')
UserAPI().create([(username, 'secret', u'Test user', u'*****@*****.**')
for username in TESTING_DATA[u'users']])
logging.info('Creating testing namespaces.')
NamespaceAPI(admin).create([(namespace, u'Used for testing purposes.')
for namespace in TESTING_DATA[u'namespaces']])
logging.info('Creating testing tags.')
TagAPI(admin).create([(tag, u'Used for testing purposes.')
for tag in TESTING_DATA[u'tags']])
getMainStore().commit()
def createTagPermission(tag):
"""Create a L{TagPermission}.
Permissions are inherited as follows:
- L{Operation.UPDATE_TAG} inherits its L{Policy} and exceptions list from
the permissions for L{Operation.UPDATE_NAMESPACE}.
- L{Operation.DELETE_TAG} inherits its L{Policy} and exceptions list from
the permissions for L{Operation.DELETE_NAMESPACE}.
- L{Operation.CONTROL_TAG} inherits its L{Policy} and exceptions list
from the permissions for L{Operation.CONTROL_NAMESPACE}.
- L{Operation.WRITE_TAG_VALUE} inherits its L{Policy} and exceptions list
from the permissions for L{Operation.UPDATE_NAMESPACE}.
- L{Operation.READ_TAG_VALUE} inherits its L{Policy} and exceptions list
from the permissions for L{Operation.LIST_NAMESPACE}.
- L{Operation.DELETE_TAG_VALUE} inherits its L{Policy} and exceptions
list from the permissions for L{Operation.DELETE_NAMESPACE}.
- L{Operation.CONTROL_TAG_VALUE} inherits its L{Policy} and exceptions
list from the permissions for L{Operation.CONTROL_NAMESPACE}.
@param tag: The L{Tag} to create permissions for.
@return: A new L{TagPermission} instance.
"""
store = getMainStore()
permission = TagPermission(tag.namespace.creator.id, tag.id)
permissionTemplate = tag.namespace.permission
if permissionTemplate is not None:
for operation in TAG_PERMISSION_INHERITANCE_MAP.iterkeys():
matchingOperation = TAG_PERMISSION_INHERITANCE_MAP.get(operation)
policy, exceptions = permissionTemplate.get(matchingOperation)
permission.set(operation, policy, exceptions)
return store.add(permission)
def _getAllTagValues(updatedSince):
"""
Get all L{Tag} and L{TagValue}s that have been updated since the
provided C{datetime}.
@param updatedSince: An inclusive C{datetime} offset from which to
update new tag-values.
@return: A C{generator} that yields each updated tag individually.
"""
store = getMainStore()
CHUNK_SIZE = 500000
result = store.find(TagValue, TagValue.creationTime >= updatedSince)
totalRows = result.count()
chunks = totalRows / CHUNK_SIZE
if chunks == 0:
chunks = 1
for i in range(chunks):
limit = CHUNK_SIZE if (i != chunks - 1) else None
offset = i * CHUNK_SIZE
result = store.find((Tag.path, TagValue.value, TagValue.objectID,
TagValue.creationTime),
Tag.id == TagValue.tagID,
TagValue.creationTime >= updatedSince)
result = result.order_by(TagValue.objectID, TagValue.creationTime)
result = result.config(limit=limit, offset=offset)
for row in result:
yield row[:-1]
def deleteComment(objectID):
"""Deletes a L{Comment}.
@param objectID: The object ID of the comment.
@return: An C{int} count of the number of comments removed.
"""
store = getMainStore()
return store.find(Comment, Comment.objectID == objectID).remove()
def testGetMainStore(self):
"""
L{getMainStore} returns a C{Store} instance for the main store, when
one has been properly configured.
"""
store = getMainStore()
self.assertTrue(isinstance(store, Store))
self.assertIdentical(self.store, store)
def createTwitterUser(user, uid):
"""Create a L{TwitterUser}.
@param user: The L{User} to link to a Twitter account.
@param uid: The Twitter UID for the user.
@return: A new L{TwitterUser} instance persisted in the main store.
"""
store = getMainStore()
return store.add(TwitterUser(user.id, uid))
def createAboutTagValue(objectID, value):
"""Create a new L{AboutTagValue}.
@param objectID: The object ID this value is associated with.
@param value: The value to store.
@return: An L{AboutTagValue} instance, added to the database.
"""
store = getMainStore()
return store.add(AboutTagValue(objectID, value))
def getOpaqueValues(valueIDs):
"""Get L{OpaqueValue}s for the given L{TagValue}s.
@param valueIDs: A sequence of L{TagValue.id}s.
@return: A C{ResultSet} with L{OpaqueValue}s.
"""
store = getMainStore()
return store.find(OpaqueValue,
OpaqueValue.fileID == OpaqueValueLink.fileID,
OpaqueValueLink.valueID.is_in(valueIDs))
def _getTagIDs(self):
"""Get a L{Tag.id}s for path filtering criteria.
@return: A C{list} of L{Tag.id}s for paths or C{None} if no path
filtering criteria are available.
"""
paths = self._criteria.get('paths')
if paths:
store = getMainStore()
return list(store.find(Tag.id, Tag.path.is_in(paths)))
def getChildTags(paths):
"""Get child L{Tag}s.
@param paths: A sequence of L{Namespace.path}s to get child L{Tag}s for.
@return: A C{ResultSet} with matching L{Namespace} children.
"""
store = getMainStore()
result = getNamespaces(paths)
subselect = result.get_select_expr(Namespace.id)
return store.find(Tag, Tag.namespaceID.is_in(subselect))
def summarizeObject(self, about):
"""Get summary information for an object.
@param about: The about value of the object to summarize.
@return: A C{dict} matching the following format::
{'commentCount': <count>,
'followers': [<username>, ...],
'relatedObjects': {'<about>': <count>, ...}}
"""
# List followers.
result = self._objects.get([about])
objectID = result.get(about)
if objectID is None:
return {'commentCount': 0,
'followers': [],
'relatedObjects': {}}
paths = self._objects.getTagsForObjects([objectID])
followers = []
for path in paths:
parent = getParentPath(path)
if parent and not u'/' in parent and path.endswith('/follows'):
followers.append(parent)
# Count comments.
store = getMainStore()
result = store.find(CommentObjectLink.commentID,
CommentObjectLink.objectID == objectID)
commentCount = result.count()
# Count related objects. I'm using raw SQL here because don't know how
# to translate this query to Storm.
result = store.execute("""
SELECT about_tag_values."value", summary.count
FROM (
SELECT comment_object_link.object_id AS object_id,
COUNT(comment_object_link.comment_id) AS count
FROM comment_object_link
WHERE comment_object_link.comment_id IN (
SELECT comment_object_link.comment_id
FROM comment_object_link
WHERE comment_object_link.object_id = '{objectID}')
AND comment_object_link.object_id != '{objectID}'
GROUP BY comment_object_link.object_id
) AS summary JOIN about_tag_values
ON summary.object_id = about_tag_values.object_id;
""".format(objectID=objectID))
relatedObjects = dict(result)
return {'commentCount': commentCount,
'followers': followers,
'relatedObjects': relatedObjects}
def getObjectIDs(paths):
"""Get object IDs for L{Tag.path}s.
@param tags: A sequence of L{Tag.path}s.
@return: A C{ResultSet} yielding C{TagValue.objectID}s.
"""
if not paths:
return EmptyResultSet()
store = getMainStore()
return store.find(TagValue.objectID, Tag.id == TagValue.tagID,
Tag.path.is_in(paths))
def getTagPathsAndObjectIDs(objectIDs):
"""Get L{Tag.path}s for object IDs.
@param objectIDs: A sequence of object IDs.
@return: A C{ResultSet} yielding C{(Tag.path, objectID)} 2-tuples.
"""
if not objectIDs:
return EmptyResultSet()
store = getMainStore()
return store.find((Tag.path, TagValue.objectID), Tag.id == TagValue.tagID,
TagValue.objectID.is_in(objectIDs))
def createTagValue(creatorID, tagID, objectID, value):
"""Create a new L{TagValue}.
@param creatorID: The L{User.id} of the person creating this value.
@param tagID: The L{Tag.id} this value is associated with.
@param objectID: The object ID this value is associated with.
@param value: The value to store.
@return: A L{TagValue} instance, added to the database.
"""
store = getMainStore()
return store.add(TagValue(creatorID, tagID, objectID, value))
def values(self):
"""
Get L{Tag} values that match the filtering criteria defined for this
collection.
@return: A L{ResultSet} yielding C{(Tag, TagValue)} 2-tuples.
"""
store = getMainStore()
tagIDs = self._getTagIDs()
where = self._getWhereClause(tagIDs)
return store.find((Tag, TagValue), *where)
def createTagValue(creatorID, tagID, objectID, value):
"""Create a new L{TagValue}.
@param creatorID: The L{User.id} of the person creating this value.
@param tagID: The L{Tag.id} this value is associated with.
@param objectID: The object ID this value is associated with.
@param value: The value to store.
@return: A L{TagValue} instance, added to the database.
"""
store = getMainStore()
return store.add(TagValue(creatorID, tagID, objectID, value))
def getChildNamespaces(paths):
"""Get child L{Namespace}s.
@param paths: A sequence of L{Namespace.path}s to get child L{Namespace}s
for.
@return: A C{ResultSet} with matching L{Namespace} children.
"""
store = getMainStore()
result = getNamespaces(paths)
subselect = result.get_select_expr(Namespace.id)
return store.find(Namespace, Namespace.parentID.is_in(subselect))
def getTagPermissions(paths):
"""Get L{Tag}s and L{TagPermission}s for the specified paths.
@param paths: A sequence of L{Tag.path}s to get L{Tag}s and
L{TagPermission}s for.
@return: A C{ResultSet} yielding C{(Tag, TagPermission)} 2-tuples for the
specified L{Tag.path}s.
"""
store = getMainStore()
return store.find((Tag, TagPermission), TagPermission.tagID == Tag.id,
Tag.path.is_in(paths))
def update(self, importer, username, when, newText):
"""Updates the text of a comment.
All object associations previously extracted from the old comment's
text are removed and new associations extracted from the new text are
added. Object associations not extracted from the text are kept.
@param importer: A C{unicode} string giving the name of the importer.
@param username: The C{unicode} username of the commenter.
@param when: A C{datetime.datetime} instance.
@param text: The new text for the comment.
@return: A C{dict} as follows:
{
fluidinfo.com/info/about: A C{list} of all the about values
(i.e., URLs and hashtags) in the comment text, including
the thing the comment was about (if anything). The hashtags
are in lowercase.
fluidinfo.com/info/timestamp: The C{int} UTC timestamp (seconds
since the epoch) the comment was created at.
fluidinfo.com/info/url: The C{url}, as received.
fluidinfo.com/info/username: The C{username}, as received.
}
"""
# 1. Get the object ID of the comment object.
isoTime = when.isoformat()
commentObjectAbout = u'%s %s %s' % (importer, username, isoTime)
aboutValue = getAboutTagValues(values=[commentObjectAbout]).one()
if aboutValue is None:
raise RuntimeError('Comment does not exist.')
commentID = aboutValue.objectID
# 2. Get the old text and url of the comment.
result = self._tagValues.get(
[commentID],
[u'fluidinfo.com/info/text', u'fluidinfo.com/info/url'])
oldText = result[commentID][u'fluidinfo.com/info/text'].value
url = result[commentID][u'fluidinfo.com/info/url'].value
# 3. Get abouts in comment's text.
aboutsInText = self._extractAbouts(oldText)
# 4. Get all the about values associated with the comment.
store = getMainStore()
allAbouts = store.find(
AboutTagValue.value, CommentObjectLink.commentID == commentID,
AboutTagValue.objectID == CommentObjectLink.objectID)
# 5. Get abouts not in comment's text:
aboutsNotInText = set(allAbouts) - set(aboutsInText)
self.delete(importer, username, when)
return self.create(newText, username, aboutsNotInText, importer, when,
url)
def getDirtyObjects(objectIDs=None):
"""Get L{DirtyObject}s.
@param objectIDs: Optionally, a sequence of L{DirtyObject.objectID}s to
filter the result with.
@return: A C{ResultSet} with matching L{DirtyObject}s.
"""
store = getMainStore()
where = []
if objectIDs:
where.append(DirtyObject.objectID.is_in(objectIDs))
return store.find(DirtyObject, *where)
def update(self, importer, username, when, newText):
"""Updates the text of a comment.
All object associations previously extracted from the old comment's
text are removed and new associations extracted from the new text are
added. Object associations not extracted from the text are kept.
@param importer: A C{unicode} string giving the name of the importer.
@param username: The C{unicode} username of the commenter.
@param when: A C{datetime.datetime} instance.
@param text: The new text for the comment.
@return: A C{dict} as follows:
{
fluidinfo.com/info/about: A C{list} of all the about values
(i.e., URLs and hashtags) in the comment text, including
the thing the comment was about (if anything). The hashtags
are in lowercase.
fluidinfo.com/info/timestamp: The C{int} UTC timestamp (seconds
since the epoch) the comment was created at.
fluidinfo.com/info/url: The C{url}, as received.
fluidinfo.com/info/username: The C{username}, as received.
}
"""
# 1. Get the object ID of the comment object.
isoTime = when.isoformat()
commentObjectAbout = u'%s %s %s' % (importer, username, isoTime)
aboutValue = getAboutTagValues(values=[commentObjectAbout]).one()
if aboutValue is None:
raise RuntimeError('Comment does not exist.')
commentID = aboutValue.objectID
# 2. Get the old text and url of the comment.
result = self._tagValues.get([commentID], [u'fluidinfo.com/info/text',
u'fluidinfo.com/info/url'])
oldText = result[commentID][u'fluidinfo.com/info/text'].value
url = result[commentID][u'fluidinfo.com/info/url'].value
# 3. Get abouts in comment's text.
aboutsInText = self._extractAbouts(oldText)
# 4. Get all the about values associated with the comment.
store = getMainStore()
allAbouts = store.find(
AboutTagValue.value,
CommentObjectLink.commentID == commentID,
AboutTagValue.objectID == CommentObjectLink.objectID)
# 5. Get abouts not in comment's text:
aboutsNotInText = set(allAbouts) - set(aboutsInText)
self.delete(importer, username, when)
return self.create(newText, username, aboutsNotInText, importer, when,
url)
def getTagPermissions(paths):
"""Get L{Tag}s and L{TagPermission}s for the specified paths.
@param paths: A sequence of L{Tag.path}s to get L{Tag}s and
L{TagPermission}s for.
@return: A C{ResultSet} yielding C{(Tag, TagPermission)} 2-tuples for the
specified L{Tag.path}s.
"""
store = getMainStore()
return store.find((Tag, TagPermission),
TagPermission.tagID == Tag.id,
Tag.path.is_in(paths))
def getNamespacePermissions(paths):
"""Get L{Namespace}s and L{NamespacePermission}s for the specified paths.
@param paths: A sequence of L{Namespace.path}s to get L{Namespace}s and
L{NamespacePermission}s for.
@return: A C{ResultSet} yielding C{(Namespace, NamespacePermission)}
2-tuples for the specified L{Namespace.path}s.
"""
store = getMainStore()
return store.find((Namespace, NamespacePermission),
NamespacePermission.namespaceID == Namespace.id,
Namespace.path.is_in(paths))
def getObjectIDs(paths):
"""Get object IDs for L{Tag.path}s.
@param tags: A sequence of L{Tag.path}s.
@return: A C{ResultSet} yielding C{TagValue.objectID}s.
"""
if not paths:
return EmptyResultSet()
store = getMainStore()
return store.find(TagValue.objectID,
Tag.id == TagValue.tagID,
Tag.path.is_in(paths))
def getNamespacePermissions(paths):
"""Get L{Namespace}s and L{NamespacePermission}s for the specified paths.
@param paths: A sequence of L{Namespace.path}s to get L{Namespace}s and
L{NamespacePermission}s for.
@return: A C{ResultSet} yielding C{(Namespace, NamespacePermission)}
2-tuples for the specified L{Namespace.path}s.
"""
store = getMainStore()
return store.find((Namespace, NamespacePermission),
NamespacePermission.namespaceID == Namespace.id,
Namespace.path.is_in(paths))
def getTagPathsAndObjectIDs(objectIDs):
"""Get L{Tag.path}s for object IDs.
@param objectIDs: A sequence of object IDs.
@return: A C{ResultSet} yielding C{(Tag.path, objectID)} 2-tuples.
"""
if not objectIDs:
return EmptyResultSet()
store = getMainStore()
return store.find((Tag.path, TagValue.objectID),
Tag.id == TagValue.tagID,
TagValue.objectID.is_in(objectIDs))
def getTwitterUsers(uids=None):
"""Get C{(User, TwitterUser)} 2-tuples matching specified Twitter UIDs.
@param uids: Optionally, a sequence of L{TwitterUser.uid}s to filter the
results with.
@return: A C{ResultSet} with matching C{(User, TwitterUser)} 2-tuples.
"""
store = getMainStore()
where = []
if uids:
where.append(TwitterUser.uid.is_in(uids))
return store.find((User, TwitterUser), User.id == TwitterUser.userID,
*where)
def getTagPathsForObjectIDs(objectIDs):
"""Get L{Tag.path}s for object IDs.
@param objectIDs: A sequence of object IDs.
@return: A C{ResultSet} yield L{Tag.path} values.
"""
if not objectIDs:
return EmptyResultSet()
store = getMainStore()
result = store.find(Tag.path, Tag.id == TagValue.tagID,
TagValue.objectID.is_in(objectIDs))
result.config(distinct=True)
return result
def createOpaqueValue(valueID, content):
"""Create a new L{OpaqueValue} associated with the given L{TagValue}.
@param valueID: The L{TagValue.id} for the associated value.
@param content: The binary content of the opaque value.
"""
fileID = sha256(content).hexdigest()
store = getMainStore()
opaque = store.find(OpaqueValue, OpaqueValue.fileID == fileID).one()
if opaque is None:
opaque = OpaqueValue(fileID, content)
store.add(opaque)
store.add(OpaqueValueLink(valueID, fileID))
return opaque
def getOAuthConsumers(userIDs=None):
"""Get C{(User, OAuthConsumer)} 2-tuples matching specified L{User.id}s.
@param userIDs: Optionally, a sequence of L{User.id}s to filter the
results with.
@return: A C{ResultSet} with matching C{(User, OAuthConsumer)} results.
"""
store = getMainStore()
where = []
if userIDs:
where.append(OAuthConsumer.userID.is_in(userIDs))
return store.find((User, OAuthConsumer), OAuthConsumer.userID == User.id,
*where)
def getTwitterUsers(uids=None):
"""Get C{(User, TwitterUser)} 2-tuples matching specified Twitter UIDs.
@param uids: Optionally, a sequence of L{TwitterUser.uid}s to filter the
results with.
@return: A C{ResultSet} with matching C{(User, TwitterUser)} 2-tuples.
"""
store = getMainStore()
where = []
if uids:
where.append(TwitterUser.uid.is_in(uids))
return store.find((User, TwitterUser),
User.id == TwitterUser.userID,
*where)
def createOpaqueValue(valueID, content):
"""Create a new L{OpaqueValue} associated with the given L{TagValue}.
@param valueID: The L{TagValue.id} for the associated value.
@param content: The binary content of the opaque value.
"""
fileID = sha256(content).hexdigest()
store = getMainStore()
opaque = store.find(OpaqueValue, OpaqueValue.fileID == fileID).one()
if opaque is None:
opaque = OpaqueValue(fileID, content)
store.add(opaque)
store.add(OpaqueValueLink(valueID, fileID))
return opaque
def getTagPathsForObjectIDs(objectIDs):
"""Get L{Tag.path}s for object IDs.
@param objectIDs: A sequence of object IDs.
@return: A C{ResultSet} yield L{Tag.path} values.
"""
if not objectIDs:
return EmptyResultSet()
store = getMainStore()
result = store.find(Tag.path,
Tag.id == TagValue.tagID,
TagValue.objectID.is_in(objectIDs))
result.config(distinct=True)
return result
def getRecentActivity(objectIDs=None, usernames=None, limit=20):
"""Get information about recent tag values.
@param objectIDs: Optionally, a sequence of L{TagValue.objectID} to get
recent tag value information for.
@param usernames: Optionally, a sequence of L{User.username}s to get
recent tag value information for.
@param limit: Optionally, a limit to the number of rows returned by this
function.
@return: A generator yielding C{(Tag.path, TagValue.objectID,
AboutTagValue.value, TagValue.value, User.username,
value.creationTime)} 6-tuples with the information about the recent
tag values. The tuples are sorted by creation time.
"""
if objectIDs and usernames:
mainCondition = Or(User.username.is_in(usernames),
TagValue.objectID.is_in(objectIDs))
elif objectIDs:
mainCondition = (TagValue.objectID.is_in(objectIDs))
elif usernames:
# If we're only requesting one user, we use a special query which is
# optimized by the use the tag_values_creator_creation_idx two-column
# index in Postgres.
if len(usernames) == 1:
[username] = usernames
subselect = Select(User.id, User.username == username)
mainCondition = (TagValue.creatorID == subselect)
else:
mainCondition = (User.username.is_in(usernames))
else:
return
store = getMainStore()
join = LeftJoin(TagValue, AboutTagValue,
TagValue.objectID == AboutTagValue.objectID)
result = store.using(User, Tag, join).find(
(Tag, TagValue, AboutTagValue, User),
mainCondition,
TagValue.creatorID == User.id,
TagValue.tagID == Tag.id)
result = result.order_by(Desc(TagValue.creationTime))
result = result.config(limit=limit)
# FIXME: We have to do this because Storm doesn's support getting null
# values for about_tag_values in the LEFT JOIN.
for tag, value, aboutValue, user in result:
about = aboutValue.value if aboutValue else None
yield (tag.path, value.objectID, about, value.value,
user.username, value.creationTime)
def getTagValues(values=None):
"""Get L{TagValue}s.
@param values: Optionally, a sequence of C{(objectID, Tag.id)} 2-tuples to
filter the result with.
@return: A C{ResultSet} with L{TagValue}s.
"""
store = getMainStore()
where = []
if values:
expressions = [
And(TagValue.objectID == objectID, TagValue.tagID == tagID)
for objectID, tagID in values]
where = [Or(*expressions)]
return store.find(TagValue, *where)
def getOAuthConsumers(userIDs=None):
"""Get C{(User, OAuthConsumer)} 2-tuples matching specified L{User.id}s.
@param userIDs: Optionally, a sequence of L{User.id}s to filter the
results with.
@return: A C{ResultSet} with matching C{(User, OAuthConsumer)} results.
"""
store = getMainStore()
where = []
if userIDs:
where.append(OAuthConsumer.userID.is_in(userIDs))
return store.find((User, OAuthConsumer),
OAuthConsumer.userID == User.id,
*where)
def getForUser(self,
username,
limit=20,
olderThan=None,
newerThan=None,
filterTags=None,
filterAbout=None,
additionalTags=None):
"""Get the comments made by a particular user or on the user object.
@param username: The user to get the comments for.
@param limit: Optionally, The maximum number of comments to return.
@param olderThan: Optionally a C{datetime} indicating to return only
comments older than it.
@param filterTags: Optionally a C{list} of tag paths. If not C{None},
return only comment objects with _all_ of the specified tag paths.
@param filterAbout: Optionally, return only comments made on a
given object.
@param additionalTags: Optionally, a list of paths of additional tags
to retrieve.
@return: A C{list} of comments represented by a C{dict} with the
following format::
{
'fluidinfo.com/info/about': <about-value-list>,
'fluidinfo.com/info/text': <comment-text>,
'fluidinfo.com/info/timestamp': <float-timestamp>,
'fluidinfo.com/info/url': <url>,
'fluidinfo.com/info/username': <username>,
}
"""
store = getMainStore()
result = store.find(
CommentObjectLink.commentID,
CommentObjectLink.objectID == AboutTagValue.objectID,
AboutTagValue.value == u'@' + username)
subselect = result.get_select_expr(CommentObjectLink.commentID)
where = [
Or(Comment.username == username, Comment.objectID.is_in(subselect))
]
return self._findComments(where,
limit,
olderThan,
newerThan,
filterTags=filterTags,
filterAbout=filterAbout,
additionalTags=additionalTags)
def getTags(paths=None, objectIDs=None):
"""Get L{Tag}s.
@param paths: Optionally, a sequence of L{Tag.path}s to filter the result
with.
@param objectIDs: Optionally, a sequence of L{Tag.objectID}s to filter the
result with.
@return: A C{ResultSet} with matching L{Tag}s.
"""
store = getMainStore()
where = []
if paths:
where.append(Tag.path.is_in(paths))
if objectIDs:
where.append(Tag.objectID.is_in(objectIDs))
return store.find(Tag, *where)
def createComment(objectID, targetObjectIDs, username, creationTime=None):
"""Creates a new L{Comment}.
@param objectID: The objectID of the new comment.
@param targetObjectIDs: A list with all the target objectIDs
@param username: The username of the creator of the comment.
@param creationTime: Optionally a timestamp for the comment.
"""
store = getMainStore()
store.find(Comment, Comment.objectID == objectID).remove()
creationTime = creationTime or datetime.utcnow()
comment = Comment(objectID, username, creationTime)
store.add(comment)
for targetID in targetObjectIDs:
store.add(CommentObjectLink(objectID, targetID))
return comment
def createTag(creator, namespace, name):
"""Create a new L{Tag}.
@param creator: The L{User} that owns the L{Tag}.
@param namespace: The parent L{Namespace}.
@param name: The C{unicode} name of the L{Tag}.
@raise MalformedPathError: Raised if C{path} is empty or has unnacceptable
characters.
@return: A new L{Tag} instance persisted in the main store.
"""
store = getMainStore()
path = u'/'.join([namespace.path, name])
if not isValidPath(path):
raise MalformedPathError("'%s' is not a valid path." % path)
tag = Tag(creator, namespace, path, name)
return store.add(tag)
def getTags(paths=None, objectIDs=None):
"""Get L{Tag}s.
@param paths: Optionally, a sequence of L{Tag.path}s to filter the result
with.
@param objectIDs: Optionally, a sequence of L{Tag.objectID}s to filter the
result with.
@return: A C{ResultSet} with matching L{Tag}s.
"""
store = getMainStore()
where = []
if paths:
where.append(Tag.path.is_in(paths))
if objectIDs:
where.append(Tag.objectID.is_in(objectIDs))
return store.find(Tag, *where)
def createNamespace(creator, path, parentID=None):
"""Create a new root-level L{Namespace}.
@param creator: The L{User} that owns the namespace.
@param path: The C{unicode} path (and name) of the namespace.
@param parentID: Optionally, the L{Namespace.id} of the parent namespace.
@raise MalformedPathError: Raised if C{path} is empty or has unacceptable
characters.
@return: A new L{Namespace} instance persisted in the main store.
"""
if not isValidPath(path):
raise MalformedPathError("'%s' is not a valid path." % path)
store = getMainStore()
name = getPathName(path)
namespace = Namespace(creator, path, name, parentID)
return store.add(namespace)
def createTag(creator, namespace, name):
"""Create a new L{Tag}.
@param creator: The L{User} that owns the L{Tag}.
@param namespace: The parent L{Namespace}.
@param name: The C{unicode} name of the L{Tag}.
@raise MalformedPathError: Raised if C{path} is empty or has unnacceptable
characters.
@return: A new L{Tag} instance persisted in the main store.
"""
store = getMainStore()
path = u'/'.join([namespace.path, name])
if not isValidPath(path):
raise MalformedPathError("'%s' is not a valid path." % path)
tag = Tag(creator, namespace, path, name)
return store.add(tag)
def getTagValues(values=None):
"""Get L{TagValue}s.
@param values: Optionally, a sequence of C{(objectID, Tag.id)} 2-tuples to
filter the result with.
@return: A C{ResultSet} with L{TagValue}s.
"""
store = getMainStore()
where = []
if values:
expressions = [
And(TagValue.objectID == objectID, TagValue.tagID == tagID)
for objectID, tagID in values
]
where = [Or(*expressions)]
return store.find(TagValue, *where)
def getRecentActivity(objectIDs=None, usernames=None, limit=20):
"""Get information about recent tag values.
@param objectIDs: Optionally, a sequence of L{TagValue.objectID} to get
recent tag value information for.
@param usernames: Optionally, a sequence of L{User.username}s to get
recent tag value information for.
@param limit: Optionally, a limit to the number of rows returned by this
function.
@return: A generator yielding C{(Tag.path, TagValue.objectID,
AboutTagValue.value, TagValue.value, User.username,
value.creationTime)} 6-tuples with the information about the recent
tag values. The tuples are sorted by creation time.
"""
if objectIDs and usernames:
mainCondition = Or(User.username.is_in(usernames),
TagValue.objectID.is_in(objectIDs))
elif objectIDs:
mainCondition = (TagValue.objectID.is_in(objectIDs))
elif usernames:
# If we're only requesting one user, we use a special query which is
# optimized by the use the tag_values_creator_creation_idx two-column
# index in Postgres.
if len(usernames) == 1:
[username] = usernames
subselect = Select(User.id, User.username == username)
mainCondition = (TagValue.creatorID == subselect)
else:
mainCondition = (User.username.is_in(usernames))
else:
return
store = getMainStore()
join = LeftJoin(TagValue, AboutTagValue,
TagValue.objectID == AboutTagValue.objectID)
result = store.using(User, Tag, join).find(
(Tag, TagValue, AboutTagValue, User), mainCondition,
TagValue.creatorID == User.id, TagValue.tagID == Tag.id)
result = result.order_by(Desc(TagValue.creationTime))
result = result.config(limit=limit)
# FIXME: We have to do this because Storm doesn's support getting null
# values for about_tag_values in the LEFT JOIN.
for tag, value, aboutValue, user in result:
about = aboutValue.value if aboutValue else None
yield (tag.path, value.objectID, about, value.value, user.username,
value.creationTime)
def getAllFollowed(self,
username,
limit=20,
olderThan=None,
newerThan=None):
"""
Get all the comments on the followed objects, by the followed users and
by the requested user.
@param username: The user to get the comments for.
@param limit: Optionally, The maximum number of comments to return.
@param olderThan: Optionally a C{datetime} indicating to return only
comments older than it.
@param newerThan: A C{datetime} indicating to return only
comments newer than it.
@return: A C{list} of comments represented by a C{dict} with the
following format::
{
'fluidinfo.com/info/about': <about-value-list>,
'fluidinfo.com/info/text': <comment-text>,
'fluidinfo.com/info/timestamp': <float-timestamp>,
'fluidinfo.com/info/url': <url>,
'fluidinfo.com/info/username': <username>,
}
"""
store = getMainStore()
result = getObjectIDs([username + u'/follows'])
objectsSubselect = result.get_select_expr(TagValue.objectID)
result = store.find(User.username, User.objectID == TagValue.objectID,
Tag.id == TagValue.tagID,
Tag.path == username + u'/follows')
usersSubselect = result.get_select_expr(User.username)
where = [
Comment.objectID == CommentObjectLink.commentID,
Or(CommentObjectLink.objectID.is_in(objectsSubselect),
Comment.username.is_in(usersSubselect),
Comment.username == username)
]
if olderThan is not None:
where.append(Comment.creationTime < olderThan)
return self._findComments(where, limit, olderThan, newerThan)
def createNamespacePermission(namespace, permissionTemplate=None):
"""Create a L{NamespacePermission}.
@param namespace: The L{Namespace} to create permissions for.
@param permissionTemplate: The L{NamespacePermission} to use as a template
for this one. By default, permissions are set using the system-wide
policy.
@return: A new L{NamespacePermission} instance.
"""
store = getMainStore()
permission = NamespacePermission(namespace.creator.id, namespace.id)
if permissionTemplate:
for operation in NamespacePermission.operations.iterkeys():
policy, exceptions = permissionTemplate.get(operation)
permission.set(operation, policy, exceptions)
return store.add(permission)
def createNamespacePermission(namespace, permissionTemplate=None):
"""Create a L{NamespacePermission}.
@param namespace: The L{Namespace} to create permissions for.
@param permissionTemplate: The L{NamespacePermission} to use as a template
for this one. By default, permissions are set using the system-wide
policy.
@return: A new L{NamespacePermission} instance.
"""
store = getMainStore()
permission = NamespacePermission(namespace.creator.id, namespace.id)
if permissionTemplate:
for operation in NamespacePermission.operations.iterkeys():
policy, exceptions = permissionTemplate.get(operation)
permission.set(operation, policy, exceptions)
return store.add(permission)
def updateIndex(url, createdAfterTime=datetime.min, stream=sys.stderr):
"""
Build documents in an L{ObjectIndex} for data in the main store
that has been updated since the provided C{datetime}.
@param url: The URL of the Solr index to create documents in.
@param createdAfterTime: An inclusive C{datetime} offset from which to
update new tag-values.
@param stream: The file descriptor to send progress updates to. Defaults to
C{sys.stderr}.
@return: A C{Deferred} that will fire with the number of new documents
that were created in the index.
"""
client = SolrClient(url)
index = ObjectIndex(client)
MAX_DOCUMENTS = 1000
# setup progress bar
progressbarWidth = 78
totalRows = getMainStore().find(TagValue).count()
documentsPerDash = totalRows / progressbarWidth
stream.write("[%s]" % (" " * progressbarWidth))
stream.flush()
stream.write("\b" * (progressbarWidth + 1)) # return to start of bar
documents = {}
documentsProcessed = 0
result = groupby(_getAllTagValues(createdAfterTime), itemgetter(2))
for objectID, values in result:
tagValues = dict((path, value) for path, value, _ in values)
documents.update({objectID: tagValues})
if len(documents) >= MAX_DOCUMENTS:
yield index.update(documents)
documents = {}
documentsProcessed += 1
if documentsProcessed == documentsPerDash:
stream.write("-")
stream.flush()
documentsProcessed = 0
if documents:
yield index.update(documents)
yield client.commit()
def getAboutTagValues(objectIDs=None, values=None):
"""Get L{AboutTagValue}s.
@param objectIDs: Optionally, a sequence of C{objectID}s to filter the
result with.
@param values: Optionally, a sequence of C{AboutTagValue.value}s to filter
the result with.
"""
store = getMainStore()
where = []
if objectIDs:
where.append(AboutTagValue.objectID.is_in(objectIDs))
if values:
where.append(AboutTagValue.value.is_in(values))
if where:
# Can't pass an Or expression to store.find if where is empty
# i.e, no filtering is requested
return store.find(AboutTagValue, Or(*where))
return store.find(AboutTagValue)
def getUsers(usernames=None, ids=None, objectIDs=None):
"""Get L{User}s.
@param usernames: Optionally, a sequence of L{User.username}s to filter
the results with.
@param ids: Optionally, a sequence of L{User.id}s to filter the results
with.
@param objectIDs: Optionally, a sequence of L{User.objectID}s to filter the
result with.
@return: A C{ResultSet} with matching L{User}s.
"""
store = getMainStore()
where = []
if ids:
where.append(User.id.is_in(ids))
if usernames:
where.append(User.username.is_in(usernames))
if objectIDs:
where.append(User.objectID.is_in(objectIDs))
return store.find(User, *where)
def createOAuthConsumer(user, secret=None):
"""Create a new L{OAuthConsumer} with a randomly generated secret.
A 16-character secret is generated randomly, if one isn't explicitly
provided. It's combined with the Fluidinfo C{access-secret} to generate
the final 32-character key that is used to generate a
C{fluiddb.util.minitoken} token.
@param user: The L{User} to associated with the L{OAuthConsumer}.
@param secret: Optionally, a C{str} with a secret. If not passed, a random
secret is generated.
@return: A new L{OAuthConsumer} instance persisted in the main store.
"""
store = getMainStore()
if secret is None:
secret = ''.join(sample(ALPHABET, 16))
elif len(secret) != 16:
raise ValueError('Consumer secret must be exactly 16 characters in '
'length.')
return store.add(OAuthConsumer(user.id, secret))
def checkIntegrity(maxRowsPerQuery=10000):
"""
Check the integrity of the database for cases which the database
engine can't detect.
@param maxRowsPerQuery: Limit the number of rows fetched by SQL queries to
avoid excessive use of memory.
"""
results = _splitResult(getNamespaces(), Namespace.id, maxRowsPerQuery)
for result in results:
namespaces = list(result)
NamespaceIntegrityChecker().check(namespaces)
results = _splitResult(getTags(), Tag.id, maxRowsPerQuery)
for result in results:
tags = list(result)
TagIntegrityChecker().check(tags)
results = _splitResult(getUsers(), User.id, maxRowsPerQuery)
for result in results:
users = list(result)
UserIntegrityChecker().check(users)
results = _splitResult(getAboutTagValues(), AboutTagValue.objectID,
maxRowsPerQuery)
for result in results:
aboutTagValues = list(result)
AboutTagValueIntegrityChecker().check(aboutTagValues)
# In the case of TagValues we limit the query to only tag paths starting
# with "fluiddb" because these are the only ones we're checking and we
# don't want a huge result.
store = getMainStore()
result = store.find(TagValue, TagValue.tagID == Tag.id,
Tag.path.startswith(u'fluiddb/'))
results = _splitResult(getTagValues(), TagValue.id, maxRowsPerQuery)
for result in results:
tagValues = list(result)
TagValueIntegrityChecker().check(tagValues)
def createUser(username, password, fullname, email=None, role=None):
"""Create a L{User} called C{name} with C{role}.
@param username: A C{unicode} username for the user.
@param password: A C{unicode} password in plain text for the user. The
password will be hashed before being stored. The password will be
disabled if C{None} is provided.
@param email: Optionally, an email address for the user.
@param role: Optionally, a role for the user, defaults to L{Role.USER}.
@raise MalformedUsernameError: Raised if C{username} is not valid.
@raise DuplicateUserError: Raised if a user with the given C{username}
already exists.
@return: A new L{User} instance persisted in the main store.
"""
if not isValidUsername(username):
raise MalformedUsernameError(username)
store = getMainStore()
if store.find(User.id, User.username == username).any():
raise DuplicateUserError([username])
passwordHash = '!' if password is None else hashPassword(password)
role = role if role is not None else Role.USER
return store.add(User(username, passwordHash, fullname, email, role))
def getFollowedObjects(self,
username,
limit=20,
olderThan=None,
objectType=None):
"""Get the objects followed by the specified user.
@param username: The user to get the followed objects for.
@param limit: Optionally, The maximum number of objects to return.
@param olderThan: Optionally a C{datetime} indicating to return only
objects followed before the specified time.
@param objectType: Optionally, a C{str} representing the object type
to filter from the objects. The allowed values are C{url},
C{user} and C{hashtag}.
@return: A C{list} of objects followed by C{username} represented by
a C{dict} with the following format::
[
{
'about': '<about>',
'creationTime': <float_timestamp>,
'following': True
},
...
]
"""
store = getMainStore()
where = [
TagValue.tagID == Tag.id,
TagValue.objectID == AboutTagValue.objectID,
Tag.path == username + u'/follows'
]
if olderThan is not None:
where.append(TagValue.creationTime < olderThan)
if objectType is not None:
if objectType == 'user':
where.append(Like(AboutTagValue.value, u'@%'))
elif objectType == 'url':
where.append(Like(AboutTagValue.value, u'http%'))
elif objectType == 'hashtag':
where.append(Like(AboutTagValue.value, u'#%'))
else:
raise FeatureError('Unknown object type.')
result = store.find(
(TagValue.objectID, AboutTagValue.value, TagValue.creationTime),
where)
result = result.order_by(Desc(TagValue.creationTime))
result = list(result.config(limit=limit))
objectIDs = [objectID for objectID, _, _ in result]
if self._user.username != username:
callerObjectIDs = set(
store.find(TagValue.objectID, Tag.id == TagValue.tagID,
Tag.path == self._user.username + u'/follows',
TagValue.objectID.is_in(objectIDs)))
else:
callerObjectIDs = None
return [{
u'about':
about,
u'following': (objectID in callerObjectIDs
if callerObjectIDs is not None else True),
u'creationTime': (timegm(creationTime.utctimetuple()) +
float(creationTime.strftime('0.%f')))
} for objectID, about, creationTime in result]
def _findComments(self,
where,
limit,
olderThan,
newerThan,
username=None,
followedByUsername=None,
filterTags=None,
filterAbout=None,
additionalTags=None):
"""Find comments in the database and format the result.
@param where: The conditions for querying the comments table.
@param limit: The maximum number of comments to return.
@param olderThan: A C{datetime} indicating to return only
comments older than it.
@param newerThan: A C{datetime} indicating to return only
comments newer than it.
@param username: Optionally, only return comments made by the
specified L{User.username}.
@param followedByUsername: Optionally, only return comments made by
L{User}s that the specified L{User.username} follows.
@param filterTags: Optionally a C{list} of tag paths. If not C{None},
return only comment objects with _all_ of the specified tag paths.
@param filterAbout: Optionally, return only comments made on a
given object.
@param additionalTags: Optionally, a list of paths of additional tags
to retrieve.
@return: A C{list} of comments represented by a C{dict} with the
following format::
{
'fluidinfo.com/info/about': <about-value-list>,
'fluidinfo.com/info/text': <comment-text>,
'fluidinfo.com/info/timestamp': <float-timestamp>,
'fluidinfo.com/info/url': <url>,
'fluidinfo.com/info/username': <username>,
}
"""
store = getMainStore()
if olderThan is not None:
where.append(Comment.creationTime < olderThan)
if newerThan is not None:
where.append(Comment.creationTime > newerThan)
if username is not None:
where.append(Comment.username == username)
if followedByUsername:
result = store.find(User.username,
User.objectID == TagValue.objectID,
Tag.id == TagValue.tagID,
Tag.path == followedByUsername + u'/follows')
subselect = result.get_select_expr(User.username)
where.append(Comment.username.is_in(subselect))
if filterTags is not None:
# Partial SQL, because Storm doesn't do "= ALL()"
where.append(
SQL(
"""
comments.object_id IN (
SELECT tag_values.object_id
FROM tag_values, tags
WHERE
tags.id = tag_values.tag_id
AND tags.path = ANY(?)
GROUP BY tag_values.object_id
HAVING COUNT(*) = ?
)
""", [filterTags, len(filterTags)]))
if filterAbout is not None:
result = store.find(
CommentObjectLink.commentID,
CommentObjectLink.objectID == AboutTagValue.objectID,
AboutTagValue.value == filterAbout)
subselect = result.get_select_expr(CommentObjectLink.commentID)
where.append(Comment.objectID.is_in(subselect))
result = store.find(Comment.objectID, *where)
# Use GROUP BY and MIN here to return unique object IDs. It's not
# possible to use DISTINCT because postgres expects an ORDER BY
# expression to be in the select list. -- ceronman
result = result.group_by(Comment.objectID)
result = result.order_by(Desc(Min(Comment.creationTime)))
result = result.config(limit=limit)
commentIDs = list(result)
if not commentIDs:
return []
paths = self.COMMENT_TAGS + (additionalTags or [])
tagValues = self._tagValues.get(objectIDs=commentIDs, paths=paths)
result = []
for commentID in commentIDs:
valuesByTag = {}
for path in paths:
if path in tagValues[commentID]:
value = tagValues[commentID][path].value
if isinstance(value, dict):
del value['contents']
value['id'] = str(commentID)
valuesByTag[path] = value
result.append(valuesByTag)
return result
def run():
store = getMainStore()
store.execute(u'SELECT * FROM \N{HIRAGANA LETTER A}')
