def removeTestingData(): """ Delete L{User}s, L{Namespace}s and L{Tag}s used for testing purposes. """ admin = getUser(u'fluiddb') logging.info('Deleting testing tags.') result = TagAPI(admin).get(TESTING_DATA[u'tags']) if result: TagAPI(admin).delete(result.keys()) logging.info('Deleting testing namespaces.') result = NamespaceAPI(admin).get(TESTING_DATA[u'namespaces']) # we must delete namespaces one by one, otherwise we'll get NotEmptyError. for path in sorted(result.keys(), reverse=True): NamespaceAPI(admin).delete([path]) logging.info('Deleting testing users.') result = UserAPI().get(TESTING_DATA[u'users']) if result: for username in result: path = '%s/private' % username try: NamespaceAPI(admin).delete([path]) except FeatureError: # FIXME This is a bit crap, but it's faster than checking to # see if the namespace exists before attempting to delete it. continue if result: UserAPI().delete(result.keys()) getMainStore().commit()
def prepareForTesting(): """ Create a set of L{User}s, L{Namespace}s and L{Tag}s for testing purposes. """ admin = getUser(u'fluiddb') logging.info('Creating testing users.') UserAPI().create([(username, 'secret', u'Test user', u'*****@*****.**') for username in TESTING_DATA[u'users']]) logging.info('Creating testing namespaces.') NamespaceAPI(admin).create([(namespace, u'Used for testing purposes.') for namespace in TESTING_DATA[u'namespaces']]) logging.info('Creating testing tags.') TagAPI(admin).create([(tag, u'Used for testing purposes.') for tag in TESTING_DATA[u'tags']]) getMainStore().commit()
def createTagPermission(tag): """Create a L{TagPermission}. Permissions are inherited as follows: - L{Operation.UPDATE_TAG} inherits its L{Policy} and exceptions list from the permissions for L{Operation.UPDATE_NAMESPACE}. - L{Operation.DELETE_TAG} inherits its L{Policy} and exceptions list from the permissions for L{Operation.DELETE_NAMESPACE}. - L{Operation.CONTROL_TAG} inherits its L{Policy} and exceptions list from the permissions for L{Operation.CONTROL_NAMESPACE}. - L{Operation.WRITE_TAG_VALUE} inherits its L{Policy} and exceptions list from the permissions for L{Operation.UPDATE_NAMESPACE}. - L{Operation.READ_TAG_VALUE} inherits its L{Policy} and exceptions list from the permissions for L{Operation.LIST_NAMESPACE}. - L{Operation.DELETE_TAG_VALUE} inherits its L{Policy} and exceptions list from the permissions for L{Operation.DELETE_NAMESPACE}. - L{Operation.CONTROL_TAG_VALUE} inherits its L{Policy} and exceptions list from the permissions for L{Operation.CONTROL_NAMESPACE}. @param tag: The L{Tag} to create permissions for. @return: A new L{TagPermission} instance. """ store = getMainStore() permission = TagPermission(tag.namespace.creator.id, tag.id) permissionTemplate = tag.namespace.permission if permissionTemplate is not None: for operation in TAG_PERMISSION_INHERITANCE_MAP.iterkeys(): matchingOperation = TAG_PERMISSION_INHERITANCE_MAP.get(operation) policy, exceptions = permissionTemplate.get(matchingOperation) permission.set(operation, policy, exceptions) return store.add(permission)
def _getAllTagValues(updatedSince): """ Get all L{Tag} and L{TagValue}s that have been updated since the provided C{datetime}. @param updatedSince: An inclusive C{datetime} offset from which to update new tag-values. @return: A C{generator} that yields each updated tag individually. """ store = getMainStore() CHUNK_SIZE = 500000 result = store.find(TagValue, TagValue.creationTime >= updatedSince) totalRows = result.count() chunks = totalRows / CHUNK_SIZE if chunks == 0: chunks = 1 for i in range(chunks): limit = CHUNK_SIZE if (i != chunks - 1) else None offset = i * CHUNK_SIZE result = store.find((Tag.path, TagValue.value, TagValue.objectID, TagValue.creationTime), Tag.id == TagValue.tagID, TagValue.creationTime >= updatedSince) result = result.order_by(TagValue.objectID, TagValue.creationTime) result = result.config(limit=limit, offset=offset) for row in result: yield row[:-1]
def deleteComment(objectID): """Deletes a L{Comment}. @param objectID: The object ID of the comment. @return: An C{int} count of the number of comments removed. """ store = getMainStore() return store.find(Comment, Comment.objectID == objectID).remove()
def testGetMainStore(self): """ L{getMainStore} returns a C{Store} instance for the main store, when one has been properly configured. """ store = getMainStore() self.assertTrue(isinstance(store, Store)) self.assertIdentical(self.store, store)
def createTwitterUser(user, uid): """Create a L{TwitterUser}. @param user: The L{User} to link to a Twitter account. @param uid: The Twitter UID for the user. @return: A new L{TwitterUser} instance persisted in the main store. """ store = getMainStore() return store.add(TwitterUser(user.id, uid))
def createAboutTagValue(objectID, value): """Create a new L{AboutTagValue}. @param objectID: The object ID this value is associated with. @param value: The value to store. @return: An L{AboutTagValue} instance, added to the database. """ store = getMainStore() return store.add(AboutTagValue(objectID, value))
def getOpaqueValues(valueIDs): """Get L{OpaqueValue}s for the given L{TagValue}s. @param valueIDs: A sequence of L{TagValue.id}s. @return: A C{ResultSet} with L{OpaqueValue}s. """ store = getMainStore() return store.find(OpaqueValue, OpaqueValue.fileID == OpaqueValueLink.fileID, OpaqueValueLink.valueID.is_in(valueIDs))
def _getTagIDs(self): """Get a L{Tag.id}s for path filtering criteria. @return: A C{list} of L{Tag.id}s for paths or C{None} if no path filtering criteria are available. """ paths = self._criteria.get('paths') if paths: store = getMainStore() return list(store.find(Tag.id, Tag.path.is_in(paths)))
def getChildTags(paths): """Get child L{Tag}s. @param paths: A sequence of L{Namespace.path}s to get child L{Tag}s for. @return: A C{ResultSet} with matching L{Namespace} children. """ store = getMainStore() result = getNamespaces(paths) subselect = result.get_select_expr(Namespace.id) return store.find(Tag, Tag.namespaceID.is_in(subselect))
def summarizeObject(self, about): """Get summary information for an object. @param about: The about value of the object to summarize. @return: A C{dict} matching the following format:: {'commentCount': <count>, 'followers': [<username>, ...], 'relatedObjects': {'<about>': <count>, ...}} """ # List followers. result = self._objects.get([about]) objectID = result.get(about) if objectID is None: return {'commentCount': 0, 'followers': [], 'relatedObjects': {}} paths = self._objects.getTagsForObjects([objectID]) followers = [] for path in paths: parent = getParentPath(path) if parent and not u'/' in parent and path.endswith('/follows'): followers.append(parent) # Count comments. store = getMainStore() result = store.find(CommentObjectLink.commentID, CommentObjectLink.objectID == objectID) commentCount = result.count() # Count related objects. I'm using raw SQL here because don't know how # to translate this query to Storm. result = store.execute(""" SELECT about_tag_values."value", summary.count FROM ( SELECT comment_object_link.object_id AS object_id, COUNT(comment_object_link.comment_id) AS count FROM comment_object_link WHERE comment_object_link.comment_id IN ( SELECT comment_object_link.comment_id FROM comment_object_link WHERE comment_object_link.object_id = '{objectID}') AND comment_object_link.object_id != '{objectID}' GROUP BY comment_object_link.object_id ) AS summary JOIN about_tag_values ON summary.object_id = about_tag_values.object_id; """.format(objectID=objectID)) relatedObjects = dict(result) return {'commentCount': commentCount, 'followers': followers, 'relatedObjects': relatedObjects}
def getObjectIDs(paths): """Get object IDs for L{Tag.path}s. @param tags: A sequence of L{Tag.path}s. @return: A C{ResultSet} yielding C{TagValue.objectID}s. """ if not paths: return EmptyResultSet() store = getMainStore() return store.find(TagValue.objectID, Tag.id == TagValue.tagID, Tag.path.is_in(paths))
def getTagPathsAndObjectIDs(objectIDs): """Get L{Tag.path}s for object IDs. @param objectIDs: A sequence of object IDs. @return: A C{ResultSet} yielding C{(Tag.path, objectID)} 2-tuples. """ if not objectIDs: return EmptyResultSet() store = getMainStore() return store.find((Tag.path, TagValue.objectID), Tag.id == TagValue.tagID, TagValue.objectID.is_in(objectIDs))
def createTagValue(creatorID, tagID, objectID, value): """Create a new L{TagValue}. @param creatorID: The L{User.id} of the person creating this value. @param tagID: The L{Tag.id} this value is associated with. @param objectID: The object ID this value is associated with. @param value: The value to store. @return: A L{TagValue} instance, added to the database. """ store = getMainStore() return store.add(TagValue(creatorID, tagID, objectID, value))
def values(self): """ Get L{Tag} values that match the filtering criteria defined for this collection. @return: A L{ResultSet} yielding C{(Tag, TagValue)} 2-tuples. """ store = getMainStore() tagIDs = self._getTagIDs() where = self._getWhereClause(tagIDs) return store.find((Tag, TagValue), *where)
def createTagValue(creatorID, tagID, objectID, value): """Create a new L{TagValue}. @param creatorID: The L{User.id} of the person creating this value. @param tagID: The L{Tag.id} this value is associated with. @param objectID: The object ID this value is associated with. @param value: The value to store. @return: A L{TagValue} instance, added to the database. """ store = getMainStore() return store.add(TagValue(creatorID, tagID, objectID, value))
def getChildNamespaces(paths): """Get child L{Namespace}s. @param paths: A sequence of L{Namespace.path}s to get child L{Namespace}s for. @return: A C{ResultSet} with matching L{Namespace} children. """ store = getMainStore() result = getNamespaces(paths) subselect = result.get_select_expr(Namespace.id) return store.find(Namespace, Namespace.parentID.is_in(subselect))
def getTagPermissions(paths): """Get L{Tag}s and L{TagPermission}s for the specified paths. @param paths: A sequence of L{Tag.path}s to get L{Tag}s and L{TagPermission}s for. @return: A C{ResultSet} yielding C{(Tag, TagPermission)} 2-tuples for the specified L{Tag.path}s. """ store = getMainStore() return store.find((Tag, TagPermission), TagPermission.tagID == Tag.id, Tag.path.is_in(paths))
def update(self, importer, username, when, newText): """Updates the text of a comment. All object associations previously extracted from the old comment's text are removed and new associations extracted from the new text are added. Object associations not extracted from the text are kept. @param importer: A C{unicode} string giving the name of the importer. @param username: The C{unicode} username of the commenter. @param when: A C{datetime.datetime} instance. @param text: The new text for the comment. @return: A C{dict} as follows: { fluidinfo.com/info/about: A C{list} of all the about values (i.e., URLs and hashtags) in the comment text, including the thing the comment was about (if anything). The hashtags are in lowercase. fluidinfo.com/info/timestamp: The C{int} UTC timestamp (seconds since the epoch) the comment was created at. fluidinfo.com/info/url: The C{url}, as received. fluidinfo.com/info/username: The C{username}, as received. } """ # 1. Get the object ID of the comment object. isoTime = when.isoformat() commentObjectAbout = u'%s %s %s' % (importer, username, isoTime) aboutValue = getAboutTagValues(values=[commentObjectAbout]).one() if aboutValue is None: raise RuntimeError('Comment does not exist.') commentID = aboutValue.objectID # 2. Get the old text and url of the comment. result = self._tagValues.get( [commentID], [u'fluidinfo.com/info/text', u'fluidinfo.com/info/url']) oldText = result[commentID][u'fluidinfo.com/info/text'].value url = result[commentID][u'fluidinfo.com/info/url'].value # 3. Get abouts in comment's text. aboutsInText = self._extractAbouts(oldText) # 4. Get all the about values associated with the comment. store = getMainStore() allAbouts = store.find( AboutTagValue.value, CommentObjectLink.commentID == commentID, AboutTagValue.objectID == CommentObjectLink.objectID) # 5. Get abouts not in comment's text: aboutsNotInText = set(allAbouts) - set(aboutsInText) self.delete(importer, username, when) return self.create(newText, username, aboutsNotInText, importer, when, url)
def getDirtyObjects(objectIDs=None): """Get L{DirtyObject}s. @param objectIDs: Optionally, a sequence of L{DirtyObject.objectID}s to filter the result with. @return: A C{ResultSet} with matching L{DirtyObject}s. """ store = getMainStore() where = [] if objectIDs: where.append(DirtyObject.objectID.is_in(objectIDs)) return store.find(DirtyObject, *where)
def update(self, importer, username, when, newText): """Updates the text of a comment. All object associations previously extracted from the old comment's text are removed and new associations extracted from the new text are added. Object associations not extracted from the text are kept. @param importer: A C{unicode} string giving the name of the importer. @param username: The C{unicode} username of the commenter. @param when: A C{datetime.datetime} instance. @param text: The new text for the comment. @return: A C{dict} as follows: { fluidinfo.com/info/about: A C{list} of all the about values (i.e., URLs and hashtags) in the comment text, including the thing the comment was about (if anything). The hashtags are in lowercase. fluidinfo.com/info/timestamp: The C{int} UTC timestamp (seconds since the epoch) the comment was created at. fluidinfo.com/info/url: The C{url}, as received. fluidinfo.com/info/username: The C{username}, as received. } """ # 1. Get the object ID of the comment object. isoTime = when.isoformat() commentObjectAbout = u'%s %s %s' % (importer, username, isoTime) aboutValue = getAboutTagValues(values=[commentObjectAbout]).one() if aboutValue is None: raise RuntimeError('Comment does not exist.') commentID = aboutValue.objectID # 2. Get the old text and url of the comment. result = self._tagValues.get([commentID], [u'fluidinfo.com/info/text', u'fluidinfo.com/info/url']) oldText = result[commentID][u'fluidinfo.com/info/text'].value url = result[commentID][u'fluidinfo.com/info/url'].value # 3. Get abouts in comment's text. aboutsInText = self._extractAbouts(oldText) # 4. Get all the about values associated with the comment. store = getMainStore() allAbouts = store.find( AboutTagValue.value, CommentObjectLink.commentID == commentID, AboutTagValue.objectID == CommentObjectLink.objectID) # 5. Get abouts not in comment's text: aboutsNotInText = set(allAbouts) - set(aboutsInText) self.delete(importer, username, when) return self.create(newText, username, aboutsNotInText, importer, when, url)
def getTagPermissions(paths): """Get L{Tag}s and L{TagPermission}s for the specified paths. @param paths: A sequence of L{Tag.path}s to get L{Tag}s and L{TagPermission}s for. @return: A C{ResultSet} yielding C{(Tag, TagPermission)} 2-tuples for the specified L{Tag.path}s. """ store = getMainStore() return store.find((Tag, TagPermission), TagPermission.tagID == Tag.id, Tag.path.is_in(paths))
def getNamespacePermissions(paths): """Get L{Namespace}s and L{NamespacePermission}s for the specified paths. @param paths: A sequence of L{Namespace.path}s to get L{Namespace}s and L{NamespacePermission}s for. @return: A C{ResultSet} yielding C{(Namespace, NamespacePermission)} 2-tuples for the specified L{Namespace.path}s. """ store = getMainStore() return store.find((Namespace, NamespacePermission), NamespacePermission.namespaceID == Namespace.id, Namespace.path.is_in(paths))
def getObjectIDs(paths): """Get object IDs for L{Tag.path}s. @param tags: A sequence of L{Tag.path}s. @return: A C{ResultSet} yielding C{TagValue.objectID}s. """ if not paths: return EmptyResultSet() store = getMainStore() return store.find(TagValue.objectID, Tag.id == TagValue.tagID, Tag.path.is_in(paths))
def getNamespacePermissions(paths): """Get L{Namespace}s and L{NamespacePermission}s for the specified paths. @param paths: A sequence of L{Namespace.path}s to get L{Namespace}s and L{NamespacePermission}s for. @return: A C{ResultSet} yielding C{(Namespace, NamespacePermission)} 2-tuples for the specified L{Namespace.path}s. """ store = getMainStore() return store.find((Namespace, NamespacePermission), NamespacePermission.namespaceID == Namespace.id, Namespace.path.is_in(paths))
def getTagPathsAndObjectIDs(objectIDs): """Get L{Tag.path}s for object IDs. @param objectIDs: A sequence of object IDs. @return: A C{ResultSet} yielding C{(Tag.path, objectID)} 2-tuples. """ if not objectIDs: return EmptyResultSet() store = getMainStore() return store.find((Tag.path, TagValue.objectID), Tag.id == TagValue.tagID, TagValue.objectID.is_in(objectIDs))
def getTwitterUsers(uids=None): """Get C{(User, TwitterUser)} 2-tuples matching specified Twitter UIDs. @param uids: Optionally, a sequence of L{TwitterUser.uid}s to filter the results with. @return: A C{ResultSet} with matching C{(User, TwitterUser)} 2-tuples. """ store = getMainStore() where = [] if uids: where.append(TwitterUser.uid.is_in(uids)) return store.find((User, TwitterUser), User.id == TwitterUser.userID, *where)
def getTagPathsForObjectIDs(objectIDs): """Get L{Tag.path}s for object IDs. @param objectIDs: A sequence of object IDs. @return: A C{ResultSet} yield L{Tag.path} values. """ if not objectIDs: return EmptyResultSet() store = getMainStore() result = store.find(Tag.path, Tag.id == TagValue.tagID, TagValue.objectID.is_in(objectIDs)) result.config(distinct=True) return result
def createOpaqueValue(valueID, content): """Create a new L{OpaqueValue} associated with the given L{TagValue}. @param valueID: The L{TagValue.id} for the associated value. @param content: The binary content of the opaque value. """ fileID = sha256(content).hexdigest() store = getMainStore() opaque = store.find(OpaqueValue, OpaqueValue.fileID == fileID).one() if opaque is None: opaque = OpaqueValue(fileID, content) store.add(opaque) store.add(OpaqueValueLink(valueID, fileID)) return opaque
def getOAuthConsumers(userIDs=None): """Get C{(User, OAuthConsumer)} 2-tuples matching specified L{User.id}s. @param userIDs: Optionally, a sequence of L{User.id}s to filter the results with. @return: A C{ResultSet} with matching C{(User, OAuthConsumer)} results. """ store = getMainStore() where = [] if userIDs: where.append(OAuthConsumer.userID.is_in(userIDs)) return store.find((User, OAuthConsumer), OAuthConsumer.userID == User.id, *where)
def getTwitterUsers(uids=None): """Get C{(User, TwitterUser)} 2-tuples matching specified Twitter UIDs. @param uids: Optionally, a sequence of L{TwitterUser.uid}s to filter the results with. @return: A C{ResultSet} with matching C{(User, TwitterUser)} 2-tuples. """ store = getMainStore() where = [] if uids: where.append(TwitterUser.uid.is_in(uids)) return store.find((User, TwitterUser), User.id == TwitterUser.userID, *where)
def createOpaqueValue(valueID, content): """Create a new L{OpaqueValue} associated with the given L{TagValue}. @param valueID: The L{TagValue.id} for the associated value. @param content: The binary content of the opaque value. """ fileID = sha256(content).hexdigest() store = getMainStore() opaque = store.find(OpaqueValue, OpaqueValue.fileID == fileID).one() if opaque is None: opaque = OpaqueValue(fileID, content) store.add(opaque) store.add(OpaqueValueLink(valueID, fileID)) return opaque
def getTagPathsForObjectIDs(objectIDs): """Get L{Tag.path}s for object IDs. @param objectIDs: A sequence of object IDs. @return: A C{ResultSet} yield L{Tag.path} values. """ if not objectIDs: return EmptyResultSet() store = getMainStore() result = store.find(Tag.path, Tag.id == TagValue.tagID, TagValue.objectID.is_in(objectIDs)) result.config(distinct=True) return result
def getRecentActivity(objectIDs=None, usernames=None, limit=20): """Get information about recent tag values. @param objectIDs: Optionally, a sequence of L{TagValue.objectID} to get recent tag value information for. @param usernames: Optionally, a sequence of L{User.username}s to get recent tag value information for. @param limit: Optionally, a limit to the number of rows returned by this function. @return: A generator yielding C{(Tag.path, TagValue.objectID, AboutTagValue.value, TagValue.value, User.username, value.creationTime)} 6-tuples with the information about the recent tag values. The tuples are sorted by creation time. """ if objectIDs and usernames: mainCondition = Or(User.username.is_in(usernames), TagValue.objectID.is_in(objectIDs)) elif objectIDs: mainCondition = (TagValue.objectID.is_in(objectIDs)) elif usernames: # If we're only requesting one user, we use a special query which is # optimized by the use the tag_values_creator_creation_idx two-column # index in Postgres. if len(usernames) == 1: [username] = usernames subselect = Select(User.id, User.username == username) mainCondition = (TagValue.creatorID == subselect) else: mainCondition = (User.username.is_in(usernames)) else: return store = getMainStore() join = LeftJoin(TagValue, AboutTagValue, TagValue.objectID == AboutTagValue.objectID) result = store.using(User, Tag, join).find( (Tag, TagValue, AboutTagValue, User), mainCondition, TagValue.creatorID == User.id, TagValue.tagID == Tag.id) result = result.order_by(Desc(TagValue.creationTime)) result = result.config(limit=limit) # FIXME: We have to do this because Storm doesn's support getting null # values for about_tag_values in the LEFT JOIN. for tag, value, aboutValue, user in result: about = aboutValue.value if aboutValue else None yield (tag.path, value.objectID, about, value.value, user.username, value.creationTime)
def getTagValues(values=None): """Get L{TagValue}s. @param values: Optionally, a sequence of C{(objectID, Tag.id)} 2-tuples to filter the result with. @return: A C{ResultSet} with L{TagValue}s. """ store = getMainStore() where = [] if values: expressions = [ And(TagValue.objectID == objectID, TagValue.tagID == tagID) for objectID, tagID in values] where = [Or(*expressions)] return store.find(TagValue, *where)
def getOAuthConsumers(userIDs=None): """Get C{(User, OAuthConsumer)} 2-tuples matching specified L{User.id}s. @param userIDs: Optionally, a sequence of L{User.id}s to filter the results with. @return: A C{ResultSet} with matching C{(User, OAuthConsumer)} results. """ store = getMainStore() where = [] if userIDs: where.append(OAuthConsumer.userID.is_in(userIDs)) return store.find((User, OAuthConsumer), OAuthConsumer.userID == User.id, *where)
def getForUser(self, username, limit=20, olderThan=None, newerThan=None, filterTags=None, filterAbout=None, additionalTags=None): """Get the comments made by a particular user or on the user object. @param username: The user to get the comments for. @param limit: Optionally, The maximum number of comments to return. @param olderThan: Optionally a C{datetime} indicating to return only comments older than it. @param filterTags: Optionally a C{list} of tag paths. If not C{None}, return only comment objects with _all_ of the specified tag paths. @param filterAbout: Optionally, return only comments made on a given object. @param additionalTags: Optionally, a list of paths of additional tags to retrieve. @return: A C{list} of comments represented by a C{dict} with the following format:: { 'fluidinfo.com/info/about': <about-value-list>, 'fluidinfo.com/info/text': <comment-text>, 'fluidinfo.com/info/timestamp': <float-timestamp>, 'fluidinfo.com/info/url': <url>, 'fluidinfo.com/info/username': <username>, } """ store = getMainStore() result = store.find( CommentObjectLink.commentID, CommentObjectLink.objectID == AboutTagValue.objectID, AboutTagValue.value == u'@' + username) subselect = result.get_select_expr(CommentObjectLink.commentID) where = [ Or(Comment.username == username, Comment.objectID.is_in(subselect)) ] return self._findComments(where, limit, olderThan, newerThan, filterTags=filterTags, filterAbout=filterAbout, additionalTags=additionalTags)
def getTags(paths=None, objectIDs=None): """Get L{Tag}s. @param paths: Optionally, a sequence of L{Tag.path}s to filter the result with. @param objectIDs: Optionally, a sequence of L{Tag.objectID}s to filter the result with. @return: A C{ResultSet} with matching L{Tag}s. """ store = getMainStore() where = [] if paths: where.append(Tag.path.is_in(paths)) if objectIDs: where.append(Tag.objectID.is_in(objectIDs)) return store.find(Tag, *where)
def createComment(objectID, targetObjectIDs, username, creationTime=None): """Creates a new L{Comment}. @param objectID: The objectID of the new comment. @param targetObjectIDs: A list with all the target objectIDs @param username: The username of the creator of the comment. @param creationTime: Optionally a timestamp for the comment. """ store = getMainStore() store.find(Comment, Comment.objectID == objectID).remove() creationTime = creationTime or datetime.utcnow() comment = Comment(objectID, username, creationTime) store.add(comment) for targetID in targetObjectIDs: store.add(CommentObjectLink(objectID, targetID)) return comment
def createTag(creator, namespace, name): """Create a new L{Tag}. @param creator: The L{User} that owns the L{Tag}. @param namespace: The parent L{Namespace}. @param name: The C{unicode} name of the L{Tag}. @raise MalformedPathError: Raised if C{path} is empty or has unnacceptable characters. @return: A new L{Tag} instance persisted in the main store. """ store = getMainStore() path = u'/'.join([namespace.path, name]) if not isValidPath(path): raise MalformedPathError("'%s' is not a valid path." % path) tag = Tag(creator, namespace, path, name) return store.add(tag)
def getTags(paths=None, objectIDs=None): """Get L{Tag}s. @param paths: Optionally, a sequence of L{Tag.path}s to filter the result with. @param objectIDs: Optionally, a sequence of L{Tag.objectID}s to filter the result with. @return: A C{ResultSet} with matching L{Tag}s. """ store = getMainStore() where = [] if paths: where.append(Tag.path.is_in(paths)) if objectIDs: where.append(Tag.objectID.is_in(objectIDs)) return store.find(Tag, *where)
def createNamespace(creator, path, parentID=None): """Create a new root-level L{Namespace}. @param creator: The L{User} that owns the namespace. @param path: The C{unicode} path (and name) of the namespace. @param parentID: Optionally, the L{Namespace.id} of the parent namespace. @raise MalformedPathError: Raised if C{path} is empty or has unacceptable characters. @return: A new L{Namespace} instance persisted in the main store. """ if not isValidPath(path): raise MalformedPathError("'%s' is not a valid path." % path) store = getMainStore() name = getPathName(path) namespace = Namespace(creator, path, name, parentID) return store.add(namespace)
def createTag(creator, namespace, name): """Create a new L{Tag}. @param creator: The L{User} that owns the L{Tag}. @param namespace: The parent L{Namespace}. @param name: The C{unicode} name of the L{Tag}. @raise MalformedPathError: Raised if C{path} is empty or has unnacceptable characters. @return: A new L{Tag} instance persisted in the main store. """ store = getMainStore() path = u'/'.join([namespace.path, name]) if not isValidPath(path): raise MalformedPathError("'%s' is not a valid path." % path) tag = Tag(creator, namespace, path, name) return store.add(tag)
def getTagValues(values=None): """Get L{TagValue}s. @param values: Optionally, a sequence of C{(objectID, Tag.id)} 2-tuples to filter the result with. @return: A C{ResultSet} with L{TagValue}s. """ store = getMainStore() where = [] if values: expressions = [ And(TagValue.objectID == objectID, TagValue.tagID == tagID) for objectID, tagID in values ] where = [Or(*expressions)] return store.find(TagValue, *where)
def getRecentActivity(objectIDs=None, usernames=None, limit=20): """Get information about recent tag values. @param objectIDs: Optionally, a sequence of L{TagValue.objectID} to get recent tag value information for. @param usernames: Optionally, a sequence of L{User.username}s to get recent tag value information for. @param limit: Optionally, a limit to the number of rows returned by this function. @return: A generator yielding C{(Tag.path, TagValue.objectID, AboutTagValue.value, TagValue.value, User.username, value.creationTime)} 6-tuples with the information about the recent tag values. The tuples are sorted by creation time. """ if objectIDs and usernames: mainCondition = Or(User.username.is_in(usernames), TagValue.objectID.is_in(objectIDs)) elif objectIDs: mainCondition = (TagValue.objectID.is_in(objectIDs)) elif usernames: # If we're only requesting one user, we use a special query which is # optimized by the use the tag_values_creator_creation_idx two-column # index in Postgres. if len(usernames) == 1: [username] = usernames subselect = Select(User.id, User.username == username) mainCondition = (TagValue.creatorID == subselect) else: mainCondition = (User.username.is_in(usernames)) else: return store = getMainStore() join = LeftJoin(TagValue, AboutTagValue, TagValue.objectID == AboutTagValue.objectID) result = store.using(User, Tag, join).find( (Tag, TagValue, AboutTagValue, User), mainCondition, TagValue.creatorID == User.id, TagValue.tagID == Tag.id) result = result.order_by(Desc(TagValue.creationTime)) result = result.config(limit=limit) # FIXME: We have to do this because Storm doesn's support getting null # values for about_tag_values in the LEFT JOIN. for tag, value, aboutValue, user in result: about = aboutValue.value if aboutValue else None yield (tag.path, value.objectID, about, value.value, user.username, value.creationTime)
def getAllFollowed(self, username, limit=20, olderThan=None, newerThan=None): """ Get all the comments on the followed objects, by the followed users and by the requested user. @param username: The user to get the comments for. @param limit: Optionally, The maximum number of comments to return. @param olderThan: Optionally a C{datetime} indicating to return only comments older than it. @param newerThan: A C{datetime} indicating to return only comments newer than it. @return: A C{list} of comments represented by a C{dict} with the following format:: { 'fluidinfo.com/info/about': <about-value-list>, 'fluidinfo.com/info/text': <comment-text>, 'fluidinfo.com/info/timestamp': <float-timestamp>, 'fluidinfo.com/info/url': <url>, 'fluidinfo.com/info/username': <username>, } """ store = getMainStore() result = getObjectIDs([username + u'/follows']) objectsSubselect = result.get_select_expr(TagValue.objectID) result = store.find(User.username, User.objectID == TagValue.objectID, Tag.id == TagValue.tagID, Tag.path == username + u'/follows') usersSubselect = result.get_select_expr(User.username) where = [ Comment.objectID == CommentObjectLink.commentID, Or(CommentObjectLink.objectID.is_in(objectsSubselect), Comment.username.is_in(usersSubselect), Comment.username == username) ] if olderThan is not None: where.append(Comment.creationTime < olderThan) return self._findComments(where, limit, olderThan, newerThan)
def createNamespacePermission(namespace, permissionTemplate=None): """Create a L{NamespacePermission}. @param namespace: The L{Namespace} to create permissions for. @param permissionTemplate: The L{NamespacePermission} to use as a template for this one. By default, permissions are set using the system-wide policy. @return: A new L{NamespacePermission} instance. """ store = getMainStore() permission = NamespacePermission(namespace.creator.id, namespace.id) if permissionTemplate: for operation in NamespacePermission.operations.iterkeys(): policy, exceptions = permissionTemplate.get(operation) permission.set(operation, policy, exceptions) return store.add(permission)
def createNamespacePermission(namespace, permissionTemplate=None): """Create a L{NamespacePermission}. @param namespace: The L{Namespace} to create permissions for. @param permissionTemplate: The L{NamespacePermission} to use as a template for this one. By default, permissions are set using the system-wide policy. @return: A new L{NamespacePermission} instance. """ store = getMainStore() permission = NamespacePermission(namespace.creator.id, namespace.id) if permissionTemplate: for operation in NamespacePermission.operations.iterkeys(): policy, exceptions = permissionTemplate.get(operation) permission.set(operation, policy, exceptions) return store.add(permission)
def updateIndex(url, createdAfterTime=datetime.min, stream=sys.stderr): """ Build documents in an L{ObjectIndex} for data in the main store that has been updated since the provided C{datetime}. @param url: The URL of the Solr index to create documents in. @param createdAfterTime: An inclusive C{datetime} offset from which to update new tag-values. @param stream: The file descriptor to send progress updates to. Defaults to C{sys.stderr}. @return: A C{Deferred} that will fire with the number of new documents that were created in the index. """ client = SolrClient(url) index = ObjectIndex(client) MAX_DOCUMENTS = 1000 # setup progress bar progressbarWidth = 78 totalRows = getMainStore().find(TagValue).count() documentsPerDash = totalRows / progressbarWidth stream.write("[%s]" % (" " * progressbarWidth)) stream.flush() stream.write("\b" * (progressbarWidth + 1)) # return to start of bar documents = {} documentsProcessed = 0 result = groupby(_getAllTagValues(createdAfterTime), itemgetter(2)) for objectID, values in result: tagValues = dict((path, value) for path, value, _ in values) documents.update({objectID: tagValues}) if len(documents) >= MAX_DOCUMENTS: yield index.update(documents) documents = {} documentsProcessed += 1 if documentsProcessed == documentsPerDash: stream.write("-") stream.flush() documentsProcessed = 0 if documents: yield index.update(documents) yield client.commit()
def getAboutTagValues(objectIDs=None, values=None): """Get L{AboutTagValue}s. @param objectIDs: Optionally, a sequence of C{objectID}s to filter the result with. @param values: Optionally, a sequence of C{AboutTagValue.value}s to filter the result with. """ store = getMainStore() where = [] if objectIDs: where.append(AboutTagValue.objectID.is_in(objectIDs)) if values: where.append(AboutTagValue.value.is_in(values)) if where: # Can't pass an Or expression to store.find if where is empty # i.e, no filtering is requested return store.find(AboutTagValue, Or(*where)) return store.find(AboutTagValue)
def getUsers(usernames=None, ids=None, objectIDs=None): """Get L{User}s. @param usernames: Optionally, a sequence of L{User.username}s to filter the results with. @param ids: Optionally, a sequence of L{User.id}s to filter the results with. @param objectIDs: Optionally, a sequence of L{User.objectID}s to filter the result with. @return: A C{ResultSet} with matching L{User}s. """ store = getMainStore() where = [] if ids: where.append(User.id.is_in(ids)) if usernames: where.append(User.username.is_in(usernames)) if objectIDs: where.append(User.objectID.is_in(objectIDs)) return store.find(User, *where)
def createOAuthConsumer(user, secret=None): """Create a new L{OAuthConsumer} with a randomly generated secret. A 16-character secret is generated randomly, if one isn't explicitly provided. It's combined with the Fluidinfo C{access-secret} to generate the final 32-character key that is used to generate a C{fluiddb.util.minitoken} token. @param user: The L{User} to associated with the L{OAuthConsumer}. @param secret: Optionally, a C{str} with a secret. If not passed, a random secret is generated. @return: A new L{OAuthConsumer} instance persisted in the main store. """ store = getMainStore() if secret is None: secret = ''.join(sample(ALPHABET, 16)) elif len(secret) != 16: raise ValueError('Consumer secret must be exactly 16 characters in ' 'length.') return store.add(OAuthConsumer(user.id, secret))
def checkIntegrity(maxRowsPerQuery=10000): """ Check the integrity of the database for cases which the database engine can't detect. @param maxRowsPerQuery: Limit the number of rows fetched by SQL queries to avoid excessive use of memory. """ results = _splitResult(getNamespaces(), Namespace.id, maxRowsPerQuery) for result in results: namespaces = list(result) NamespaceIntegrityChecker().check(namespaces) results = _splitResult(getTags(), Tag.id, maxRowsPerQuery) for result in results: tags = list(result) TagIntegrityChecker().check(tags) results = _splitResult(getUsers(), User.id, maxRowsPerQuery) for result in results: users = list(result) UserIntegrityChecker().check(users) results = _splitResult(getAboutTagValues(), AboutTagValue.objectID, maxRowsPerQuery) for result in results: aboutTagValues = list(result) AboutTagValueIntegrityChecker().check(aboutTagValues) # In the case of TagValues we limit the query to only tag paths starting # with "fluiddb" because these are the only ones we're checking and we # don't want a huge result. store = getMainStore() result = store.find(TagValue, TagValue.tagID == Tag.id, Tag.path.startswith(u'fluiddb/')) results = _splitResult(getTagValues(), TagValue.id, maxRowsPerQuery) for result in results: tagValues = list(result) TagValueIntegrityChecker().check(tagValues)
def createUser(username, password, fullname, email=None, role=None): """Create a L{User} called C{name} with C{role}. @param username: A C{unicode} username for the user. @param password: A C{unicode} password in plain text for the user. The password will be hashed before being stored. The password will be disabled if C{None} is provided. @param email: Optionally, an email address for the user. @param role: Optionally, a role for the user, defaults to L{Role.USER}. @raise MalformedUsernameError: Raised if C{username} is not valid. @raise DuplicateUserError: Raised if a user with the given C{username} already exists. @return: A new L{User} instance persisted in the main store. """ if not isValidUsername(username): raise MalformedUsernameError(username) store = getMainStore() if store.find(User.id, User.username == username).any(): raise DuplicateUserError([username]) passwordHash = '!' if password is None else hashPassword(password) role = role if role is not None else Role.USER return store.add(User(username, passwordHash, fullname, email, role))
def getFollowedObjects(self, username, limit=20, olderThan=None, objectType=None): """Get the objects followed by the specified user. @param username: The user to get the followed objects for. @param limit: Optionally, The maximum number of objects to return. @param olderThan: Optionally a C{datetime} indicating to return only objects followed before the specified time. @param objectType: Optionally, a C{str} representing the object type to filter from the objects. The allowed values are C{url}, C{user} and C{hashtag}. @return: A C{list} of objects followed by C{username} represented by a C{dict} with the following format:: [ { 'about': '<about>', 'creationTime': <float_timestamp>, 'following': True }, ... ] """ store = getMainStore() where = [ TagValue.tagID == Tag.id, TagValue.objectID == AboutTagValue.objectID, Tag.path == username + u'/follows' ] if olderThan is not None: where.append(TagValue.creationTime < olderThan) if objectType is not None: if objectType == 'user': where.append(Like(AboutTagValue.value, u'@%')) elif objectType == 'url': where.append(Like(AboutTagValue.value, u'http%')) elif objectType == 'hashtag': where.append(Like(AboutTagValue.value, u'#%')) else: raise FeatureError('Unknown object type.') result = store.find( (TagValue.objectID, AboutTagValue.value, TagValue.creationTime), where) result = result.order_by(Desc(TagValue.creationTime)) result = list(result.config(limit=limit)) objectIDs = [objectID for objectID, _, _ in result] if self._user.username != username: callerObjectIDs = set( store.find(TagValue.objectID, Tag.id == TagValue.tagID, Tag.path == self._user.username + u'/follows', TagValue.objectID.is_in(objectIDs))) else: callerObjectIDs = None return [{ u'about': about, u'following': (objectID in callerObjectIDs if callerObjectIDs is not None else True), u'creationTime': (timegm(creationTime.utctimetuple()) + float(creationTime.strftime('0.%f'))) } for objectID, about, creationTime in result]
def _findComments(self, where, limit, olderThan, newerThan, username=None, followedByUsername=None, filterTags=None, filterAbout=None, additionalTags=None): """Find comments in the database and format the result. @param where: The conditions for querying the comments table. @param limit: The maximum number of comments to return. @param olderThan: A C{datetime} indicating to return only comments older than it. @param newerThan: A C{datetime} indicating to return only comments newer than it. @param username: Optionally, only return comments made by the specified L{User.username}. @param followedByUsername: Optionally, only return comments made by L{User}s that the specified L{User.username} follows. @param filterTags: Optionally a C{list} of tag paths. If not C{None}, return only comment objects with _all_ of the specified tag paths. @param filterAbout: Optionally, return only comments made on a given object. @param additionalTags: Optionally, a list of paths of additional tags to retrieve. @return: A C{list} of comments represented by a C{dict} with the following format:: { 'fluidinfo.com/info/about': <about-value-list>, 'fluidinfo.com/info/text': <comment-text>, 'fluidinfo.com/info/timestamp': <float-timestamp>, 'fluidinfo.com/info/url': <url>, 'fluidinfo.com/info/username': <username>, } """ store = getMainStore() if olderThan is not None: where.append(Comment.creationTime < olderThan) if newerThan is not None: where.append(Comment.creationTime > newerThan) if username is not None: where.append(Comment.username == username) if followedByUsername: result = store.find(User.username, User.objectID == TagValue.objectID, Tag.id == TagValue.tagID, Tag.path == followedByUsername + u'/follows') subselect = result.get_select_expr(User.username) where.append(Comment.username.is_in(subselect)) if filterTags is not None: # Partial SQL, because Storm doesn't do "= ALL()" where.append( SQL( """ comments.object_id IN ( SELECT tag_values.object_id FROM tag_values, tags WHERE tags.id = tag_values.tag_id AND tags.path = ANY(?) GROUP BY tag_values.object_id HAVING COUNT(*) = ? ) """, [filterTags, len(filterTags)])) if filterAbout is not None: result = store.find( CommentObjectLink.commentID, CommentObjectLink.objectID == AboutTagValue.objectID, AboutTagValue.value == filterAbout) subselect = result.get_select_expr(CommentObjectLink.commentID) where.append(Comment.objectID.is_in(subselect)) result = store.find(Comment.objectID, *where) # Use GROUP BY and MIN here to return unique object IDs. It's not # possible to use DISTINCT because postgres expects an ORDER BY # expression to be in the select list. -- ceronman result = result.group_by(Comment.objectID) result = result.order_by(Desc(Min(Comment.creationTime))) result = result.config(limit=limit) commentIDs = list(result) if not commentIDs: return [] paths = self.COMMENT_TAGS + (additionalTags or []) tagValues = self._tagValues.get(objectIDs=commentIDs, paths=paths) result = [] for commentID in commentIDs: valuesByTag = {} for path in paths: if path in tagValues[commentID]: value = tagValues[commentID][path].value if isinstance(value, dict): del value['contents'] value['id'] = str(commentID) valuesByTag[path] = value result.append(valuesByTag) return result
def run(): store = getMainStore() store.execute(u'SELECT * FROM \N{HIRAGANA LETTER A}')