def run(): tagValues = SecureTagValueAPI(session.auth.user) try: result = tagValues.get([objectID], [path]) except PermissionDeniedError as error: session.log.exception(error) path_, operation = error.pathsAndOperations[0] raise TNonexistentTag(path_) except UnknownPathError as error: session.log.exception(error) raise TNonexistentTag(path) if not result: raise TNoInstanceOnObject(path, objectId) else: tagValue = result[objectID][path] value = tagValue.value # FIXME This is a bit crap, but its easier than modifying # Thrift-related logic. if isinstance(value, dict): mimeType = value['mime-type'].encode('utf-8') value = createBinaryThriftValue(value['contents'], mimeType) elif isinstance(value, UUID): value = createThriftValue(str(value)) else: value = createThriftValue(value) return (value, tagValue)
def run(): tagValues = SecureTagValueAPI(session.auth.user) objects = SecureObjectAPI(session.auth.user) objectIDs = self._resolveQuery(session, objects, parsedQuery) values = [] if tags is None: # delete all tags user has permissions for result = objects.getTagsByObjects(objectIDs, Operation.DELETE_TAG_VALUE) for objectID, paths in result.iteritems(): for path in paths: values.append((objectID, path)) else: # delete only tags requested by user result = objects.getTagsByObjects(objectIDs) for objectID, paths in result.iteritems(): for path in paths: if tags is None or path in tags: values.append((objectID, path)) if values: try: tagValues.delete(values) except UnknownPathError as error: session.log.exception(error) path = error.paths[0] raise TNonexistentTag(path.encode('utf-8')) except PermissionDeniedError as error: session.log.exception(error) path_, operation = error.pathsAndOperations[0] category, action = getCategoryAndAction(operation) raise TPathPermissionDenied(category, action, path_)
def setUp(self): super(SecureTagValueAPIWithSuperuserRoleTest, self).setUp() system = createSystemData() self.superuser = system.users[u'fluiddb'] TagAPI(self.superuser).create([(u'fluiddb/tag', u'description')]) self.tagValues = SecureTagValueAPI(self.superuser) self.permissions = CachingPermissionAPI(self.superuser)
def setUp(self): super(DatasetImporterTest, self).setUp() self.system = createSystemData() UserAPI().create([(u'user', u'secret', u'User', u'*****@*****.**')]) user = getUser(u'user') self.objects = SecureObjectAPI(user) self.values = SecureTagValueAPI(user)
def run(): objects = SecureObjectAPI(session.auth.user) try: searchQueries = objects.search(valuesByQuery.keys()) except UnknownPathError as error: session.log.exception(error) unknownPath = error.paths[0] raise TNonexistentTag(unknownPath.encode('utf-8')) except PermissionDeniedError as error: session.log.exception(error) path_, operation = error.pathsAndOperations[0] if operation == Operation.CREATE_OBJECT: raise TUnauthorized() else: raise TNonexistentTag(path_) # Run queries. try: with session.timer.track('index-search'): result = blockingCallFromThread(reactor, searchQueries.get) except SearchError as error: session.log.exception(error) raise TParseError(query, error.message) # Build a result set from the searches. values = {} for parsedQuery, objectIDs in result.iteritems(): for objectID in objectIDs: for tagAndValue in valuesByQuery[parsedQuery]: value = guessValue(tagAndValue.value) # FIXME: this code sucks, but I rather not having # to modify guessValue to return a list, as that # would break other code. # Hopefully, we'll be able to remove this pretty # soon. if isinstance(value, list): value = [item.decode('utf-8') for item in value] if objectID not in values: values[objectID] = {} values[objectID][tagAndValue.path] = value # Update values. if values: tagValues = SecureTagValueAPI(session.auth.user) try: result = tagValues.set(values) except UnknownPathError as error: session.log.exception(error) path = error.paths[0] raise TNonexistentTag(path.encode('utf-8')) except MalformedPathError as error: # FIXME: Modify MalformedPathError to have a path field. raise TInvalidPath(str(error).encode('utf-8')) except PermissionDeniedError as error: session.log.exception(error) path_, operation = error.pathsAndOperations[0] category, action = getCategoryAndAction(operation) raise TPathPermissionDenied(category, action, path_)
def setUp(self): super(SecureTagValueAPIWithUserRoleTest, self).setUp() createSystemData() UserAPI().create([(u'username', u'password', u'User', u'*****@*****.**')]) self.user = getUser(u'username') self.permissions = CachingPermissionAPI(self.user) TagAPI(self.user).create([(u'username/tag', u'description')]) self.tagValues = SecureTagValueAPI(self.user)
def run(): tagValues = SecureTagValueAPI(session.auth.user) try: values = tagValues.get([objectID], [path]) return createThriftValue(len(values.keys()) > 0) except UnknownPathError as error: session.log.exception(error) raise TNonexistentTag(path.encode('utf-8')) except PermissionDeniedError as error: session.log.exception(error) path_, operation = error.pathsAndOperations[0] raise TNonexistentTag(path_)
def run(): result = TObjectInfo() objects = SecureObjectAPI(session.auth.user) tagPaths = objects.getTagsByObjects([objectID]) if not tagPaths: result.tagPaths = [] return result tagPaths = tagPaths[objectID] if showAbout and u'fluiddb/about' in tagPaths: tagValues = SecureTagValueAPI(session.auth.user) values = tagValues.get([objectID], [u'fluiddb/about']) about = values[objectID][u'fluiddb/about'].value result.about = about.encode('utf-8') result.tagPaths = tagPaths return result
def testGetObjectAllPathsAreDenied(self): """ L{FacadeObjectAPI.getObject} returns a L{TObjectInfo} with the L{Tag.path}s for which the L{User} has L{Operation.READ_TAG_VALUE}, if all of them are denied, L{FacadeObjectAPI.getObject} must return an empty L{TObjectInfo}. """ SecureTagAPI(self.user).create([(u'username/tag1', u'description'), (u'username/tag2', u'description')]) objectID = uuid4() values = { objectID: { u'username/tag1': 16 }, objectID: { u'username/tag2': 16 } } SecureTagValueAPI(self.user).set(values) self.permissions.set([ (u'username/tag1', Operation.READ_TAG_VALUE, Policy.CLOSED, []), (u'username/tag2', Operation.READ_TAG_VALUE, Policy.CLOSED, []) ]) self.store.commit() with login(self.user.username, uuid4(), self.transact) as session: objectInfo = yield self.facade.getObject(session, str(objectID)) self.assertEqual([], objectInfo.tagPaths) self.assertEqual(None, objectInfo.about)
def setUp(self): super(DatasetImporterTest, self).setUp() self.system = createSystemData() UserAPI().create([(u'user', u'secret', u'User', u'*****@*****.**')]) user = getUser(u'user') self.objects = SecureObjectAPI(user) self.values = SecureTagValueAPI(user)
def setUp(self): super(SecureTagValueAPIWithBrokenCacheTest, self).setUp() self.system = createSystemData() UserAPI().create([(u'username', u'password', u'User', u'*****@*****.**')]) self.user = getUser(u'username') self.permissions = CachingPermissionAPI(self.user) self.tagValues = SecureTagValueAPI(self.user)
def run(): try: SecureTagValueAPI(session.auth.user).delete(values) except UnknownPathError as error: session.log.exception(error) raise TNonexistentTag(path.encode('utf-8')) except PermissionDeniedError as error: session.log.exception(error) path_, operation = error.pathsAndOperations[0] category, action = getCategoryAndAction(operation) raise TPathPermissionDenied(category, action, path_)
def testGetTagsByObjectsPathIsAllowed(self): """ L{SecureObjectAPI.getTagsByObjects} will return all the tags for which the anonymous user has C{Operation.READ_TAG_VALUE} permissions. """ objectID = uuid4() values = {objectID: {u'username/tag': 16}} SecureTagValueAPI(self.user).set(values) self.permissions.set([(u'username/tag', Operation.READ_TAG_VALUE, Policy.OPEN, [])]) self.assertEqual({objectID: [u'username/tag']}, self.objects.getTagsByObjects([objectID]))
def run(): tagValues = SecureTagValueAPI(session.auth.user) objects = SecureObjectAPI(session.auth.user) objectIDs = self._resolveQuery(session, objects, parsedQuery) if not objectIDs: return dumps({'results': {'id': {}}}) try: values = tagValues.get(objectIDs, tags) except UnknownPathError as error: # One or more of the requested return Tag's doesn't exist. # We'll filter them out and try again because we don't want to # fail the request just because of a missing tag. filteredTags = set(tags) - set(error.paths) if not filteredTags: return dumps({'results': {'id': {}}}) values = tagValues.get(objectIDs, filteredTags) except PermissionDeniedError as error: session.log.exception(error) path_, operation = error.pathsAndOperations[0] raise TNonexistentTag(path_) valuesByObjectID = {} for objectID, tagPaths in values.iteritems(): for tagPath, tagValue in tagPaths.iteritems(): value = tagValue.value if isinstance(value, dict): size = len(value[u'contents']) mimeType = value[u'mime-type'] value = {u'value-type': mimeType, u'size': size} elif isinstance(value, UUID): value = {'value': str(value)} else: value = {'value': value} value['updated-at'] = tagValue.creationTime.isoformat() value['username'] = tagValue.creator.username objectID = str(objectID) valuesByObjectID.setdefault(objectID, {})[tagPath] = value result = {'results': {'id': valuesByObjectID}} return dumps(result)
def upload(self, username, objects, message=None): """Upload batch object data directly into Fluidinfo. The data to be uploaded must be provided in the following format:: [{'about': <about-tag-value>, 'values': {<tag-path>: <tag-value>, ...}}, ...] @param username: The name of the L{User} to import data on behalf of. @param objects: A C{list} of C{dict}s representing tags and values, organized as objects, to upload to Fluidinfo. @param message: An optional C{unicode} message used to make logging output less generic. """ message = (u'%s: ' % message) if message else u'' nObjects = len(objects) nImported = 0 logging.info('%sImporting %d new objects.', message, nObjects) startTime = time() start, end = 0, min(len(objects), self._batchSize) if end: while start < len(objects): # NOTE: Be careful here. An obvious refactoring, at first # glance, is to move the logic to get the user and create the # tag value API out of the loop, but doing so will cause the # user object to be used across transaction boundaries, which # we don't want to do. It's important that the database # interactions for each batch processed here are managed in a # single transaction. try: user = getUser(username) if user is None: raise UnknownUserError([username]) data = self._getObjectData(user, objects[start:end]) SecureTagValueAPI(user).set(data) transaction.commit() nImported += (end - start) logging.info('%sImported %d/%d new objects.', message, nImported, nObjects) except: transaction.abort() logging.info('%sImport failed. Aborting.', message) raise start, end = end, min(len(objects), end + self._batchSize) elapsed = time() - startTime logging.info( '%sImported %d objects in %s seconds, %.3f ' 'objects/second.', message, nObjects, elapsed, float(nObjects) / elapsed)
def testGetIsAllowed(self): """ L{SecureTagAPI.get} is allowed if the anonymous user has C{Operation.READ_TAG_VALUE} permissions. """ objectID = uuid4() values = {objectID: {u'username/tag': 16}} SecureTagValueAPI(self.user).set(values) self.permissions.set([(u'username/tag', Operation.READ_TAG_VALUE, Policy.OPEN, [])]) result = self.tagValues.get(objectIDs=[objectID], paths=[u'username/tag']) self.assertEqual(16, result[objectID][u'username/tag'].value)
def testGetTagsForObjectsOnlyReturnsAccessibleTags(self): """ L{SecureObjectAPI.getTagsForObjects} only returns L{Tag.path}s that the user has C{Operation.READ_TAG_VALUE} permissions for. """ TagAPI(self.user).create([(u'username/tag1', u'description'), (u'username/tag2', u'description')]) objectID = uuid4() SecureTagValueAPI(self.user).set({objectID: {u'username/tag1': 13, u'username/tag2': 17}}) self.permissions.set([(u'username/tag2', Operation.READ_TAG_VALUE, Policy.CLOSED, [])]) self.assertEqual([u'username/tag1'], self.objects.getTagsForObjects([objectID]))
def run(): objects = SecureObjectAPI(session.auth.user) # _resolveQuery is implemented in FacadeTagValueMixin objectIDs = self._resolveQuery(session, objects, parsedQuery) if not objectIDs: return [] # FIXME: This sucks, but right now if the query returns too many # objects, RecentActivityAPI will blow up. While we fix this, it's # better to return a 400 than a 500. if len(objectIDs) > 2000: raise TBadRequest('The given query returns to many objects.') values = SecureTagValueAPI(session.auth.user) result = values.get(objectIDs, [u'fluiddb/users/username']) usernames = [result[objectID][u'fluiddb/users/username'].value for objectID in result] recentActivity = SecureRecentActivityAPI(session.auth.user) result = recentActivity.getForUsers(usernames) return self._formatResult(result)
def testGetIsDenied(self): """ L{SecureTagAPI.get} raises L{PermissionDeniedError} if the anonymous user doesn't have C{Operation.READ_TAG_VALUE} permissions. """ objectID = uuid4() values = {objectID: {u'username/tag': 16}} SecureTagValueAPI(self.user).set(values) self.permissions.set([(u'username/tag', Operation.READ_TAG_VALUE, Policy.CLOSED, [])]) error = self.assertRaises(PermissionDeniedError, self.tagValues.get, objectIDs=[objectID], paths=[u'username/tag']) self.assertEqual([(u'username/tag', Operation.READ_TAG_VALUE)], error.pathsAndOperations)
def testGetTagsByObjectsWithCustomPermission(self): """ L{SecureObjectAPI.getTagsByObjects} optionally accepts a permission type to check for instead of L{Operation.READ_TAG_VALUE}). """ TagAPI(self.user).create([(u'username/open', u'An accessible tag'), (u'username/closed', u'A denied tag')]) objectID = uuid4() SecureTagValueAPI(self.user).set({objectID: {u'username/open': 13, u'username/closed': 17}}) self.permissions.set([(u'username/closed', Operation.DELETE_TAG_VALUE, Policy.CLOSED, [])]) result = self.objects.getTagsByObjects( [objectID], permission=Operation.DELETE_TAG_VALUE) self.assertEqual({objectID: [u'username/open']}, result)
def testGetObject(self): """ L{FacadeObjectAPI.getObject} returns a L{TObjectInfo} with the L{Tag.path}s for which the L{User} has L{Operation.READ_TAG_VALUE}. """ objectID = uuid4() SecureTagAPI(self.user).create([(u'username/foo', u'A description')]) values = {objectID: {u'username/foo': u'bar'}} SecureTagValueAPI(self.user).set(values) self.store.commit() with login(self.user.username, uuid4(), self.transact) as session: objectInfo = yield self.facade.getObject(session, str(objectID)) self.assertEqual([u'username/foo'], objectInfo.tagPaths) self.assertEqual(None, objectInfo.about)
def testCreateObjectWithDuplicateAboutValue(self): """ L{FacadeObjectAPI.createObject} returns a valid C{UUID} in a C{str} for an existing object ID if an about value is given and already exists in the database. """ objectID = uuid4() createAboutTagValue(objectID, u'bar') values = {objectID: {u'fluiddb/about': u'bar'}} SecureTagValueAPI(self.system.superuser).set(values) self.store.commit() with login(self.user.username, uuid4(), self.transact) as session: resultObjectID = yield self.facade.createObject(session, about='bar') self.assertEqual(str(objectID), resultObjectID)
def testGetTagsForObjectsIsAlwaysAllowed(self): """ L{SecureObjectAPI.getTagsForObjects} is always allowed for the superuser. """ TagAPI(self.superuser).create([(u'username/tag1', u'description'), (u'username/tag2', u'description')]) objectID = uuid4() values = {objectID: {u'username/tag1': 16, u'username/tag2': 16}} SecureTagValueAPI(self.superuser).set(values) self.permissions.set([(u'username/tag1', Operation.READ_TAG_VALUE, Policy.CLOSED, []), (u'username/tag2', Operation.READ_TAG_VALUE, Policy.OPEN, [])]) self.assertEqual( sorted([u'username/tag1', u'username/tag2']), sorted(self.objects.getTagsForObjects([objectID])))
def testGetTagsByObjectsReturnsNoneAllowedTags(self): """ L{SecureObjectAPI.getTagsByObjects} will return all the tags for which the user has C{Operation.READ_TAG_VALUE} permissions, but not those for which the user doesn't have. """ TagAPI(self.user).create([(u'username/tag1', u'description'), (u'username/tag2', u'description')]) objectID = uuid4() values = {objectID: {u'username/tag1': 16, u'username/tag2': 16}} SecureTagValueAPI(self.user).set(values) self.permissions.set([(u'username/tag1', Operation.READ_TAG_VALUE, Policy.CLOSED, []), (u'username/tag2', Operation.READ_TAG_VALUE, Policy.CLOSED, [])]) result = self.objects.getTagsByObjects([objectID]) self.assertEqual({}, result)
def testGetForUserWithAdditionalTagsUnreadable(self): """ L{SecureCommentAPI.getForUser} raises a L{PermissionDeniedError} if a L{User} tries to retrieve C{additionalTags} which are unreadable to them. """ objectID = uuid4() values = {objectID: {u'username/tag': 16}} SecureTagValueAPI(self.user).set(values) CachingPermissionAPI(self.user).set([(u'username/tag', Operation.READ_TAG_VALUE, Policy.CLOSED, [])]) error = self.assertRaises(PermissionDeniedError, self.comments.getForUser, u'username', additionalTags=[u'username/tag']) self.assertEqual(u'username', error.username) self.assertEqual( [(u'username/tag', Operation.READ_TAG_VALUE)], error.pathsAndOperations)
def testGetTagsByObjectsReturnsNoneIfDenied(self): """ L{SecureObjectAPI.getTagsByObjects} will return an empty C{dict} if the L{User} does not have C{Operation.READ_TAG_VALUE} permission on none of the L{Tag}s an object has. """ self.tags.create([(u'username/tag1', u'description'), (u'username/tag2', u'description')]) objectID = uuid4() values = {objectID: {u'username/tag1': 16, u'username/tag2': 16}} SecureTagValueAPI(self.user).set(values) self.permissions.set([(u'username/tag1', Operation.READ_TAG_VALUE, Policy.CLOSED, []), (u'username/tag2', Operation.READ_TAG_VALUE, Policy.CLOSED, [])]) result = self.objects.getTagsByObjects([objectID]) self.assertEqual({}, result)
def testGetObjectWithAboutValue(self): """ L{FacadeObjectAPI.getObject} returns a L{TObjectInfo} with the L{Tag.path}s for which the L{User} has L{Operation.READ_TAG_VALUE}, and the L{AboutTagValue.value} if it has one and C{showAbout} is C{True}. """ objectID = uuid4() SecureTagAPI(self.user).create([(u'username/foo', u'A description')]) values = {objectID: {u'username/foo': u'bar'}} SecureTagValueAPI(self.user).set(values) aboutTag = self.system.tags[u'fluiddb/about'] createAboutTagValue(objectID, u'bar') createTagValue(self.user.id, aboutTag.id, objectID, u'about value') self.store.commit() with login(self.user.username, uuid4(), self.transact) as session: objectInfo = yield self.facade.getObject(session, str(objectID), showAbout=True) self.assertEqual([u'fluiddb/about', u'username/foo'], sorted(objectInfo.tagPaths)) self.assertEqual('about value', objectInfo.about)
class CommentImporterTest(FluidinfoTestCase): resources = [('cache', CacheResource()), ('config', ConfigResource()), ('log', LoggingResource(format='%(message)s')), ('store', DatabaseResource())] def setUp(self): super(CommentImporterTest, self).setUp() self.system = createSystemData() UserAPI().create([ (u'fluidinfo.com', u'secret', u'User', u'*****@*****.**')]) user = getUser(u'fluidinfo.com') self.objects = SecureObjectAPI(user) self.values = SecureTagValueAPI(user) def testUpload(self): """ Comments are uploaded directly to Fluidinfo. After performing an upload, all the inserted values must be present in Fluidinfo. """ when = datetime.utcnow() floatTime = timegm(when.utctimetuple()) + float(when.strftime('0.%f')) isoTime = when.isoformat() client = CommentImporter(100) client.upload([ {'about': [u'one', u'two'], 'importer': u'fluidinfo.com', 'text': u'Here is my #wonderful comment', 'timestamp': when, 'url': u'http://twitter.com/status/9373973', 'username': u'joe'} ]) about = u'fluidinfo.com joe %s' % isoTime result = self.objects.get([about]) objectID = result[about] result = self.values.get([objectID], [u'fluidinfo.com/info/about', u'fluidinfo.com/info/text', u'fluidinfo.com/info/timestamp', u'fluidinfo.com/info/url', u'fluidinfo.com/info/username', ]) comment = result[objectID] self.assertEqual([u'one', u'two', u'#wonderful'], comment[u'fluidinfo.com/info/about'].value) self.assertEqual(u'Here is my #wonderful comment', comment[u'fluidinfo.com/info/text'].value) self.assertEqual(floatTime, comment[u'fluidinfo.com/info/timestamp'].value) self.assertEqual(u'http://twitter.com/status/9373973', comment[u'fluidinfo.com/info/url'].value) self.assertEqual(u'joe', comment[u'fluidinfo.com/info/username'].value) def testUploadWithoutAboutValues(self): """ When no explicit about values are in the uploaded comment, there must be no about values stored in Fluidinfo. """ when = datetime.utcnow() isoTime = when.isoformat() client = CommentImporter(100) client.upload([ {'importer': u'fluidinfo.com', 'text': u'Here is my comment', 'timestamp': when, 'url': u'http://twitter.com/status/9373973', 'username': u'joe'} ]) about = u'fluidinfo.com joe %s' % isoTime result = self.objects.get([about]) objectID = result[about] result = self.values.get([objectID], [u'fluidinfo.com/info/about']) comment = result[objectID] self.assertEqual([], comment[u'fluidinfo.com/info/about'].value) def testUploadMultipleComments(self): """Multiple comments are inserted in batches.""" when = datetime.utcnow() isoTime = when.isoformat() client = CommentImporter(100) client.upload([ {'importer': u'fluidinfo.com', 'text': u'Here is my #wonderful comment', 'timestamp': when, 'url': u'http://twitter.com/status/9373973', 'username': u'joe'}, {'importer': u'fluidinfo.com', 'text': u'A #crazy comment', 'timestamp': when, 'url': u'http://twitter.com/status/9279479379', 'username': u'mike'} ]) about1 = u'fluidinfo.com joe %s' % isoTime about2 = u'fluidinfo.com mike %s' % isoTime result = self.objects.get([about1, about2]) objectID1 = result[about1] objectID2 = result[about2] comments = self.values.get([objectID1, objectID2], [u'fluidinfo.com/info/text']) comment1 = comments[objectID1] self.assertEqual(u'Here is my #wonderful comment', comment1[u'fluidinfo.com/info/text'].value) comment2 = comments[objectID2] self.assertEqual(u'A #crazy comment', comment2[u'fluidinfo.com/info/text'].value) self.assertTrue(self.log.getvalue().startswith( 'Importing 2 new comments.\nImported 2/2 new comments.\n' 'Imported 2 comments in ')) def testUploadUsesBatchSize(self): """ Comments are uploaded in batches when possible, depending on the batch size. """ when = datetime.utcnow() client = CommentImporter(1) client.upload([ {'importer': u'fluidinfo.com', 'text': u'Here is my #wonderful comment', 'timestamp': when, 'url': u'http://twitter.com/status/9373973', 'username': u'joe'}, {'importer': u'fluidinfo.com', 'text': u'A #crazy comment', 'timestamp': when, 'url': u'http://twitter.com/status/9279479379', 'username': u'mike'} ]) self.assertTrue(self.log.getvalue().startswith( 'Importing 2 new comments.\nImported 1/2 new comments.\n' 'Imported 2/2 new comments.\nImported 2 comments in ')) def testUploadLogsMessage(self): """ Uploads must prefix log output with the passed message. """ when = datetime.utcnow() client = CommentImporter(100) client.upload([ {'importer': u'fluidinfo.com', 'text': u'Here is my #wonderful comment', 'timestamp': when, 'url': u'http://twitter.com/status/9373973', 'username': u'joe'}], 'message-xxx') self.assertTrue(self.log.getvalue().startswith( 'message-xxx: Importing 1 new comments.\n' 'message-xxx: Imported 1/1 new comments.\n' 'message-xxx: Imported 1 comments in '))
class DatasetImporterTest(FluidinfoTestCase): resources = [('cache', CacheResource()), ('config', ConfigResource()), ('log', LoggingResource(format='%(message)s')), ('store', DatabaseResource())] def setUp(self): super(DatasetImporterTest, self).setUp() self.system = createSystemData() UserAPI().create([(u'user', u'secret', u'User', u'*****@*****.**')]) user = getUser(u'user') self.objects = SecureObjectAPI(user) self.values = SecureTagValueAPI(user) def testUpload(self): """ Object data is converted into a format compatible with the L{TagValueAPI.set} method before being uploaded directly into Fluidinfo. """ client = DatasetImporter(100) client.upload(u'user', [{ 'about': u'hello world', 'values': { u'user/bar': 13 } }]) result = self.objects.get([u'hello world']) objectID = result[u'hello world'] result = self.values.get([objectID], [u'user/bar']) value = result[objectID][u'user/bar'] self.assertEqual(13, value.value) def testUploadMultipleObjects(self): """Multiple objects are inserted in batches.""" client = DatasetImporter(100) client.upload(u'user', [{ 'about': u'hello world', 'values': { u'user/bar': 13 } }, { 'about': u'wubble', 'values': { u'user/quux': 42 } }]) aboutValues = self.objects.get([u'hello world', u'wubble']) objectID = aboutValues[u'hello world'] result = self.values.get([objectID], [u'user/bar']) value = result[objectID][u'user/bar'] self.assertEqual(13, value.value) objectID = aboutValues[u'wubble'] result = self.values.get([objectID], [u'user/quux']) value = result[objectID][u'user/quux'] self.assertEqual(42, value.value) self.assertTrue(self.log.getvalue().startswith( 'Importing 2 new objects.\nImported 2/2 new objects.\n' 'Imported 2 objects in ')) def testUploadUsesBatchSize(self): """ Objects are uploaded in batches when possible, depending on the batch size. """ client = DatasetImporter(1) client.upload(u'user', [{ 'about': u'hello world', 'values': { u'user/bar': 13 } }, { 'about': u'wubble', 'values': { u'user/quux': 42 } }]) self.assertTrue(self.log.getvalue().startswith( 'Importing 2 new objects.\nImported 1/2 new objects.\n' 'Imported 2/2 new objects.\nImported 2 objects in ')) def testUploadWithUncreatablePath(self): """ L{DatasetImporter.upload} checks permissions when importing data. An L{UnknownPathError} is raised if a specified tag doesn't exist and the L{User} doesn't have permissions to create it. """ client = DatasetImporter(100) self.assertRaises(UnknownPathError, client.upload, u'user', [{ 'about': u'hello world', 'values': { u'foo/bar': 13 } }]) def testUploadWithPermissionViolation(self): """L{DatasetImporter.upload} checks permissions when importing data.""" UserAPI().create([(u'user1', u'pwd', u'User 1', u'*****@*****.**')]) client = DatasetImporter(100) self.assertRaises(PermissionDeniedError, client.upload, u'user', [{ 'about': u'hello world', 'values': { u'user1/bar': 13 } }]) def testUploadWithUnknownUser(self): """ L{DatasetImporter.upload} raises an L{UnknownUserError} if the specified L{User} doesn't exist. """ client = DatasetImporter(100) self.assertRaises(UnknownUserError, client.upload, u'unknown', [{ 'about': u'hello world', 'values': { u'unknown/bar': 13 } }]) def testUploadLogsMessage(self): """ Uploads must prefix log output with the passed message. """ client = DatasetImporter(100) client.upload(u'user', [{ 'about': u'hello world', 'values': { u'user/bar': 13 } }], 'message-xxx') self.assertTrue(self.log.getvalue().startswith( 'message-xxx: Importing 1 new objects.\n' 'message-xxx: Imported 1/1 new objects.\n' 'message-xxx: Imported 1 objects in '))
class CommentImporterTest(FluidinfoTestCase): resources = [('cache', CacheResource()), ('config', ConfigResource()), ('log', LoggingResource(format='%(message)s')), ('store', DatabaseResource())] def setUp(self): super(CommentImporterTest, self).setUp() self.system = createSystemData() UserAPI().create([(u'fluidinfo.com', u'secret', u'User', u'*****@*****.**')]) user = getUser(u'fluidinfo.com') self.objects = SecureObjectAPI(user) self.values = SecureTagValueAPI(user) def testUpload(self): """ Comments are uploaded directly to Fluidinfo. After performing an upload, all the inserted values must be present in Fluidinfo. """ when = datetime.utcnow() floatTime = timegm(when.utctimetuple()) + float(when.strftime('0.%f')) isoTime = when.isoformat() client = CommentImporter(100) client.upload([{ 'about': [u'one', u'two'], 'importer': u'fluidinfo.com', 'text': u'Here is my #wonderful comment', 'timestamp': when, 'url': u'http://twitter.com/status/9373973', 'username': u'joe' }]) about = u'fluidinfo.com joe %s' % isoTime result = self.objects.get([about]) objectID = result[about] result = self.values.get([objectID], [ u'fluidinfo.com/info/about', u'fluidinfo.com/info/text', u'fluidinfo.com/info/timestamp', u'fluidinfo.com/info/url', u'fluidinfo.com/info/username', ]) comment = result[objectID] self.assertEqual([u'one', u'two', u'#wonderful'], comment[u'fluidinfo.com/info/about'].value) self.assertEqual(u'Here is my #wonderful comment', comment[u'fluidinfo.com/info/text'].value) self.assertEqual(floatTime, comment[u'fluidinfo.com/info/timestamp'].value) self.assertEqual(u'http://twitter.com/status/9373973', comment[u'fluidinfo.com/info/url'].value) self.assertEqual(u'joe', comment[u'fluidinfo.com/info/username'].value) def testUploadWithoutAboutValues(self): """ When no explicit about values are in the uploaded comment, there must be no about values stored in Fluidinfo. """ when = datetime.utcnow() isoTime = when.isoformat() client = CommentImporter(100) client.upload([{ 'importer': u'fluidinfo.com', 'text': u'Here is my comment', 'timestamp': when, 'url': u'http://twitter.com/status/9373973', 'username': u'joe' }]) about = u'fluidinfo.com joe %s' % isoTime result = self.objects.get([about]) objectID = result[about] result = self.values.get([objectID], [u'fluidinfo.com/info/about']) comment = result[objectID] self.assertEqual([], comment[u'fluidinfo.com/info/about'].value) def testUploadMultipleComments(self): """Multiple comments are inserted in batches.""" when = datetime.utcnow() isoTime = when.isoformat() client = CommentImporter(100) client.upload([{ 'importer': u'fluidinfo.com', 'text': u'Here is my #wonderful comment', 'timestamp': when, 'url': u'http://twitter.com/status/9373973', 'username': u'joe' }, { 'importer': u'fluidinfo.com', 'text': u'A #crazy comment', 'timestamp': when, 'url': u'http://twitter.com/status/9279479379', 'username': u'mike' }]) about1 = u'fluidinfo.com joe %s' % isoTime about2 = u'fluidinfo.com mike %s' % isoTime result = self.objects.get([about1, about2]) objectID1 = result[about1] objectID2 = result[about2] comments = self.values.get([objectID1, objectID2], [u'fluidinfo.com/info/text']) comment1 = comments[objectID1] self.assertEqual(u'Here is my #wonderful comment', comment1[u'fluidinfo.com/info/text'].value) comment2 = comments[objectID2] self.assertEqual(u'A #crazy comment', comment2[u'fluidinfo.com/info/text'].value) self.assertTrue(self.log.getvalue().startswith( 'Importing 2 new comments.\nImported 2/2 new comments.\n' 'Imported 2 comments in ')) def testUploadUsesBatchSize(self): """ Comments are uploaded in batches when possible, depending on the batch size. """ when = datetime.utcnow() client = CommentImporter(1) client.upload([{ 'importer': u'fluidinfo.com', 'text': u'Here is my #wonderful comment', 'timestamp': when, 'url': u'http://twitter.com/status/9373973', 'username': u'joe' }, { 'importer': u'fluidinfo.com', 'text': u'A #crazy comment', 'timestamp': when, 'url': u'http://twitter.com/status/9279479379', 'username': u'mike' }]) self.assertTrue(self.log.getvalue().startswith( 'Importing 2 new comments.\nImported 1/2 new comments.\n' 'Imported 2/2 new comments.\nImported 2 comments in ')) def testUploadLogsMessage(self): """ Uploads must prefix log output with the passed message. """ when = datetime.utcnow() client = CommentImporter(100) client.upload([{ 'importer': u'fluidinfo.com', 'text': u'Here is my #wonderful comment', 'timestamp': when, 'url': u'http://twitter.com/status/9373973', 'username': u'joe' }], 'message-xxx') self.assertTrue(self.log.getvalue().startswith( 'message-xxx: Importing 1 new comments.\n' 'message-xxx: Imported 1/1 new comments.\n' 'message-xxx: Imported 1 comments in '))
class SecureTagValueAPIWithUserRoleTest(FluidinfoTestCase): resources = [('cache', CacheResource()), ('config', ConfigResource()), ('store', DatabaseResource())] def setUp(self): super(SecureTagValueAPIWithUserRoleTest, self).setUp() createSystemData() UserAPI().create([(u'username', u'password', u'User', u'*****@*****.**')]) self.user = getUser(u'username') self.permissions = CachingPermissionAPI(self.user) TagAPI(self.user).create([(u'username/tag', u'description')]) self.tagValues = SecureTagValueAPI(self.user) def testGetIsAllowed(self): """ L{SecureTagAPI.get} is allowed if the user has C{Operation.READ_TAG_VALUE} permissions. """ objectID = uuid4() values = {objectID: {u'username/tag': 16}} self.tagValues.set(values) self.permissions.set([(u'username/tag', Operation.READ_TAG_VALUE, Policy.OPEN, [])]) result = self.tagValues.get(objectIDs=[objectID], paths=[u'username/tag']) self.assertEqual(16, result[objectID][u'username/tag'].value) def testGetIsDenied(self): """ L{SecureTagAPI.get} raises L{PermissionDeniedError} if the user doesn't have C{Operation.READ_TAG_VALUE} permissions. """ objectID = uuid4() values = {objectID: {u'username/tag': 16}} self.tagValues.set(values) self.permissions.set([(u'username/tag', Operation.READ_TAG_VALUE, Policy.CLOSED, [])]) error = self.assertRaises(PermissionDeniedError, self.tagValues.get, objectIDs=[objectID], paths=[u'username/tag']) self.assertEqual([(u'username/tag', Operation.READ_TAG_VALUE)], error.pathsAndOperations) def testSetIsAllowed(self): """ L{SecureTagAPI.set} is allowed if the user has C{Operation.WRITE_TAG_VALUE} permissions. """ self.permissions.set([(u'username/tag', Operation.WRITE_TAG_VALUE, Policy.OPEN, [])]) objectID = uuid4() values = {objectID: {u'username/tag': 16}} self.tagValues.set(values) result = self.tagValues.get(objectIDs=[objectID], paths=[u'username/tag']) self.assertEqual(16, result[objectID][u'username/tag'].value) def testSetIsDenied(self): """ L{SecureTagAPI.set} raises L{PermissionDeniedError} if the user doesn't have C{Operation.WRITE_TAG_VALUE} permissions. """ self.permissions.set([(u'username/tag', Operation.WRITE_TAG_VALUE, Policy.CLOSED, [])]) values = {uuid4(): {u'username/tag': 16}} error = self.assertRaises(PermissionDeniedError, self.tagValues.set, values) self.assertEqual([(u'username/tag', Operation.WRITE_TAG_VALUE)], error.pathsAndOperations) def testDeleteIsAllowed(self): """ L{SecureTagAPI.delete} is allowed if the user has C{Operation.DELETE_TAG_VALUE} permissions. """ self.permissions.set([(u'username/tag', Operation.DELETE_TAG_VALUE, Policy.OPEN, [])]) objectID = uuid4() values = {objectID: {u'username/tag': 16}} self.tagValues.set(values) self.tagValues.delete([(objectID, u'username/tag')]) result = self.tagValues.get(objectIDs=[objectID], paths=[u'username/tag']) self.assertEqual({}, result) def testDeleteIsDenied(self): """ L{SecureTagAPI.delete} raises L{PermissionDeniedError} if the user doesn't have C{Operation.DELETE_TAG_VALUE} permissions. """ objectID = uuid4() values = {objectID: {u'username/tag': 16}} self.tagValues.set(values) self.permissions.set([(u'username/tag', Operation.DELETE_TAG_VALUE, Policy.CLOSED, [])]) error = self.assertRaises(PermissionDeniedError, self.tagValues.delete, [(objectID, u'username/tag')]) self.assertEqual([(u'username/tag', Operation.DELETE_TAG_VALUE)], error.pathsAndOperations)
class SecureTagValueAPIWithSuperuserRoleTest(FluidinfoTestCase): resources = [('cache', CacheResource()), ('config', ConfigResource()), ('store', DatabaseResource())] def setUp(self): super(SecureTagValueAPIWithSuperuserRoleTest, self).setUp() system = createSystemData() self.superuser = system.users[u'fluiddb'] TagAPI(self.superuser).create([(u'fluiddb/tag', u'description')]) self.tagValues = SecureTagValueAPI(self.superuser) self.permissions = CachingPermissionAPI(self.superuser) def testGetIsAllowed(self): """L{SecureTagValueAPI.get} is always allowed for the superuser.""" objectID = uuid4() values = {objectID: {u'fluiddb/tag': 16}} self.tagValues.set(values) self.permissions.set([(u'fluiddb/tag', Operation.READ_TAG_VALUE, Policy.CLOSED, [])]) result = self.tagValues.get(objectIDs=[objectID], paths=[u'fluiddb/tag']) self.assertEqual(16, result[objectID][u'fluiddb/tag'].value) def testSetIsAllowed(self): """L{SecureTagValueAPI.set} is always allowed for the superuser.""" self.permissions.set([(u'fluiddb/tag', Operation.WRITE_TAG_VALUE, Policy.CLOSED, [])]) objectID = uuid4() values = {objectID: {u'fluiddb/tag': 16}} self.tagValues.set(values) result = self.tagValues.get(objectIDs=[objectID], paths=[u'fluiddb/tag']) self.assertEqual(16, result[objectID][u'fluiddb/tag'].value) def testDeleteIsAllowed(self): """L{SecureTagValueAPI.delete} is always allowed for the superuser.""" self.permissions.set([(u'fluiddb/tag', Operation.DELETE_TAG_VALUE, Policy.CLOSED, [])]) objectID = uuid4() values = {objectID: {u'fluiddb/tag': 16}} self.tagValues.set(values) self.tagValues.delete([(objectID, u'fluiddb/tag')]) result = self.tagValues.get(objectIDs=[objectID], paths=[u'fluiddb/tag']) self.assertEqual({}, result)
class DatasetImporterTest(FluidinfoTestCase): resources = [('cache', CacheResource()), ('config', ConfigResource()), ('log', LoggingResource(format='%(message)s')), ('store', DatabaseResource())] def setUp(self): super(DatasetImporterTest, self).setUp() self.system = createSystemData() UserAPI().create([(u'user', u'secret', u'User', u'*****@*****.**')]) user = getUser(u'user') self.objects = SecureObjectAPI(user) self.values = SecureTagValueAPI(user) def testUpload(self): """ Object data is converted into a format compatible with the L{TagValueAPI.set} method before being uploaded directly into Fluidinfo. """ client = DatasetImporter(100) client.upload(u'user', [{'about': u'hello world', 'values': {u'user/bar': 13}}]) result = self.objects.get([u'hello world']) objectID = result[u'hello world'] result = self.values.get([objectID], [u'user/bar']) value = result[objectID][u'user/bar'] self.assertEqual(13, value.value) def testUploadMultipleObjects(self): """Multiple objects are inserted in batches.""" client = DatasetImporter(100) client.upload(u'user', [{'about': u'hello world', 'values': {u'user/bar': 13}}, {'about': u'wubble', 'values': {u'user/quux': 42}}]) aboutValues = self.objects.get([u'hello world', u'wubble']) objectID = aboutValues[u'hello world'] result = self.values.get([objectID], [u'user/bar']) value = result[objectID][u'user/bar'] self.assertEqual(13, value.value) objectID = aboutValues[u'wubble'] result = self.values.get([objectID], [u'user/quux']) value = result[objectID][u'user/quux'] self.assertEqual(42, value.value) self.assertTrue(self.log.getvalue().startswith( 'Importing 2 new objects.\nImported 2/2 new objects.\n' 'Imported 2 objects in ')) def testUploadUsesBatchSize(self): """ Objects are uploaded in batches when possible, depending on the batch size. """ client = DatasetImporter(1) client.upload(u'user', [{'about': u'hello world', 'values': {u'user/bar': 13}}, {'about': u'wubble', 'values': {u'user/quux': 42}}]) self.assertTrue(self.log.getvalue().startswith( 'Importing 2 new objects.\nImported 1/2 new objects.\n' 'Imported 2/2 new objects.\nImported 2 objects in ')) def testUploadWithUncreatablePath(self): """ L{DatasetImporter.upload} checks permissions when importing data. An L{UnknownPathError} is raised if a specified tag doesn't exist and the L{User} doesn't have permissions to create it. """ client = DatasetImporter(100) self.assertRaises( UnknownPathError, client.upload, u'user', [{'about': u'hello world', 'values': {u'foo/bar': 13}}]) def testUploadWithPermissionViolation(self): """L{DatasetImporter.upload} checks permissions when importing data.""" UserAPI().create([(u'user1', u'pwd', u'User 1', u'*****@*****.**')]) client = DatasetImporter(100) self.assertRaises( PermissionDeniedError, client.upload, u'user', [{'about': u'hello world', 'values': {u'user1/bar': 13}}]) def testUploadWithUnknownUser(self): """ L{DatasetImporter.upload} raises an L{UnknownUserError} if the specified L{User} doesn't exist. """ client = DatasetImporter(100) self.assertRaises( UnknownUserError, client.upload, u'unknown', [{'about': u'hello world', 'values': {u'unknown/bar': 13}}]) def testUploadLogsMessage(self): """ Uploads must prefix log output with the passed message. """ client = DatasetImporter(100) client.upload(u'user', [{'about': u'hello world', 'values': {u'user/bar': 13}}], 'message-xxx') self.assertTrue(self.log.getvalue().startswith( 'message-xxx: Importing 1 new objects.\n' 'message-xxx: Imported 1/1 new objects.\n' 'message-xxx: Imported 1 objects in '))