def run():
tagValues = SecureTagValueAPI(session.auth.user)
try:
result = tagValues.get([objectID], [path])
except PermissionDeniedError as error:
session.log.exception(error)
path_, operation = error.pathsAndOperations[0]
raise TNonexistentTag(path_)
except UnknownPathError as error:
session.log.exception(error)
raise TNonexistentTag(path)
if not result:
raise TNoInstanceOnObject(path, objectId)
else:
tagValue = result[objectID][path]
value = tagValue.value
# FIXME This is a bit crap, but its easier than modifying
# Thrift-related logic.
if isinstance(value, dict):
mimeType = value['mime-type'].encode('utf-8')
value = createBinaryThriftValue(value['contents'],
mimeType)
elif isinstance(value, UUID):
value = createThriftValue(str(value))
else:
value = createThriftValue(value)
return (value, tagValue)
def run():
tagValues = SecureTagValueAPI(session.auth.user)
objects = SecureObjectAPI(session.auth.user)
objectIDs = self._resolveQuery(session, objects, parsedQuery)
values = []
if tags is None:
# delete all tags user has permissions for
result = objects.getTagsByObjects(objectIDs,
Operation.DELETE_TAG_VALUE)
for objectID, paths in result.iteritems():
for path in paths:
values.append((objectID, path))
else:
# delete only tags requested by user
result = objects.getTagsByObjects(objectIDs)
for objectID, paths in result.iteritems():
for path in paths:
if tags is None or path in tags:
values.append((objectID, path))
if values:
try:
tagValues.delete(values)
except UnknownPathError as error:
session.log.exception(error)
path = error.paths[0]
raise TNonexistentTag(path.encode('utf-8'))
except PermissionDeniedError as error:
session.log.exception(error)
path_, operation = error.pathsAndOperations[0]
category, action = getCategoryAndAction(operation)
raise TPathPermissionDenied(category, action, path_)
def setUp(self):
super(SecureTagValueAPIWithSuperuserRoleTest, self).setUp()
system = createSystemData()
self.superuser = system.users[u'fluiddb']
TagAPI(self.superuser).create([(u'fluiddb/tag', u'description')])
self.tagValues = SecureTagValueAPI(self.superuser)
self.permissions = CachingPermissionAPI(self.superuser)
def setUp(self):
super(DatasetImporterTest, self).setUp()
self.system = createSystemData()
UserAPI().create([(u'user', u'secret', u'User', u'*****@*****.**')])
user = getUser(u'user')
self.objects = SecureObjectAPI(user)
self.values = SecureTagValueAPI(user)
def run():
objects = SecureObjectAPI(session.auth.user)
try:
searchQueries = objects.search(valuesByQuery.keys())
except UnknownPathError as error:
session.log.exception(error)
unknownPath = error.paths[0]
raise TNonexistentTag(unknownPath.encode('utf-8'))
except PermissionDeniedError as error:
session.log.exception(error)
path_, operation = error.pathsAndOperations[0]
if operation == Operation.CREATE_OBJECT:
raise TUnauthorized()
else:
raise TNonexistentTag(path_)
# Run queries.
try:
with session.timer.track('index-search'):
result = blockingCallFromThread(reactor, searchQueries.get)
except SearchError as error:
session.log.exception(error)
raise TParseError(query, error.message)
# Build a result set from the searches.
values = {}
for parsedQuery, objectIDs in result.iteritems():
for objectID in objectIDs:
for tagAndValue in valuesByQuery[parsedQuery]:
value = guessValue(tagAndValue.value)
# FIXME: this code sucks, but I rather not having
# to modify guessValue to return a list, as that
# would break other code.
# Hopefully, we'll be able to remove this pretty
# soon.
if isinstance(value, list):
value = [item.decode('utf-8') for item in value]
if objectID not in values:
values[objectID] = {}
values[objectID][tagAndValue.path] = value
# Update values.
if values:
tagValues = SecureTagValueAPI(session.auth.user)
try:
result = tagValues.set(values)
except UnknownPathError as error:
session.log.exception(error)
path = error.paths[0]
raise TNonexistentTag(path.encode('utf-8'))
except MalformedPathError as error:
# FIXME: Modify MalformedPathError to have a path field.
raise TInvalidPath(str(error).encode('utf-8'))
except PermissionDeniedError as error:
session.log.exception(error)
path_, operation = error.pathsAndOperations[0]
category, action = getCategoryAndAction(operation)
raise TPathPermissionDenied(category, action, path_)
def setUp(self):
super(SecureTagValueAPIWithUserRoleTest, self).setUp()
createSystemData()
UserAPI().create([(u'username', u'password', u'User',
u'*****@*****.**')])
self.user = getUser(u'username')
self.permissions = CachingPermissionAPI(self.user)
TagAPI(self.user).create([(u'username/tag', u'description')])
self.tagValues = SecureTagValueAPI(self.user)
def run():
tagValues = SecureTagValueAPI(session.auth.user)
try:
values = tagValues.get([objectID], [path])
return createThriftValue(len(values.keys()) > 0)
except UnknownPathError as error:
session.log.exception(error)
raise TNonexistentTag(path.encode('utf-8'))
except PermissionDeniedError as error:
session.log.exception(error)
path_, operation = error.pathsAndOperations[0]
raise TNonexistentTag(path_)
def run():
result = TObjectInfo()
objects = SecureObjectAPI(session.auth.user)
tagPaths = objects.getTagsByObjects([objectID])
if not tagPaths:
result.tagPaths = []
return result
tagPaths = tagPaths[objectID]
if showAbout and u'fluiddb/about' in tagPaths:
tagValues = SecureTagValueAPI(session.auth.user)
values = tagValues.get([objectID], [u'fluiddb/about'])
about = values[objectID][u'fluiddb/about'].value
result.about = about.encode('utf-8')
result.tagPaths = tagPaths
return result
def testGetObjectAllPathsAreDenied(self):
"""
L{FacadeObjectAPI.getObject} returns a L{TObjectInfo} with the
L{Tag.path}s for which the L{User} has L{Operation.READ_TAG_VALUE}, if
all of them are denied, L{FacadeObjectAPI.getObject} must return an
empty L{TObjectInfo}.
"""
SecureTagAPI(self.user).create([(u'username/tag1', u'description'),
(u'username/tag2', u'description')])
objectID = uuid4()
values = {
objectID: {
u'username/tag1': 16
},
objectID: {
u'username/tag2': 16
}
}
SecureTagValueAPI(self.user).set(values)
self.permissions.set([
(u'username/tag1', Operation.READ_TAG_VALUE, Policy.CLOSED, []),
(u'username/tag2', Operation.READ_TAG_VALUE, Policy.CLOSED, [])
])
self.store.commit()
with login(self.user.username, uuid4(), self.transact) as session:
objectInfo = yield self.facade.getObject(session, str(objectID))
self.assertEqual([], objectInfo.tagPaths)
self.assertEqual(None, objectInfo.about)
def setUp(self):
super(DatasetImporterTest, self).setUp()
self.system = createSystemData()
UserAPI().create([(u'user', u'secret', u'User', u'*****@*****.**')])
user = getUser(u'user')
self.objects = SecureObjectAPI(user)
self.values = SecureTagValueAPI(user)
def setUp(self):
super(SecureTagValueAPIWithBrokenCacheTest, self).setUp()
self.system = createSystemData()
UserAPI().create([(u'username', u'password', u'User',
u'*****@*****.**')])
self.user = getUser(u'username')
self.permissions = CachingPermissionAPI(self.user)
self.tagValues = SecureTagValueAPI(self.user)
def run():
try:
SecureTagValueAPI(session.auth.user).delete(values)
except UnknownPathError as error:
session.log.exception(error)
raise TNonexistentTag(path.encode('utf-8'))
except PermissionDeniedError as error:
session.log.exception(error)
path_, operation = error.pathsAndOperations[0]
category, action = getCategoryAndAction(operation)
raise TPathPermissionDenied(category, action, path_)
def testGetTagsByObjectsPathIsAllowed(self):
"""
L{SecureObjectAPI.getTagsByObjects} will return all the tags for
which the anonymous user has C{Operation.READ_TAG_VALUE} permissions.
"""
objectID = uuid4()
values = {objectID: {u'username/tag': 16}}
SecureTagValueAPI(self.user).set(values)
self.permissions.set([(u'username/tag', Operation.READ_TAG_VALUE,
Policy.OPEN, [])])
self.assertEqual({objectID: [u'username/tag']},
self.objects.getTagsByObjects([objectID]))
def run():
tagValues = SecureTagValueAPI(session.auth.user)
objects = SecureObjectAPI(session.auth.user)
objectIDs = self._resolveQuery(session, objects, parsedQuery)
if not objectIDs:
return dumps({'results': {'id': {}}})
try:
values = tagValues.get(objectIDs, tags)
except UnknownPathError as error:
# One or more of the requested return Tag's doesn't exist.
# We'll filter them out and try again because we don't want to
# fail the request just because of a missing tag.
filteredTags = set(tags) - set(error.paths)
if not filteredTags:
return dumps({'results': {'id': {}}})
values = tagValues.get(objectIDs, filteredTags)
except PermissionDeniedError as error:
session.log.exception(error)
path_, operation = error.pathsAndOperations[0]
raise TNonexistentTag(path_)
valuesByObjectID = {}
for objectID, tagPaths in values.iteritems():
for tagPath, tagValue in tagPaths.iteritems():
value = tagValue.value
if isinstance(value, dict):
size = len(value[u'contents'])
mimeType = value[u'mime-type']
value = {u'value-type': mimeType,
u'size': size}
elif isinstance(value, UUID):
value = {'value': str(value)}
else:
value = {'value': value}
value['updated-at'] = tagValue.creationTime.isoformat()
value['username'] = tagValue.creator.username
objectID = str(objectID)
valuesByObjectID.setdefault(objectID, {})[tagPath] = value
result = {'results': {'id': valuesByObjectID}}
return dumps(result)
def upload(self, username, objects, message=None):
"""Upload batch object data directly into Fluidinfo.
The data to be uploaded must be provided in the following format::
[{'about': <about-tag-value>,
'values': {<tag-path>: <tag-value>, ...}},
...]
@param username: The name of the L{User} to import data on behalf of.
@param objects: A C{list} of C{dict}s representing tags and values,
organized as objects, to upload to Fluidinfo.
@param message: An optional C{unicode} message used to make logging
output less generic.
"""
message = (u'%s: ' % message) if message else u''
nObjects = len(objects)
nImported = 0
logging.info('%sImporting %d new objects.', message, nObjects)
startTime = time()
start, end = 0, min(len(objects), self._batchSize)
if end:
while start < len(objects):
# NOTE: Be careful here. An obvious refactoring, at first
# glance, is to move the logic to get the user and create the
# tag value API out of the loop, but doing so will cause the
# user object to be used across transaction boundaries, which
# we don't want to do. It's important that the database
# interactions for each batch processed here are managed in a
# single transaction.
try:
user = getUser(username)
if user is None:
raise UnknownUserError([username])
data = self._getObjectData(user, objects[start:end])
SecureTagValueAPI(user).set(data)
transaction.commit()
nImported += (end - start)
logging.info('%sImported %d/%d new objects.', message,
nImported, nObjects)
except:
transaction.abort()
logging.info('%sImport failed. Aborting.', message)
raise
start, end = end, min(len(objects), end + self._batchSize)
elapsed = time() - startTime
logging.info(
'%sImported %d objects in %s seconds, %.3f '
'objects/second.', message, nObjects, elapsed,
float(nObjects) / elapsed)
def testGetIsAllowed(self):
"""
L{SecureTagAPI.get} is allowed if the anonymous user has
C{Operation.READ_TAG_VALUE} permissions.
"""
objectID = uuid4()
values = {objectID: {u'username/tag': 16}}
SecureTagValueAPI(self.user).set(values)
self.permissions.set([(u'username/tag', Operation.READ_TAG_VALUE,
Policy.OPEN, [])])
result = self.tagValues.get(objectIDs=[objectID],
paths=[u'username/tag'])
self.assertEqual(16, result[objectID][u'username/tag'].value)
def testGetTagsForObjectsOnlyReturnsAccessibleTags(self):
"""
L{SecureObjectAPI.getTagsForObjects} only returns L{Tag.path}s that
the user has C{Operation.READ_TAG_VALUE} permissions for.
"""
TagAPI(self.user).create([(u'username/tag1', u'description'),
(u'username/tag2', u'description')])
objectID = uuid4()
SecureTagValueAPI(self.user).set({objectID: {u'username/tag1': 13,
u'username/tag2': 17}})
self.permissions.set([(u'username/tag2', Operation.READ_TAG_VALUE,
Policy.CLOSED, [])])
self.assertEqual([u'username/tag1'],
self.objects.getTagsForObjects([objectID]))
def run():
objects = SecureObjectAPI(session.auth.user)
# _resolveQuery is implemented in FacadeTagValueMixin
objectIDs = self._resolveQuery(session, objects, parsedQuery)
if not objectIDs:
return []
# FIXME: This sucks, but right now if the query returns too many
# objects, RecentActivityAPI will blow up. While we fix this, it's
# better to return a 400 than a 500.
if len(objectIDs) > 2000:
raise TBadRequest('The given query returns to many objects.')
values = SecureTagValueAPI(session.auth.user)
result = values.get(objectIDs, [u'fluiddb/users/username'])
usernames = [result[objectID][u'fluiddb/users/username'].value
for objectID in result]
recentActivity = SecureRecentActivityAPI(session.auth.user)
result = recentActivity.getForUsers(usernames)
return self._formatResult(result)
def testGetIsDenied(self):
"""
L{SecureTagAPI.get} raises L{PermissionDeniedError} if the anonymous
user doesn't have C{Operation.READ_TAG_VALUE} permissions.
"""
objectID = uuid4()
values = {objectID: {u'username/tag': 16}}
SecureTagValueAPI(self.user).set(values)
self.permissions.set([(u'username/tag', Operation.READ_TAG_VALUE,
Policy.CLOSED, [])])
error = self.assertRaises(PermissionDeniedError,
self.tagValues.get, objectIDs=[objectID],
paths=[u'username/tag'])
self.assertEqual([(u'username/tag', Operation.READ_TAG_VALUE)],
error.pathsAndOperations)
def testGetTagsByObjectsWithCustomPermission(self):
"""
L{SecureObjectAPI.getTagsByObjects} optionally accepts a permission
type to check for instead of L{Operation.READ_TAG_VALUE}).
"""
TagAPI(self.user).create([(u'username/open', u'An accessible tag'),
(u'username/closed', u'A denied tag')])
objectID = uuid4()
SecureTagValueAPI(self.user).set({objectID: {u'username/open': 13,
u'username/closed': 17}})
self.permissions.set([(u'username/closed', Operation.DELETE_TAG_VALUE,
Policy.CLOSED, [])])
result = self.objects.getTagsByObjects(
[objectID], permission=Operation.DELETE_TAG_VALUE)
self.assertEqual({objectID: [u'username/open']}, result)
def testGetObject(self):
"""
L{FacadeObjectAPI.getObject} returns a L{TObjectInfo} with the
L{Tag.path}s for which the L{User} has L{Operation.READ_TAG_VALUE}.
"""
objectID = uuid4()
SecureTagAPI(self.user).create([(u'username/foo', u'A description')])
values = {objectID: {u'username/foo': u'bar'}}
SecureTagValueAPI(self.user).set(values)
self.store.commit()
with login(self.user.username, uuid4(), self.transact) as session:
objectInfo = yield self.facade.getObject(session, str(objectID))
self.assertEqual([u'username/foo'], objectInfo.tagPaths)
self.assertEqual(None, objectInfo.about)
def testCreateObjectWithDuplicateAboutValue(self):
"""
L{FacadeObjectAPI.createObject} returns a valid C{UUID} in a C{str} for
an existing object ID if an about value is given and already exists in
the database.
"""
objectID = uuid4()
createAboutTagValue(objectID, u'bar')
values = {objectID: {u'fluiddb/about': u'bar'}}
SecureTagValueAPI(self.system.superuser).set(values)
self.store.commit()
with login(self.user.username, uuid4(), self.transact) as session:
resultObjectID = yield self.facade.createObject(session,
about='bar')
self.assertEqual(str(objectID), resultObjectID)
def testGetTagsForObjectsIsAlwaysAllowed(self):
"""
L{SecureObjectAPI.getTagsForObjects} is always allowed for the
superuser.
"""
TagAPI(self.superuser).create([(u'username/tag1', u'description'),
(u'username/tag2', u'description')])
objectID = uuid4()
values = {objectID: {u'username/tag1': 16,
u'username/tag2': 16}}
SecureTagValueAPI(self.superuser).set(values)
self.permissions.set([(u'username/tag1', Operation.READ_TAG_VALUE,
Policy.CLOSED, []),
(u'username/tag2', Operation.READ_TAG_VALUE,
Policy.OPEN, [])])
self.assertEqual(
sorted([u'username/tag1', u'username/tag2']),
sorted(self.objects.getTagsForObjects([objectID])))
def testGetTagsByObjectsReturnsNoneAllowedTags(self):
"""
L{SecureObjectAPI.getTagsByObjects} will return all the tags for
which the user has C{Operation.READ_TAG_VALUE} permissions, but not
those for which the user doesn't have.
"""
TagAPI(self.user).create([(u'username/tag1', u'description'),
(u'username/tag2', u'description')])
objectID = uuid4()
values = {objectID: {u'username/tag1': 16,
u'username/tag2': 16}}
SecureTagValueAPI(self.user).set(values)
self.permissions.set([(u'username/tag1', Operation.READ_TAG_VALUE,
Policy.CLOSED, []),
(u'username/tag2', Operation.READ_TAG_VALUE,
Policy.CLOSED, [])])
result = self.objects.getTagsByObjects([objectID])
self.assertEqual({}, result)
def testGetForUserWithAdditionalTagsUnreadable(self):
"""
L{SecureCommentAPI.getForUser} raises a L{PermissionDeniedError} if a
L{User} tries to retrieve C{additionalTags} which are unreadable to
them.
"""
objectID = uuid4()
values = {objectID: {u'username/tag': 16}}
SecureTagValueAPI(self.user).set(values)
CachingPermissionAPI(self.user).set([(u'username/tag',
Operation.READ_TAG_VALUE,
Policy.CLOSED, [])])
error = self.assertRaises(PermissionDeniedError,
self.comments.getForUser, u'username',
additionalTags=[u'username/tag'])
self.assertEqual(u'username', error.username)
self.assertEqual(
[(u'username/tag', Operation.READ_TAG_VALUE)],
error.pathsAndOperations)
def testGetTagsByObjectsReturnsNoneIfDenied(self):
"""
L{SecureObjectAPI.getTagsByObjects} will return an empty C{dict} if
the L{User} does not have C{Operation.READ_TAG_VALUE} permission on
none of the L{Tag}s an object has.
"""
self.tags.create([(u'username/tag1', u'description'),
(u'username/tag2', u'description')])
objectID = uuid4()
values = {objectID: {u'username/tag1': 16,
u'username/tag2': 16}}
SecureTagValueAPI(self.user).set(values)
self.permissions.set([(u'username/tag1', Operation.READ_TAG_VALUE,
Policy.CLOSED, []),
(u'username/tag2', Operation.READ_TAG_VALUE,
Policy.CLOSED, [])])
result = self.objects.getTagsByObjects([objectID])
self.assertEqual({}, result)
def testGetObjectWithAboutValue(self):
"""
L{FacadeObjectAPI.getObject} returns a L{TObjectInfo} with the
L{Tag.path}s for which the L{User} has L{Operation.READ_TAG_VALUE},
and the L{AboutTagValue.value} if it has one and C{showAbout} is
C{True}.
"""
objectID = uuid4()
SecureTagAPI(self.user).create([(u'username/foo', u'A description')])
values = {objectID: {u'username/foo': u'bar'}}
SecureTagValueAPI(self.user).set(values)
aboutTag = self.system.tags[u'fluiddb/about']
createAboutTagValue(objectID, u'bar')
createTagValue(self.user.id, aboutTag.id, objectID, u'about value')
self.store.commit()
with login(self.user.username, uuid4(), self.transact) as session:
objectInfo = yield self.facade.getObject(session,
str(objectID),
showAbout=True)
self.assertEqual([u'fluiddb/about', u'username/foo'],
sorted(objectInfo.tagPaths))
self.assertEqual('about value', objectInfo.about)
class CommentImporterTest(FluidinfoTestCase):
resources = [('cache', CacheResource()),
('config', ConfigResource()),
('log', LoggingResource(format='%(message)s')),
('store', DatabaseResource())]
def setUp(self):
super(CommentImporterTest, self).setUp()
self.system = createSystemData()
UserAPI().create([
(u'fluidinfo.com', u'secret', u'User', u'*****@*****.**')])
user = getUser(u'fluidinfo.com')
self.objects = SecureObjectAPI(user)
self.values = SecureTagValueAPI(user)
def testUpload(self):
"""
Comments are uploaded directly to Fluidinfo. After performing an
upload, all the inserted values must be present in Fluidinfo.
"""
when = datetime.utcnow()
floatTime = timegm(when.utctimetuple()) + float(when.strftime('0.%f'))
isoTime = when.isoformat()
client = CommentImporter(100)
client.upload([
{'about': [u'one', u'two'],
'importer': u'fluidinfo.com',
'text': u'Here is my #wonderful comment',
'timestamp': when,
'url': u'http://twitter.com/status/9373973',
'username': u'joe'}
])
about = u'fluidinfo.com joe %s' % isoTime
result = self.objects.get([about])
objectID = result[about]
result = self.values.get([objectID], [u'fluidinfo.com/info/about',
u'fluidinfo.com/info/text',
u'fluidinfo.com/info/timestamp',
u'fluidinfo.com/info/url',
u'fluidinfo.com/info/username',
])
comment = result[objectID]
self.assertEqual([u'one', u'two', u'#wonderful'],
comment[u'fluidinfo.com/info/about'].value)
self.assertEqual(u'Here is my #wonderful comment',
comment[u'fluidinfo.com/info/text'].value)
self.assertEqual(floatTime,
comment[u'fluidinfo.com/info/timestamp'].value)
self.assertEqual(u'http://twitter.com/status/9373973',
comment[u'fluidinfo.com/info/url'].value)
self.assertEqual(u'joe',
comment[u'fluidinfo.com/info/username'].value)
def testUploadWithoutAboutValues(self):
"""
When no explicit about values are in the uploaded comment, there
must be no about values stored in Fluidinfo.
"""
when = datetime.utcnow()
isoTime = when.isoformat()
client = CommentImporter(100)
client.upload([
{'importer': u'fluidinfo.com',
'text': u'Here is my comment',
'timestamp': when,
'url': u'http://twitter.com/status/9373973',
'username': u'joe'}
])
about = u'fluidinfo.com joe %s' % isoTime
result = self.objects.get([about])
objectID = result[about]
result = self.values.get([objectID], [u'fluidinfo.com/info/about'])
comment = result[objectID]
self.assertEqual([], comment[u'fluidinfo.com/info/about'].value)
def testUploadMultipleComments(self):
"""Multiple comments are inserted in batches."""
when = datetime.utcnow()
isoTime = when.isoformat()
client = CommentImporter(100)
client.upload([
{'importer': u'fluidinfo.com',
'text': u'Here is my #wonderful comment',
'timestamp': when,
'url': u'http://twitter.com/status/9373973',
'username': u'joe'},
{'importer': u'fluidinfo.com',
'text': u'A #crazy comment',
'timestamp': when,
'url': u'http://twitter.com/status/9279479379',
'username': u'mike'}
])
about1 = u'fluidinfo.com joe %s' % isoTime
about2 = u'fluidinfo.com mike %s' % isoTime
result = self.objects.get([about1, about2])
objectID1 = result[about1]
objectID2 = result[about2]
comments = self.values.get([objectID1, objectID2],
[u'fluidinfo.com/info/text'])
comment1 = comments[objectID1]
self.assertEqual(u'Here is my #wonderful comment',
comment1[u'fluidinfo.com/info/text'].value)
comment2 = comments[objectID2]
self.assertEqual(u'A #crazy comment',
comment2[u'fluidinfo.com/info/text'].value)
self.assertTrue(self.log.getvalue().startswith(
'Importing 2 new comments.\nImported 2/2 new comments.\n'
'Imported 2 comments in '))
def testUploadUsesBatchSize(self):
"""
Comments are uploaded in batches when possible, depending on the batch
size.
"""
when = datetime.utcnow()
client = CommentImporter(1)
client.upload([
{'importer': u'fluidinfo.com',
'text': u'Here is my #wonderful comment',
'timestamp': when,
'url': u'http://twitter.com/status/9373973',
'username': u'joe'},
{'importer': u'fluidinfo.com',
'text': u'A #crazy comment',
'timestamp': when,
'url': u'http://twitter.com/status/9279479379',
'username': u'mike'}
])
self.assertTrue(self.log.getvalue().startswith(
'Importing 2 new comments.\nImported 1/2 new comments.\n'
'Imported 2/2 new comments.\nImported 2 comments in '))
def testUploadLogsMessage(self):
"""
Uploads must prefix log output with the passed message.
"""
when = datetime.utcnow()
client = CommentImporter(100)
client.upload([
{'importer': u'fluidinfo.com',
'text': u'Here is my #wonderful comment',
'timestamp': when,
'url': u'http://twitter.com/status/9373973',
'username': u'joe'}], 'message-xxx')
self.assertTrue(self.log.getvalue().startswith(
'message-xxx: Importing 1 new comments.\n'
'message-xxx: Imported 1/1 new comments.\n'
'message-xxx: Imported 1 comments in '))
class DatasetImporterTest(FluidinfoTestCase):
resources = [('cache', CacheResource()), ('config', ConfigResource()),
('log', LoggingResource(format='%(message)s')),
('store', DatabaseResource())]
def setUp(self):
super(DatasetImporterTest, self).setUp()
self.system = createSystemData()
UserAPI().create([(u'user', u'secret', u'User', u'*****@*****.**')])
user = getUser(u'user')
self.objects = SecureObjectAPI(user)
self.values = SecureTagValueAPI(user)
def testUpload(self):
"""
Object data is converted into a format compatible with the
L{TagValueAPI.set} method before being uploaded directly into
Fluidinfo.
"""
client = DatasetImporter(100)
client.upload(u'user', [{
'about': u'hello world',
'values': {
u'user/bar': 13
}
}])
result = self.objects.get([u'hello world'])
objectID = result[u'hello world']
result = self.values.get([objectID], [u'user/bar'])
value = result[objectID][u'user/bar']
self.assertEqual(13, value.value)
def testUploadMultipleObjects(self):
"""Multiple objects are inserted in batches."""
client = DatasetImporter(100)
client.upload(u'user', [{
'about': u'hello world',
'values': {
u'user/bar': 13
}
}, {
'about': u'wubble',
'values': {
u'user/quux': 42
}
}])
aboutValues = self.objects.get([u'hello world', u'wubble'])
objectID = aboutValues[u'hello world']
result = self.values.get([objectID], [u'user/bar'])
value = result[objectID][u'user/bar']
self.assertEqual(13, value.value)
objectID = aboutValues[u'wubble']
result = self.values.get([objectID], [u'user/quux'])
value = result[objectID][u'user/quux']
self.assertEqual(42, value.value)
self.assertTrue(self.log.getvalue().startswith(
'Importing 2 new objects.\nImported 2/2 new objects.\n'
'Imported 2 objects in '))
def testUploadUsesBatchSize(self):
"""
Objects are uploaded in batches when possible, depending on the batch
size.
"""
client = DatasetImporter(1)
client.upload(u'user', [{
'about': u'hello world',
'values': {
u'user/bar': 13
}
}, {
'about': u'wubble',
'values': {
u'user/quux': 42
}
}])
self.assertTrue(self.log.getvalue().startswith(
'Importing 2 new objects.\nImported 1/2 new objects.\n'
'Imported 2/2 new objects.\nImported 2 objects in '))
def testUploadWithUncreatablePath(self):
"""
L{DatasetImporter.upload} checks permissions when importing data. An
L{UnknownPathError} is raised if a specified tag doesn't exist and the
L{User} doesn't have permissions to create it.
"""
client = DatasetImporter(100)
self.assertRaises(UnknownPathError, client.upload, u'user', [{
'about': u'hello world',
'values': {
u'foo/bar': 13
}
}])
def testUploadWithPermissionViolation(self):
"""L{DatasetImporter.upload} checks permissions when importing data."""
UserAPI().create([(u'user1', u'pwd', u'User 1', u'*****@*****.**')])
client = DatasetImporter(100)
self.assertRaises(PermissionDeniedError, client.upload, u'user',
[{
'about': u'hello world',
'values': {
u'user1/bar': 13
}
}])
def testUploadWithUnknownUser(self):
"""
L{DatasetImporter.upload} raises an L{UnknownUserError} if the
specified L{User} doesn't exist.
"""
client = DatasetImporter(100)
self.assertRaises(UnknownUserError, client.upload, u'unknown',
[{
'about': u'hello world',
'values': {
u'unknown/bar': 13
}
}])
def testUploadLogsMessage(self):
"""
Uploads must prefix log output with the passed message.
"""
client = DatasetImporter(100)
client.upload(u'user', [{
'about': u'hello world',
'values': {
u'user/bar': 13
}
}], 'message-xxx')
self.assertTrue(self.log.getvalue().startswith(
'message-xxx: Importing 1 new objects.\n'
'message-xxx: Imported 1/1 new objects.\n'
'message-xxx: Imported 1 objects in '))
class CommentImporterTest(FluidinfoTestCase):
resources = [('cache', CacheResource()), ('config', ConfigResource()),
('log', LoggingResource(format='%(message)s')),
('store', DatabaseResource())]
def setUp(self):
super(CommentImporterTest, self).setUp()
self.system = createSystemData()
UserAPI().create([(u'fluidinfo.com', u'secret', u'User',
u'*****@*****.**')])
user = getUser(u'fluidinfo.com')
self.objects = SecureObjectAPI(user)
self.values = SecureTagValueAPI(user)
def testUpload(self):
"""
Comments are uploaded directly to Fluidinfo. After performing an
upload, all the inserted values must be present in Fluidinfo.
"""
when = datetime.utcnow()
floatTime = timegm(when.utctimetuple()) + float(when.strftime('0.%f'))
isoTime = when.isoformat()
client = CommentImporter(100)
client.upload([{
'about': [u'one', u'two'],
'importer': u'fluidinfo.com',
'text': u'Here is my #wonderful comment',
'timestamp': when,
'url': u'http://twitter.com/status/9373973',
'username': u'joe'
}])
about = u'fluidinfo.com joe %s' % isoTime
result = self.objects.get([about])
objectID = result[about]
result = self.values.get([objectID], [
u'fluidinfo.com/info/about',
u'fluidinfo.com/info/text',
u'fluidinfo.com/info/timestamp',
u'fluidinfo.com/info/url',
u'fluidinfo.com/info/username',
])
comment = result[objectID]
self.assertEqual([u'one', u'two', u'#wonderful'],
comment[u'fluidinfo.com/info/about'].value)
self.assertEqual(u'Here is my #wonderful comment',
comment[u'fluidinfo.com/info/text'].value)
self.assertEqual(floatTime,
comment[u'fluidinfo.com/info/timestamp'].value)
self.assertEqual(u'http://twitter.com/status/9373973',
comment[u'fluidinfo.com/info/url'].value)
self.assertEqual(u'joe', comment[u'fluidinfo.com/info/username'].value)
def testUploadWithoutAboutValues(self):
"""
When no explicit about values are in the uploaded comment, there
must be no about values stored in Fluidinfo.
"""
when = datetime.utcnow()
isoTime = when.isoformat()
client = CommentImporter(100)
client.upload([{
'importer': u'fluidinfo.com',
'text': u'Here is my comment',
'timestamp': when,
'url': u'http://twitter.com/status/9373973',
'username': u'joe'
}])
about = u'fluidinfo.com joe %s' % isoTime
result = self.objects.get([about])
objectID = result[about]
result = self.values.get([objectID], [u'fluidinfo.com/info/about'])
comment = result[objectID]
self.assertEqual([], comment[u'fluidinfo.com/info/about'].value)
def testUploadMultipleComments(self):
"""Multiple comments are inserted in batches."""
when = datetime.utcnow()
isoTime = when.isoformat()
client = CommentImporter(100)
client.upload([{
'importer': u'fluidinfo.com',
'text': u'Here is my #wonderful comment',
'timestamp': when,
'url': u'http://twitter.com/status/9373973',
'username': u'joe'
}, {
'importer': u'fluidinfo.com',
'text': u'A #crazy comment',
'timestamp': when,
'url': u'http://twitter.com/status/9279479379',
'username': u'mike'
}])
about1 = u'fluidinfo.com joe %s' % isoTime
about2 = u'fluidinfo.com mike %s' % isoTime
result = self.objects.get([about1, about2])
objectID1 = result[about1]
objectID2 = result[about2]
comments = self.values.get([objectID1, objectID2],
[u'fluidinfo.com/info/text'])
comment1 = comments[objectID1]
self.assertEqual(u'Here is my #wonderful comment',
comment1[u'fluidinfo.com/info/text'].value)
comment2 = comments[objectID2]
self.assertEqual(u'A #crazy comment',
comment2[u'fluidinfo.com/info/text'].value)
self.assertTrue(self.log.getvalue().startswith(
'Importing 2 new comments.\nImported 2/2 new comments.\n'
'Imported 2 comments in '))
def testUploadUsesBatchSize(self):
"""
Comments are uploaded in batches when possible, depending on the batch
size.
"""
when = datetime.utcnow()
client = CommentImporter(1)
client.upload([{
'importer': u'fluidinfo.com',
'text': u'Here is my #wonderful comment',
'timestamp': when,
'url': u'http://twitter.com/status/9373973',
'username': u'joe'
}, {
'importer': u'fluidinfo.com',
'text': u'A #crazy comment',
'timestamp': when,
'url': u'http://twitter.com/status/9279479379',
'username': u'mike'
}])
self.assertTrue(self.log.getvalue().startswith(
'Importing 2 new comments.\nImported 1/2 new comments.\n'
'Imported 2/2 new comments.\nImported 2 comments in '))
def testUploadLogsMessage(self):
"""
Uploads must prefix log output with the passed message.
"""
when = datetime.utcnow()
client = CommentImporter(100)
client.upload([{
'importer': u'fluidinfo.com',
'text': u'Here is my #wonderful comment',
'timestamp': when,
'url': u'http://twitter.com/status/9373973',
'username': u'joe'
}], 'message-xxx')
self.assertTrue(self.log.getvalue().startswith(
'message-xxx: Importing 1 new comments.\n'
'message-xxx: Imported 1/1 new comments.\n'
'message-xxx: Imported 1 comments in '))
class SecureTagValueAPIWithUserRoleTest(FluidinfoTestCase):
resources = [('cache', CacheResource()),
('config', ConfigResource()),
('store', DatabaseResource())]
def setUp(self):
super(SecureTagValueAPIWithUserRoleTest, self).setUp()
createSystemData()
UserAPI().create([(u'username', u'password', u'User',
u'*****@*****.**')])
self.user = getUser(u'username')
self.permissions = CachingPermissionAPI(self.user)
TagAPI(self.user).create([(u'username/tag', u'description')])
self.tagValues = SecureTagValueAPI(self.user)
def testGetIsAllowed(self):
"""
L{SecureTagAPI.get} is allowed if the user has
C{Operation.READ_TAG_VALUE} permissions.
"""
objectID = uuid4()
values = {objectID: {u'username/tag': 16}}
self.tagValues.set(values)
self.permissions.set([(u'username/tag', Operation.READ_TAG_VALUE,
Policy.OPEN, [])])
result = self.tagValues.get(objectIDs=[objectID],
paths=[u'username/tag'])
self.assertEqual(16, result[objectID][u'username/tag'].value)
def testGetIsDenied(self):
"""
L{SecureTagAPI.get} raises L{PermissionDeniedError} if the user doesn't
have C{Operation.READ_TAG_VALUE} permissions.
"""
objectID = uuid4()
values = {objectID: {u'username/tag': 16}}
self.tagValues.set(values)
self.permissions.set([(u'username/tag', Operation.READ_TAG_VALUE,
Policy.CLOSED, [])])
error = self.assertRaises(PermissionDeniedError,
self.tagValues.get, objectIDs=[objectID],
paths=[u'username/tag'])
self.assertEqual([(u'username/tag', Operation.READ_TAG_VALUE)],
error.pathsAndOperations)
def testSetIsAllowed(self):
"""
L{SecureTagAPI.set} is allowed if the user has
C{Operation.WRITE_TAG_VALUE} permissions.
"""
self.permissions.set([(u'username/tag', Operation.WRITE_TAG_VALUE,
Policy.OPEN, [])])
objectID = uuid4()
values = {objectID: {u'username/tag': 16}}
self.tagValues.set(values)
result = self.tagValues.get(objectIDs=[objectID],
paths=[u'username/tag'])
self.assertEqual(16, result[objectID][u'username/tag'].value)
def testSetIsDenied(self):
"""
L{SecureTagAPI.set} raises L{PermissionDeniedError} if the user doesn't
have C{Operation.WRITE_TAG_VALUE} permissions.
"""
self.permissions.set([(u'username/tag', Operation.WRITE_TAG_VALUE,
Policy.CLOSED, [])])
values = {uuid4(): {u'username/tag': 16}}
error = self.assertRaises(PermissionDeniedError,
self.tagValues.set, values)
self.assertEqual([(u'username/tag', Operation.WRITE_TAG_VALUE)],
error.pathsAndOperations)
def testDeleteIsAllowed(self):
"""
L{SecureTagAPI.delete} is allowed if the user has
C{Operation.DELETE_TAG_VALUE} permissions.
"""
self.permissions.set([(u'username/tag', Operation.DELETE_TAG_VALUE,
Policy.OPEN, [])])
objectID = uuid4()
values = {objectID: {u'username/tag': 16}}
self.tagValues.set(values)
self.tagValues.delete([(objectID, u'username/tag')])
result = self.tagValues.get(objectIDs=[objectID],
paths=[u'username/tag'])
self.assertEqual({}, result)
def testDeleteIsDenied(self):
"""
L{SecureTagAPI.delete} raises L{PermissionDeniedError} if the user
doesn't have C{Operation.DELETE_TAG_VALUE} permissions.
"""
objectID = uuid4()
values = {objectID: {u'username/tag': 16}}
self.tagValues.set(values)
self.permissions.set([(u'username/tag', Operation.DELETE_TAG_VALUE,
Policy.CLOSED, [])])
error = self.assertRaises(PermissionDeniedError,
self.tagValues.delete,
[(objectID, u'username/tag')])
self.assertEqual([(u'username/tag', Operation.DELETE_TAG_VALUE)],
error.pathsAndOperations)
class SecureTagValueAPIWithSuperuserRoleTest(FluidinfoTestCase):
resources = [('cache', CacheResource()),
('config', ConfigResource()),
('store', DatabaseResource())]
def setUp(self):
super(SecureTagValueAPIWithSuperuserRoleTest, self).setUp()
system = createSystemData()
self.superuser = system.users[u'fluiddb']
TagAPI(self.superuser).create([(u'fluiddb/tag', u'description')])
self.tagValues = SecureTagValueAPI(self.superuser)
self.permissions = CachingPermissionAPI(self.superuser)
def testGetIsAllowed(self):
"""L{SecureTagValueAPI.get} is always allowed for the superuser."""
objectID = uuid4()
values = {objectID: {u'fluiddb/tag': 16}}
self.tagValues.set(values)
self.permissions.set([(u'fluiddb/tag', Operation.READ_TAG_VALUE,
Policy.CLOSED, [])])
result = self.tagValues.get(objectIDs=[objectID],
paths=[u'fluiddb/tag'])
self.assertEqual(16, result[objectID][u'fluiddb/tag'].value)
def testSetIsAllowed(self):
"""L{SecureTagValueAPI.set} is always allowed for the superuser."""
self.permissions.set([(u'fluiddb/tag', Operation.WRITE_TAG_VALUE,
Policy.CLOSED, [])])
objectID = uuid4()
values = {objectID: {u'fluiddb/tag': 16}}
self.tagValues.set(values)
result = self.tagValues.get(objectIDs=[objectID],
paths=[u'fluiddb/tag'])
self.assertEqual(16, result[objectID][u'fluiddb/tag'].value)
def testDeleteIsAllowed(self):
"""L{SecureTagValueAPI.delete} is always allowed for the superuser."""
self.permissions.set([(u'fluiddb/tag', Operation.DELETE_TAG_VALUE,
Policy.CLOSED, [])])
objectID = uuid4()
values = {objectID: {u'fluiddb/tag': 16}}
self.tagValues.set(values)
self.tagValues.delete([(objectID, u'fluiddb/tag')])
result = self.tagValues.get(objectIDs=[objectID],
paths=[u'fluiddb/tag'])
self.assertEqual({}, result)
class DatasetImporterTest(FluidinfoTestCase):
resources = [('cache', CacheResource()),
('config', ConfigResource()),
('log', LoggingResource(format='%(message)s')),
('store', DatabaseResource())]
def setUp(self):
super(DatasetImporterTest, self).setUp()
self.system = createSystemData()
UserAPI().create([(u'user', u'secret', u'User', u'*****@*****.**')])
user = getUser(u'user')
self.objects = SecureObjectAPI(user)
self.values = SecureTagValueAPI(user)
def testUpload(self):
"""
Object data is converted into a format compatible with the
L{TagValueAPI.set} method before being uploaded directly into
Fluidinfo.
"""
client = DatasetImporter(100)
client.upload(u'user',
[{'about': u'hello world', 'values': {u'user/bar': 13}}])
result = self.objects.get([u'hello world'])
objectID = result[u'hello world']
result = self.values.get([objectID], [u'user/bar'])
value = result[objectID][u'user/bar']
self.assertEqual(13, value.value)
def testUploadMultipleObjects(self):
"""Multiple objects are inserted in batches."""
client = DatasetImporter(100)
client.upload(u'user',
[{'about': u'hello world', 'values': {u'user/bar': 13}},
{'about': u'wubble', 'values': {u'user/quux': 42}}])
aboutValues = self.objects.get([u'hello world', u'wubble'])
objectID = aboutValues[u'hello world']
result = self.values.get([objectID], [u'user/bar'])
value = result[objectID][u'user/bar']
self.assertEqual(13, value.value)
objectID = aboutValues[u'wubble']
result = self.values.get([objectID], [u'user/quux'])
value = result[objectID][u'user/quux']
self.assertEqual(42, value.value)
self.assertTrue(self.log.getvalue().startswith(
'Importing 2 new objects.\nImported 2/2 new objects.\n'
'Imported 2 objects in '))
def testUploadUsesBatchSize(self):
"""
Objects are uploaded in batches when possible, depending on the batch
size.
"""
client = DatasetImporter(1)
client.upload(u'user',
[{'about': u'hello world', 'values': {u'user/bar': 13}},
{'about': u'wubble', 'values': {u'user/quux': 42}}])
self.assertTrue(self.log.getvalue().startswith(
'Importing 2 new objects.\nImported 1/2 new objects.\n'
'Imported 2/2 new objects.\nImported 2 objects in '))
def testUploadWithUncreatablePath(self):
"""
L{DatasetImporter.upload} checks permissions when importing data. An
L{UnknownPathError} is raised if a specified tag doesn't exist and the
L{User} doesn't have permissions to create it.
"""
client = DatasetImporter(100)
self.assertRaises(
UnknownPathError, client.upload, u'user',
[{'about': u'hello world', 'values': {u'foo/bar': 13}}])
def testUploadWithPermissionViolation(self):
"""L{DatasetImporter.upload} checks permissions when importing data."""
UserAPI().create([(u'user1', u'pwd', u'User 1', u'*****@*****.**')])
client = DatasetImporter(100)
self.assertRaises(
PermissionDeniedError, client.upload, u'user',
[{'about': u'hello world', 'values': {u'user1/bar': 13}}])
def testUploadWithUnknownUser(self):
"""
L{DatasetImporter.upload} raises an L{UnknownUserError} if the
specified L{User} doesn't exist.
"""
client = DatasetImporter(100)
self.assertRaises(
UnknownUserError, client.upload, u'unknown',
[{'about': u'hello world', 'values': {u'unknown/bar': 13}}])
def testUploadLogsMessage(self):
"""
Uploads must prefix log output with the passed message.
"""
client = DatasetImporter(100)
client.upload(u'user',
[{'about': u'hello world', 'values': {u'user/bar': 13}}],
'message-xxx')
self.assertTrue(self.log.getvalue().startswith(
'message-xxx: Importing 1 new objects.\n'
'message-xxx: Imported 1/1 new objects.\n'
'message-xxx: Imported 1 objects in '))