def testUploadMultipleObjects(self):
"""Multiple objects are inserted in batches."""
client = DatasetImporter(100)
client.upload(u'user', [{
'about': u'hello world',
'values': {
u'user/bar': 13
}
}, {
'about': u'wubble',
'values': {
u'user/quux': 42
}
}])
aboutValues = self.objects.get([u'hello world', u'wubble'])
objectID = aboutValues[u'hello world']
result = self.values.get([objectID], [u'user/bar'])
value = result[objectID][u'user/bar']
self.assertEqual(13, value.value)
objectID = aboutValues[u'wubble']
result = self.values.get([objectID], [u'user/quux'])
value = result[objectID][u'user/quux']
self.assertEqual(42, value.value)
self.assertTrue(self.log.getvalue().startswith(
'Importing 2 new objects.\nImported 2/2 new objects.\n'
'Imported 2 objects in '))
def testUploadUsesBatchSize(self):
"""
Objects are uploaded in batches when possible, depending on the batch
size.
"""
client = DatasetImporter(1)
client.upload(u'user',
[{'about': u'hello world', 'values': {u'user/bar': 13}},
{'about': u'wubble', 'values': {u'user/quux': 42}}])
self.assertTrue(self.log.getvalue().startswith(
'Importing 2 new objects.\nImported 1/2 new objects.\n'
'Imported 2/2 new objects.\nImported 2 objects in '))
def testUploadLogsMessage(self):
"""
Uploads must prefix log output with the passed message.
"""
client = DatasetImporter(100)
client.upload(u'user',
[{'about': u'hello world', 'values': {u'user/bar': 13}}],
'message-xxx')
self.assertTrue(self.log.getvalue().startswith(
'message-xxx: Importing 1 new objects.\n'
'message-xxx: Imported 1/1 new objects.\n'
'message-xxx: Imported 1 objects in '))
def testUpload(self):
"""
Object data is converted into a format compatible with the
L{TagValueAPI.set} method before being uploaded directly into
Fluidinfo.
"""
client = DatasetImporter(100)
client.upload(u'user',
[{'about': u'hello world', 'values': {u'user/bar': 13}}])
result = self.objects.get([u'hello world'])
objectID = result[u'hello world']
result = self.values.get([objectID], [u'user/bar'])
value = result[objectID][u'user/bar']
self.assertEqual(13, value.value)
def testUploadLogsMessage(self):
"""
Uploads must prefix log output with the passed message.
"""
client = DatasetImporter(100)
client.upload(u'user', [{
'about': u'hello world',
'values': {
u'user/bar': 13
}
}], 'message-xxx')
self.assertTrue(self.log.getvalue().startswith(
'message-xxx: Importing 1 new objects.\n'
'message-xxx: Imported 1/1 new objects.\n'
'message-xxx: Imported 1 objects in '))
def testUpload(self):
"""
Object data is converted into a format compatible with the
L{TagValueAPI.set} method before being uploaded directly into
Fluidinfo.
"""
client = DatasetImporter(100)
client.upload(u'user', [{
'about': u'hello world',
'values': {
u'user/bar': 13
}
}])
result = self.objects.get([u'hello world'])
objectID = result[u'hello world']
result = self.values.get([objectID], [u'user/bar'])
value = result[objectID][u'user/bar']
self.assertEqual(13, value.value)
def run(self, path_list, username=None, database_uri=None,
batch_size=None):
if database_uri is None:
database_uri = 'postgres:///fluidinfo'
if batch_size is None:
batch_size = 100
if username is None:
print >> self.outf, 'You must provide a username.'
sys.exit(1)
setConfig(setupConfig(None))
setupLogging(self.outf)
setupStore(database_uri, 'main')
client = DatasetImporter(batch_size)
for filename in path_list:
with open(filename, 'r') as file:
data = load(file)
client.upload(username, data['objects'])
def testUploadUsesBatchSize(self):
"""
Objects are uploaded in batches when possible, depending on the batch
size.
"""
client = DatasetImporter(1)
client.upload(u'user', [{
'about': u'hello world',
'values': {
u'user/bar': 13
}
}, {
'about': u'wubble',
'values': {
u'user/quux': 42
}
}])
self.assertTrue(self.log.getvalue().startswith(
'Importing 2 new objects.\nImported 1/2 new objects.\n'
'Imported 2/2 new objects.\nImported 2 objects in '))
def testUploadWithPermissionViolation(self):
"""L{DatasetImporter.upload} checks permissions when importing data."""
UserAPI().create([(u'user1', u'pwd', u'User 1', u'*****@*****.**')])
client = DatasetImporter(100)
self.assertRaises(PermissionDeniedError, client.upload, u'user',
[{
'about': u'hello world',
'values': {
u'user1/bar': 13
}
}])
def testUploadMultipleObjects(self):
"""Multiple objects are inserted in batches."""
client = DatasetImporter(100)
client.upload(u'user',
[{'about': u'hello world', 'values': {u'user/bar': 13}},
{'about': u'wubble', 'values': {u'user/quux': 42}}])
aboutValues = self.objects.get([u'hello world', u'wubble'])
objectID = aboutValues[u'hello world']
result = self.values.get([objectID], [u'user/bar'])
value = result[objectID][u'user/bar']
self.assertEqual(13, value.value)
objectID = aboutValues[u'wubble']
result = self.values.get([objectID], [u'user/quux'])
value = result[objectID][u'user/quux']
self.assertEqual(42, value.value)
self.assertTrue(self.log.getvalue().startswith(
'Importing 2 new objects.\nImported 2/2 new objects.\n'
'Imported 2 objects in '))
def testUploadWithUncreatablePath(self):
"""
L{DatasetImporter.upload} checks permissions when importing data. An
L{UnknownPathError} is raised if a specified tag doesn't exist and the
L{User} doesn't have permissions to create it.
"""
client = DatasetImporter(100)
self.assertRaises(UnknownPathError, client.upload, u'user', [{
'about': u'hello world',
'values': {
u'foo/bar': 13
}
}])
def testUploadWithUnknownUser(self):
"""
L{DatasetImporter.upload} raises an L{UnknownUserError} if the
specified L{User} doesn't exist.
"""
client = DatasetImporter(100)
self.assertRaises(UnknownUserError, client.upload, u'unknown',
[{
'about': u'hello world',
'values': {
u'unknown/bar': 13
}
}])
def run(self,
path_list,
username=None,
database_uri=None,
batch_size=None):
if database_uri is None:
database_uri = 'postgres:///fluidinfo'
if batch_size is None:
batch_size = 100
if username is None:
print >> self.outf, 'You must provide a username.'
sys.exit(1)
setConfig(setupConfig(None))
setupLogging(self.outf)
setupStore(database_uri, 'main')
client = DatasetImporter(batch_size)
for filename in path_list:
with open(filename, 'r') as file:
data = load(file)
client.upload(username, data['objects'])