class GetPatchStatusTest(TemporaryDatabaseMixin, FluidinfoTestCase):
resources = [('fs', TemporaryDirectoryResource()),
('python', PythonPackageBuilderResource())]
def setUp(self):
super(GetPatchStatusTest, self).setUp()
self.packageBuilder = self.python.createPackage('sample_schema')
import sample_schema
self.patchPackage = sample_schema
def testGetPatchStatusWithoutSchema(self):
"""
L{getPatchStatus} raises an C{OperationalError} exception if the
database being checked doesn't have a schema in place.
"""
schema = createSchema(self.patchPackage)
self.assertRaises(OperationalError, getPatchStatus, self.store, schema)
def testGetPatchStatus(self):
"""
L{getPatchStatus} returns a L{PatchStatus} with information about
outstanding and unknown patches. If the database is up-to-date and no
unknown patches have been applied to it, both of these values are
empty C{list}s.
"""
schema = createSchema(self.patchPackage)
patchDatabase(self.store, schema)
status = getPatchStatus(self.store, schema)
self.assertEqual([], status.unappliedPatches)
self.assertEqual([], status.unknownPatches)
def testGetPatchStatusWithUnappliedPatches(self):
"""
L{getPatchStatus} returns information about patch versions that need
to be applied to a database to make it up-to-date with the code base.
"""
schema = createSchema(self.patchPackage)
patchDatabase(self.store, schema)
self.packageBuilder.createModule('patch_1', dedent("""\
def apply(store):
store.execute('INSERT INTO person (name) VALUES (\\'Bob\\')')
"""))
status = getPatchStatus(self.store, schema)
self.assertEqual([1], status.unappliedPatches)
self.assertEqual([], status.unknownPatches)
def testGetPatchStatusWithUnknownPatches(self):
"""
L{getPatchStatus} returns information about patch versions that exist
in the database, but not in the code base.
"""
schema = createSchema(self.patchPackage)
patchDatabase(self.store, schema)
self.store.execute('INSERT INTO patch (version) VALUES (1)')
status = getPatchStatus(self.store, schema)
self.assertEqual([], status.unappliedPatches)
self.assertEqual([1], status.unknownPatches)
class TraceLogParserTest(FluidinfoTestCase):
resources = [('fs', TemporaryDirectoryResource())]
def setUp(self):
super(TraceLogParserTest, self).setUp()
self.parser = TraceLogParser()
def testParse(self):
"""
L{TraceLogParser.parse} gets the root endpoint and stores it. A query
string, when present, is automatically removed.
"""
path = sibpath(__file__, 'simple-trace-logs')
filename = os.path.join(path, 'API-9000-20110630-165151-000901')
[trace] = list(self.parser.parse(filename))
self.assertEqual('API-9000-20110630-165151-000901', trace.sessionID)
self.assertTrue(isinstance(trace.duration, timedelta))
self.assertEqual('/objects', trace.endpoint)
def testParseWithURIAndQueryString(self):
"""
L{TraceLogParser.parse} gets the root endpoint and stores it. A query
string, when present, is automatically removed.
"""
path = sibpath(__file__, 'query-string-trace-logs')
filename = os.path.join(path, 'API-9000-20110630-165209-001507')
[trace] = list(self.parser.parse(filename))
self.assertEqual('API-9000-20110630-165209-001507', trace.sessionID)
self.assertTrue(isinstance(trace.duration, timedelta))
self.assertEqual('/values', trace.endpoint)
class LoadTraceLogsTest(FluidinfoTestCase):
resources = [('fs', TemporaryDirectoryResource()),
('store', LogsDatabaseResource())]
def setUp(self):
super(LoadTraceLogsTest, self).setUp()
self.parser = TraceLogParser()
def testLoadWithFiles(self):
"""L{loadTraceLogs} loads trace logs from the specified path."""
path = sibpath(__file__, 'simple-trace-logs')
filename = os.path.join(path, 'API-9000-20110630-165151-000901')
loadTraceLogs(filename, self.store)
trace = self.store.find(TraceLog).one()
self.assertEqual('API-9000-20110630-165151-000901', trace.sessionID)
self.assertTrue(isinstance(trace.duration, timedelta))
self.assertEqual('/objects', trace.endpoint)
class SessionStorageTest(FluidinfoTestCase):
resources = [('fs', TemporaryDirectoryResource())]
def testDump(self):
"""
L{SessionStorage.dump} appends JSON representations of L{Session}
instances to a log file.
"""
path = self.fs.makePath()
session = Session('id')
session.start()
session.stop()
storage = SessionStorage()
storage.dump(session, path)
with open(path, 'r') as stream:
data = stream.read()
self.assertEqual(data, session.dumps() + '\n')
class PatchDatabaseTest(TemporaryDatabaseMixin, FluidinfoTestCase):
resources = [('fs', TemporaryDirectoryResource()),
('python', PythonPackageBuilderResource())]
def setUp(self):
super(PatchDatabaseTest, self).setUp()
self.packageBuilder = self.python.createPackage('sample_schema')
import sample_schema
self.patchPackage = sample_schema
def testPatchDatabaseWithoutSchema(self):
"""L{patchDatabase} creates a schema if one isn't already in place."""
schema = createSchema(self.patchPackage)
self.assertRaises(OperationalError, self.store.execute,
'SELECT * FROM patch')
patchDatabase(self.store, schema)
self.assertEqual([], list(self.store.execute('SELECT * FROM patch')))
def testPatchDatabaseWithoutPatches(self):
"""
L{patchDatabase} is basically a no-op if no patches are available.
"""
schema = createSchema(self.patchPackage)
patchDatabase(self.store, schema)
self.assertEqual([], list(self.store.execute('SELECT * FROM patch')))
patchDatabase(self.store, schema)
self.assertEqual([], list(self.store.execute('SELECT * FROM patch')))
def testPatchDatabase(self):
"""
L{patchDatabase} applies all outstanding patches. The C{patch} table
contains a row for every patch version that has been applied to the
database.
"""
schema = createSchema(self.patchPackage)
patchDatabase(self.store, schema)
self.packageBuilder.createModule('patch_1', dedent("""\
def apply(store):
store.execute('INSERT INTO person (name) VALUES (\\'Bob\\')')
"""))
patchDatabase(self.store, schema)
self.assertEqual([(1,)],
list(self.store.execute('SELECT * FROM patch')))
self.assertEqual([('Bob',)],
list(self.store.execute('SELECT * FROM person')))
def testPatchDatabaseWithoutUnappliedPatches(self):
"""L{patchDatabase} only applies unapplied patches."""
schema = createSchema(self.patchPackage)
patchDatabase(self.store, schema)
self.packageBuilder.createModule('patch_1', dedent("""\
def apply(store):
store.execute('INSERT INTO person (name) VALUES (\\'Bob\\')')
"""))
self.store.execute('INSERT INTO patch (version) VALUES (1)')
patchDatabase(self.store, schema)
self.assertEqual([(1,)],
list(self.store.execute('SELECT * FROM patch')))
self.assertEqual([], list(self.store.execute('SELECT * FROM person')))
class FluidinfoSessionTest(FluidinfoTestCase):
resources = [('config', ConfigResource()),
('fs', TemporaryDirectoryResource()),
('store', DatabaseResource()),
('threadPool', ThreadPoolResource())]
def setUp(self):
super(FluidinfoSessionTest, self).setUp()
self.transact = Transact(self.threadPool)
def testLogin(self):
"""L{AuthenticationPlugin.login} stores the provided credentials."""
objectID = uuid4()
session = FluidinfoSession('id', self.transact)
session.start()
try:
session.auth.login(u'username', objectID)
finally:
session.stop()
self.assertEqual(u'username', session.auth.username)
self.assertEqual(objectID, session.auth.objectID)
def testLoginWithoutStart(self):
"""
L{AuthenticationPlugin.login} raises a C{RuntimeError} if the session
hasn't been started.
"""
session = FluidinfoSession('id', self.transact)
self.assertRaises(RuntimeError, session.auth.login, u'username',
uuid4())
def testUser(self):
"""
The L{AuthenticationPlugin.user} property returns the L{User} instance
for the session.
"""
createSystemData()
[(objectID, username)] = UserAPI().create([
(u'user', u'secret', u'User', u'*****@*****.**')
])
session = FluidinfoSession('id', self.transact)
session.start()
try:
session.auth.login(username, objectID)
finally:
session.stop()
self.assertEqual(u'user', session.auth.user.username)
def testDumpsAndLoads(self):
"""
Data stored by a L{LoggingPlugin} can be dumped to and loaded from
JSON.
"""
objectID = uuid4()
session = FluidinfoSession('id', self.transact)
session.start()
try:
session.auth.login(u'username', objectID)
finally:
session.stop()
data = session.dumps()
loadedSession = FluidinfoSession('another-id', self.transact)
loadedSession.loads(data)
self.assertEqual(session.auth.username, loadedSession.auth.username)
self.assertEqual(session.auth.objectID, loadedSession.auth.objectID)
class SetupLoggingTest(FluidinfoTestCase):
resources = [('fs', TemporaryDirectoryResource())]
def tearDown(self):
log = logging.getLogger()
while log.handlers:
log.removeHandler(log.handlers.pop())
super(SetupLoggingTest, self).tearDown()
def testSetupLoggingWithStream(self):
"""
L{setupLogging} can configure logging to write messages to a stream.
"""
stream = StringIO()
setupLogging(stream=stream)
logging.info('Test log message.')
self.assertIn('Test log message.', stream.getvalue())
def testSetupLoggingWithStreamSupportsUnicode(self):
"""
The log configured by L{setupLogging} handles C{unicode} messages
properly.
"""
stream = StringIO()
setupLogging(stream=stream)
logging.info(u'\N{HIRAGANA LETTER A}')
logging.getLogger().handlers[0].flush()
self.assertIn(u'\N{HIRAGANA LETTER A}'.encode('utf-8'),
stream.getvalue())
def testSetupLoggingUsesRootLogger(self):
"""L{setupLogging} configures the C{logging} module's root logger."""
stream = StringIO()
log = setupLogging(stream=stream)
self.assertIdentical(log, logging.getLogger())
def testSetupLoggingUsesDebugLevel(self):
"""L{setupLogging} uses the C{logging.INFO} log level, by default."""
stream = StringIO()
setupLogging(stream=stream)
self.assertEqual(logging.INFO, logging.getLogger().level)
def testSetupLoggingWithCustomLevel(self):
"""L{setupLogging} sets the log level passed by the caller."""
stream = StringIO()
log = setupLogging(stream=stream, level=logging.CRITICAL)
self.assertEqual(logging.CRITICAL, log.level)
def testSetupLoggingWithFile(self):
"""
L{setupLogging} can configure logging to write messages to a file.
"""
path = self.fs.makePath()
setupLogging(path=path)
logging.info('Log message.')
logging.getLogger().handlers[0].flush()
with open(path, 'r') as log:
self.assertIn('Log message.', log.read())
def testSetupLoggingWithFileSupportsUnicode(self):
"""
The log configured by L{setupLogging} handles C{unicode} messages
properly.
"""
path = self.fs.makePath()
setupLogging(path=path)
logging.info(u'\N{HIRAGANA LETTER A}')
logging.getLogger().handlers[0].flush()
with open(path, 'r') as log:
self.assertIn(u'\N{HIRAGANA LETTER A}'.encode('utf-8'), log.read())
def testSetupLoggingWithFileSupportsLogRotation(self):
"""
L{setupLogging} uses a C{WatchedFileHandler} when a path is used. The
handler automatically reopens the log file if it gets moved, by
logrotate for example.
"""
path = self.fs.makePath()
setupLogging(path=path)
logging.info('Log message 1.')
os.rename(path, '%s.1' % path)
logging.info('Log message 2.')
logging.getLogger().handlers[0].flush()
with open(path, 'r') as log:
self.assertIn('Log message 2.', log.read())
with open('%s.1' % path, 'r') as log:
self.assertIn('Log message 1.', log.read())
class SetupConfigTest(FluidinfoTestCase):
resources = [('fs', TemporaryDirectoryResource())]
def setUp(self):
super(SetupConfigTest, self).setUp()
content = dedent("""\
[service]
temp-path = var/run
trace-path = var/log
max-threads = 2
port = 9000
[store]
main-uri = postgres://fluiddb:fluiddb@localhost/fluiddb-test
[index]
url = http://localhost:8080/solr
shards = localhost:8080/solr
[cache]
host = 127.0.0.1
port = 6379
db = 0
expire-timeout = 3600
[oauth]
access-secret = testaccesssecret
renewal-secret = testrenewalsecrt
renewal-token-duration = 24
[comments]
extract-atnames = true
extract-hashtags = true
extract-plustags = true
extract-urls = true
extract-files = true
file-object = :files:
""")
self.path = self.fs.makePath(content)
def testSetupConfig(self):
"""L{setupConfig} loads data from a configuration file."""
config = setupConfig(self.path)
self.assertEqual('var/run', config.get('service', 'temp-path'))
self.assertEqual('var/log', config.get('service', 'trace-path'))
self.assertEqual('2', config.get('service', 'max-threads'))
self.assertEqual('9000', config.get('service', 'port'))
self.assertEqual('False', config.get('service', 'development'))
self.assertEqual('http://*****:*****@localhost/fluiddb-test',
config.get('store', 'main-uri'))
self.assertEqual('127.0.0.1', config.get('cache', 'host'))
self.assertEqual(6379, config.getint('cache', 'port'))
self.assertEqual(0, config.getint('cache', 'db'))
self.assertEqual(3600, config.getint('cache', 'expire-timeout'))
self.assertEqual('testaccesssecret',
config.get('oauth', 'access-secret'))
self.assertEqual('testrenewalsecrt',
config.get('oauth', 'renewal-secret'))
self.assertEqual('24', config.get('oauth', 'renewal-token-duration'))
self.assertTrue(config.getboolean('comments', 'extract-atnames'))
self.assertTrue(config.getboolean('comments', 'extract-hashtags'))
self.assertTrue(config.getboolean('comments', 'extract-plustags'))
self.assertTrue(config.getboolean('comments', 'extract-urls'))
self.assertTrue(config.getboolean('comments', 'extract-files'))
self.assertEqual(':files:', config.get('comments', 'file-object'))
def testSetupConfigWithCustomPort(self):
"""
L{setupConfig} can override the port defined in the configuration
file.
"""
config = setupConfig(self.path, port=9002)
self.assertEqual('9002', config.get('service', 'port'))
def testSetupConfigWithCustomDevelopmentMode(self):
"""L{setupConfig} can override the development mode flag."""
config = setupConfig(self.path, development=True)
self.assertEqual('True', config.get('service', 'development'))
class SetupOptionsTest(FluidinfoTestCase):
resources = [('fs', TemporaryDirectoryResource())]
def setUp(self):
super(SetupOptionsTest, self).setUp()
content = dedent("""\
[service]
temp-path = var/run
trace-path = var/log
max-threads = 2
port = 9000
allow-anonymous-access = True
[store]
main-uri = postgres://fluiddb:fluiddb@localhost/fluiddb-test
[index]
url = http://localhost:8080/solr
shards = localhost:8080/solr
[cache]
host = 127.0.0.1
port = 6379
db = 0
expire-timeout = 3600
[oauth]
access-secret = testaccesssecret
renewal-secret = testrenewalsecrt
renewal-token-duration = 24
[comments]
extract-atnames = true
extract-hashtags = true
extract-plustags = true
extract-urls = true
extract-files = true
file-object = :files:
""")
self.path = self.fs.makePath(content)
def testSetupOptions(self):
"""
L{setupOptions} loads a configuration using information from an
L{APIServiceOptions} instance.
"""
options = APIServiceOptions()
options.parseOptions(['--config', self.path])
config = setupOptions(options)
self.assertEqual('var/run', config.get('service', 'temp-path'))
self.assertEqual('var/log', config.get('service', 'trace-path'))
self.assertEqual('2', config.get('service', 'max-threads'))
self.assertEqual('9000', config.get('service', 'port'))
self.assertEqual('True', config.get('service',
'allow-anonymous-access'))
self.assertEqual('False', config.get('service', 'development'))
self.assertEqual('postgres://*****:*****@localhost/fluiddb-test',
config.get('store', 'main-uri'))
self.assertEqual('http://*****:*****@localhost/fluidinfo-test',
config.get('store', 'main-uri'))
self.assertEqual('http://localhost:8080/solr',
config.get('index', 'url'))
self.assertEqual('localhost:8080/solr', config.get('index', 'shards'))
self.assertEqual('127.0.0.1', config.get('cache', 'host'))
self.assertEqual(6379, config.getint('cache', 'port'))
self.assertEqual(0, config.getint('cache', 'db'))
self.assertEqual(3600, config.getint('cache', 'expire-timeout'))
self.assertEqual('', config.get('oauth', 'access-secret'))
self.assertEqual('', config.get('oauth', 'renewal-secret'))
self.assertEqual('168', config.get('oauth', 'renewal-token-duration'))
def testSetupOptionsOverridesPort(self):
"""
If an explicit port is defined in the L{APIServiceOptions} instance it
will be used to override the value in the configuration file.
"""
options = APIServiceOptions()
options.parseOptions(['--config', self.path, '--port', '9003'])
config = setupOptions(options)
self.assertEqual('9003', config.get('service', 'port'))
def testSetupOptionsOverridesDevelopmentMode(self):
"""
If the development mode flag is defined in the L{APIServiceOptions}
instance it will be used to set the development mode.
"""
options = APIServiceOptions()
options.parseOptions(['--config', self.path, '--development'])
config = setupOptions(options)
self.assertEqual('True', config.get('service', 'development'))
class LoadLogsTest(FluidinfoTestCase):
resources = [('fs', TemporaryDirectoryResource()),
('store', LogsDatabaseResource())]
def testLoadWithEmptyFile(self):
"""Calling L{loadLogs} with an empty file is essentially a no-op."""
path = self.fs.makePath('')
loadLogs(path, self.store)
self.assertTrue(self.store.find(ErrorLine).is_empty())
self.assertTrue(self.store.find(StatusLine).is_empty())
def testLoadWithStatus(self):
"""
L{loadLogs} stores L{StatusLine} instances loaded from the log file in
the database.
"""
path = sibpath(__file__, 'status-line.log')
loadLogs(path, self.store)
status = self.store.find(StatusLine).one()
self.assertEqual(datetime(2011, 6, 14, 6, 36, 47, 68), status.time)
self.assertEqual(201, status.code)
self.assertEqual('POST', status.method)
self.assertEqual('/objects', status.endpoint)
self.assertEqual('fom/0.9.2', status.agent)
self.assertEqual(131, status.contentLength)
def testLoadWithUnexpectedContent(self):
"""L{loadLogs} skips lines in the log that can't be parsed."""
path = sibpath(__file__, 'unknown-line.log')
loadLogs(path, self.store)
self.assertTrue(self.store.find(ErrorLine).is_empty())
self.assertTrue(self.store.find(StatusLine).is_empty())
def testLoadWithError(self):
"""
L{loadLogs} stores L{ErrorLine}s instances loaded from the log file in
the database.
"""
path = sibpath(__file__, 'error-line.log')
loadLogs(path, self.store)
error = self.store.find(ErrorLine).one()
self.assertEqual(datetime(2011, 6, 14, 10, 33, 33, 312), error.time)
self.assertEqual("Unknown path u'fluidinfo/chrome-ext.xml'.",
error.message)
self.assertIdentical(None, error.exceptionClass)
self.assertIdentical(None, error.traceback)
def testLoadWithTraceback(self):
"""
L{loadLogs} stores the tracebacks loaded with errors in the database.
"""
path = sibpath(__file__, 'error-traceback.log')
loadLogs(path, self.store)
error = self.store.find(ErrorLine).one()
self.assertEqual(datetime(2011, 6, 14, 10, 33, 33, 312), error.time)
self.assertEqual("Unknown path u'fluidinfo/chrome-ext.xml'.",
error.message)
self.assertEqual('UnknownPathError', error.exceptionClass)
with open(path, 'r') as stream:
traceback = stream.read()
traceback = '\n'.join(traceback.split('\n')[1:])
self.assertEqual(traceback.strip(), error.traceback.strip())
def testLoadWithMixedLines(self):
"""L{loadLogs} correctly parses and skips lines in a log."""
path = sibpath(__file__, 'mixed-lines.log')
loadLogs(path, self.store)
error = self.store.find(ErrorLine).one()
self.assertEqual(datetime(2011, 6, 14, 10, 33, 33, 312), error.time)
self.assertEqual("Unknown path u'fluidinfo/chrome-ext.xml'.",
error.message)
with open(path, 'r') as stream:
traceback = stream.read()
traceback = '\n'.join(traceback.split('\n')[2:-3])
self.assertEqual(traceback.strip(), error.traceback.strip())
status = self.store.find(StatusLine).one()
self.assertEqual(datetime(2011, 6, 14, 11, 36, 47, 68), status.time)
self.assertEqual(201, status.code)
self.assertEqual('POST', status.method)
self.assertEqual('/objects', status.endpoint)
self.assertEqual('fom/0.9.2', status.agent)
self.assertEqual(131, status.contentLength)
class LogParserTest(FluidinfoTestCase):
resources = [('fs', TemporaryDirectoryResource())]
def setUp(self):
super(LogParserTest, self).setUp()
self.parser = LogParser()
def testParseWithEmptyFile(self):
"""
Passing an empty file to L{LogParser.parse} is essentially a no-op.
"""
path = self.fs.makePath('')
self.assertEqual([], list(self.parser.parse(path)))
def testParseWithStatus(self):
"""
L{LogParser.parse} parses status lines from the log and yields
L{StatusLine} instances for each one.
"""
path = sibpath(__file__, 'status-line.log')
[status] = list(self.parser.parse(path))
self.assertEqual(datetime(2011, 6, 14, 6, 36, 47, 68), status.time)
self.assertEqual('POST', status.method)
self.assertEqual('/objects', status.endpoint)
self.assertEqual(201, status.code)
self.assertEqual(131, status.contentLength)
self.assertEqual('fom/0.9.2', status.agent)
def testParseWithUnexpectedContent(self):
"""L{LogParser.parse} skips lines in the log that it can't parse."""
path = sibpath(__file__, 'unknown-line.log')
self.assertEqual([], list(self.parser.parse(path)))
def testParseWithError(self):
"""
L{LogParser.parse} parses error lines from the log and yields
L{ErrorLine} instances for each one.
"""
path = sibpath(__file__, 'error-line.log')
[error] = list(self.parser.parse(path))
self.assertEqual(datetime(2011, 6, 14, 10, 33, 33, 312), error.time)
self.assertEqual("Unknown path u'fluidinfo/chrome-ext.xml'.",
error.message)
self.assertIdentical(None, error.exceptionClass)
self.assertIdentical(None, error.traceback)
def testParseWithTraceback(self):
"""
L{LogParser.parse} extracts full tracebacks from the logs when they're
encountered with error lines.
"""
path = sibpath(__file__, 'error-traceback.log')
[error] = list(self.parser.parse(path))
self.assertEqual(datetime(2011, 6, 14, 10, 33, 33, 312), error.time)
self.assertEqual("Unknown path u'fluidinfo/chrome-ext.xml'.",
error.message)
self.assertEqual('UnknownPathError', error.exceptionClass)
with open(path, 'r') as stream:
traceback = stream.read()
traceback = '\n'.join(traceback.split('\n')[1:])
self.assertEqual(traceback.strip(), error.traceback.strip())
def testParseWithMixedLines(self):
"""L{LogParser.parse} correctly parses and skips lines in a log."""
path = sibpath(__file__, 'mixed-lines.log')
[error, status] = list(self.parser.parse(path))
self.assertEqual(datetime(2011, 6, 14, 10, 33, 33, 312), error.time)
self.assertEqual("Unknown path u'fluidinfo/chrome-ext.xml'.",
error.message)
with open(path, 'r') as stream:
traceback = stream.read()
traceback = '\n'.join(traceback.split('\n')[2:-3])
self.assertEqual(traceback.strip(), error.traceback.strip())
self.assertEqual(datetime(2011, 6, 14, 11, 36, 47, 68), status.time)
self.assertEqual(201, status.code)
self.assertEqual('POST', status.method)
self.assertEqual('/objects', status.endpoint)
self.assertEqual('fom/0.9.2', status.agent)
self.assertEqual(131, status.contentLength)