def initialize_handler(configuration, engine):
"""Initialize the handler from configuration."""
metadata = MetaData()
files = {}
parsers = {}
sinks = {}
for filename, patterns in configuration.get_file_to_patterns().items():
files[filename] = LogFile(filename)
parsers[filename] = []
for name, pattern in patterns.items():
parser = Parser(pattern['regex'], pattern['fields'], pattern['datetime-format'])
parsers[filename].append(parser)
(table, model) = get_table(metadata, name, pattern['fields'], pattern['order'])
sinks[parser] = Sink(sessionmaker(bind=engine), table, model)
# SqlAlchemy will auto create tables, but it will not update existing ones.
metadata.create_all(engine)
handler = Handler(files, parsers, sinks)
notifier = Handler.get_notifier(handler)
# Kick off a scan before blocking on file updates.
handler.handle_all()
return notifier
def setUp(self): #pylint: disable=C0103
"""Initialize"""
self.test_file = MagicMock()
self.test_parser = MagicMock()
self.test_sink = MagicMock()
self.files = { '/test': self.test_file }
self.parsers = { '/test': [ self.test_parser ] }
self.sinks = { self.test_parser: self.test_sink }
self.handler = Handler(self.files, self.parsers, self.sinks)
def main():
"""Begin watching logs."""
logging.basicConfig(level=logging.INFO)
if len(sys.argv) != 2:
sys.exit('Usage: %s config.yml' % sys.argv[0])
configuration = Configuration()
configuration.load_yaml(sys.argv[1])
engine = create_engine(configuration.get_database_uri(), pool_recycle=3600)
sanity_check_configuration(configuration.get_file_to_patterns())
notifier = initialize_handler(configuration, engine)
while True:
try:
Handler.handle_events(notifier)
except KeyboardInterrupt:
break
class TestHandler(TestCase):
"""Test the Handler."""
def setUp(self): #pylint: disable=C0103
"""Initialize"""
self.test_file = MagicMock()
self.test_parser = MagicMock()
self.test_sink = MagicMock()
self.files = { '/test': self.test_file }
self.parsers = { '/test': [ self.test_parser ] }
self.sinks = { self.test_parser: self.test_sink }
self.handler = Handler(self.files, self.parsers, self.sinks)
def test_reload(self):
"""Test Handler reloading"""
self.handler.reload('/test')
self.test_file.reload.assert_called_with()
def test_handle(self):
"""Test Handler handle"""
lines = [ 'test' ]
results = [ {'test': 'test'} ]
self.test_file.read.return_value = lines
self.test_parser.parse.return_value = results
self.handler.handle('/test')
self.test_file.read.assert_called_with()
self.test_parser.parse.assert_called_with(lines)
self.test_sink.sink.assert_called_with(results)
def test_handle_all(self):
"""Test Handler handle all"""
self.handler.handle = MagicMock()
self.handler.handle_all()
self.handler.handle.assert_called_with('/test')