def test_missing_line(self):
"""Should not be possible to parse file with lines out of order."""
file_content = """Latitude: 30°02′59″S -30.04982864
Distance: 2.2959 km Bearing: 137.352°
Longitude: 51°14′23″W -51.23976111
Distance: 4.2397 km Bearing: 210.121°
"""
log_file = StringIO(file_content)
with self.assertRaises(InvalidLogLineException):
list(parse_file(log_file))
def test_wrong_log_line_format(self):
"""Should not be possible to parse a line which is not on the expected
format.
"""
file_content = """Latitude: 30°02′59″S -30.04982864
Distance: 2.2959 km Bearing: 137.352°
Longitude: 51°14′23″W -51.23976111
Distance: 4.2397 km Bearing: 210.121°
"""
log_file = StringIO(file_content)
with self.assertRaises(InvalidLogLineException):
list(parse_file(log_file))
def test_parse_correct_file(self):
"""Should be possible to parse the file if it follows the expected
strcture.
"""
file_content = """Latitude: 30°02′59″S -30.04982864
Longitude: 51°12′05″W -51.20150245
Distance: 2.2959 km Bearing: 137.352°
Latitude: 30°04′03″S -30.06761588
Longitude: 51°14′23″W -51.23976111
Distance: 4.2397 km Bearing: 210.121°
"""
log_file = StringIO(file_content)
log_values = list(parse_file(log_file))
self.assertEqual(len(log_values), 2)
self.assertEqual(log_values[0], "-30.04982864\t-51.20150245\t137.352°")
self.assertEqual(log_values[1], "-30.06761588\t-51.23976111\t210.121°")
def test_blank_lines(self):
"""Should be possible to parse file with blank lines, they must be
ignored.
"""
file_content = """
Latitude: 30°02′59″S -30.04982864
Longitude: 51°12′05″W -51.20150245
Distance: 2.2959 km Bearing: 137.352°
"""
log_file = StringIO(file_content)
log_values = list(parse_file(log_file))
self.assertEqual(len(log_values), 1)
self.assertEqual(log_values[0], "-30.04982864\t-51.20150245\t137.352°")
"""Collect log messages and send them to the broker."""
import logging
import os
from collector.parser import parse_file
from collector.services import MessageBrokerService
from collector.settings import MESSAGE_BROKER_ADDRESS, QUEUE_NAME
if __name__ == "__main__":
logs_folder = "{0}/logs".format(os.getcwd())
logging.debug('Reading logs from: {0}'.format(logs_folder))
# instantiate a message broker to send logs to processors
message_broker = MessageBrokerService(MESSAGE_BROKER_ADDRESS, QUEUE_NAME)
# TODO: Think about a way to scale file reading too
for file_name in os.listdir(logs_folder):
logging.info('Reading file {0}'.format(file_name))
with open("logs/{0}".format(file_name), "r", encoding="utf-8") as log_file:
# TODO: Send logs in batches and compact them
for log in parse_file(file_object=log_file):
message_broker.send_message(log)
logging.info('File ended.')
message_broker.close()
logging.info('Finished collecting logs.')