def read(self): """Reads the log file and adds entries that happened during the monitored time frame""" # Store lastReadTime value temporarily (attribute updated next line) lastReadTime = self.lastReadTime # Update the lastReadTime before reading file # so as not to miss any value next time now = datetime.now() # Remove microseconds self.lastReadTime = datetime(now.year, now.month, now.day, now.hour, now.minute, now.second) try: with open(self.logPath, "r") as logFile: lines = logFile.readlines() # iterate over lines in reversed order (most recent first) # of entries in the monitored time frame i = 1 while i <= len(lines) and LogEntry(lines[-i]).time >= lastReadTime: logEntry = LogEntry(lines[-i]) if logEntry.time > lastReadTime: self.add_entry(logEntry) # Entries dated at the same second might have been added elif logEntry.time == lastReadTime: if logEntry not in self.log: self.add_entry(logEntry) i += 1 except: self.stop("ERROR: LogMonitor cannot read the log file")
def test_drop_old_entries(self): """Test the removal of entries older than the monitored period""" print("test_drop_old_entries()") # Create an old entry object in the log oldEntryStr = self.logGenerator.generate_entry(datetime.now() - timedelta(hours=10)) oldEntry = LogEntry(oldEntryStr) # Create a recent entry newEntryString = self.logGenerator.generate_entry(datetime.now()) newEntry = LogEntry(newEntryString) # Add them manually to the LogMonitor self.logMonitor.add_entry(oldEntry) self.logMonitor.add_entry(newEntry) self.assertEqual(len(self.logMonitor.log), 2) self.logMonitor.drop_old_entries() # Only one of the entries should be left self.assertEqual(len(self.logMonitor.log), 1) self.assertEqual(self.logMonitor.hits, 1)
def test_add_entry(self): """Tests adding entries to the logMonitor""" print("test_add_logEntry()") unparsedEntry = self.logGenerator.generate_entry(datetime.now()) logEntry = LogEntry(unparsedEntry) self.logMonitor.add_entry(logEntry) # Check that all the attributes of the entry have been processed self.assertEqual(self.logMonitor.log[0], logEntry) self.assertEqual(self.logMonitor.hits, 1) self.assertEqual(self.logMonitor.size, logEntry.size) self.assertEqual(self.logMonitor.sections, {logEntry.section: 1}) self.assertEqual(self.logMonitor.ips, {logEntry.ip: 1}) self.assertEqual(self.logMonitor.methods, {logEntry.method: 1}) self.assertEqual(self.logMonitor.codes, {logEntry.code: 1}) # Put an entry that is not formatted # to check if it's correctly dropped logEntry = LogEntry("This is not a formatted entry\n") self.logMonitor.add_entry(logEntry) self.assertEqual(len(self.logMonitor.log), 1) self.assertEqual(self.logMonitor.hits, 1)
def test_parse_log_1(self): line = '188.45.108.168 - - [12/Dec/2015:19:44:09 +0100] "GET /images/stories/raith/almhuette_raith.jpg HTTP/1.1" 200 43300 "http://www.almhuette-raith.at/" "Mozilla/5.0 (Linux; Android 4.4.2; de-at; SAMSUNG GT-I9301I Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Version/1.5 Chrome/28.0.1500.94 Mobile Safari/537.36" "-"' entry = LogEntry(line) self.assertEqual(entry.clientIp,'188.45.108.168') self.assertEqual(entry.clientId, '-') self.assertEqual(entry.userName, '-') self.assertEqual(entry.requestLine, 'GET /images/stories/raith/almhuette_raith.jpg HTTP/1.1') self.assertEqual(entry.requestUrl, '/images/stories/raith/almhuette_raith.jpg') self.assertEqual(entry.urlSection, '/images/') self.assertEqual(entry.statusCode, 200) self.assertEqual(entry.sizeBytes, 43300)
def test_parse_log_3(self): line = '2607:f0d0:1002:0051:0000:0000:0000:0004 - - [23/Jan/2016:15:41:52 +0100] "POST /administrator/index.php HTTP/1.1" 200 "-" "-" "-" "-"' entry = LogEntry(line) self.assertEqual(entry.clientIp, '2607:f0d0:1002:0051:0000:0000:0000:0004') self.assertEqual(entry.clientId, '-') self.assertEqual(entry.userName, '-') self.assertEqual(entry.requestLine, 'POST /administrator/index.php HTTP/1.1') self.assertEqual(entry.requestUrl, '/administrator/index.php') self.assertEqual(entry.urlSection, '/administrator/') self.assertEqual(entry.statusCode, 200) self.assertEqual(entry.sizeBytes, 0)
def test_parse_log_2(self): line = 'hmu4.cs.auckland.ac.nz - - [09/Feb/2016:02:50:20 -0500] "GET /docs/GCDOAR/EnergyStar.html HTTP/1.0" 200 6829' entry = LogEntry(line) self.assertEqual(entry.clientIp, 'hmu4.cs.auckland.ac.nz') self.assertEqual(entry.clientId, '-') self.assertEqual(entry.userName, '-') self.assertEqual(entry.requestLine, 'GET /docs/GCDOAR/EnergyStar.html HTTP/1.0') self.assertEqual(entry.requestUrl, '/docs/GCDOAR/EnergyStar.html') self.assertEqual(entry.urlSection, '/docs/') self.assertEqual(entry.statusCode, 200) self.assertEqual(entry.sizeBytes, 6829)
def test_delete_entry(self): """Tests deleting entries from the LogMonitor""" print("test_delete_logEntry()") # Add 2 entries and delete the oldest unparsedEntry = self.logGenerator.generate_entry(datetime.now()) logEntry1 = LogEntry(unparsedEntry) unparsedEntry = self.logGenerator.generate_entry(datetime.now()) logEntry2 = LogEntry(unparsedEntry) self.logMonitor.add_entry(logEntry1) self.logMonitor.add_entry(logEntry2) self.logMonitor.delete_entry() # logEntry1 should be deleted (oldest) # Check that only logEntry2 is left self.assertEqual(self.logMonitor.log[0], logEntry2) self.assertEqual(self.logMonitor.hits, 1) self.assertEqual(self.logMonitor.size, logEntry2.size) self.assertEqual(self.logMonitor.sections, {logEntry2.section: 1}) self.assertEqual(self.logMonitor.ips, {logEntry2.ip: 1}) self.assertEqual(self.logMonitor.methods, {logEntry2.method: 1}) self.assertEqual(self.logMonitor.codes, {logEntry2.code: 1})
def test_createLogFileStoreAndReadBack(self): print("Starting Test") log_file = LogFile("logfile.json") for i in range(1000): entry = LogEntry('test.txt', 'add', None) log_file.addLogEntry(entry) log_file.writeToJSON() log_file = LogFile("logfile2.json") log_file.readFromJSON("logfile.json") log_file.writeToJSON() self.assertTrue(filecmp.cmp('logfile.json', 'logfile2.json')) os.remove('logfile.json') os.remove('logfile2.json')
def test_logEntry(self): """Tests creation of a LogEntry instance from one line of the log""" print("test_logEntry()") now = datetime.now() now = datetime(now.year, now.month, now.day, now.hour, now.minute, now.second) # Truncate current datetime to remove microseconds # (for the test to succeed) entryLine = '127.0.0.1 - - [%s +1000] "GET /icons/blank.gif HTTP/1.1" \ 403 301' % now.strftime("%d/%b/%Y:%H:%M:%S") logEntry = LogEntry(entryLine) self.assertEqual(logEntry.ip, "127.0.0.1") self.assertEqual(logEntry.time, now) self.assertEqual(logEntry.method, "GET") self.assertEqual(logEntry.section, "icons") self.assertEqual(logEntry.size, 301)
def addFileToMVD(self, _file, _version): if os.path.exists(_file) is False: logger.error("The file " + _file + " does not exist and can therefore not be added to version control.") return if _version is None: version = VersionNumber(0, 9, 0) else: version = _version mvd_folder, log_file, trunk_folder = findMVDFolder(_file) if mvd_folder is None: logger.error("No valid .mvd folder found for "+_file + ". Please make sure that the folder has been initialized.") return None # we create a log entry for this operation log_entry = LogEntry(_file, "add") # We add the current file to the trunk. self.addFileToTrunk(_file, trunk_folder, log_entry.get_UUID)
def test_not_formatted_entry(self): """Checks that non formatted input are handled correctly""" print("test_not_formatted_entry()") entryLine = "This is not a log entry!" logEntry = LogEntry(entryLine) self.assertFalse(logEntry.parsed)
def getRandomLogEntry(fileName): line = random.choice(open(fileName).readlines()) entry = LogEntry(line) now = datetime.now(timezone.utc) entry.timeStamp = now.astimezone() return entry
def generateLogEntry(): line = '188.127.233.220 - - [22/Jan/2016:09:35:54 +0100] "GET /almhuette-raith.at.tgz HTTP/1.1" 200 228' entry = LogEntry(line) now = datetime.now(timezone.utc) entry.timeStamp = now.astimezone() return entry
sys.exit(2) logFile = "" for o, a in myopts: if o == '-i': logFile = a if (not os.path.exists(logFile)): print('Input http-access-log file "%s" does not exist' % (logFile)) sys.exit(2) lock = threading.Lock() alarmQueue = deque() analyticsQueue = queue.Queue() thread1 = AlarmThread(ALARM_REQUESTS_PER_S, ALARM_AVERAGE_PERIOD_S, alarmQueue, lock) thread2 = AnalyticsThread(ANALYTICS_UPDATE_S, analyticsQueue, lock) thread1.start() thread2.start() logFile = open(logFile, "r") logLines = follow(logFile) for line in logLines: entry = LogEntry(line) lock.acquire() analyticsQueue.put(entry) alarmQueue.append(entry) lock.release()