Example #1
0
    def read(self):
        """Reads the log file and adds entries
        that happened during the monitored time frame"""
        # Store lastReadTime value temporarily (attribute updated next line)
        lastReadTime = self.lastReadTime
        # Update the lastReadTime before reading file
        # so as not to miss any value next time
        now = datetime.now()
        # Remove microseconds
        self.lastReadTime = datetime(now.year, now.month, now.day, now.hour,
                                     now.minute, now.second)
        try:
            with open(self.logPath, "r") as logFile:
                lines = logFile.readlines()

            # iterate over lines in reversed order (most recent first)
            # of entries in the monitored time frame
            i = 1
            while i <= len(lines) and LogEntry(lines[-i]).time >= lastReadTime:
                logEntry = LogEntry(lines[-i])
                if logEntry.time > lastReadTime:
                    self.add_entry(logEntry)

                # Entries dated at the same second might have been added
                elif logEntry.time == lastReadTime:
                    if logEntry not in self.log:
                        self.add_entry(logEntry)
                i += 1
        except:
            self.stop("ERROR: LogMonitor cannot read the log file")
Example #2
0
 def test_drop_old_entries(self):
     """Test the removal of entries older than the monitored period"""
     print("test_drop_old_entries()")
     # Create an old entry object in the log
     oldEntryStr = self.logGenerator.generate_entry(datetime.now() -
                                                    timedelta(hours=10))
     oldEntry = LogEntry(oldEntryStr)
     # Create a recent entry
     newEntryString = self.logGenerator.generate_entry(datetime.now())
     newEntry = LogEntry(newEntryString)
     # Add them manually to the LogMonitor
     self.logMonitor.add_entry(oldEntry)
     self.logMonitor.add_entry(newEntry)
     self.assertEqual(len(self.logMonitor.log), 2)
     self.logMonitor.drop_old_entries()
     # Only one of the entries should be left
     self.assertEqual(len(self.logMonitor.log), 1)
     self.assertEqual(self.logMonitor.hits, 1)
Example #3
0
 def test_add_entry(self):
     """Tests adding entries to the logMonitor"""
     print("test_add_logEntry()")
     unparsedEntry = self.logGenerator.generate_entry(datetime.now())
     logEntry = LogEntry(unparsedEntry)
     self.logMonitor.add_entry(logEntry)
     # Check that all the attributes of the entry have been processed
     self.assertEqual(self.logMonitor.log[0], logEntry)
     self.assertEqual(self.logMonitor.hits, 1)
     self.assertEqual(self.logMonitor.size, logEntry.size)
     self.assertEqual(self.logMonitor.sections, {logEntry.section: 1})
     self.assertEqual(self.logMonitor.ips, {logEntry.ip: 1})
     self.assertEqual(self.logMonitor.methods, {logEntry.method: 1})
     self.assertEqual(self.logMonitor.codes, {logEntry.code: 1})
     # Put an entry that is not formatted
     # to check if it's correctly dropped
     logEntry = LogEntry("This is not a formatted entry\n")
     self.logMonitor.add_entry(logEntry)
     self.assertEqual(len(self.logMonitor.log), 1)
     self.assertEqual(self.logMonitor.hits, 1)
Example #4
0
 def test_parse_log_1(self):
     line = '188.45.108.168 - - [12/Dec/2015:19:44:09 +0100] "GET /images/stories/raith/almhuette_raith.jpg HTTP/1.1" 200 43300 "http://www.almhuette-raith.at/" "Mozilla/5.0 (Linux; Android 4.4.2; de-at; SAMSUNG GT-I9301I Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Version/1.5 Chrome/28.0.1500.94 Mobile Safari/537.36" "-"'
     entry = LogEntry(line)
     self.assertEqual(entry.clientIp,'188.45.108.168')
     self.assertEqual(entry.clientId, '-')
     self.assertEqual(entry.userName, '-')
     self.assertEqual(entry.requestLine, 'GET /images/stories/raith/almhuette_raith.jpg HTTP/1.1')
     self.assertEqual(entry.requestUrl, '/images/stories/raith/almhuette_raith.jpg')
     self.assertEqual(entry.urlSection, '/images/')
     self.assertEqual(entry.statusCode, 200)
     self.assertEqual(entry.sizeBytes, 43300)
Example #5
0
 def test_parse_log_3(self):
     line = '2607:f0d0:1002:0051:0000:0000:0000:0004 - - [23/Jan/2016:15:41:52 +0100] "POST /administrator/index.php HTTP/1.1" 200 "-" "-" "-" "-"'            
     entry = LogEntry(line)
     self.assertEqual(entry.clientIp, '2607:f0d0:1002:0051:0000:0000:0000:0004')
     self.assertEqual(entry.clientId, '-')
     self.assertEqual(entry.userName, '-')
     self.assertEqual(entry.requestLine, 'POST /administrator/index.php HTTP/1.1')
     self.assertEqual(entry.requestUrl, '/administrator/index.php')
     self.assertEqual(entry.urlSection, '/administrator/')
     self.assertEqual(entry.statusCode, 200)
     self.assertEqual(entry.sizeBytes, 0)
Example #6
0
 def test_parse_log_2(self):
     line = 'hmu4.cs.auckland.ac.nz - - [09/Feb/2016:02:50:20 -0500] "GET /docs/GCDOAR/EnergyStar.html HTTP/1.0" 200 6829'
     entry = LogEntry(line)
     self.assertEqual(entry.clientIp, 'hmu4.cs.auckland.ac.nz')
     self.assertEqual(entry.clientId, '-')
     self.assertEqual(entry.userName, '-')
     self.assertEqual(entry.requestLine, 'GET /docs/GCDOAR/EnergyStar.html HTTP/1.0')
     self.assertEqual(entry.requestUrl, '/docs/GCDOAR/EnergyStar.html')
     self.assertEqual(entry.urlSection, '/docs/')
     self.assertEqual(entry.statusCode, 200)
     self.assertEqual(entry.sizeBytes, 6829)
Example #7
0
    def test_delete_entry(self):
        """Tests deleting entries from the LogMonitor"""
        print("test_delete_logEntry()")
        # Add 2 entries and delete the oldest
        unparsedEntry = self.logGenerator.generate_entry(datetime.now())
        logEntry1 = LogEntry(unparsedEntry)
        unparsedEntry = self.logGenerator.generate_entry(datetime.now())
        logEntry2 = LogEntry(unparsedEntry)

        self.logMonitor.add_entry(logEntry1)
        self.logMonitor.add_entry(logEntry2)
        self.logMonitor.delete_entry()  # logEntry1 should be deleted (oldest)

        # Check that only logEntry2 is left
        self.assertEqual(self.logMonitor.log[0], logEntry2)
        self.assertEqual(self.logMonitor.hits, 1)
        self.assertEqual(self.logMonitor.size, logEntry2.size)
        self.assertEqual(self.logMonitor.sections, {logEntry2.section: 1})
        self.assertEqual(self.logMonitor.ips, {logEntry2.ip: 1})
        self.assertEqual(self.logMonitor.methods, {logEntry2.method: 1})
        self.assertEqual(self.logMonitor.codes, {logEntry2.code: 1})
Example #8
0
    def test_createLogFileStoreAndReadBack(self):
        print("Starting Test")
        log_file = LogFile("logfile.json")
        for i in range(1000):
            entry = LogEntry('test.txt', 'add', None)
            log_file.addLogEntry(entry)
        log_file.writeToJSON()

        log_file = LogFile("logfile2.json")
        log_file.readFromJSON("logfile.json")
        log_file.writeToJSON()
        self.assertTrue(filecmp.cmp('logfile.json', 'logfile2.json'))
        os.remove('logfile.json')
        os.remove('logfile2.json')
Example #9
0
    def test_logEntry(self):
        """Tests creation of a LogEntry instance from one line of the log"""
        print("test_logEntry()")
        now = datetime.now()
        now = datetime(now.year, now.month, now.day, now.hour, now.minute,
                       now.second)
        # Truncate current datetime to remove microseconds
        # (for the test to succeed)
        entryLine = '127.0.0.1 - - [%s +1000] "GET /icons/blank.gif HTTP/1.1" \
403 301' % now.strftime("%d/%b/%Y:%H:%M:%S")
        logEntry = LogEntry(entryLine)
        self.assertEqual(logEntry.ip, "127.0.0.1")
        self.assertEqual(logEntry.time, now)
        self.assertEqual(logEntry.method, "GET")
        self.assertEqual(logEntry.section, "icons")
        self.assertEqual(logEntry.size, 301)
Example #10
0
    def addFileToMVD(self, _file, _version):
        if os.path.exists(_file) is False:
            logger.error("The file " + _file +
                         " does not exist and can therefore not be added to version control.")
            return

        if _version is None:
            version = VersionNumber(0, 9, 0)
        else:
            version = _version

        mvd_folder, log_file, trunk_folder = findMVDFolder(_file)
        if mvd_folder is None:
            logger.error("No valid .mvd folder found for "+_file +
                         ". Please make sure that the folder has been initialized.")
            return None

        # we create a log entry for this operation
        log_entry = LogEntry(_file, "add")

        # We add the current file to the trunk.
        self.addFileToTrunk(_file, trunk_folder, log_entry.get_UUID)
Example #11
0
 def test_not_formatted_entry(self):
     """Checks that non formatted input are handled correctly"""
     print("test_not_formatted_entry()")
     entryLine = "This is not a log entry!"
     logEntry = LogEntry(entryLine)
     self.assertFalse(logEntry.parsed)
Example #12
0
def getRandomLogEntry(fileName):
    line = random.choice(open(fileName).readlines())
    entry = LogEntry(line)
    now = datetime.now(timezone.utc)
    entry.timeStamp = now.astimezone()
    return entry
Example #13
0
def generateLogEntry():
    line = '188.127.233.220 - - [22/Jan/2016:09:35:54 +0100] "GET /almhuette-raith.at.tgz HTTP/1.1" 200 228'
    entry = LogEntry(line)
    now = datetime.now(timezone.utc)
    entry.timeStamp = now.astimezone()
    return entry
Example #14
0
        sys.exit(2)

    logFile = ""
    for o, a in myopts:
        if o == '-i':
            logFile = a

    if (not os.path.exists(logFile)):
        print('Input http-access-log file "%s" does not exist' % (logFile))
        sys.exit(2)

    lock = threading.Lock()
    alarmQueue = deque()
    analyticsQueue = queue.Queue()

    thread1 = AlarmThread(ALARM_REQUESTS_PER_S, ALARM_AVERAGE_PERIOD_S,
                          alarmQueue, lock)
    thread2 = AnalyticsThread(ANALYTICS_UPDATE_S, analyticsQueue, lock)

    thread1.start()
    thread2.start()

    logFile = open(logFile, "r")
    logLines = follow(logFile)
    for line in logLines:
        entry = LogEntry(line)
        lock.acquire()
        analyticsQueue.put(entry)
        alarmQueue.append(entry)
        lock.release()
Example #15
0
def getRandomLogEntry(fileName):
    line = random.choice(open(fileName).readlines())
    entry = LogEntry(line)
    now = datetime.now(timezone.utc)
    entry.timeStamp = now.astimezone()
    return entry
Example #16
0
def generateLogEntry():
    line = '188.127.233.220 - - [22/Jan/2016:09:35:54 +0100] "GET /almhuette-raith.at.tgz HTTP/1.1" 200 228'
    entry = LogEntry(line)
    now = datetime.now(timezone.utc)
    entry.timeStamp = now.astimezone()
    return entry