def getDate(self, line): date = None dateMatch = self.matchDate(line) if dateMatch: try: # Try first with 'C' locale date = list(time.strptime(dateMatch.group(), self.getPattern())) except ValueError: # Try to convert date string to 'C' locale conv = self.convertLocale(dateMatch.group()) try: date = list(time.strptime(conv, self.getPattern())) except ValueError, e: # Try to add the current year to the pattern. Should fix # the "Feb 29" issue. conv += " %s" % MyTime.gmtime()[0] pattern = "%s %%Y" % self.getPattern() date = list(time.strptime(conv, pattern)) if date[0] < 2000: # There is probably no year field in the logs date[0] = MyTime.gmtime()[0] # Bug fix for #1241756 # If the date is greater than the current time, we suppose # that the log is not from this year but from the year before if time.mktime(date) > MyTime.time(): date[0] -= 1 elif date[1] == 1 and date[2] == 1: # If it is Jan 1st, it is either really Jan 1st or there # is neither month nor day in the log. date[1] = MyTime.gmtime()[1] date[2] = MyTime.gmtime()[2]
def getDate(self, line): date = None dateMatch = self.matchDate(line) if dateMatch: try: date = list(time.strptime(dateMatch.group(), self.getPattern())) except ValueError: conv = self.convertLocale(dateMatch.group()) try: date = list(time.strptime(conv, self.getPattern())) except ValueError, e: conv += " %s" % MyTime.gmtime()[0] pattern = "%s %%Y" % self.getPattern() date = list(time.strptime(conv, pattern)) if date[0] < 2000: date[0] = MyTime.gmtime()[0] if time.mktime(date) > MyTime.time(): date[0] -= 1 elif date[1] == 1 and date[2] == 1: date[1] = MyTime.gmtime()[1] date[2] = MyTime.gmtime()[2]
def addBannedIP(self, ip): unixTime = MyTime.time() for i in xrange(self.failManager.getMaxRetry()): self.failManager.addFailure(FailTicket(ip, unixTime)) # Perform the banning of the IP now. try: # pragma: no branch - exception is the only way out while True: ticket = self.failManager.toBan() self.jail.putFailTicket(ticket) except FailManagerEmpty: self.failManager.cleanup(MyTime.time()) return ip
def processLineAndAdd(self, line): for element in self.processLine(line): ip = element[0] unixTime = element[1] logSys.debug("Processing line with time:%s and ip:%s" % (unixTime, ip)) if unixTime < MyTime.time() - self.getFindTime(): logSys.debug("Ignore line since time %s < %s - %s" % (unixTime, MyTime.time(), self.getFindTime())) break if self.inIgnoreIPList(ip): logSys.debug("Ignore %s" % ip) continue logSys.debug("Found %s" % ip) self.failManager.addFailure(FailTicket(ip, unixTime, [line]))
def processLineAndAdd(self, line): for element in self.processLine(line): ip = element[0] unixTime = element[1] logSys.debug("Processing line with time:%s and ip:%s" % (unixTime, ip)) if unixTime < MyTime.time() - self.getFindTime(): logSys.debug("Ignore line since time %s < %s - %s" % (unixTime, MyTime.time(), self.getFindTime())) break if self.inIgnoreIPList(ip): logSys.debug("Ignore %s" % ip) continue logSys.debug("Found %s" % ip) self.failManager.addFailure(FailTicket(ip, unixTime))
def run(self): self.setActive(True) while self._isActive(): if not self.getIdle(): # Get file modification for container in self.getLogPath(): filename = container.getFileName() if self.isModified(filename): self.getFailures(filename) self.__modified = True if self.__modified: try: while True: ticket = self.failManager.toBan() self.jail.putFailTicket(ticket) except FailManagerEmpty: self.failManager.cleanup(MyTime.time()) self.dateDetector.sortTemplate() self.__modified = False time.sleep(self.getSleepTime()) else: time.sleep(self.getSleepTime()) logSys.debug((self.jail and self.jail.getName() or "jailless") + " filter terminated") return True
def run(self): self.setActive(True) while self._isActive(): if not self.getIdle(): # We cannot block here because we want to be able to # exit. if self.monitor.event_pending(): self.monitor.handle_events() if self.__modified: try: while True: ticket = self.failManager.toBan() self.jail.putFailTicket(ticket) except FailManagerEmpty: self.failManager.cleanup(MyTime.time()) self.dateDetector.sortTemplate() self.__modified = False time.sleep(self.getSleepTime()) else: time.sleep(self.getSleepTime()) # Cleanup Gamin self.__cleanup() logSys.debug(self.jail.getName() + ": filter terminated") return True
def getDate(self, line): date = None dateMatch = self.matchDate(line) if dateMatch: # extract part of format which represents seconds since epoch date = list(MyTime.localtime(float(dateMatch.group()))) return date
def createBanTicket(ticket): ip = ticket.getIP() #lastTime = ticket.getTime() lastTime = MyTime.time() banTicket = BanTicket(ip, lastTime) banTicket.setAttempt(ticket.getAttempt()) return banTicket
def createBanTicket(ticket): ip = ticket.getIP() # lastTime = ticket.getTime() lastTime = MyTime.time() banTicket = BanTicket(ip, lastTime, ticket.getMatches()) banTicket.setAttempt(ticket.getAttempt()) return banTicket
def getFailures(self, filename): if not self.__openLogFile(filename): logSys.error("Unable to get failures in " + filename) return False self.__setFilePos() lastLine = None for line in self.__crtHandler: if not self._isActive(): break try: line = line.decode('utf-8') except UnicodeDecodeError: pass if not self.dateDetector.matchTime(line): continue lastLine = line for element in self.findFailure(line): ip = element[0] unixTime = element[1] if unixTime < MyTime.time()-self.__findTime: break if self.inIgnoreIPList(ip): logSys.debug("Ignore "+ip) continue logSys.debug("Found "+ip) self.failManager.addFailure(FailTicket(ip, unixTime)) self.__lastPos[filename] = self.__getFilePos() if lastLine: self.__lastDate[filename] = self.dateDetector.getUnixTime(lastLine) self.__closeLogFile() return True
def processLineAndAdd(self, line): """Processes the line for failures and populates failManager """ for element in self.processLine(line)[1]: failregex = element[0] ip = element[1] unixTime = element[2] logSys.debug("Processing line with time:%s and ip:%s" % (unixTime, ip)) if unixTime < MyTime.time() - self.getFindTime(): logSys.debug("Ignore line since time %s < %s - %s" % (unixTime, MyTime.time(), self.getFindTime())) break if self.inIgnoreIPList(ip): logSys.debug("Ignore %s" % ip) continue logSys.debug("Found %s" % ip) ## print "D: Adding a ticket for %s" % ((ip, unixTime, [line]),) self.failManager.addFailure(FailTicket(ip, unixTime, [line]))
def addBannedIP(self, ip): if self.inIgnoreIPList(ip): logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.' % ip) unixTime = MyTime.time() for i in xrange(self.failManager.getMaxRetry()): self.failManager.addFailure(FailTicket(ip, unixTime)) # Perform the banning of the IP now. try: # pragma: no branch - exception is the only way out while True: ticket = self.failManager.toBan() self.jail.putFailTicket(ticket) except FailManagerEmpty: self.failManager.cleanup(MyTime.time()) return ip
def getDate(self, line): date = None dateMatch = self.matchDate(line) if dateMatch: try: date = list(time.strptime(dateMatch.group(), self.getPattern())) except ValueError: conv = self.convertLocale(dateMatch.group()) date = list(time.strptime(conv, self.getPattern())) if date[0] < 2000: date[0] = MyTime.gmtime()[0] if time.mktime(date) > MyTime.time(): date[0] -= 1 return date
def getDate(self, line): date = None dateMatch = self.matchDate(line) if dateMatch: # extract part of format which represents seconds since epoch value = dateMatch.group() seconds_since_epoch = value[2:17] # convert seconds from HEX into local time stamp date = list(MyTime.localtime(int(seconds_since_epoch, 16))) return date
def callback(self, path): self.getFailures(path) try: while True: ticket = self.failManager.toBan() self.jail.putFailTicket(ticket) except FailManagerEmpty: self.failManager.cleanup(MyTime.time()) self.dateDetector.sortTemplate() self.__modified = False
def processLineAndAdd(self, line): for element in self.processLine(line): ip = element[0] unixTime = element[1] if unixTime < MyTime.time() - self.getFindTime(): break if self.inIgnoreIPList(ip): logSys.debug("Ignore %s" % ip) continue logSys.debug("Found %s" % ip) self.failManager.addFailure(FailTicket(ip, unixTime))
def storeAllMenus(tot_days): db.delete(db.GqlQuery("SELECT * FROM MenuDatabase")) successCode = "Updated: \n" for dh in UrlRepo.dhs: #2 = Today and tmrw for i in range(tot_days): d = MyTime.getTheTimeNowPlus(i) menu = MyMenuParser.getMenuFor(dh, d) successCode += MenuStorage.store(dh, menu, d) successCode += UrlRepo.getUrl(dh, d) + "\n" return successCode[:-2] #[:-2] removes the last ", "
def getTime(self, line): try: date = list(time.strptime(line, self.getPattern())) except ValueError: conv = self.convertLocale(line) try: date = list(time.strptime(conv, self.getPattern())) except ValueError: conv += " %s" % MyTime.gmtime()[0] pattern = "%s %%Y" % self.getPattern() date = list(time.strptime(conv, pattern)) if date[0] < 2000: date[0] = MyTime.gmtime()[0] if time.mktime(date) > MyTime.time(): date[0] -= 1 return date
def getDate(self, line): date = None dateMatch = self.matchDate(line) if dateMatch: try: # Try first with 'C' locale date = list(time.strptime(dateMatch.group(), self.getPattern())) except ValueError: # Try to convert date string to 'C' locale conv = self.convertLocale(dateMatch.group()) try: date = list(time.strptime(conv, self.getPattern())) except ValueError, e: # Try to add the current year to the pattern. Should fix # the "Feb 29" issue. conv += " %s" % MyTime.gmtime()[0] pattern = "%s %%Y" % self.getPattern() date = list(time.strptime(conv, pattern)) if date[0] < 2000: # There is probably no year field in the logs date[0] = MyTime.gmtime()[0] # Bug fix for #1241756 # If the date is greater than the current time, we suppose # that the log is not from this year but from the year before if time.mktime(date) > MyTime.time(): logSys.debug( u"Correcting deduced year from %d to %d since %f > %f" % (date[0], date[0]-1, time.mktime(date), MyTime.time())) date[0] -= 1 elif date[1] == 1 and date[2] == 1: # If it is Jan 1st, it is either really Jan 1st or there # is neither month nor day in the log. date[1] = MyTime.gmtime()[1] date[2] = MyTime.gmtime()[2]
def _process_file(self, path): """Process a given file TODO -- RF: this is a common logic and must be shared/provided by FileFilter """ self.getFailures(path) try: while True: ticket = self.failManager.toBan() self.jail.putFailTicket(ticket) except FailManagerEmpty: self.failManager.cleanup(MyTime.time()) self.dateDetector.sortTemplate() self.__modified = False
def getTime(self, line): try: # Try first with 'C' locale date = list(time.strptime(line, self.getPattern())) except ValueError: # Try to convert date string to 'C' locale conv = self.convertLocale(line) try: date = list(time.strptime(conv, self.getPattern())) except ValueError: # Try to add the current year to the pattern. Should fix # the "Feb 29" issue. conv += " %s" % MyTime.gmtime()[0] pattern = "%s %%Y" % self.getPattern() date = list(time.strptime(conv, pattern)) if date[0] < 2000: # There is probably no year field in the logs date[0] = MyTime.gmtime()[0] # Bug fix for #1241756 # If the date is greater than the current time, we suppose # that the log is not from this year but from the year before if time.mktime(date) > MyTime.time(): date[0] -= 1 return date
def processLineAndAdd(self, line): try: # Decode line to UTF-8 l = line.decode('utf-8') except UnicodeDecodeError: l = line for element in self.findFailure(l): ip = element[0] unixTime = element[1] if unixTime < MyTime.time() - self.getFindTime(): break if self.inIgnoreIPList(ip): logSys.debug("Ignore %s" % ip) continue logSys.debug("Found %s" % ip) self.failManager.addFailure(FailTicket(ip, unixTime))
def processLineAndAdd(self, line): try: # Decode line to UTF-8 l = line.decode("utf-8") except UnicodeDecodeError: l = line for element in self.findFailure(l): ip = element[0] unixTime = element[1] if unixTime < MyTime.time() - self.getFindTime(): break if self.inIgnoreIPList(ip): logSys.debug("Ignore %s" % ip) continue logSys.debug("Found %s" % ip) self.failManager.addFailure(FailTicket(ip, unixTime))
def getDate(self, line): date = None dateMatch = self.matchDate(line) if dateMatch: try: # Try first with 'C' locale date = list(time.strptime(dateMatch.group(), self.getPattern())) except ValueError: # Try to convert date string to 'C' locale conv = self.convertLocale(dateMatch.group()) try: date = list(time.strptime(conv, self.getPattern())) except (ValueError, re.error), e: # Try to add the current year to the pattern. Should fix # the "Feb 29" issue. opattern = self.getPattern() # makes sense only if %Y is not in already: if not '%Y' in opattern: pattern = "%s %%Y" % opattern conv += " %s" % MyTime.gmtime()[0] date = list(time.strptime(conv, pattern)) else: # we are helpless here raise ValueError( "Given pattern %r does not match. Original " "exception was %r and Feb 29 workaround could not " "be tested due to already present year mark in the " "pattern" % (opattern, e)) if date[0] < 2000: # There is probably no year field in the logs # NOTE: Possibly makes week/year day incorrect date[0] = MyTime.gmtime()[0] # Bug fix for #1241756 # If the date is greater than the current time, we suppose # that the log is not from this year but from the year before if time.mktime(date) > MyTime.time(): logSys.debug( u"Correcting deduced year from %d to %d since %f > %f" % (date[0], date[0] - 1, time.mktime(date), MyTime.time())) # NOTE: Possibly makes week/year day incorrect date[0] -= 1 elif date[1] == 1 and date[2] == 1: # If it is Jan 1st, it is either really Jan 1st or there # is neither month nor day in the log. # NOTE: Possibly makes week/year day incorrect date[1] = MyTime.gmtime()[1] date[2] = MyTime.gmtime()[2]
def getDate(self, line): date = None dateMatch = self.matchDate(line) if dateMatch: try: # Try first with 'C' locale date = list(time.strptime(dateMatch.group(), self.getPattern())) except ValueError: # Try to convert date string to 'C' locale conv = self.convertLocale(dateMatch.group()) try: date = list(time.strptime(conv, self.getPattern())) except (ValueError, re.error), e: # Try to add the current year to the pattern. Should fix # the "Feb 29" issue. opattern = self.getPattern() # makes sense only if %Y is not in already: if not '%Y' in opattern: pattern = "%s %%Y" % opattern conv += " %s" % MyTime.gmtime()[0] date = list(time.strptime(conv, pattern)) else: # we are helpless here raise ValueError( "Given pattern %r does not match. Original " "exception was %r and Feb 29 workaround could not " "be tested due to already present year mark in the " "pattern" % (opattern, e)) if date[0] < 2000: # There is probably no year field in the logs # NOTE: Possibly makes week/year day incorrect date[0] = MyTime.gmtime()[0] # Bug fix for #1241756 # If the date is greater than the current time, we suppose # that the log is not from this year but from the year before if time.mktime(date) > MyTime.time(): logSys.debug( u"Correcting deduced year from %d to %d since %f > %f" % (date[0], date[0]-1, time.mktime(date), MyTime.time())) # NOTE: Possibly makes week/year day incorrect date[0] -= 1 elif date[1] == 1 and date[2] == 1: # If it is Jan 1st, it is either really Jan 1st or there # is neither month nor day in the log. # NOTE: Possibly makes week/year day incorrect date[1] = MyTime.gmtime()[1] date[2] = MyTime.gmtime()[2]
def run(self): self.setActive(True) while self._isActive(): if not self.getIdle(): for container in self.getLogPath(): if self.isModified(container.getFileName()): self.getFailures(container.getFileName()) self.__modified = True if self.__modified: try: while True: ticket = self.failManager.toBan() self.jail.putFailTicket(ticket) except FailManagerEmpty: self.failManager.cleanup(MyTime.time()) self.__modified = False time.sleep(self.getSleepTime()) else: time.sleep(self.getSleepTime()) logSys.debug(self.jail.getName() + ": filter terminated") return True
def run(self): self.setActive(True) while self._isActive(): if not self.getIdle(): if self.monitor.event_pending(): self.monitor.handle_events() if self.__modified: try: while True: ticket = self.failManager.toBan() self.jail.putFailTicket(ticket) except FailManagerEmpty: self.failManager.cleanup(MyTime.time()) self.__modified = False time.sleep(self.getSleepTime()) else: time.sleep(self.getSleepTime()) self.__cleanup() logSys.debug(self.jail.getName() + ": filter terminated") return True
def main(): for dh in UrlRepo.dhs: d = MyTime.getTheTimeNowPlus(8) menu = MyMenuParser.getMenuFor(dh, d) MenuStorage.store(dh, menu, d)
def getMenu(self): dh = self.request.get("dh") self.response.headers["Content-Type"] = "application/json" if self.request.get('time') == "true": self.response.out.write(datetime.now()) self.response.out.write("\n") self.response.out.write(MyTime.getTheTimeNow()) self.response.out.write("\n") #"Hack" to allow first-time storage of menus, #where necessary url-command is: slugmenu.appspot.com/getmenu.py?exe=storeAllMenus[&num_to_store=#] if self.request.get('exe') == "storeAllMenus": num_dh = 8; if self.request.get('num_to_store') != "": num_dh = self.request.get('num_to_store') self.response.out.write( MenuStorage.storeAllMenus(num_dh) ) return if dh == "": self.response.out.write( json.dumps( {"request":{"success":0}, "response":{"message":"Error! Null Dining Hall!"} } ) ) return if dh not in UrlRepo.dhs: self.response.out.write( json.dumps( {"request":{"success":0}, "response":{"message":"Invalid Dining Hall: "+dh} } ) ) return #For testing! if self.request.get('debug') == "url": self.response.out.write("#URL") self.response.out.write("\n") self.response.out.write(UrlRepo.getUrl(dh, MyTime.getTheTimeNow())) self.response.out.write("\n") if self.request.get('debug') == "simple": self.response.out.write("#MENU") self.response.out.write("\n") self.response.out.write(UrlRepo.getUrl(dh, MyTime.getTheTimeNow())) self.response.out.write("\n") self.response.out.write( json.dumps( MyMenuParser.getMenuFor(dh, MyTime.getTheTimeNow()) , indent = 4, sort_keys = True )) self.response.out.write("\n") if self.request.get('debug') == "verbose": self.response.out.write("#HTML") self.response.out.write("\n") html = MyMenuParser.getHtmlFrom( UrlRepo.getUrl(dh, MyTime.getTheTimeNow()) ) self.response.out.write(html) self.response.out.write("\n") dtdate = 0 if self.request.get('dtdate') != '': dtdate = int(self.request.get('dtdate')) if dtdate > 7: self.response.out.write( json.dumps( {"request":{"success":0}, "response":{"message":"Cannot get more than 1 week ahead!"} } ) ) return q = db.GqlQuery( "SELECT * FROM MenuDatabase " + "WHERE dh=:1 AND time=:2", dh, MyTime.getTheTimeNowPlus(dtdate)) json_str = '' for i in q: json_str += i.menu try: self.response.out.write( json.dumps( json.loads(json_str), indent=4, sort_keys=True ) ) except ValueError as ve: self.response.out.write( json.dumps( {"request":{"success":0}, "response":{"message":ve.args[0]} } ) )
def __checkUnBan(self): for ticket in self.__banManager.unBanList(MyTime.time()): self.__unBan(ticket)