def open(*args): """Overload built in open so we could assure sufficiently large buffer Explicit .flush would be needed to assure that changes leave the buffer """ if len(args) == 2: # ~50kB buffer should be sufficient for all tests here. args = args + (50000,) if sys.version_info >= (3,): return fopen(*args, **{'encoding': 'utf-8', 'errors': 'ignore'}) else: return fopen(*args)
def open(*args): """Overload built in open so we could assure sufficiently large buffer Explicit .flush would be needed to assure that changes leave the buffer """ if len(args) == 2: # ~50kB buffer should be sufficient for all tests here. args = args + (50000, ) if sys.version_info >= (3, ): return fopen(*args, **{'encoding': 'utf-8', 'errors': 'ignore'}) else: return fopen(*args)
def testGetFailuresWrongChar(self): # write wrong utf-8 char: fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf') fout = fopen(fname, 'wb') try: # write: for l in ( b'2015-01-14 20:00:58 user \"test\xf1ing\" from \"192.0.2.0\"\n', # wrong utf-8 char b'2015-01-14 20:00:59 user \"\xd1\xe2\xe5\xf2\xe0\" from \"192.0.2.0\"\n', # wrong utf-8 chars b'2015-01-14 20:01:00 user \"testing\" from \"192.0.2.0\"\n' # correct utf-8 chars ): fout.write(l) fout.close() # output = ('192.0.2.0', 3, 1421262060.0) failregex = "^\s*user \"[^\"]*\" from \"<HOST>\"\s*$" # test encoding auto or direct set of encoding: for enc in (None, 'utf-8', 'ascii'): if enc is not None: self.tearDown();self.setUp(); self.filter.setLogEncoding(enc); self.assertNotLogged('Error decoding line'); self.filter.addLogPath(fname) self.filter.addFailRegex(failregex) self.filter.getFailures(fname) _assert_correct_last_attempt(self, self.filter, output) self.assertLogged('Error decoding line'); self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:58 user '); self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:59 user '); finally: _killfile(fout, fname)
def open(*args): """Overload built in open so we could assure sufficiently large buffer Explicit .flush would be needed to assure that changes leave the buffer """ if len(args) == 2: # ~50kB buffer should be sufficient for all tests here. args = args + (50000,) return fopen(*args)
def Import(self, path): """ Import file line by line """ curObject = None with fopen(path, "r") as infile: self.maptextdata = infile.read() formatter = dict(_static_patterns) for var, pattern in _var_patterns.viewitems(): for num in range(4): formatter[var + "{0:d}".format(num)] = pattern self.lines = re.split("\n", self.maptextdata) self.match = list() lineCache = "" for linenum, line in zip(range(len(self.lines)), self.lines): self.curlinenum = linenum lineCache = line if lineCache == "" else lineCache + "\n" + line matched = False tag = re.search("^#?(\S*)", lineCache).group(1) if self.debug: print ("{0:d}".format(linenum) + ":" + lineCache + ":" + _valid_lines[tag] + ":") try: match = re.match(_valid_lines[tag].format(**formatter), lineCache, re.MULTILINE + re.DOTALL) except Exception as e: print tag, lineCache, partial raise (e) if match: self.match.append((linenum, tag, match.groups())) matched = True lineCache = "" elif (not matched) and (tag == "description"): continue else: partial = _valid_lines[tag] for term in formatter: partial = partial.replace("{" + term + "}", formatter[term]) print ("{0:d}".format(linenum) + ":" + lineCache + ":" + partial + ":" + _valid_lines[tag] + ":") raise Exception( "InvalidLine", "Line {0:d} ::{1:s}:: matched no pattern for ::{2:s}::".format(linenum, line, tag) ) import_functions.get(tag).__call__(match, self, curObject) curObject = focus_functions.get(tag).__call__(match, self, curObject) or curObject return tag, match
def testGetFailuresWrongChar(self): # write wrong utf-8 char: fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf') fout = fopen(fname, 'wb') try: # write: for l in ( b'2015-01-14 20:00:58 user \"test\xf1ing\" from \"192.0.2.0\"\n', # wrong utf-8 char b'2015-01-14 20:00:59 user \"\xd1\xe2\xe5\xf2\xe0\" from \"192.0.2.0\"\n', # wrong utf-8 chars b'2015-01-14 20:01:00 user \"testing\" from \"192.0.2.0\"\n' # correct utf-8 chars ): fout.write(l) fout.close() # output = ('192.0.2.0', 3, 1421262060.0) failregex = "^\s*user \"[^\"]*\" from \"<HOST>\"\s*$" # test encoding auto or direct set of encoding: for enc in (None, 'utf-8', 'ascii'): if enc is not None: self.tearDown() self.setUp() self.filter.setLogEncoding(enc) self.assertNotLogged('Error decoding line') self.filter.addLogPath(fname) self.filter.addFailRegex(failregex) self.filter.getFailures(fname) _assert_correct_last_attempt(self, self.filter, output) self.assertLogged('Error decoding line') self.assertLogged( 'Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:58 user ') self.assertLogged( 'Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:59 user ') finally: _killfile(fout, fname)
def testGetFailuresWrongChar(self): # write wrong utf-8 char: fname = tempfile.mktemp(prefix="tmp_fail2ban", suffix="crlf") fout = fopen(fname, "wb") try: # write: for l in ( b'2015-01-14 20:00:58 user "test\xf1ing" from "192.0.2.0"\n', # wrong utf-8 char b'2015-01-14 20:00:59 user "\xd1\xe2\xe5\xf2\xe0" from "192.0.2.0"\n', # wrong utf-8 chars b'2015-01-14 20:01:00 user "testing" from "192.0.2.0"\n', # correct utf-8 chars ): fout.write(l) fout.close() # output = ("192.0.2.0", 3, 1421262060.0) failregex = '^\s*user "[^"]*" from "<HOST>"\s*$' # test encoding auto or direct set of encoding: for enc in (None, "utf-8", "ascii"): if enc is not None: self.tearDown() self.setUp() self.filter.setLogEncoding(enc) self.assertNotLogged("Error decoding line") self.filter.addLogPath(fname) self.filter.addFailRegex(failregex) self.filter.getFailures(fname) _assert_correct_last_attempt(self, self.filter, output) self.assertLogged("Error decoding line") self.assertLogged( "Continuing to process line ignoring invalid characters:", "2015-01-14 20:00:58 user " ) self.assertLogged( "Continuing to process line ignoring invalid characters:", "2015-01-14 20:00:59 user " ) finally: _killfile(fout, fname)
def __init__( self, filename=None, status="unknown", imagefile=None, title=None, domversion=428, description=None, maptextdata=None, header=None, debug=False, ): self.__inits = (not filename, not imagefile, not title, not description, not maptextdata, not header) self.debug = debug self.filename = "newMap.map" if not filename else filename self.status = status self.maptextdata = maptextdata self.header = _defaults.header if not header else header self.periodic = PeriodicNone # these values are required for valid map - even the Null Map self.dom2title = title self.imagefile = imagefile self.Image = None self.domversion = domversion self.description = description self.scenario = None self.maptextcol = None self.emptylines = list() self.comments = list() self.allowedplayer = list() self.specstart = list() # raw data storage self.provinces = list() self.connections = list() self.connection_data = dict() self.connection_types = dict() self.specials = list() if self.__inits == (False, True, True, True, True, True): if self.status == "old": with fopen(self.filename, "r") as filehandle: self.maptextdata = filehandle.read() self.maptextdata_lines = self.maptextdata.split("\n") self.parse() if self.status == "new": # It's a little bit of file safety try: stat = os.stat(filename) except: # Must not be there if status == "new": stat = None if not stat: e = Exception("IOError", "{filename} declared as new, should not exist".format(filename=filename)) raise e if self.status == "new" or self.status == "unknown": # new, unknown - init maptextdata, wait for fill self.maptextdata = "" if self.__inits == (True, True, True, True, False, True): # assign default filename, parse mapfiledata self.maptextdata = maptextdata self.maptextdata_lines = self.maptextdata.split("\n") self.parse() # current text state of the map data self.newmaptextdata = maptextdata
if __name__ == "__main__": print ("Testing MapData.py") TEST_OPEN = False TEST_NEEDS_OPEN = False TEST_OVERWRITING_SAVE = False TEST_NEW_SAVE = False TEST_REMOVE_PROVINCE = False if TEST_OPEN or TEST_NEEDS_OPEN: print ("Testing open") M = open(os.path.expanduser("~/dominions4/maps/hexawyr.map")) if TEST_OVERWRITING_SAVE: print ("Testing overwriting save") with fopen(os.path.expanduser("~/dominions4/maps/hexawyr_copy.map"), "w") as outfile: M.Export(outfile, new=False) if TEST_NEW_SAVE: print ("Testing new save") with fopen(os.path.expanduser("~/dominions4/maps/hexawyr_copy2.map"), "w") as outfile: M.Export(outfile) if TEST_REMOVE_PROVINCE: print ("Testing remove province") M.removeProvince(Province(10, -1)) with fopen(os.path.expanduser("~/dominions4/maps/hexawyr_no10.map"), "w") as outfile: M.Export(outfile, new=False) print ("Testing Import") Tester = new("example.map", "Example Map", "example.tga")