def open(*args):
"""Overload built in open so we could assure sufficiently large buffer
Explicit .flush would be needed to assure that changes leave the buffer
"""
if len(args) == 2:
# ~50kB buffer should be sufficient for all tests here.
args = args + (50000,)
if sys.version_info >= (3,):
return fopen(*args, **{'encoding': 'utf-8', 'errors': 'ignore'})
else:
return fopen(*args)
def open(*args):
"""Overload built in open so we could assure sufficiently large buffer
Explicit .flush would be needed to assure that changes leave the buffer
"""
if len(args) == 2:
# ~50kB buffer should be sufficient for all tests here.
args = args + (50000, )
if sys.version_info >= (3, ):
return fopen(*args, **{'encoding': 'utf-8', 'errors': 'ignore'})
else:
return fopen(*args)
def testGetFailuresWrongChar(self):
# write wrong utf-8 char:
fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf')
fout = fopen(fname, 'wb')
try:
# write:
for l in (
b'2015-01-14 20:00:58 user \"test\xf1ing\" from \"192.0.2.0\"\n', # wrong utf-8 char
b'2015-01-14 20:00:59 user \"\xd1\xe2\xe5\xf2\xe0\" from \"192.0.2.0\"\n', # wrong utf-8 chars
b'2015-01-14 20:01:00 user \"testing\" from \"192.0.2.0\"\n' # correct utf-8 chars
):
fout.write(l)
fout.close()
#
output = ('192.0.2.0', 3, 1421262060.0)
failregex = "^\s*user \"[^\"]*\" from \"<HOST>\"\s*$"
# test encoding auto or direct set of encoding:
for enc in (None, 'utf-8', 'ascii'):
if enc is not None:
self.tearDown();self.setUp();
self.filter.setLogEncoding(enc);
self.assertNotLogged('Error decoding line');
self.filter.addLogPath(fname)
self.filter.addFailRegex(failregex)
self.filter.getFailures(fname)
_assert_correct_last_attempt(self, self.filter, output)
self.assertLogged('Error decoding line');
self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:58 user ');
self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:59 user ');
finally:
_killfile(fout, fname)
def open(*args): """Overload built in open so we could assure sufficiently large buffer Explicit .flush would be needed to assure that changes leave the buffer """ if len(args) == 2: # ~50kB buffer should be sufficient for all tests here. args = args + (50000,) return fopen(*args)
def open(*args): """Overload built in open so we could assure sufficiently large buffer Explicit .flush would be needed to assure that changes leave the buffer """ if len(args) == 2: # ~50kB buffer should be sufficient for all tests here. args = args + (50000,) return fopen(*args)
def Import(self, path):
""" Import file line by line """
curObject = None
with fopen(path, "r") as infile:
self.maptextdata = infile.read()
formatter = dict(_static_patterns)
for var, pattern in _var_patterns.viewitems():
for num in range(4):
formatter[var + "{0:d}".format(num)] = pattern
self.lines = re.split("\n", self.maptextdata)
self.match = list()
lineCache = ""
for linenum, line in zip(range(len(self.lines)), self.lines):
self.curlinenum = linenum
lineCache = line if lineCache == "" else lineCache + "\n" + line
matched = False
tag = re.search("^#?(\S*)", lineCache).group(1)
if self.debug:
print ("{0:d}".format(linenum) + ":" + lineCache + ":" + _valid_lines[tag] + ":")
try:
match = re.match(_valid_lines[tag].format(**formatter), lineCache, re.MULTILINE + re.DOTALL)
except Exception as e:
print tag, lineCache, partial
raise (e)
if match:
self.match.append((linenum, tag, match.groups()))
matched = True
lineCache = ""
elif (not matched) and (tag == "description"):
continue
else:
partial = _valid_lines[tag]
for term in formatter:
partial = partial.replace("{" + term + "}", formatter[term])
print ("{0:d}".format(linenum) + ":" + lineCache + ":" + partial + ":" + _valid_lines[tag] + ":")
raise Exception(
"InvalidLine", "Line {0:d} ::{1:s}:: matched no pattern for ::{2:s}::".format(linenum, line, tag)
)
import_functions.get(tag).__call__(match, self, curObject)
curObject = focus_functions.get(tag).__call__(match, self, curObject) or curObject
return tag, match
def testGetFailuresWrongChar(self):
# write wrong utf-8 char:
fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf')
fout = fopen(fname, 'wb')
try:
# write:
for l in (
b'2015-01-14 20:00:58 user \"test\xf1ing\" from \"192.0.2.0\"\n', # wrong utf-8 char
b'2015-01-14 20:00:59 user \"\xd1\xe2\xe5\xf2\xe0\" from \"192.0.2.0\"\n', # wrong utf-8 chars
b'2015-01-14 20:01:00 user \"testing\" from \"192.0.2.0\"\n' # correct utf-8 chars
):
fout.write(l)
fout.close()
#
output = ('192.0.2.0', 3, 1421262060.0)
failregex = "^\s*user \"[^\"]*\" from \"<HOST>\"\s*$"
# test encoding auto or direct set of encoding:
for enc in (None, 'utf-8', 'ascii'):
if enc is not None:
self.tearDown()
self.setUp()
self.filter.setLogEncoding(enc)
self.assertNotLogged('Error decoding line')
self.filter.addLogPath(fname)
self.filter.addFailRegex(failregex)
self.filter.getFailures(fname)
_assert_correct_last_attempt(self, self.filter, output)
self.assertLogged('Error decoding line')
self.assertLogged(
'Continuing to process line ignoring invalid characters:',
'2015-01-14 20:00:58 user ')
self.assertLogged(
'Continuing to process line ignoring invalid characters:',
'2015-01-14 20:00:59 user ')
finally:
_killfile(fout, fname)
def testGetFailuresWrongChar(self):
# write wrong utf-8 char:
fname = tempfile.mktemp(prefix="tmp_fail2ban", suffix="crlf")
fout = fopen(fname, "wb")
try:
# write:
for l in (
b'2015-01-14 20:00:58 user "test\xf1ing" from "192.0.2.0"\n', # wrong utf-8 char
b'2015-01-14 20:00:59 user "\xd1\xe2\xe5\xf2\xe0" from "192.0.2.0"\n', # wrong utf-8 chars
b'2015-01-14 20:01:00 user "testing" from "192.0.2.0"\n', # correct utf-8 chars
):
fout.write(l)
fout.close()
#
output = ("192.0.2.0", 3, 1421262060.0)
failregex = '^\s*user "[^"]*" from "<HOST>"\s*$'
# test encoding auto or direct set of encoding:
for enc in (None, "utf-8", "ascii"):
if enc is not None:
self.tearDown()
self.setUp()
self.filter.setLogEncoding(enc)
self.assertNotLogged("Error decoding line")
self.filter.addLogPath(fname)
self.filter.addFailRegex(failregex)
self.filter.getFailures(fname)
_assert_correct_last_attempt(self, self.filter, output)
self.assertLogged("Error decoding line")
self.assertLogged(
"Continuing to process line ignoring invalid characters:", "2015-01-14 20:00:58 user "
)
self.assertLogged(
"Continuing to process line ignoring invalid characters:", "2015-01-14 20:00:59 user "
)
finally:
_killfile(fout, fname)
def __init__(
self,
filename=None,
status="unknown",
imagefile=None,
title=None,
domversion=428,
description=None,
maptextdata=None,
header=None,
debug=False,
):
self.__inits = (not filename, not imagefile, not title, not description, not maptextdata, not header)
self.debug = debug
self.filename = "newMap.map" if not filename else filename
self.status = status
self.maptextdata = maptextdata
self.header = _defaults.header if not header else header
self.periodic = PeriodicNone
# these values are required for valid map - even the Null Map
self.dom2title = title
self.imagefile = imagefile
self.Image = None
self.domversion = domversion
self.description = description
self.scenario = None
self.maptextcol = None
self.emptylines = list()
self.comments = list()
self.allowedplayer = list()
self.specstart = list()
# raw data storage
self.provinces = list()
self.connections = list()
self.connection_data = dict()
self.connection_types = dict()
self.specials = list()
if self.__inits == (False, True, True, True, True, True):
if self.status == "old":
with fopen(self.filename, "r") as filehandle:
self.maptextdata = filehandle.read()
self.maptextdata_lines = self.maptextdata.split("\n")
self.parse()
if self.status == "new":
# It's a little bit of file safety
try:
stat = os.stat(filename)
except:
# Must not be there
if status == "new":
stat = None
if not stat:
e = Exception("IOError", "{filename} declared as new, should not exist".format(filename=filename))
raise e
if self.status == "new" or self.status == "unknown":
# new, unknown - init maptextdata, wait for fill
self.maptextdata = ""
if self.__inits == (True, True, True, True, False, True):
# assign default filename, parse mapfiledata
self.maptextdata = maptextdata
self.maptextdata_lines = self.maptextdata.split("\n")
self.parse()
# current text state of the map data
self.newmaptextdata = maptextdata
if __name__ == "__main__":
print ("Testing MapData.py")
TEST_OPEN = False
TEST_NEEDS_OPEN = False
TEST_OVERWRITING_SAVE = False
TEST_NEW_SAVE = False
TEST_REMOVE_PROVINCE = False
if TEST_OPEN or TEST_NEEDS_OPEN:
print ("Testing open")
M = open(os.path.expanduser("~/dominions4/maps/hexawyr.map"))
if TEST_OVERWRITING_SAVE:
print ("Testing overwriting save")
with fopen(os.path.expanduser("~/dominions4/maps/hexawyr_copy.map"), "w") as outfile:
M.Export(outfile, new=False)
if TEST_NEW_SAVE:
print ("Testing new save")
with fopen(os.path.expanduser("~/dominions4/maps/hexawyr_copy2.map"), "w") as outfile:
M.Export(outfile)
if TEST_REMOVE_PROVINCE:
print ("Testing remove province")
M.removeProvince(Province(10, -1))
with fopen(os.path.expanduser("~/dominions4/maps/hexawyr_no10.map"), "w") as outfile:
M.Export(outfile, new=False)
print ("Testing Import")
Tester = new("example.map", "Example Map", "example.tga")