def read_files(inp: FileInput) -> Seq[FileRecord]:
with inp:
f = n = None
acc = []
for line in inp:
if inp.isfirstline():
debug(1, f"Loading file {inp.filename()}")
if acc:
yield (FileLine(f, n), acc)
acc = []
f, n = inp.filename(), inp.filelineno()
i = line.find('#')
if i >= 0: line = line[:i]
line = line.rstrip()
if not line: continue
if line[0].isspace():
if acc:
acc.append(line)
else:
parse_error((
inp.filename(), inp.filelineno()
), 'Non-continuation (first in file) line starts with whitespace'
)
else:
if acc:
yield (FileLine(f, n), acc)
acc = [line]
n = inp.filelineno()
if acc:
yield (FileLine(f, n), acc)
def test_files_that_dont_end_with_newline(self):
t1 = self.writeTmp("A\nB\nC")
t2 = self.writeTmp("D\nE\nF")
fi = FileInput(files=(t1, t2))
lines = list(fi)
self.assertEqual(lines, ["A\n", "B\n", "C", "D\n", "E\n", "F"])
self.assertEqual(fi.filelineno(), 3)
self.assertEqual(fi.lineno(), 6)
def test_files_that_dont_end_with_newline(self):
t1 = self.writeTmp("A\nB\nC")
t2 = self.writeTmp("D\nE\nF")
fi = FileInput(files=(t1, t2))
lines = list(fi)
self.assertEqual(lines, ["A\n", "B\n", "C", "D\n", "E\n", "F"])
self.assertEqual(fi.filelineno(), 3)
self.assertEqual(fi.lineno(), 6)
def test_zero_byte_files(self):
t1 = self.writeTmp("")
t2 = self.writeTmp("")
t3 = self.writeTmp("The only line there is.\n")
t4 = self.writeTmp("")
fi = FileInput(files=(t1, t2, t3, t4))
line = fi.readline()
self.assertEqual(line, 'The only line there is.\n')
self.assertEqual(fi.lineno(), 1)
self.assertEqual(fi.filelineno(), 1)
self.assertEqual(fi.filename(), t3)
line = fi.readline()
self.assertFalse(line)
self.assertEqual(fi.lineno(), 1)
self.assertEqual(fi.filelineno(), 0)
self.assertEqual(fi.filename(), t4)
fi.close()
def test_zero_byte_files(self):
t1 = self.writeTmp("")
t2 = self.writeTmp("")
t3 = self.writeTmp("The only line there is.\n")
t4 = self.writeTmp("")
fi = FileInput(files=(t1, t2, t3, t4))
line = fi.readline()
self.assertEqual(line, 'The only line there is.\n')
self.assertEqual(fi.lineno(), 1)
self.assertEqual(fi.filelineno(), 1)
self.assertEqual(fi.filename(), t3)
line = fi.readline()
self.assertFalse(line)
self.assertEqual(fi.lineno(), 1)
self.assertEqual(fi.filelineno(), 0)
self.assertEqual(fi.filename(), t4)
fi.close()
def test_files_that_dont_end_with_newline(self):
try:
t1 = writeTmp(1, ["A\nB\nC"])
t2 = writeTmp(2, ["D\nE\nF"])
fi = FileInput(files=(t1, t2))
lines = list(fi)
self.assertEqual(lines, ["A\n", "B\n", "C", "D\n", "E\n", "F"])
self.assertEqual(fi.filelineno(), 3)
self.assertEqual(fi.lineno(), 6)
finally:
remove_tempfiles(t1, t2)
def test_files_that_dont_end_with_newline(self):
try:
t1 = writeTmp(1, ["A\nB\nC"])
t2 = writeTmp(2, ["D\nE\nF"])
fi = FileInput(files=(t1, t2))
lines = list(fi)
self.assertEqual(lines, ["A\n", "B\n", "C", "D\n", "E\n", "F"])
self.assertEqual(fi.filelineno(), 3)
self.assertEqual(fi.lineno(), 6)
finally:
remove_tempfiles(t1, t2)
def test_zero_byte_files(self):
t1 = t2 = t3 = t4 = None
try:
t1 = writeTmp(1, [''])
t2 = writeTmp(2, [''])
t3 = writeTmp(3, ['The only line there is.\n'])
t4 = writeTmp(4, [''])
fi = FileInput(files=(t1, t2, t3, t4))
line = fi.readline()
self.assertEqual(line, 'The only line there is.\n')
self.assertEqual(fi.lineno(), 1)
self.assertEqual(fi.filelineno(), 1)
self.assertEqual(fi.filename(), t3)
line = fi.readline()
self.assertFalse(line)
self.assertEqual(fi.lineno(), 1)
self.assertEqual(fi.filelineno(), 0)
self.assertEqual(fi.filename(), t4)
fi.close()
finally:
remove_tempfiles(t1, t2, t3, t4)
def test_files_that_dont_end_with_newline(self):
t1 = t2 = None
try:
t1 = writeTmp(1, ['A\nB\nC'])
t2 = writeTmp(2, ['D\nE\nF'])
fi = FileInput(files=(t1, t2))
lines = list(fi)
self.assertEqual(lines, ['A\n', 'B\n', 'C', 'D\n', 'E\n', 'F'])
self.assertEqual(fi.filelineno(), 3)
self.assertEqual(fi.lineno(), 6)
finally:
remove_tempfiles(t1, t2)
def test_zero_byte_files(self):
try:
t1 = writeTmp(1, [""])
t2 = writeTmp(2, [""])
t3 = writeTmp(3, ["The only line there is.\n"])
t4 = writeTmp(4, [""])
fi = FileInput(files=(t1, t2, t3, t4))
line = fi.readline()
self.assertEqual(line, 'The only line there is.\n')
self.assertEqual(fi.lineno(), 1)
self.assertEqual(fi.filelineno(), 1)
self.assertEqual(fi.filename(), t3)
line = fi.readline()
self.failIf(line)
self.assertEqual(fi.lineno(), 1)
self.assertEqual(fi.filelineno(), 0)
self.assertEqual(fi.filename(), t4)
fi.close()
finally:
remove_tempfiles(t1, t2, t3, t4)
finally:
remove_tempfiles(t1, t2, t3, t4)
# Next, check for proper behavior with 0-byte files.
if verbose:
print "13. 0-byte files"
try:
t1 = writeTmp(1, [""])
t2 = writeTmp(2, [""])
t3 = writeTmp(3, ["The only line there is.\n"])
t4 = writeTmp(4, [""])
fi = FileInput(files=(t1, t2, t3, t4))
line = fi.readline()
verify(line == 'The only line there is.\n')
verify(fi.lineno() == 1)
verify(fi.filelineno() == 1)
verify(fi.filename() == t3)
line = fi.readline()
verify(not line)
verify(fi.lineno() == 1)
verify(fi.filelineno() == 0)
verify(fi.filename() == t4)
fi.close()
finally:
remove_tempfiles(t1, t2, t3, t4)
if verbose:
print "14. Files that don't end with newline"
try:
t1 = writeTmp(1, ["A\nB\nC"])
t2 = writeTmp(2, ["D\nE\nF"])
class ManyFile(object):
"""A generalized version of file input that allows for stopping
reading the current file and switching to a different file.
Reading the old file is resumed once the current file is complete.
Useful for #include or similiar constructs.
"""
def __init__(self, files=None, hook=None):
self.fi = [
] # an array of file input objects. We push and pop onto this so we can resume.
self.files = [] # an array of files that need to be read later.
self.hook = None
self.current = None # the current file input object.
self.line = 0
if files is not None:
self.current = FileInput(files, openhook=self.hook)
def include(self, files):
if self.current is not None:
self.fi.push(self.current)
self.current = FileInput(files, openhook=self.hook)
def fileno(self):
if self.current is None:
return None
return self.current.fileno()
def filename(self):
if self.current is None:
return None
return self.current.filename()
def lineno(self):
return self.line
def filelineno(self):
if self.current is None:
return None
return self.current.filelineno()
def isfirstline(self):
if self.current is None:
return None
return self.current.isfirstline()
def isstdin(self):
return False
def nextfile(self):
if self.current is not None:
self.current.nextfile()
def close(self):
if self.current:
self.current.close()
self.fi = []
self.files = []
def readline(self):
if self.current is not None:
l = self.current.readline()
if len(l) > 0:
self.line = self.line + 1
return l
# Advance to the next file
if len(self.fi) > 0:
self.current = self.fi.pop()
return self.readline()
# see if there are any files left to open
if len(self.files) > 0:
self.current = FileInput(self.files,
openhook=fileinput.hook_compressed)
self.files = []
return self.readline()
return ""
def __getimem__(self):
return self.readline()
def __iter__(self):
return self
def next(self):
l = self.__getimem__()
if l != '':
return l
raise StopIteration
'''
def runTests(t1, t2, t3, t4, bs=0, round=0):
start = 1 + round * 6
if verbose:
print '%s. Simple iteration (bs=%s)' % (start + 0, bs)
fi = FileInput(files=(t1, t2, t3, t4), bufsize=bs)
lines = list(fi)
fi.close()
verify(len(lines) == 31)
verify(lines[4] == 'Line 5 of file 1\n')
verify(lines[30] == 'Line 1 of file 4\n')
verify(fi.lineno() == 31)
verify(fi.filename() == t4)
if verbose:
print '%s. Status variables (bs=%s)' % (start + 1, bs)
fi = FileInput(files=(t1, t2, t3, t4), bufsize=bs)
s = "x"
while s and s != 'Line 6 of file 2\n':
s = fi.readline()
verify(fi.filename() == t2)
verify(fi.lineno() == 21)
verify(fi.filelineno() == 6)
verify(not fi.isfirstline())
verify(not fi.isstdin())
if verbose:
print '%s. Nextfile (bs=%s)' % (start + 2, bs)
fi.nextfile()
verify(fi.readline() == 'Line 1 of file 3\n')
verify(fi.lineno() == 22)
fi.close()
if verbose:
print '%s. Stdin (bs=%s)' % (start + 3, bs)
fi = FileInput(files=(t1, t2, t3, t4, '-'), bufsize=bs)
savestdin = sys.stdin
try:
sys.stdin = StringIO("Line 1 of stdin\nLine 2 of stdin\n")
lines = list(fi)
verify(len(lines) == 33)
verify(lines[32] == 'Line 2 of stdin\n')
verify(fi.filename() == '<stdin>')
fi.nextfile()
finally:
sys.stdin = savestdin
if verbose:
print '%s. Boundary conditions (bs=%s)' % (start + 4, bs)
fi = FileInput(files=(t1, t2, t3, t4), bufsize=bs)
verify(fi.lineno() == 0)
verify(fi.filename() == None)
fi.nextfile()
verify(fi.lineno() == 0)
verify(fi.filename() == None)
if verbose:
print '%s. Inplace (bs=%s)' % (start + 5, bs)
savestdout = sys.stdout
try:
fi = FileInput(files=(t1, t2, t3, t4), inplace=1, bufsize=bs)
for line in fi:
line = line[:-1].upper()
print line
fi.close()
finally:
sys.stdout = savestdout
fi = FileInput(files=(t1, t2, t3, t4), bufsize=bs)
for line in fi:
verify(line[-1] == '\n')
m = pat.match(line[:-1])
verify(m != None)
verify(int(m.group(1)) == fi.filelineno())
fi.close()
finally:
remove_tempfiles(t1, t2, t3, t4)
# Next, check for proper behavior with 0-byte files.
if verbose:
print "13. 0-byte files"
try:
t1 = writeTmp(1, [""])
t2 = writeTmp(2, [""])
t3 = writeTmp(3, ["The only line there is.\n"])
t4 = writeTmp(4, [""])
fi = FileInput(files=(t1, t2, t3, t4))
line = fi.readline()
verify(line == 'The only line there is.\n')
verify(fi.lineno() == 1)
verify(fi.filelineno() == 1)
verify(fi.filename() == t3)
line = fi.readline()
verify(not line)
verify(fi.lineno() == 1)
verify(fi.filelineno() == 0)
verify(fi.filename() == t4)
fi.close()
finally:
remove_tempfiles(t1, t2, t3, t4)
if verbose:
print "14. Files that don't end with newline"
try:
t1 = writeTmp(1, ["A\nB\nC"])
t2 = writeTmp(2, ["D\nE\nF"])
class ManyFile(object):
"""A generalized version of file input that allows for stopping
reading the current file and switching to a different file.
Reading the old file is resumed once the current file is complete.
Useful for #include or similiar constructs.
"""
def __init__(self, files=None, hook = None):
self.fi = [] # an array of file input objects. We push and pop onto this so we can resume.
self.files = [] # an array of files that need to be read later.
self.hook = None
self.current = None # the current file input object.
self.line = 0
if files is not None:
self.current = FileInput(files, openhook = self.hook)
def include(self, files):
if self.current is not None:
self.fi.push(self.current)
self.current = FileInput(files, openhook = self.hook)
def fileno(self):
if self.current is None:
return None
return self.current.fileno()
def filename(self):
if self.current is None:
return None
return self.current.filename()
def lineno(self):
return self.line
def filelineno(self):
if self.current is None:
return None
return self.current.filelineno()
def isfirstline(self):
if self.current is None:
return None
return self.current.isfirstline()
def isstdin(self):
return False
def nextfile(self):
if self.current is not None:
self.current.nextfile()
def close(self):
if self.current:
self.current.close()
self.fi = []
self.files = []
def readline(self):
if self.current is not None:
l = self.current.readline()
if len(l) > 0:
self.line = self.line + 1
return l
# Advance to the next file
if len (self.fi) > 0:
self.current = self.fi.pop()
return self.readline()
# see if there are any files left to open
if len (self.files) > 0:
self.current = FileInput(self.files, openhook = fileinput.hook_compressed)
self.files = []
return self.readline()
return ""
def __getimem__ (self):
return self.readline()
def __iter__(self):
return self
def next(self):
l = self.__getimem__()
if l != '':
return l
raise StopIteration
def buffer_size_test(self, t1, t2, t3, t4, bs=0, round=0):
pat = re.compile(r'LINE (\d+) OF FILE (\d+)')
start = 1 + round*6
if verbose:
print('%s. Simple iteration (bs=%s)' % (start+0, bs))
fi = FileInput(files=(t1, t2, t3, t4), bufsize=bs)
lines = list(fi)
fi.close()
self.assertEqual(len(lines), 31)
self.assertEqual(lines[4], 'Line 5 of file 1\n')
self.assertEqual(lines[30], 'Line 1 of file 4\n')
self.assertEqual(fi.lineno(), 31)
self.assertEqual(fi.filename(), t4)
if verbose:
print('%s. Status variables (bs=%s)' % (start+1, bs))
fi = FileInput(files=(t1, t2, t3, t4), bufsize=bs)
s = "x"
while s and s != 'Line 6 of file 2\n':
s = fi.readline()
self.assertEqual(fi.filename(), t2)
self.assertEqual(fi.lineno(), 21)
self.assertEqual(fi.filelineno(), 6)
self.assertFalse(fi.isfirstline())
self.assertFalse(fi.isstdin())
if verbose:
print('%s. Nextfile (bs=%s)' % (start+2, bs))
fi.nextfile()
self.assertEqual(fi.readline(), 'Line 1 of file 3\n')
self.assertEqual(fi.lineno(), 22)
fi.close()
if verbose:
print('%s. Stdin (bs=%s)' % (start+3, bs))
fi = FileInput(files=(t1, t2, t3, t4, '-'), bufsize=bs)
savestdin = sys.stdin
try:
sys.stdin = StringIO("Line 1 of stdin\nLine 2 of stdin\n")
lines = list(fi)
self.assertEqual(len(lines), 33)
self.assertEqual(lines[32], 'Line 2 of stdin\n')
self.assertEqual(fi.filename(), '<stdin>')
fi.nextfile()
finally:
sys.stdin = savestdin
if verbose:
print('%s. Boundary conditions (bs=%s)' % (start+4, bs))
fi = FileInput(files=(t1, t2, t3, t4), bufsize=bs)
self.assertEqual(fi.lineno(), 0)
self.assertEqual(fi.filename(), None)
fi.nextfile()
self.assertEqual(fi.lineno(), 0)
self.assertEqual(fi.filename(), None)
if verbose:
print('%s. Inplace (bs=%s)' % (start+5, bs))
savestdout = sys.stdout
try:
fi = FileInput(files=(t1, t2, t3, t4), inplace=1, bufsize=bs)
for line in fi:
line = line[:-1].upper()
print(line)
fi.close()
finally:
sys.stdout = savestdout
fi = FileInput(files=(t1, t2, t3, t4), bufsize=bs)
for line in fi:
self.assertEqual(line[-1], '\n')
m = pat.match(line[:-1])
self.assertNotEqual(m, None)
self.assertEqual(int(m.group(1)), fi.filelineno())
fi.close()
def test_buffer_sizes(self):
t1 = self.writeTmp(''.join("Line %s of file 1\n" % (i + 1)
for i in range(15)))
t2 = self.writeTmp(''.join("Line %s of file 2\n" % (i + 1)
for i in range(10)))
t3 = self.writeTmp(''.join("Line %s of file 3\n" % (i + 1)
for i in range(5)))
t4 = self.writeTmp(''.join("Line %s of file 4\n" % (i + 1)
for i in range(1)))
pat = re.compile(r'LINE (\d+) OF FILE (\d+)')
if verbose:
print('1. Simple iteration')
fi = FileInput(files=(t1, t2, t3, t4))
lines = list(fi)
fi.close()
self.assertEqual(len(lines), 31)
self.assertEqual(lines[4], 'Line 5 of file 1\n')
self.assertEqual(lines[30], 'Line 1 of file 4\n')
self.assertEqual(fi.lineno(), 31)
self.assertEqual(fi.filename(), t4)
if verbose:
print('2. Status variables')
fi = FileInput(files=(t1, t2, t3, t4))
s = "x"
while s and s != 'Line 6 of file 2\n':
s = fi.readline()
self.assertEqual(fi.filename(), t2)
self.assertEqual(fi.lineno(), 21)
self.assertEqual(fi.filelineno(), 6)
self.assertFalse(fi.isfirstline())
self.assertFalse(fi.isstdin())
if verbose:
print('3. Nextfile')
fi.nextfile()
self.assertEqual(fi.readline(), 'Line 1 of file 3\n')
self.assertEqual(fi.lineno(), 22)
fi.close()
if verbose:
print('4. Stdin')
fi = FileInput(files=(t1, t2, t3, t4, '-'))
savestdin = sys.stdin
try:
sys.stdin = StringIO("Line 1 of stdin\nLine 2 of stdin\n")
lines = list(fi)
self.assertEqual(len(lines), 33)
self.assertEqual(lines[32], 'Line 2 of stdin\n')
self.assertEqual(fi.filename(), '<stdin>')
fi.nextfile()
finally:
sys.stdin = savestdin
if verbose:
print('5. Boundary conditions')
fi = FileInput(files=(t1, t2, t3, t4))
self.assertEqual(fi.lineno(), 0)
self.assertEqual(fi.filename(), None)
fi.nextfile()
self.assertEqual(fi.lineno(), 0)
self.assertEqual(fi.filename(), None)
if verbose:
print('6. Inplace')
savestdout = sys.stdout
try:
fi = FileInput(files=(t1, t2, t3, t4), inplace=1)
for line in fi:
line = line[:-1].upper()
print(line)
fi.close()
finally:
sys.stdout = savestdout
fi = FileInput(files=(t1, t2, t3, t4))
for line in fi:
self.assertEqual(line[-1], '\n')
m = pat.match(line[:-1])
self.assertNotEqual(m, None)
self.assertEqual(int(m.group(1)), fi.filelineno())
fi.close()
def runTests(t1, t2, t3, t4, bs=0, round=0):
start = 1 + round*6
if verbose:
print '%s. Simple iteration (bs=%s)' % (start+0, bs)
fi = FileInput(files=(t1, t2, t3, t4), bufsize=bs)
lines = list(fi)
fi.close()
verify(len(lines) == 31)
verify(lines[4] == 'Line 5 of file 1\n')
verify(lines[30] == 'Line 1 of file 4\n')
verify(fi.lineno() == 31)
verify(fi.filename() == t4)
if verbose:
print '%s. Status variables (bs=%s)' % (start+1, bs)
fi = FileInput(files=(t1, t2, t3, t4), bufsize=bs)
s = "x"
while s and s != 'Line 6 of file 2\n':
s = fi.readline()
verify(fi.filename() == t2)
verify(fi.lineno() == 21)
verify(fi.filelineno() == 6)
verify(not fi.isfirstline())
verify(not fi.isstdin())
if verbose:
print '%s. Nextfile (bs=%s)' % (start+2, bs)
fi.nextfile()
verify(fi.readline() == 'Line 1 of file 3\n')
verify(fi.lineno() == 22)
fi.close()
if verbose:
print '%s. Stdin (bs=%s)' % (start+3, bs)
fi = FileInput(files=(t1, t2, t3, t4, '-'), bufsize=bs)
savestdin = sys.stdin
try:
sys.stdin = StringIO("Line 1 of stdin\nLine 2 of stdin\n")
lines = list(fi)
verify(len(lines) == 33)
verify(lines[32] == 'Line 2 of stdin\n')
verify(fi.filename() == '<stdin>')
fi.nextfile()
finally:
sys.stdin = savestdin
if verbose:
print '%s. Boundary conditions (bs=%s)' % (start+4, bs)
fi = FileInput(files=(t1, t2, t3, t4), bufsize=bs)
verify(fi.lineno() == 0)
verify(fi.filename() == None)
fi.nextfile()
verify(fi.lineno() == 0)
verify(fi.filename() == None)
if verbose:
print '%s. Inplace (bs=%s)' % (start+5, bs)
savestdout = sys.stdout
try:
fi = FileInput(files=(t1, t2, t3, t4), inplace=1, bufsize=bs)
for line in fi:
line = line[:-1].upper()
print line
fi.close()
finally:
sys.stdout = savestdout
fi = FileInput(files=(t1, t2, t3, t4), bufsize=bs)
for line in fi:
verify(line[-1] == '\n')
m = pat.match(line[:-1])
verify(m != None)
verify(int(m.group(1)) == fi.filelineno())
fi.close()
def buffer_size_test(self, t1, t2, t3, t4, bs=0, round=0):
pat = re.compile(r'LINE (\d+) OF FILE (\d+)')
start = 1 + round * 6
if verbose:
print('%s. Simple iteration (bs=%s)' % (start + 0, bs))
fi = FileInput(files=(t1, t2, t3, t4), bufsize=bs)
lines = list(fi)
fi.close()
self.assertEqual(len(lines), 31)
self.assertEqual(lines[4], 'Line 5 of file 1\n')
self.assertEqual(lines[30], 'Line 1 of file 4\n')
self.assertEqual(fi.lineno(), 31)
self.assertEqual(fi.filename(), t4)
if verbose:
print('%s. Status variables (bs=%s)' % (start + 1, bs))
fi = FileInput(files=(t1, t2, t3, t4), bufsize=bs)
s = "x"
while s and s != 'Line 6 of file 2\n':
s = fi.readline()
self.assertEqual(fi.filename(), t2)
self.assertEqual(fi.lineno(), 21)
self.assertEqual(fi.filelineno(), 6)
self.assertFalse(fi.isfirstline())
self.assertFalse(fi.isstdin())
if verbose:
print('%s. Nextfile (bs=%s)' % (start + 2, bs))
fi.nextfile()
self.assertEqual(fi.readline(), 'Line 1 of file 3\n')
self.assertEqual(fi.lineno(), 22)
fi.close()
if verbose:
print('%s. Boundary conditions (bs=%s)' % (start + 4, bs))
fi = FileInput(files=(t1, t2, t3, t4), bufsize=bs)
self.assertEqual(fi.lineno(), 0)
self.assertEqual(fi.filename(), None)
fi.nextfile()
self.assertEqual(fi.lineno(), 0)
self.assertEqual(fi.filename(), None)
if verbose:
print('%s. Inplace (bs=%s)' % (start + 5, bs))
savestdout = sys.stdout
try:
fi = FileInput(files=(t1, t2, t3, t4), inplace=1, bufsize=bs)
for line in fi:
line = line[:-1].upper()
print(line)
fi.close()
finally:
pass ###
fi = FileInput(files=(t1, t2, t3, t4), bufsize=bs)
for line in fi:
self.assertEqual(line[-1], '\n')
m = pat.match(line[:-1])
self.assertNotEqual(m, None)
self.assertEqual(int(m.group(1)), fi.filelineno())
fi.close()
'''