def test_utf_char_in_code(log):
''' Check that we cope with Fortran code that contains UTF characters. This
is not valid Fortran but most compilers cope with it. '''
log.reset()
fort_file = os.path.join(os.path.dirname(__file__), "utf_in_code.f90")
reader = FortranFileReader(fort_file, ignore_comments=True)
out_line = reader.get_item()
while out_line:
out_line = reader.get_item()
assert log.messages['critical'] == []
def _parse_file(self, fpath):
"""Get a node tree from a fortran file."""
reader = FortranFileReader(str(fpath), ignore_comments=False)
reader.exit_on_error = False # don't call sys.exit, it messes up the multi-processing
try:
tree = self.f2008_parser(reader)
return tree
except FortranSyntaxError as err:
# we can't return the FortranSyntaxError, it breaks multiprocessing!
logger.error(f"\nsyntax error in {fpath}\n{err}")
raise Exception(f"syntax error in {fpath}\n{err}")
except Exception as err:
logger.error(f"\nunhandled error '{type(err)}' in {fpath}\n{err}")
raise Exception(f"unhandled error '{type(err)}' in {fpath}\n{err}")
def runner(_, options, args):
''' Function to read, parse and output fortran source code '''
from fparser.two.parser import ParserFactory
from fparser.two.Fortran2003 import FortranSyntaxError, InternalError
from fparser.common.readfortran import FortranFileReader
if not args:
print("Error: No fortran files specified")
raise SystemExit(1)
for filename in args:
try:
reader = FortranFileReader(filename)
except IOError as error:
print(error)
return
if options.mode != 'auto':
reader.format.from_mode(options.mode)
try:
f2003_parser = ParserFactory().create()
program = f2003_parser(reader)
print(program)
except FortranSyntaxError as msg:
print("Syntax error: {0}".format(str(msg)))
try:
# protect the access to fifo_item[-1] in case the fifo
# buffer is empty
print('parsing %r failed at %s' %
(filename, reader.fifo_item[-1]))
print('started at %s' % (reader.fifo_item[0]))
except IndexError:
pass
raise SystemExit(1)
except InternalError as msg:
print("Internal error in fparser: {0}".format(str(msg)))
raise SystemExit(1)
def runner(_, options, args):
'''Call the Fortran File reader for each filename in args and print
out its content.
:param options: command line argument information from the options \
parser
:type options: :py:class:`optparse.Values`
:param args: a list of Fortran filepaths
:type args: list of str
:raises NotImplementedError: if the task option is not set to \
"show".
'''
from fparser.common.readfortran import FortranFileReader
for filename in args:
reader = FortranFileReader(filename)
if options.task == 'show':
for item in reader:
print(item)
sys.stdout.flush()
else:
raise NotImplementedError(
"The task option '{0}' is invalid. Currently only "
"'show' is supported.".format(repr(options.task)))
def parse_fp2(filename):
'''Parse a Fortran source file contained in the file 'filename' using
fparser2.
:param str filename: source file (including path) to read.
:returns: fparser2 AST for the source file.
:rtype: :py:class:`fparser.two.Fortran2003.Program`
:raises ParseError: if the file could not be parsed.
'''
parser = ParserFactory().create(std="f2008")
# We get the directories to search for any Fortran include files from
# our configuration object.
config = Config.get()
try:
reader = FortranFileReader(filename, include_dirs=config.include_paths)
except IOError as error:
raise ParseError(
"algorithm.py:parse_fp2: Failed to parse file '{0}'. Error "
"returned was ' {1} '.".format(filename, error))
try:
parse_tree = parser(reader)
except FortranSyntaxError as msg:
raise ParseError(
"algorithm.py:parse_fp2: Syntax error in file '{0}':\n"
"{1}".format(filename, str(msg)))
return parse_tree
def test_bad_file_reader():
'''
Tests that the file reader can spot when it is given something to read
which is neither file nor filename.
'''
with pytest.raises(ValueError) as ex:
unit_under_test = FortranFileReader(42)
expected = 'FortranFileReader is used with a filename or file-like object.'
assert expected in str(ex)
def check_include_works(fortran_filename,
fortran_code,
include_info,
expected,
tmpdir,
ignore_comments=True):
'''Utility function used by a number of tests to check that include
files work as expected.
:param str fortran_filename: the name of the fortran file that is \
going to be created in the 'tmpdir' directory.
:param str fortran_code: the fortran code to put in the fortran \
file specified by 'fortran_filename'.
:param include_info: a list of 2-tuples each with an include \
filename as a string followed by include code as a string.
:type include_info: list of (str, str)
:param str expected: the expected output after parsing the code.
:param str tmpdir: the temporary directory in which to create and \
process the Fortran files.
:param bool ignore_comments: whether to ignore (skip) comments in \
the Fortran code or not. Defaults to ignore them.
'''
try:
oldpwd = tmpdir.chdir()
cwd = str(tmpdir)
# Create the program
with open(os.path.join(cwd, fortran_filename), "w") as cfile:
cfile.write(fortran_code)
for include_filename in include_info.keys():
with open(os.path.join(cwd, include_filename), "w") as cfile:
cfile.write(include_info[include_filename])
reader = FortranFileReader(fortran_filename,
ignore_comments=ignore_comments)
for orig_line in expected.split("\n"):
new_line = reader.next().line
assert new_line == orig_line
with pytest.raises(StopIteration):
reader.next()
finally:
oldpwd.chdir()
def test_filename_reader():
'''
Tests that a Fortran source file can be read given its filename.
'''
handle, filename = tempfile.mkstemp(suffix='.f90', text=True)
os.close(handle)
try:
with io.open(filename, mode='w', encoding='UTF-8') as source_file:
source_file.write(FULL_FREE_SOURCE)
unit_under_test = FortranFileReader(filename)
expected = fparser.common.sourceinfo.FortranFormat(True, False)
assert unit_under_test.format == expected
for expected in FULL_FREE_EXPECTED:
found = unit_under_test.get_single_line(ignore_empty=True)
assert found == expected
except Exception:
os.unlink(filename)
raise
def test_file_reader():
'''
Tests that a Fortran source file can be read given a file object of it.
'''
handle, filename = tempfile.mkstemp(suffix='.f90', text=True)
os.close(handle)
try:
with open(filename, mode='w') as source_file:
print(FULL_FREE_SOURCE, file=source_file)
with open(filename, mode='r') as source_file:
unit_under_test = FortranFileReader(source_file)
expected = fparser.common.sourceinfo.FortranFormat(True, False)
assert unit_under_test.format == expected
for expected in FULL_FREE_EXPECTED:
assert unit_under_test.get_single_line(ignore_empty=True) \
== expected
except Exception:
os.unlink(filename)
raise
def test_none_in_fifo(log):
''' Check that a None entry in the reader FIFO buffer is handled
correctly. '''
log.reset()
handle, filename = tempfile.mkstemp(suffix='.f90', text=True)
os.close(handle)
with io.open(filename, mode='w', encoding='UTF-8') as source_file:
source_file.write(FULL_FREE_SOURCE)
with io.open(filename, mode='r', encoding='UTF-8') as source_file:
unit_under_test = FortranFileReader(source_file)
while True:
try:
_ = unit_under_test.next(ignore_comments=False)
except StopIteration:
break
# Erroneously push a None to the FIFO buffer
unit_under_test.put_item(None)
# Attempt to read the next item
with pytest.raises(StopIteration):
_ = unit_under_test.next(ignore_comments=False)
# Check that nothing has been logged
for log_level in ["debug", "info", "warning", "error", "critical"]:
assert log.messages[log_level] == []
def test_new_kernel_file(tmpdir, monkeypatch):
''' Check that we write out the transformed kernel to the CWD. '''
from fparser.two import Fortran2003, parser
from fparser.common.readfortran import FortranFileReader
# Ensure kernel-output directory is uninitialised
config = Config.get()
monkeypatch.setattr(config, "_kernel_output_dir", "")
monkeypatch.setattr(config, "_kernel_naming", "multiple")
# Change to temp dir (so kernel written there)
old_cwd = tmpdir.chdir()
psy, invoke = get_invoke("nemolite2d_alg_mod.f90", api="gocean1.0", idx=0)
sched = invoke.schedule
kern = sched.children[0].loop_body[0].loop_body[0]
rtrans = ACCRoutineTrans()
_, _ = rtrans.apply(kern)
# Generate the code (this triggers the generation of a new kernel)
code = str(psy.gen).lower()
# Work out the value of the tag used to re-name the kernel
tag = re.search('use continuity(.+?)_mod', code).group(1)
assert ("use continuity{0}_mod, only: continuity{0}_code".format(tag)
in code)
assert "call continuity{0}_code(".format(tag) in code
# The kernel and module name should have gained the tag just identified
# and be written to the CWD
filename = os.path.join(str(tmpdir), "continuity{0}_mod.f90".format(tag))
assert os.path.isfile(filename)
# Parse the new kernel file
f2003_parser = parser.ParserFactory().create()
reader = FortranFileReader(filename)
prog = f2003_parser(reader)
# Check that the module has the right name
modules = walk_ast(prog.content, [Fortran2003.Module_Stmt])
assert str(modules[0].items[1]) == "continuity{0}_mod".format(tag)
# Check that the subroutine has the right name
subs = walk_ast(prog.content, [Fortran2003.Subroutine_Stmt])
found = False
for sub in subs:
if str(sub.items[1]) == "continuity{0}_code".format(tag):
found = True
break
assert found
# Check that the kernel type has been re-named
dtypes = walk_ast(prog.content, [Fortran2003.Derived_Type_Def])
names = walk_ast(dtypes[0].content, [Fortran2003.Type_Name])
assert str(names[0]) == "continuity{0}_type".format(tag)
from gocean1p0_build import GOcean1p0Build
# If compilation fails this will raise an exception
GOcean1p0Build(tmpdir).compile_file(filename)
old_cwd.chdir()
def test_nemo_find_container_symbol(parser):
''' Check that find_or_create_symbol() works for the NEMO API when the
searched-for symbol is declared in the parent module. '''
reader = FortranFileReader(
os.path.join(
os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
"test_files", "gocean1p0", "kernel_with_global_mod.f90"))
prog = parser(reader)
psy = PSyFactory("nemo", distributed_memory=False).create(prog)
# Get a node from the schedule
bops = psy._invokes.invoke_list[0].schedule.walk(BinaryOperation)
# Use it as the starting point for the search
symbol = _find_or_create_imported_symbol(bops[0], "alpha")
assert symbol.datatype.intrinsic == ScalarType.Intrinsic.REAL
def runner(parser, options, args):
from fparser.two import Fortran2003
from fparser.common.readfortran import FortranFileReader
for filename in args:
reader = FortranFileReader(filename)
if options.mode != 'auto':
mode = fparser.common.sourceinfo\
.FortranFormat.from_mode(options.mode)
reader.format.set_mode(mode)
try:
program = Fortran2003.Program(reader)
print(program)
except Fortran2003.NoMatchError as msg:
print('parsing %r failed at %s' % (filename, reader.fifo_item[-1]))
print('started at %s' % (reader.fifo_item[0]))
print('quiting')
return
def runner(parser, options, args):
from fparser.common.readfortran import FortranFileReader
from fparser.one.parsefortran import FortranParser
for filename in args:
reader = FortranFileReader(filename)
if options.mode != 'auto':
mode = fparser.common.sourceinfo\
.FortranFormat.from_mode(options.mode)
reader.format.set_mode(mode)
parser = FortranParser(reader)
parser.parse()
parser.analyze()
if options.task == 'show':
print(parser.block.torepr(4))
elif options.task == 'none':
pass
else:
raise NotImplementedError(repr(options.task))
def test_inherited_f77():
'''
A grab bag of functional tests inherited from readfortran.py.
'''
string_f77 = """c -*- f77 -*-
c12346 comment
subroutine foo
call foo
'bar
a 'g
abc=2
cf2py call me ! hey
call you ! hi
end
'"""
expected = ["Comment('c -*- f77 -*-',(1, 1))",
"Comment('c12346 comment',(2, 2))",
"line #3'subroutine foo'",
"line #4'call foobar'",
'Comment("a \'g",(6, 6))',
"line #7'abc=2'",
"line #9'call you ! hi'",
"line #10'end'"]
# Reading from buffer
reader = FortranStringReader(
string_f77, ignore_comments=False)
assert reader.format.mode == 'f77', repr(reader.format.mode)
stack = expected[:]
for item in reader:
assert str(item) == stack.pop(0)
# Reading from file
handle, filename = tempfile.mkstemp(suffix='.f', text=True)
os.close(handle)
with open(filename, 'w') as fortran_file:
print(string_f77, file=fortran_file)
reader = FortranFileReader(
filename, ignore_comments=False)
stack = expected[:]
for item in reader:
assert str(item) == stack.pop(0)
def runner(_, options, args):
'''
Function to read, parse and output Fortran source code.
:param options: object constructed by OptionParser with cmd-line flags.
:param args: list of Fortran files to parse.
:type args: list of str
'''
import six
from fparser.two.parser import ParserFactory
from fparser.two.Fortran2003 import FortranSyntaxError, InternalError
from fparser.common.readfortran import FortranFileReader
if not args:
print("Error: No fortran files specified", file=sys.stderr)
raise SystemExit(1)
for filename in args:
print("File: '{0}'".format(filename), file=sys.stderr)
try:
reader = FortranFileReader(filename, ignore_comments=False)
except IOError as error:
print(error, file=sys.stderr)
continue
try:
fparser = ParserFactory().create(std=options.std)
program = fparser(reader)
if options.task == "show":
print(six.text_type(program))
if options.task == "repr":
print(repr(program))
except FortranSyntaxError as msg:
print("Syntax error: {0}".format(six.text_type(msg)),
file=sys.stderr)
except InternalError as msg:
print("Internal error in fparser: {0}".format(six.text_type(msg)),
file=sys.stderr)
def get_reader(source,
isfree=None,
isstrict=None,
include_dirs=None,
source_only=None,
ignore_comments=True):
'''
Returns Fortran reader instance.
If ``source`` is a C filename then the functions searches for comment
lines starting with ``/*f2py`` and reads following lines as PYF file
content until a line ``*/`` is found.
:param str source: Specify a string or filename containing Fortran code.
:param bool isfree: True if Fortran is free format
:param bool isstrict: True if we are to strictly enforce free/fixed format
:param list include_dirs: Specify a list of include directories. The
default list (when include_dirs=None) contains
the current working directory and the directory
of ``source``.
:param list source_only: Specify a list of Fortran file names that are
searched when the ``USE`` statement is
encountered.
:param bool ignore_comments: Whether or not to ignore (and discard)
comments when parsing the source.
:returns: a reader instance
:rtype: :py:class:`fparser.common.readfortran.FortranReader`
'''
import os
import re
from fparser.common.readfortran import FortranFileReader, \
FortranStringReader
from fparser.common.sourceinfo import FortranFormat
if os.path.isfile(source):
_name, ext = os.path.splitext(source)
if ext.lower() in ['.c']:
# get signatures from C file comments starting with
# `/*f2py` and ending with `*/`.
# TODO: improve parser to take line number offset making line
# numbers in parser messages correct.
f2py_c_comments = re.compile(r'/[*]\s*f2py\s.*[*]/', re.I | re.M)
handle = open(source, 'r')
c_input = ''
for line in f2py_c_comments.findall(handle.read()):
c_input += line[2:-2].lstrip()[4:] + '\n'
handle.close()
if isfree is None:
isfree = True
if isstrict is None:
isstrict = True
return parse(c_input, isfree, isstrict, include_dirs)
reader = FortranFileReader(source,
include_dirs=include_dirs,
source_only=source_only,
ignore_comments=ignore_comments)
elif isinstance(source, string_types):
reader = FortranStringReader(source,
include_dirs=include_dirs,
source_only=source_only,
ignore_comments=ignore_comments)
else:
raise TypeError('Expected string or filename input but got %s' %
(type(input)))
if isfree is None:
isfree = reader.format.is_free
if isstrict is None:
isstrict = reader.format.is_strict
reader.set_format(FortranFormat(isfree, isstrict))
return reader
if __name__ == "__main__":
if len(sys.argv) < 2:
usage()
# Apparently pylint thinks that `all_files` is a constant, and complains
# that it should be capitalised.
# pylint: disable=invalid-name
all_files = sys.argv[1:]
# Sort the input file names, so that they are output alphabetically
all_files.sort()
for filename in all_files:
# Parse the current source file:
try:
reader = FortranFileReader(filename)
except IOError:
print("Could not open file '{0}'.".format(filename),
file=sys.stderr)
sys.exit(-1)
parser = ParserFactory().create(std="f2003")
parse_tree = parser(reader)
# Collect all used modules in a list
all_use = []
for node in walk(parse_tree, Use_Stmt):
use_name = str(node.items[2])
# Nothing else to do if the name is already in the list:
if use_name + ".o" in all_use:
continue