def get_filename_from_import(module, output_folder=''):
"""Returns a valid filename with absolute path, that corresponds to the
definition of module.
The priority order is:
- header files (extension == pyh)
- python files (extension == py)
"""
filename_pyh = '{}.pyh'.format(module.replace('.', '/'))
filename_py = '{}.py'.format(module.replace('.', '/'))
if is_valid_filename_pyh(filename_pyh):
return os.path.abspath(filename_pyh)
if is_valid_filename_py(filename_py):
return os.path.abspath(filename_py)
folders = output_folder.split(""".""")
for i in range(len(folders)):
poss_dirname = os.path.join(*folders[:i + 1])
poss_filename_pyh = os.path.join(poss_dirname, filename_pyh)
poss_filename_py = os.path.join(poss_dirname, filename_py)
if is_valid_filename_pyh(poss_filename_pyh):
return os.path.abspath(poss_filename_pyh)
if is_valid_filename_py(poss_filename_py):
return os.path.abspath(poss_filename_py)
source = module
if len(module.split(""".""")) > 1:
# we remove the last entry, since it can be a pyh file
source = """.""".join(i for i in module.split(""".""")[:-1])
_module = module.split(""".""")[-1]
filename_pyh = '{}.pyh'.format(_module)
filename_py = '{}.py'.format(_module)
try:
package = importlib.import_module(source)
package_dir = str(package.__path__[0])
except:
errors = Errors()
errors.report(PYCCEL_UNFOUND_IMPORTED_MODULE,
symbol=source,
severity='fatal')
filename_pyh = os.path.join(package_dir, filename_pyh)
filename_py = os.path.join(package_dir, filename_py)
if os.path.isfile(filename_pyh):
return filename_pyh
elif os.path.isfile(filename_py):
return filename_py
errors = Errors()
errors.report(PYCCEL_UNFOUND_IMPORTED_MODULE,
symbol=module,
severity='fatal')
def test_semantic_errors():
print('*********************************')
print('*** ***')
print('*** TESTING SEMANTIC ERRORS ***')
print('*** ***')
print('*********************************')
init_dir = os.getcwd()
base_dir = os.path.dirname(os.path.realpath(__file__))
path_dir = os.path.join(base_dir, 'semantic')
files = sorted(os.listdir(path_dir))
files = [f for f in files if (f.endswith(".py"))]
os.chdir(path_dir)
for f in files:
print('> testing {0}'.format(str(f)))
pyccel = Parser(f, show_traceback=False)
ast = pyccel.parse()
try:
settings = {}
ast = pyccel.annotate(**settings)
except:
pass
# reset Errors singleton
errors = Errors()
errors.reset()
os.chdir(init_dir)
print('\n')
def test_syntax_errors():
print('*********************************')
print('*** ***')
print('*** TESTING SYNTAX ERRORS ***')
print('*** ***')
print('*********************************')
init_dir = os.getcwd()
base_dir = os.path.dirname(os.path.realpath(__file__))
path_dir = os.path.join(base_dir, 'syntax')
files = sorted(os.listdir(path_dir))
files = [f for f in files if (f.endswith(".py"))]
os.chdir(path_dir)
for f in files:
print('> testing {0}'.format(str(f)))
pyccel = Parser(f)
try:
ast = pyccel.parse()
except:
pass
# reset Errors singleton
errors = Errors()
errors.reset()
os.chdir(init_dir)
print('\n')
def parse(self, verbose=False):
"""converts redbaron fst to sympy ast."""
if self.syntax_done:
print('> syntax analysis already done')
return self.ast
# TODO - add settings to Errors
# - filename
errors = Errors()
if self.filename:
errors.set_target(self.filename, 'file')
errors.set_parser_stage('syntax')
# we add the try/except to allow the parser to find all possible errors
try:
ast = self._visit(self.fst)
except Exception as e:
errors.check()
if self.show_traceback:
traceback.print_exc()
raise SystemExit(0)
self._ast = ast
errors.check()
self._visit_done = True
return ast
def test_semantic():
print('*********************************')
print('*** ***')
print('*** TESTING SEMANTIC ***')
print('*** ***')
print('*********************************')
init_dir = os.getcwd()
base_dir = os.path.dirname(os.path.realpath(__file__))
path_dir = os.path.join(base_dir, 'scripts')
files = sorted(os.listdir(path_dir))
files = [f for f in files if (f.endswith(".py"))]
os.chdir(path_dir)
for f in files:
print('> testing {0}'.format(str(f)))
pyccel = Parser(f)
ast = pyccel.parse()
settings = {}
ast = pyccel.annotate(**settings)
# reset Errors singleton
errors = Errors()
errors.reset()
os.chdir(init_dir)
print('\n')
def test_preprocess():
print('*********************************')
print('*** ***')
print('*** TESTING preprocess ***')
print('*** ***')
print('*********************************')
init_dir = os.getcwd()
base_dir = os.path.dirname(os.path.realpath(__file__))
path_dir = os.path.join(base_dir, 'scripts')
files = sorted(os.listdir(path_dir))
files = [f for f in files if (f.endswith(".py"))]
os.chdir(path_dir)
for f in files:
print('> testing {0}'.format(str(f)))
pyccel = Parser(f)
pyccel.parse()
print(pyccel.fst)
# reset Errors singleton
errors = Errors()
errors.reset()
os.chdir(init_dir)
print('\n')
def doprint(self, **settings):
"""Prints the code in the target language."""
# ... finds the target language
language = settings.pop('language', 'fortran')
if not language in ['fortran', 'c']:
raise ValueError('the language {} not available'.format(lanugage))
self._language = language
# ... define the printing function to be used
printer = printer_registry[language]
# ...
errors = Errors()
errors.set_parser_stage('codegen')
# ...
code = printer(self.expr, self.parser, **settings)
# ...
errors.check()
self._code = code
return code
def __init__(self, inputs, **kwargs):
BasicParser.__init__(self, **kwargs)
# check if inputs is a file
code = inputs
if os.path.isfile(inputs):
# we don't use is_valid_filename_py since it uses absolute path
# file extension
ext = inputs.split(""".""")[-1]
if not ext in ['py', 'pyh']:
errors = Errors()
errors.report(INVALID_FILE_EXTENSION,
symbol=ext,
severity='fatal')
errors.check()
raise SystemExit(0)
code = read_file(inputs)
self._filename = inputs
self._code = code
try:
code = self.code
red = RedBaron(code)
except Exception as e:
errors = Errors()
errors.report(INVALID_PYTHON_SYNTAX,
symbol='\n' + str(e),
severity='fatal')
errors.check()
raise SystemExit(0)
red = fst_move_directives(red)
self._fst = red
self.parse(verbose=True)
def test_codegen_warnings():
print('*********************************')
print('*** ***')
print('*** testing codegen warnings ***')
print('*** ***')
print('*********************************')
init_dir = os.getcwd()
base_dir = os.path.dirname(os.path.realpath(__file__))
path_dir = os.path.join(base_dir, 'codegen')
files = sorted(os.listdir(path_dir))
files = [f for f in files if (f.endswith(".py"))]
os.chdir(path_dir)
for f in files:
print('> testing {0}'.format(str(f)))
pyccel = Parser(f, show_traceback=False)
ast = pyccel.parse()
settings = {}
ast = pyccel.annotate(**settings)
try:
name = os.path.basename(f)
name = os.path.splitext(name)[0]
codegen = Codegen(ast, name)
code = codegen.doprint()
except:
pass
# reset Errors singleton
errors = Errors()
errors.reset()
os.chdir(init_dir)
print('\n')
def test_codegen():
print('*********************************')
print('*** ***')
print('*** TESTING CODEGEN ***')
print('*** ***')
print('*********************************')
init_dir = os.getcwd()
base_dir = os.path.dirname(os.path.realpath(__file__))
path_dir = os.path.join(base_dir, 'scripts')
files = sorted(os.listdir(path_dir))
files = [f for f in files if (f.endswith(".py"))]
os.chdir(path_dir)
for f in files:
print('> testing {0}'.format(str(f)))
pyccel = Parser(f)
ast = pyccel.parse()
settings = {}
ast = pyccel.annotate(**settings)
name = os.path.basename(f)
name = os.path.splitext(name)[0]
codegen = Codegen(ast, name)
code = codegen.doprint()
# reset Errors singleton
errors = Errors()
errors.reset()
os.chdir(init_dir)
print('\n')
def pyccel(files=None, openmp=None, openacc=None, output_dir=None, compiler='gfortran'):
"""
pyccel console command.
"""
parser = MyParser(description='pyccel command line')
parser.add_argument('files', metavar='N', type=str, nargs='+',
help='a Pyccel file')
# ... compiler syntax, semantic and codegen
group = parser.add_argument_group('Pyccel compiling stages')
group.add_argument('-x', '--syntax-only', action='store_true',
help='Using pyccel for Syntax Checking')
group.add_argument('-e', '--semantic-only', action='store_true',
help='Using pyccel for Semantic Checking')
group.add_argument('-t', '--convert-only', action='store_true',
help='Converts pyccel files only without build')
# ...
# ... backend compiler options
group = parser.add_argument_group('Backend compiler options')
group.add_argument('--compiler', type=str, \
help='Used compiler')
group.add_argument('--fflags', type=str, \
help='Fortran compiler flags.')
group.add_argument('--debug', action='store_true', \
help='compiles the code in a debug mode.')
group.add_argument('--include', type=str, \
help='path to include directory.')
group.add_argument('--libdir', type=str, \
help='path to lib directory.')
group.add_argument('--libs', type=str, \
help='list of libraries to link with.')
group.add_argument('--output', type=str, default = '',\
help='folder in which the output is stored.')
group.add_argument('--prefix', type=str, default = '',\
help='add prefix to the generated file.')
group.add_argument('--prefix-module', type=str, default = '',\
help='add prefix module name.')
group.add_argument('--language', type=str, help='target language')
# ...
# ... Accelerators
group = parser.add_argument_group('Accelerators options')
group.add_argument('--openmp', action='store_true', \
help='uses openmp')
group.add_argument('--openacc', action='store_true', \
help='uses openacc')
# ...
# ... Other options
group = parser.add_argument_group('Other options')
group.add_argument('--verbose', action='store_true', \
help='enables verbose mode.')
group.add_argument('--developer-mode', action='store_true', \
help='shows internal messages')
# ...
# TODO move to another cmd line
parser.add_argument('--analysis', action='store_true', \
help='enables code analysis mode.')
# ...
# ...
args = parser.parse_args()
# ...
# ...
if not files:
files = args.files
if args.compiler:
compiler = args.compiler
if not openmp:
openmp = args.openmp
if not openacc:
openacc = args.openacc
if args.convert_only or args.syntax_only or args.semantic_only:
compiler = None
# ...
# ...
if not files:
raise ValueError("a python filename must be provided.")
if len(files) > 1:
raise ValueError('Expecting one single file for the moment.')
# ...
filename = files[0]
# ... report error
if os.path.isfile(filename):
# we don't use is_valid_filename_py since it uses absolute path
# file extension
ext = filename.split('.')[-1]
if not(ext in ['py', 'pyh']):
errors = Errors()
errors.report(INVALID_FILE_EXTENSION,
symbol=ext,
severity='fatal')
errors.check()
raise SystemExit(0)
else:
# we use Pyccel error manager, although we can do it in other ways
errors = Errors()
errors.report(INVALID_FILE_DIRECTORY,
symbol=filename,
severity='fatal')
errors.check()
raise SystemExit(0)
# ...
if compiler:
if _which(compiler) is None:
raise ValueError('Could not find {0}'.format(compiler))
accelerator = None
if openmp:
accelerator = "openmp"
if openacc:
accelerator = "openacc"
debug = args.debug
verbose = args.verbose
include = args.include
fflags = args.fflags
libdir = args.libdir
libs = args.libs
output_folder = args.output
prefix = args.prefix
prefix_module = args.prefix_module
language = args.language
if (len(output_folder)>0 and output_folder[-1]!='/'):
output_folder+='/'
if not include:
include = []
if not libdir:
libdir = []
if not libs:
libs = []
# ...
# ...
if args.developer_mode:
# this will initialize the singelton ErrorsMode
# making this settings available everywhere
err_mode = ErrorsMode()
err_mode.set_mode('developer')
# ...
# ...
from pyccel.parser import Parser
from pyccel.codegen import Codegen
if args.syntax_only:
pyccel = Parser(filename)
ast = pyccel.parse()
elif args.semantic_only:
pyccel = Parser(filename)
ast = pyccel.parse()
settings = {}
ast = pyccel.annotate(**settings)
elif args.convert_only:
pyccel = Parser(filename)
ast = pyccel.parse()
settings = {}
if args.language:
settings['language'] = args.language
ast = pyccel.annotate(**settings)
name = os.path.basename(filename)
name = os.path.splitext(name)[0]
codegen = Codegen(ast, name)
settings['prefix_module'] = prefix_module
code = codegen.doprint(**settings)
if prefix:
name = '{prefix}{name}'.format(prefix=prefix, name=name)
codegen.export(output_folder+name)
for son in pyccel.sons:
if 'print' in son.metavars.keys():
name = son.filename.split('/')[-1].strip('.py')
name = 'mod_'+name
codegen = Codegen(son.ast, name)
code = codegen.doprint()
codegen.export()
elif args.analysis:
# TODO move to another cmd line
from pyccel.complexity.arithmetic import OpComplexity
complexity = OpComplexity(filename)
print(" arithmetic cost ~ " + str(complexity.cost()))
else:
# TODO shall we add them in the cmd line?
modules = []
binary = None
execute_pyccel(filename,
compiler=compiler,
fflags=fflags,
debug=False,
verbose=verbose,
accelerator=accelerator,
include=include,
libdir=libdir,
modules=modules,
libs=libs,
binary=binary,
output=output_folder)
from sympy import ceiling, floor, Mod from sympy import Min, Max from sympy import oo as INF from sympy import Pow as sp_Pow from sympy import Integer, Float from sympy import true, false from sympy import Tuple from sympy import Lambda from sympy import Atom from sympy import Expr from sympy import Dict from sympy import Not from sympy import cache errors = Errors() #============================================================================== strip_ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]|[\n\t\r]') redbaron.ipython_behavior = False # use this to delete ansi_escape characters from a string # Useful for very coarse version differentiation. #============================================================================== from pyccel.parser.base import BasicParser from pyccel.parser.base import is_ignored_module
def execute_pyccel(filename,
compiler=None,
fflags=None,
debug=False,
verbose=False,
accelerator=None,
include=[],
libdir=[],
modules=[],
libs=[],
binary=None,
output='',
convert_only=False,
return_ast=False):
"""Executes the full process:
- parsing the python code
- annotating the python code
- converting from python to fortran
- compiling the fortran code.
"""
pyccel = Parser(filename, output_folder=output.replace('/', '.'))
ast = pyccel.parse()
settings = {}
ast = pyccel.annotate(**settings)
name = os.path.basename(filename)
name = os.path.splitext(name)[0]
codegen = Codegen(ast, name)
code = codegen.doprint()
#S.H we return the Codegen instance instead of the ast
if convert_only:
if not return_ast:
return code
else:
return code, codegen
else:
fname = os.path.join(output, name)
fname = codegen.export(fname)
# reset Errors singleton
errors = Errors()
errors.reset()
# ... constructs the compiler flags
if compiler is None:
compiler = 'gfortran'
flags = construct_flags(compiler,
fflags=fflags,
debug=debug,
accelerator=accelerator,
include=include,
libdir=libdir)
# ...
# ... compile fortran code
output, cmd = compile_fortran(fname,
compiler,
flags,
binary=binary,
verbose=verbose,
modules=modules,
is_module=codegen.is_module,
output=output,
libs=libs)
# ...
if not return_ast:
return output, cmd
else:
return output, cmd, codegen
def epyccel_old(func,
inputs=None,
verbose=False,
modules=[],
libs=[],
libdirs=[],
name=None,
compiler=None,
mpi=False,
static=None,
only=None,
openmp=False):
"""Pyccelize a python function and wrap it using f2py.
func: function, str
a Python function or source code defining the function
inputs: str, list, tuple, dict
inputs can be the function header as a string, or a list/tuple of
strings or the globals() dictionary
verbose: bool
talk more
modules: list, tuple
list of dependencies
libs: list, tuple
list of libraries
libdirs: list, tuple
list of paths for libraries
name: str
name of the function, if it is given as a string
static: list/tuple
a list of 'static' functions as strings
only: list/tuple
a list of what should be exposed by f2py as strings
openmp: bool
True if openmp is used. Note that in this case, one need to use nogil
from cython
Examples
The following example shows how to use Pyccel within an IPython session
>>> #$ header procedure static f_static(int [:]) results(int)
>>> def f_static(x):
>>> y = x[0] - 1
>>> return y
>>> from test_epyccel import epyccel
>>> f = epyccel(f_static, globals()) # appending IPython history
>>> header = '#$ header procedure static f_static(int [:]) results(int)'
>>> f = epyccel(f_static, header) # giving the header explicitly
Now, **f** is a Fortran function that has been wrapped. It is compatible
with numpy and you can call it
>>> import numpy as np
>>> x = np.array([3, 4, 5, 6], dtype=int)
>>> y = f(x)
You can also call it with a list instead of numpy arrays
>>> f([3, 4, 5])
2
"""
if compiler is None:
compiler = 'gfortran'
is_module = False
is_function = False
if isinstance(func, ModuleType):
is_module = True
if callable(func):
is_function = True
assert (callable(func) or isinstance(func, str)
or isinstance(func, ModuleType))
# ...
if callable(func) or isinstance(func, ModuleType):
name = func.__name__
elif name is None:
# case of func as a string
raise ValueError(
'function name must be provided, in the case of func string')
# ...
output_folder = name.rsplit('.', 1)[0] if '.' in name else ''
# fortran module name
modname = 'epyccel__' + name
# ...
if is_module:
mod = func
is_sharedlib = isinstance(getattr(mod, '__loader__', None),
ExtensionFileLoader)
if is_sharedlib:
module_filename = inspect.getfile(mod)
# clean
cmd = 'rm -f {}'.format(module_filename)
output = subprocess.check_output(cmd, shell=True)
# then re-run again
mod = importlib.import_module(name)
# we must reload the module, otherwise it is still the .so one
importlib.reload(mod)
epyccel_old(mod,
inputs=inputs,
verbose=verbose,
modules=modules,
libs=libs,
name=name,
compiler=compiler,
mpi=mpi,
static=static,
only=only,
openmp=openmp)
# ...
# ...
ignored_funcs = None
if not static:
if isinstance(func, ModuleType):
mod = func
funcs = [
i for i in dir(mod)
if isinstance(getattr(mod, i), FunctionType)
]
# remove pyccel.decorators
ignored_funcs = [
i for i in funcs
if getattr(mod, i).__module__ == 'pyccel.decorators'
]
static = [i for i in funcs if not (i in ignored_funcs)]
else:
static = [name]
# ...
# ...
headers = None
if inputs:
if isinstance(inputs, str):
headers = inputs
elif isinstance(inputs, (tuple, list)):
# find all possible headers
lines = [
str(i) for i in inputs
if (isinstance(i, str) and i.lstrip().startswith('#$ header'))
]
# TODO take the last occurence for f => use zip
headers = "\n".join([str(i) for i in lines])
elif isinstance(inputs, dict):
# case of globals() history from ipython
if not 'In' in inputs.keys():
raise ValueError('Expecting `In` key in the inputs dictionary')
inputs = inputs['In']
# TODO shall we reverse the list
# find all possible headers
lines = [
str(i) for i in inputs if i.lstrip().startswith('#$ header')
]
# TODO take the last occurence for f => use zip
headers = "\n".join([str(i) for i in lines])
# we parse all headers then convert them to static function
d_headers = {}
if headers:
hdr = parse(stmts=headers)
if isinstance(hdr, FunctionHeader):
header = hdr.to_static()
d_headers = {str(name): header}
elif isinstance(hdr, (tuple, list)):
hs = [h.to_static() for h in hdr]
hs = [h for h in hs if hs.func == name]
# TODO improve
header = hs[0]
raise NotImplementedError('TODO')
else:
raise NotImplementedError('TODO')
# ...
# ...
if not static:
raise NotImplementedError('TODO')
# ...
# ... get the function source code
if callable(func):
code = get_source_function(func)
elif isinstance(func, ModuleType):
lines = inspect.getsourcelines(func)[0]
code = ''.join(lines)
else:
code = func
# ...
if verbose:
print('------')
print(code)
print('------')
extra_args = ''
include_args = ''
if libdirs:
libdirs = ['-L{}'.format(i) for i in libdirs]
extra_args += ' '.join(i for i in libdirs)
try:
# ...
pyccel = Parser(code,
headers=d_headers,
static=static,
output_folder=output_folder)
ast = pyccel.parse()
settings = {}
ast = pyccel.annotate(**settings)
codegen = Codegen(ast, modname)
code = codegen.doprint()
# ...
# reset Errors singleton
errors = Errors()
errors.reset()
except:
# reset Errors singleton
errors = Errors()
errors.reset()
raise PyccelError('Could not convert to Fortran')
# Change module name to avoid name clashes: Python cannot import two modules with the same name
if is_module:
head, sep, tail = name.rpartition('.')
name = sep.join([head, '__epyccel__' + tail])
else:
name = '__epyccel__' + name
# Find directory where Fortran extension module should be created
if is_module:
dirname = os.path.dirname(os.path.abspath(mod.__file__))
else:
dirname = os.path.dirname(
os.path.abspath(sys.modules[func.__module__].__file__))
# Move into working directory, create extension module, then move back to original directory
origin = os.path.abspath(os.curdir)
os.chdir(dirname)
output, cmd = compile_fortran(code,
name,
extra_args=extra_args,
libs=libs,
compiler=compiler,
mpi=mpi,
openmp=openmp,
includes=include_args,
only=only)
os.chdir(origin)
if verbose:
print(cmd)
if verbose:
print(code)
# ...
# ...
try:
if PY_VERSION == (3, 7):
dirname = os.path.relpath(dirname).replace('/', '.')
package = dirname + '.' + name
package = importlib.import_module('..' + name, package=package)
clean_extension_module(package, modname)
else:
os.chdir(dirname)
package = importlib.import_module(name)
clean_extension_module(package, modname)
os.chdir(origin)
except:
raise ImportError('could not import {0}'.format(name))
# ...
if is_module:
return package
else:
return getattr(package, func.__name__.lower())