def execute_pyccel(fname,
*,
syntax_only=False,
semantic_only=False,
convert_only=False,
recursive=False,
verbose=False,
folder=None,
language=None,
compiler=None,
mpi_compiler=None,
fflags=None,
includes=(),
libdirs=(),
modules=(),
libs=(),
debug=False,
extra_args='',
accelerator=None,
output_name=None):
# Reset Errors singleton before parsing a new file
errors = Errors()
errors.reset()
# TODO [YG, 03.02.2020]: test validity of function arguments
# Copy list arguments to local lists to avoid unexpected behavior
includes = [*includes]
libdirs = [*libdirs]
modules = [*modules]
libs = [*libs]
# Store current directory and add it to sys.path
# variable to imitate Python's import behavior
base_dirpath = os.getcwd()
sys.path.insert(0, base_dirpath)
# Unified way to handle errors: print formatted error message, then move
# to original working directory. Caller should then raise exception.
def handle_error(stage):
print('\nERROR at {} stage'.format(stage))
errors.check()
os.chdir(base_dirpath)
# Identify absolute path, directory, and filename
pymod_filepath = os.path.abspath(fname)
pymod_dirpath, pymod_filename = os.path.split(pymod_filepath)
# Extract module name
module_name = os.path.splitext(pymod_filename)[0]
# Define working directory 'folder'
if folder is None or folder == "":
folder = pymod_dirpath
else:
folder = os.path.abspath(folder)
# Define directory name and path for pyccel & cpython build
pyccel_dirname = '__pyccel__'
pyccel_dirpath = os.path.join(folder, pyccel_dirname)
# Create new directories if not existing
os.makedirs(folder, exist_ok=True)
os.makedirs(pyccel_dirpath, exist_ok=True)
# Change working directory to 'folder'
os.chdir(folder)
if language is None:
language = 'fortran'
# Choose Fortran compiler
if compiler is None:
if language == 'fortran':
compiler = 'gfortran'
elif language == 'c':
compiler = 'gcc'
f90exec = mpi_compiler if mpi_compiler else compiler
if (language == "c"):
libs = libs + ['m']
if accelerator == 'openmp':
if compiler in ["gcc", "gfortran"]:
if sys.platform == "darwin" and compiler == "gcc":
libs = libs + ['omp']
else:
libs = libs + ['gomp']
elif compiler == 'ifort':
libs.append('iomp5')
# ...
# Construct flags for the Fortran compiler
if fflags is None:
fflags = construct_flags(f90exec,
fflags=None,
debug=debug,
accelerator=accelerator,
includes=())
# Build position-independent code, suited for use in shared library
fflags = ' {} -fPIC '.format(fflags)
# ...
# Parse Python file
try:
parser = Parser(pymod_filepath, show_traceback=verbose)
parser.parse(verbose=verbose)
except NotImplementedError as error:
msg = str(error)
errors.report(msg + '\n' + PYCCEL_RESTRICTION_TODO, severity='error')
except PyccelError:
handle_error('parsing (syntax)')
raise
if errors.has_errors():
handle_error('parsing (syntax)')
raise PyccelSyntaxError('Syntax step failed')
if syntax_only:
return
# Annotate abstract syntax Tree
try:
settings = {'verbose': verbose}
parser.annotate(**settings)
except NotImplementedError as error:
msg = str(error)
errors.report(msg + '\n' + PYCCEL_RESTRICTION_TODO, severity='error')
except PyccelError:
handle_error('annotation (semantic)')
# Raise a new error to avoid a large traceback
raise PyccelSemanticError('Semantic step failed') from None
if errors.has_errors():
handle_error('annotation (semantic)')
raise PyccelSemanticError('Semantic step failed')
if semantic_only:
return
if parser.module_parser:
parsers = [parser.module_parser, parser]
program_name = os.path.basename(os.path.splitext(parser.filename)[0])
module_names = [module_name, program_name]
else:
parsers = [parser]
module_names = [module_name]
for parser, module_name in zip(parsers, module_names):
semantic_parser = parser.semantic_parser
# Generate .f90 file
try:
codegen = Codegen(semantic_parser, module_name)
fname = os.path.join(pyccel_dirpath, module_name)
fname = codegen.export(fname, language=language)
except NotImplementedError as error:
msg = str(error)
errors.report(msg + '\n' + PYCCEL_RESTRICTION_TODO,
severity='error')
except PyccelError:
handle_error('code generation')
# Raise a new error to avoid a large traceback
raise PyccelCodegenError('Code generation failed') from None
if errors.has_errors():
handle_error('code generation')
raise PyccelCodegenError('Code generation failed')
#------------------------------------------------------
# TODO: collect dependencies and proceed recursively
# if recursive:
# for dep in parser.sons:
# # Call same function on 'dep'
# pass
#------------------------------------------------------
if convert_only:
continue
# ...
# Determine all .o files and all folders needed by executable
def get_module_dependencies(parser, mods=(), folders=()):
mod_folder = os.path.join(os.path.dirname(parser.filename),
"__pyccel__")
mod_base = os.path.splitext(os.path.basename(parser.filename))[0]
# Stop conditions
if parser.metavars.get('ignore_at_import', False) or \
parser.metavars.get('module_name', None) == 'omp_lib':
return mods, folders
# Update lists
mods = [*mods, os.path.join(mod_folder, mod_base)]
folders = [*folders, mod_folder]
# Proceed recursively
for son in parser.sons:
mods, folders = get_module_dependencies(son, mods, folders)
return mods, folders
dep_mods, inc_dirs = get_module_dependencies(parser)
# Remove duplicates without changing order
dep_mods = tuple(OrderedDict.fromkeys(dep_mods))
inc_dirs = tuple(OrderedDict.fromkeys(inc_dirs))
# ...
includes += inc_dirs
if codegen.is_program:
modules += [os.path.join(pyccel_dirpath, m) for m in dep_mods[1:]]
# Construct compiler flags
flags = construct_flags(f90exec,
fflags=fflags,
debug=debug,
accelerator=accelerator,
includes=includes)
# Compile Fortran code
#
# TODO: stop at object files, do not compile executable
# This allows for properly linking program to modules
#
try:
compile_files(fname,
f90exec,
flags,
binary=None,
verbose=verbose,
modules=modules,
is_module=codegen.is_module,
output=pyccel_dirpath,
libs=libs,
libdirs=libdirs,
language=language)
except Exception:
handle_error('Fortran compilation')
raise
# For a program stop here
if codegen.is_program:
if verbose:
exec_filepath = os.path.join(folder, module_name)
print(
'> Executable has been created: {}'.format(exec_filepath))
os.chdir(base_dirpath)
continue
# Create shared library
try:
sharedlib_filepath = create_shared_library(
codegen, language, pyccel_dirpath, compiler, mpi_compiler,
accelerator, dep_mods, libs, libdirs, includes, flags,
extra_args, output_name, verbose)
except NotImplementedError as error:
msg = str(error)
errors.report(msg + '\n' + PYCCEL_RESTRICTION_TODO,
severity='error')
handle_error('code generation (wrapping)')
raise PyccelCodegenError(msg) from None
except PyccelError:
handle_error('code generation (wrapping)')
raise
except Exception:
handle_error('shared library generation')
raise
if errors.has_errors():
handle_error('code generation (wrapping)')
raise PyccelCodegenError('Code generation failed')
# Move shared library to folder directory
# (First construct absolute path of target location)
sharedlib_filename = os.path.basename(sharedlib_filepath)
target = os.path.join(folder, sharedlib_filename)
shutil.move(sharedlib_filepath, target)
sharedlib_filepath = target
if verbose:
print('> Shared library has been created: {}'.format(
sharedlib_filepath))
# Print all warnings now
if errors.has_warnings():
errors.check()
# Change working directory back to starting point
os.chdir(base_dirpath)
def pyccel(files=None, openmp=None, openacc=None, output_dir=None, compiler=None):
"""
pyccel console command.
"""
parser = MyParser(description='pyccel command line')
parser.add_argument('files', metavar='N', type=str, nargs='+',
help='a Pyccel file')
#... Version
import pyccel
version = pyccel.__version__
libpath = pyccel.__path__[0]
python = 'python {}.{}'.format(*sys.version_info)
message = "pyccel {} from {} ({})".format(version, libpath, python)
parser.add_argument('-V', '--version', action='version', version=message)
# ...
# ... compiler syntax, semantic and codegen
group = parser.add_argument_group('Pyccel compiling stages')
group.add_argument('-x', '--syntax-only', action='store_true',
help='Using pyccel for Syntax Checking')
group.add_argument('-e', '--semantic-only', action='store_true',
help='Using pyccel for Semantic Checking')
group.add_argument('-t', '--convert-only', action='store_true',
help='Converts pyccel files only without build')
# group.add_argument('-f', '--f2py-compatible', action='store_true',
# help='Converts pyccel files to be compiled by f2py')
# ...
# ... backend compiler options
group = parser.add_argument_group('Backend compiler options')
group.add_argument('--language', choices=('fortran', 'c', 'python'), help='Generated language')
group.add_argument('--compiler', choices=('gfortran', 'ifort', 'pgfortran', \
'gcc', 'icc'), help='Compiler name')
group.add_argument('--mpi-compiler', help='MPI compiler wrapper')
group.add_argument('--flags', type=str, \
help='Compiler flags.')
group.add_argument('--debug', action='store_true', \
help='compiles the code in a debug mode.')
group.add_argument('--include',
type=str,
nargs='*',
dest='includes',
default=(),
help='list of include directories.')
group.add_argument('--libdir',
type=str,
nargs='*',
dest='libdirs',
default=(),
help='list of library directories.')
group.add_argument('--libs',
type=str,
nargs='*',
dest='libs',
default=(),
help='list of libraries to link with.')
group.add_argument('--output', type=str, default = '',\
help='folder in which the output is stored.')
# ...
# ... Accelerators
group = parser.add_argument_group('Accelerators options')
group.add_argument('--openmp', action='store_true', \
help='uses openmp')
group.add_argument('--openacc', action='store_true', \
help='uses openacc')
# ...
# ... Other options
group = parser.add_argument_group('Other options')
group.add_argument('--verbose', action='store_true', \
help='enables verbose mode.')
group.add_argument('--developer-mode', action='store_true', \
help='shows internal messages')
# ...
# TODO move to another cmd line
parser.add_argument('--analysis', action='store_true', \
help='enables code analysis mode.')
# ...
# ...
args = parser.parse_args()
# ...
# Imports
from pyccel.errors.errors import Errors, PyccelError
from pyccel.errors.errors import ErrorsMode
from pyccel.errors.messages import INVALID_FILE_DIRECTORY, INVALID_FILE_EXTENSION
from pyccel.codegen.pipeline import execute_pyccel
# ...
if not files:
files = args.files
if args.compiler:
compiler = args.compiler
if not openmp:
openmp = args.openmp
if not openacc:
openacc = args.openacc
if args.convert_only or args.syntax_only or args.semantic_only:
compiler = None
# ...
# ...
if len(files) > 1:
errors = Errors()
# severity is error to avoid needing to catch exception
errors.report('Pyccel can currently only handle 1 file at a time',
severity='error')
errors.check()
sys.exit(1)
# ...
filename = files[0]
# ... report error
if os.path.isfile(filename):
# we don't use is_valid_filename_py since it uses absolute path
# file extension
ext = filename.split('.')[-1]
if not(ext in ['py', 'pyh']):
errors = Errors()
# severity is error to avoid needing to catch exception
errors.report(INVALID_FILE_EXTENSION,
symbol=ext,
severity='error')
errors.check()
sys.exit(1)
else:
# we use Pyccel error manager, although we can do it in other ways
errors = Errors()
# severity is error to avoid needing to catch exception
errors.report(INVALID_FILE_DIRECTORY,
symbol=filename,
severity='error')
errors.check()
sys.exit(1)
# ...
if compiler:
if _which(compiler) is None:
errors = Errors()
# severity is error to avoid needing to catch exception
errors.report('Could not find compiler',
symbol=compiler,
severity='error')
errors.check()
sys.exit(1)
accelerator = None
if openmp:
accelerator = "openmp"
if openacc:
accelerator = "openacc"
# ...
# ...
if args.developer_mode:
# this will initialize the singelton ErrorsMode
# making this settings available everywhere
err_mode = ErrorsMode()
err_mode.set_mode('developer')
# ...
base_dirpath = os.getcwd()
try:
# TODO: prune options
execute_pyccel(filename,
syntax_only = args.syntax_only,
semantic_only = args.semantic_only,
convert_only = args.convert_only,
verbose = args.verbose,
language = args.language,
compiler = compiler,
mpi_compiler = args.mpi_compiler,
fflags = args.flags,
includes = args.includes,
libdirs = args.libdirs,
modules = (),
libs = args.libs,
debug = args.debug,
extra_args = '',
accelerator = accelerator,
folder = args.output)
except PyccelError:
sys.exit(1)
finally:
os.chdir(base_dirpath)
return
def execute_pyccel(fname,
*,
syntax_only=False,
semantic_only=False,
convert_only=False,
verbose=False,
folder=None,
language=None,
compiler=None,
mpi_compiler=None,
fflags=None,
includes=(),
libdirs=(),
modules=(),
libs=(),
debug=False,
accelerator=None,
output_name=None):
"""
Carries out the main steps required to execute pyccel
- Parses the python file (syntactic stage)
- Annotates the abstract syntax tree (semantic stage)
- Generates the translated file(s) (codegen stage)
- Compiles the files to generate an executable and/or a shared library
Parameters
----------
fname : str
Name of python file to be translated
syntax_only : bool
Boolean indicating whether the pipeline should stop
after the syntax stage
Default : False
semantic_only : bool
Boolean indicating whether the pipeline should stop
after the semantic stage
Default : False
convert_only : bool
Boolean indicating whether the pipeline should stop
after the codegen stage
Default : False
verbose : bool
Boolean indicating whether debugging messages should be printed
Default : False
folder : str
Path to the working directory
Default : folder containing the file to be translated
language : str
The language which pyccel is translating to
Default : fortran
compiler : str
The compiler used to compile the generated files
Default : GNU
mpi_compiler : str
The compiler used to compile the generated files when mpi is needed.
This value must be provided to compile with mpi
Default : None (compile with 'compiler')
fflags : str
The flags passed to the compiler
Default : provided by codegen.utilities.construct_flags
includes : list
list of include directories paths
libdirs : list
list of paths to directories containing the required libraries
modules : list
list of files which must also be compiled in order to compile this module
libs : list
list of required libraries
debug : bool
Boolean indicating whether the file should be compiled in debug mode
(currently this only implies that the flag -fcheck=bounds is added)
Default : False
accelerator : str
Tool used to accelerate the code (e.g. openmp openacc)
output_name : str
Name of the generated module
Default : Same name as the file which was translated
"""
# Reset Errors singleton before parsing a new file
errors = Errors()
errors.reset()
# TODO [YG, 03.02.2020]: test validity of function arguments
# Copy list arguments to local lists to avoid unexpected behavior
includes = [*includes]
libdirs = [*libdirs]
modules = [*modules]
libs = [*libs]
# Store current directory and add it to sys.path
# variable to imitate Python's import behavior
base_dirpath = os.getcwd()
sys.path.insert(0, base_dirpath)
# Unified way to handle errors: print formatted error message, then move
# to original working directory. Caller should then raise exception.
def handle_error(stage):
print('\nERROR at {} stage'.format(stage))
errors.check()
os.chdir(base_dirpath)
# Identify absolute path, directory, and filename
pymod_filepath = os.path.abspath(fname)
pymod_dirpath, pymod_filename = os.path.split(pymod_filepath)
# Extract module name
module_name = os.path.splitext(pymod_filename)[0]
# Define working directory 'folder'
if folder is None or folder == "":
folder = pymod_dirpath
else:
folder = os.path.abspath(folder)
# Define directory name and path for pyccel & cpython build
pyccel_dirname = '__pyccel__'
pyccel_dirpath = os.path.join(folder, pyccel_dirname)
# Create new directories if not existing
os.makedirs(folder, exist_ok=True)
os.makedirs(pyccel_dirpath, exist_ok=True)
# Change working directory to 'folder'
os.chdir(folder)
if language is None:
language = 'fortran'
# Choose Fortran compiler
if compiler is None:
if language == 'fortran':
compiler = 'gfortran'
elif language == 'c':
compiler = 'gcc'
f90exec = mpi_compiler if mpi_compiler else compiler
if (language == "c"):
libs = libs + ['m']
if accelerator == 'openmp':
if compiler in ["gcc", "gfortran"]:
if sys.platform == "darwin" and compiler == "gcc":
libs = libs + ['omp']
else:
libs = libs + ['gomp']
elif compiler == 'ifort':
libs.append('iomp5')
# ...
# Construct flags for the Fortran compiler
if fflags is None:
fflags = construct_flags(f90exec,
fflags=None,
debug=debug,
accelerator=accelerator,
includes=())
# Build position-independent code, suited for use in shared library
fflags = ' {} -fPIC '.format(fflags)
# ...
# Parse Python file
try:
parser = Parser(pymod_filepath, show_traceback=verbose)
parser.parse(verbose=verbose)
except NotImplementedError as error:
msg = str(error)
errors.report(msg + '\n' + PYCCEL_RESTRICTION_TODO, severity='error')
except PyccelError:
handle_error('parsing (syntax)')
raise
if errors.has_errors():
handle_error('parsing (syntax)')
raise PyccelSyntaxError('Syntax step failed')
if syntax_only:
return
# Annotate abstract syntax Tree
try:
settings = {'verbose': verbose}
parser.annotate(**settings)
except NotImplementedError as error:
msg = str(error)
errors.report(msg + '\n' + PYCCEL_RESTRICTION_TODO, severity='error')
except PyccelError:
handle_error('annotation (semantic)')
# Raise a new error to avoid a large traceback
raise PyccelSemanticError('Semantic step failed') from None
if errors.has_errors():
handle_error('annotation (semantic)')
raise PyccelSemanticError('Semantic step failed')
if semantic_only:
return
if parser.module_parser:
parsers = [parser.module_parser, parser]
program_name = os.path.basename(os.path.splitext(parser.filename)[0])
module_names = [module_name, program_name]
else:
parsers = [parser]
module_names = [module_name]
# -------------------------------------------------------------------------
# get path to pyccel/stdlib/lib_name
stdlib_path = os.path.dirname(stdlib_folder.__file__)
internal_libs_name = set()
internal_libs_path = []
internal_libs_files = []
for parser, module_name in zip(parsers, module_names):
semantic_parser = parser.semantic_parser
# Generate .f90 file
try:
codegen = Codegen(semantic_parser, module_name)
fname = os.path.join(pyccel_dirpath, module_name)
fname = codegen.export(fname, language=language)
except NotImplementedError as error:
msg = str(error)
errors.report(msg + '\n' + PYCCEL_RESTRICTION_TODO,
severity='error')
except PyccelError:
handle_error('code generation')
# Raise a new error to avoid a large traceback
raise PyccelCodegenError('Code generation failed') from None
if errors.has_errors():
handle_error('code generation')
raise PyccelCodegenError('Code generation failed')
#------------------------------------------------------
# TODO: collect dependencies and proceed recursively
# if recursive:
# for dep in parser.sons:
# # Call same function on 'dep'
# pass
#------------------------------------------------------
# Iterate over the internal_libs list and determine if the printer
# requires an internal lib to be included.
for lib in internal_libs:
if lib in codegen.get_printer_imports():
# get the include folder path and library files
if lib not in internal_libs_name:
# get the library folder name
lib_name = internal_libs[lib]
# get lib path (stdlib_path/lib_name)
lib_path = os.path.join(stdlib_path, lib_name)
# remove library folder to avoid missing files and copy
# new one from pyccel stdlib
lib_dest_path = os.path.join(pyccel_dirpath, lib_name)
if os.path.exists(lib_dest_path):
shutil.rmtree(lib_dest_path)
shutil.copytree(lib_path, lib_dest_path)
# stop after copying lib to __pyccel__ directory for
# convert only
if convert_only:
continue
# get library source files
source_files = []
for e in os.listdir(lib_dest_path):
if e.endswith(lang_ext_dict[language]):
source_files.append(os.path.join(lib_dest_path, e))
# compile library source files
flags = construct_flags(f90exec,
fflags=fflags,
debug=debug,
includes=[lib_dest_path])
try:
for f in source_files:
compile_files(f,
f90exec,
flags,
binary=None,
verbose=verbose,
is_module=True,
output=lib_dest_path,
language=language)
except Exception:
handle_error('C {} library compilation'.format(lib))
raise
# Add internal lib to internal_libs_name set
internal_libs_name.add(lib)
# add source file without extension to internal_libs_files
internal_libs_files.extend(
os.path.splitext(f)[0] for f in source_files)
# add library path to internal_libs_path
internal_libs_path.append(lib_dest_path)
if convert_only:
continue
# ...
# Determine all .o files and all folders needed by executable
def get_module_dependencies(parser, mods=(), folders=()):
mod_folder = os.path.join(os.path.dirname(parser.filename),
"__pyccel__")
mod_base = os.path.splitext(os.path.basename(parser.filename))[0]
# Stop conditions
if parser.metavars.get('ignore_at_import', False) or \
parser.metavars.get('module_name', None) == 'omp_lib':
return mods, folders
# Update lists
mods = [*mods, os.path.join(mod_folder, mod_base)]
folders = [*folders, mod_folder]
# Proceed recursively
for son in parser.sons:
mods, folders = get_module_dependencies(son, mods, folders)
return mods, folders
dep_mods, inc_dirs = get_module_dependencies(parser)
# Add internal dependencies
dep_mods = [*dep_mods, *internal_libs_files]
inc_dirs = [*inc_dirs, *internal_libs_path]
# Remove duplicates without changing order
dep_mods = tuple(OrderedDict.fromkeys(dep_mods))
inc_dirs = tuple(OrderedDict.fromkeys(inc_dirs))
# ...
includes += inc_dirs
if codegen.is_program:
modules += [os.path.join(pyccel_dirpath, m) for m in dep_mods[1:]]
# Construct compiler flags
flags = construct_flags(f90exec,
fflags=fflags,
debug=debug,
accelerator=accelerator,
includes=includes)
# Compile Fortran code
#
# TODO: stop at object files, do not compile executable
# This allows for properly linking program to modules
#
try:
compile_files(fname,
f90exec,
flags,
binary=None,
verbose=verbose,
modules=modules,
is_module=codegen.is_module,
output=pyccel_dirpath,
libs=libs,
libdirs=libdirs,
language=language)
except Exception:
handle_error('Fortran compilation')
raise
# For a program stop here
if codegen.is_program:
if verbose:
exec_filepath = os.path.join(folder, module_name)
print(
'> Executable has been created: {}'.format(exec_filepath))
os.chdir(base_dirpath)
continue
# Create shared library
try:
sharedlib_filepath = create_shared_library(
codegen, language, pyccel_dirpath, compiler, mpi_compiler,
accelerator, dep_mods, libs, libdirs, includes, flags,
output_name, verbose)
except NotImplementedError as error:
msg = str(error)
errors.report(msg + '\n' + PYCCEL_RESTRICTION_TODO,
severity='error')
handle_error('code generation (wrapping)')
raise PyccelCodegenError(msg) from None
except PyccelError:
handle_error('code generation (wrapping)')
raise
except Exception:
handle_error('shared library generation')
raise
if errors.has_errors():
handle_error('code generation (wrapping)')
raise PyccelCodegenError('Code generation failed')
# Move shared library to folder directory
# (First construct absolute path of target location)
sharedlib_filename = os.path.basename(sharedlib_filepath)
target = os.path.join(folder, sharedlib_filename)
shutil.move(sharedlib_filepath, target)
sharedlib_filepath = target
if verbose:
print('> Shared library has been created: {}'.format(
sharedlib_filepath))
# Print all warnings now
if errors.has_warnings():
errors.check()
# Change working directory back to starting point
os.chdir(base_dirpath)