def opt_invocation(self, opt):
self.attrs['invocation']['triples'] = []
for line in opt:
for invocation in line.split(','):
t = invocation.split(':')
if len(t) != 3:
raise UserException(
'Wrong invocation syntax: expected <mpi ranks>:<openmp numbers>:invocations but used %s'
% invocation)
triple = []
for pair in t:
r = pair.split('-')
if len(r) == 1:
triple.append((r[0], r[0]))
elif len(r) == 2:
triple.append(r)
else:
raise UserException(
'Wrong invocation syntax: expected a single number or "number-number" format but used %s'
% pair)
try:
int(triple[2][0])
int(triple[2][1])
except:
raise UserException(
'The last item in invocation triple should be number.')
self.attrs['invocation']['triples'].append(triple)
if not self.attrs['invocation']['triples']:
self.attrs['invocation']['triples'] = [(('0', '0'), ('0', '0'),
('0', '0'))]
def opt_timing(self, opt):
for time in opt.split(','):
key, value = time.split('=', 1)
if key in [ 'repeat' ] :
try:
self.attrs['timing'][key] = value
except:
raise UserException('repeat sub-flag should be integer value: %s'%value)
else:
raise UserException('Unknown timing option: %s' % time)
def handle_include(self, lines):
import re
import os
insert_lines = []
for i, line in enumerate(lines):
match = re.match(r'^\s*include\s*("[^"]+"|\'[^\']+\')\s*\Z', line,
re.I)
#if not match:
# match = re.match(r'\s*#include\s*("[^"]+"|\<[^\']+\>)\s*\Z', line, re.I)
if match:
if Config.include['file'].has_key(self.abspath):
include_dirs = Config.include['file'][
self.abspath]['path'] + Config.include['path']
else:
include_dirs = Config.include['path']
filename = match.group(1)[1:-1].strip()
path = filename
for incl_dir in include_dirs + [os.path.dirname(self.abspath)]:
path = os.path.join(incl_dir, filename)
if os.path.exists(path):
break
if os.path.isfile(path):
with open(path, 'r') as f:
included_lines = f.read()
insert_lines.extend(
self.handle_include(included_lines.split('\n')))
else:
raise UserException(
'Can not find %s in include paths of %s.' %
(filename, self.abspath))
else:
insert_lines.append(line)
return insert_lines
def analyze_callsite():
from block_statements import EndStatement, Subroutine, Function, Interface
from statements import SpecificBinding
from kgen_search import f2003_search_unknowns
# read source file that contains callsite stmt
cs_file = SrcFile(Config.callsite['filepath'])
#process_directive(cs_file.tree)
if len(State.callsite['stmts']) == 0:
raise UserException('Can not find callsite')
# ancestors of callsite stmt
ancs = State.callsite['stmts'][0].ancestors()
# add geninfo for ancestors
prevstmt = State.callsite['stmts'][0]
prevname = None
for anc in reversed(ancs):
if not hasattr(anc, 'geninfo'):
anc.geninfo = OrderedDict()
if len(anc.content)>0 and isinstance(anc.content[-1], EndStatement) and \
not hasattr(anc.content[-1], 'geninfo'):
anc.content[-1].geninfo = OrderedDict()
if prevname:
dummy_req = ResState(KGGenType.STATE_IN, KGName(prevname), None,
[anc])
dummy_req.res_stmts = [prevstmt]
anc.check_spec_stmts(dummy_req.uname, dummy_req)
if hasattr(anc, 'name'): prevname = anc.name
else: prevname = None
prevstmt = anc
# populate parent block parameters
State.parentblock['stmt'] = ancs[-1]
# populate top block parameters
State.topblock['stmt'] = ancs[0]
for cs_stmt in State.callsite['stmts']:
#resolve cs_stmt
f2003_search_unknowns(cs_stmt, cs_stmt.f2003)
for uname, req in cs_stmt.unknowns.iteritems():
cs_stmt.resolve(req)
if not req.res_stmts:
raise ProgramException('Resolution fail.')
# update state info of callsite and its upper blocks
update_state_info(State.parentblock['stmt'])
# update state info of modules
for modname, moddict in State.modules.iteritems():
modstmt = moddict['stmt']
if modstmt != State.topblock['stmt']:
update_state_info(moddict['stmt'])
def opt_state_build(self, opt):
for line in opt:
for build in line.split(','):
key, value = build.split('=', 1)
if key in [ 'cmds' ] :
self.attrs['state_build'][key] = value
else:
raise UserException('Unknown state-build option: %s' % build)
def opt_prerun(self, opt):
for line in opt:
for comp in line.split(','):
key, value = comp.split('=', 1)
if key in [ 'clean', 'build', 'run', 'kernel_build', 'kernel_run' ] :
self.attrs['prerun'][key] = value
else:
raise UserException('Unknown prerun option: %s' % comp)
def opt_state_run(self, opt):
for line in opt:
for run in line.split(','):
key, value = run.split('=', 1)
if key in ['cmds']:
self.attrs['state_run'][key] = value
else:
raise UserException('Unknown state-run option: %s' % run)
def opt_kernel_compile(self, opt):
for line in opt:
for comp in line.split(','):
key, value = comp.split('=', 1)
if key in ['FC', 'FC_FLAGS', 'PRERUN']:
self.attrs['kernel_compile'][key] = value
else:
raise UserException('Unknown kernel compile option: %s' %
comp)
def opt_openmp(self, opt):
self.attrs['openmp']['enabled'] = True
for line in opt:
for openmp in line.split(','):
if openmp == 'enable':
pass
else:
# key, value = openmp.split('=')
# if key=='enabled':
# pass
# else:
raise UserException('Unknown OpenMP option: %s' % openmp)
def opt_kernel_option(self, opt):
for line in opt:
for kopt in line.split(','):
split_kopt = kopt.split('=', 1)
if len(split_kopt)==1:
self.attrs['kernel_option']['compiler']['add'][split_kopt[0]] = None
elif len(split_kopt)==2:
if split_kopt[1] in [ 'FC', 'FC_FLAGS' ]:
self.attrs['kernel_option'][split_kopt[1]] = split_kopt[0]
elif split_kopt[1] in [ 'add', 'remove' ]:
self.attrs['kernel_option']['compiler'][split_kopt[1]].append(split_kopt[0])
elif split_kopt[1]=='link':
self.attrs['kernel_option']['linker']['add'].append(split_kopt[0])
else:
raise UserException('Unknown state-switch option: %s' % run)
def opt_openmp(self, opt):
self.attrs['openmp']['enabled'] = True
for line in opt:
for openmp in line.split(','):
if openmp=='enable':
pass
else:
key, value = openmp.split('=')
if key=='kernel-in-critical-region':
if value=='no':
self.attrs['openmp']['critical'] = False
elif key=='omp_num_threads':
if isinstance(value, str) and value.isdigit():
self.attrs['openmp']['maxnum_threads'] = int(value)
else:
raise UserException('Unknown OpenMP option: %s' % openmp)
def opt_mpi(self, opt):
self.attrs['mpi']['enabled'] = True
for line in opt:
for mpi in line.split(','):
if mpi=='enable':
pass
else:
key, value = mpi.split('=', 1)
if key=='comm':
self.attrs['mpi'][key] = value
elif key=='use':
mod_name, identifier = value.split(':')
self.attrs['mpi']['use_stmts'].append((mod_name, [identifier]))
elif key=='ranks':
print 'ranks subflag for mpi is not supported. Please use invocation flag instead'
sys.exit(-1)
#self.attrs['mpi'][key] = value.split(':')
#self.attrs['mpi']['size'] = len(self.attrs['mpi'][key])
elif key=='header':
self.attrs['mpi'][key] = value
else:
raise UserException('Unknown MPI option: %s' % mpi)
def check_mode():
from kgen_utils import Config, exec_cmd
from utils import module_file_extensions
from api import parse, walk
from statements import Comment
from kgen_search import f2003_search_unknowns, SearchException
import logging
logger = logging.getLogger('kgen') # KGEN addition
logger.setLevel(logging.WARNING)
files = []
# collect source files
for path in Config.check_mode:
if os.path.basename(path).startswith('.'): continue
if os.path.isdir(path):
for root, dirnames, filenames in os.walk(os.path.abspath(path)):
for filename in filenames:
if os.path.basename(filename).startswith('.'): continue
fname, fext = os.path.splitext(filename)
if len(fext) > 1 and fext.lower(
) in module_file_extensions:
files.append(os.path.join(root, filename))
elif os.path.isfile(path):
if os.path.isfile(path):
files.append(os.path.abspath(path))
else:
raise '%s is not a direcotory nor a file' % path
# TODO: support #include cpp directive
# parse source files
for n, file in enumerate(files):
print 'Reading(%d/%d): ' % (n + 1, len(files)), file
# fsrc = open(file, 'rb')
# prepare include paths and macro definitions
path_src = []
macros_src = []
if Config.include['file'].has_key(self.abspath):
path_src = Config.include['file'][self.abspath]['path'] + [
os.path.dirname(self.abspath)
]
for k, v in Config.include['file'][
self.abspath]['macro'].iteritems():
if v:
macros_src.append('-D%s=%s' % (k, v))
else:
macros_src.append('-D%s' % k)
includes = '-I' + ' -I'.join(Config.include['path'] + path_src)
macros_common = []
for k, v in Config.include['macro'].iteritems():
if v:
macros_common.append('-D%s=%s' % (k, v))
else:
macros_common.append('-D%s' % k)
macros = ' '.join(macros_common + macros_src)
# execute preprocessing
prep = Config.bin['pp']
if prep.endswith('fpp'): flags = Config.bin['fpp_flags']
elif prep.endswith('cpp'): flags = Config.bin['cpp_flags']
else: raise UserException('Preprocessor is not either fpp or cpp')
output = exec_cmd('%s %s %s %s %s' %
(prep, flags, includes, macros, file))
# convert the preprocessed for fparser
prep = map(lambda l: '!KGEN' + l if l.startswith('#') else l,
output.split('\n'))
# fparse
tree = parse('\n'.join(prep), ignore_comments=False, analyze=False, isfree=True, isstrict=False, \
include_dirs=None, source_only=None )
# parse f2003
Config.search['promote_exception'] = True
lineno = 0
linediff = 0
for stmt, depth in walk(tree, -1):
try:
if isinstance(
stmt,
Comment) and stmt.item.comment.startswith('!KGEN#'):
comment_split = stmt.item.comment.split(' ')
lineno = int(comment_split[1])
stmt.item.span = (0, 0)
else:
if lineno > 0:
linediff = stmt.item.span[0] - lineno
lineno = 0
stmt.item.span = (stmt.item.span[0] - linediff,
stmt.item.span[1] - linediff)
stmt.parse_f2003()
if stmt.f2003.__class__ not in exclude_list:
f2003_search_unknowns(stmt,
stmt.f2003,
gentype=KGGenType.KERNEL)
except (NoMatchError, AttributeError) as e:
if file not in not_parsed:
not_parsed[file] = []
not_parsed[file].append(stmt)
except NameError as e:
errmsg = str(e)
pos = errmsg.find('search_')
if len(errmsg) > 7 and pos > 0:
clsname = errmsg[pos + 7:-16]
#print "NOT SUPPORTED: '%s' Fortran statement is not supported yet"%clsname
if file not in not_supported:
not_supported[file] = []
not_supported[file].append((clsname, stmt.item.span[0]))
except Exception as e:
print 'WARNING: Following statement is not correctly parsed'
print stmt
print ''
print ''
print '********************'
print '*** CHECK RESULT ***'
print '********************'
print ''
print 'NOTE: KGEN may be able to extract kernel even though not all source code lines are parsed or supported.'
print ''
print '*** KGEN Parsing Error(s) ***'
print ''
for file, stmts in not_parsed.iteritems():
print file
lines = []
for stmt in stmts:
if hasattr(stmt, 'item'):
lines.append('Near line # %d:' % stmt.item.span[0])
lines.append(stmt.tokgen() + '\n')
else:
lines.append(str(stmt) + '\n')
print '\n'.join(lines), '\n'
print '*** Not Supported Fortran Statement(s) ***'
print ''
for file, clsnames in not_supported.iteritems():
print file
lines = []
for clsname, lineno in clsnames:
lines.append("'%s' Fortran statment near line # %d" %
(clsname, lineno))
print '\n'.join(lines), '\n'
if len(not_parsed) == 0 and len(not_supported) == 0:
print 'Current KGEN version can support all source code lines.'
def preprocess():
from kgen_state import SrcFile
if Config.check_mode:
check_mode()
sys.exit(0)
else:
if Config.mpi['enabled']:
# get path of mpif.h
mpifpath = ''
if os.path.isabs(Config.mpi['header']):
if os.path.exists(Config.mpi['header']):
mpifpath = Config.mpi['header']
else:
raise UserException('Can not find %s' %
Config.mpi['header'])
else:
for p in Config.include['path']: # KGEN addition
fp = os.path.join(p, Config.mpi['header'])
if os.path.exists(fp):
mpifpath = fp
break
# collect required information
if mpifpath:
try:
reader = FortranFileReader(
mpifpath, include_dirs=Config.include['path'])
spec = Specification_Part(reader)
bag = {}
if Config.mpi['comm'] is None:
bag['key'] = 'MPI_COMM_WORLD'
bag[bag['key']] = []
traverse(spec, get_MPI_PARAM, bag, subnode='content')
if bag.has_key(bag['key']):
Config.mpi['comm'] = bag[bag['key']][-1]
else:
raise UserException(
'Can not find MPI_COMM_WORLD in mpif.h')
if Config.mpi['logical'] is None:
bag['key'] = 'MPI_LOGICAL'
bag[bag['key']] = []
traverse(spec, get_MPI_PARAM, bag, subnode='content')
if bag.has_key(bag['key']):
Config.mpi['logical'] = bag[bag['key']][-1]
else:
raise UserException(
'Can not find MPI_LOGICAL in mpif.h')
if Config.mpi['status_size'] is None:
bag['key'] = 'MPI_STATUS_SIZE'
bag[bag['key']] = []
traverse(spec, get_MPI_PARAM, bag, subnode='content')
if bag.has_key(bag['key']):
Config.mpi['status_size'] = bag[bag['key']][-1]
else:
raise UserException(
'Can not find MPI_STATUS_SIZE in mpif.h')
if Config.mpi['any_source'] is None:
bag['key'] = 'MPI_ANY_SOURCE'
bag[bag['key']] = []
traverse(spec, get_MPI_PARAM, bag, subnode='content')
if bag.has_key(bag['key']):
Config.mpi['any_source'] = bag[bag['key']][-1]
else:
raise UserException(
'Can not find MPI_ANY_SOURCE in mpif.h')
if Config.mpi['source'] is None:
bag['key'] = 'MPI_SOURCE'
bag[bag['key']] = []
traverse(spec, get_MPI_PARAM, bag, subnode='content')
if bag.has_key(bag['key']):
Config.mpi['source'] = bag[bag['key']][-1]
else:
raise UserException(
'Can not find MPI_SOURCE in mpif.h')
except Exception as e:
raise UserException('Error occurred during reading %s.' %
mpifpath)
else:
raise UserException(
'Can not find mpif.h. Please provide a path to the file')
# parse imported source files through include.ini
for path, import_type in Config.include['import'].iteritems():
if 'source' == import_type:
State.imported['source'].append(SrcFile(path))
def locate_callsite(cs_tree):
from statements import Comment
from block_statements import executable_construct
import re
def get_next_non_comment(stmt):
if not stmt: return
if not hasattr(stmt, 'parent'): return
started = False
for s in stmt.parent.content:
if s == stmt:
if not isinstance(s, Comment): return s
started = True
elif started:
if not isinstance(s, Comment): return s
def get_names(node, bag, depth):
from Fortran2003 import Name
if isinstance(node, Name) and not node.string in bag:
bag.append(node.string)
# collect directives
directs = []
for stmt, depth in walk(cs_tree):
if isinstance(stmt, Comment):
line = stmt.item.comment.strip()
match = re.match(r'^[c!*]\$kgen\s+(.+)$', line, re.IGNORECASE)
if match:
dsplit = match.group(1).split(' ', 1)
dname = dsplit[0].strip()
if len(dsplit) > 1: clause = dsplit[1].strip()
else: clause = None
if dname.startswith('begin_'):
sname = dname[6:]
directs.append(sname)
State.kernel['name'] = clause
elif dname.startswith('end_'):
ename = dname[4:]
if directs[-1] == ename:
directs.pop()
if ename == 'callsite':
pass
else:
raise UserException(
'WARNING: Not supported KGEN directive: %s' %
ename)
else:
raise UserException('Directive name mismatch: %s, %s' %
(dname_stack[-1], ename))
elif dname == 'callsite':
next_fort_stmt = get_next_non_comment(stmt)
if next_fort_stmt:
State.kernel['name'] = clause
State.callsite['stmts'].append(next_fort_stmt)
else:
raise UserException('WARNING: callsite is not found')
else:
if Config.callsite[
'namepath'] and stmt.__class__ in executable_construct:
names = []
traverse(stmt.f2003, get_names, names)
for name in names:
if match_namepath(Config.callsite['namepath'],
pack_exnamepath(stmt, name),
internal=False):
State.kernel['name'] = name
for _s, _d in walk(stmt):
State.callsite['stmts'].append(_s)
return
elif len(directs) > 0 and directs[-1] == 'callsite':
State.callsite['stmts'].append(stmt)
if len(State.callsite['stmts']) == 0:
raise UserException('Can not find callsite')
def preprocess():
from kgen_state import SrcFile
if Config.check_mode:
check_mode()
sys.exit(0)
else:
if Config.mpi['enabled']:
# get path of mpif.h
mpifpath = ''
if os.path.isabs(Config.mpi['header']):
if os.path.exists(Config.mpi['header']):
mpifpath = Config.mpi['header']
else:
raise UserException('Can not find %s'%Config.mpi['header'])
else:
for p in Config.include['path']:
fp = os.path.join(p, Config.mpi['header'])
if os.path.exists(fp):
mpifpath = fp
break
if not mpifpath:
for incpath, incdict in Config.include['file'].items():
for p in incdict['path']:
fp = os.path.join(p, Config.mpi['header'])
if os.path.exists(fp):
mpifpath = fp
break
if mpifpath: break
# collect required information
# TODO: FortranFileReader should be replaced with FortranStringReader after preprocessing
# TODO: include keyword should be handdled properly too.
if mpifpath:
try:
reader = FortranFileReader(mpifpath, include_dirs = Config.include['path'])
spec = Specification_Part(reader)
bag = {}
config_name_mapping = [
('comm', 'MPI_COMM_WORLD'),
('logical', 'MPI_LOGICAL'),
('status_size', 'MPI_STATUS_SIZE'),
('any_source', 'MPI_ANY_SOURCE'),
('source', 'MPI_SOURCE'),
]
for config_key, name in config_name_mapping:
if not Config.mpi.has_key(config_key) or Config.mpi[config_key] is None:
bag['key'] = name
bag[name] = []
traverse(spec, get_MPI_PARAM, bag, subnode='content')
if len(bag[name]) > 0:
Config.mpi[config_key] = bag[name][-1]
else:
raise UserException('Can not find {name} in mpif.h'.format(name=name))
except UserException:
raise # Reraise this exception rather than catching it below
except Exception as e:
raise UserException('Error occurred during reading %s.'%mpifpath)
else:
raise UserException('Can not find mpif.h. Please provide a path to the file')
# parse imported source files through include.ini
for path, import_type in Config.include['import'].iteritems():
if 'source'==import_type:
State.imported['source'].append(SrcFile(path))
def __init__(self, srcpath, preprocess=True):
import os.path
from kgen_utils import run_shcmd
from statements import Comment
from block_statements import Module, Program
# set default values
self.tree = None
self.srcpath = srcpath
self.abspath = os.path.abspath(self.srcpath)
# set source file format
isfree = True
isstrict = False
if self.abspath in Config.source['file'].keys():
if Config.source['file'][self.abspath].has_key('isfree'):
isfree = Config.source['file'][self.abspath]['isfree']
if Config.source['file'][self.abspath].has_key('isstrict'):
isstrict = Config.source['file'][self.abspath]['isstrict']
else:
if Config.source['isstrict']: isstrict = Config.source['isstrict']
if Config.source['isfree']: isfree = Config.source['isfree']
# prepare include paths and macro definitions
path_src = []
macros_src = []
if Config.include['file'].has_key(self.abspath):
path_src = Config.include['file'][self.abspath]['path'] + [
os.path.dirname(self.abspath)
]
path_src = [path for path in path_src if len(path) > 0]
for k, v in Config.include['file'][
self.abspath]['macro'].iteritems():
if v:
macros_src.append('-D%s=%s' % (k, v))
else:
macros_src.append('-D%s' % k)
includes = '-I' + ' -I'.join(Config.include['path'] + path_src)
macros_common = []
for k, v in Config.include['macro'].iteritems():
if v:
macros_common.append('-D%s=%s' % (k, v))
else:
macros_common.append('-D%s' % k)
macros = ' '.join(macros_common + macros_src)
# execute preprocessing
Logger.info('Reading %s' % self.srcpath, stdout=True)
new_lines = []
with open(self.abspath, 'r') as f:
if preprocess:
pp = Config.bin['pp']
if pp.endswith('fpp'):
if isfree: srcfmt = ' -free'
else: srcfmt = ' -fixed'
flags = Config.bin['fpp_flags'] + srcfmt
elif pp.endswith('cpp'):
flags = Config.bin['cpp_flags']
else:
raise UserException(
'Preprocessor is not either fpp or cpp')
output, err, retcode = run_shcmd('%s %s %s %s' %
(pp, flags, includes, macros),
input=f.read())
prep = map(lambda l: '!KGEN' + l if l.startswith('#') else l,
output.split('\n'))
new_lines = self.handle_include(prep)
else:
new_lines = f.read().split('\n')
# add include paths
if Config.include['file'].has_key(
self.abspath) and Config.include['file'][self.abspath].has_key(
'path'):
include_dirs = Config.include['file'][self.abspath]['path'] + [
os.path.dirname(self.abspath)
]
else:
include_dirs = None
# fparse
self.tree = parse('\n'.join(new_lines), ignore_comments=False, analyze=True, isfree=isfree, \
isstrict=isstrict, include_dirs=include_dirs, source_only=None )
self.tree.prep = new_lines
self.tree.used4genstate = False
# parse f2003
lineno = 0
linediff = 0
for stmt, depth in walk(self.tree, -1):
stmt.parse_f2003()
# rename reader.id
self.tree.reader.id = self.abspath
# collect module information
for mod_name, mod_stmt in self.tree.a.module.iteritems():
if not State.modules.has_key(mod_name):
State.modules[mod_name] = OrderedDict()
State.modules[mod_name]['stmt'] = mod_stmt
State.modules[mod_name]['file'] = self
State.modules[mod_name]['path'] = self.abspath
# collect program unit information
for item in self.tree.content:
if item.__class__ not in [Module, Comment, Program]:
if item.reader.id not in State.program_units.keys():
State.program_units[item.reader.id] = []
State.program_units[item.reader.id].append(item)
# create a tuple for file dependency
State.srcfiles[self.abspath] = (self, [], [])
self.process_directive()
