def execute(self):
self._trees = []
self.genfiles = []
self.kernel_name = State.kernel_driver['name']
try:
print ''
self.initialize()
preprocess()
Logger.info('Pre-processing is done', stdout=True)
analyze()
Logger.info('Program is analyzed', stdout=True)
self.transform()
self.output()
postprocess()
Logger.info('Post-processing is done', stdout=True)
Logger.info('Completed.', stdout=True)
except UserException as e:
print 'ERROR: %s' % str(e)
Logger.info(e)
#Logger.critical(e)
except ProgramException as e:
Logger.critical(e)
except Exception as e:
Logger.critical(e)
finally:
pass
def fini(self):
import ConfigParser
Logger.info('KCoverage is finished.', stdout=True)
#return { 'invocation': '0:0:0' }
return {'invocation': ','.join(self.invocation)}
def output(self):
# generate source files from each node of the tree
for kfile, sfile, filepath in self.genfiles:
filename = os.path.basename(filepath)
self.set_indent('')
klines = kfile.tostring()
if klines is not None:
with open('%s/%s' % (Config.path['kernel'], filename),
'wb') as fd:
fd.write(klines)
if sfile.kgen_stmt.used4genstate:
self.set_indent('')
slines = sfile.tostring()
if slines is not None:
with open('%s/%s' % (Config.path['state'], filename),
'wb') as fd:
fd.write(slines)
with open('%s/%s.f90' % (Config.path['kernel'], self.kernel_name),
'wb') as fd:
self.set_indent('')
lines = self.driver.tostring()
if lines is not None: fd.write(lines)
# generate kgen_utils.f90 in kernel directory
self.generate_kgen_utils()
generate_makefiles()
Logger.info('Makefiles are generated', stdout=True)
def main(self):
Logger.info('Starting KCoverage', stdout=True)
if not os.path.exists(self.config.coverage['cwd']):
os.mkdir(self.config.coverage['cwd'])
coverage_srcfiles = []
# generate instrumented source files
# generate makefile
generate_makefile(coverage_srcfiles)
def output(self):
# generate source files from each node of the tree
for obj, filepath in self.genfiles:
filename = os.path.basename(filepath)
self.set_indent('')
lines = obj.tostring()
if lines is not None:
with open('%s/%s'%(Config.path['state'], filename), 'wb') as fd:
fd.write(lines)
generate_makefile()
Logger.info('Makefiles are generated', stdout=True)
def output(self):
# generate source files from each node of the tree
for obj, filepath in self.genfiles:
filename = os.path.basename(filepath)
self.set_indent('')
lines = obj.tostring()
if lines is not None:
with open('%s/%s' % (Config.path['state'], filename),
'wb') as fd:
fd.write(lines)
generate_makefile()
Logger.info('Makefiles are generated', stdout=True)
def fini(self):
kernel_files = []
state_files = []
# generate source files from each node of the tree
for kfile, sfile, filepath in self.genfiles:
filename = os.path.basename(filepath)
self.set_indent('')
klines = kfile.tostring()
if klines is not None:
klines = self.remove_multiblanklines(klines)
kernel_files.append(filename)
with open('%s/%s' % (Config.path['kernel'], filename),
'wb') as fd:
fd.write(klines)
if sfile.kgen_stmt.used4genstate:
self.set_indent('')
slines = sfile.tostring()
if slines is not None:
slines = self.remove_multiblanklines(slines)
state_files.append(filename)
with open('%s/%s' % (Config.path['state'], filename),
'wb') as fd:
fd.write(slines)
kernel_files.append(self.kernel_name)
with open('%s/%s.f90' % (Config.path['kernel'], self.kernel_name),
'wb') as fd:
self.set_indent('')
lines = self.driver.tostring()
if lines is not None:
lines = self.remove_multiblanklines(lines)
fd.write(lines)
Logger.info('Kernel generation and instrumentation is completed.',
stdout=True)
# generate kgen_utils.f90 in kernel directory
kernel_files.append('kgen_utils.f90')
self.generate_kgen_utils()
kernel_files.append(TPROF)
shutil.copyfile('%s/%s' % (KGEN_BASE, TPROF),
'%s/%s' % (Config.path['kernel'], TPROF))
kernel_files.append('Makefile')
state_files.append('Makefile')
generate_makefiles()
Logger.info('Makefiles are generated', stdout=True)
postprocess()
Logger.info('Post-processing is done', stdout=True)
Logger.info('KExt is finished.', stdout=True)
return {'kernel_files': kernel_files, 'state_files': state_files}
def fini(self):
kernel_files = []
state_files = []
# generate source files from each node of the tree
for kfile, sfile, filepath in self.genfiles:
filename = os.path.basename(filepath)
self.set_indent('')
klines = kfile.tostring()
if klines is not None:
klines = self.remove_multiblanklines(klines)
kernel_files.append(filename)
with open('%s/%s'%(Config.path['kernel'], filename), 'wb') as fd:
fd.write(klines)
if sfile.kgen_stmt.used4genstate:
self.set_indent('')
slines = sfile.tostring()
if slines is not None:
slines = self.remove_multiblanklines(slines)
state_files.append(filename)
with open('%s/%s'%(Config.path['state'], filename), 'wb') as fd:
fd.write(slines)
kernel_files.append(self.kernel_name)
with open('%s/%s.f90'%(Config.path['kernel'], self.kernel_name), 'wb') as fd:
self.set_indent('')
lines = self.driver.tostring()
if lines is not None:
lines = self.remove_multiblanklines(lines)
fd.write(lines)
Logger.info('Kernel generation and instrumentation is completed.', stdout=True)
# generate kgen_utils.f90 in kernel directory
kernel_files.append('kgen_utils.f90')
self.generate_kgen_utils()
kernel_files.append(TPROF)
shutil.copyfile('%s/%s'%(KGEN_BASE, TPROF), '%s/%s'%(Config.path['kernel'], TPROF))
kernel_files.append('Makefile')
state_files.append('Makefile')
generate_makefiles()
Logger.info('Makefiles are generated', stdout=True)
postprocess()
Logger.info('Post-processing is done', stdout=True)
Logger.info('KExt is finished.', stdout=True)
return { 'kernel_files': kernel_files, 'state_files': state_files }
def main(self):
Logger.info('Starting KOption', stdout=True)
cwd = self.config.build['cwd']
slog = os.path.join(cwd, self.config.strace)
if not os.path.exists(slog) or \
'all' in self.config.rebuild or 'strace' in self.config.rebuild:
Logger.info('Building application to collect strace log file: %s'%slog, stdout=True)
with open(SH%cwd, 'w') as f:
f.write(TEMP_SH%(self.config.prerun['clean'], self.config.build['clean'], \
self.config.prerun['build'], self.config.build['cmdline']))
st = os.stat(SH%cwd)
os.chmod(SH%cwd, st.st_mode | stat.S_IEXEC)
shcmds = 'strace -o %s -f -q -s 100000 -e trace=execve -v -- %s'%\
(self.config.strace, SH%cwd)
out, err, retcode = run_shcmd(shcmds, cwd=cwd)
#os.remove(SH%self.config.build['cwd'])
if retcode!=0:
#print 'CMDS: ', shcmds
#print 'RETCODE: ', retcode
#print 'STDOUT: ', out
#print 'STDERR: ', err
raise ProgramException('Application build command is failed: %s'%err)
else:
Logger.info('Reusing strace log file: %s'%self.config.strace, stdout=True)
def main():
preprocess()
Logger.info('Pre-processing is done', stdout=True)
locate_callsite()
Logger.info('Call-site location is found', stdout=True)
collect_kernel_info()
Logger.info('Kernel information is collected', stdout=True)
mark_generation_info()
Logger.info('Kernel generation information is marked')
generate_srcfiles()
Logger.info('Source files are generated', stdout=True)
# generate_state()
# Logger.info('Instrumented files are generated', stdout=True)
#
# generate_kernel()
# Logger.info('Kernel files are generated', stdout=True)
generate_makefiles()
Logger.info('Makefiles are generated', stdout=True)
postprocess()
Logger.info('Post-processing is done', stdout=True)
Logger.info('Completed.', stdout=True)
def main():
version = [0, 1, '0']
outdir = '.'
retval = 0
varid = 0
Logger.info('Starting simplify_ifcond', stdout=True)
try:
# option parser
parser = optparse.OptionParser(
version='simplify_ifcond version %d.%d.%s' % tuple(version))
# common options
parser.add_option("--outdir",
dest="outdir",
action='store',
type='string',
default='output',
help="path to create outputs")
parser.add_option("--add-ext",
dest="ext",
action='store',
type='string',
default=None,
help="File extensions to parse")
parser.add_option(
"-t",
"--threshold",
dest="threshold",
action='store',
type='int',
default=20,
help="Max number of identifiers if condition before simplifying.")
opts, args = parser.parse_args()
if len(args) < 1:
print 'ERROR: Target source folders are not provided.'
print 'Usage: simplify_ifcond [options] <target folder path[, target folder path, ...]>'
sys.exit(-1)
# create output directory
outpath = os.path.abspath(opts.outdir)
if not os.path.exists(outpath):
os.makedirs(outpath)
outsrcpath = '%s/src' % outpath
shutil.rmtree(outsrcpath)
os.makedirs(outsrcpath)
# walk through source directory tree
for srcdir in args:
abssrcpath = os.path.abspath(srcdir)
for dirName, subdirList, fileList in os.walk(abssrcpath):
relpath = os.path.relpath(dirName, start=abssrcpath)
outfilepath = '%s/%s' % (outsrcpath, relpath.replace('.', ''))
if not os.path.exists(outfilepath):
os.makedirs(outfilepath)
for srcfile in fileList:
if any(srcfile.endswith(ext) for ext in file_exts):
try:
# read source file
parsed = SrcFile(os.path.join(dirName, srcfile),
preprocess=False)
# create analysis container
parstmts = []
# anlyze
last_span = (1, 1)
for stmt, depth in api.walk(parsed.tree):
if isinstance(stmt, Comment):
if stmt.item.span[0] >= last_span[
0] and stmt.item.span[
1] <= last_span[1]:
stmt.ignore = True
elif isinstance(stmt, (IfThen, If, ElseIf)):
if ifcondcheck(stmt.f2003.items[0],
opts.threshold):
p = stmt.ancestors()[-1]
if p not in parstmts:
parstmts.append(p)
if not hasattr(p, 'simplify_ifstmts'):
p.simplify_ifstmts = []
if stmt not in p.simplify_ifstmts:
p.simplify_ifstmts.append(stmt)
last_span = stmt.item.span
# modify
for parstmt in parstmts:
# add openmp directive
add_specstmts(parstmt,
len(parstmt.simplify_ifstmts))
# simplify
for i, ifstmt in enumerate(
parstmt.simplify_ifstmts):
simplify(ifstmt, ifstmt.f2003.items[0], i)
# generate modified source files
if len(parstmts) > 0:
lines = []
for stmt, depth in api.walk(parsed.tree):
if hasattr(stmt, 'forced_lines'):
lines.extend(stmt.forced_lines)
elif not stmt.ignore:
start = stmt.item.span[0] - 1
end = stmt.item.span[1]
for line in stmt.top.prep[start:end]:
split = line.split()
if len(split) > 2 and split[
0].startswith(
'!KGEN#'
) and split[1].isdigit():
continue
lines.append(line)
with open(os.path.join(outfilepath, srcfile),
'w') as f:
print 'Generating %s\n' % outfilepath
f.write('\n'.join(lines))
f.write('\n')
except Exception as e:
#import pdb; pdb.set_trace()
raise
pass
# switch source files if directed
except UserException as e:
print 'ERROR: %s' % str(e)
Logger.info(e)
#Logger.critical(e)
retval = -1
except ProgramException as e:
Logger.critical(e)
retval = -1
except Exception as e:
Logger.critical(e)
retval = -1
finally:
pass
Logger.info('simplify_ifcond is finished.', stdout=True)
return retval
def main():
from compflag_tool import CompFlagDetect
from kext_tool import KExtTool
version = [ 0, 7, '1' ]
outdir = '.'
retval = 0
Logger.info('Starting KGen', stdout=True)
try:
# option parser
parser = optparse.OptionParser(version='KGEN version %d.%d.%s'%tuple(version))
# common options
parser.add_option("--outdir", dest="outdir", action='store', type='string', default=None, help="path to create outputs")
parser.add_option("--rebuild", dest="rebuild", action='append', type='string', default=None, help="force to rebuild")
parser.add_option("--prerun", dest="prerun", action='append', type='string', default=None, help="prerun commands")
parser.add_option("-i", "--include-ini", dest="include_ini", action='store', type='string', default=None, help="INI options")
parser.add_option("--debug", dest="debug", action='append', type='string', help=optparse.SUPPRESS_HELP)
# kapp options
parser.add_option("-c", "--cmd-clean", dest="cmd_clean", action='store', type='string', default=None, help="Linux command(s) to clean a target application build")
parser.add_option("-b", "--cmd-build", dest="cmd_build", action='store', type='string', default=None, help="Linux command(s) to build a target application")
parser.add_option("-r", "--cmd-run", dest="cmd_run", action='store', type='string', default=None, help="Linux command(s) to run a target application")
# compflag options
parser.add_option("--strace", dest="strace", action='store', type='string', default=None, help="strace options")
# kext options
parser.add_option("--invocation", dest="invocation", action='append', type='string', default=None, help="(process, thread, invocation) pairs of kernel for data collection")
parser.add_option("-e", "--exclude-ini", dest="exclude_ini", action='store', type='string', default=None, help="information excluded for analysis")
parser.add_option("--kernel-option", dest="kernel_option", action='append', type='string', default=None, help="compiler and linker options for Kgen-generated kernel")
parser.add_option("--openmp", dest="openmp", action='append', type='string', default=None, help="Specifying OpenMP options")
parser.add_option("--mpi", dest="mpi", action='append', type='string', default=None, help="MPI information for data collection")
parser.add_option("--timing", dest="timing", action='append', type='string', default=None, help="Timing measurement information")
parser.add_option("--intrinsic", dest="intrinsic", action='append', type='string', default=None, help="Specifying resolution for intrinsic procedures during searching")
parser.add_option("--check", dest="check", action='append', type='string', default=None, help="Kernel correctness check information")
parser.add_option("--verbose", dest="verbose", action='store', type='string', default=None, help="Set the verbose level for verification output")
parser.add_option("--add-mpi-frame", dest="add_mpi_frame", type='string', default=None, help='Add MPI frame codes in kernel_driver.')
parser.add_option("--source", dest="source", action='append', type='string', default=None, help="Setting source file related properties")
parser.add_option("--logging", dest="logging", action='append', type='string', help=optparse.SUPPRESS_HELP)
opts, args = parser.parse_args()
if len(args)<1:
print 'ERROR: Target source files is not provided.'
print 'Usage: kgen [options] <target file path[:namepath]> --cmd-clean <commands> --cmd-build <commands> --cmd-run <commands>'
sys.exit(-1)
if opts.cmd_clean is None:
print 'ERROR: No clean command is prvoided in command line. Please add --cmd-clean option.'
print 'Usage: kgen [options] <target file path[:namepath]> --cmd-clean <commands> --cmd-build <commands> --cmd-run <commands>'
sys.exit(-1)
if opts.cmd_build is None:
print 'ERROR: No build command is prvoided in command line. Please add --cmd-build option.'
print 'Usage: kgen [options] <target file path[:namepath]> --cmd-clean <commands> --cmd-build <commands> --cmd-run <commands>'
sys.exit(-1)
if opts.cmd_run is None:
print 'ERROR: No run command is prvoided in command line. Please add --cmd-run option.'
print 'Usage: kgen [options] <target file path[:namepath]> --cmd-clean <commands> --cmd-build <commands> --cmd-run <commands>'
sys.exit(-1)
kext_argv = []
compflag_argv = []
# collect common options
if opts.outdir:
kext_argv.append('--outdir')
kext_argv.append(opts.outdir)
compflag_argv.append('--build')
compflag_argv.append('cwd="%s",clean="%s"'%(opts.outdir, opts.cmd_clean))
outdir = opts.outdir
else:
kext_argv.append('--outdir')
kext_argv.append(os.getcwd())
compflag_argv.append('--build')
compflag_argv.append('cwd="%s",clean="%s"'%(os.getcwd(), opts.cmd_clean))
outdir = os.getcwd()
if opts.prerun:
compflag_argv.append('--prerun')
compflag_argv.extend(opts.prerun)
kext_argv.append('--prerun')
kext_argv.extend(opts.prerun)
if opts.rebuild:
compflag_argv.append('--rebuild')
compflag_argv.extend(opts.rebuild)
kext_argv.append('--rebuild')
kext_argv.extend(opts.rebuild)
if opts.include_ini:
compflag_argv.append('--include_ini')
compflag_argv.append(opts.include_ini)
kext_argv.append('--include-ini')
kext_argv.append(opts.include_ini)
if opts.debug:
compflag_argv.append('--debug')
compflag_argv.extend(opts.debug)
kext_argv.append('--debug')
kext_argv.extend(opts.debug)
# collect compflag options
if opts.strace:
compflag_argv.append('--strace')
compflag_argv.append(opts.strace)
compflag_argv.append(opts.cmd_build)
# collect kext options
if opts.invocation:
kext_argv.append('--invocation')
kext_argv.extend(opts.invocation)
if opts.exclude_ini:
kext_argv.append('--exclude-ini')
kext_argv.append(opts.exclude_ini)
if opts.source:
kext_argv.append('--source')
kext_argv.extend(opts.source)
if opts.mpi:
kext_argv.append('--mpi')
kext_argv.extend(opts.mpi)
if opts.openmp:
kext_argv.append('--openmp')
kext_argv.extend(opts.openmp)
if opts.timing:
kext_argv.append('--timing')
kext_argv.extend(opts.timing)
if opts.intrinsic:
kext_argv.append('--intrinsic')
kext_argv.extend(opts.intrinsic)
if opts.check:
kext_argv.append('--check')
kext_argv.extend(opts.check)
if opts.kernel_option:
kext_argv.append('--kernel-option')
kext_argv.extend(opts.kernel_option)
if opts.verbose:
kext_argv.append('--verbose')
kext_argv.append(opts.verbose)
if opts.add_mpi_frame:
kext_argv.append('--add-mpi-frame')
kext_argv.append(opts.add_mpi_frame)
if opts.logging:
kext_argv.append('--logging')
kext_argv.extend(opts.logging)
kext_argv.append('--state-clean')
kext_argv.append('cmds=%s'%opts.cmd_clean)
kext_argv.append('--state-build')
kext_argv.append('cmds=%s'%opts.cmd_build)
kext_argv.append('--state-run')
kext_argv.append('cmds=%s'%opts.cmd_run)
kext_argv.append(args[0])
# run compflag
compflag = CompFlagDetect()
compflag.init(argv=compflag_argv)
compflag.main()
flags = compflag.fini()
# run kext
kext = KExtTool()
kext.init()
kext_argv.extend( [ '-i', flags['incini'] ] )
Config.apply(argv=kext_argv)
kext.main()
extracts = kext.fini()
# extracts contain kernel files, state files
# parse rebuild option
is_rebuild = False
if opts.rebuild:
for r in opts.rebuild:
if isinstance(r, str):
subopts = r.split(',')
for subopt in subopts:
if subopt in [ 'all', 'state' ]:
is_rebuild = True
if is_rebuild: break
# check if state files exist
has_statefiles = False
if os.path.exists('%s/kernel'%outdir):
statefiles = glob.glob('%s/kernel/%s.*.*.*'%(outdir, State.kernel['name']))
if len(statefiles)>0:
has_statefiles = True
# generate state
if is_rebuild or not has_statefiles:
Logger.info('Generating state data files at %s/state.'%outdir, stdout=True)
out, err, retcode = run_shcmd('make', cwd='%s/state'%outdir)
if retcode != 0:
Logger.info('FAILED: %s'%err, stdout=True)
else:
Logger.info('Reusing state data files at %s/kernel'%outdir, stdout=True)
except UserException as e:
print 'ERROR: %s'%str(e)
Logger.info(e)
#Logger.critical(e)
retval = -1
except ProgramException as e:
Logger.critical(e)
retval = -1
except Exception as e:
Logger.critical(e)
retval = -1
finally:
if os.path.exists('%s/state/Makefile'%outdir):
out, err, retcode = run_shcmd('make recover_from_locals', cwd='%s/state'%outdir)
Logger.info('KGen is finished.', stdout=True)
return retval
def get_name_or_defer(stmt, node, resolvers, defer=True, gentype=None):
"""Select a name to be searched, or defer to lower level of nodes in AST.
Parameters
----------
stmt : F2PY parser statement object
Specify a statement object to be searched
node : F2PY Fortran2003 parser object
Specify an expression object to be searched
resolvers : A list of statement classes for resolver
Limits the classes of resolver
defer : bool
check if to search lower level of nodes in AST.
gentype : Type of state data (IN or OUT)
Specify the type of state data
Returns
-------
None
See also
--------
f2003_search_unknowns
get_name
defer
defer_names
"""
from kgen_utils import KGName, pack_innamepath, get_innamepath, match_namepath
from kgen_state import ResState
if node is None: return
# uncomment below line for debug
#print node.__class__, str(node)
if isinstance(node, Fortran2003.Name):
# # skip if intrinsic
# if node.string.lower() in Intrinsic_Procedures:
# if Config.search['skip_intrinsic'] and not is_except(node, stmt):
# if hasattr(node, 'parent') and not isinstance(node.parent, Fortran2003.Part_Ref) and \
# not (isinstance(node.parent, Fortran2003.Function_Reference) and node.string.lower()=='null') and \
# not (isinstance(node.parent, Fortran2003.Specific_Binding) and node.string.lower()=='null'):
# Logger.info('Intrinsic procedure name of "%s" is used for name resolution'% \
# (node.string.lower()), stdout=True)
# Logger.info('\tnear "%s"'% stmt.item.line, stdout=True)
# Logger.info('\tin %s'% stmt.reader.id, stdout=True)
# else:
# #if node.string.lower()!='null':
# # Logger.info('Intrinsic procedure name of "%s" is skipped from name resolution'% \
# # (node.string.lower()), stdout=True)
# #Logger.info('\tnear "%s"'% stmt.item.line, stdout=True)
# #Logger.info('\tin %s'% stmt.reader.id, stdout=True)
# return
#
# elif not Config.search['skip_intrinsic'] and is_except(node, stmt):
# if hasattr(node, 'parent') and not isinstance(node.parent, Fortran2003.Part_Ref) and \
# not (isinstance(node.parent, Fortran2003.Function_Reference) and node.string.lower()=='null') and \
# not (isinstance(node.parent, Fortran2003.Specific_Binding) and node.string.lower()=='null'):
# #Logger.info('Intrinsic procedure name of "%s" is NOT skipped from name resolution'% \
# # (node.string.lower()), stdout=True)
# #Logger.info('\tnear "%s"'% stmt.item.line, stdout=True)
# #Logger.info('\tin %s'% stmt.reader.id, stdout=True)
# pass
# else:
# if node.string.lower()!='null':
# Logger.info('Intrinsic procedure name of "%s" is skipped from name resolution'% \
# (node.string.lower()), stdout=True)
# Logger.info('\tnear "%s"'% stmt.item.line, stdout=True)
# Logger.info('\tin %s'% stmt.reader.id, stdout=True)
# return
# skip if excluded
#if Config.exclude.has_key('namepath') and stmt.__class__ in execution_part:
if Config.exclude.has_key('namepath'):
for pattern, actions in Config.exclude['namepath'].iteritems():
name = node.string.lower()
namepath = pack_innamepath(stmt, name)
#Logger.info('%s and %s are being checked for exclusion'%(pattern, namepath))
if match_namepath(pattern, namepath):
#Logger.info('%s and %s are mathched for exclusion'%(pattern, namepath))
if not hasattr(stmt, 'exclude_names'):
stmt.exclude_names = OrderedDict()
if stmt.exclude_names.has_key(name):
stmt.exclude_names[name].extend(actions)
else:
stmt.exclude_names[name] = actions
node.skip_search = True
if hasattr(node, 'parent'): node.parent.skip_search = True
return
ukey = KGName(pack_innamepath(stmt, node.string.lower()),
node=node,
stmt=stmt)
if gentype is None: gentype = KGGenType.STATE_IN
if resolvers is None:
stmt.unknowns[ukey] = ResState(gentype, ukey, stmt, res_default)
else:
stmt.unknowns[ukey] = ResState(gentype, ukey, stmt, resolvers)
Logger.info('%s is saved as unknown' % node.string.lower(),
name=ukey,
stmt=stmt)
elif defer:
f2003_search_unknowns(stmt, node, resolvers, gentype=gentype)
def get_name_or_defer(stmt, node, resolvers, defer=True, gentype=None):
from kgen_utils import KGName, pack_innamepath, get_innamepath, match_namepath
from kgen_state import ResState
if node is None: return
# uncomment below line for debug
#print node.__class__, str(node)
if isinstance(node, Fortran2003.Name):
# skip if intrinsic
if node.string.lower() in Intrinsic_Procedures:
if Config.search['skip_intrinsic'] and not is_except(node, stmt):
if hasattr(node, 'parent') and not isinstance(node.parent, Fortran2003.Part_Ref) and \
not (isinstance(node.parent, Fortran2003.Function_Reference) and node.string.lower()=='null') and \
not (isinstance(node.parent, Fortran2003.Specific_Binding) and node.string.lower()=='null'):
Logger.info('Intrinsic procedure name of "%s" is used for name resolution'% \
(node.string.lower()), stdout=True)
Logger.info('\tnear "%s"' % stmt.item.line, stdout=True)
Logger.info('\tin %s' % stmt.reader.id, stdout=True)
else:
#if node.string.lower()!='null':
# Logger.info('Intrinsic procedure name of "%s" is skipped from name resolution'% \
# (node.string.lower()), stdout=True)
#Logger.info('\tnear "%s"'% stmt.item.line, stdout=True)
#Logger.info('\tin %s'% stmt.reader.id, stdout=True)
return
elif not Config.search['skip_intrinsic'] and is_except(node, stmt):
if hasattr(node, 'parent') and not isinstance(node.parent, Fortran2003.Part_Ref) and \
not (isinstance(node.parent, Fortran2003.Function_Reference) and node.string.lower()=='null') and \
not (isinstance(node.parent, Fortran2003.Specific_Binding) and node.string.lower()=='null'):
#Logger.info('Intrinsic procedure name of "%s" is NOT skipped from name resolution'% \
# (node.string.lower()), stdout=True)
#Logger.info('\tnear "%s"'% stmt.item.line, stdout=True)
#Logger.info('\tin %s'% stmt.reader.id, stdout=True)
pass
else:
if node.string.lower() != 'null':
Logger.info('Intrinsic procedure name of "%s" is skipped from name resolution'% \
(node.string.lower()), stdout=True)
Logger.info('\tnear "%s"' % stmt.item.line, stdout=True)
Logger.info('\tin %s' % stmt.reader.id, stdout=True)
return
# skip if excluded
#if Config.exclude.has_key('namepath') and stmt.__class__ in execution_part:
if Config.exclude.has_key('namepath'):
for pattern, actions in Config.exclude['namepath'].iteritems():
name = node.string.lower()
namepath = pack_innamepath(stmt, name)
if match_namepath(pattern, namepath):
if not hasattr(stmt, 'exclude_names'):
stmt.exclude_names = OrderedDict()
if stmt.exclude_names.has_key(name):
stmt.exclude_names[name].extend(actions)
else:
stmt.exclude_names[name] = actions
node.skip_search = True
if hasattr(node, 'parent'): node.parent.skip_search = True
return
ukey = KGName(pack_innamepath(stmt, node.string.lower()),
node=node,
stmt=stmt)
if gentype is None: gentype = KGGenType.STATE_IN
if resolvers is None:
stmt.unknowns[ukey] = ResState(gentype, ukey, stmt, res_default)
else:
stmt.unknowns[ukey] = ResState(gentype, ukey, stmt, resolvers)
Logger.info('%s is saved as unknown' % node.string.lower(),
name=ukey,
stmt=stmt)
elif defer:
f2003_search_unknowns(stmt, node, resolvers, gentype=gentype)
def main(self):
Logger.info('Starting KExt', stdout=True)
# create state directories
if not os.path.exists(Config.path['state']):
os.makedirs(Config.path['state'])
# create kernel directories
if not os.path.exists(Config.path['kernel']):
os.makedirs(Config.path['kernel'])
os.system('rm -f %s/kgen_statefile.lst' % Config.path['kernel'])
os.system('rm -f %s/done.*' % Config.path['kernel'])
preprocess()
Logger.info('Pre-processing is done', stdout=True)
analyze()
Logger.info('Program is analyzed', stdout=True)
# generate kgen_driver.f90 in kernel directory
self.driver = self.create_tree()
program = self.create_program(self.driver)
program.name = self.kernel_name
self.append_program_in_tree(self.driver, program)
# init plugin framework
init_plugins([KERNEL_ID_0])
# construct a generation tree
for filepath, (srcobj, mods_used,
units_used) in State.srcfiles.iteritems():
if hasattr(srcobj.tree, 'geninfo') and KGGenType.has_state(
srcobj.tree.geninfo):
kfile = genkobj(None, srcobj.tree, KERNEL_ID_0)
sfile = gensobj(None, srcobj.tree, KERNEL_ID_0)
if kfile is None or sfile is None:
raise ProgramException(
'Kernel source file is not generated for %s.' %
filepath)
self.genfiles.append((kfile, sfile, filepath))
State.used_srcfiles[filepath] = (srcobj, mods_used, units_used)
# process each nodes in the tree
for plugin_name in event_register.keys():
for kfile, sfile, filepath in self.genfiles:
kfile.created([plugin_name])
sfile.created([plugin_name])
for tree in self._trees:
tree.created([plugin_name])
for kfile, sfile, filepath in self.genfiles:
kfile.process([plugin_name])
sfile.process([plugin_name])
for tree in self._trees:
tree.process([plugin_name])
for kfile, sfile, filepath in self.genfiles:
kfile.finalize([plugin_name])
sfile.finalize([plugin_name])
for tree in self._trees:
tree.finalize([plugin_name])
for kfile, sfile, filepath in self.genfiles:
kfile.flatten(KERNEL_ID_0, [plugin_name])
sfile.flatten(KERNEL_ID_0, [plugin_name])
for tree in self._trees:
tree.flatten(KERNEL_ID_0, [plugin_name])
def __init__(self, srcpath, preprocess=True):
import os.path
from kgen_utils import run_shcmd
from statements import Comment
from block_statements import Module, Program
# set default values
self.tree = None
self.srcpath = srcpath
self.abspath = os.path.abspath(self.srcpath)
# set source file format
isfree = True
isstrict = False
if self.abspath in Config.source['file'].keys():
if Config.source['file'][self.abspath].has_key('isfree'):
isfree = Config.source['file'][self.abspath]['isfree']
if Config.source['file'][self.abspath].has_key('isstrict'):
isstrict = Config.source['file'][self.abspath]['isstrict']
else:
if Config.source['isstrict']: isstrict = Config.source['isstrict']
if Config.source['isfree']: isfree = Config.source['isfree']
# prepare include paths and macro definitions
path_src = []
macros_src = []
if Config.include['file'].has_key(self.abspath):
path_src = Config.include['file'][self.abspath]['path'] + [
os.path.dirname(self.abspath)
]
path_src = [path for path in path_src if len(path) > 0]
for k, v in Config.include['file'][
self.abspath]['macro'].iteritems():
if v:
macros_src.append('-D%s=%s' % (k, v))
else:
macros_src.append('-D%s' % k)
includes = '-I' + ' -I'.join(Config.include['path'] + path_src)
macros_common = []
for k, v in Config.include['macro'].iteritems():
if v:
macros_common.append('-D%s=%s' % (k, v))
else:
macros_common.append('-D%s' % k)
macros = ' '.join(macros_common + macros_src)
# execute preprocessing
Logger.info('Reading %s' % self.srcpath, stdout=True)
new_lines = []
with open(self.abspath, 'r') as f:
if preprocess:
pp = Config.bin['pp']
if pp.endswith('fpp'):
if isfree: srcfmt = ' -free'
else: srcfmt = ' -fixed'
flags = Config.bin['fpp_flags'] + srcfmt
elif pp.endswith('cpp'):
flags = Config.bin['cpp_flags']
else:
raise UserException(
'Preprocessor is not either fpp or cpp')
output, err, retcode = run_shcmd('%s %s %s %s' %
(pp, flags, includes, macros),
input=f.read())
prep = map(lambda l: '!KGEN' + l if l.startswith('#') else l,
output.split('\n'))
new_lines = self.handle_include(prep)
else:
new_lines = f.read().split('\n')
# add include paths
if Config.include['file'].has_key(
self.abspath) and Config.include['file'][self.abspath].has_key(
'path'):
include_dirs = Config.include['file'][self.abspath]['path'] + [
os.path.dirname(self.abspath)
]
else:
include_dirs = None
# fparse
self.tree = parse('\n'.join(new_lines), ignore_comments=False, analyze=True, isfree=isfree, \
isstrict=isstrict, include_dirs=include_dirs, source_only=None )
self.tree.prep = new_lines
self.tree.used4genstate = False
# parse f2003
lineno = 0
linediff = 0
for stmt, depth in walk(self.tree, -1):
stmt.parse_f2003()
# rename reader.id
self.tree.reader.id = self.abspath
# collect module information
for mod_name, mod_stmt in self.tree.a.module.iteritems():
if not State.modules.has_key(mod_name):
State.modules[mod_name] = OrderedDict()
State.modules[mod_name]['stmt'] = mod_stmt
State.modules[mod_name]['file'] = self
State.modules[mod_name]['path'] = self.abspath
# collect program unit information
for item in self.tree.content:
if item.__class__ not in [Module, Comment, Program]:
if item.reader.id not in State.program_units.keys():
State.program_units[item.reader.id] = []
State.program_units[item.reader.id].append(item)
# create a tuple for file dependency
State.srcfiles[self.abspath] = (self, [], [])
self.process_directive()
def get_name_or_defer(stmt, node, resolvers, defer=True, gentype=None):
"""Select a name to be searched, or defer to lower level of nodes in AST.
Parameters
----------
stmt : F2PY parser statement object
Specify a statement object to be searched
node : F2PY Fortran2003 parser object
Specify an expression object to be searched
resolvers : A list of statement classes for resolver
Limits the classes of resolver
defer : bool
check if to search lower level of nodes in AST.
gentype : Type of state data (IN or OUT)
Specify the type of state data
Returns
-------
None
See also
--------
f2003_search_unknowns
get_name
defer
defer_names
"""
from kgen_utils import KGName, pack_innamepath, get_innamepath, match_namepath
from kgen_state import ResState
if node is None: return
# uncomment below line for debug
#print node.__class__, str(node)
if isinstance(node, Fortran2003.Name):
# skip if intrinsic
if node.string.lower() in Intrinsic_Procedures:
if Config.search['skip_intrinsic'] and not is_except(node, stmt):
if hasattr(node, 'parent') and not isinstance(node.parent, Fortran2003.Part_Ref) and \
not (isinstance(node.parent, Fortran2003.Function_Reference) and node.string.lower()=='null') and \
not (isinstance(node.parent, Fortran2003.Specific_Binding) and node.string.lower()=='null'):
Logger.info('Intrinsic procedure name of "%s" is used for name resolution'% \
(node.string.lower()), stdout=True)
Logger.info('\tnear "%s"'% stmt.item.line, stdout=True)
Logger.info('\tin %s'% stmt.reader.id, stdout=True)
else:
#if node.string.lower()!='null':
# Logger.info('Intrinsic procedure name of "%s" is skipped from name resolution'% \
# (node.string.lower()), stdout=True)
#Logger.info('\tnear "%s"'% stmt.item.line, stdout=True)
#Logger.info('\tin %s'% stmt.reader.id, stdout=True)
return
elif not Config.search['skip_intrinsic'] and is_except(node, stmt):
if hasattr(node, 'parent') and not isinstance(node.parent, Fortran2003.Part_Ref) and \
not (isinstance(node.parent, Fortran2003.Function_Reference) and node.string.lower()=='null') and \
not (isinstance(node.parent, Fortran2003.Specific_Binding) and node.string.lower()=='null'):
#Logger.info('Intrinsic procedure name of "%s" is NOT skipped from name resolution'% \
# (node.string.lower()), stdout=True)
#Logger.info('\tnear "%s"'% stmt.item.line, stdout=True)
#Logger.info('\tin %s'% stmt.reader.id, stdout=True)
pass
else:
if node.string.lower()!='null':
Logger.info('Intrinsic procedure name of "%s" is skipped from name resolution'% \
(node.string.lower()), stdout=True)
Logger.info('\tnear "%s"'% stmt.item.line, stdout=True)
Logger.info('\tin %s'% stmt.reader.id, stdout=True)
return
# skip if excluded
#if Config.exclude.has_key('namepath') and stmt.__class__ in execution_part:
if Config.exclude.has_key('namepath'):
for pattern, actions in Config.exclude['namepath'].iteritems():
name = node.string.lower()
namepath = pack_innamepath(stmt, name)
#Logger.info('%s and %s are being checked for exclusion'%(pattern, namepath))
if match_namepath(pattern, namepath):
#Logger.info('%s and %s are mathched for exclusion'%(pattern, namepath))
if not hasattr(stmt, 'exclude_names'): stmt.exclude_names = OrderedDict()
if stmt.exclude_names.has_key(name):
stmt.exclude_names[name].extend(actions)
else:
stmt.exclude_names[name] = actions
node.skip_search = True
if hasattr(node, 'parent'): node.parent.skip_search = True
return
ukey = KGName(pack_innamepath(stmt, node.string.lower()), node=node, stmt=stmt)
if gentype is None: gentype = KGGenType.STATE_IN
if resolvers is None:
stmt.unknowns[ukey] = ResState(gentype, ukey, stmt, res_default)
else:
stmt.unknowns[ukey] = ResState(gentype, ukey, stmt, resolvers)
Logger.info('%s is saved as unknown' % node.string.lower(), name=ukey, stmt=stmt)
elif defer:
f2003_search_unknowns(stmt, node, resolvers, gentype=gentype)
def main(self):
Logger.info('Starting KExt', stdout=True)
# create state directories
if not os.path.exists(Config.path['state']):
os.makedirs(Config.path['state'])
# create kernel directories
if not os.path.exists(Config.path['kernel']):
os.makedirs(Config.path['kernel'])
os.system('rm -f %s/kgen_statefile.lst'%Config.path['kernel'])
os.system('rm -f %s/done.*'%Config.path['kernel'])
preprocess()
Logger.info('Pre-processing is done', stdout=True)
analyze()
Logger.info('Program is analyzed', stdout=True)
# generate kgen_driver.f90 in kernel directory
self.driver = self.create_tree()
program = self.create_program(self.driver)
program.name = self.kernel_name
self.append_program_in_tree(self.driver, program)
# init plugin framework
init_plugins([KERNEL_ID_0])
# construct a generation tree
for filepath, (srcobj, mods_used, units_used) in State.srcfiles.iteritems():
if hasattr(srcobj.tree, 'geninfo') and KGGenType.has_state(srcobj.tree.geninfo):
kfile = genkobj(None, srcobj.tree, KERNEL_ID_0)
sfile = gensobj(None, srcobj.tree, KERNEL_ID_0)
if kfile is None or sfile is None:
raise ProgramException('Kernel source file is not generated for %s.'%filepath)
self.genfiles.append((kfile, sfile, filepath))
State.used_srcfiles[filepath] = (srcobj, mods_used, units_used)
# process each nodes in the tree
for plugin_name in event_register.keys():
for kfile, sfile, filepath in self.genfiles:
kfile.created([plugin_name])
sfile.created([plugin_name])
for tree in self._trees:
tree.created([plugin_name])
for kfile, sfile, filepath in self.genfiles:
kfile.process([plugin_name])
sfile.process([plugin_name])
for tree in self._trees:
tree.process([plugin_name])
for kfile, sfile, filepath in self.genfiles:
kfile.finalize([plugin_name])
sfile.finalize([plugin_name])
for tree in self._trees:
tree.finalize([plugin_name])
for kfile, sfile, filepath in self.genfiles:
kfile.flatten(KERNEL_ID_0, [plugin_name])
sfile.flatten(KERNEL_ID_0, [plugin_name])
for tree in self._trees:
tree.flatten(KERNEL_ID_0, [plugin_name])
def main():
version = [ 0, 1, '0' ]
outdir = '.'
retval = 0
varid = 0
Logger.info('Starting simplify_ifcond', stdout=True)
try:
# option parser
parser = optparse.OptionParser(version='simplify_ifcond version %d.%d.%s'%tuple(version))
# common options
parser.add_option("--outdir", dest="outdir", action='store', type='string', default='output', help="path to create outputs")
parser.add_option("--add-ext", dest="ext", action='store', type='string', default=None, help="File extensions to parse")
parser.add_option("-t", "--threshold", dest="threshold", action='store', type='int', default=20, help="Max number of identifiers if condition before simplifying.")
opts, args = parser.parse_args()
if len(args)<1:
print 'ERROR: Target source folders are not provided.'
print 'Usage: simplify_ifcond [options] <target folder path[, target folder path, ...]>'
sys.exit(-1)
# create output directory
outpath = os.path.abspath(opts.outdir)
if not os.path.exists(outpath):
os.makedirs(outpath)
outsrcpath = '%s/src'%outpath
shutil.rmtree(outsrcpath)
os.makedirs(outsrcpath)
# walk through source directory tree
for srcdir in args:
abssrcpath = os.path.abspath(srcdir)
for dirName, subdirList, fileList in os.walk(abssrcpath):
relpath = os.path.relpath(dirName, start=abssrcpath)
outfilepath = '%s/%s'%(outsrcpath, relpath.replace('.', ''))
if not os.path.exists(outfilepath):
os.makedirs(outfilepath)
for srcfile in fileList:
if any(srcfile.endswith(ext) for ext in file_exts):
try:
# read source file
parsed = SrcFile(os.path.join(dirName, srcfile), preprocess=False)
# create analysis container
parstmts = []
# anlyze
last_span = ( 1, 1 )
for stmt, depth in api.walk(parsed.tree):
if isinstance(stmt, Comment):
if stmt.item.span[0] >= last_span[0] and stmt.item.span[1] <= last_span[1]:
stmt.ignore = True
elif isinstance(stmt, (IfThen, If, ElseIf)):
if ifcondcheck(stmt.f2003.items[0], opts.threshold):
p = stmt.ancestors()[-1]
if p not in parstmts:
parstmts.append(p)
if not hasattr(p, 'simplify_ifstmts'):
p.simplify_ifstmts = []
if stmt not in p.simplify_ifstmts:
p.simplify_ifstmts.append(stmt)
last_span = stmt.item.span
# modify
for parstmt in parstmts:
# add openmp directive
add_specstmts(parstmt, len(parstmt.simplify_ifstmts))
# simplify
for i, ifstmt in enumerate(parstmt.simplify_ifstmts):
simplify(ifstmt, ifstmt.f2003.items[0], i)
# generate modified source files
if len(parstmts) > 0:
lines = []
for stmt, depth in api.walk(parsed.tree):
if hasattr(stmt, 'forced_lines'):
lines.extend(stmt.forced_lines)
elif not stmt.ignore:
start = stmt.item.span[0]-1
end = stmt.item.span[1]
for line in stmt.top.prep[start:end]:
split = line.split()
if len(split) > 2 and split[0].startswith('!KGEN#') and split[1].isdigit():
continue
lines.append(line)
with open(os.path.join(outfilepath, srcfile), 'w') as f:
print 'Generating %s\n'%outfilepath
f.write('\n'.join(lines))
f.write('\n')
except Exception as e:
#import pdb; pdb.set_trace()
raise
pass
# switch source files if directed
except UserException as e:
print 'ERROR: %s'%str(e)
Logger.info(e)
#Logger.critical(e)
retval = -1
except ProgramException as e:
Logger.critical(e)
retval = -1
except Exception as e:
Logger.critical(e)
retval = -1
finally:
pass
Logger.info('simplify_ifcond is finished.', stdout=True)
return retval
# generate_state()
# Logger.info('Instrumented files are generated', stdout=True)
#
# generate_kernel()
# Logger.info('Kernel files are generated', stdout=True)
generate_makefiles()
Logger.info('Makefiles are generated', stdout=True)
postprocess()
Logger.info('Post-processing is done', stdout=True)
Logger.info('Completed.', stdout=True)
if __name__ == "__main__":
try:
print ''
main()
except UserException as e:
print 'ERROR: %s'%str(e)
Logger.info(e)
#Logger.critical(e)
except ProgramException as e:
Logger.critical(e)
except Exception as e:
Logger.critical(e)
finally:
pass
def fini(self):
import ConfigParser
cwd = self.config.build['cwd']
incini = os.path.join(cwd, self.config.ini)
if not os.path.exists(incini) or 'all' in self.config.rebuild or 'include' in self.config.rebuild:
Logger.info('Creating KGen include file: %s'%incini, stdout=True)
Config = ConfigParser.RawConfigParser()
Config.optionxform = str
if len(self.config.include)>0:
Config.add_section('include')
for inc in self.config.include.keys():
for i in inc.split(':'):
Config.set('include', i, '')
if len(self.config.macro)>0:
Config.add_section('macro')
for key, value in self.config.macro.items():
Config.set('macro', key, value)
if len(self.config.object)>0:
Config.add_section('import')
for key, value in self.config.macro.items():
Config.set('import', key, value)
flags = {}
with open(os.path.join(cwd, self.config.strace), 'r') as f:
line = f.readline()
while(line):
pos_execve = line.find(STR_EX)
if pos_execve >= 0:
pos_enoent = line.rfind(STR_EN)
if pos_enoent < 0:
pos_last = line.rfind(STR_UF)
if pos_last < 0:
pos_last = line.rfind(')')
if pos_last >= 0:
try:
exec('exepath, cmdlist, env = %s'%line[pos_execve+len(STR_EX):pos_last])
compid = cmdlist[0].split('/')[-1]
if exepath and cmdlist and compid==cmdlist[0].split('/')[-1]:
compiler = CompilerFactory.createCompiler(compid)
if compiler:
srcs, incs, macros, openmp, options = compiler.parse_option(cmdlist, _getpwd(env))
if len(srcs)>0:
for src in srcs:
if src in flags:
flags[src].append((exepath, incs, macros, openmp, options))
else:
flags[src] = [ (exepath, incs, macros, openmp, options) ]
except:
pass
line = f.readline()
for fname, incitems in flags.items():
if len(incitems)>0:
# save the last compiler set
compiler = incitems[-1][0]
incs = incitems[-1][1]
macros = incitems[-1][2]
options = incitems[-1][4]
if Config.has_section(fname):
print 'Warning: %s section is dupulicated.' % fname
else:
Config.add_section(fname)
Config.set(fname,'compiler', compiler)
Config.set(fname,'compiler_options', ' '.join(options))
Config.set(fname,'include',':'.join(incs))
for name, value in macros:
Config.set(fname, name, value)
if len(Config.sections())>0:
with open(incini, 'w') as f:
Config.write(f)
else:
Logger.info('Reusing KGen include file: %s'%incini, stdout=True)
Logger.info('KOption is finished.', stdout=True)
return { 'incini': incini }
def main():
preprocess()
Logger.info('Pre-processing is done', stdout=True)
locate_callsite()
Logger.info('Call-site location is found', stdout=True)
collect_kernel_info()
Logger.info('Kernel information is collected', stdout=True)
mark_generation_info()
Logger.info('Kernel generation information is marked')
generate_srcfiles()
Logger.info('Source files are generated', stdout=True)
# generate_state()
# Logger.info('Instrumented files are generated', stdout=True)
#
# generate_kernel()
# Logger.info('Kernel files are generated', stdout=True)
generate_makefiles()
Logger.info('Makefiles are generated', stdout=True)
postprocess()
Logger.info('Post-processing is done', stdout=True)
Logger.info('Completed.', stdout=True)
def __init__(self, srcpath, preprocess=True):
import os.path
from kgen_utils import run_shcmd
from statements import Comment
from block_statements import Module, Program
# set default values
self.tree = None
self.srcpath = srcpath
self.abspath = os.path.abspath(self.srcpath)
# set source file format
isfree = True
isstrict = False
if self.abspath in Config.source["file"].keys():
if Config.source["file"][self.abspath].has_key("isfree"):
isfree = Config.source["file"][self.abspath]["isfree"]
if Config.source["file"][self.abspath].has_key("isstrict"):
isstrict = Config.source["file"][self.abspath]["isstrict"]
else:
if Config.source["isstrict"]:
isstrict = Config.source["isstrict"]
if Config.source["isfree"]:
isfree = Config.source["isfree"]
# prepare include paths and macro definitions
path_src = []
macros_src = []
if Config.include["file"].has_key(self.abspath):
path_src = Config.include["file"][self.abspath]["path"] + [os.path.dirname(self.abspath)]
for k, v in Config.include["file"][self.abspath]["macro"].iteritems():
if v:
macros_src.append("-D%s=%s" % (k, v))
else:
macros_src.append("-D%s" % k)
includes = "-I" + " -I".join(Config.include["path"] + path_src)
macros_common = []
for k, v in Config.include["macro"].iteritems():
if v:
macros_common.append("-D%s=%s" % (k, v))
else:
macros_common.append("-D%s" % k)
macros = " ".join(macros_common + macros_src)
# execute preprocessing
Logger.info("Reading %s" % self.srcpath, stdout=True)
new_lines = []
with open(self.abspath, "r") as f:
if preprocess:
pp = Config.bin["pp"]
if pp.endswith("fpp"):
if isfree:
srcfmt = " -free"
else:
srcfmt = " -fixed"
flags = Config.bin["fpp_flags"] + srcfmt
elif pp.endswith("cpp"):
flags = Config.bin["cpp_flags"]
else:
raise UserException("Preprocessor is not either fpp or cpp")
output, err, retcode = run_shcmd("%s %s %s %s" % (pp, flags, includes, macros), input=f.read())
prep = map(lambda l: "!KGEN" + l if l.startswith("#") else l, output.split("\n"))
new_lines = self.handle_include(prep)
else:
new_lines = f.read().split("\n")
# add include paths
if Config.include["file"].has_key(self.abspath) and Config.include["file"][self.abspath].has_key("path"):
include_dirs = Config.include["file"][self.abspath]["path"] + [os.path.dirname(self.abspath)]
else:
include_dirs = None
# if self.abspath=='/glade/scratch/youngsun/kgen_system_test/branches/initial/MPAS-Release/src/framework/mpas_derived_types.F':
# print '\n'.join(new_lines)
# sys.exit()
# import pdb ; pdb.set_trace()
# fparse
self.tree = parse(
"\n".join(new_lines),
ignore_comments=False,
analyze=True,
isfree=isfree,
isstrict=isstrict,
include_dirs=include_dirs,
source_only=None,
)
self.tree.prep = new_lines
self.tree.used4genstate = False
# if self.abspath=='/glade/scratch/youngsun/kgen_system_test/branches/initial/MPAS-Release/src/framework/mpas_derived_types.F':
# print self.tree
# sys.exit()
# parse f2003
lineno = 0
linediff = 0
for stmt, depth in walk(self.tree, -1):
stmt.parse_f2003()
# rename reader.id
self.tree.reader.id = self.abspath
# collect module information
for mod_name, mod_stmt in self.tree.a.module.iteritems():
if not State.modules.has_key(mod_name):
State.modules[mod_name] = OrderedDict()
# State.modules[mod_name]['num'] = State.mod_num
# State.mod_num += 1
State.modules[mod_name]["stmt"] = mod_stmt
State.modules[mod_name]["file"] = self
State.modules[mod_name]["path"] = self.abspath
# State.modules[mod_name]['extern'] = OrderedDict()
# State.modules[mod_name]['extern']['names'] = []
# State.modules[mod_name]['extern']['typedecl_stmt'] = OrderedDict()
# State.modules[mod_name]['extern']['tkdpat'] = []
# State.modules[mod_name]['mod_rw_var_depends'] = []
# State.modules[mod_name]['dtype'] = []
# collect program unit information
for item in self.tree.content:
if item.__class__ not in [Module, Comment, Program]:
if item.reader.id not in State.program_units.keys():
State.program_units[item.reader.id] = []
State.program_units[item.reader.id].append(item)
# create a tuple for file dependency
State.srcfiles[self.abspath] = (self, [], [])
def main():
from compflag_tool import CompFlagDetect
from coverage_tool import CoverageDetect
from kext_tool import KExtTool
version = [0, 7, '2']
outdir = '.'
retval = 0
Logger.info('Starting KGen', stdout=True)
# add options for all of kgen tools
try:
# option parser
parser = optparse.OptionParser(version='KGEN version %d.%d.%s' %
tuple(version))
# common options
parser.add_option("--outdir",
dest="outdir",
action='store',
type='string',
default=None,
help="path to create outputs")
parser.add_option("--rebuild",
dest="rebuild",
action='append',
type='string',
default=None,
help="force to rebuild")
parser.add_option("--prerun",
dest="prerun",
action='append',
type='string',
default=None,
help="prerun commands")
parser.add_option("-i",
"--include-ini",
dest="include_ini",
action='store',
type='string',
default=None,
help="INI options")
parser.add_option("--debug",
dest="debug",
action='append',
type='string',
help=optparse.SUPPRESS_HELP)
# kapp options
parser.add_option(
"-c",
"--cmd-clean",
dest="cmd_clean",
action='store',
type='string',
default=None,
help="Linux command(s) to clean a target application build")
parser.add_option(
"-b",
"--cmd-build",
dest="cmd_build",
action='store',
type='string',
default=None,
help="Linux command(s) to build a target application")
parser.add_option("-r",
"--cmd-run",
dest="cmd_run",
action='store',
type='string',
default=None,
help="Linux command(s) to run a target application")
# compflag options
parser.add_option("--strace",
dest="strace",
action='store',
type='string',
default=None,
help="strace options")
# coverage options
#parser.add_option("--strace", dest="strace", action='store', type='string', default=None, help="strace options")
# kext options
parser.add_option(
"--invocation",
dest="invocation",
action='append',
type='string',
default=None,
help=
"(process, thread, invocation) pairs of kernel for data collection"
)
parser.add_option("-e",
"--exclude-ini",
dest="exclude_ini",
action='store',
type='string',
default=None,
help="information excluded for analysis")
parser.add_option(
"--kernel-option",
dest="kernel_option",
action='append',
type='string',
default=None,
help="compiler and linker options for Kgen-generated kernel")
parser.add_option("--openmp",
dest="openmp",
action='append',
type='string',
default=None,
help="Specifying OpenMP options")
parser.add_option("--mpi",
dest="mpi",
action='append',
type='string',
default=None,
help="MPI information for data collection")
parser.add_option("--timing",
dest="timing",
action='append',
type='string',
default=None,
help="Timing measurement information")
parser.add_option(
"--intrinsic",
dest="intrinsic",
action='append',
type='string',
default=None,
help=
"Specifying resolution for intrinsic procedures during searching")
parser.add_option("--check",
dest="check",
action='append',
type='string',
default=None,
help="Kernel correctness check information")
parser.add_option("--verbose",
dest="verbose",
action='store',
type='string',
default=None,
help="Set the verbose level for verification output")
parser.add_option("--add-mpi-frame",
dest="add_mpi_frame",
type='string',
default=None,
help='Add MPI frame codes in kernel_driver.')
parser.add_option("--source",
dest="source",
action='append',
type='string',
default=None,
help="Setting source file related properties")
parser.add_option("--logging",
dest="logging",
action='append',
type='string',
help=optparse.SUPPRESS_HELP)
opts, args = parser.parse_args()
if len(args) < 1:
print 'ERROR: Target source files is not provided.'
print 'Usage: kgen [options] <target file path[:namepath]> --cmd-clean <commands> --cmd-build <commands> --cmd-run <commands>'
sys.exit(-1)
if opts.cmd_clean is None:
print 'ERROR: No clean command is prvoided in command line. Please add --cmd-clean option.'
print 'Usage: kgen [options] <target file path[:namepath]> --cmd-clean <commands> --cmd-build <commands> --cmd-run <commands>'
sys.exit(-1)
if opts.cmd_build is None:
print 'ERROR: No build command is prvoided in command line. Please add --cmd-build option.'
print 'Usage: kgen [options] <target file path[:namepath]> --cmd-clean <commands> --cmd-build <commands> --cmd-run <commands>'
sys.exit(-1)
if opts.cmd_run is None:
print 'ERROR: No run command is prvoided in command line. Please add --cmd-run option.'
print 'Usage: kgen [options] <target file path[:namepath]> --cmd-clean <commands> --cmd-build <commands> --cmd-run <commands>'
sys.exit(-1)
# seletively copy user inputs for each kgen tools
compflag_argv = []
coverage_argv = []
kext_argv = []
#kgen_argv = []
# collect common options
if opts.outdir:
kext_argv.append('--outdir')
kext_argv.append(opts.outdir)
compflag_argv.append('--build')
compflag_argv.append('cwd="%s",clean="%s"' %
(opts.outdir, opts.cmd_clean))
coverage_argv.append('--outdir')
coverage_argv.append(opts.outdir)
outdir = opts.outdir
if not os.path.exists(outdir):
os.mkdir(outdir)
else:
kext_argv.append('--outdir')
kext_argv.append(os.getcwd())
compflag_argv.append('--build')
compflag_argv.append('cwd="%s",clean="%s"' %
(os.getcwd(), opts.cmd_clean))
coverage_argv.append('--outdir')
coverage_argv.append(os.getcwd())
outdir = os.getcwd()
if opts.prerun:
coverage_argv.append('--prerun')
coverage_argv.extend(opts.prerun)
compflag_argv.append('--prerun')
compflag_argv.extend(opts.prerun)
kext_argv.append('--prerun')
kext_argv.extend(opts.prerun)
if opts.rebuild:
coverage_argv.append('--rebuild')
coverage_argv.extend(opts.rebuild)
compflag_argv.append('--rebuild')
compflag_argv.extend(opts.rebuild)
kext_argv.append('--rebuild')
kext_argv.extend(opts.rebuild)
if opts.include_ini:
compflag_argv.append('--include_ini')
compflag_argv.append(opts.include_ini)
kext_argv.append('--include-ini')
kext_argv.append(opts.include_ini)
if opts.debug:
coverage_argv.append('--debug')
coverage_argv.extend(opts.debug)
compflag_argv.append('--debug')
compflag_argv.extend(opts.debug)
kext_argv.append('--debug')
kext_argv.extend(opts.debug)
# collect compflag options
if opts.strace:
compflag_argv.append('--strace')
compflag_argv.append(opts.strace)
compflag_argv.append(opts.cmd_build)
# collect coverage options
coverage_argv.append('--cmd-clean')
coverage_argv.append(opts.cmd_clean)
coverage_argv.append('--cmd-build')
coverage_argv.append(opts.cmd_build)
coverage_argv.append('--cmd-run')
coverage_argv.append(opts.cmd_run)
coverage_argv.append(args[0])
# collect kext options
if opts.invocation:
kext_argv.append('--invocation')
kext_argv.extend(opts.invocation)
if opts.exclude_ini:
kext_argv.append('--exclude-ini')
kext_argv.append(opts.exclude_ini)
if opts.source:
kext_argv.append('--source')
kext_argv.extend(opts.source)
if opts.mpi:
kext_argv.append('--mpi')
kext_argv.extend(opts.mpi)
if opts.openmp:
kext_argv.append('--openmp')
kext_argv.extend(opts.openmp)
if opts.timing:
kext_argv.append('--timing')
kext_argv.extend(opts.timing)
if opts.intrinsic:
kext_argv.append('--intrinsic')
kext_argv.extend(opts.intrinsic)
if opts.check:
kext_argv.append('--check')
kext_argv.extend(opts.check)
if opts.kernel_option:
kext_argv.append('--kernel-option')
kext_argv.extend(opts.kernel_option)
if opts.verbose:
kext_argv.append('--verbose')
kext_argv.append(opts.verbose)
if opts.add_mpi_frame:
kext_argv.append('--add-mpi-frame')
kext_argv.append(opts.add_mpi_frame)
if opts.logging:
kext_argv.append('--logging')
kext_argv.extend(opts.logging)
kext_argv.append('--state-clean')
kext_argv.append('cmds=%s' % opts.cmd_clean)
kext_argv.append('--state-build')
kext_argv.append('cmds=%s' % opts.cmd_build)
kext_argv.append('--state-run')
kext_argv.append('cmds=%s' % opts.cmd_run)
kext_argv.append(args[0])
# run compflag
compflag = CompFlagDetect()
compflag.init(argv=compflag_argv)
compflag.main()
CF_flags = compflag.fini()
Config.apply()
analyze()
#kext = KExtTool()
#kext.init()
# collect command line inputs and parse
#kgen_argv.extend(kext_argv)
#Config.apply(argv=kgen_argv)
if not opts.invocation:
coverage = CoverageDetect()
coverage.init(argv=coverage_argv)
#Logger.info('Program is analyzed', stdout=True)
coverage_argv.append(opts.cmd_clean)
coverage_argv.append(opts.cmd_build)
coverage_argv.append(opts.cmd_run)
# run coverage
coverage.main()
# parse coverage rebuild option
is_rebuild_coverage = False
if opts.rebuild:
for r in opts.rebuild:
if isinstance(r, str):
subopts = r.split(',')
for subopt in subopts:
if subopt in ['all', 'coverage']:
is_rebuild_coverage = True
if is_rebuild_coverage: break
# check if coverage file exist
has_coveragefile = False
if os.path.exists('%s/coverage' % outdir):
coveragefile = glob.glob('%s/coverage/%s.*' %
(outdir, State.kernel['name']))
if len(coveragefile) > 0:
has_coveragefile = True
# generate coverage file
if is_rebuild_coverage or not has_coveragefile:
Logger.info('Generating coverage file at %s/coverage' % outdir,
stdout=True)
out, err, retcode = run_shcmd('make',
cwd='%s/coverage' % outdir)
if retcode != 0:
Logger.info('FAILED: %s' % err, stdout=True)
else:
Logger.info('Reusing coverage at %s/coverage' % outdir,
stdout=True)
CV_flags = coverage.fini()
kext_argv.extend(['--invocation', CV_flags['invocation']])
kext_argv.extend(['-i', CF_flags['incini']])
kext = KExtTool()
kext.init()
# run kext
kext.main()
extracts = kext.fini()
# extracts contain kernel files, state files
# parse kext rebuild option
is_rebuild_kext = False
if opts.rebuild:
for r in opts.rebuild:
if isinstance(r, str):
subopts = r.split(',')
for subopt in subopts:
if subopt in ['all', 'state']:
is_rebuild_kext = True
if is_rebuild_kext: break
# check if state files exist
has_statefiles = False
if os.path.exists('%s/kernel' % outdir):
statefiles = glob.glob('%s/kernel/%s.*.*.*' %
(outdir, State.kernel['name']))
if len(statefiles) > 0:
has_statefiles = True
# generate state
if is_rebuild_kext or not has_statefiles:
Logger.info('Generating state data files at %s/state.' % outdir,
stdout=True)
out, err, retcode = run_shcmd('make clean',
cwd='%s/state' % outdir)
out, err, retcode = run_shcmd('make run', cwd='%s/state' % outdir)
if retcode != 0:
Logger.info('FAILED: %s' % err, stdout=True)
else:
Logger.info('Reusing state data files at %s/kernel' % outdir,
stdout=True)
except UserException as e:
print 'ERROR: %s' % str(e)
Logger.info(e)
#Logger.critical(e)
retval = -1
except ProgramException as e:
Logger.critical(e)
retval = -1
except Exception as e:
Logger.critical(e)
retval = -1
finally:
if os.path.exists('%s/state/Makefile' % outdir):
out, err, retcode = run_shcmd('make recover_from_locals',
cwd='%s/state' % outdir)
Logger.info('KGen is finished.', stdout=True)
return retval
# generate_state()
# Logger.info('Instrumented files are generated', stdout=True)
#
# generate_kernel()
# Logger.info('Kernel files are generated', stdout=True)
generate_makefiles()
Logger.info('Makefiles are generated', stdout=True)
postprocess()
Logger.info('Post-processing is done', stdout=True)
Logger.info('Completed.', stdout=True)
if __name__ == "__main__":
try:
print ''
main()
except UserException as e:
print 'ERROR: %s' % str(e)
Logger.info(e)
#Logger.critical(e)
except ProgramException as e:
Logger.critical(e)
except Exception as e:
Logger.critical(e)
finally:
pass