def download(self, myname, result):
systestdir = result['mkdir_task']['sysdir']
workdir = result['mkdir_task']['workdir']
appsrc = '%s/cesm_ref'%systestdir
if not os.path.exists(appsrc):
os.mkdir(appsrc)
# check if cesm exists in appsrc dir
out, err, retcode = run_shcmd('svn info | grep URL', cwd=appsrc)
if retcode != 0 or not out or len(out)<3 or not out.startswith('URL'):
out, err, retcode = run_shcmd('svn checkout -r 76722 https://svn-ccsm-models.cgd.ucar.edu/cesm1/tags/cesm1_4_beta06 .', cwd=appsrc)
# copy cesm src into test specific src dir
tmpsrc = '%s/cesm_work'%systestdir
if not os.path.exists(tmpsrc):
shutil.copytree(appsrc, tmpsrc)
result[myname]['appsrc'] = appsrc
result[myname]['tmpsrc'] = tmpsrc
if os.path.exists(os.path.join(self.TEST_DIR, 'exclude.ini')):
shutil.copy(os.path.join(self.TEST_DIR, 'exclude.ini'), workdir)
self.set_status(result, myname, self.PASSED)
return result
def config(self, myname, result):
workdir = result['mkdir_task']['workdir']
systestdir = result['mkdir_task']['sysdir']
tmpsrc = result['download_task']['tmpsrc']
scriptdir = '%s/cime/scripts'%tmpsrc
casename = 'KINTCESM'
casedir = '%s/%s'%(systestdir, casename)
datadir = '%s/data'%workdir
# NOTE: svn co https://svn-ccsm-models.cgd.ucar.edu/cesm1/tags/cesm1_4_beta07/ systestdir/cesm_ref
if self.REBUILD or not os.path.exists(datadir) or len([name for name in os.listdir(datadir) if os.path.isfile(os.path.join(datadir, name))])==0 or \
not os.path.exists(casedir):
# check if project option exists
if 'project' not in self.OPTIONS:
self.set_status(result, myname, self.FAILED, errmsg='"project" user option is not provided. Use "-o project=<your porject id>"')
return result
# create a case
if not os.path.exists(casedir):
casecmd = './create_newcase -project %s -mach yellowstone -compset FC5 -res ne16_ne16 -compiler intel -case %s'%(self.OPTIONS['project'], casedir)
out, err, retcode = run_shcmd(casecmd, cwd=scriptdir)
if retcode!=0:
self.set_status(result, myname, self.FAILED, errmsg='MG2 case generation is failed: %s\n\n%s'%(err, out))
return result
# modify env_build.xml to enable MG2
out, err, retcode = run_shcmd('grep mg2 env_build.xml', cwd=casedir)
if retcode!=0:
xmlchange = './xmlchange -f env_build.xml -id CAM_CONFIG_OPTS -val "-microphys mg2 -clubb_sgs" -a'
out, err, retcode = run_shcmd(xmlchange, cwd=casedir)
if retcode!=0:
self.set_status(result, myname, self.FAILED, errmsg='Modification of env_build.xml is failed: %s, %s '%(err, out))
return result
# cesm.setup
if not os.path.exists('%s/case.run'%casedir):
#import pdb; pdb.set_trace()
out, err, retcode = run_shcmd('./case.setup', cwd=casedir)
if retcode!=0:
self.set_status(result, myname, self.FAILED, errmsg='case.setup is failed: %s\n\n%s'%(err, out))
return result
for fname in os.listdir('%s/SourceMods'%casedir):
if fname.startswith('src.') and os.path.isdir(os.path.join('%s/SourceMods'%casedir, fname)):
for srcfile in os.listdir('%s/SourceMods/%s'%(casedir, fname)):
if os.path.isfile(os.path.join('%s/SourceMods/%s'%(casedir, fname), srcfile)):
os.remove(os.path.join('%s/SourceMods/%s'%(casedir, fname), srcfile))
# include.ini was created manually
result[myname]['srcmods'] = '%s/SourceMods'%casedir
result[myname]['casedir'] = casedir
result[myname]['casename'] = casename
self.set_status(result, myname, self.PASSED)
return result
def download(self, myname, result):
systestdir = result['mkdir_task']['sysdir']
workdir = result['mkdir_task']['workdir']
#appsrc = '%s/cesm_ref'%systestdir
appsrc = '/glade/u/home/youngsun/apps/cesm/cesm1_5_beta07'
if not os.path.exists(appsrc):
os.mkdir(appsrc)
# check if cesm exists in appsrc dir
out, err, retcode = run_shcmd('svn info | grep URL', cwd=appsrc)
if retcode != 0 or not out or len(out)<3 or not out.startswith('URL'):
out, err, retcode = run_shcmd('svn checkout -r 82434 https://svn-ccsm-models.cgd.ucar.edu/cesm1/tags/cesm1_5_beta07 .', cwd=appsrc)
# copy cesm src into test specific src dir
tmpsrc = '%s/cesm_work'%systestdir
if not os.path.exists(tmpsrc):
shutil.copytree(appsrc, tmpsrc)
result[myname]['appsrc'] = appsrc
result[myname]['tmpsrc'] = tmpsrc
if os.path.exists(os.path.join(self.TEST_DIR, 'exclude.ini')):
shutil.copy(os.path.join(self.TEST_DIR, 'exclude.ini'), workdir)
self.set_status(result, myname, self.PASSED)
return result
def config(self, myname, result):
workdir = result['mkdir_task']['workdir']
systestdir = result['mkdir_task']['sysdir']
tmpsrc = result['download_task']['tmpsrc']
scriptdir = '%s/cime/scripts'%tmpsrc
casename = 'KINTCESM'
casedir = '%s/%s'%(systestdir, casename)
datadir = '%s/data'%workdir
# NOTE: svn co https://svn-ccsm-models.cgd.ucar.edu/cesm1/tags/cesm1_4_beta07/ systestdir/cesm_ref
if self.REBUILD or not os.path.exists(datadir) or len([name for name in os.listdir(datadir) if os.path.isfile(os.path.join(datadir, name))])==0:
# check if project option exists
if 'project' not in self.OPTIONS:
self.set_status(result, myname, self.FAILED, errmsg='"project" user option is not provided. Use "-o project=<your porject id>"')
return result
# create a case
if not os.path.exists(casedir):
casecmd = './create_newcase -project %s -mach yellowstone -compset FC5 -res ne16_ne16 -compiler intel -case %s'%(self.OPTIONS['project'], casedir)
out, err, retcode = run_shcmd(casecmd, cwd=scriptdir)
if retcode!=0:
self.set_status(result, myname, self.FAILED, errmsg='MG2 case generation is failed: %s\n\n%s'%(err, out))
return result
# modify env_build.xml to enable MG2
out, err, retcode = run_shcmd('grep mg2 env_build.xml', cwd=casedir)
if retcode!=0:
xmlchange = './xmlchange -f env_build.xml -id CAM_CONFIG_OPTS -val "-microphys mg2 -clubb_sgs" -a'
out, err, retcode = run_shcmd(xmlchange, cwd=casedir)
if retcode!=0:
self.set_status(result, myname, self.FAILED, errmsg='Modification of env_build.xml is failed: '%(err, out))
return result
# cesm.setup
if not os.path.exists('%s/%s.run'%(casedir, casename)):
out, err, retcode = run_shcmd('./cesm_setup', cwd=casedir)
if retcode!=0:
self.set_status(result, myname, self.FAILED, errmsg='cesm.setup is failed: %s\n\n%s'%(err, out))
return result
for fname in os.listdir('%s/SourceMods'%casedir):
if fname.startswith('src.') and os.path.isdir(os.path.join('%s/SourceMods'%casedir, fname)):
for srcfile in os.listdir('%s/SourceMods/%s'%(casedir, fname)):
if os.path.isfile(os.path.join('%s/SourceMods/%s'%(casedir, fname), srcfile)):
os.remove(os.path.join('%s/SourceMods/%s'%(casedir, fname), srcfile))
# include.ini was created manually
result[myname]['srcmods'] = '%s/SourceMods'%casedir
result[myname]['casedir'] = casedir
result[myname]['casename'] = casename
self.set_status(result, myname, self.PASSED)
return result
def postprocess():
# TODO: display summary for kernel generation
from kgen_utils import run_shcmd
# copy object files into kernel folder
if Config.include.has_key('import'):
for path, import_type in Config.include['import'].iteritems():
if 'object'==import_type:
run_shcmd('cp -f %s %s'%(path, Config.path['kernel']))
def postprocess():
# TODO: display summary for kernel generation
from kgen_utils import run_shcmd
# copy object files into kernel folder
if Config.include.has_key('import'):
for path, import_type in Config.include['import'].iteritems():
if 'object' == import_type:
run_shcmd('cp -f %s %s' % (path, Config.path['kernel']))
def genstate(self, myname, result):
workdir = result['mkdir_task']['workdir']
reuse_data = result['mkdir_task']['reuse_data']
if not reuse_data:
# find jobid
jobid = None
for iter in range(120):
#print('Waiting for "KGENMPAS" job to be initiated.')
time.sleep(5)
out, err, retcode = run_shcmd('bjobs')
for line in out.split('\n'):
items = line.split()
if any(item=='KGENMPAS' for item in items):
jobid = items[0]
#print('"KGENMPAS" job is initiated.')
break
if jobid: break
if jobid is None:
self.set_status(result, myname, self.FAILED, errmsg='Job id is not found.')
return result
status = ''
maxiter = 3600
iter = 0
sleep_sec = 1
while status not in [ 'DONE', 'PSUSP', 'USUSP', 'SSUSP', 'EXIT', 'UNKWN', 'ZOMBI', 'FINISHED' ]:
#if iter % 10 == 0:
# print('Waiting for "KGENMPAS" job to be finished. %d seconds has been passed.'%iter*sleep_sec)
time.sleep(sleep_sec)
out, err, retcode = run_shcmd('bjobs %s'%jobid)
if retcode==0:
for line in out.split('\n'):
items = line.split()
if len(items)>3 and items[0]==jobid:
status = items[2]
elif len(items)>0 and items[-1]=='found':
print('"KGENMPAS" job is finished.')
status = 'FINISHED'
else:
print('DEBUG: ', out, err, retcode)
iter += 1
if iter>=maxiter:
break
if status=='DONE' or 'FINISHED':
self.set_status(result, myname, self.PASSED)
else:
self.set_status(result, myname, self.FAILED, errmsg='Job completion status is not expected.')
else:
self.set_status(result, myname, self.PASSED)
return result
def genstate(self, myname, result):
workdir = result['mkdir_task']['workdir']
reuse_data = result['mkdir_task']['reuse_data']
if not reuse_data:
# find jobid
jobid = None
for iter in range(120):
time.sleep(5)
out, err, retcode = run_shcmd('squeue -u %s'%getpass.getuser())
for line in out.split('\n'):
items = line.split()
if any(item=='KGCALC' for item in items):
#if len(items)>6 and items[6].endswith('KHOMME'):
jobid = items[0]
break
if jobid: break
if jobid is None:
self.set_status(result, myname, self.FAILED, errmsg='Job id is not found.')
return result
status = ''
maxiter = 3600
iter = 0
while status not in [ 'BF', 'CA', 'CD', 'F', 'NF', 'PR', 'SE', 'ST', 'TO' ]:
time.sleep(1)
out, err, retcode = run_shcmd('squeue -j %s'%jobid)
if retcode==0:
lines = out.strip().split('\n')
if len(lines) == 1: break
for line in lines[1:]:
items = line.split()
if len(items) > 9:
if items[0]==jobid:
status = items[9]
else:
print('DEBUG: ', out, err, retcode)
break
iter += 1
if iter>=maxiter:
break
if status=='F' or 'CD' or 'CG':
self.set_status(result, myname, self.PASSED)
else:
self.set_status(result, myname, self.FAILED, errmsg='Job completion status is not expected.')
else:
self.set_status(result, myname, self.PASSED)
return result
def genstate(self, myname, result):
casedir = result['config_task']['casedir']
casename = result['config_task']['casename']
workdir = result['mkdir_task']['workdir']
reuse_data = result['mkdir_task']['reuse_data']
if not reuse_data:
# find jobid
jobid = None
for iter in range(120):
time.sleep(5)
out, err, retcode = run_shcmd('bjobs')
for line in out.split('\n'):
items = line.split()
if any(item==casename for item in items):
#if len(items)>6 and items[6].endswith(casename):
jobid = items[0]
break
if jobid: break
if jobid is None:
self.set_status(result, myname, self.FAILED, errmsg='Job id is not found.')
return result
status = ''
maxiter = 3600
iter = 0
while status not in [ 'DONE', 'PSUSP', 'USUSP', 'SSUSP', 'EXIT', 'UNKWN', 'ZOMBI', 'FINISHED' ]:
time.sleep(1)
out, err, retcode = run_shcmd('bjobs %s'%jobid)
if retcode==0:
for line in out.split('\n'):
items = line.split()
if len(items)>3 and items[0]==jobid:
status = items[2]
elif len(items)>0 and items[-1]=='found':
status = 'FINISHED'
else:
print('DEBUG: ', out, err, retcode)
iter += 1
if iter>=maxiter:
break
if status=='DONE' or 'FINISHED':
self.set_status(result, myname, self.PASSED)
else:
self.set_status(result, myname, self.FAILED, errmsg='Job completion status is not expected.')
else:
self.set_status(result, myname, self.PASSED)
return result
def genstate(self, myname, result):
casedir = result['config_task']['casedir']
casename = result['config_task']['casename']
workdir = result['mkdir_task']['workdir']
reuse_data = result['mkdir_task']['reuse_data']
if not reuse_data:
# find jobid
jobid = None
for iter in range(120):
time.sleep(5)
out, err, retcode = run_shcmd('bjobs')
for line in out.split('\n'):
items = line.split()
if any(item==casename for item in items):
#if len(items)>6 and items[6].endswith(casename):
jobid = items[0]
break
if jobid: break
if jobid is None:
self.set_status(result, myname, self.FAILED, errmsg='Job id is not found.')
return result
status = ''
maxiter = 3600
iter = 0
while status not in [ 'DONE', 'PSUSP', 'USUSP', 'SSUSP', 'EXIT', 'UNKWN', 'ZOMBI', 'FINISHED' ]:
time.sleep(1)
out, err, retcode = run_shcmd('bjobs %s'%jobid)
if retcode==0:
for line in out.split('\n'):
items = line.split()
if len(items)>3 and items[0]==jobid:
status = items[2]
elif len(items)>0 and items[-1]=='found':
status = 'FINISHED'
else:
print('DEBUG: ', out, err, retcode)
iter += 1
if iter>=maxiter:
break
if status=='DONE' or 'FINISHED':
self.set_status(result, myname, self.PASSED)
else:
self.set_status(result, myname, self.FAILED, errmsg='Job completion status is not expected.')
else:
self.set_status(result, myname, self.PASSED)
return result
def extract_kernel(self, target, namepath, *args, **kwargs):
cmds = [ '%s/bin/kext'%self.KGEN_HOME ]
for kw, kwarg in kwargs.iteritems():
flag = kw.replace('_', '-').replace('UNDERSCORE', '_')
cmds.append('%s %s'%(flag, kwarg))
if namepath:
cmds.append('%s:%s'%(target, namepath))
else:
cmds.append(target)
out, err, retcode = run_shcmd(' '.join(cmds), cwd=self.TEST_DIR)
# debug
#print ' '.join(cmds)
#print out
if self.LEAVE_TEMP:
with open('%s/kgen_cmds.sh'%self.TEST_DIR, 'w') as f:
f.write('#!/bin/bash\n')
f.write('\n')
for cmd in cmds[:-1]:
f.write(' %s \\\n'%cmd)
f.write(' %s'%cmds[-1])
os.chmod('%s/kgen_cmds.sh'%self.TEST_DIR, 0755)
if not out or out.find('ERROR')>=0 or out.find('CRITICAL')>=0 or err or retcode!=0:
return False, out, err
return True, out, err
def extract_kernel(self, target, namepath, *args, **kwargs):
cmds = ['%s/bin/kext' % self.KGEN_HOME]
for kw, kwarg in kwargs.iteritems():
flag = kw.replace('_', '-').replace('UNDERSCORE', '_')
cmds.append('%s %s' % (flag, kwarg))
if namepath:
cmds.append('%s:%s' % (target, namepath))
else:
cmds.append(target)
out, err, retcode = run_shcmd(' '.join(cmds), cwd=self.TEST_DIR)
# debug
#print ' '.join(cmds)
#print out
if self.LEAVE_TEMP:
with open('%s/kgen_cmds.sh' % self.TEST_DIR, 'w') as f:
f.write('#!/bin/bash\n')
f.write('\n')
for cmd in cmds[:-1]:
f.write(' %s \\\n' % cmd)
f.write(' %s' % cmds[-1])
os.chmod('%s/kgen_cmds.sh' % self.TEST_DIR, 0755)
if not out or out.find('ERROR') >= 0 or out.find(
'CRITICAL') >= 0 or err or retcode != 0:
return False, out, err
return True, out, err
def genstate(self, myname, result):
workdir = result['mkdir_task']['workdir']
statefiles = result['generate_task']['statefiles']
FC = result['config_task']['FC']
FC_FLAGS = result['config_task']['FC_FLAGS']
PRERUN = result['config_task']['PRERUN']
out, err, retcode = run_shcmd(
'make FC="%s" FC_FLAGS="%s" PRERUN="%s"' % (FC, FC_FLAGS, PRERUN),
cwd='%s/state' % workdir)
result[myname]['stdout'] = out
result[myname]['stderr'] = err
if retcode != 0:
print(out, err, retcode)
self.set_status(result, myname, self.FAILED, err)
return result
outfiles = []
for statefile in statefiles:
outfile = os.path.join('%s/kernel' % workdir, statefile)
if os.path.exists(outfile):
outfiles.append(outfile)
result[myname]['outfiles'] = outfiles
if len(outfiles) == len(statefiles):
self.set_status(result, myname, self.PASSED)
else:
self.set_status(result, myname, self.FAILED, errmsg=str(outfiles))
return result
def main(self):
Logger.info('Starting KOption', stdout=True)
cwd = self.config.build['cwd']
slog = os.path.join(cwd, self.config.strace)
if not os.path.exists(slog) or \
'all' in self.config.rebuild or 'strace' in self.config.rebuild:
Logger.info('Building application to collect strace log file: %s'%slog, stdout=True)
with open(SH%cwd, 'w') as f:
f.write(TEMP_SH%(self.config.prerun['clean'], self.config.build['clean'], \
self.config.prerun['build'], self.config.build['cmdline']))
st = os.stat(SH%cwd)
os.chmod(SH%cwd, st.st_mode | stat.S_IEXEC)
shcmds = 'strace -o %s -f -q -s 100000 -e trace=execve -v -- %s'%\
(self.config.strace, SH%cwd)
out, err, retcode = run_shcmd(shcmds, cwd=cwd)
#os.remove(SH%self.config.build['cwd'])
if retcode!=0:
#print 'CMDS: ', shcmds
#print 'RETCODE: ', retcode
#print 'STDOUT: ', out
#print 'STDERR: ', err
raise ProgramException('Application build command is failed: %s'%err)
else:
Logger.info('Reusing strace log file: %s'%self.config.strace, stdout=True)
def genstate(self, myname, result):
workdir = result['mkdir_task']['workdir']
statefiles = result['generate_task']['statefiles']
FC = result['config_task']['FC']
FC_FLAGS = result['config_task']['FC_FLAGS']
PRERUN = result['config_task']['PRERUN']
out, err, retcode = run_shcmd('make FC="%s" FC_FLAGS="%s" PRERUN="%s"'%(FC, FC_FLAGS, PRERUN), cwd='%s/state'%workdir)
result[myname]['stdout'] = out
result[myname]['stderr'] = err
if retcode != 0:
print (out, err, retcode)
self.set_status(result, myname, self.FAILED, err)
return result
outfiles = []
for statefile in statefiles:
outfile = os.path.join('%s/kernel'%workdir, statefile)
if os.path.exists(outfile):
outfiles.append(outfile)
result[myname]['outfiles'] = outfiles
if len(outfiles)==len(statefiles):
self.set_status(result, myname, self.PASSED)
else:
self.set_status(result, myname, self.FAILED, errmsg=str(outfiles))
return result
def download(self, myname, result):
systestdir = result['mkdir_task']['sysdir']
workdir = result['mkdir_task']['workdir']
#appsrc = '%s/homme_ref'%systestdir
appsrc = '/glade/u/home/youngsun/apps/homme/trunk'
if not os.path.exists(appsrc):
os.mkdir(appsrc)
# check if homme exists in appsrc dir
out, err, retcode = run_shcmd('svn info | grep URL', cwd=appsrc)
if retcode != 0 or not out or len(out) < 3 or not out.startswith(
'URL'):
#out, err, retcode = run_shcmd('svn checkout -r 4971 https://svn-homme-model.cgd.ucar.edu/trunk/ .', cwd=appsrc) # r 4971 has broken pio external link
#out, err, retcode = run_shcmd('svn checkout -r 5438 https://svn-homme-model.cgd.ucar.edu/trunk/ .', cwd=appsrc)
out, err, retcode = run_shcmd(
'svn checkout -r 5704 https://svn-homme-model.cgd.ucar.edu/trunk/ .',
cwd=appsrc)
#out, err, retcode = run_shcmd('svn checkout -r 5650 https://svn-homme-model.cgd.ucar.edu/branch_tags/dungeon_tags/dungeon06 .', cwd=appsrc)
# copy homme src into test specific src dir
tmpsrc = '%s/homme_work' % systestdir
# if os.path.exists(tmpsrc):
# shutil.rmtree(tmpsrc)
# shutil.copytree(appsrc, tmpsrc)
if not os.path.exists(tmpsrc):
shutil.copytree(appsrc, tmpsrc)
else:
for fname in os.listdir('%s/src' % tmpsrc):
if fname.endswith('.kgen'):
shutil.copyfile(
os.path.join('%s/src' % tmpsrc, fname),
os.path.join('%s/src' % tmpsrc, fname[:-5]))
for fname in os.listdir('%s/src/share' % tmpsrc):
if fname.endswith('.kgen'):
shutil.copyfile(
os.path.join('%s/src/share' % tmpsrc, fname),
os.path.join('%s/src/share' % tmpsrc, fname[:-5]))
result[myname]['appsrc'] = appsrc
result[myname]['tmpsrc'] = tmpsrc
self.set_status(result, myname, self.PASSED)
return result
def recover(self, myname, result):
workdir = result['mkdir_task']['workdir']
out, err, retcode = run_shcmd('make recover_from_locals', cwd='%s/state'%workdir)
self.set_status(result, myname, self.PASSED)
return result
def preprocess(self, myname, result):
out, err, retcode = run_shcmd("sinfo | cut -d ' ' -f 1")
if retcode != 0 or out.find('debug')<0 or out.find('realtime')<0 or out.find('special')<0 or out.find('shared')<0:
errmsg = 'Current system is not Edison of NERSC'
self.set_status(result, myname, self.FAILED, errmsg)
else:
self.set_status(result, myname, self.PASSED)
return result
def preprocess(self, myname, result):
out, err, retcode = run_shcmd('bqueues')
if retcode != 0 or out.find('caldera')<0 or out.find('geyser')<0 or out.find('regular')<0 or out.find('premium')<0:
errmsg = 'Current system is not Yellowstone of NCAR'
self.set_status(result, myname, self.FAILED, errmsg)
else:
self.set_status(result, myname, self.PASSED)
return result
def recover(self, myname, result):
workdir = result['mkdir_task']['workdir']
out, err, retcode = run_shcmd('make recover_from_locals',
cwd='%s/state' % workdir)
self.set_status(result, myname, self.PASSED)
return result
def preprocess(self, myname, result):
out, err, retcode = run_shcmd('bqueues')
if retcode != 0 or out.find('caldera') < 0 or out.find(
'geyser') < 0 or out.find('regular') < 0 or out.find(
'premium') < 0:
errmsg = 'Current system is not Yellowstone of NCAR'
self.set_status(result, myname, self.FAILED, errmsg)
else:
self.set_status(result, myname, self.PASSED)
return result
def runkernel(self, myname, result):
workdir = result['mkdir_task']['workdir']
out, err, retcode = run_shcmd('make clean; make', cwd='%s/kernel'%workdir)
result[myname]['stdout'] = out
result[myname]['stderr'] = err
if retcode != 0 or err:
self.set_status(result, myname, self.FAILED, errmsg='kernel execution is failed: %s'%err)
else:
self.set_status(result, myname, self.PASSED)
return result
def download(self, myname, result):
systestdir = result['mkdir_task']['sysdir']
workdir = result['mkdir_task']['workdir']
appsrc = '%s/homme_ref'%systestdir
if not os.path.exists(appsrc):
os.mkdir(appsrc)
# check if homme exists in appsrc dir
out, err, retcode = run_shcmd('svn info | grep URL', cwd=appsrc)
if retcode != 0 or not out or len(out)<3 or not out.startswith('URL'):
#out, err, retcode = run_shcmd('svn checkout -r 4971 https://svn-homme-model.cgd.ucar.edu/trunk/ .', cwd=appsrc) # r 4971 has broken pio external link
out, err, retcode = run_shcmd('svn checkout -r 5438 https://svn-homme-model.cgd.ucar.edu/trunk/ .', cwd=appsrc)
# copy homme src into test specific src dir
tmpsrc = '%s/homme_work'%systestdir
# if os.path.exists(tmpsrc):
# shutil.rmtree(tmpsrc)
# shutil.copytree(appsrc, tmpsrc)
if not os.path.exists(tmpsrc):
shutil.copytree(appsrc, tmpsrc)
else:
for fname in os.listdir('%s/src'%tmpsrc):
if fname.endswith('.kgen'):
shutil.copyfile(os.path.join('%s/src'%tmpsrc, fname), os.path.join('%s/src'%tmpsrc, fname[:-5]))
for fname in os.listdir('%s/src/share'%tmpsrc):
if fname.endswith('.kgen'):
shutil.copyfile(os.path.join('%s/src/share'%tmpsrc, fname), os.path.join('%s/src/share'%tmpsrc, fname[:-5]))
result[myname]['appsrc'] = appsrc
result[myname]['tmpsrc'] = tmpsrc
self.set_status(result, myname, self.PASSED)
return result
def run_compflag(self, build_cmds, cwd, *args, **kwargs):
cmds = [ '%s/bin/compflag'%self.KGEN_HOME ]
for kw, kwarg in kwargs.iteritems():
flag = kw.replace('_', '-').replace('UNDERSCORE', '_')
cmds.append('%s %s'%(flag, kwarg))
cmds.append('-- %s'%build_cmds)
out, err, retcode = run_shcmd(' '.join(cmds), cwd=cwd)
#print('CMD: ', ' '.join(cmds))
#print('RETURN CODE: ', retcode)
#print 'STDOUT: ', out
#print 'STDERR: ', err
if not out or out.find('ERROR')>=0 or out.find('CRITICAL')>=0 or err or retcode!=0:
return False, out, err
return True, out, err
def runkernel(self, myname, result):
workdir = result['mkdir_task']['workdir']
out, err, retcode = run_shcmd('make clean; make',
cwd='%s/kernel' % workdir)
result[myname]['stdout'] = out
result[myname]['stderr'] = err
if retcode != 0 or err:
self.set_status(result,
myname,
self.FAILED,
errmsg='kernel execution is failed: %s' % err)
else:
self.set_status(result, myname, self.PASSED)
return result
def extract_kernel(self, target, namepath, *args, **kwargs):
outdir = '.'
cmds = ['%s/bin/kgen' % self.KGEN_HOME]
for kw, kwarg in kwargs.iteritems():
flag = kw.replace('_', '-').replace('UNDERSCORE', '_')
cmds.append('%s %s' % (flag, kwarg))
if flag == '--outdir':
outdir = kwarg
if namepath:
cmds.append('%s:%s' % (target, namepath))
else:
cmds.append(target)
for arg in args:
cmds.append(arg)
if self.LEAVE_TEMP:
with open('%s/kgen_cmds.sh' % outdir, 'w') as f:
f.write('#!/bin/bash\n')
f.write('\n')
for cmd in cmds[:-1]:
f.write(' %s \\\n' % cmd)
f.write(' %s' % cmds[-1])
os.chmod('%s/kgen_cmds.sh' % outdir, 0755)
out, err, retcode = run_shcmd(' '.join(cmds))
# debug
#print 'CMDS: ', ' '.join(cmds)
#print 'STDOUT: ', out
#print 'STDERR: ', err
#print 'RETCODE: ', retcode
if not out or out.find('ERROR') >= 0 or out.find(
'CRITICAL') >= 0 or err or retcode != 0:
return False, out, err
return True, out, err
def extract_kernel(self, target, namepath, *args, **kwargs):
outdir = '.'
cmds = [ '%s/bin/kgen'%self.KGEN_HOME ]
for kw, kwarg in kwargs.iteritems():
flag = kw.replace('_', '-').replace('UNDERSCORE', '_')
cmds.append('%s %s'%(flag, kwarg))
if flag=='--outdir':
outdir = kwarg
if namepath:
cmds.append('%s:%s'%(target, namepath))
else:
cmds.append(target)
for arg in args:
cmds.append(arg)
if self.LEAVE_TEMP:
with open('%s/kgen_cmds.sh'%outdir, 'w') as f:
f.write('#!/bin/bash\n')
f.write('\n')
for cmd in cmds[:-1]:
f.write(' %s \\\n'%cmd)
f.write(' %s'%cmds[-1])
os.chmod('%s/kgen_cmds.sh'%outdir, 0755)
out, err, retcode = run_shcmd(' '.join(cmds))
# debug
#print 'CMDS: ', ' '.join(cmds)
#print 'STDOUT: ', out
#print 'STDERR: ', err
#print 'RETCODE: ', retcode
if not out or out.find('ERROR')>=0 or out.find('CRITICAL')>=0 or err or retcode!=0:
return False, out, err
return True, out, err
def genstate(self, myname, result):
workdir = result['mkdir_task']['workdir']
reuse_data = result['mkdir_task']['reuse_data']
if not reuse_data:
# find jobid
jobid = None
for iter in range(120):
#print('Waiting for "KGENMPAS" job to be initiated.')
time.sleep(5)
out, err, retcode = run_shcmd('bjobs')
for line in out.split('\n'):
items = line.split()
if any(item == 'KGENMPAS' for item in items):
jobid = items[0]
#print('"KGENMPAS" job is initiated.')
break
if jobid: break
if jobid is None:
self.set_status(result,
myname,
self.FAILED,
errmsg='Job id is not found.')
return result
status = ''
maxiter = 3600
iter = 0
sleep_sec = 1
while status not in [
'DONE', 'PSUSP', 'USUSP', 'SSUSP', 'EXIT', 'UNKWN',
'ZOMBI', 'FINISHED'
]:
#if iter % 10 == 0:
# print('Waiting for "KGENMPAS" job to be finished. %d seconds has been passed.'%iter*sleep_sec)
time.sleep(sleep_sec)
out, err, retcode = run_shcmd('bjobs %s' % jobid)
if retcode == 0:
for line in out.split('\n'):
items = line.split()
if len(items) > 3 and items[0] == jobid:
status = items[2]
elif len(items) > 0 and items[-1] == 'found':
print('"KGENMPAS" job is finished.')
status = 'FINISHED'
else:
print('DEBUG: ', out, err, retcode)
iter += 1
if iter >= maxiter:
break
if status == 'DONE' or 'FINISHED':
self.set_status(result, myname, self.PASSED)
else:
self.set_status(
result,
myname,
self.FAILED,
errmsg='Job completion status is not expected.')
else:
self.set_status(result, myname, self.PASSED)
return result
def config(self, myname, result):
workdir = result['mkdir_task']['workdir']
systestdir = result['mkdir_task']['sysdir']
tmpsrc = result['download_task']['tmpsrc']
blddir = '%s/bld'%workdir
if not os.path.exists(blddir):
os.mkdir(blddir)
rundir = '%s/run'%workdir
if not os.path.exists(rundir):
os.mkdir(rundir)
result[myname]['blddir'] = blddir
result[myname]['rundir'] = rundir
# setup
if not os.path.exists('%s/movies'%rundir):
os.mkdir('%s/movies'%rundir)
if os.path.exists('%s/vcoord'%rundir):
os.system('rm -f %s/vcoord'%rundir)
#os.system('ln -s %s/test/vcoord %s/vcoord'%(tmpsrc, rundir))
os.system('ln -s %s/../../config/perfTestWACCM_64_1_4/vcoord %s/vcoord'%(self.TEST_DIR, rundir))
# create namelist
if os.path.exists('%s/perfTestWACCM.nl'%rundir):
os.remove('%s/perfTestWACCM.nl'%rundir)
#shutil.copy('%s/test/reg_test/namelists/perfTestWACCM.nl'%tmpsrc, rundir)
shutil.copy('%s/../../config/perfTestWACCM_64_1_4/perfTestWACCM-ne8.nl'%self.TEST_DIR, rundir)
#if os.path.exists('%s/camBench.nl'%rundir):
# os.remove('%s/camBench.nl'%rundir)
#shutil.copy('%s/test/perftest/camBench.nl'%tmpsrc, rundir)
# copy exclude.ini
if os.path.exists('%s/exclude.ini'%workdir):
os.remove('%s/exclude.ini'%workdir)
shutil.copy('%s/exclude.ini'%self.TEST_DIR, workdir)
result[myname]['prerun_config'] = self.get_prerun_cmds() + ['rm -rf CMakeFiles CMakeCache.txt']
result[myname]['prerun_build'] = self.get_prerun_cmds()
result[myname]['prerun_run'] = self.get_prerun_cmds() + ['export OMP_NUM_THREADS=2', \
'ulimit -s unlimited', 'export LD_LIBRARY_PATH=${NETCDF}/lib:${LD_LIBRARY_PATH}' ]
result[myname]['prerun_kernel'] = self.get_prerun_kernel_cmds()
result[myname]['mpirun'] = 'mpirun'
#'export LD_LIBRARY_PATH=$NETCDF/lib:/glade/apps/opt/hdf5/1.8.12/intel/12.1.5/lib:$LD_LIBRARY_PATH',
# create job submit script
with open('%s/homme.submit'%rundir, 'w') as fd:
#fd.write(job_script%('16', '16', '\n'.join(result[myname]['prerun_run']), result[myname]['mpirun'], '%s/test_execs/perfTest/perfTest'%blddir, '%s/camBench.nl'%rundir))
fd.write(job_script%('16', '16', '\n'.join(result[myname]['prerun_run']), result[myname]['mpirun'], '%s/test_execs/perfTestWACCM/perfTestWACCM'%blddir, '%s/perfTestWACCM-ne8.nl'%rundir))
if self.REBUILD or not os.path.exists(blddir) or len([name for name in os.listdir(blddir) if os.path.isfile(os.path.join(blddir, name))]) < 3:
# prepare prerun command
prerun_cmds = result[myname]['prerun_config']
# prepare cmake command
cmake_cmd = ['cmake']
cmake_cmd.append('-DHOMME_PROJID="STDD0002"')
cmake_cmd.append('-DENABLE_PERFTEST=TRUE')
cmake_cmd.append('-DENABLE_OPENMP=TRUE')
cmake_cmd.append('-DUSE_MPIEXEC="mpirun"')
cmake_cmd.append('-DCMAKE_C_COMPILER="mpiicc"')
cmake_cmd.append('-DCMAKE_CXX_COMPILER="mpiicpc"')
cmake_cmd.append('-DCMAKE_Fortran_COMPILER="mpiifort"')
cmake_cmd.append('-DNETCDF_DIR:PATH=$NETCDF')
cmake_cmd.append('-DWITH_PNETCDF=FALSE')
cmake_cmd.append(tmpsrc)
# cmake_cmd.append('-DPNETCDF_DIR:PATH=$PNETCDF')
# cmake_cmd.append('-DHDF5_DIR:PATH=/glade/apps/opt/hdf5/1.8.12/intel/12.1.5')
# cmake_cmd.append('-DSZIP_DIR:PATH=/glade/apps/opt/szip/2.1/intel/12.1.5')
if self.LEAVE_TEMP:
with open('%s/config_cmds.sh'%blddir, 'w') as f:
f.write('#!/bin/bash\n')
f.write('\n')
for cmd in prerun_cmds:
f.write(' %s\n'%cmd)
for cmd in cmake_cmd[:-1]:
f.write(' %s \\\n'%cmd)
f.write(' %s &> config.log'%cmake_cmd[-1])
os.chmod('%s/config_cmds.sh'%blddir, 0755)
out, err, retcode = run_shcmd('%s; %s'%('; '.join(prerun_cmds), ' '.join(cmake_cmd)), cwd=blddir)
if retcode != 0:
self.set_status(result, myname, self.FAILED, errmsg=err)
return result
# include.ini was created manually
self.set_status(result, myname, self.PASSED)
return result
def __init__(self, srcpath, preprocess=True):
import os.path
from kgen_utils import run_shcmd
from statements import Comment
from block_statements import Module, Program
# set default values
self.tree = None
self.srcpath = srcpath
self.abspath = os.path.abspath(self.srcpath)
# set source file format
isfree = True
isstrict = False
if self.abspath in Config.source['file'].keys():
if Config.source['file'][self.abspath].has_key('isfree'):
isfree = Config.source['file'][self.abspath]['isfree']
if Config.source['file'][self.abspath].has_key('isstrict'):
isstrict = Config.source['file'][self.abspath]['isstrict']
else:
if Config.source['isstrict']: isstrict = Config.source['isstrict']
if Config.source['isfree']: isfree = Config.source['isfree']
# prepare include paths and macro definitions
path_src = []
macros_src = []
if Config.include['file'].has_key(self.abspath):
path_src = Config.include['file'][self.abspath]['path'] + [
os.path.dirname(self.abspath)
]
path_src = [path for path in path_src if len(path) > 0]
for k, v in Config.include['file'][
self.abspath]['macro'].iteritems():
if v:
macros_src.append('-D%s=%s' % (k, v))
else:
macros_src.append('-D%s' % k)
includes = '-I' + ' -I'.join(Config.include['path'] + path_src)
macros_common = []
for k, v in Config.include['macro'].iteritems():
if v:
macros_common.append('-D%s=%s' % (k, v))
else:
macros_common.append('-D%s' % k)
macros = ' '.join(macros_common + macros_src)
# execute preprocessing
Logger.info('Reading %s' % self.srcpath, stdout=True)
new_lines = []
with open(self.abspath, 'r') as f:
if preprocess:
pp = Config.bin['pp']
if pp.endswith('fpp'):
if isfree: srcfmt = ' -free'
else: srcfmt = ' -fixed'
flags = Config.bin['fpp_flags'] + srcfmt
elif pp.endswith('cpp'):
flags = Config.bin['cpp_flags']
else:
raise UserException(
'Preprocessor is not either fpp or cpp')
output, err, retcode = run_shcmd('%s %s %s %s' %
(pp, flags, includes, macros),
input=f.read())
prep = map(lambda l: '!KGEN' + l if l.startswith('#') else l,
output.split('\n'))
new_lines = self.handle_include(prep)
else:
new_lines = f.read().split('\n')
# add include paths
if Config.include['file'].has_key(
self.abspath) and Config.include['file'][self.abspath].has_key(
'path'):
include_dirs = Config.include['file'][self.abspath]['path'] + [
os.path.dirname(self.abspath)
]
else:
include_dirs = None
# fparse
self.tree = parse('\n'.join(new_lines), ignore_comments=False, analyze=True, isfree=isfree, \
isstrict=isstrict, include_dirs=include_dirs, source_only=None )
self.tree.prep = new_lines
self.tree.used4genstate = False
# parse f2003
lineno = 0
linediff = 0
for stmt, depth in walk(self.tree, -1):
stmt.parse_f2003()
# rename reader.id
self.tree.reader.id = self.abspath
# collect module information
for mod_name, mod_stmt in self.tree.a.module.iteritems():
if not State.modules.has_key(mod_name):
State.modules[mod_name] = OrderedDict()
State.modules[mod_name]['stmt'] = mod_stmt
State.modules[mod_name]['file'] = self
State.modules[mod_name]['path'] = self.abspath
# collect program unit information
for item in self.tree.content:
if item.__class__ not in [Module, Comment, Program]:
if item.reader.id not in State.program_units.keys():
State.program_units[item.reader.id] = []
State.program_units[item.reader.id].append(item)
# create a tuple for file dependency
State.srcfiles[self.abspath] = (self, [], [])
self.process_directive()
def config(self, myname, result):
workdir = result['mkdir_task']['workdir']
systestdir = result['mkdir_task']['sysdir']
tmpsrc = result['download_task']['tmpsrc']
blddir = '%s/bld'%workdir
if not os.path.exists(blddir):
os.mkdir(blddir)
rundir = '%s/run'%workdir
if not os.path.exists(rundir):
os.mkdir(rundir)
result[myname]['blddir'] = blddir
result[myname]['rundir'] = rundir
# setup
if not os.path.exists('%s/movies'%rundir):
os.mkdir('%s/movies'%rundir)
if os.path.exists('%s/vcoord'%rundir):
os.system('unlink %s/vcoord'%rundir)
os.system('ln -s %s/test/vcoord %s/vcoord'%(tmpsrc, rundir))
# create namelist
if os.path.exists('%s/camBench.nl'%rundir):
os.remove('%s/camBench.nl'%rundir)
shutil.copy('%s/test/perftest/camBench.nl'%tmpsrc, rundir)
# copy exclude.ini
if os.path.exists('%s/exclude.ini'%workdir):
os.remove('%s/exclude.ini'%workdir)
shutil.copy('%s/exclude.ini'%self.TEST_DIR, workdir)
result[myname]['prerun_config'] = self.get_prerun_cmds() + ['rm -rf CMakeFiles CMakeCache.txt']
result[myname]['prerun_build'] = self.get_prerun_cmds()
result[myname]['prerun_run'] = self.get_prerun_cmds() + ['export OMP_NUM_THREADS=2', \
'ulimit -s unlimited', 'export LD_LIBRARY_PATH=${NETCDF}/lib:${LD_LIBRARY_PATH}' ]
result[myname]['prerun_kernel'] = self.get_prerun_kernel_cmds()
result[myname]['mpirun'] = 'mpirun.lsf'
#'export LD_LIBRARY_PATH=$NETCDF/lib:/glade/apps/opt/hdf5/1.8.12/gnu/4.8.12/lib:$LD_LIBRARY_PATH',
# create job submit script
with open('%s/homme.submit'%rundir, 'w') as fd:
fd.write(job_script%('16', '16', '\n'.join(result[myname]['prerun_run']), result[myname]['mpirun'], '%s/test_execs/perfTest/perfTest'%blddir, '%s/camBench.nl'%rundir))
if self.REBUILD or not os.path.exists(blddir) or len([name for name in os.listdir(blddir) if os.path.isfile(os.path.join(blddir, name))])==0:
# prepare prerun command
prerun_cmds = result[myname]['prerun_config']
# prepare cmake command
cmake_cmd = ['cmake']
cmake_cmd.append('-DHOMME_PROJID="STDD0002"')
cmake_cmd.append('-DENABLE_PERFTEST=TRUE')
cmake_cmd.append('-DENABLE_OPENMP=TRUE')
cmake_cmd.append('-DUSE_MPIEXEC="mpirun.lsf"')
cmake_cmd.append('-DCMAKE_C_COMPILER="/glade/apps/opt/modulefiles/ys/cmpwrappers/mpipcc"')
cmake_cmd.append('-DCMAKE_Fortran_COMPILER="/glade/apps/opt/modulefiles/ys/cmpwrappers/mpipf90"')
cmake_cmd.append('-DNETCDF_DIR:PATH=$NETCDF')
cmake_cmd.append('-DPNETCDF_DIR:PATH=$PNETCDF')
cmake_cmd.append('-DHDF5_DIR:PATH=/glade/apps/opt/hdf5/1.8.12/pgi/13.3')
cmake_cmd.append(tmpsrc)
if self.LEAVE_TEMP:
with open('%s/config_cmds.sh'%blddir, 'w') as f:
f.write('#!/bin/bash\n')
f.write('\n')
for cmd in prerun_cmds:
f.write(' %s\n'%cmd)
for cmd in cmake_cmd[:-1]:
f.write(' %s \\\n'%cmd)
f.write(' %s &> config.log'%cmake_cmd[-1])
os.chmod('%s/config_cmds.sh'%blddir, 0755)
out, err, retcode = run_shcmd('%s; %s'%('; '.join(prerun_cmds), ' '.join(cmake_cmd)), cwd=blddir)
if retcode != 0:
self.set_status(result, myname, self.FAILED, errmsg=err)
return result
# include.ini was created manually
self.set_status(result, myname, self.PASSED)
return result
def check_mode():
from kgen_utils import Config, run_shcmd
from utils import module_file_extensions
from api import parse, walk
from statements import Comment
from kgen_search import f2003_search_unknowns, SearchException
import logging
logger = logging.getLogger('kgen') # KGEN addition
logger.setLevel(logging.WARNING)
files = []
# collect source files
for path in Config.check_mode:
if os.path.basename(path).startswith('.'): continue
if os.path.isdir(path):
for root, dirnames, filenames in os.walk(os.path.abspath(path)):
for filename in filenames:
if os.path.basename(filename).startswith('.'): continue
fname, fext = os.path.splitext(filename)
if len(fext)>1 and fext.lower() in module_file_extensions:
files.append(os.path.join(root, filename))
elif os.path.isfile(path):
if os.path.isfile(path):
files.append(os.path.abspath(path))
else:
raise '%s is not a direcotory nor a file'%path
# TODO: support #include cpp directive
# parse source files
for n, file in enumerate(files):
print 'Reading(%d/%d): '%(n+1, len(files)), file
# fsrc = open(file, 'rb')
# prepare include paths and macro definitions
path_src = []
macros_src = []
if Config.include['file'].has_key(self.abspath):
path_src = Config.include['file'][self.abspath]['path']+[os.path.dirname(self.abspath)]
for k, v in Config.include['file'][self.abspath]['macro'].iteritems():
if v:
macros_src.append('-D%s=%s'%(k,v))
else:
macros_src.append('-D%s'%k)
includes = '-I'+' -I'.join(Config.include['path']+path_src)
macros_common = []
for k, v in Config.include['macro'].iteritems():
if v:
macros_common.append('-D%s=%s'%(k,v))
else:
macros_common.append('-D%s'%k)
macros = ' '.join(macros_common + macros_src)
# execute preprocessing
prep = Config.bin['pp']
if prep.endswith('fpp'): flags = Config.bin['fpp_flags']
elif prep.endswith('cpp'): flags = Config.bin['cpp_flags']
else: raise UserException('Preprocessor is not either fpp or cpp')
output, err, retcode = run_shcmd('%s %s %s %s %s' % (prep, flags, includes, macros, file))
# convert the preprocessed for fparser
prep = map(lambda l: '!KGEN'+l if l.startswith('#') else l, output.split('\n'))
# fparse
tree = parse('\n'.join(prep), ignore_comments=False, analyze=False, isfree=True, isstrict=False, \
include_dirs=None, source_only=None )
# parse f2003
Config.search['promote_exception'] = True
lineno = 0
linediff = 0
for stmt, depth in walk(tree, -1):
try:
if isinstance(stmt, Comment) and stmt.item.comment.startswith('!KGEN#'):
comment_split = stmt.item.comment.split(' ')
lineno = int(comment_split[1])
stmt.item.span = ( 0, 0 )
else:
if lineno>0:
linediff = stmt.item.span[0] - lineno
lineno = 0
stmt.item.span = ( stmt.item.span[0]-linediff, stmt.item.span[1]-linediff )
stmt.parse_f2003()
if stmt.f2003.__class__ not in exclude_list:
f2003_search_unknowns(stmt, stmt.f2003, gentype=KGGenType.KERNEL)
except (NoMatchError, AttributeError) as e:
if file not in not_parsed:
not_parsed[file] = []
not_parsed[file].append(stmt)
except NameError as e:
errmsg = str(e)
pos = errmsg.find('search_')
if len(errmsg)>7 and pos>0:
clsname = errmsg[pos+7:-16]
#print "NOT SUPPORTED: '%s' Fortran statement is not supported yet"%clsname
if file not in not_supported:
not_supported[file] = []
not_supported[file].append((clsname, stmt.item.span[0]))
except Exception as e:
print 'WARNING: Following statement is not correctly parsed'
print stmt
print ''
print ''
print '********************'
print '*** CHECK RESULT ***'
print '********************'
print ''
print 'NOTE: KGEN may be able to extract kernel even though not all source code lines are parsed or supported.'
print ''
print '*** KGEN Parsing Error(s) ***'
print ''
for file, stmts in not_parsed.iteritems():
print file
lines = []
for stmt in stmts:
if hasattr(stmt, 'item'):
lines.append('Near line # %d:'%stmt.item.span[0])
lines.append(stmt.tokgen()+'\n')
else:
lines.append(str(stmt)+'\n')
print '\n'.join(lines), '\n'
print '*** Not Supported Fortran Statement(s) ***'
print ''
for file, clsnames in not_supported.iteritems():
print file
lines = []
for clsname, lineno in clsnames:
lines.append("'%s' Fortran statment near line # %d"%(clsname, lineno))
print '\n'.join(lines), '\n'
if len(not_parsed)==0 and len(not_supported)==0:
print 'Current KGEN version can support all source code lines.'
def check_mode():
from kgen_utils import Config, run_shcmd
from utils import module_file_extensions
from api import parse, walk
from statements import Comment
from kgen_search import f2003_search_unknowns, SearchException
import logging
logger = logging.getLogger('kgen') # KGEN addition
logger.setLevel(logging.WARNING)
files = []
# collect source files
for path in Config.check_mode:
if os.path.basename(path).startswith('.'): continue
if os.path.isdir(path):
for root, dirnames, filenames in os.walk(os.path.abspath(path)):
for filename in filenames:
if os.path.basename(filename).startswith('.'): continue
fname, fext = os.path.splitext(filename)
if len(fext) > 1 and fext.lower(
) in module_file_extensions:
files.append(os.path.join(root, filename))
elif os.path.isfile(path):
if os.path.isfile(path):
files.append(os.path.abspath(path))
else:
raise '%s is not a direcotory nor a file' % path
# TODO: support #include cpp directive
# parse source files
for n, file in enumerate(files):
print 'Reading(%d/%d): ' % (n + 1, len(files)), file
# fsrc = open(file, 'rb')
# prepare include paths and macro definitions
path_src = []
macros_src = []
if Config.include['file'].has_key(self.abspath):
path_src = Config.include['file'][self.abspath]['path'] + [
os.path.dirname(self.abspath)
]
for k, v in Config.include['file'][
self.abspath]['macro'].iteritems():
if v:
macros_src.append('-D%s=%s' % (k, v))
else:
macros_src.append('-D%s' % k)
includes = '-I' + ' -I'.join(Config.include['path'] + path_src)
macros_common = []
for k, v in Config.include['macro'].iteritems():
if v:
macros_common.append('-D%s=%s' % (k, v))
else:
macros_common.append('-D%s' % k)
macros = ' '.join(macros_common + macros_src)
# execute preprocessing
prep = Config.bin['pp']
if prep.endswith('fpp'): flags = Config.bin['fpp_flags']
elif prep.endswith('cpp'): flags = Config.bin['cpp_flags']
else: raise UserException('Preprocessor is not either fpp or cpp')
output, err, retcode = run_shcmd('%s %s %s %s %s' %
(prep, flags, includes, macros, file))
# convert the preprocessed for fparser
prep = map(lambda l: '!KGEN' + l if l.startswith('#') else l,
output.split('\n'))
# fparse
tree = parse('\n'.join(prep), ignore_comments=False, analyze=False, isfree=True, isstrict=False, \
include_dirs=None, source_only=None )
# parse f2003
Config.search['promote_exception'] = True
lineno = 0
linediff = 0
for stmt, depth in walk(tree, -1):
try:
if isinstance(
stmt,
Comment) and stmt.item.comment.startswith('!KGEN#'):
comment_split = stmt.item.comment.split(' ')
lineno = int(comment_split[1])
stmt.item.span = (0, 0)
else:
if lineno > 0:
linediff = stmt.item.span[0] - lineno
lineno = 0
stmt.item.span = (stmt.item.span[0] - linediff,
stmt.item.span[1] - linediff)
stmt.parse_f2003()
if stmt.f2003.__class__ not in exclude_list:
f2003_search_unknowns(stmt,
stmt.f2003,
gentype=KGGenType.KERNEL)
except (NoMatchError, AttributeError) as e:
if file not in not_parsed:
not_parsed[file] = []
not_parsed[file].append(stmt)
except NameError as e:
errmsg = str(e)
pos = errmsg.find('search_')
if len(errmsg) > 7 and pos > 0:
clsname = errmsg[pos + 7:-16]
#print "NOT SUPPORTED: '%s' Fortran statement is not supported yet"%clsname
if file not in not_supported:
not_supported[file] = []
not_supported[file].append((clsname, stmt.item.span[0]))
except Exception as e:
print 'WARNING: Following statement is not correctly parsed'
print stmt
print ''
print ''
print '********************'
print '*** CHECK RESULT ***'
print '********************'
print ''
print 'NOTE: KGEN may be able to extract kernel even though not all source code lines are parsed or supported.'
print ''
print '*** KGEN Parsing Error(s) ***'
print ''
for file, stmts in not_parsed.iteritems():
print file
lines = []
for stmt in stmts:
if hasattr(stmt, 'item'):
lines.append('Near line # %d:' % stmt.item.span[0])
lines.append(stmt.tokgen() + '\n')
else:
lines.append(str(stmt) + '\n')
print '\n'.join(lines), '\n'
print '*** Not Supported Fortran Statement(s) ***'
print ''
for file, clsnames in not_supported.iteritems():
print file
lines = []
for clsname, lineno in clsnames:
lines.append("'%s' Fortran statment near line # %d" %
(clsname, lineno))
print '\n'.join(lines), '\n'
if len(not_parsed) == 0 and len(not_supported) == 0:
print 'Current KGEN version can support all source code lines.'
def main():
from compflag_tool import CompFlagDetect
from kext_tool import KExtTool
version = [ 0, 7, '1' ]
outdir = '.'
retval = 0
Logger.info('Starting KGen', stdout=True)
try:
# option parser
parser = optparse.OptionParser(version='KGEN version %d.%d.%s'%tuple(version))
# common options
parser.add_option("--outdir", dest="outdir", action='store', type='string', default=None, help="path to create outputs")
parser.add_option("--rebuild", dest="rebuild", action='append', type='string', default=None, help="force to rebuild")
parser.add_option("--prerun", dest="prerun", action='append', type='string', default=None, help="prerun commands")
parser.add_option("-i", "--include-ini", dest="include_ini", action='store', type='string', default=None, help="INI options")
parser.add_option("--debug", dest="debug", action='append', type='string', help=optparse.SUPPRESS_HELP)
# kapp options
parser.add_option("-c", "--cmd-clean", dest="cmd_clean", action='store', type='string', default=None, help="Linux command(s) to clean a target application build")
parser.add_option("-b", "--cmd-build", dest="cmd_build", action='store', type='string', default=None, help="Linux command(s) to build a target application")
parser.add_option("-r", "--cmd-run", dest="cmd_run", action='store', type='string', default=None, help="Linux command(s) to run a target application")
# compflag options
parser.add_option("--strace", dest="strace", action='store', type='string', default=None, help="strace options")
# kext options
parser.add_option("--invocation", dest="invocation", action='append', type='string', default=None, help="(process, thread, invocation) pairs of kernel for data collection")
parser.add_option("-e", "--exclude-ini", dest="exclude_ini", action='store', type='string', default=None, help="information excluded for analysis")
parser.add_option("--kernel-option", dest="kernel_option", action='append', type='string', default=None, help="compiler and linker options for Kgen-generated kernel")
parser.add_option("--openmp", dest="openmp", action='append', type='string', default=None, help="Specifying OpenMP options")
parser.add_option("--mpi", dest="mpi", action='append', type='string', default=None, help="MPI information for data collection")
parser.add_option("--timing", dest="timing", action='append', type='string', default=None, help="Timing measurement information")
parser.add_option("--intrinsic", dest="intrinsic", action='append', type='string', default=None, help="Specifying resolution for intrinsic procedures during searching")
parser.add_option("--check", dest="check", action='append', type='string', default=None, help="Kernel correctness check information")
parser.add_option("--verbose", dest="verbose", action='store', type='string', default=None, help="Set the verbose level for verification output")
parser.add_option("--add-mpi-frame", dest="add_mpi_frame", type='string', default=None, help='Add MPI frame codes in kernel_driver.')
parser.add_option("--source", dest="source", action='append', type='string', default=None, help="Setting source file related properties")
parser.add_option("--logging", dest="logging", action='append', type='string', help=optparse.SUPPRESS_HELP)
opts, args = parser.parse_args()
if len(args)<1:
print 'ERROR: Target source files is not provided.'
print 'Usage: kgen [options] <target file path[:namepath]> --cmd-clean <commands> --cmd-build <commands> --cmd-run <commands>'
sys.exit(-1)
if opts.cmd_clean is None:
print 'ERROR: No clean command is prvoided in command line. Please add --cmd-clean option.'
print 'Usage: kgen [options] <target file path[:namepath]> --cmd-clean <commands> --cmd-build <commands> --cmd-run <commands>'
sys.exit(-1)
if opts.cmd_build is None:
print 'ERROR: No build command is prvoided in command line. Please add --cmd-build option.'
print 'Usage: kgen [options] <target file path[:namepath]> --cmd-clean <commands> --cmd-build <commands> --cmd-run <commands>'
sys.exit(-1)
if opts.cmd_run is None:
print 'ERROR: No run command is prvoided in command line. Please add --cmd-run option.'
print 'Usage: kgen [options] <target file path[:namepath]> --cmd-clean <commands> --cmd-build <commands> --cmd-run <commands>'
sys.exit(-1)
kext_argv = []
compflag_argv = []
# collect common options
if opts.outdir:
kext_argv.append('--outdir')
kext_argv.append(opts.outdir)
compflag_argv.append('--build')
compflag_argv.append('cwd="%s",clean="%s"'%(opts.outdir, opts.cmd_clean))
outdir = opts.outdir
else:
kext_argv.append('--outdir')
kext_argv.append(os.getcwd())
compflag_argv.append('--build')
compflag_argv.append('cwd="%s",clean="%s"'%(os.getcwd(), opts.cmd_clean))
outdir = os.getcwd()
if opts.prerun:
compflag_argv.append('--prerun')
compflag_argv.extend(opts.prerun)
kext_argv.append('--prerun')
kext_argv.extend(opts.prerun)
if opts.rebuild:
compflag_argv.append('--rebuild')
compflag_argv.extend(opts.rebuild)
kext_argv.append('--rebuild')
kext_argv.extend(opts.rebuild)
if opts.include_ini:
compflag_argv.append('--include_ini')
compflag_argv.append(opts.include_ini)
kext_argv.append('--include-ini')
kext_argv.append(opts.include_ini)
if opts.debug:
compflag_argv.append('--debug')
compflag_argv.extend(opts.debug)
kext_argv.append('--debug')
kext_argv.extend(opts.debug)
# collect compflag options
if opts.strace:
compflag_argv.append('--strace')
compflag_argv.append(opts.strace)
compflag_argv.append(opts.cmd_build)
# collect kext options
if opts.invocation:
kext_argv.append('--invocation')
kext_argv.extend(opts.invocation)
if opts.exclude_ini:
kext_argv.append('--exclude-ini')
kext_argv.append(opts.exclude_ini)
if opts.source:
kext_argv.append('--source')
kext_argv.extend(opts.source)
if opts.mpi:
kext_argv.append('--mpi')
kext_argv.extend(opts.mpi)
if opts.openmp:
kext_argv.append('--openmp')
kext_argv.extend(opts.openmp)
if opts.timing:
kext_argv.append('--timing')
kext_argv.extend(opts.timing)
if opts.intrinsic:
kext_argv.append('--intrinsic')
kext_argv.extend(opts.intrinsic)
if opts.check:
kext_argv.append('--check')
kext_argv.extend(opts.check)
if opts.kernel_option:
kext_argv.append('--kernel-option')
kext_argv.extend(opts.kernel_option)
if opts.verbose:
kext_argv.append('--verbose')
kext_argv.append(opts.verbose)
if opts.add_mpi_frame:
kext_argv.append('--add-mpi-frame')
kext_argv.append(opts.add_mpi_frame)
if opts.logging:
kext_argv.append('--logging')
kext_argv.extend(opts.logging)
kext_argv.append('--state-clean')
kext_argv.append('cmds=%s'%opts.cmd_clean)
kext_argv.append('--state-build')
kext_argv.append('cmds=%s'%opts.cmd_build)
kext_argv.append('--state-run')
kext_argv.append('cmds=%s'%opts.cmd_run)
kext_argv.append(args[0])
# run compflag
compflag = CompFlagDetect()
compflag.init(argv=compflag_argv)
compflag.main()
flags = compflag.fini()
# run kext
kext = KExtTool()
kext.init()
kext_argv.extend( [ '-i', flags['incini'] ] )
Config.apply(argv=kext_argv)
kext.main()
extracts = kext.fini()
# extracts contain kernel files, state files
# parse rebuild option
is_rebuild = False
if opts.rebuild:
for r in opts.rebuild:
if isinstance(r, str):
subopts = r.split(',')
for subopt in subopts:
if subopt in [ 'all', 'state' ]:
is_rebuild = True
if is_rebuild: break
# check if state files exist
has_statefiles = False
if os.path.exists('%s/kernel'%outdir):
statefiles = glob.glob('%s/kernel/%s.*.*.*'%(outdir, State.kernel['name']))
if len(statefiles)>0:
has_statefiles = True
# generate state
if is_rebuild or not has_statefiles:
Logger.info('Generating state data files at %s/state.'%outdir, stdout=True)
out, err, retcode = run_shcmd('make', cwd='%s/state'%outdir)
if retcode != 0:
Logger.info('FAILED: %s'%err, stdout=True)
else:
Logger.info('Reusing state data files at %s/kernel'%outdir, stdout=True)
except UserException as e:
print 'ERROR: %s'%str(e)
Logger.info(e)
#Logger.critical(e)
retval = -1
except ProgramException as e:
Logger.critical(e)
retval = -1
except Exception as e:
Logger.critical(e)
retval = -1
finally:
if os.path.exists('%s/state/Makefile'%outdir):
out, err, retcode = run_shcmd('make recover_from_locals', cwd='%s/state'%outdir)
Logger.info('KGen is finished.', stdout=True)
return retval
def main():
from compflag_tool import CompFlagDetect
from coverage_tool import CoverageDetect
from kext_tool import KExtTool
version = [0, 7, '2']
outdir = '.'
retval = 0
Logger.info('Starting KGen', stdout=True)
# add options for all of kgen tools
try:
# option parser
parser = optparse.OptionParser(version='KGEN version %d.%d.%s' %
tuple(version))
# common options
parser.add_option("--outdir",
dest="outdir",
action='store',
type='string',
default=None,
help="path to create outputs")
parser.add_option("--rebuild",
dest="rebuild",
action='append',
type='string',
default=None,
help="force to rebuild")
parser.add_option("--prerun",
dest="prerun",
action='append',
type='string',
default=None,
help="prerun commands")
parser.add_option("-i",
"--include-ini",
dest="include_ini",
action='store',
type='string',
default=None,
help="INI options")
parser.add_option("--debug",
dest="debug",
action='append',
type='string',
help=optparse.SUPPRESS_HELP)
# kapp options
parser.add_option(
"-c",
"--cmd-clean",
dest="cmd_clean",
action='store',
type='string',
default=None,
help="Linux command(s) to clean a target application build")
parser.add_option(
"-b",
"--cmd-build",
dest="cmd_build",
action='store',
type='string',
default=None,
help="Linux command(s) to build a target application")
parser.add_option("-r",
"--cmd-run",
dest="cmd_run",
action='store',
type='string',
default=None,
help="Linux command(s) to run a target application")
# compflag options
parser.add_option("--strace",
dest="strace",
action='store',
type='string',
default=None,
help="strace options")
# coverage options
#parser.add_option("--strace", dest="strace", action='store', type='string', default=None, help="strace options")
# kext options
parser.add_option(
"--invocation",
dest="invocation",
action='append',
type='string',
default=None,
help=
"(process, thread, invocation) pairs of kernel for data collection"
)
parser.add_option("-e",
"--exclude-ini",
dest="exclude_ini",
action='store',
type='string',
default=None,
help="information excluded for analysis")
parser.add_option(
"--kernel-option",
dest="kernel_option",
action='append',
type='string',
default=None,
help="compiler and linker options for Kgen-generated kernel")
parser.add_option("--openmp",
dest="openmp",
action='append',
type='string',
default=None,
help="Specifying OpenMP options")
parser.add_option("--mpi",
dest="mpi",
action='append',
type='string',
default=None,
help="MPI information for data collection")
parser.add_option("--timing",
dest="timing",
action='append',
type='string',
default=None,
help="Timing measurement information")
parser.add_option(
"--intrinsic",
dest="intrinsic",
action='append',
type='string',
default=None,
help=
"Specifying resolution for intrinsic procedures during searching")
parser.add_option("--check",
dest="check",
action='append',
type='string',
default=None,
help="Kernel correctness check information")
parser.add_option("--verbose",
dest="verbose",
action='store',
type='string',
default=None,
help="Set the verbose level for verification output")
parser.add_option("--add-mpi-frame",
dest="add_mpi_frame",
type='string',
default=None,
help='Add MPI frame codes in kernel_driver.')
parser.add_option("--source",
dest="source",
action='append',
type='string',
default=None,
help="Setting source file related properties")
parser.add_option("--logging",
dest="logging",
action='append',
type='string',
help=optparse.SUPPRESS_HELP)
opts, args = parser.parse_args()
if len(args) < 1:
print 'ERROR: Target source files is not provided.'
print 'Usage: kgen [options] <target file path[:namepath]> --cmd-clean <commands> --cmd-build <commands> --cmd-run <commands>'
sys.exit(-1)
if opts.cmd_clean is None:
print 'ERROR: No clean command is prvoided in command line. Please add --cmd-clean option.'
print 'Usage: kgen [options] <target file path[:namepath]> --cmd-clean <commands> --cmd-build <commands> --cmd-run <commands>'
sys.exit(-1)
if opts.cmd_build is None:
print 'ERROR: No build command is prvoided in command line. Please add --cmd-build option.'
print 'Usage: kgen [options] <target file path[:namepath]> --cmd-clean <commands> --cmd-build <commands> --cmd-run <commands>'
sys.exit(-1)
if opts.cmd_run is None:
print 'ERROR: No run command is prvoided in command line. Please add --cmd-run option.'
print 'Usage: kgen [options] <target file path[:namepath]> --cmd-clean <commands> --cmd-build <commands> --cmd-run <commands>'
sys.exit(-1)
# seletively copy user inputs for each kgen tools
compflag_argv = []
coverage_argv = []
kext_argv = []
#kgen_argv = []
# collect common options
if opts.outdir:
kext_argv.append('--outdir')
kext_argv.append(opts.outdir)
compflag_argv.append('--build')
compflag_argv.append('cwd="%s",clean="%s"' %
(opts.outdir, opts.cmd_clean))
coverage_argv.append('--outdir')
coverage_argv.append(opts.outdir)
outdir = opts.outdir
if not os.path.exists(outdir):
os.mkdir(outdir)
else:
kext_argv.append('--outdir')
kext_argv.append(os.getcwd())
compflag_argv.append('--build')
compflag_argv.append('cwd="%s",clean="%s"' %
(os.getcwd(), opts.cmd_clean))
coverage_argv.append('--outdir')
coverage_argv.append(os.getcwd())
outdir = os.getcwd()
if opts.prerun:
coverage_argv.append('--prerun')
coverage_argv.extend(opts.prerun)
compflag_argv.append('--prerun')
compflag_argv.extend(opts.prerun)
kext_argv.append('--prerun')
kext_argv.extend(opts.prerun)
if opts.rebuild:
coverage_argv.append('--rebuild')
coverage_argv.extend(opts.rebuild)
compflag_argv.append('--rebuild')
compflag_argv.extend(opts.rebuild)
kext_argv.append('--rebuild')
kext_argv.extend(opts.rebuild)
if opts.include_ini:
compflag_argv.append('--include_ini')
compflag_argv.append(opts.include_ini)
kext_argv.append('--include-ini')
kext_argv.append(opts.include_ini)
if opts.debug:
coverage_argv.append('--debug')
coverage_argv.extend(opts.debug)
compflag_argv.append('--debug')
compflag_argv.extend(opts.debug)
kext_argv.append('--debug')
kext_argv.extend(opts.debug)
# collect compflag options
if opts.strace:
compflag_argv.append('--strace')
compflag_argv.append(opts.strace)
compflag_argv.append(opts.cmd_build)
# collect coverage options
coverage_argv.append('--cmd-clean')
coverage_argv.append(opts.cmd_clean)
coverage_argv.append('--cmd-build')
coverage_argv.append(opts.cmd_build)
coverage_argv.append('--cmd-run')
coverage_argv.append(opts.cmd_run)
coverage_argv.append(args[0])
# collect kext options
if opts.invocation:
kext_argv.append('--invocation')
kext_argv.extend(opts.invocation)
if opts.exclude_ini:
kext_argv.append('--exclude-ini')
kext_argv.append(opts.exclude_ini)
if opts.source:
kext_argv.append('--source')
kext_argv.extend(opts.source)
if opts.mpi:
kext_argv.append('--mpi')
kext_argv.extend(opts.mpi)
if opts.openmp:
kext_argv.append('--openmp')
kext_argv.extend(opts.openmp)
if opts.timing:
kext_argv.append('--timing')
kext_argv.extend(opts.timing)
if opts.intrinsic:
kext_argv.append('--intrinsic')
kext_argv.extend(opts.intrinsic)
if opts.check:
kext_argv.append('--check')
kext_argv.extend(opts.check)
if opts.kernel_option:
kext_argv.append('--kernel-option')
kext_argv.extend(opts.kernel_option)
if opts.verbose:
kext_argv.append('--verbose')
kext_argv.append(opts.verbose)
if opts.add_mpi_frame:
kext_argv.append('--add-mpi-frame')
kext_argv.append(opts.add_mpi_frame)
if opts.logging:
kext_argv.append('--logging')
kext_argv.extend(opts.logging)
kext_argv.append('--state-clean')
kext_argv.append('cmds=%s' % opts.cmd_clean)
kext_argv.append('--state-build')
kext_argv.append('cmds=%s' % opts.cmd_build)
kext_argv.append('--state-run')
kext_argv.append('cmds=%s' % opts.cmd_run)
kext_argv.append(args[0])
# run compflag
compflag = CompFlagDetect()
compflag.init(argv=compflag_argv)
compflag.main()
CF_flags = compflag.fini()
Config.apply()
analyze()
#kext = KExtTool()
#kext.init()
# collect command line inputs and parse
#kgen_argv.extend(kext_argv)
#Config.apply(argv=kgen_argv)
if not opts.invocation:
coverage = CoverageDetect()
coverage.init(argv=coverage_argv)
#Logger.info('Program is analyzed', stdout=True)
coverage_argv.append(opts.cmd_clean)
coverage_argv.append(opts.cmd_build)
coverage_argv.append(opts.cmd_run)
# run coverage
coverage.main()
# parse coverage rebuild option
is_rebuild_coverage = False
if opts.rebuild:
for r in opts.rebuild:
if isinstance(r, str):
subopts = r.split(',')
for subopt in subopts:
if subopt in ['all', 'coverage']:
is_rebuild_coverage = True
if is_rebuild_coverage: break
# check if coverage file exist
has_coveragefile = False
if os.path.exists('%s/coverage' % outdir):
coveragefile = glob.glob('%s/coverage/%s.*' %
(outdir, State.kernel['name']))
if len(coveragefile) > 0:
has_coveragefile = True
# generate coverage file
if is_rebuild_coverage or not has_coveragefile:
Logger.info('Generating coverage file at %s/coverage' % outdir,
stdout=True)
out, err, retcode = run_shcmd('make',
cwd='%s/coverage' % outdir)
if retcode != 0:
Logger.info('FAILED: %s' % err, stdout=True)
else:
Logger.info('Reusing coverage at %s/coverage' % outdir,
stdout=True)
CV_flags = coverage.fini()
kext_argv.extend(['--invocation', CV_flags['invocation']])
kext_argv.extend(['-i', CF_flags['incini']])
kext = KExtTool()
kext.init()
# run kext
kext.main()
extracts = kext.fini()
# extracts contain kernel files, state files
# parse kext rebuild option
is_rebuild_kext = False
if opts.rebuild:
for r in opts.rebuild:
if isinstance(r, str):
subopts = r.split(',')
for subopt in subopts:
if subopt in ['all', 'state']:
is_rebuild_kext = True
if is_rebuild_kext: break
# check if state files exist
has_statefiles = False
if os.path.exists('%s/kernel' % outdir):
statefiles = glob.glob('%s/kernel/%s.*.*.*' %
(outdir, State.kernel['name']))
if len(statefiles) > 0:
has_statefiles = True
# generate state
if is_rebuild_kext or not has_statefiles:
Logger.info('Generating state data files at %s/state.' % outdir,
stdout=True)
out, err, retcode = run_shcmd('make clean',
cwd='%s/state' % outdir)
out, err, retcode = run_shcmd('make run', cwd='%s/state' % outdir)
if retcode != 0:
Logger.info('FAILED: %s' % err, stdout=True)
else:
Logger.info('Reusing state data files at %s/kernel' % outdir,
stdout=True)
except UserException as e:
print 'ERROR: %s' % str(e)
Logger.info(e)
#Logger.critical(e)
retval = -1
except ProgramException as e:
Logger.critical(e)
retval = -1
except Exception as e:
Logger.critical(e)
retval = -1
finally:
if os.path.exists('%s/state/Makefile' % outdir):
out, err, retcode = run_shcmd('make recover_from_locals',
cwd='%s/state' % outdir)
Logger.info('KGen is finished.', stdout=True)
return retval
def __init__(self, srcpath, preprocess=True):
import os.path
from kgen_utils import run_shcmd
from statements import Comment
from block_statements import Module, Program
# set default values
self.tree = None
self.srcpath = srcpath
self.abspath = os.path.abspath(self.srcpath)
# set source file format
isfree = True
isstrict = False
if self.abspath in Config.source["file"].keys():
if Config.source["file"][self.abspath].has_key("isfree"):
isfree = Config.source["file"][self.abspath]["isfree"]
if Config.source["file"][self.abspath].has_key("isstrict"):
isstrict = Config.source["file"][self.abspath]["isstrict"]
else:
if Config.source["isstrict"]:
isstrict = Config.source["isstrict"]
if Config.source["isfree"]:
isfree = Config.source["isfree"]
# prepare include paths and macro definitions
path_src = []
macros_src = []
if Config.include["file"].has_key(self.abspath):
path_src = Config.include["file"][self.abspath]["path"] + [os.path.dirname(self.abspath)]
for k, v in Config.include["file"][self.abspath]["macro"].iteritems():
if v:
macros_src.append("-D%s=%s" % (k, v))
else:
macros_src.append("-D%s" % k)
includes = "-I" + " -I".join(Config.include["path"] + path_src)
macros_common = []
for k, v in Config.include["macro"].iteritems():
if v:
macros_common.append("-D%s=%s" % (k, v))
else:
macros_common.append("-D%s" % k)
macros = " ".join(macros_common + macros_src)
# execute preprocessing
Logger.info("Reading %s" % self.srcpath, stdout=True)
new_lines = []
with open(self.abspath, "r") as f:
if preprocess:
pp = Config.bin["pp"]
if pp.endswith("fpp"):
if isfree:
srcfmt = " -free"
else:
srcfmt = " -fixed"
flags = Config.bin["fpp_flags"] + srcfmt
elif pp.endswith("cpp"):
flags = Config.bin["cpp_flags"]
else:
raise UserException("Preprocessor is not either fpp or cpp")
output, err, retcode = run_shcmd("%s %s %s %s" % (pp, flags, includes, macros), input=f.read())
prep = map(lambda l: "!KGEN" + l if l.startswith("#") else l, output.split("\n"))
new_lines = self.handle_include(prep)
else:
new_lines = f.read().split("\n")
# add include paths
if Config.include["file"].has_key(self.abspath) and Config.include["file"][self.abspath].has_key("path"):
include_dirs = Config.include["file"][self.abspath]["path"] + [os.path.dirname(self.abspath)]
else:
include_dirs = None
# if self.abspath=='/glade/scratch/youngsun/kgen_system_test/branches/initial/MPAS-Release/src/framework/mpas_derived_types.F':
# print '\n'.join(new_lines)
# sys.exit()
# import pdb ; pdb.set_trace()
# fparse
self.tree = parse(
"\n".join(new_lines),
ignore_comments=False,
analyze=True,
isfree=isfree,
isstrict=isstrict,
include_dirs=include_dirs,
source_only=None,
)
self.tree.prep = new_lines
self.tree.used4genstate = False
# if self.abspath=='/glade/scratch/youngsun/kgen_system_test/branches/initial/MPAS-Release/src/framework/mpas_derived_types.F':
# print self.tree
# sys.exit()
# parse f2003
lineno = 0
linediff = 0
for stmt, depth in walk(self.tree, -1):
stmt.parse_f2003()
# rename reader.id
self.tree.reader.id = self.abspath
# collect module information
for mod_name, mod_stmt in self.tree.a.module.iteritems():
if not State.modules.has_key(mod_name):
State.modules[mod_name] = OrderedDict()
# State.modules[mod_name]['num'] = State.mod_num
# State.mod_num += 1
State.modules[mod_name]["stmt"] = mod_stmt
State.modules[mod_name]["file"] = self
State.modules[mod_name]["path"] = self.abspath
# State.modules[mod_name]['extern'] = OrderedDict()
# State.modules[mod_name]['extern']['names'] = []
# State.modules[mod_name]['extern']['typedecl_stmt'] = OrderedDict()
# State.modules[mod_name]['extern']['tkdpat'] = []
# State.modules[mod_name]['mod_rw_var_depends'] = []
# State.modules[mod_name]['dtype'] = []
# collect program unit information
for item in self.tree.content:
if item.__class__ not in [Module, Comment, Program]:
if item.reader.id not in State.program_units.keys():
State.program_units[item.reader.id] = []
State.program_units[item.reader.id].append(item)
# create a tuple for file dependency
State.srcfiles[self.abspath] = (self, [], [])