def flat_items(items):
if isinstance(items[0], str):
return [SrcFile.applymap(t) for t in items]
elif isinstance(items[0], tuple):
return [SrcFile.applymap(t[0]) for t, _ in items]
else:
raise ProgramException('Unknown type: %s' % items[0].__class__)
def openmp(inputfile, target_stmt, sentinel, directive, clauses, span):
line = ''
new_target_stmt = None
for stmt, depth in walk(inputfile.tree, -1):
if stmt.item.span==span:
new_target_stmt = stmt
if clauses:
mapped_clauses = SrcFile.applymap(clauses[0])
else:
mapped_clauses = ''
line = '%s %s %s'%(SrcFile.applymap(sentinel[0]), SrcFile.applymap(directive[0]), mapped_clauses)
break
if line:
parsed = parse(line, analyze=False, ignore_comments=False)
if len(parsed.content)>0:
#parsed.content[0].parent = target_stmt.parent
#import pdb; pdb.set_trace()
for stmt, depth in walk(parsed, -1): stmt.parse_f2003()
insert_content(new_target_stmt, parsed.content, remove_olditem=True)
def directive(inputfile, target_stmt, label, sentinel, directive, span):
line = ''
new_target_stmt = None
for stmt, depth in walk(inputfile.tree, -1):
if stmt.item.span==span:
new_target_stmt = stmt
line = '!%s$ %s'%(SrcFile.applymap(sentinel[0]), SrcFile.applymap(directive[0]))
break
if line:
parsed = parse(line, analyze=False, ignore_comments=False)
if len(parsed.content)>0:
for stmt, depth in walk(parsed, 1):
if isinstance(stmt, Comment):
stmt.label = int(label[0])
else:
stmt.item.label = int(label[0])
stmt.parse_f2003()
insert_content(new_target_stmt, parsed.content, remove_olditem=True)
def __init__(self):
self.refcase = None
self.ranking = [] # (casenum, caseorder, performance )
self.failed = [] # (casenum, caseorder, performance )
self.casenums = []
# configure for case generation
if State.direct.has_key('casegen'):
if len(State.direct['casegen'])!=1:
raise UserException('Only one CASEGEN directive is allowed')
sub = State.direct['casegen'][0][0] # (sub, stmt, span)
subsub = sub.cases[0][0][0]
item = SrcFile.applymap(subsub.case[0][0][0])
attrs = SrcFile.applymap(subsub.case[0][1])
if item=='rand':
self.selectfunc, self.prefunc, self.postfunc, self.updatefunc = casegen_functions_random
elif item=='dtree':
self.selectfunc, self.prefunc, self.postfunc, self.updatefunc = casegen_functions_dtree
else: raise UserException('%s is not valid cage generation algorithm'%item)
else:
self.selectfunc, self.prefunc, self.postfunc, self.updatefunc = casegen_functions_random
# configure for measurment
self.measure = {}
for sub, stmt, span in State.direct['measure']:
for subsub in sub.cases[0][0]:
item = SrcFile.applymap(subsub.case[0][0][0])
attrs = SrcFile.applymap(subsub.case[0][1])
self.measure[item] = attrs
# configure for verification
self.verify = {}
for sub, stmt, span in State.direct['verify']:
for subsub in sub.cases[0][0]:
item = SrcFile.applymap(subsub.case[0][0][0])
attrs = SrcFile.applymap(subsub.case[0][1])
self.verify[item] = attrs
self.rank_var = State.direct['rank'][0][0].cases[0][0][0].case[0][0][0]
self.rank_attrs = State.direct['rank'][0][0].cases[0][0][0].case[0][1]
# build Neural Network
casenum, casenumseq, directs, objs = get_directs( self.selectfunc, self.prefunc, self.postfunc)
self.casesizes = []
for caseidx, caseobj in casenumseq:
self.casesizes.append(max(1, int(math.ceil(math.log(caseobj.size, 2)))))
self.NN_input_size = sum(self.casesizes)
self.NN_hidden_layers = 4
self.NN_target_size = 1
self.NN_net = buildNetwork( self.NN_input_size, self.NN_hidden_layers, self.NN_target_size, bias = True )
self.NN_trainer = BackpropTrainer(self.NN_net, momentum=0.1, weightdecay=0.01, learningrate=0.01)
self.NN_ds = SupervisedDataSet( self.NN_input_size, self.NN_target_size )
self.NN_basket_size = 1000
self.NN_min_dataset_size = min(20, State.cases['size'])
self.NN_amp_factor = 1.0
def directive(inputfile, target_stmt, label, sentinel, directive, span):
line = ''
new_target_stmt = None
for stmt, depth in walk(inputfile.tree, -1):
if stmt.item.span == span:
new_target_stmt = stmt
line = '!%s$ %s' % (SrcFile.applymap(
sentinel[0]), SrcFile.applymap(directive[0]))
break
if line:
parsed = parse(line, analyze=False, ignore_comments=False)
if len(parsed.content) > 0:
for stmt, depth in walk(parsed, 1):
if isinstance(stmt, Comment):
stmt.label = int(label[0])
else:
stmt.item.label = int(label[0])
stmt.parse_f2003()
insert_content(new_target_stmt,
parsed.content,
remove_olditem=True)
def generate_script(casenum, directs, srcfileids):
# collect directs
for direct in [ 'prerun', 'clean', 'build', 'execute', 'postrun' ]:
exec('%s = []'%direct)
for key, value in directs.iteritems():
if key.lower().startswith(direct):
exec('%s.extend(value[0])'%direct)
script_file = Config.path['workdir']+'/case_'+str(casenum)+'.sh'
fscr = open(script_file, 'w+')
# start of script
fscr.write('#!/bin/bash\n')
fscr.write('\n')
fscr.write('echo start of case ' + str(casenum)+ '\n')
# cd to workdir
fscr.write('\n')
fscr.write('cd %s\n'%Config.path['workdir'])
# prerun
fscr.write('\n')
fscr.write('# prerun\n')
for cmds, attr in prerun:
for cmd in cmds:
fscr.write(SrcFile.applymap(cmd)+'\n')
# create symlink
# fscr.write('\n')
# fscr.write('# symlink\n')
# for i, inputfile in enumerate(State.inputfile):
# #fscr.write('rm -f %s\n'%inputfile.filename)
# fscr.write('rm -f %s\n'%inputfile.abspath)
# if i in srcfileids:
# outpath = os.path.join(Config.path['outdir'], case_filename(inputfile.filename, casenum))
# fscr.write('ln -s %s %s\n'%(outpath, inputfile.filename))
# else:
# fscr.write('ln -s %s.oc_org %s\n'%(inputfile.filename, inputfile.filename))
# clean
fscr.write('\n')
fscr.write('# clean\n')
if len(clean)!=1: raise Exception('Only one clean directive is allowed')
clean_cmds, clean_attrs = clean[0]
if len(clean_cmds)!=1: raise Exception('Only one clean command is allowed')
clean_cmd = clean_cmds[0]
cline = clean_cmd
clean_makefile = 'Makefile'
clean_target = 'clean'
if clean_attrs.has_key('makefile'):
clean_makefile = SrcFile.applymap(clean_attrs['makefile'][0])
cline += ' -f %s'%clean_makefile
if clean_attrs.has_key('target'):
clean_target = SrcFile.applymap(clean_attrs['target'][0])
cline += ' %s'%clean_target
#dstpath = Config.path['outdir']+'/'+os.path.basename(clean_makefile)
#if os.path.exists(dstpath): os.remove(dstpath)
#if os.path.isabs(clean_makefile):
# copyfile(clean_makefile, dstpath)
#else:
# copyfile(Config.path['workdir']+'/'+clean_makefile, dstpath)
for macro, value in clean_attrs.iteritems():
if macro not in [ 'makefile', 'target']:
items = collect_items(value)
cline += ' %s="%s"'%(macro, ' '.join([ SrcFile.applymap(item) for item in items ]))
fscr.write(cline)
fscr.write('\n')
# build
fscr.write('\n')
fscr.write('# build\n')
if len(build)!=1: raise Exception('Only one build directive is allowed')
build_cmds, build_attrs = build[0]
if len(build_cmds)!=1: raise Exception('Only one build command is allowed')
build_cmd = build_cmds[0]
cline = build_cmd
build_makefile = 'Makefile'
build_target = 'build'
if build_attrs.has_key('makefile'):
build_makefile = SrcFile.applymap(build_attrs['makefile'][0])
cline += ' -f %s'%build_makefile
if build_attrs.has_key('target'):
build_target = SrcFile.applymap(build_attrs['target'][0])
cline += ' %s'%build_target
#if os.path.abspath(build_makefile)!=os.path.abspath(clean_makefile):
# dstpath = Config.path['outdir']+'/'+os.path.basename(build_makefile)
# if os.path.exists(dstpath): os.remove(dstpath)
# if os.path.isabs(build_makefile):
# copyfile(build_makefile, dstpath)
# else:
# copyfile(Config.path['workdir']+'/'+build_makefile, dstpath)
for macro, value in build_attrs.iteritems():
if macro not in [ 'makefile', 'target']:
items = collect_items(value)
cline += ' %s="%s"'%(macro, ' '.join([ SrcFile.applymap(item) for item in items ]))
fscr.write(cline)
fscr.write('\n')
# execute
fscr.write('\n')
fscr.write('# execute\n')
if len(execute)!=1: raise Exception('Only one execute directive is allowed')
execute_cmds, execute_attrs = execute[0]
if len(execute_cmds)!=1: raise Exception('Only one execute command is allowed')
execute_cmd = execute_cmds[0]
repeat = 1
sleep = 0.3
if execute_attrs.has_key('repeat'):
repeat = int(execute_attrs['repeat'][0])
if execute_attrs.has_key('sleep'):
sleep = float(execute_attrs['sleep'][0])
if repeat>1: fscr.write("for i in `seq 1 %s`; do\n"%repeat)
if repeat>1: cline = ' %s '%execute_cmd
else: cline = '%s '%execute_cmd
execute_makefile = 'Makefile'
execute_target = 'run'
if execute_attrs.has_key('makefile'):
execute_makefile = SrcFile.applymap(execute_attrs['makefile'][0])
cline += ' -f %s'%execute_makefile
if execute_attrs.has_key('target'):
execute_target = SrcFile.applymap(execute_attrs['target'][0])
cline += ' %s'%execute_target
#if os.path.abspath(execute_makefile)!=os.path.abspath(clean_makefile) and \
# os.path.abspath(execute_makefile)!=os.path.abspath(build_makefile):
# dstpath = Config.path['outdir']+'/'+os.path.basename(execute_makefile)
# if os.path.exists(dstpath): os.remove(dstpath)
# if os.path.isabs(build_makefile):
# copyfile(execute_makefile, dstpath)
# else:
# copyfile(Config.path['workdir']+'/'+execute_makefile, dstpath)
for macro, value in execute_attrs.iteritems():
if macro not in [ 'makefile', 'target', 'repeat', 'sleep']:
items = collect_items(value)
cline += ' %s="%s"'%(macro, ','.join([ SrcFile.applymap(item) for item in items ]))
fscr.write(cline+'\n')
if repeat>1:
fscr.write(" python -c 'import time; time.sleep(%s)'\n"%sleep)
fscr.write("done\n")
fscr.flush()
st = os.stat(script_file)
os.chmod(script_file, st.st_mode | stat.S_IEXEC)
return script_file
def flat_items(items):
if isinstance(items[0], str):
return [ SrcFile.applymap(t) for t in items ]
elif isinstance(items[0], tuple):
return [ SrcFile.applymap(t[0]) for t, _ in items]
else: raise ProgramException('Unknown type: %s'%items[0].__class__)
def prefunc_printelem(node, **kwargs):
if hasattr(node, 'default_items'):
print SrcFile.applymap(str(node.default_items)), SrcFile.applymap(str(node.default_attrs))
def execute(self):
if self.casenum==Case.NOT_INTIALIZED:
raise ProgramException('Case is not initialized')
output = None
if self.casenum==Case.REFCASE:
# execute refcase
stdout = []
cmd = State.direct['refcase'][0][0].cases[0][0][0].case[0][0][0]
refcmd = 'cd %s; '%Config.path['refdir'] + SrcFile.applymap(cmd)
for j in range(self.ref_outer_iter):
for i in range(self.ref_inner_iter):
stdout.append(exec_cmd(refcmd))
time.sleep(0.1)
time.sleep(0.3)
output = '\n'.join(stdout)
else:
print 'Executing case %d of %d'%(self.casenum, State.cases['size'])
# transform source
srcgen = [ value for key, value in self.directs.iteritems() if key.lower().startswith('srcgen') ]
srcfiles = generate_source(self.casenum, srcgen)
for fileid in srcfiles:
src = '%s/%s'%(Config.path['workdir'], State.inputfile[fileid].relpath)
dst = '%s/%s.%d'%(Config.path['outdir'], State.inputfile[fileid].relpath, self.casenum)
shutil.copyfile(src, dst)
# generate shell script
script = generate_script(self.casenum, self.directs, srcfiles)
src = '%s/case_%d.sh'%(Config.path['workdir'],self.casenum)
dst = '%s/case_%d.sh'%(Config.path['outdir'],self.casenum)
shutil.copyfile(src, dst)
# execute shell script
output = exec_cmd(script)
#print 'OUTPUT: ', output
if not output:
self.result = Case.EXECUTION_FAIL
return
# measure
self.measured = {}
for var, attrs in self.parent.measure.iteritems():
self.measured[var] = []
prefix = attrs['prefix']
len_prefix = len(prefix)
match_prefix = findall(prefix, output)
if match_prefix:
for start in match_prefix:
valuestr = output[start+len_prefix:].lstrip()
match_value = re.search(r'[\s\r\n\z]', valuestr)
if match_value:
self.measured[var].append(valuestr[:match_value.start()])
else:
self.result = Case.MEASURMENT_FAIL
return
else:
self.result = Case.MEASURMENT_FAIL
return
if any([ len(v)==0 for k,v in self.measured.iteritems()]):
self.result = Case.MEASURMENT_FAIL
return
# verify
for var, attrs in self.parent.verify.iteritems():
if not self.measured.has_key(var):
self.result = Case.NO_MEASUREMENT_FAIL
return
method = attrs['method']
if method=='match':
pattern = attrs['pattern']
if any( [ value!=pattern for value in self.measured[var] ] ):
self.result = Case.VERIFICATION_FAIL
return
elif method=='diff':
refval = float(attrs['refval'])
maxdiff = float(attrs['maxdiff'])
if any( [ abs(float(value)-refval)>maxdiff for value in self.measured[var] ] ):
self.result = Case.VERIFICATION_FAIL
return
else: raise ProgramException('Unsupported method: %s'%method)
self.result = Case.VERIFIED
return
def prefunc_printelem(node, **kwargs):
if hasattr(node, 'default_items'):
print SrcFile.applymap(str(node.default_items)), SrcFile.applymap(
str(node.default_attrs))
def execute(self):
if self.casenum == Case.NOT_INTIALIZED:
raise ProgramException('Case is not initialized')
output = None
if self.casenum == Case.REFCASE:
# execute refcase
stdout = []
cmd = State.direct['refcase'][0][0].cases[0][0][0].case[0][0][0]
refcmd = 'cd %s; ' % Config.path['refdir'] + SrcFile.applymap(cmd)
for j in range(self.ref_outer_iter):
for i in range(self.ref_inner_iter):
stdout.append(exec_cmd(refcmd))
time.sleep(0.1)
time.sleep(0.3)
output = '\n'.join(stdout)
else:
print 'Executing case %d of %d' % (self.casenum,
State.cases['size'])
# transform source
srcgen = [
value for key, value in self.directs.iteritems()
if key.lower().startswith('srcgen')
]
srcfiles = generate_source(self.casenum, srcgen)
for fileid in srcfiles:
src = '%s/%s' % (Config.path['workdir'],
State.inputfile[fileid].relpath)
dst = '%s/%s.%d' % (Config.path['outdir'],
State.inputfile[fileid].relpath,
self.casenum)
shutil.copyfile(src, dst)
# generate shell script
script = generate_script(self.casenum, self.directs, srcfiles)
src = '%s/case_%d.sh' % (Config.path['workdir'], self.casenum)
dst = '%s/case_%d.sh' % (Config.path['outdir'], self.casenum)
shutil.copyfile(src, dst)
# execute shell script
output = exec_cmd(script)
#print 'OUTPUT: ', output
if not output:
self.result = Case.EXECUTION_FAIL
return
# measure
self.measured = {}
for var, attrs in self.parent.measure.iteritems():
self.measured[var] = []
prefix = attrs['prefix']
len_prefix = len(prefix)
match_prefix = findall(prefix, output)
if match_prefix:
for start in match_prefix:
valuestr = output[start + len_prefix:].lstrip()
match_value = re.search(r'[\s\r\n\z]', valuestr)
if match_value:
self.measured[var].append(
valuestr[:match_value.start()])
else:
self.result = Case.MEASURMENT_FAIL
return
else:
self.result = Case.MEASURMENT_FAIL
return
if any([len(v) == 0 for k, v in self.measured.iteritems()]):
self.result = Case.MEASURMENT_FAIL
return
# verify
for var, attrs in self.parent.verify.iteritems():
if not self.measured.has_key(var):
self.result = Case.NO_MEASUREMENT_FAIL
return
method = attrs['method']
if method == 'match':
pattern = attrs['pattern']
if any([value != pattern for value in self.measured[var]]):
self.result = Case.VERIFICATION_FAIL
return
elif method == 'diff':
refval = float(attrs['refval'])
maxdiff = float(attrs['maxdiff'])
if any([
abs(float(value) - refval) > maxdiff
for value in self.measured[var]
]):
self.result = Case.VERIFICATION_FAIL
return
else:
raise ProgramException('Unsupported method: %s' % method)
self.result = Case.VERIFIED
return