def GenerateRange(self, ast, releases, options):
# Get list of out files
outlist = GetOption('out')
if outlist: outlist = outlist.split(',')
skipList = []
cnt = 0
for filenode in ast.GetListOf('File'):
# Ignore files with errors
if filenode in self.skip_list:
continue
# Skip this file if not required
if outlist and filenode.GetName() not in outlist:
continue
# Create the output file and increment out count if there was a delta
if self.GenerateFile(filenode, releases, options):
cnt = cnt + 1
for filenode in skipList:
errcnt = filenode.GetProperty('ERRORS')
ErrOut.Log('%s : Skipped because of %d errors.' %
(filenode.GetName(), errcnt))
if skipList:
return -len(skipList)
if GetOption('diff'):
return -cnt
return cnt
def DetermineInterfaces(self, ast, releases):
"""Get a list of interfaces along with whatever metadata we need.
"""
iface_releases = []
for filenode in ast.GetListOf('File'):
# If this file has errors, skip it
if filenode in self.skip_list:
if GetOption('verbose'):
InfoOut.Log('WrapperGen: Skipping %s due to errors\n' %
filenode.GetName())
continue
file_name = self.GetHeaderName(filenode.GetName())
ifaces = filenode.GetListOf('Interface')
for iface in ifaces:
releases_for_iface = iface.GetUniqueReleases(releases)
for release in releases_for_iface:
version = iface.GetVersion(release)
struct_name = self.cgen.GetStructName(iface,
release,
include_version=True)
needs_wrap = self.InterfaceVersionNeedsWrapping(
iface, version)
if not needs_wrap:
if GetOption('verbose'):
InfoOut.Log(
'Interface %s ver %s does not need wrapping' %
(struct_name, version))
iface_releases.append(
Interface(iface, release, version, struct_name,
needs_wrap, file_name))
return iface_releases
def GenerateRange(self, ast, releases, options):
"""Generate entry point code for a range of releases.
"""
self._skip_opt = GetOption('disable_pepper_opt')
self._skip_region_gen = GetOption('disable_region_gen')
self.SetOutputFile(GetOption('pepperfile'))
return WrapperGen.GenerateRange(self, ast, releases, options)
def ParseFiles(filenames):
parser = IDLParser()
filenodes = []
if not filenames:
filenames = []
srcroot = GetOption('srcroot')
dirs = default_dirs
if GetOption('include_private'):
dirs += ['private']
for dirname in dirs:
srcdir = os.path.join(srcroot, dirname, '*.idl')
srcdir = os.path.normpath(srcdir)
filenames += sorted(glob.glob(srcdir))
if not filenames:
ErrOut.Log('No sources provided.')
for filename in filenames:
filenode = parser.ParseFile(filename)
filenodes.append(filenode)
ast = IDLAst(filenodes)
if GetOption('dump_tree'): ast.Dump(0)
Lint(ast)
return ast
def Generate(self, ast, options):
self.errors = 0
rangestr = GetOption('range')
releasestr = GetOption('release')
print "Found releases: %s" % ast.releases
# Generate list of files to ignore due to errors
for filenode in ast.GetListOf('File'):
# If this file has errors, skip it
if filenode.GetProperty('ERRORS') > 0:
self.skip_list.append(filenode)
continue
# Check for a range option which over-rides a release option
if not releasestr and rangestr:
range_list = rangestr.split(',')
if len(range_list) != 2:
self.Error('Failed to generate for %s, incorrect range: "%s"' %
(self.name, rangestr))
else:
vmin = range_list[0]
vmax = range_list[1]
# Generate 'start' and 'end' represent first and last found.
if vmin == 'start':
vmin = ast.releases[0]
if vmax == 'end':
vmax = ast.releases[-1]
vmin = ast.releases.index(vmin)
vmax = ast.releases.index(vmax) + 1
releases = ast.releases[vmin:vmax]
InfoOut.Log('Generate range %s of %s.' % (rangestr, self.name))
ret = self.GenerateRange(ast, releases, options)
if ret < 0:
self.Error('Failed to generate range %s : %s.' %(vmin, vmax))
else:
InfoOut.Log('%s wrote %d files.' % (self.name, ret))
# Otherwise this should be a single release generation
else:
if releasestr == 'start':
releasestr = ast.releases[0]
if releasestr == 'end':
releasestr = ast.releases[-1]
if releasestr:
InfoOut.Log('Generate release %s of %s.' % (releasestr, self.name))
ret = self.GenerateRelease(ast, releasestr, options)
if ret < 0:
self.Error('Failed to generate release %s.' % releasestr)
else:
InfoOut.Log('%s wrote %d files.' % (self.name, ret))
else:
self.Error('No range or release specified for %s.' % releasestr)
return self.errors
def __init__(self):
IDLLexer.__init__(self)
self.yaccobj = yacc.yacc(module=self, tabmodule=None, debug=False,
optimize=0, write_tables=0)
self.build_debug = GetOption('build_debug')
self.parse_debug = GetOption('parse_debug')
self.token_debug = GetOption('token_debug')
self.verbose = GetOption('verbose')
self.parse_errors = 0
def GenerateTail(self, out, filenode, releases, options):
__pychecker__ = 'unusednames=options,releases'
gpath = GetOption('guard')
def_guard = GetPathFromNode(filenode, relpath=gpath, ext='.h')
def_guard = def_guard.replace(os.sep, '_').replace('.',
'_').upper() + '_'
out.Write('#endif /* %s */\n\n' % def_guard)
def TestSame(values1):
# Recreate the source from the tokens. We use newline instead of whitespace
# since the '//' and #inline regex are line sensitive.
text = '\n'.join(values1)
values2 = TextToTokens(text)
count1 = len(values1)
count2 = len(values2)
if count1 != count2:
print "Size mismatch original %d vs %d\n" % (count1, count2)
if count1 > count2: count1 = count2
for i in range(count1):
if values1[i] != values2[i]:
print "%d >>%s<< >>%s<<" % (i, values1[i], values2[i])
if GetOption('output'):
sys.stdout.write('Generating original.txt and tokenized.txt\n')
open('original.txt', 'w').write(src1)
open('tokenized.txt', 'w').write(src2)
if values1 == values2:
sys.stdout.write('Same: Pass\n')
return 0
print "****************\n%s\n%s***************\n" % (src1, src2)
sys.stdout.write('Same: Failed\n')
return -1
def GetItemName(self, enum, item):
suppress_prefix = GetOption('cs-enum_prefix')
if suppress_prefix:
enumeration = item
if item.upper().startswith(enum.upper() + "_"):
enumeration = item[len(enum + "_"):]
elif item.upper().startswith(enum.upper()):
enumeration = item[len(enum):]
if self.is_number(enumeration):
enumeration = "_" + enumeration
if enumeration.startswith('PP_'):
enumeration = enumeration[3:]
if enumeration.startswith('PPB_'):
enumeration = enumeration[4:]
enumeration = enumeration.lower()
enumeration = enumeration.replace('_', ' ')
enumeration = string.capwords(enumeration)
enumeration = "".join(enumeration.split(' '))
if self.is_number(enumeration):
enumeration = "_" + enumeration
if enumeration in self.RemapEnum:
enumeration = self.RemapEnum[enumeration]
item = enumeration
return item
def FilterLinesIn(output):
was = []
now = []
filter = []
for index in range(len(output)):
filter.append(False)
line = output[index]
if len(line) < 2: continue
if line[0] == '<':
if line[2:].strip() == '': continue
was.append((index, line[2:]))
elif line[0] == '>':
if line[2:].strip() == '': continue
now.append((index, line[2:]))
for windex, wline in was:
for nindex, nline in now:
if filter[nindex]: continue
if filter[windex]: continue
if wline == nline:
filter[nindex] = True
filter[windex] = True
if GetOption('verbose'):
print "Found %d, %d >>%s<<" % (windex + 1, nindex + 1, wline)
out = []
for index in range(len(output)):
if not filter[index]:
out.append(output[index])
return out
def IsRelease(self, release):
if self.rmax and self.rmax <= release:
return False
if self.rmin and self.rmin > release:
return False
if GetOption('release_debug'):
InfoOut.Log('%f is in %s' % (release, self))
return True
def Lint(ast):
options = ['wcomment', 'wenum', 'winline', 'wparam', 'wpass', 'wname']
wnone = GetOption('wnone')
for opt in options:
if wnone or GetOption(opt): ast.SetProperty(opt, True)
skipList = []
for filenode in ast.GetListOf('File'):
name = filenode.GetProperty('NAME')
if filenode.GetProperty('ERRORS') > 0:
ErrOut.Log('%s : Skipped due to errors.' % name)
skipList.append(filenode)
continue
warnings = IDLLinter().Visit(filenode, 0)
if warnings:
WarnOut.Log('%s warning(s) for %s\n' % (warnings, name))
return skipList
def Main(args):
filenames = ParseOptions(args)
try:
tokens = FilesToTokens(filenames, GetOption('verbose'))
values = [tok.value for tok in tokens]
if GetOption('output'): sys.stdout.write(' <> '.join(values) + '\n')
if GetOption('test'):
if TestSame(values):
return -1
if TestExpect(tokens):
return -1
return 0
except lex.LexError as le:
sys.stderr.write('%s\n' % str(le))
return -1
def Close(self):
filename = os.path.realpath(self.filename)
self.open = False
outtext = ''.join(self.outlist)
oldtext = ''
if not self.always_write:
if os.path.isfile(filename):
oldtext = open(filename, 'rb').read()
if self.IsEquivalent_(oldtext):
if GetOption('verbose'):
InfoOut.Log('Output %s unchanged.' % self.filename)
return False
if GetOption('diff'):
for line in difflib.unified_diff(oldtext.split('\n'),
outtext.split('\n'),
'OLD ' + self.filename,
'NEW ' + self.filename,
n=1,
lineterm=''):
ErrOut.Log(line)
try:
# If the directory does not exit, try to create it, if we fail, we
# still get the exception when the file is openned.
basepath, leafname = os.path.split(filename)
if basepath and not os.path.isdir(basepath) and self.create_dir:
InfoOut.Log('Creating directory: %s\n' % basepath)
os.makedirs(basepath)
if not GetOption('test'):
outfile = open(filename, 'wb')
outfile.write(outtext)
outfile.close()
InfoOut.Log('Output %s written.' % self.filename)
return True
except IOError as e:
ErrOut.Log("I/O error(%d): %s" % (e.errno, e.strerror))
except:
ErrOut.Log("Unexpected error: %s" % sys.exc_info()[0])
raise
return False
def AddNode(self, node):
if GetOption('release_debug'):
InfoOut.Log('\nAdding %s %s' % (node.Location(), node))
last = None
# Check current releases in that namespace
for cver in self.nodes:
if GetOption('release_debug'): InfoOut.Log(' Checking %s' % cver)
# We should only be missing a 'release' tag for the first item.
if not node.rmin:
node.Error('Missing release on overload of previous %s.' %
cver.Location())
return False
# If the node has no max, then set it to this one
if not cver.rmax:
cver.rmax = node.rmin
if GetOption('release_debug'):
InfoOut.Log(' Update %s' % cver)
# if the max and min overlap, than's an error
if cver.rmax > node.rmin:
if node.rmax and cver.rmin >= node.rmax:
node.Error('Declarations out of order.')
else:
node.Error('Overlap in releases: %s vs %s when adding %s' %
(cver.rmax, node.rmin, node))
return False
last = cver
# Otherwise, the previous max and current min should match
# unless this is the unlikely case of something being only
# temporarily deprecated.
if last and last.rmax != node.rmin:
node.Warn('Gap in release numbers.')
# If we made it here, this new node must be the 'newest'
# and does not overlap with anything previously added, so
# we can add it to the end of the list.
if GetOption('release_debug'): InfoOut.Log('Done %s' % node)
self.nodes.append(node)
return True
def Depart(self, node, data, childdata):
if node.IsA('LabelItem'):
return (node.GetName(), node.GetProperty('VALUE'))
if node.IsA('Label') and node.GetName() == GetOption('label'):
try:
self.release_map = IDLReleaseMap(childdata)
node.parent.release_map = self.release_map
except Exception as err:
node.Error('Unable to build release map: %s' % str(err))
return None
def main(args):
filenames = ParseOptions(args)
if GetOption('test'):
return TestFiles(filenames)
ast = ParseFiles(filenames)
cgen = CSGen()
for f in ast.GetListOf('File'):
if f.GetProperty('ERRORS') > 0:
print 'Skipping %s' % f.GetName()
continue
for node in f.GetChildren()[2:]:
print cgen.Define(node, ast.releases, comment=True, prefix='tst_')
def Main(args):
filenames = ParseOptions(args)
if GetOption('test'):
return TestFiles(filenames)
ast = ParseFiles(filenames)
for f in ast.GetListOf('File'):
if f.GetProperty('ERRORS') > 0:
print 'Skipping %s' % f.GetName()
continue
print DefineDepends(node)
for node in f.GetChildren()[2:]:
print Define(node, comment=True, prefix='tst_')
def Generate(self, ast, options):
self.errors = 0
rangestr = GetOption('range')
releasestr = GetOption('release')
# Check for a range option which over-rides a release option
if not releasestr and rangestr:
range_list = rangestr.split(',')
if len(range_list) != 2:
self.Error('Failed to generate for %s, incorrect range: "%s"' %
(self.name, rangestr))
else:
vmin = range_list[0]
vmax = range_list[1]
vmin = ast.releases.index(vmin)
vmax = ast.releases.index(vmax) + 1
range = ast.releases[vmin:vmax]
InfoOut.Log('Generate range %s of %s.' % (range, self.name))
ret = self.GenerateRange(ast, range, options)
if ret < 0:
self.Error('Failed to generate range %s : %s.' %
(vmin, vmax))
else:
InfoOut.Log('%s wrote %d files.' % (self.name, ret))
# Otherwise this should be a single release generation
else:
if releasestr:
InfoOut.Log('Generate release %s of %s.' %
(releasestr, self.name))
ret = self.GenerateRelease(ast, releasestr, options)
if ret < 0:
self.Error('Failed to generate release %s.' % releasestr)
else:
InfoOut.Log('%s wrote %d files.' % (self.name, ret))
else:
self.Error('No range or release specified for %s.' %
releasestr)
return self.errors
def ParseFiles(filenames):
parser = IDLParser()
filenodes = []
errors = 0
if not filenames:
filenames = []
srcroot = GetOption('srcroot')
for dir in default_dirs:
srcdir = os.path.join(srcroot, dir, '*.idl')
srcdir = os.path.normpath(srcdir)
filenames += sorted(glob.glob(srcdir))
for filename in filenames:
filenode = parser.ParseFile(filename)
filenodes.append(filenode)
ast = IDLAst(filenodes)
if GetOption('dump_tree'): ast.Dump(0)
Lint(ast)
return ast
def GenerateFile(self, filenode, releases, options):
savename = GetOutFileName(filenode, GetOption('dstroot'))
unique_releases = filenode.GetUniqueReleases(releases)
if not unique_releases:
if os.path.isfile(savename):
print "Removing stale %s for this range." % filenode.GetName()
os.remove(os.path.realpath(savename))
return False
out = IDLOutFile(savename)
self.GenerateHead(out, filenode, releases, options)
self.GenerateBody(out, filenode, releases, options)
self.GenerateTail(out, filenode, releases, options)
return out.Close()
def GenerateFile(self, filenode, releases, options):
savename = GetOutFileName(filenode, GetOption('dstroot'))
my_min, my_max = filenode.GetMinMax(releases)
if my_min > releases[-1] or my_max < releases[0]:
if os.path.isfile(savename):
print "Removing stale %s for this range." % filenode.GetName()
os.remove(os.path.realpath(savename))
return False
out = IDLOutFile(savename)
self.GenerateHead(out, filenode, releases, options)
self.GenerateBody(out, filenode, releases, options)
self.GenerateTail(out, filenode, releases, options)
return out.Close()
def InRange(self, rmin, rmax):
assert (rmin == None) or rmin < rmax
# An min of None always passes a min bound test
# An max of None always passes a max bound test
if rmin is not None and self.rmax is not None:
if self.rmax <= rmin:
return False
if rmax is not None and self.rmin is not None:
if self.rmin >= rmax:
return False
if GetOption('release_debug'):
InfoOut.Log('%f to %f is in %s' % (rmin, rmax, self))
return True
def GenerateRange(self, ast, releases, options):
# Get list of out files
outlist = GetOption('out')
if outlist: outlist = outlist.split(',')
skipList = []
cnt = 0
for filenode in ast.GetListOf('File'):
# Skip this file if not required
if outlist and filenode.GetName() not in outlist:
continue
# If this file has errors, skip it
if filenode.GetProperty('ERRORS') > 0:
skipList.append(filenode)
continue
# Create output file
out = self.GetOutFile(filenode, options)
self.GenerateHead(out, filenode, releases, options)
self.GenerateBody(out, filenode, releases, options)
self.GenerateTail(out, filenode, releases, options)
if out.Close(): cnt = cnt + 1
for filenode in skipList:
errcnt = filenode.GetProperty('ERRORS')
ErrOut.Log('%s : Skipped because of %d errors.' %
(filenode.GetName(), errcnt))
if skipList:
return -len(skipList)
if GetOption('diff'):
return -cnt
return cnt
def Main(args):
filenames = ParseOptions(args)
test_releases = ['M13', 'M14', 'M15']
if not filenames:
idldir = os.path.split(sys.argv[0])[0]
idldir = os.path.join(idldir, 'test_gen_pnacl', '*.idl')
filenames = glob.glob(idldir)
filenames = sorted(filenames)
if GetOption('test'):
# Run the tests.
return TestFiles(filenames, test_releases)
# Otherwise, generate the output file (for potential use as golden file).
ast = ParseFiles(filenames)
return pnaclgen.GenerateRange(ast, test_releases, filenames)
def GenerateHeader(out, filenode, releases):
gpath = GetOption('guard')
cgen = CGen()
pref = ''
do_comments = True
# Generate definitions.
last_group = None
top_types = ['Typedef', 'Interface', 'Struct', 'Enum', 'Inline']
for node in filenode.GetListOf(*top_types):
# Skip if this node is not in this release
if not node.InReleases(releases):
print "Skiping %s" % node
continue
# End/Start group marker
if do_comments:
last_group = WriteGroupMarker(out, node, last_group)
if node.IsA('Inline'):
item = node.GetProperty('VALUE')
# If 'C++' use __cplusplus wrapper
if node.GetName() == 'cc':
item = '#ifdef __cplusplus\n%s\n#endif // __cplusplus\n\n' % item
# If not C++ or C, then skip it
elif not node.GetName() == 'c':
continue
if item: out.Write(item)
continue
#
# Otherwise we are defining a file level object, so generate the
# correct document notation.
#
item = cgen.Define(node, releases, prefix=pref, comment=True)
if not item: continue
asize = node.GetProperty('assert_size()')
if asize:
name = '%s%s' % (pref, node.GetName())
if node.IsA('Struct'):
form = 'PP_COMPILE_ASSERT_STRUCT_SIZE_IN_BYTES(%s, %s);\n'
else:
form = 'PP_COMPILE_ASSERT_SIZE_IN_BYTES(%s, %s);\n'
item += form % (name, asize[0])
if item: out.Write(item)
if last_group:
out.Write(CommentLines(['*', ' @}', '']) + '\n')
def GetEnumValue(self, enum, value):
suppress_prefix = GetOption('cs-enum_prefix')
if suppress_prefix:
enumValue = value
if value.upper().startswith(enum.upper() + "_"):
enumValue = value[len(enum + "_"):]
elif value.upper().startswith(enum.upper()):
enumValue = value[len(enum):]
enumValue = enumValue.lower()
enumValue = enumValue.replace('_', ' ')
enumValue = string.capwords(enumValue)
enumValue = "".join(enumValue.split(' '))
value = enumValue
return value
def main(args):
filenames = ParseOptions(args)
ast = ParseFiles(filenames)
rpcgen = RPCGen()
files = ast.GetListOf('File')
outname = GetOption('out')
if outname == '':
outname = 'out.cc'
outfile = IDLOutFile(outname)
emitted = set()
for f in files:
if f.GetName() == 'pp_macros.idl':
GenerateDep(outfile, emitted, f, ast.releases)
for f in files:
GenerateDep(outfile, emitted, f, ast.releases)
out = ''
out += '#include "host/rpc.h"\n'
out += '#include "json/json.cpp"\n'
out += 'using namespace std;\n'
out += 'using namespace JSON;\n'
for declareOnly in [True, False]:
for f in files:
if f.GetProperty('ERRORS') > 0:
print 'Skipping %s' % f.GetName()
continue
for node in f.GetChildren()[2:]:
out += rpcgen.Define(node, ast.releases, declareOnly)
out += 'static map<string, void*> gInterfaces;\n'
out += '\n'
out += 'typedef char* (*InterfaceMemberCallFunc)(const void*, JSONIterator&);\n'
out += 'static map<string, InterfaceMemberCallFunc> gInterfaceMemberCallers;\n'
out += '\n'
out += 'void InitializeInterfaceList() {\n'
for (interfaceString, interfaceStruct) in rpcgen.interfaceStructs:
if interfaceString == "PPB_Flash_File_FileRef;2.0":
interfaceString = "PPB_Flash_File_FileRef;2"
elif interfaceString == "PPB_Flash_File_ModuleLocal;3.0":
interfaceString = "PPB_Flash_File_ModuleLocal;3"
elif interfaceString == "PPB_PDF;0.1":
interfaceString = "PPB_PDF;1"
out += ' gInterfaces.insert(pair<string, void*>("' + interfaceString + '", &' + interfaceStruct + '));\n'
for (interfaceString, caller) in rpcgen.pppInterfaceGetters:
out += ' gInterfaceMemberCallers.insert(pair<string, InterfaceMemberCallFunc>("' + interfaceString + '", ' + caller + '));\n'
out += '};\n'
out += '\n'
outfile.Write(out)
outfile.Close()
def Main(args):
filenames = ParseOptions(args)
if not filenames:
gendir = os.path.join(GetOption('gen'), '*.h')
filenames = sorted(glob.glob(gendir))
srcdir = os.path.join(GetOption('src'), '*.h')
srcs = sorted(glob.glob(srcdir))
for name in srcs:
name = os.path.split(name)[1]
name = os.path.join(GetOption('gen'), name)
if name not in filenames:
print 'Missing: %s' % name
for filename in filenames:
gen = filename
filename = filename[len(GetOption('gen')) + 1:]
src = os.path.join(GetOption('src'), filename)
diff = os.path.join(GetOption('diff'), filename)
p = subprocess.Popen(['diff', src, gen], stdout=subprocess.PIPE)
output, errors = p.communicate()
try:
input = open(diff, 'rt').read()
except:
input = ''
if input != output:
changes = GetChanges(output)
else:
changes = []
if changes:
print "\n\nDelta between:\n src=%s\n gen=%s\n" % (src, gen)
for change in changes:
change.Dump()
print 'Done with %s\n\n' % src
if GetOption('ok'):
open(diff, 'wt').write(output)
if GetOption('halt'):
return 1
else:
print "\nSAME:\n src=%s\n gen=%s" % (src, gen)
if input: print ' ** Matched expected diff. **'
print '\n'
def GenerateFile(self, filenode, releases, options):
savename = _GetThunkFileName(filenode, GetOption('thunkroot'))
my_min, my_max = filenode.GetMinMax(releases)
if my_min > releases[-1] or my_max < releases[0]:
if os.path.isfile(savename):
print "Removing stale %s for this range." % filenode.GetName()
os.remove(os.path.realpath(savename))
return False
do_generate = filenode.GetProperty('generate_thunk')
if not do_generate:
return False
thunk_out = IDLOutFile(savename)
body, meta = self.GenerateBody(thunk_out, filenode, releases, options)
self.WriteHead(thunk_out, filenode, releases, options, meta)
thunk_out.Write('\n\n'.join(body))
self.WriteTail(thunk_out, filenode, releases, options)
return thunk_out.Close()
