def test_unique():
m = NameManager()
sn = m.seennames
check = [
m.uniquename('something0'),
m.uniquename('something', with_number=True),
m.uniquename('something', with_number=True),
m.uniquename('something2', with_number=True),
m.uniquename('something1'),
m.uniquename('something1_1'),
]
assert check == ['something0', 'something0_1', 'something1',
'something2_0', 'something1_1', 'something1_1_1']
class SourceGenerator:
one_source_file = True
def __init__(self, database):
self.database = database
self.extrafiles = []
self.headers_to_precompile = []
self.path = None
self.namespace = NameManager()
def set_strategy(self, path, split=True):
all_nodes = list(self.database.globalcontainers())
# split off non-function nodes. We don't try to optimize these, yet.
funcnodes = []
othernodes = []
for node in all_nodes:
if node.nodekind == 'func':
funcnodes.append(node)
else:
othernodes.append(node)
if split:
self.one_source_file = False
self.funcnodes = funcnodes
self.othernodes = othernodes
self.path = path
def uniquecname(self, name):
assert name.endswith('.c')
return self.namespace.uniquename(name[:-2]) + '.c'
def makefile(self, name):
log.writing(name)
filepath = self.path.join(name)
if name.endswith('.c'):
self.extrafiles.append(filepath)
if name.endswith('.h'):
self.headers_to_precompile.append(filepath)
return filepath.open('w')
def getextrafiles(self):
return self.extrafiles
def getothernodes(self):
return self.othernodes[:]
def getbasecfilefornode(self, node, basecname):
# For FuncNode instances, use the python source filename (relative to
# the top directory):
def invent_nice_name(g):
# Lookup the filename from the function.
# However, not all FunctionGraph objs actually have a "func":
if hasattr(g, 'func'):
if g.filename.endswith('.py'):
localpath = py.path.local(g.filename)
pypkgpath = localpath.pypkgpath()
if pypkgpath:
relpypath = localpath.relto(pypkgpath.dirname)
assert relpypath, ("%r should be relative to %r" %
(localpath, pypkgpath.dirname))
if len(relpypath.split(os.path.sep)) > 2:
# pypy detail to agregate the c files by directory,
# since the enormous number of files was causing
# memory issues linking on win32
return os.path.split(relpypath)[0] + '.c'
return relpypath.replace('.py', '.c')
return None
if hasattr(node.obj, 'graph'):
# Regular RPython functions
name = invent_nice_name(node.obj.graph)
if name is not None:
return name
elif node._funccodegen_owner is not None:
# Data nodes that belong to a known function
graph = getattr(node._funccodegen_owner, 'graph', None)
name = invent_nice_name(graph)
if name is not None:
return "data_" + name
return basecname
def splitnodesimpl(self, basecname, nodes, nextra, nbetween,
split_criteria=SPLIT_CRITERIA):
# Gather nodes by some criteria:
nodes_by_base_cfile = {}
for node in nodes:
c_filename = self.getbasecfilefornode(node, basecname)
if c_filename in nodes_by_base_cfile:
nodes_by_base_cfile[c_filename].append(node)
else:
nodes_by_base_cfile[c_filename] = [node]
# produce a sequence of nodes, grouped into files
# which have no more than SPLIT_CRITERIA lines
for basecname in sorted(nodes_by_base_cfile):
iternodes = iter(nodes_by_base_cfile[basecname])
done = [False]
def subiter():
used = nextra
for node in iternodes:
impl = '\n'.join(list(node.implementation())).split('\n')
if not impl:
continue
cost = len(impl) + nbetween
yield node, impl
del impl
if used + cost > split_criteria:
# split if criteria met, unless we would produce nothing.
raise StopIteration
used += cost
done[0] = True
while not done[0]:
yield self.uniquecname(basecname), subiter()
@contextlib.contextmanager
def write_on_included_file(self, f, name):
fi = self.makefile(name)
print >> f, '#include "%s"' % name
yield fi
fi.close()
@contextlib.contextmanager
def write_on_maybe_separate_source(self, f, name):
print >> f, '/* %s */' % name
if self.one_source_file:
yield f
else:
fi = self.makefile(name)
yield fi
fi.close()
def gen_readable_parts_of_source(self, f):
split_criteria_big = SPLIT_CRITERIA
if py.std.sys.platform != "win32":
if self.database.gcpolicy.need_no_typeptr():
pass # XXX gcc uses toooooons of memory???
else:
split_criteria_big = SPLIT_CRITERIA * 4
#
# All declarations
#
with self.write_on_included_file(f, 'structdef.h') as fi:
gen_structdef(fi, self.database)
with self.write_on_included_file(f, 'forwarddecl.h') as fi:
gen_forwarddecl(fi, self.database)
with self.write_on_included_file(f, 'preimpl.h') as fi:
gen_preimpl(fi, self.database)
#
# Implementation of functions and global structures and arrays
#
print >> f
print >> f, '/***********************************************************/'
print >> f, '/*** Implementations ***/'
print >> f
print >> f, '#define PYPY_FILE_NAME "%s"' % os.path.basename(f.name)
print >> f, '#include "src/g_include.h"'
if self.database.reverse_debugger:
print >> f, '#include "revdb_def.h"'
print >> f
nextralines = 11 + 1
for name, nodeiter in self.splitnodesimpl('nonfuncnodes.c',
self.othernodes,
nextralines, 1):
with self.write_on_maybe_separate_source(f, name) as fc:
if fc is not f:
print >> fc, '/***********************************************************/'
print >> fc, '/*** Non-function Implementations ***/'
print >> fc
print >> fc, '#include "common_header.h"'
print >> fc, '#include "structdef.h"'
print >> fc, '#include "forwarddecl.h"'
print >> fc, '#include "preimpl.h"'
print >> fc
print >> fc, '#include "src/g_include.h"'
print >> fc
print >> fc, MARKER
for node, impl in nodeiter:
print >> fc, '\n'.join(impl)
print >> fc, MARKER
print >> fc, '/***********************************************************/'
nextralines = 12
for name, nodeiter in self.splitnodesimpl('implement.c',
self.funcnodes,
nextralines, 1,
split_criteria_big):
with self.write_on_maybe_separate_source(f, name) as fc:
if fc is not f:
print >> fc, '/***********************************************************/'
print >> fc, '/*** Implementations ***/'
print >> fc
print >> fc, '#include "common_header.h"'
print >> fc, '#include "structdef.h"'
print >> fc, '#include "forwarddecl.h"'
print >> fc, '#include "preimpl.h"'
print >> fc, '#define PYPY_FILE_NAME "%s"' % name
print >> fc, '#include "src/g_include.h"'
if self.database.reverse_debugger:
print >> fc, '#include "revdb_def.h"'
print >> fc
print >> fc, MARKER
for node, impl in nodeiter:
print >> fc, '\n'.join(impl)
print >> fc, MARKER
print >> fc, '/***********************************************************/'
print >> f
class SourceGenerator:
one_source_file = True
def __init__(self, database):
self.database = database
self.extrafiles = []
self.headers_to_precompile = []
self.path = None
self.namespace = NameManager()
def set_strategy(self, path, split=True):
all_nodes = list(self.database.globalcontainers())
# split off non-function nodes. We don't try to optimize these, yet.
funcnodes = []
othernodes = []
for node in all_nodes:
if node.nodekind == 'func':
funcnodes.append(node)
else:
othernodes.append(node)
if split:
self.one_source_file = False
self.funcnodes = funcnodes
self.othernodes = othernodes
self.path = path
def uniquecname(self, name):
assert name.endswith('.c')
return self.namespace.uniquename(name[:-2]) + '.c'
def makefile(self, name):
log.writing(name)
filepath = self.path.join(name)
if name.endswith('.c'):
self.extrafiles.append(filepath)
if name.endswith('.h'):
self.headers_to_precompile.append(filepath)
return filepath.open('w')
def getextrafiles(self):
return self.extrafiles
def getothernodes(self):
return self.othernodes[:]
def getbasecfilefornode(self, node, basecname):
# For FuncNode instances, use the python source filename (relative to
# the top directory):
def invent_nice_name(g):
# Lookup the filename from the function.
# However, not all FunctionGraph objs actually have a "func":
if hasattr(g, 'func'):
if g.filename.endswith('.py'):
localpath = py.path.local(g.filename)
pypkgpath = localpath.pypkgpath()
if pypkgpath:
relpypath = localpath.relto(pypkgpath.dirname)
assert relpypath, ("%r should be relative to %r" %
(localpath, pypkgpath.dirname))
if len(relpypath.split(os.path.sep)) > 2:
# pypy detail to agregate the c files by directory,
# since the enormous number of files was causing
# memory issues linking on win32
return os.path.split(relpypath)[0] + '.c'
return relpypath.replace('.py', '.c')
return None
if hasattr(node.obj, 'graph'):
# Regular RPython functions
name = invent_nice_name(node.obj.graph)
if name is not None:
return name
elif node._funccodegen_owner is not None:
# Data nodes that belong to a known function
graph = getattr(node._funccodegen_owner, 'graph', None)
name = invent_nice_name(graph)
if name is not None:
return "data_" + name
return basecname
def splitnodesimpl(self, basecname, nodes, nextra, nbetween,
split_criteria=SPLIT_CRITERIA):
# Gather nodes by some criteria:
nodes_by_base_cfile = {}
for node in nodes:
c_filename = self.getbasecfilefornode(node, basecname)
if c_filename in nodes_by_base_cfile:
nodes_by_base_cfile[c_filename].append(node)
else:
nodes_by_base_cfile[c_filename] = [node]
# produce a sequence of nodes, grouped into files
# which have no more than SPLIT_CRITERIA lines
for basecname in sorted(nodes_by_base_cfile):
iternodes = iter(nodes_by_base_cfile[basecname])
done = [False]
def subiter():
used = nextra
for node in iternodes:
impl = '\n'.join(list(node.implementation())).split('\n')
if not impl:
continue
cost = len(impl) + nbetween
yield node, impl
del impl
if used + cost > split_criteria:
# split if criteria met, unless we would produce nothing.
raise StopIteration
used += cost
done[0] = True
while not done[0]:
yield self.uniquecname(basecname), subiter()
@contextlib.contextmanager
def write_on_included_file(self, f, name):
fi = self.makefile(name)
print >> f, '#include "%s"' % name
yield fi
fi.close()
@contextlib.contextmanager
def write_on_maybe_separate_source(self, f, name):
print >> f, '/* %s */' % name
if self.one_source_file:
yield f
else:
fi = self.makefile(name)
yield fi
fi.close()
def gen_readable_parts_of_source(self, f):
split_criteria_big = SPLIT_CRITERIA
if py.std.sys.platform != "win32":
if self.database.gcpolicy.need_no_typeptr():
pass # XXX gcc uses toooooons of memory???
else:
split_criteria_big = SPLIT_CRITERIA * 4
#
# All declarations
#
with self.write_on_included_file(f, 'structdef.h') as fi:
gen_structdef(fi, self.database)
with self.write_on_included_file(f, 'forwarddecl.h') as fi:
gen_forwarddecl(fi, self.database)
with self.write_on_included_file(f, 'preimpl.h') as fi:
gen_preimpl(fi, self.database)
#
# Implementation of functions and global structures and arrays
#
print >> f
print >> f, '/***********************************************************/'
print >> f, '/*** Implementations ***/'
print >> f
print >> f, '#define PYPY_FILE_NAME "%s"' % os.path.basename(f.name)
print >> f, '#include "src/g_include.h"'
print >> f
nextralines = 11 + 1
for name, nodeiter in self.splitnodesimpl('nonfuncnodes.c',
self.othernodes,
nextralines, 1):
with self.write_on_maybe_separate_source(f, name) as fc:
if fc is not f:
print >> fc, '/***********************************************************/'
print >> fc, '/*** Non-function Implementations ***/'
print >> fc
print >> fc, '#include "common_header.h"'
print >> fc, '#include "structdef.h"'
print >> fc, '#include "forwarddecl.h"'
print >> fc, '#include "preimpl.h"'
print >> fc
print >> fc, '#include "src/g_include.h"'
print >> fc
print >> fc, MARKER
for node, impl in nodeiter:
print >> fc, '\n'.join(impl)
print >> fc, MARKER
print >> fc, '/***********************************************************/'
nextralines = 12
for name, nodeiter in self.splitnodesimpl('implement.c',
self.funcnodes,
nextralines, 1,
split_criteria_big):
with self.write_on_maybe_separate_source(f, name) as fc:
if fc is not f:
print >> fc, '/***********************************************************/'
print >> fc, '/*** Implementations ***/'
print >> fc
print >> fc, '#include "common_header.h"'
print >> fc, '#include "structdef.h"'
print >> fc, '#include "forwarddecl.h"'
print >> fc, '#include "preimpl.h"'
print >> fc, '#define PYPY_FILE_NAME "%s"' % name
print >> fc, '#include "src/g_include.h"'
print >> fc
print >> fc, MARKER
for node, impl in nodeiter:
print >> fc, '\n'.join(impl)
print >> fc, MARKER
print >> fc, '/***********************************************************/'
print >> f
