def task_database_c(self):
translator = self.translator
if translator.annotator is not None:
translator.frozen = True
if self.libdef is not None:
cbuilder = self.libdef.getcbuilder(self.translator, self.config)
self.standalone = False
standalone = False
else:
standalone = self.standalone
if standalone:
from pypy.translator.c.genc import CStandaloneBuilder as CBuilder
else:
from pypy.translator.c.genc import CExtModuleBuilder as CBuilder
cbuilder = CBuilder(self.translator, self.entry_point,
config=self.config)
cbuilder.stackless = self.config.translation.stackless
if not standalone: # xxx more messy
cbuilder.modulename = self.extmod_name
database = cbuilder.build_database()
self.log.info("database for generating C source was created")
self.cbuilder = cbuilder
self.database = database
def test_stackless_simple():
def g(x):
return x + 1
class A(object):
pass
def entrypoint(argv):
a = A()
a.b = g(1)
return a.b
from pypy.rpython.llinterp import LLInterpreter
from pypy.translator.c.genc import CStandaloneBuilder
from pypy.translator.c import gc
from pypy.annotation.listdef import s_list_of_strings
t = rtype(entrypoint, [s_list_of_strings])
cbuild = CStandaloneBuilder(t, entrypoint, t.config,
gcpolicy=StacklessFrameworkGcPolicy2)
db = cbuild.generate_graphs_for_llinterp()
entrypointptr = cbuild.getentrypointptr()
entrygraph = entrypointptr._obj.graph
r_list_of_strings = t.rtyper.getrepr(s_list_of_strings)
ll_argv = r_list_of_strings.convert_const([])
llinterp = LLInterpreter(t.rtyper)
# FIIIIISH
setupgraph = db.gctransformer.frameworkgc_setup_ptr.value._obj.graph
llinterp.eval_graph(setupgraph, [])
res = llinterp.eval_graph(entrygraph, [ll_argv])
assert res == 2
def test_framework_simple():
def g(x):
return x + 1
class A(object):
pass
def entrypoint(argv):
a = A()
a.b = g(1)
return str(a.b)
from pypy.rpython.llinterp import LLInterpreter
from pypy.translator.c.genc import CStandaloneBuilder
from pypy.translator.c import gc
from pypy.annotation.listdef import s_list_of_strings
t = rtype(entrypoint, [s_list_of_strings])
cbuild = CStandaloneBuilder(t, entrypoint, t.config,
gcpolicy=FrameworkGcPolicy2)
db = cbuild.generate_graphs_for_llinterp()
entrypointptr = cbuild.getentrypointptr()
entrygraph = entrypointptr._obj.graph
r_list_of_strings = t.rtyper.getrepr(s_list_of_strings)
ll_argv = r_list_of_strings.convert_const([])
llinterp = LLInterpreter(t.rtyper)
# FIIIIISH
setupgraph = db.gctransformer.frameworkgc_setup_ptr.value._obj.graph
llinterp.eval_graph(setupgraph, [])
res = llinterp.eval_graph(entrygraph, [ll_argv])
assert ''.join(res.chars) == "2"
def task_database_c(self):
translator = self.translator
if translator.annotator is not None:
translator.frozen = True
if self.libdef is not None:
cbuilder = self.libdef.getcbuilder(self.translator, self.config)
self.standalone = False
standalone = False
else:
standalone = self.standalone
if standalone:
from pypy.translator.c.genc import CStandaloneBuilder as CBuilder
else:
from pypy.translator.c.genc import CExtModuleBuilder as CBuilder
cbuilder = CBuilder(self.translator, self.entry_point,
config=self.config)
cbuilder.stackless = self.config.translation.stackless
if not standalone: # xxx more messy
cbuilder.modulename = self.extmod_name
database = cbuilder.build_database()
self.log.info("database for generating C source was created")
self.cbuilder = cbuilder
self.database = database
def _makefunc_str_int(cls, func):
def main(argv):
arg0 = argv[1]
arg1 = int(argv[2])
try:
res = func(arg0, arg1)
except MemoryError:
print 'Result: MemoryError'
else:
print 'Result: "%s"' % (res,)
return 0
from pypy.config.pypyoption import get_pypy_config
config = get_pypy_config(translating=True)
config.translation.gc = cls.gcpolicy
config.translation.gcrootfinder = "asmgcc"
if sys.platform == 'win32':
config.translation.cc = 'mingw32'
t = TranslationContext(config=config)
a = t.buildannotator()
a.build_types(main, [s_list_of_strings])
t.buildrtyper().specialize()
t.checkgraphs()
cbuilder = CStandaloneBuilder(t, main, config=config)
c_source_filename = cbuilder.generate_source(
defines = cbuilder.DEBUG_DEFINES)
cls._patch_makefile(cbuilder.targetdir)
if conftest.option.view:
t.view()
exe_name = cbuilder.compile()
def run(arg0, arg1):
lines = []
print >> sys.stderr, 'RUN: starting', exe_name
if sys.platform == 'win32':
redirect = ' 2> NUL'
else:
redirect = ''
g = os.popen('"%s" %s %d%s' % (exe_name, arg0, arg1, redirect), 'r')
for line in g:
print >> sys.stderr, 'RUN:', line.rstrip()
lines.append(line)
g.close()
if not lines:
py.test.fail("no output from subprocess")
if not lines[-1].startswith('Result:'):
py.test.fail("unexpected output from subprocess")
result = lines[-1][len('Result:'):].strip()
if result == 'MemoryError':
raise MemoryError("subprocess got an RPython MemoryError")
if result.startswith('"') and result.endswith('"'):
return result[1:-1]
else:
return int(result)
return run
def _compile_and_run(self, t, entry_point, entry_point_graph, args):
from pypy.translator.c.genc import CStandaloneBuilder as CBuilder
# XXX patch exceptions
cbuilder = CBuilder(t, entry_point, config=t.config)
cbuilder.generate_source()
exe_name = cbuilder.compile()
log('---------- Test starting ----------')
stdout = cbuilder.cmdexec(" ".join([str(arg) for arg in args]))
res = int(stdout)
log('---------- Test done (%d) ----------' % (res, ))
return res
def _compile_and_run(self, t, entry_point, entry_point_graph, args):
from pypy.translator.c.genc import CStandaloneBuilder as CBuilder
# XXX patch exceptions
cbuilder = CBuilder(t, entry_point, config=t.config)
cbuilder.generate_source()
exe_name = cbuilder.compile()
log("---------- Test starting ----------")
stdout = cbuilder.cmdexec(" ".join([str(arg) for arg in args]))
res = int(stdout)
log("---------- Test done (%d) ----------" % (res,))
return res
def test_os_setpgrp(self):
def entry_point(argv):
os.setpgrp()
return 0
t = TranslationContext(self.config)
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
cbuilder.generate_source()
cbuilder.compile()
def runner(self, f, nbargs=0, statistics=False, transformer=False,
**extraconfigopts):
if nbargs == 2:
def entrypoint(args):
x = args[0]
y = args[1]
r = f(x, y)
return r
elif nbargs == 0:
def entrypoint(args):
return f()
else:
raise NotImplementedError("pure laziness")
from pypy.rpython.llinterp import LLInterpreter
from pypy.translator.c.genc import CStandaloneBuilder
ARGS = lltype.FixedSizeArray(lltype.Signed, nbargs)
s_args = annmodel.SomePtr(lltype.Ptr(ARGS))
t = rtype(entrypoint, [s_args], gcname=self.gcname,
stacklessgc=self.stacklessgc,
**extraconfigopts)
cbuild = CStandaloneBuilder(t, entrypoint, config=t.config,
gcpolicy=self.gcpolicy)
db = cbuild.generate_graphs_for_llinterp()
entrypointptr = cbuild.getentrypointptr()
entrygraph = entrypointptr._obj.graph
if conftest.option.view:
t.viewcg()
llinterp = LLInterpreter(t.rtyper)
# FIIIIISH
setupgraph = db.gctransformer.frameworkgc_setup_ptr.value._obj.graph
llinterp.eval_graph(setupgraph, [])
def run(args):
ll_args = lltype.malloc(ARGS, immortal=True)
for i in range(nbargs):
ll_args[i] = args[i]
res = llinterp.eval_graph(entrygraph, [ll_args])
return res
if statistics:
statisticsgraph = db.gctransformer.statistics_ptr.value._obj.graph
ll_gc = db.gctransformer.c_const_gc.value
def statistics(index):
return llinterp.eval_graph(statisticsgraph, [ll_gc, index])
return run, statistics
elif transformer:
return run, db.gctransformer
else:
return run
def runner(self, f, nbargs=0, statistics=False, transformer=False,
**extraconfigopts):
if nbargs == 2:
def entrypoint(args):
x = args[0]
y = args[1]
r = f(x, y)
return r
elif nbargs == 0:
def entrypoint(args):
return f()
else:
raise NotImplementedError("pure laziness")
from pypy.rpython.llinterp import LLInterpreter
from pypy.translator.c.genc import CStandaloneBuilder
ARGS = lltype.FixedSizeArray(lltype.Signed, nbargs)
s_args = annmodel.SomePtr(lltype.Ptr(ARGS))
t = rtype(entrypoint, [s_args], gcname=self.gcname,
stacklessgc=self.stacklessgc,
**extraconfigopts)
cbuild = CStandaloneBuilder(t, entrypoint, config=t.config,
gcpolicy=self.gcpolicy)
db = cbuild.generate_graphs_for_llinterp()
entrypointptr = cbuild.getentrypointptr()
entrygraph = entrypointptr._obj.graph
if conftest.option.view:
t.viewcg()
llinterp = LLInterpreter(t.rtyper)
# FIIIIISH
setupgraph = db.gctransformer.frameworkgc_setup_ptr.value._obj.graph
llinterp.eval_graph(setupgraph, [])
def run(args):
ll_args = lltype.malloc(ARGS, immortal=True)
for i in range(nbargs):
ll_args[i] = args[i]
res = llinterp.eval_graph(entrygraph, [ll_args])
return res
if statistics:
statisticsgraph = db.gctransformer.statistics_ptr.value._obj.graph
ll_gc = db.gctransformer.c_const_gc.value
def statistics(index):
return llinterp.eval_graph(statisticsgraph, [ll_gc, index])
return run, statistics
elif transformer:
return run, db.gctransformer
else:
return run
def compile(self, entry_point):
t = TranslationContext(self.config)
t.config.translation.gc = "semispace"
t.config.translation.gcrootfinder = self.gcrootfinder
t.config.translation.thread = True
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
#
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES)
cbuilder.compile()
#
return t, cbuilder
def getcompiled(self, func):
def main(argv):
try:
res = func()
except MemoryError:
print 'Result: MemoryError'
else:
if isinstance(res, int):
print 'Result:', res
else:
print 'Result: "%s"' % (res,)
return 0
from pypy.config.pypyoption import get_pypy_config
config = get_pypy_config(translating=True)
config.translation.gc = self.gcpolicy
config.translation.gcrootfinder = "asmgcc"
t = TranslationContext(config=config)
self.t = t
a = t.buildannotator()
a.build_types(main, [s_list_of_strings])
t.buildrtyper().specialize()
t.checkgraphs()
cbuilder = CStandaloneBuilder(t, main, config=config)
c_source_filename = cbuilder.generate_source(
defines = cbuilder.DEBUG_DEFINES)
self.patch_makefile(cbuilder.targetdir)
if conftest.option.view:
t.view()
exe_name = cbuilder.compile()
def run():
lines = []
print >> sys.stderr, 'RUN: starting', exe_name
g = os.popen("'%s'" % (exe_name,), 'r')
for line in g:
print >> sys.stderr, 'RUN:', line.rstrip()
lines.append(line)
g.close()
if not lines:
py.test.fail("no output from subprocess")
if not lines[-1].startswith('Result:'):
py.test.fail("unexpected output from subprocess")
result = lines[-1][len('Result:'):].strip()
if result == 'MemoryError':
raise MemoryError("subprocess got an RPython MemoryError")
if result.startswith('"') and result.endswith('"'):
return result[1:-1]
else:
return int(result)
return run
def wrap_stackless_function(self, fn):
def entry_point(argv):
os.write(1, str(fn()) + "\n")
return 0
from pypy.config.pypyoption import get_pypy_config
config = get_pypy_config(translating=True)
config.translation.gc = self.gcpolicy
config.translation.stackless = True
if self.stacklessgc:
config.translation.gcrootfinder = "stackless"
t = TranslationContext(config=config)
self.t = t
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
if self.backendopt:
backend_optimizations(t)
from pypy.translator.transform import insert_ll_stackcheck
insert_ll_stackcheck(t)
cbuilder = CStandaloneBuilder(t, entry_point, config=config)
cbuilder.stackless = True
cbuilder.generate_source()
cbuilder.compile()
res = cbuilder.cmdexec('')
return int(res.strip())
def llinterpreter_for_transformed_graph(self, f, args_s):
from pypy.rpython.llinterp import LLInterpreter
from pypy.translator.c.genc import CStandaloneBuilder
from pypy.translator.c import gc
t = rtype(f, args_s)
# XXX we shouldn't need an actual gcpolicy here.
cbuild = CStandaloneBuilder(t, f, t.config, gcpolicy=self.gcpolicy)
db = cbuild.generate_graphs_for_llinterp()
graph = cbuild.getentrypointptr()._obj.graph
llinterp = LLInterpreter(t.rtyper)
if conftest.option.view:
t.view()
return llinterp, graph
def test_no_collect_detection():
from pypy.rlib import rgc
from pypy.translator.c.genc import CStandaloneBuilder
from pypy.translator.c import gc
class A(object):
def __init__(self, x):
self.x = x
@rgc.no_collect
def g():
return A(1).x
assert g._dont_inline_
assert g._gc_no_collect_
def entrypoint(argv):
return g() + 2
t = rtype(entrypoint, [s_list_of_strings])
cbuild = CStandaloneBuilder(t,
entrypoint,
t.config,
gcpolicy=FrameworkGcPolicy2)
f = py.test.raises(Exception, cbuild.generate_graphs_for_llinterp)
assert str(f.value) == 'no_collect function can trigger collection: g'
def setup(self, func):
""" setup all nodes
create c file for externs
create ll file for c file
create codewriter """
# XXX please dont ask!
from pypy.translator.c.genc import CStandaloneBuilder
cbuild = CStandaloneBuilder(self.translator, func, config=self.config)
#cbuild.stackless = self.stackless
c_db = cbuild.generate_graphs_for_llinterp()
self.db = Database(self, self.translator)
# XXX hardcoded for now
self.db.gcpolicy = GcPolicy.new(self.db, 'boehm')
# get entry point
entry_point = self.get_entry_point(func)
self._checkpoint('get_entry_point')
# set up all nodes
self.db.setup_all()
self.entrynode = self.db.set_entrynode(entry_point)
self._checkpoint('setup_all all nodes')
# set up externs nodes
self.extern_decls = setup_externs(c_db, self.db)
self.translator.rtyper.specialize_more_blocks()
self.db.setup_all()
self._checkpoint('setup_all externs')
for node in self.db.getnodes():
node.post_setup_transform()
self._print_node_stats()
# create ll file from c code
self.generate_ll_externs()
self._checkpoint('setup_externs')
# open file & create codewriter
codewriter, self.filename = self.create_codewriter()
self._checkpoint('open file and create codewriter')
return codewriter
def wrap_stackless_function(self, fn):
def entry_point(argv):
os.write(1, str(fn())+"\n")
return 0
from pypy.config.pypyoption import get_pypy_config
config = get_pypy_config(translating=True)
config.translation.gc = self.gcpolicy
config.translation.stackless = True
if self.stacklessgc:
config.translation.gcrootfinder = "stackless"
t = TranslationContext(config=config)
self.t = t
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
if self.backendopt:
backend_optimizations(t)
from pypy.translator.transform import insert_ll_stackcheck
insert_ll_stackcheck(t)
cbuilder = CStandaloneBuilder(t, entry_point, config=config)
cbuilder.stackless = True
cbuilder.generate_source()
cbuilder.compile()
res = cbuilder.cmdexec('')
return int(res.strip())
def test_counters():
from pypy.rpython.lltypesystem import lltype
from pypy.rpython.lltypesystem.lloperation import llop
def entry_point(argv):
llop.instrument_count(lltype.Void, 'test', 2)
llop.instrument_count(lltype.Void, 'test', 1)
llop.instrument_count(lltype.Void, 'test', 1)
llop.instrument_count(lltype.Void, 'test', 2)
llop.instrument_count(lltype.Void, 'test', 1)
return 0
t = TranslationContext()
t.config.translation.instrument = True
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
cbuilder = CStandaloneBuilder(t, entry_point, config=t.config) # xxx
cbuilder.generate_source()
cbuilder.compile()
counters_fname = udir.join("_counters_")
os.putenv('_INSTRUMENT_COUNTERS', str(counters_fname))
try:
data = cbuilder.cmdexec()
finally:
os.unsetenv('_INSTRUMENT_COUNTERS')
f = counters_fname.open('rb')
counters_data = f.read()
f.close()
import struct
counters = struct.unpack("LLL", counters_data)
assert counters == (0,3,2)
def test_standalone_large_files(self):
from pypy.module.posix.test.test_posix2 import need_sparse_files
need_sparse_files()
filename = str(udir.join('test_standalone_largefile'))
r4800000000 = r_longlong(4800000000L)
def entry_point(argv):
fd = os.open(filename, os.O_RDWR | os.O_CREAT, 0644)
os.lseek(fd, r4800000000, 0)
os.write(fd, "$")
newpos = os.lseek(fd, 0, 1)
if newpos == r4800000000 + 1:
print "OK"
else:
print "BAD POS"
os.close(fd)
return 0
t = TranslationContext(self.config)
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
cbuilder.generate_source()
cbuilder.compile()
data = cbuilder.cmdexec('hi there')
assert data.strip() == "OK"
def setup(self, func):
""" setup all nodes
create c file for externs
create ll file for c file
create codewriter """
if self.standalone:
func = augment_entrypoint(self.translator, func)
# XXX please dont ask!
from pypy.translator.c.genc import CStandaloneBuilder
cbuild = CStandaloneBuilder(self.translator, func, config=self.config)
c_db = cbuild.generate_graphs_for_llinterp()
self.db = Database(self, self.translator)
self.db.gcpolicy = GcPolicy.new(self.db, self.config)
self.db.gctransformer = c_db.gctransformer
# get entry point
entry_point = self.get_entry_point(func)
self._checkpoint('get_entry_point')
# set up all nodes
self.db.setup_all()
self.entrynode = self.db.set_entrynode(entry_point)
self._checkpoint('setup_all all nodes')
# set up externs nodes
self.setup_externs(c_db, self.db)
self.db.setup_all()
self._checkpoint('setup_all externs')
self._print_node_stats()
# open file & create codewriter
codewriter, self.filename = self.create_codewriter()
self._checkpoint('open file and create codewriter')
# create ll file from c code
self.generate_ll_externs(codewriter)
return codewriter
def test_no_collect():
from pypy.rlib import rgc
from pypy.translator.c.genc import CStandaloneBuilder
from pypy.translator.c import gc
@rgc.no_collect
def g():
return 1
assert g._dont_inline_
assert g._gc_no_collect_
def entrypoint(argv):
return g() + 2
t = rtype(entrypoint, [s_list_of_strings])
cbuild = CStandaloneBuilder(t, entrypoint, t.config,
gcpolicy=FrameworkGcPolicy2)
db = cbuild.generate_graphs_for_llinterp()
def llinterpreter_for_transformed_graph(self, f, args_s):
from pypy.rpython.llinterp import LLInterpreter
from pypy.translator.c.genc import CStandaloneBuilder
from pypy.translator.c import gc
t = rtype(f, args_s)
# XXX we shouldn't need an actual gcpolicy here.
cbuild = CStandaloneBuilder(t, f, t.config, gcpolicy=self.gcpolicy)
db = cbuild.generate_graphs_for_llinterp()
graph = cbuild.getentrypointptr()._obj.graph
# arguments cannot be GC objects because nobody would put a
# proper header on them
for v in graph.getargs():
if isinstance(v.concretetype, lltype.Ptr):
assert v.concretetype.TO._gckind != 'gc', "fix the test!"
llinterp = LLInterpreter(t.rtyper)
if conftest.option.view:
t.view()
return llinterp, graph
def llinterpreter_for_transformed_graph(self, f, args_s):
from pypy.rpython.llinterp import LLInterpreter
from pypy.translator.c.genc import CStandaloneBuilder
from pypy.translator.c import gc
t = rtype(f, args_s)
# XXX we shouldn't need an actual gcpolicy here.
cbuild = CStandaloneBuilder(t, f, t.config, gcpolicy=self.gcpolicy)
db = cbuild.generate_graphs_for_llinterp()
graph = cbuild.getentrypointptr()._obj.graph
# arguments cannot be GC objects because nobody would put a
# proper header on them
for v in graph.getargs():
if isinstance(v.concretetype, lltype.Ptr):
assert v.concretetype.TO._gckind != 'gc', "fix the test!"
llinterp = LLInterpreter(t.rtyper)
if conftest.option.view:
t.view()
return llinterp, graph
def test_no_collect():
from pypy.rlib import rgc
from pypy.translator.c.genc import CStandaloneBuilder
from pypy.translator.c import gc
@rgc.no_collect
def g():
return 1
assert g._dont_inline_
assert g._gc_no_collect_
def entrypoint(argv):
return g() + 2
t = rtype(entrypoint, [s_list_of_strings])
cbuild = CStandaloneBuilder(t,
entrypoint,
t.config,
gcpolicy=FrameworkGcPolicy2)
db = cbuild.generate_graphs_for_llinterp()
def test_counters(self):
from pypy.rpython.lltypesystem import lltype
from pypy.rpython.lltypesystem.lloperation import llop
def entry_point(argv):
llop.instrument_count(lltype.Void, 'test', 2)
llop.instrument_count(lltype.Void, 'test', 1)
llop.instrument_count(lltype.Void, 'test', 1)
llop.instrument_count(lltype.Void, 'test', 2)
llop.instrument_count(lltype.Void, 'test', 1)
return 0
t = TranslationContext(self.config)
t.config.translation.instrument = True
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
cbuilder = CStandaloneBuilder(t, entry_point, config=t.config) # xxx
cbuilder.generate_source()
cbuilder.compile()
counters_fname = udir.join("_counters_")
os.putenv('_INSTRUMENT_COUNTERS', str(counters_fname))
try:
data = cbuilder.cmdexec()
finally:
os.unsetenv('_INSTRUMENT_COUNTERS')
f = counters_fname.open('rb')
counters_data = f.read()
f.close()
import struct
counters = struct.unpack("LLL", counters_data)
assert counters == (0,3,2)
def compile(self, entry_point, debug=True):
t = TranslationContext(self.config)
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
if debug:
cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES)
else:
cbuilder.generate_source()
cbuilder.compile()
if option.view:
t.view()
return t, cbuilder
def test_separate_files(self):
# One file in translator/c/src
fname = py.path.local(pypydir).join('translator', 'c', 'src',
'll_strtod.h')
# One file in (another) subdir of the temp directory
dirname = udir.join("test_dir").ensure(dir=1)
fname2 = dirname.join("test_genc.c")
fname2.write("""
void f() {
LL_strtod_formatd("%5f", 12.3);
}""")
files = [fname, fname2]
def entry_point(argv):
return 0
t = TranslationContext(self.config)
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
cbuilder.eci = cbuilder.eci.merge(
ExternalCompilationInfo(separate_module_files=files))
cbuilder.generate_source()
makefile = udir.join(cbuilder.modulename, 'Makefile').read()
# generated files are compiled in the same directory
assert " ../test_dir/test_genc.c" in makefile
assert " ../test_dir/test_genc.o" in makefile
# but files from pypy source dir must be copied
assert "translator/c/src" not in makefile
assert " ll_strtod.h" in makefile
assert " ll_strtod.o" in makefile
def test_separate_files(self):
# One file in translator/c/src
fname = py.path.local(pypydir).join(
'translator', 'c', 'src', 'll_strtod.h')
# One file in (another) subdir of the temp directory
dirname = udir.join("test_dir").ensure(dir=1)
fname2 = dirname.join("test_genc.c")
fname2.write("""
void f() {
LL_strtod_formatd("%5f", 12.3);
}""")
files = [fname, fname2]
def entry_point(argv):
return 0
t = TranslationContext(self.config)
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
cbuilder.eci = cbuilder.eci.merge(
ExternalCompilationInfo(separate_module_files=files))
cbuilder.generate_source()
makefile = udir.join(cbuilder.modulename, 'Makefile').read()
# generated files are compiled in the same directory
assert " ../test_dir/test_genc.c" in makefile
assert " ../test_dir/test_genc.o" in makefile
# but files from pypy source dir must be copied
assert "translator/c/src" not in makefile
assert " ll_strtod.h" in makefile
assert " ll_strtod.o" in makefile
def compile(self, entry_point, debug=True):
t = TranslationContext(self.config)
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
if debug:
cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES)
else:
cbuilder.generate_source()
cbuilder.compile()
if option.view:
t.view()
return t, cbuilder
def test_build():
def entry_point(argv):
res = interp_pyexpat.XML_ErrorString(3)
os.write(1, rffi.charp2str(res))
return 0
t = TranslationContext()
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
builder = CStandaloneBuilder(t, entry_point, t.config)
builder.generate_source()
builder.compile()
data = builder.cmdexec()
assert data == pyexpat.ErrorString(3)
def test_os_setpgrp(self):
def entry_point(argv):
os.setpgrp()
return 0
t = TranslationContext(self.config)
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
cbuilder.generate_source()
cbuilder.compile()
def compile(self, entry_point):
t = TranslationContext(self.config)
t.config.translation.gc = "semispace"
t.config.translation.gcrootfinder = self.gcrootfinder
t.config.translation.thread = True
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
#
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES)
cbuilder.compile()
#
return t, cbuilder
def test_frexp():
import math
def entry_point(argv):
m, e = math.frexp(0)
x, y = math.frexp(0)
print m, x
return 0
t = TranslationContext()
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
cbuilder.generate_source()
cbuilder.compile()
data = cbuilder.cmdexec('hi there')
assert map(float, data.split()) == [0.0, 0.0]
def compile(self, entry_point, debug=True, shared=False,
stackcheck=False):
t = TranslationContext(self.config)
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
if stackcheck:
from pypy.translator.transform import insert_ll_stackcheck
insert_ll_stackcheck(t)
t.config.translation.shared = shared
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
if debug:
cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES)
else:
cbuilder.generate_source()
cbuilder.compile()
if option.view:
t.view()
return t, cbuilder
def test_hello_world():
def entry_point(argv):
os.write(1, "hello world\n")
argv = argv[1:]
os.write(1, "argument count: " + str(len(argv)) + "\n")
for s in argv:
os.write(1, " '" + str(s) + "'\n")
return 0
t = TranslationContext()
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
cbuilder.generate_source()
cbuilder.compile()
data = cbuilder.cmdexec('hi there')
assert data.startswith('''hello world\nargument count: 2\n 'hi'\n 'there'\n''')
def run_stackless_function(fn):
def entry_point(argv):
r = fn()
os.write(1, str(r) + '\n')
return 0
t = rtype_stackless_function(entry_point)
cbuilder = CStandaloneBuilder(t,
entry_point,
config=t.config,
gcpolicy=gc.BoehmGcPolicy)
cbuilder.generate_source()
if conftest.option.view:
t.view()
cbuilder.compile()
res = cbuilder.cmdexec('')
return int(res.strip())
class I386PortalTestMixin(object):
RGenOp = RI386GenOp
def postprocess_timeshifting(self):
annhelper = self.hrtyper.annhelper
convert_result = getattr(self.main, 'convert_result', str)
annotator = self.rtyper.annotator
args_s = [annmodel.lltype_to_annotation(v.concretetype)
for v in self.maingraph.getargs()]
retvar = self.maingraph.getreturnvar()
s_result = annmodel.lltype_to_annotation(retvar.concretetype)
main_fnptr = self.rtyper.type_system.getcallable(self.maingraph)
main = PseudoHighLevelCallable(main_fnptr, args_s, s_result)
if hasattr(self.main, 'convert_arguments'):
decoders = self.main.convert_arguments
assert len(decoders) == len(args_s)
else:
decoders = [int] * len(args_s)
decoders = unrolling_iterable(decoders)
def ll_main(argv):
args = ()
i = 1
for decoder in decoders:
args += (decoder(argv[i]),)
i = i + 1
try:
res = main(*args)
except Exception, e:
os.write(1, 'EXCEPTION: %s\n' % (e,))
return 0
os.write(1, convert_result(res) + '\n')
return 0
annhelper.getgraph(ll_main, [s_list_of_strings],
annmodel.SomeInteger())
annhelper.finish()
t = self.rtyper.annotator.translator
t.config.translation.gc = 'boehm'
self.cbuilder = CStandaloneBuilder(t, ll_main, config=t.config)
self.cbuilder.generate_source()
self.cbuilder.compile()
def run_stackless_function(fn):
def entry_point(argv):
r = fn()
os.write(1, str(r) + "\n")
return 0
t = rtype_stackless_function(entry_point)
cbuilder = CStandaloneBuilder(t, entry_point, config=t.config, gcpolicy=gc.BoehmGcPolicy)
cbuilder.generate_source()
if conftest.option.view:
t.view()
cbuilder.compile()
res = cbuilder.cmdexec("")
return int(res.strip())
def test_build():
def entry_point(argv):
res = interp_pyexpat.XML_ErrorString(3)
os.write(1, rffi.charp2str(res))
return 0
t = TranslationContext()
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
builder = CStandaloneBuilder(t, entry_point, t.config)
builder.generate_source()
builder.compile()
data = builder.cmdexec()
assert data == pyexpat.ErrorString(3)
def test_frexp():
import math
def entry_point(argv):
m, e = math.frexp(0)
x, y = math.frexp(0)
print m, x
return 0
t = TranslationContext()
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
cbuilder.generate_source()
cbuilder.compile()
data = cbuilder.cmdexec('hi there')
assert map(float, data.split()) == [0.0, 0.0]
def compile(self, entry_point, debug=True, shared=False,
stackcheck=False):
t = TranslationContext(self.config)
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
if stackcheck:
from pypy.translator.transform import insert_ll_stackcheck
insert_ll_stackcheck(t)
t.config.translation.shared = shared
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
if debug:
cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES)
else:
cbuilder.generate_source()
cbuilder.compile()
if option.view:
t.view()
return t, cbuilder
def test_hello_world():
def entry_point(argv):
os.write(1, "hello world\n")
argv = argv[1:]
os.write(1, "argument count: " + str(len(argv)) + "\n")
for s in argv:
os.write(1, " '" + str(s) + "'\n")
return 0
t = TranslationContext()
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
cbuilder.generate_source()
cbuilder.compile()
data = cbuilder.cmdexec('hi there')
assert data.startswith('''hello world\nargument count: 2\n 'hi'\n 'there'\n''')
def test_print():
def entry_point(argv):
print "hello simpler world"
argv = argv[1:]
print "argument count:", len(argv)
print "arguments:", argv
print "argument lengths:",
print [len(s) for s in argv]
return 0
t = TranslationContext()
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
cbuilder.generate_source()
cbuilder.compile()
data = cbuilder.cmdexec('hi there')
assert data.startswith('''hello simpler world\n'''
'''argument count: 2\n'''
'''arguments: [hi, there]\n'''
'''argument lengths: [2, 5]\n''')
def test_standalone_large_files():
from pypy.module.posix.test.test_posix2 import need_sparse_files
need_sparse_files()
filename = str(udir.join('test_standalone_largefile'))
r4800000000 = r_longlong(4800000000L)
def entry_point(argv):
fd = os.open(filename, os.O_RDWR | os.O_CREAT, 0644)
os.lseek(fd, r4800000000, 0)
os.write(fd, "$")
newpos = os.lseek(fd, 0, 1)
if newpos == r4800000000 + 1:
print "OK"
else:
print "BAD POS"
os.close(fd)
return 0
t = TranslationContext()
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
cbuilder.generate_source()
cbuilder.compile()
data = cbuilder.cmdexec('hi there')
assert data.strip() == "OK"
def test_stack_size(self):
import time
from pypy.module.thread import ll_thread
class State:
pass
state = State()
def recurse(n):
if n > 0:
return recurse(n-1)+1
else:
state.ll_lock.release()
time.sleep(0.2)
state.ll_lock.acquire(True)
return 0
def bootstrap():
# recurse a lot, like 2500 times
state.ll_lock.acquire(True)
recurse(2500)
state.count += 1
state.ll_lock.release()
def entry_point(argv):
os.write(1, "hello world\n")
error = ll_thread.set_stacksize(int(argv[1]))
assert error == 0
# malloc a bit
s1 = State(); s2 = State(); s3 = State()
s1.x = 0x11111111; s2.x = 0x22222222; s3.x = 0x33333333
# start 3 new threads
state.ll_lock = ll_thread.Lock(ll_thread.allocate_ll_lock())
state.count = 0
ident1 = ll_thread.start_new_thread(bootstrap, ())
ident2 = ll_thread.start_new_thread(bootstrap, ())
ident3 = ll_thread.start_new_thread(bootstrap, ())
# wait for the 3 threads to finish
while True:
state.ll_lock.acquire(True)
if state.count == 3:
break
state.ll_lock.release()
time.sleep(0.1)
# check that the malloced structures were not overwritten
assert s1.x == 0x11111111
assert s2.x == 0x22222222
assert s3.x == 0x33333333
os.write(1, "done\n")
return 0
t = TranslationContext()
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
cbuilder.generate_source()
cbuilder.compile()
# recursing should crash with only 32 KB of stack,
# and it should eventually work with more stack
for test_kb in [32, 128, 512, 1024, 2048, 4096, 8192, 16384]:
print >> sys.stderr, 'Trying with %d KB of stack...' % (test_kb,),
try:
data = cbuilder.cmdexec(str(test_kb * 1024))
except Exception, e:
if e.__class__ is not Exception:
raise
print >> sys.stderr, 'segfault'
# got a segfault! try with the next stack size...
else:
# it worked
print >> sys.stderr, 'ok'
assert data == 'hello world\ndone\n'
assert test_kb > 32 # it cannot work with just 32 KB of stack
break # finish
def setup_class(cls):
funcs0 = []
funcs2 = []
cleanups = []
name_to_func = {}
mixlevelstuff = []
for fullname in dir(cls):
if not fullname.startswith('define'):
continue
definefunc = getattr(cls, fullname)
_, name = fullname.split('_', 1)
func_fixup = definefunc.im_func(cls)
cleanup = None
if isinstance(func_fixup, tuple):
func, cleanup, fixup = func_fixup
mixlevelstuff.append(fixup)
else:
func = func_fixup
func.func_name = "f_%s" % name
if cleanup:
cleanup.func_name = "clean_%s" % name
nargs = len(inspect.getargspec(func)[0])
name_to_func[name] = len(funcs0)
if nargs == 2:
funcs2.append(func)
funcs0.append(None)
elif nargs == 0:
funcs0.append(func)
funcs2.append(None)
else:
raise NotImplementedError(
"defined test functions should have 0/2 arguments")
# used to let test cleanup static root pointing to runtime
# allocated stuff
cleanups.append(cleanup)
def entrypoint(args):
num = args[0]
func = funcs0[num]
if func:
res = func()
else:
func = funcs2[num]
res = func(args[1], args[2])
cleanup = cleanups[num]
if cleanup:
cleanup()
return res
from pypy.translator.c.genc import CStandaloneBuilder
s_args = annmodel.SomePtr(lltype.Ptr(ARGS))
t = rtype(entrypoint, [s_args], gcname=cls.gcname,
stacklessgc=cls.stacklessgc,
taggedpointers=cls.taggedpointers)
for fixup in mixlevelstuff:
if fixup:
fixup(t)
cbuild = CStandaloneBuilder(t, entrypoint, config=t.config,
gcpolicy=cls.gcpolicy)
db = cbuild.generate_graphs_for_llinterp()
entrypointptr = cbuild.getentrypointptr()
entrygraph = entrypointptr._obj.graph
if conftest.option.view:
t.viewcg()
cls.name_to_func = name_to_func
cls.entrygraph = entrygraph
cls.rtyper = t.rtyper
cls.db = db
class I386TimeshiftingTestMixin(object):
RGenOp = RI386GenOp
SEPLINE = 'running residual graph...\n'
def annotate_interface_functions(self):
annhelper = self.hrtyper.annhelper
RGenOp = self.RGenOp
SEPLINE = self.SEPLINE
ml_generate_code = self.ml_generate_code
argcolors = list(self.argcolors)
if hasattr(self.ll_function, 'convert_arguments'):
decoders = self.ll_function.convert_arguments
assert len(decoders) == len(argcolors)
else:
decoders = [int] * len(argcolors)
argcolors_decoders = zip(argcolors, decoders)
argcolors_decoders = unrolling_iterable(argcolors_decoders)
convert_result = getattr(self.ll_function, 'convert_result', str)
def ll_main(argv):
i = 1
mainargs = ()
residualargs = ()
if len(argv) == 2 and argv[1] == '--help':
os.write(1, 'usage: ' + argv[0])
for color, decoder in argcolors_decoders:
os.write(1, ' ')
if color == 'green':
os.write(1, decoder.__name__)
else:
os.write(1, "-const|-var "+decoder.__name__)
os.write(1, '\n')
return 0
for color, decoder in argcolors_decoders:
try:
if color == 'green':
llvalue = decoder(argv[i])
mainargs += (llvalue,)
i = i + 1
else:
if argv[i] == '-const':
is_const = True
elif argv[i] == '-var':
is_const = False
else:
raise ValueError()
i += 1
llvalue = decoder(argv[i])
mainargs += (is_const, llvalue)
residualargs += (llvalue,)
i += 1
except (ValueError, IndexError):
j = 1
while j < len(argv):
arg = argv[j]
if j == i:
os.write(1, '--> ')
else:
os.write(1, ' ')
os.write(1, arg+'\n')
j += 1
if j == i:
os.write(1, '-->\n')
return 1
rgenop = RGenOp()
generated = ml_generate_code(rgenop, *mainargs)
os.write(1, SEPLINE)
bench = Benchmark()
while 1:
try:
res = generated(*residualargs)
except Exception, e:
os.write(1, 'EXCEPTION: %s\n' % (e,))
return 0
if bench.stop():
break
os.write(1, convert_result(res) + '\n')
rgenop.check_no_open_mc()
keepalive_until_here(rgenop) # to keep the code blocks alive
return 0
annhelper.getgraph(ll_main, [s_list_of_strings],
annmodel.SomeInteger())
annhelper.finish()
t = self.rtyper.annotator.translator
t.config.translation.gc = 'boehm'
cbuilder = CStandaloneBuilder(t, ll_main, config=t.config)
cbuilder.generate_source()
cbuilder.compile()
self.main_cbuilder= cbuilder
def _makefunc_str_int(cls, func):
def main(argv):
arg0 = argv[1]
arg1 = int(argv[2])
try:
res = func(arg0, arg1)
except MemoryError:
print 'Result: MemoryError'
else:
print 'Result: "%s"' % (res,)
return 0
config = cls.make_config()
t = TranslationContext(config=config)
a = t.buildannotator()
sec_ep = getattr(cls, 'secondary_entrypoints', [])
for f, inputtypes in sec_ep:
a.build_types(f, inputtypes, False)
a.build_types(main, [s_list_of_strings])
t.buildrtyper().specialize()
t.checkgraphs()
cbuilder = CStandaloneBuilder(t, main, config=config,
secondary_entrypoints=sec_ep)
c_source_filename = cbuilder.generate_source(
defines = cbuilder.DEBUG_DEFINES)
cls._patch_makefile(cbuilder.targetdir)
if conftest.option.view:
t.view()
exe_name = cbuilder.compile()
def run(arg0, arg1):
lines = []
print >> sys.stderr, 'RUN: starting', exe_name, arg0, arg1
if sys.platform == 'win32':
redirect = ' 2> NUL'
else:
redirect = ''
if config.translation.shared and os.name == 'posix':
env = 'LD_LIBRARY_PATH="%s" ' % (exe_name.dirpath(),)
else:
env = ''
cwd = os.getcwd()
try:
os.chdir(str(exe_name.dirpath()))
g = os.popen(
'%s"%s" %s %d%s' % (env, exe_name, arg0, arg1, redirect), 'r')
finally:
os.chdir(cwd)
for line in g:
print >> sys.stderr, 'RUN:', line.rstrip()
lines.append(line)
g.close()
if not lines:
py.test.fail("no output from subprocess")
if not lines[-1].startswith('Result:'):
py.test.fail("unexpected output from subprocess")
result = lines[-1][len('Result:'):].strip()
if result == 'MemoryError':
raise MemoryError("subprocess got an RPython MemoryError")
if result.startswith('"') and result.endswith('"'):
return result[1:-1]
else:
return int(result)
return run
def _makefunc_str_int(cls, func):
def main(argv):
arg0 = argv[1]
arg1 = int(argv[2])
try:
res = func(arg0, arg1)
except MemoryError:
print 'Result: MemoryError'
else:
print 'Result: "%s"' % (res, )
return 0
config = cls.make_config()
t = TranslationContext(config=config)
a = t.buildannotator()
sec_ep = getattr(cls, 'secondary_entrypoints', [])
for f, inputtypes in sec_ep:
a.build_types(f, inputtypes, False)
a.build_types(main, [s_list_of_strings])
t.buildrtyper().specialize()
t.checkgraphs()
cbuilder = CStandaloneBuilder(t,
main,
config=config,
secondary_entrypoints=sec_ep)
c_source_filename = cbuilder.generate_source(
defines=cbuilder.DEBUG_DEFINES)
cls._patch_makefile(cbuilder.targetdir)
if conftest.option.view:
t.view()
exe_name = cbuilder.compile()
def run(arg0, arg1):
lines = []
print >> sys.stderr, 'RUN: starting', exe_name, arg0, arg1
if sys.platform == 'win32':
redirect = ' 2> NUL'
else:
redirect = ''
if config.translation.shared and os.name == 'posix':
env = 'LD_LIBRARY_PATH="%s" ' % (exe_name.dirpath(), )
else:
env = ''
cwd = os.getcwd()
try:
os.chdir(str(exe_name.dirpath()))
g = os.popen(
'%s"%s" %s %d%s' % (env, exe_name, arg0, arg1, redirect),
'r')
finally:
os.chdir(cwd)
for line in g:
print >> sys.stderr, 'RUN:', line.rstrip()
lines.append(line)
g.close()
if not lines:
py.test.fail("no output from subprocess")
if not lines[-1].startswith('Result:'):
py.test.fail("unexpected output from subprocess")
result = lines[-1][len('Result:'):].strip()
if result == 'MemoryError':
raise MemoryError("subprocess got an RPython MemoryError")
if result.startswith('"') and result.endswith('"'):
return result[1:-1]
else:
return int(result)
return run
def test_print():
def entry_point(argv):
print "hello simpler world"
argv = argv[1:]
print "argument count:", len(argv)
print "arguments:", argv
print "argument lengths:",
print [len(s) for s in argv]
return 0
t = TranslationContext()
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
cbuilder.generate_source()
cbuilder.compile()
data = cbuilder.cmdexec('hi there')
assert data.startswith('''hello simpler world\n'''
'''argument count: 2\n'''
'''arguments: [hi, there]\n'''
'''argument lengths: [2, 5]\n''')
# NB. RPython has only str, not repr, so str() on a list of strings
# gives the strings unquoted in the list
def test_counters():
if sys.platform == 'win32':
py.test.skip("instrument counters support is unix only for now")
from pypy.rpython.lltypesystem import lltype
from pypy.rpython.lltypesystem.lloperation import llop
