def get_external_function_sandbox_graph(fnobj, db, force_stub=False):
"""Build the graph of a helper trampoline function to be used
in place of real calls to the external function 'fnobj'. The
trampoline marshals its input arguments, dumps them to STDOUT,
and waits for an answer on STDIN.
"""
fnname = fnobj._name
if hasattr(fnobj, 'graph'):
# get the annotation of the input arguments and the result
graph = fnobj.graph
annotator = db.translator.annotator
args_s = [annotator.binding(v) for v in graph.getargs()]
s_result = annotator.binding(graph.getreturnvar())
else:
# pure external function - fall back to the annotations
# corresponding to the ll types
FUNCTYPE = lltype.typeOf(fnobj)
args_s = [annmodel.lltype_to_annotation(ARG) for ARG in FUNCTYPE.ARGS]
s_result = annmodel.lltype_to_annotation(FUNCTYPE.RESULT)
try:
if force_stub: # old case - don't try to support suggested_primitive
raise NotImplementedError("sandboxing for external function '%s'"
% (fnname,))
dump_arguments = rmarshal.get_marshaller(tuple(args_s))
load_result = rmarshal.get_loader(s_result)
except (NotImplementedError,
rmarshal.CannotMarshal,
rmarshal.CannotUnmarshall), e:
msg = 'Not Implemented: %s' % (e,)
log.WARNING(msg)
def execute(*args):
not_implemented_stub(msg)
def helper_func(self, FUNCPTR, func):
if not self.cpu.translate_support_code:
return llhelper(FUNCPTR, func)
FUNC = FUNCPTR.TO
args_s = [annmodel.lltype_to_annotation(ARG) for ARG in FUNC.ARGS]
s_result = annmodel.lltype_to_annotation(FUNC.RESULT)
graph = self.annhelper.getgraph(func, args_s, s_result)
return self.annhelper.graph2delayed(graph, FUNC)
def gendirectcall(self, ll_function, *args_v):
rtyper = self.rtyper
args_s = []
newargs_v = []
for v in args_v:
if v.concretetype is Void:
s_value = rtyper.binding(v, default=annmodel.s_None)
if not s_value.is_constant():
raise TyperError("non-constant variable of type Void")
if not isinstance(s_value, annmodel.SomePBC):
raise TyperError("non-PBC Void argument: %r", (s_value,))
args_s.append(s_value)
else:
args_s.append(annmodel.lltype_to_annotation(v.concretetype))
newargs_v.append(v)
self.rtyper.call_all_setups() # compute ForwardReferences now
# hack for bound methods
if hasattr(ll_function, 'im_func'):
bk = rtyper.annotator.bookkeeper
args_s.insert(0, bk.immutablevalue(ll_function.im_self))
newargs_v.insert(0, inputconst(Void, ll_function.im_self))
ll_function = ll_function.im_func
graph = annotate_lowlevel_helper(rtyper.annotator, ll_function, args_s,
rtyper.lowlevel_ann_policy)
self.record_extra_call(graph)
# build the 'direct_call' operation
f = self.rtyper.getcallable(graph)
c = inputconst(typeOf(f), f)
fobj = self.rtyper.type_system_deref(f)
return self.genop('direct_call', [c]+newargs_v,
resulttype = typeOf(fobj).RESULT)
def _compute_annotation(t, bookkeeper=None):
from rpython.rtyper.lltypesystem import lltype
from rpython.rtyper import extregistry
if isinstance(t, SomeObject):
return t
elif isinstance(t, lltype.LowLevelType):
return lltype_to_annotation(t)
elif isinstance(t, list):
assert len(t) == 1, "We do not support type joining in list"
listdef = ListDef(bookkeeper, annotation(t[0]), mutated=True, resized=True)
return SomeList(listdef)
elif isinstance(t, tuple):
return SomeTuple(tuple([annotation(i) for i in t]))
elif isinstance(t, dict):
assert len(t) == 1, "We do not support type joining in dict"
result = SomeDict(DictDef(bookkeeper, annotation(t.keys()[0]),
annotation(t.values()[0])))
return result
elif type(t) is types.NoneType:
return s_None
elif extregistry.is_registered(t):
entry = extregistry.lookup(t)
entry.bookkeeper = bookkeeper
return entry.compute_result_annotation()
else:
return annotationoftype(t, bookkeeper)
def test_ll_calling_ll(self):
A = GcArray(Float)
B = GcArray(Signed)
def ll_make(T, n):
x = malloc(T, n)
return x
def ll_get(T, x, i):
return x[i]
def llf():
a = ll_make(A, 3)
b = ll_make(B, 2)
a[0] = 1.0
b[1] = 3
y0 = ll_get(A, a, 1)
y1 = ll_get(B, b, 1)
#
a2 = ll_make(A, 4)
a2[0] = 2.0
return ll_get(A, a2, 1)
s = self.annotate(llf, [])
a = self.a
assert s == annmodel.SomeFloat()
seen = {}
ngraphs = len(a.translator.graphs)
vTs = []
for call in annotated_calls(a):
if derived(call, "ll_"):
func, T = [x.value for x in call.args[0:2]]
if (func, T) in seen:
continue
seen[func, T] = True
desc = a.bookkeeper.getdesc(func)
g = desc.specialize([a.binding(x) for x in call.args[1:]])
args = g.getargs()
rv = g.getreturnvar()
if func is ll_get:
vT, vp, vi = args
assert a.binding(vT) == a.bookkeeper.immutablevalue(T)
assert a.binding(vi).knowntype == int
assert a.binding(vp).ll_ptrtype.TO == T
assert a.binding(rv) == annmodel.lltype_to_annotation(T.OF)
elif func is ll_make:
vT, vn = args
assert a.binding(vT) == a.bookkeeper.immutablevalue(T)
assert a.binding(vn).knowntype == int
assert a.binding(rv).ll_ptrtype.TO == T
else:
assert False, func
vTs.append(vT)
assert len(seen) == 4
return a, vTs # reused by a test in test_rtyper
def lltype_to_annotation(cls, TYPE):
if isinstance(TYPE, NativeInstance):
return SomeOOInstance(TYPE)
elif TYPE is ootype.Char:
return SomeChar()
elif TYPE is ootype.String:
return SomeString(can_be_None=True)
else:
return lltype_to_annotation(TYPE)
def getprimitiverepr(self, lltype):
try:
return self.primitive_to_repr[lltype]
except KeyError:
pass
if isinstance(lltype, Primitive):
repr = self.primitive_to_repr[lltype] = self.getrepr(annmodel.lltype_to_annotation(lltype))
return repr
raise TyperError('There is no primitive repr for %r' % (lltype,))
def lltype_to_annotation(cls, TYPE):
if isinstance(TYPE, NativeInstance):
return SomeOOInstance(TYPE)
elif TYPE is ootype.Char:
return SomeChar()
elif TYPE is ootype.String:
return SomeString(can_be_None=True)
else:
return lltype_to_annotation(TYPE)
def complete_destrptr(gctransformer):
translator = gctransformer.translator
mixlevelannotator = MixLevelHelperAnnotator(translator.rtyper)
args_s = [annmodel.lltype_to_annotation(lltype.Ptr(SUSPSTACK))]
s_result = annmodel.s_None
destrptr = mixlevelannotator.delayedfunction(suspstack_destructor,
args_s, s_result)
mixlevelannotator.finish()
lltype.attachRuntimeTypeInfo(SUSPSTACK, destrptr=destrptr)
def compute_result_annotation(self, s_RESTYPE, s_pythonfunction, *args_s):
from rpython.annotator import model as annmodel
from rpython.rtyper.lltypesystem import lltype
assert s_RESTYPE.is_constant()
assert s_pythonfunction.is_constant()
s_result = s_RESTYPE.const
if isinstance(s_result, lltype.LowLevelType):
s_result = annmodel.lltype_to_annotation(s_result)
assert isinstance(s_result, annmodel.SomeObject)
return s_result
def rtype_simple_call(self, hop):
hop2 = hop.copy()
func = self.func
s_func = hop.rtyper.annotator.bookkeeper.immutablevalue(func)
v_ptr = hop2.args_v[0]
hop2.r_s_popfirstarg()
hop2.v_s_insertfirstarg(
v_ptr, annmodel.lltype_to_annotation(self.ll_ptrtype))
hop2.v_s_insertfirstarg(flowmodel.Constant(func), s_func)
return hop2.dispatch()
def annotate_helper(self, ll_helper, ll_args, ll_result, inline=False):
assert not self.finished_helpers
args_s = map(annmodel.lltype_to_annotation, ll_args)
s_result = annmodel.lltype_to_annotation(ll_result)
graph = self.mixlevelannotator.getgraph(ll_helper, args_s, s_result)
# the produced graphs does not need to be fully transformed
self.need_minimal_transform(graph)
if inline:
self.graphs_to_inline[graph] = True
FUNCTYPE = lltype.FuncType(ll_args, ll_result)
return self.mixlevelannotator.graph2delayed(graph, FUNCTYPE=FUNCTYPE)
def consider_call_site(self, call_op):
binding = self.annotator.binding
s_callable = binding(call_op.args[0])
args_s = [binding(arg) for arg in call_op.args[1:]]
if isinstance(s_callable, SomeLLADTMeth):
adtmeth = s_callable
s_callable = self.immutablevalue(adtmeth.func)
args_s = [lltype_to_annotation(adtmeth.ll_ptrtype)] + args_s
if isinstance(s_callable, SomePBC):
s_result = binding(call_op.result, s_ImpossibleValue)
self.consider_call_site_for_pbc(s_callable, call_op.opname, args_s,
s_result, call_op)
def _get_rmarshall_support_(self): # for rlib.rmarshal
# reduce and recreate stat_result objects from 10-tuples
# (we ignore the extra values here for simplicity and portability)
def stat_result_reduce(st):
return (st[0], st[1], st[2], st[3], st[4],
st[5], st[6], st[7], st[8], st[9])
def stat_result_recreate(tup):
return make_stat_result(tup + extra_zeroes)
s_reduced = annmodel.SomeTuple([annmodel.lltype_to_annotation(TYPE)
for name, TYPE in PORTABLE_STAT_FIELDS])
extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS))
return s_reduced, stat_result_reduce, stat_result_recreate
def compile(self, fn, args, ann=None, backendopt=False):
if ann is None:
ann = [lltype_to_annotation(typeOf(x)) for x in args]
if self._func is fn and self._ann == ann:
return JvmGeneratedSourceWrapper(self._jvm_src)
else:
self._func = fn
self._ann = ann
olddefs = patch_os()
self._jvm_src = generate_source_for_function(fn, ann, backendopt)
unpatch_os(olddefs)
if not getoption('noasm'):
self._jvm_src.compile()
return JvmGeneratedSourceWrapper(self._jvm_src)
def compile(self, fn, args, ann=None, backendopt=False):
if ann is None:
ann = [lltype_to_annotation(typeOf(x)) for x in args]
if self._func is fn and self._ann == ann:
return JvmGeneratedSourceWrapper(self._jvm_src)
else:
self._func = fn
self._ann = ann
olddefs = patch_os()
self._jvm_src = generate_source_for_function(fn, ann, backendopt)
unpatch_os(olddefs)
if not getoption('noasm'):
self._jvm_src.compile()
return JvmGeneratedSourceWrapper(self._jvm_src)
def robjmodel_hlinvoke(s_repr, s_llcallable, *args_s):
from rpython.rtyper import rmodel
from rpython.rtyper.error import TyperError
assert s_repr.is_constant() and isinstance(s_repr.const, rmodel.Repr), "hlinvoke expects a constant repr as first argument"
r_func, nimplicitarg = s_repr.const.get_r_implfunc()
nbargs = len(args_s) + nimplicitarg
s_sigs = r_func.get_s_signatures((nbargs, (), False, False))
if len(s_sigs) != 1:
raise TyperError("cannot hlinvoke callable %r with not uniform"
"annotations: %r" % (s_repr.const,
s_sigs))
_, s_ret = s_sigs[0]
rresult = r_func.rtyper.getrepr(s_ret)
return lltype_to_annotation(rresult.lowleveltype)
def annotate_helper(self, ll_function, argtypes):
"""Annotate the given low-level helper function and return its graph
"""
args_s = []
for s in argtypes:
# assume 's' is a low-level type, unless it is already an annotation
if not isinstance(s, annmodel.SomeObject):
s = annmodel.lltype_to_annotation(s)
args_s.append(s)
# hack for bound methods
if hasattr(ll_function, 'im_func'):
bk = self.annotator.bookkeeper
args_s.insert(0, bk.immutablevalue(ll_function.im_self))
ll_function = ll_function.im_func
helper_graph = annotate_lowlevel_helper(self.annotator,
ll_function, args_s,
policy=self.lowlevel_ann_policy)
return helper_graph
def test_ll_calling_ll2(self):
A = GcArray(Float)
B = GcArray(Signed)
def ll_make(T, n):
x = malloc(T, n)
return x
def ll_get(x, i):
return x[i]
def makelen4(T):
return ll_make(T, 4)
def llf():
a = ll_make(A, 3)
b = ll_make(B, 2)
a[0] = 1.0
b[1] = 3
y0 = ll_get(a, 1)
y1 = ll_get(b, 1)
#
a2 = makelen4(A)
a2[0] = 2.0
return ll_get(a2, 1)
s = self.annotate(llf, [])
a = self.a
assert s == annmodel.SomeFloat()
seen = {}
def q(v):
s = a.binding(v)
if s.is_constant():
return s.const
else:
return s.ll_ptrtype
vTs = []
for call in annotated_calls(a):
if derived(call, "ll_") or derived(call, "makelen4"):
func, T = [q(x) for x in call.args[0:2]]
if (func, T) in seen:
continue
seen[func, T] = True
desc = a.bookkeeper.getdesc(func)
g = desc.specialize([a.binding(x) for x in call.args[1:]])
args = g.getargs()
rv = g.getreturnvar()
if func is ll_make:
vT, vn = args
assert a.binding(vT) == a.bookkeeper.immutablevalue(T)
assert a.binding(vn).knowntype == int
assert a.binding(rv).ll_ptrtype.TO == T
vTs.append(vT)
elif func is makelen4:
vT, = args
assert a.binding(vT) == a.bookkeeper.immutablevalue(T)
assert a.binding(rv).ll_ptrtype.TO == T
vTs.append(vT)
elif func is ll_get:
vp, vi = args
assert a.binding(vi).knowntype == int
assert a.binding(vp).ll_ptrtype == T
assert a.binding(rv) == annmodel.lltype_to_annotation(
T.TO.OF)
else:
assert False, func
assert len(seen) == 5
return a, vTs # reused by a test in test_rtyper
def call(adtmeth, args):
bookkeeper = getbookkeeper()
s_func = bookkeeper.immutablevalue(adtmeth.func)
return s_func.call(args.prepend(lltype_to_annotation(adtmeth.ll_ptrtype)))
def compute_result_annotation(self, s_TP, s_storage, s_index):
assert s_TP.is_constant()
return annmodel.lltype_to_annotation(s_TP.const)
def getattr(self, s_attr):
assert s_attr.is_constant(), "non-constant attr name in getattr()"
attrname = s_attr.const
TYPE = STAT_FIELD_TYPES[attrname]
return annmodel.lltype_to_annotation(TYPE)
def getitem((s_stat, s_int)):
assert s_int.is_constant()
name, TYPE = STATVFS_FIELDS[s_int.const]
return annmodel.lltype_to_annotation(TYPE)
def getitem((s_sta, s_int)):
assert s_int.is_constant(), "os.stat()[index]: index must be constant"
index = s_int.const
assert 0 <= index < N_INDEXABLE_FIELDS, "os.stat()[index] out of range"
name, TYPE = STAT_FIELDS[index]
return annmodel.lltype_to_annotation(TYPE)
def getattr(self, s_attr):
assert s_attr.is_constant()
TYPE = STATVFS_FIELD_TYPES[s_attr.const]
return annmodel.lltype_to_annotation(TYPE)
def compute_result_annotation(self, RESULTTYPE, *args):
from rpython.annotator.model import lltype_to_annotation
assert RESULTTYPE.is_constant()
return lltype_to_annotation(RESULTTYPE.const)
def __init__(self, adtmeth, rtyper):
self.func = adtmeth.func
self.lowleveltype = adtmeth.ll_ptrtype
self.ll_ptrtype = adtmeth.ll_ptrtype
self.lowleveltype = rtyper.getrepr(annmodel.lltype_to_annotation(adtmeth.ll_ptrtype)).lowleveltype
def compute_result_annotation(self, *args):
if (isinstance(s_result, annmodel.SomeObject) or
s_result is None):
return s_result
return annmodel.lltype_to_annotation(s_result)
def annotation(self):
return lltype_to_annotation(self.TP)
def getitem((s_taa, s_int)):
from rpython.annotator.model import lltype_to_annotation
return lltype_to_annotation(s_taa.type)
def compute_result_annotation(self, *args):
from rpython.annotator.model import lltype_to_annotation
return lltype_to_annotation(lltype.Void)
def get_annotation(x):
if isinstance(x, basestring) and len(x) > 1:
return SomeString(no_nul='\x00' not in x)
else:
return lltype_to_annotation(typeOf(x))
