def read_linklet_bundle_hash(in_port, env, cont):
from pycket.racket_entry import get_primitive
from pycket.fasl import Fasl
from pycket.util import console_log
current_load_relative_dir_path = get_primitive("current-load-relative-directory").get_cell_value(cont)
fasl_to_s_exp = get_primitive("fasl->s-exp")
with PerfRegion("fasl->s-exp"):
bundle_map = Fasl(current_load_relative_dir_path).to_sexp_from_w_port(in_port)
#return fasl_to_s_exp.call([in_port, values.w_true], env, read_linklet_cont(env, cont))
if not isinstance(bundle_map, W_HashTable):
raise SchemeException("got something that is not a table: %s" % bundle_map.tostring())
console_log("BUNDLE SEXP FASL-READ from ZO: %s" % deserialize_loop(bundle_map).tostring(), 7)
with PerfRegion("s-exp->ast"):
return return_value(deserialize_loop(bundle_map), env, cont)
def write_linklet_bundle_hash(ht, out_port, env, cont):
from pycket.util import console_log
from pycket.racket_entry import get_primitive
console_log("BUNDLE AST TO BE SERIALIZED: %s" % ht.tostring(), 7)
with PerfRegion("ast->sexp"):
l = ht.length()
keys = [None] * l
vals = [None] * l
i = 0
for k, v in ht.iteritems():
keys[i] = k
vals[i] = ast_to_sexp(v)
i += 1
bundle_s_exp = make_simple_immutable_table(W_EqImmutableHashTable,
keys, vals)
console_log("WRITING BUNDLE SEXP : %s" % bundle_s_exp.tostring(), 7)
s_exp_to_fasl = get_primitive("s-exp->fasl")
with PerfRegionCPS("s-exp->fasl"):
return s_exp_to_fasl.call([bundle_s_exp, out_port, values.w_false],
env,
finish_perf_region_cont(
"s-exp->fasl", env, cont))
def write_linklet_bundle_hash(ht, out_port, env, cont):
from pycket.util import console_log
from pycket.racket_entry import get_primitive
console_log("BUNDLE AST TO BE SERIALIZED: %s" % ht.tostring(), 7)
with PerfRegion("ast->sexp"):
l = ht.length()
keys = [None]*l
vals = [None]*l
i = 0
for k, v in ht.iteritems():
keys[i] = k
vals[i] = ast_to_sexp(v)
i += 1
bundle_s_exp = make_simple_immutable_table(W_EqImmutableHashTable, keys, vals)
console_log("WRITING BUNDLE SEXP : %s" % bundle_s_exp.tostring(), 7)
s_exp_to_fasl = get_primitive("s-exp->fasl")
with PerfRegionCPS("s-exp->fasl"):
return s_exp_to_fasl.call([bundle_s_exp, out_port, values.w_false], env,
finish_perf_region_cont("s-exp->fasl", env, cont))
def fasl_to_sexp_recursive(self, fasl_string, pos):
from pycket import values as v
from pycket.values_string import W_String
from pycket.values_regex import W_Regexp, W_PRegexp, W_ByteRegexp, W_BytePRegexp
from pycket.vector import W_Vector
from pycket.values_struct import W_Struct
from pycket.prims.general import srcloc
from pycket.hash import simple as hash_simple
from pycket.hash.equal import W_EqualHashTable
from pycket.prims.numeric import float_bytes_to_real
from pycket.prims.string import _str2num
from rpython.rlib.rbigint import rbigint
from pycket.prims.input_output import build_path, bytes_to_path_element
from pycket.ast_vs_sexp import to_rpython_list
from pycket.racket_entry import get_primitive
typ, pos = self.read_byte_no_eof(fasl_string, pos)
if typ == FASL_GRAPH_DEF_TYPE:
position, pos = self.read_fasl_integer(fasl_string, pos)
val, pos = self.fasl_to_sexp_recursive(fasl_string, pos)
if position >= self.GLOBAL_SHARED_COUNT:
raise Exception("fasl: bad graph index")
self.SHARED[position] = val
return val, pos
elif typ == FASL_GRAPH_REF_TYPE:
position, pos = self.read_fasl_integer(fasl_string, pos)
if position >= self.GLOBAL_SHARED_COUNT:
raise Exception("fasl: bad graph index")
return self.SHARED[position], pos
elif typ == FASL_FALSE_TYPE:
return v.w_false, pos
elif typ == FASL_TRUE_TYPE:
return v.w_true, pos
elif typ == FASL_NULL_TYPE:
return v.w_null, pos
elif typ == FASL_VOID_TYPE:
return v.w_void, pos
elif typ == FASL_EOF_TYPE:
return v.eof_object, pos
elif typ == FASL_INTEGER_TYPE:
num, pos = self.read_fasl_integer(fasl_string, pos)
if isinstance(num, rbigint):
return v.W_Bignum(num), pos
return v.W_Fixnum(num), pos
elif typ == FASL_FLONUM_TYPE:
num_str, pos = self.read_bytes_exactly(fasl_string, pos, 8)
return float_bytes_to_real(list(num_str), v.w_false), pos
elif typ == FASL_SINGLE_FLONUM_TYPE:
num_str, pos = self.read_bytes_exactly(fasl_string, pos, 4)
real = float_bytes_to_real(list(num_str), v.w_false)
return real.arith_exact_inexact(), pos
elif typ == FASL_EXTFLONUM_TYPE:
bstr_len, pos = self.read_fasl_integer(fasl_string, pos)
num_str, pos = self.read_bytes_exactly(fasl_string, pos, bstr_len)
return _str2num(W_String.fromstr_utf8(num_str).as_str_utf8(),
10), pos
elif typ == FASL_RATIONAL_TYPE:
num, pos = self.fasl_to_sexp_recursive(fasl_string, pos)
den, pos = self.fasl_to_sexp_recursive(fasl_string, pos)
return v.W_Rational.make(num, den), pos
elif typ == FASL_COMPLEX_TYPE:
re, pos = self.fasl_to_sexp_recursive(fasl_string, pos)
im, pos = self.fasl_to_sexp_recursive(fasl_string, pos)
return v.W_Complex.from_real_pair(re, im), pos
elif typ == FASL_CHAR_TYPE:
_chr, pos = self.read_fasl_integer(fasl_string, pos)
return v.W_Character(unichr(_chr)), pos
elif typ == FASL_SYMBOL_TYPE:
sym_str, pos = self.read_fasl_string(fasl_string, pos)
return v.W_Symbol.make(sym_str), pos
elif typ == FASL_UNREADABLE_SYMBOL_TYPE:
sym_str, pos = self.read_fasl_string(fasl_string, pos)
return v.W_Symbol.make_unreadable(sym_str), pos
elif typ == FASL_UNINTERNED_SYMBOL_TYPE:
sym_str, pos = self.read_fasl_string(fasl_string, pos)
return v.W_Symbol(sym_str), pos
elif typ == FASL_KEYWORD_TYPE:
key_str, pos = self.read_fasl_string(fasl_string, pos)
return v.W_Keyword.make(key_str), pos
elif typ == FASL_STRING_TYPE:
str_str, pos = self.read_fasl_string(fasl_string, pos)
return W_String.make(str_str), pos
elif typ == FASL_IMMUTABLE_STRING_TYPE:
str_str, pos = self.read_fasl_string(fasl_string, pos)
return W_String.make(str_str).make_immutable(), pos
elif typ == FASL_BYTES_TYPE:
byts, pos = self.read_fasl_bytes(fasl_string, pos)
return v.W_Bytes.from_string(byts, immutable=False), pos
elif typ == FASL_IMMUTABLE_BYTES_TYPE:
byts, pos = self.read_fasl_bytes(fasl_string, pos)
return v.W_Bytes.from_string(byts), pos
elif typ == FASL_PATH_TYPE:
byts, pos = self.read_fasl_bytes(fasl_string, pos)
return v.W_Path(byts), pos
elif typ == FASL_RELATIVE_PATH_TYPE:
wrt_dir = self.current_relative_dir
p_w_lst, pos = self.fasl_to_sexp_recursive(fasl_string, pos)
p_r_lst, _ = to_rpython_list(p_w_lst)
rel_elems = [
bytes_to_path_element(p) if isinstance(p, v.W_Bytes) else p
for p in p_r_lst
]
if wrt_dir:
return build_path([wrt_dir] + rel_elems), pos
elif rel_elems == []:
return build_path([v.W_Symbol.make("same")]), pos
else:
return build_path(rel_elems), pos
elif typ == FASL_PREGEXP_TYPE:
str_str, pos = self.read_fasl_string(fasl_string, pos)
reg_str = W_String.make(str_str)
pregexp = get_primitive('pregexp')
pregexp_obj = pregexp.call_interpret([reg_str])
return pregexp_obj, pos
elif typ == FASL_REGEXP_TYPE:
str_str, pos = self.read_fasl_string(fasl_string, pos)
reg_str = W_String.make(str_str)
regexp = get_primitive('regexp')
regexp_obj = regexp.call_interpret([reg_str])
return regexp_obj, pos
elif typ == FASL_BYTE_PREGEXP:
str_str, pos = self.read_fasl_string(fasl_string, pos)
reg_bytes = v.W_Bytes.from_string(str_str)
byte_pregexp = get_primitive('byte-pregexp')
byte_pregexp_obj = byte_pregexp.call_interpret([reg_bytes])
return byte_pregexp_obj, pos
elif typ == FASL_BYTE_REGEXP_TYPE:
str_str, pos = self.read_fasl_string(fasl_string, pos)
reg_bytes = v.W_Bytes.from_string(str_str)
byte_regexp = get_primitive('byte-regexp')
byte_regexp_obj = byte_regexp.call_interpret([reg_bytes])
return byte_regexp_obj, pos
elif typ == FASL_LIST_TYPE:
list_len, pos = self.read_fasl_integer(fasl_string, pos)
lst, pos = self.read_multi_into_rpython_list(
fasl_string, pos, list_len)
return v.to_list(lst), pos
elif typ == FASL_PAIR_TYPE:
car, pos = self.fasl_to_sexp_recursive(fasl_string, pos)
cdr, pos = self.fasl_to_sexp_recursive(fasl_string, pos)
return v.W_Cons.make(car, cdr), pos
elif typ == FASL_LIST_STAR_TYPE:
list_len, pos = self.read_fasl_integer(fasl_string, pos)
# list_len is the length of the proper part
lst, pos = self.read_multi_into_rpython_list(
fasl_string, pos, list_len)
# read the last element
return_list, pos = self.fasl_to_sexp_recursive(fasl_string, pos)
for i in range(list_len - 1, -1, -1):
return_list = v.W_Cons.make(lst[i], return_list)
return return_list, pos
elif typ == FASL_VECTOR_TYPE or typ == FASL_IMMUTABLE_VECTOR_TYPE:
vec_len, pos = self.read_fasl_integer(fasl_string, pos)
storage, pos = self.read_multi_into_rpython_list(
fasl_string, pos, vec_len)
if typ == FASL_IMMUTABLE_VECTOR_TYPE:
return W_Vector.fromelements(storage, immutable=True), pos
return W_Vector.fromelements(storage), pos
elif typ == FASL_BOX_TYPE:
element, pos = self.fasl_to_sexp_recursive(fasl_string, pos)
return v.W_MBox(element), pos
elif typ == FASL_IMMUTABLE_BOX_TYPE:
element, pos = self.fasl_to_sexp_recursive(fasl_string, pos)
return v.W_IBox(element), pos
elif typ == FASL_PREFAB_TYPE:
key, pos = self.fasl_to_sexp_recursive(fasl_string, pos)
length, pos = self.read_fasl_integer(fasl_string, pos)
vals, pos = self.read_multi_into_rpython_list(
fasl_string, pos, length)
return W_Struct.make_prefab(key, vals), pos
elif typ == FASL_HASH_TYPE:
variant, pos = self.read_byte_no_eof(fasl_string, pos)
length, pos = self.read_fasl_integer(fasl_string, pos)
keys, vals, pos = self.read_multi_double_into_rpython_list(
fasl_string, pos, length)
if variant == FASL_HASH_EQ_VARIANT:
return hash_simple.make_simple_mutable_table(
hash_simple.W_EqMutableHashTable, keys, vals), pos
elif variant == FASL_HASH_EQV_VARIANT:
return hash_simple.make_simple_mutable_table(
hash_simple.W_EqvMutableHashTable, keys, vals), pos
else: # variant == FASL_HASH_EQUAL_VARIANT:
return W_EqualHashTable(keys, vals, immutable=False), pos
elif typ == FASL_IMMUTABLE_HASH_TYPE:
variant, pos = self.read_byte_no_eof(fasl_string, pos)
length, pos = self.read_fasl_integer(fasl_string, pos)
keys, vals, pos = self.read_multi_double_into_rpython_list(
fasl_string, pos, length)
if variant == FASL_HASH_EQ_VARIANT:
return hash_simple.make_simple_immutable_table(
hash_simple.W_EqImmutableHashTable, keys, vals), pos
elif variant == FASL_HASH_EQV_VARIANT:
return hash_simple.make_simple_immutable_table(
hash_simple.W_EqvImmutableHashTable, keys, vals), pos
else: # variant == FASL_HASH_EQUAL_VARIANT:
return W_EqualHashTable(keys, vals, immutable=True), pos
elif typ == FASL_SRCLOC:
# difficult to create an instance of srcloc struct so defer that to the runtime
source, pos = self.fasl_to_sexp_recursive(fasl_string, pos)
line, pos = self.fasl_to_sexp_recursive(fasl_string, pos)
column, pos = self.fasl_to_sexp_recursive(fasl_string, pos)
position, pos = self.fasl_to_sexp_recursive(fasl_string, pos)
span, pos = self.fasl_to_sexp_recursive(fasl_string, pos)
return W_Struct.make([source, line, column, position, span],
srcloc), pos
else:
if typ >= FASL_SMALL_INTEGER_START:
return v.W_Fixnum((typ - FASL_SMALL_INTEGER_START) +
FASL_LOWEST_SMALL_INTEGER), pos
else:
raise Exception("unrecognized fasl tag : %s" % typ)
from pycket.test.utils import *
from pycket.config import get_testing_config
#
# basic runners
#
# This is where all the work happens
delete_temp_files = True
##############################
## NEW Pycket test helpers
##############################
instantiate_linklet = get_primitive("instantiate-linklet")
# This is where all the work happens
if pytest.config.load_expander:
# get the expander
print("Loading and initializing the expander")
initiate_boot_sequence(None, [], False)
# load the '#%kernel
print("(namespace-require '#%%kernel)")
namespace_require_kernel(None)
def run_sexp(body_sexp_str,
v=None,
just_return=False,
def read_linklet_bundle_hash(in_port, env, cont):
from pycket.racket_entry import get_primitive
fasl_to_s_exp = get_primitive("fasl->s-exp")
with PerfRegionCPS("fasl->s-exp"):
return fasl_to_s_exp.call([in_port, values.w_true], env,
read_linklet_cont(env, cont))
def load_linklet(json_file_name, set_version=False, generate_zo=False):
from pycket.expand import readfile_rpython, getkey, JsonLoader
from pycket.util import console_log
from pycket.env import w_version
import os
if generate_zo:
try:
os.remove("expander.zo")
except OSError:
pass
if set_version and os.path.exists("expander.zo"):
console_log("Loading the expander linklet from expander.zo")
# We're loading the expander, so try fasl->sexp
from pycket.racket_entry import get_primitive
from pycket.prims.input_output import open_input_file
with PerfRegion("expander-linklet"):
fasl_to_s_exp = get_primitive("fasl->s-exp")
in_file = open_input_file.w_prim.call_interpret(
[W_Path("expander.zo")])
try:
expander_zo_sexp = fasl_to_s_exp.call_interpret(
[in_file, w_true])
version = expander_zo_sexp.car()
expander_sexp = expander_zo_sexp.cdr()
expander_linkl = deserialize_loop(expander_sexp)
w_version.set_version(version.tostring())
console_log("Setting the version to %s" %
version.tostring())
return expander_linkl, None
except (SchemeException, OSError):
console_log("Couldn't read from expander.zo")
""" Expands and loads a linklet from a JSON file"""
with PerfRegion("json-load"):
data = readfile_rpython(json_file_name)
json = pycket_json.loads(data)
console_log("Finished reading JSON from %s" % json_file_name, 2)
assert json.is_object
json_python_dict = json.value_object()
assert "linklet" in json_python_dict
linklet_dict = getkey(json_python_dict, "linklet", type='o')
assert "exports" in linklet_dict and "body" in linklet_dict
# list of JsonObject
exports_list = getkey(linklet_dict, "exports", type='a')
if set_version:
from pycket.util import console_log
conf = getkey(linklet_dict, "config", type='o')
ver = conf['version'].value_string()
console_log("Setting the version to %s" % ver)
w_version.set_version(ver)
exports = {}
for exp in exports_list:
if exp.is_array:
arr = exp.value_array()
internal_str = arr[0].value_object()['quote'].value_object(
)['toplevel'].value_string()
internal_name = W_Symbol.make(internal_str)
external_name = W_Symbol.make(
arr[1].value_object()['quote'].value_object()
['toplevel'].value_string())
w_internal_name = Gensym.gensym(internal_str)
exports[internal_name] = Export(w_internal_name, external_name)
else:
exp_str = exp.value_object()['quote'].value_object(
)['toplevel'].value_string()
exp_sym = W_Symbol.make(exp_str)
w_exp_sym = Gensym.gensym(exp_str)
exports[exp_sym] = Export(w_exp_sym, exp_sym)
imports_list = getkey(linklet_dict, "importss", type='a', default=[])
importss = [None] * len(imports_list) # list of dict
if "importss" in linklet_dict:
for index, imports in enumerate(imports_list):
arr = imports.value_array()
# bootstrap linklets have no imports at all
# this is only for debugging purposes
instance_imports = []
for id_str in arr:
imp_str = id_str.value_object()['quote'].value_object(
)['toplevel'].value_string()
imp_sym = W_Symbol.make(imp_str)
w_imp_sym = Gensym.gensym(imp_str)
instance_imports.append(
Import(W_Fixnum(index), imp_sym, w_imp_sym, w_imp_sym))
importss[index] = instance_imports
console_log("Converting linklet forms to AST ...", 2)
loader = JsonLoader()
with PerfRegion("json-to-ast"):
all_forms = []
for body_form in getkey(linklet_dict, "body", type='a'):
form_2 = loader.to_ast(body_form)
form_1 = Context.normalize_term(form_2)
# if form_1.tostring() != form_2.tostring():
# import pdb;pdb.set_trace()
form = assign_convert(form_1)
all_forms.append(form)
# for each exported defined id, we need to add a
# variable-set! for the exported var with the defined id
for exp_sym, exp_obj in exports.iteritems():
rator = ModuleVar(var_set_sym, "#%kernel", var_set_sym, None)
exp_var = LinkletStaticVar(exp_obj.int_id)
top_var = ToplevelVar(exp_sym)
mode = Quote(values.w_false) # FIXME: possible optimization
rands = [exp_var, top_var, mode]
all_forms.append(App.make(rator, rands))
linkl = W_Linklet(W_Symbol.make(json_file_name), importss, exports,
all_forms)
console_log("Finished converting linklet forms to AST ...", 2)
config = {}
config_obj = getkey(linklet_dict, "config", type='o')
if config_obj is not None:
for k, v in config_obj.iteritems():
config[k] = v.value_string()
if set_version and not os.path.exists("expander.zo"):
console_log("Serializing the expander linklet into expander.zo")
from pycket.racket_entry import get_primitive
from pycket.prims.input_output import open_output_file
s_exp_to_fasl = get_primitive("s-exp->fasl")
expander_sexp = ast_to_sexp(linkl)
version = W_Symbol.make(w_version.get_version())
expander_zo_sexp = W_Cons.make(version, expander_sexp)
try:
out_file = open_output_file.w_prim.call_interpret(
[W_Path("expander.zo")])
s_exp_to_fasl.call_interpret(
[expander_zo_sexp, out_file, values.w_false])
except (SchemeException, OSError):
console_log("Couldn't write the expander.zo")
return linkl, config
from pycket.test.utils import *
from pycket.config import get_testing_config
#
# basic runners
#
# This is where all the work happens
delete_temp_files = True
##############################
## NEW Pycket test helpers
##############################
instantiate_linklet = get_primitive("instantiate-linklet")
# This is where all the work happens
try:
if not pytest.config.new_pycket:
w_global_config.set_linklet_mode_off()
if pytest.config.load_expander:
w_global_config.set_config_val('expander_loaded', 1)
# get the expander
print("Loading and initializing the expander")
initiate_boot_sequence([], False)
# load the '#%kernel
print("(namespace-require '#%%kernel)")
namespace_require_kernel()