def char_foldcase(w_char): char = ord(w_char.value) folded = unicodedb.casefold_lookup(char) if folded is None: lower = unicodedb.tolower(char) return values.W_Character(unichr(lower)) # XXX: What to do if the case folded character consists of more than one code point? return values.W_Character(unichr(folded[0]))
def string_to_list(s): data = s.as_unicode() acc = values.w_null for i in range(len(data) - 1, -1, -1): char = data[i] acc = values.W_Cons.make(values.W_Character(char), acc) return acc
def do_read_one(w_port, as_bytes, peek, env, cont): from pycket.interpreter import return_value if peek: c = w_port.peek() else: c = w_port.read(1) if len(c) == 0: return return_value(values.eof_object, env, cont) i = ord(c[0]) if as_bytes: return return_value(values.W_Fixnum(i), env, cont) else: # hmpf, poking around in internals needed = runicode.utf8_code_length[i] if peek: old = w_port.tell() c = w_port.read(needed) w_port.seek(old) elif needed > 1: c += w_port.read(needed - 1) c = c.decode("utf-8") assert len(c) == 1 return return_value(values.W_Character(c[0]), env, cont)
def do_read_one(w_port, as_bytes, peek, env, cont): from pycket.interpreter import return_value if w_port is None: w_port = current_in_param.get(cont) assert isinstance(w_port, values.W_InputPort) if peek: c = w_port.peek() else: c = w_port.read(1) if len(c) == 0: return return_value(values.eof_object, env, cont) i = ord(c[0]) if as_bytes: return return_value(values.W_Fixnum(i), env, cont) else: # hmpf, poking around in internals needed = runicode.utf8_code_length[i] c += w_port.read(needed - 1) c = c.decode("utf-8") assert len(c) == 1 return return_value(values.W_Character(c[0]), env, cont)
def fasl_to_sexp_recursive(self, fasl_string, pos): from pycket import values as v from pycket.values_string import W_String from pycket.values_regex import W_Regexp, W_PRegexp, W_ByteRegexp, W_BytePRegexp from pycket.vector import W_Vector from pycket.values_struct import W_Struct from pycket.prims.general import srcloc from pycket.hash import simple as hash_simple from pycket.hash.equal import W_EqualHashTable from pycket.prims.numeric import float_bytes_to_real from pycket.prims.string import _str2num from rpython.rlib.rbigint import rbigint from pycket.prims.input_output import build_path, bytes_to_path_element from pycket.ast_vs_sexp import to_rpython_list from pycket.racket_entry import get_primitive typ, pos = self.read_byte_no_eof(fasl_string, pos) if typ == FASL_GRAPH_DEF_TYPE: position, pos = self.read_fasl_integer(fasl_string, pos) val, pos = self.fasl_to_sexp_recursive(fasl_string, pos) if position >= self.GLOBAL_SHARED_COUNT: raise Exception("fasl: bad graph index") self.SHARED[position] = val return val, pos elif typ == FASL_GRAPH_REF_TYPE: position, pos = self.read_fasl_integer(fasl_string, pos) if position >= self.GLOBAL_SHARED_COUNT: raise Exception("fasl: bad graph index") return self.SHARED[position], pos elif typ == FASL_FALSE_TYPE: return v.w_false, pos elif typ == FASL_TRUE_TYPE: return v.w_true, pos elif typ == FASL_NULL_TYPE: return v.w_null, pos elif typ == FASL_VOID_TYPE: return v.w_void, pos elif typ == FASL_EOF_TYPE: return v.eof_object, pos elif typ == FASL_INTEGER_TYPE: num, pos = self.read_fasl_integer(fasl_string, pos) if isinstance(num, rbigint): return v.W_Bignum(num), pos return v.W_Fixnum(num), pos elif typ == FASL_FLONUM_TYPE: num_str, pos = self.read_bytes_exactly(fasl_string, pos, 8) return float_bytes_to_real(list(num_str), v.w_false), pos elif typ == FASL_SINGLE_FLONUM_TYPE: num_str, pos = self.read_bytes_exactly(fasl_string, pos, 4) real = float_bytes_to_real(list(num_str), v.w_false) return real.arith_exact_inexact(), pos elif typ == FASL_EXTFLONUM_TYPE: bstr_len, pos = self.read_fasl_integer(fasl_string, pos) num_str, pos = self.read_bytes_exactly(fasl_string, pos, bstr_len) return _str2num(W_String.fromstr_utf8(num_str).as_str_utf8(), 10), pos elif typ == FASL_RATIONAL_TYPE: num, pos = self.fasl_to_sexp_recursive(fasl_string, pos) den, pos = self.fasl_to_sexp_recursive(fasl_string, pos) return v.W_Rational.make(num, den), pos elif typ == FASL_COMPLEX_TYPE: re, pos = self.fasl_to_sexp_recursive(fasl_string, pos) im, pos = self.fasl_to_sexp_recursive(fasl_string, pos) return v.W_Complex.from_real_pair(re, im), pos elif typ == FASL_CHAR_TYPE: _chr, pos = self.read_fasl_integer(fasl_string, pos) return v.W_Character(unichr(_chr)), pos elif typ == FASL_SYMBOL_TYPE: sym_str, pos = self.read_fasl_string(fasl_string, pos) return v.W_Symbol.make(sym_str), pos elif typ == FASL_UNREADABLE_SYMBOL_TYPE: sym_str, pos = self.read_fasl_string(fasl_string, pos) return v.W_Symbol.make_unreadable(sym_str), pos elif typ == FASL_UNINTERNED_SYMBOL_TYPE: sym_str, pos = self.read_fasl_string(fasl_string, pos) return v.W_Symbol(sym_str), pos elif typ == FASL_KEYWORD_TYPE: key_str, pos = self.read_fasl_string(fasl_string, pos) return v.W_Keyword.make(key_str), pos elif typ == FASL_STRING_TYPE: str_str, pos = self.read_fasl_string(fasl_string, pos) return W_String.make(str_str), pos elif typ == FASL_IMMUTABLE_STRING_TYPE: str_str, pos = self.read_fasl_string(fasl_string, pos) return W_String.make(str_str).make_immutable(), pos elif typ == FASL_BYTES_TYPE: byts, pos = self.read_fasl_bytes(fasl_string, pos) return v.W_Bytes.from_string(byts, immutable=False), pos elif typ == FASL_IMMUTABLE_BYTES_TYPE: byts, pos = self.read_fasl_bytes(fasl_string, pos) return v.W_Bytes.from_string(byts), pos elif typ == FASL_PATH_TYPE: byts, pos = self.read_fasl_bytes(fasl_string, pos) return v.W_Path(byts), pos elif typ == FASL_RELATIVE_PATH_TYPE: wrt_dir = self.current_relative_dir p_w_lst, pos = self.fasl_to_sexp_recursive(fasl_string, pos) p_r_lst, _ = to_rpython_list(p_w_lst) rel_elems = [ bytes_to_path_element(p) if isinstance(p, v.W_Bytes) else p for p in p_r_lst ] if wrt_dir: return build_path([wrt_dir] + rel_elems), pos elif rel_elems == []: return build_path([v.W_Symbol.make("same")]), pos else: return build_path(rel_elems), pos elif typ == FASL_PREGEXP_TYPE: str_str, pos = self.read_fasl_string(fasl_string, pos) reg_str = W_String.make(str_str) pregexp = get_primitive('pregexp') pregexp_obj = pregexp.call_interpret([reg_str]) return pregexp_obj, pos elif typ == FASL_REGEXP_TYPE: str_str, pos = self.read_fasl_string(fasl_string, pos) reg_str = W_String.make(str_str) regexp = get_primitive('regexp') regexp_obj = regexp.call_interpret([reg_str]) return regexp_obj, pos elif typ == FASL_BYTE_PREGEXP: str_str, pos = self.read_fasl_string(fasl_string, pos) reg_bytes = v.W_Bytes.from_string(str_str) byte_pregexp = get_primitive('byte-pregexp') byte_pregexp_obj = byte_pregexp.call_interpret([reg_bytes]) return byte_pregexp_obj, pos elif typ == FASL_BYTE_REGEXP_TYPE: str_str, pos = self.read_fasl_string(fasl_string, pos) reg_bytes = v.W_Bytes.from_string(str_str) byte_regexp = get_primitive('byte-regexp') byte_regexp_obj = byte_regexp.call_interpret([reg_bytes]) return byte_regexp_obj, pos elif typ == FASL_LIST_TYPE: list_len, pos = self.read_fasl_integer(fasl_string, pos) lst, pos = self.read_multi_into_rpython_list( fasl_string, pos, list_len) return v.to_list(lst), pos elif typ == FASL_PAIR_TYPE: car, pos = self.fasl_to_sexp_recursive(fasl_string, pos) cdr, pos = self.fasl_to_sexp_recursive(fasl_string, pos) return v.W_Cons.make(car, cdr), pos elif typ == FASL_LIST_STAR_TYPE: list_len, pos = self.read_fasl_integer(fasl_string, pos) # list_len is the length of the proper part lst, pos = self.read_multi_into_rpython_list( fasl_string, pos, list_len) # read the last element return_list, pos = self.fasl_to_sexp_recursive(fasl_string, pos) for i in range(list_len - 1, -1, -1): return_list = v.W_Cons.make(lst[i], return_list) return return_list, pos elif typ == FASL_VECTOR_TYPE or typ == FASL_IMMUTABLE_VECTOR_TYPE: vec_len, pos = self.read_fasl_integer(fasl_string, pos) storage, pos = self.read_multi_into_rpython_list( fasl_string, pos, vec_len) if typ == FASL_IMMUTABLE_VECTOR_TYPE: return W_Vector.fromelements(storage, immutable=True), pos return W_Vector.fromelements(storage), pos elif typ == FASL_BOX_TYPE: element, pos = self.fasl_to_sexp_recursive(fasl_string, pos) return v.W_MBox(element), pos elif typ == FASL_IMMUTABLE_BOX_TYPE: element, pos = self.fasl_to_sexp_recursive(fasl_string, pos) return v.W_IBox(element), pos elif typ == FASL_PREFAB_TYPE: key, pos = self.fasl_to_sexp_recursive(fasl_string, pos) length, pos = self.read_fasl_integer(fasl_string, pos) vals, pos = self.read_multi_into_rpython_list( fasl_string, pos, length) return W_Struct.make_prefab(key, vals), pos elif typ == FASL_HASH_TYPE: variant, pos = self.read_byte_no_eof(fasl_string, pos) length, pos = self.read_fasl_integer(fasl_string, pos) keys, vals, pos = self.read_multi_double_into_rpython_list( fasl_string, pos, length) if variant == FASL_HASH_EQ_VARIANT: return hash_simple.make_simple_mutable_table( hash_simple.W_EqMutableHashTable, keys, vals), pos elif variant == FASL_HASH_EQV_VARIANT: return hash_simple.make_simple_mutable_table( hash_simple.W_EqvMutableHashTable, keys, vals), pos else: # variant == FASL_HASH_EQUAL_VARIANT: return W_EqualHashTable(keys, vals, immutable=False), pos elif typ == FASL_IMMUTABLE_HASH_TYPE: variant, pos = self.read_byte_no_eof(fasl_string, pos) length, pos = self.read_fasl_integer(fasl_string, pos) keys, vals, pos = self.read_multi_double_into_rpython_list( fasl_string, pos, length) if variant == FASL_HASH_EQ_VARIANT: return hash_simple.make_simple_immutable_table( hash_simple.W_EqImmutableHashTable, keys, vals), pos elif variant == FASL_HASH_EQV_VARIANT: return hash_simple.make_simple_immutable_table( hash_simple.W_EqvImmutableHashTable, keys, vals), pos else: # variant == FASL_HASH_EQUAL_VARIANT: return W_EqualHashTable(keys, vals, immutable=True), pos elif typ == FASL_SRCLOC: # difficult to create an instance of srcloc struct so defer that to the runtime source, pos = self.fasl_to_sexp_recursive(fasl_string, pos) line, pos = self.fasl_to_sexp_recursive(fasl_string, pos) column, pos = self.fasl_to_sexp_recursive(fasl_string, pos) position, pos = self.fasl_to_sexp_recursive(fasl_string, pos) span, pos = self.fasl_to_sexp_recursive(fasl_string, pos) return W_Struct.make([source, line, column, position, span], srcloc), pos else: if typ >= FASL_SMALL_INTEGER_START: return v.W_Fixnum((typ - FASL_SMALL_INTEGER_START) + FASL_LOWEST_SMALL_INTEGER), pos else: raise Exception("unrecognized fasl tag : %s" % typ)
def char_upcase(v): return values.W_Character(unichr(unicodedb.toupper(ord(v.value))))
def char_downcase(v): return values.W_Character(unichr(unicodedb.tolower(ord(v.value))))
def integer_to_char(v): return values.W_Character(unichr(v.value))
def string_ref(s, n): n = n.value if not 0 <= n < s.length(): raise SchemeException("string-ref: index out of bounds") return values.W_Character(s.getitem(n))
def read_token(f): while True: c = f.read(1) # FIXME: unicode if not c: return values.eof_object if c == ";": f.readline() continue if c in [" ", "\n", "\t"]: continue if c in ["(", "[", "{"]: return LParenToken(c) if c in [")", "]", "}"]: return RParenToken(c) if c == "\"": v = read_string(f) return v if c == ".": p = f.peek() if p in [" ", "\n", "\t"]: return dot_token return read_number_or_id(f, c) if c == "'": return quote_token if c == "`": return quasiquote_token if c == ",": p = f.peek() if p == "@": p = f.read(1) return unquote_splicing_token else: return unquote_token if idchar(c): return read_number_or_id(f, c) if c == "#": c2 = f.read(1) if c2 == "'": return quote_syntax_token if c2 == "`": return quasisyntax_token if c2 == ",": p = f.peek() if p == "@": p = f.read(1) return unsyntax_splicing_token return unsyntax_token if c2 == "t": return values.w_true if c2 == "f": return values.w_false if c2 in ["(", "[", "{"]: return LParenToken("#" + c2) if c2 == "\\": s = f.read(1) if not s: raise SchemeException("unexpected end of file") c = ord(s[0]) # XXX deal with unicode return values.W_Character(unichr(c)) raise SchemeException("bad token in read: %s" % c2) raise SchemeException("bad token in read: %s" % c)
def string_to_list(s): return values.to_list([values.W_Character(i) for i in s.as_unicode()])