def testAnotherQuasiquote(self):
self.pe("(define x 42)")
self.assertEquals(42, self.pe("x"))
self.assertEquals(
pair.list(Symbol("x"),
pair.list(Symbol("quote"), pair.list(Symbol("x"), 42))),
self.pe("`(x '(x ,x))"))
def ReplaceInTree(self, mask, value):
"""Replace a symbol with the symbol of the value inside the tree hosted by this node"""
has_replaced = False
print("FOUND:", self.symbol, mask, value)
if (self.symbol.mask == mask):
print("REPLACING:", self.symbol, mask, value, self.sign)
if (value not in ["T", "F"]):
sym = Symbol(value)
else:
if (not self.sign):
if (value == "T"):
sym = Symbol("F")
elif (value == "F"):
sym = Symbol("T")
self.sign = True
else:
sym = Symbol(value)
has_replaced = True
self.symbol = sym
print("CHECK REP:", self.symbol, mask, value)
if (self.left != None):
has_replaced = has_replaced | self.left.ReplaceInTree(mask, value)
if (self.right != None):
has_replaced = has_replaced | self.right.ReplaceInTree(mask, value)
print("has replaced", has_replaced)
return has_replaced
def test_equality():
"""Two symbols with the same attributes are identical."""
s1 = Symbol(1, 4)
s2 = Symbol(2, 2)
s3 = Symbol(2, 2)
assert s1 != s2
assert s2 == s3
def __init__(self, lineno, byref=False):
''' Initializes the argument data. Byref must be set
to True if this Argument is passed by reference.
'''
Symbol.__init__(self, None, 'ARGUMENT')
self.lineno = lineno
self.byref = byref
def __init__(self, symbol, _type):
Symbol.__init__(self, symbol._mangled, 'PARAMDECL')
self.entry = symbol
self.__size = TYPE_SIZES[self._type]
self.__size = self.__size + (self.__size % 2) # Make it even-sized (Float and Byte)
self.byref = OPTIONS.byref.value # By default all params By value (false)
self.offset = None # Set by PARAMLIST, contains positive offset from top of the stack
def __init__(self, value, lineno, offset=None):
global SYMBOL_TABLE
Symbol.__init__(self, value, 'ID')
self.id = value
self.filename = gl.FILENAME # In which file was first used
self.lineno = lineno # In which line was first used
self._class = None
self._mangled = '_%s' % value # This value will be overriden later
self.t = self._mangled
self.declared = False # if declared (DIM var AS <type>) this must be True
self._type = None # Unknown type
self.offset = offset # For local variables, offset from top of the stack
self.default_value = None # If defined, variable will be initialized with this value (Arrays = List of Bytes)
self.scope = 'global' # One of 'global', 'parameter', 'local'
self.byref = False # By default, it's a global var
self.default_value = None # For variables, this is the default initalized value
self.__kind = None # If not None, it should be one of 'function' or 'sub'
self.addr = None # If not None, the address of this symbol (string)
self.alias = None # If not None, this var is an alias of another
self.aliased_by = [] # Which variables are an alias of this one
self.referenced_by = [
] # Which objects do use this one (e.g. sentences using this variable)
self.references = [
] # Objects referenced by this one (e.g. variables used in this sentence)
self.accessed = False # Where this object has been accessed (if false it might be not compiled, since it is useless)
self.caseins = OPTIONS.case_insensitive.value # Whether this ID is case insensitive or not
def __init__(self, value, lineno, offset=None):
global SYMBOL_TABLE
Symbol.__init__(self, value, "ID")
self.id = value
self.filename = gl.FILENAME # In which file was first used
self.lineno = lineno # In which line was first used
self._class = None
self._mangled = "_%s" % value # This value will be overriden later
self.t = self._mangled
self.declared = False # if declared (DIM var AS <type>) this must be True
self._type = None # Unknown type
self.offset = offset # For local variables, offset from top of the stack
self.default_value = None # If defined, variable will be initialized with this value (Arrays = List of Bytes)
self.scope = "global" # One of 'global', 'parameter', 'local'
self.byref = False # By default, it's a global var
self.default_value = None # For variables, this is the default initalized value
self.__kind = None # If not None, it should be one of 'function' or 'sub'
self.addr = None # If not None, the address of this symbol (string)
self.alias = None # If not None, this var is an alias of another
self.aliased_by = [] # Which variables are an alias of this one
self.referenced_by = [] # Which objects do use this one (e.g. sentences using this variable)
self.references = [] # Objects referenced by this one (e.g. variables used in this sentence)
self.accessed = (
False
) # Where this object has been accessed (if false it might be not compiled, since it is useless)
self.caseins = OPTIONS.case_insensitive.value # Whether this ID is case insensitive or not
def __init__(self, lineno, byref = False):
''' Initializes the argument data. Byref must be set
to True if this Argument is passed by reference.
'''
Symbol.__init__(self, None, 'ARGUMENT')
self.lineno = lineno
self.byref = byref
def add_symbol(self):
symbol = Symbol()
size = len(self.symbols)
if size > 0:
last = self.symbols[-1]
# if the previous tag has started a script or a style,
# do not include the contents of those tags as they
# are large and unnecessary
# NOTE: this does not skip inline tags
if last.type == SymbolType.SCRIPT_START or last.type == SymbolType.STYLE_START:
symbol = self.tokenizer.get_next_symbol()
while (symbol.type == SymbolType.SCRIPT_START
or symbol.type == SymbolType.STYLE_START
or symbol.type == SymbolType.SCRIPT_END
or symbol.type == SymbolType.STYLE_END
or symbol.type == SymbolType.PLAIN_TEXT):
symbol = self.tokenizer.get_next_symbol()
else:
symbol = self.tokenizer.get_next_symbol()
else:
symbol = self.tokenizer.get_next_symbol()
# do not add symbols until a start of the document has been reached
# the start it represented as DOCTYPE tag
if symbol.type != SymbolType.DOCTYPE:
symbol = Symbol.empty()
if not symbol.is_empty():
# might be empty when the end of the HTML has been reached
self.symbols.append(symbol)
def parse_lexical_symbols(self):
"""Given that the token generator is at the beginning of the
lexical symbol specifications, read a series of lexical symbol
specifications, doing name and basic type analysis on the fly."""
stack = []
self.tokenizer.next().must_be('{')
for token in self.tokenizer:
stack += [ token ]
if token.text == ".":
stack[0].assert_symbol_name()
stack[1].must_be(':')
stack[2].must_match('^\\$', "regular expression")
## Name analysis
if stack[0].text in self.GlobalSymbolDict:
originalDef = self.GlobalSymbolDict[stack[0].text].defining_token
raise Exception("Symbol %s redefined at %d,%d. Originally at %d,%d" % (stack[0].text, stack[0].line, stack[0].col, \
originalDef.line, originalDef.col))
s = Symbol(stack[0])
s.is_gla = True
s.regex = Set(stack[2].text[1:])
self.GlobalSymbolDict[stack[0].text] = s
stack = []
elif token.text == "{":
raise Exception("Unexpected %s" % token)
elif token.text == "}":
if len(stack) > 1: raise Exception("Unfinished lexical specification beginning with %s" % stack[0])
return
else: pass
def SimpleDistribOR(self, leaf):
if (leaf.symbol.code == "OP_AND"):
l = leaf.left
r = leaf.right
if (l.symbol.code == "OP_OR" and r.symbol.code == "IDENTIFIER"):
op = l
ident = r
elif (r.symbol.code == "OP_OR" and l.symbol.code == "IDENTIFIER"):
op = r
ident = l
else:
return False
leaf.symbol = Symbol("v")
opL = op.left
opR = op.right
ident2, identt = ident.DuplicateTree()
self.tree += identt
nl = Leaf(Symbol("^"), True, leaf, opL, ident)
nr = Leaf(Symbol("^"), True, leaf, opR, ident2)
leaf.left = nl
leaf.right = nr
return True
def __init__(self, coordinator, hparams, shuffle):
super(DataFeeder, self).__init__()
self._coord = coordinator
self._hparams = hparams
hp = self._hparams
self._offset = 0
self._symbol = Symbol(hp.vocab_path, hp.poly_dict_path)
self.input_dim = self._symbol.input_dim
self.num_class = self._symbol.num_class
self.shuffle = shuffle
with open(hp.data_path, "r") as json_file:
data = json.load(json_file)
self._eval_feature = data["features"][:hp.eval_size]
self._eval_label = data["features"][:hp.eval_size]
self._metadata = list(
zip(data["features"][hp.eval_size:],
data["labels"][hp.eval_size:]))
self.num_samples = len(self._metadata)
self._placeholders = [
tf.placeholder(tf.float32, [None, None, self.input_dim], 'inputs'),
tf.placeholder(tf.int32, [None], 'target_lengths'),
tf.placeholder(tf.float32, [None, None, self.num_class],
'targets'),
tf.placeholder(tf.float32, [None, None, self.num_class],
'poly_mask')
]
# Create queue for buffering data:
self.queue = tf.FIFOQueue(
hp.queue_capacity, [tf.float32, tf.int32, tf.float32, tf.float32],
name='input_queue')
self._enqueue_op = self.queue.enqueue(self._placeholders)
def __init__(self, symbol, _type):
Symbol.__init__(self, symbol._mangled, "PARAMDECL")
self.entry = symbol
self.__size = TYPE_SIZES[self._type]
self.__size = self.__size + (self.__size % 2) # Make it even-sized (Float and Byte)
self.byref = OPTIONS.byref.value # By default all params By value (false)
self.offset = None # Set by PARAMLIST, contains positive offset from top of the stack
def GetSign(self, leaf):
if (not leaf.sign):
if (leaf.symbol.mask == "T"):
leaf.symbol = Symbol("F")
leaf.sign = True
elif (leaf.symbol.mask == "F"):
leaf.symbol = Symbol("T")
leaf.sign = True
def __init__(self, left, treedef_list):
''' Constructor. Requires Symbol S and a list [ ] of
TreeDefs objects
'''
Symbol.__init__(self)
self.left = left # Tree symbol (left side of the rule, S symbol)
self.defs = {}
self.add_alternatives(treedef_list)
def __init__(self, terminal, arglist = None, generator = None):
Symbol.__init__(self)
self.first = terminal # 1st symbol in the right side
self.arglist = arglist # Optional arglist
# If no generator specified, a default one is created
# which will just return a string of the terminal
self.generator = generator # Optional generator function
def render(self, renderer):
for tile in self.body:
renderer.render(10 + tile[0], 5 + tile[1],
Symbol(' ', blt.color_from_name("white")), 0,
colors.WORLD_GEN_HIGHLIGHT)
for opening in self.openings:
renderer.render(10 + opening[0], 5 + opening[1],
Symbol('V', blt.color_from_name("white")), 0,
colors.WORLD_GEN_OPENING)
def setupEnvironment():
"""Sets up a new environment with a bunch of fairly standard
Scheme built-in primitives."""
PRIMITIVE_PROCEDURES = [
["car", pair.car],
["cdr", pair.cdr],
["cons", pair.cons],
["append", pair.append],
["list", pair.list],
["set-car!", pair.setCarBang],
["set-cdr!", pair.setCdrBang],
["+", schemeAdd],
["-", schemeSubtract],
["*", schemeMultiply],
["/", schemeDivide],
["remainder", schemeRemainder],
["quotient", schemeQuotient],
["sqrt", schemeSqrt],
["floor", schemeFloor],
["ceiling", schemeCeiling],
["=", schemeNumericalEq],
["<", schemeLessThan],
["<=", schemeLessThanOrEquals],
[">", schemeGreaterThan],
[">=", schemeGreaterThanOrEquals],
["eq?", schemeEqQuestion],
["equal?", schemeEqualQuestion],
["list?", schemeListQuestion],
["pair?", schemePairQuestion],
["null?", schemeNullQuestion],
["display", schemeDisplay],
["write", schemeWrite],
["newline", schemeNewline],
["not", schemeNot],
["string->symbol", schemeStringToSymbol],
["symbol->string", schemeSymbolToString],
["number->string", schemeNumberToString],
["string-append", schemeStringAppend],
["quit", schemeQuit],
["exit", schemeQuit],
["error", schemeError],
["parse", schemeParse],
["pow", schemePower],
]
initial_environment = environment.extendEnvironment(
pair.NIL, pair.NIL, environment.THE_EMPTY_ENVIRONMENT)
for name, proc in PRIMITIVE_PROCEDURES:
installPythonFunction(name, proc, initial_environment)
## Finally, put true and false in there.
environment.defineVariable(Symbol("#t"), Symbol("#t"), initial_environment)
environment.defineVariable(Symbol("#f"), Symbol("#f"), initial_environment)
return initial_environment
def __init__(self, _type, lineno, implicit = False):
''' Implicit = True if this type has been
"inferred" by default, or by the expression surrounding
the ID.
'''
Symbol.__init__(self, _type, 'TYPE')
self._type = _type
self.size = TYPE_SIZES[self._type]
self.lineno = lineno
self.implicit = implicit
def make_grammar(self):
grammar = Grammar()
r1 = Rule(
Symbol("NP", {"AGR": "?a"}), [
Symbol("ART", {"AGR": "?a"}), Symbol("N", {"AGR": "?a"})])
r1.set_variable_code("?a", -1L)
# -1L should be default for any undefined variable
# that is referenced while constructing
grammar.add_rule(r1)
return grammar
def testSchemeEval(self):
self.assertEquals(42, self.pe("(+ 2 (eval '(+ 30 10)))"))
self.assertEquals(42, self.pe("(+ 2 (eval '(+ 30 ((lambda () 10)))))"))
self.assertEquals(Symbol("hello"),
self.pe("((eval '(lambda () 'hello)))"))
## Quick test to see that EVAL'ed expressions are also
## expanded, since AND is derived.
self.assertEquals(Symbol("foo"),
self.pe("(eval '(AND 'this 'is 'a 'foo))"))
def define(self, symbol: Symbol) -> bool:
existing = self.findLocal(symbol.name)
if existing != None:
print(symbol.range_, 'Duplicate symbol:', symbol.name)
return False
self.symbols.append(symbol)
symbol.scope = self
symbol.minScopeDepth = self.depth
return True
def testCond(self):
self.assertEquals(
Symbol("ok"),
self.pe("""(define (even? x) (= (remainder x 2) 0))"""))
self.assertEquals(Symbol("ok"),
self.pe("""(define (odd? x) (not (even? x)))"""))
self.assertEquals(
Symbol("ok"),
self.pe("""(cond ((even? 3) "not-ok")
((odd? 2) "still-not-ok")
(#t 'ok))"""))
def test_attributes():
"""A symbol has two attributes: its index and its weight."""
s1 = Symbol(4, 2)
assert s1.index == 4
assert s1.weight == 2
s2 = Symbol(4)
assert s2.index == 4
assert s2.weight == 1 # default value
s3 = Symbol(weight=2, index=4)
assert s3.index == 4
assert s3.weight == 2
def testSetCarCdr(self):
self.pe("(define x '(hello world))")
self.pe("(define y x)")
self.pe("(define z (cons 'hi 'earth))")
self.pe("(set-car! x 'hi)")
self.pe("(set-cdr! y 'earth)")
self.assertEquals(pair.cons(Symbol("hi"), Symbol("earth")),
self.pe("x"))
self.assert_(self.pe("(eq? x y)"))
self.assert_(self.pe("(not (eq? x z))"))
self.assert_(self.pe("(equal? x z)"))
def testLoad(self):
## A small test of the stack module in the 't' test directory.
self.pe('(load "t/stack.scm")')
self.pe("(define s (make-stack))")
self.pe("(push s 42)")
self.pe("(push s 'foobar)")
self.pe("(push s (lambda (x) x))")
self.assertEquals(Symbol("muhaha"), self.pe("((pop s) 'muhaha)"))
self.assertEquals(Symbol("foobar"), self.pe("(pop s)"))
self.assertEquals(42, self.pe("(pop s)"))
self.assertRaises(SchemeError, self.pe, "(pop s)")
def test_init():
s1 = Symbol(1, 4)
s2 = Symbol(2, 2)
s3 = Symbol(3, 7)
t = Tile({s1, s2, s3})
assert t.symbols == {s1, s2, s3}
assert hasattr(t, "hash")
assert t.leaf_symbol == s3
assert t.leaf_index == 3
assert t.weight == 13
assert t.string == "{1, 2, 3}"
def add_sym(self, s_type) -> int:
"""
Add an empty symbol instance to the syms list.
Returns index for new symbol at end of list. This may not be
necessary when using python.
:param str s_type:
"""
sym = Symbol()
sym.type = s_type
self.syms.append(sym)
return len(self.syms)
def count_polY_accuracy(model, num_batchs, sess, save_dir):
symbol = Symbol(hp.vocab_path, hp.poly_dict_path)
with open(hp.poly_dict_path, 'r') as f:
POLY_DICT = json.load(f)
test_poly_dict = {}
for key, value in POLY_DICT.items():
POLY_DICT[key] = sorted(value)
test_poly_dict[key] = np.zeros((len(value), len(value)))
for i in range(num_batchs):
model_inputs, model_pre, mask, poly_mask, model_correct_pred, model_acc, model_outputs, model_targets, model_target_seq \
= sess.run([model.inputs, model.pred, model.mask, model.poly_mask, model.correct_pred, model.accuracy, model.outputs, model.targets, model.target_seq])
print("model_pred", model_pre)
print("model_accruacy", model_acc)
print("model_outputs", model_outputs)
print("model_mask", mask)
print("model_poly_mask", poly_mask)
print("model_targets", model_targets)
print("model_target_seq", model_target_seq)
print("model_correct_pred", model_correct_pred)
for pred_poly_seq, ta, model_input in zip(model_pre, model_target_seq,
model_inputs):
pred_poly = symbol.sequence_to_label(pred_poly_seq)
target_poly = symbol.sequence_to_label(ta)
word_value = symbol.input_to_word_value(model_input)
print(pred_poly)
print(target_poly)
print(word_value)
if len(pred_poly_seq) != len(ta):
print("length not equal")
print("-----------------")
continue
for p, t, w in zip(pred_poly, target_poly, word_value):
if t == "-":
continue
else:
i = POLY_DICT[w].index(p)
j = POLY_DICT[w].index(t)
test_poly_dict[w][i, j] += 1
print(w)
print(test_poly_dict[w])
print("Model Accuracy: {}".format(model_acc))
poly_accuracy_out = open(os.path.join(save_dir, "accuracy.out"), "w")
poly_count = open(os.path.join(save_dir, "poly_count.out"), "w")
for key, value in test_poly_dict.items():
accuracy = np.trace(value) / np.sum(value)
poly_accuracy_out.writelines("{}\t{}\t{}\t{}\n".format(
key, int(np.sum(value)), int(np.trace(value)), accuracy))
poly_count.writelines(key + "\n")
for i, poly_i in enumerate(POLY_DICT[key]):
for j, poly_j in enumerate(POLY_DICT[key]):
poly_count.write("{}->{}:{}\n".format(poly_i, poly_j,
int(value[i, j])))
def train_string(self, string):
"""docstring for train_string"""
input_sequence = [c for c in string]
if (0 < len(input_sequence)):
self.root_production.last().insert_after(Symbol.factory(self, input_sequence.pop(0)))
while (0 < len(input_sequence)):
self.root_production.last().insert_after(Symbol.factory(self, input_sequence.pop(0)))
match = self.get_index(self.root_production.last().prev)
if not match:
self.add_index(self.root_production.last().prev)
elif match.next != self.root_production.last().prev:
self.root_production.last().prev.process_match(match)
def __init__(self, arglistOrTerminal, terminal = None):
''' Creates an argument list, by concatenaing an arglist
+ a terminal. If only a terminal is specified, then
it is converted to a list.
'''
Symbol.__init__(self)
self.list = []
if terminal is None:
self.list += [arglistOrTerminal]
else:
self.list = arglistOrTerminal.list + [terminal]
def visitClassDecl(self, node):
name = node.children[0].children[0].value
scope_type = "type_decl"
scope = Scope(node,
scope_type=scope_type,
scope_name=name,
parent=self.scope)
self.scope.scopes.setdefault(name, scope)
sym = Symbol(node, scope, name, scope_type)
type_def = Type_Def(node, name, sym.qualified_name, "scalar_type")
self.type_defs.setdefault(sym.qualified_name, type_def)
sym.type_def = type_def
self.scope.symbols.setdefault(name, sym)
def DistribOR(self, leaf):
if (leaf.symbol.code == "OP_OR"):
l = leaf.left
r = leaf.right
if (l.symbol.code == "OP_AND" and r.symbol.code == "OP_AND"):
ll1 = l.left
lr1 = l.right
rl1 = r.left
rr1 = r.right
leaf.symbol = Symbol("^")
ll2, llt = ll1.DuplicateTree()
self.tree += llt
lr2, lrt = lr1.DuplicateTree()
self.tree += lrt
rl2, rlt = rl1.DuplicateTree()
self.tree += rlt
rr2, rrt = rr1.DuplicateTree()
self.tree += rrt
nsym = Symbol("v")
nll = Leaf(nsym, True, l, ll1, rl1)
self.tree.append(nll)
ll1.upper = nll
rl1.upper = nll
nlr = Leaf(nsym, True, l, ll2, rr1)
self.tree.append(nlr)
ll2.upper = nlr
rr1.upper = nlr
nrl = Leaf(nsym, True, r, lr1, rl2)
self.tree.append(nrl)
lr1.upper = nrl
rl2.upper = nrl
nrr = Leaf(nsym, True, r, lr2, rr2)
self.tree.append(nrr)
lr2.upper = nrr
rr2.upper = nrr
l.left = nll
l.right = nlr
r.left = nrl
r.right = nrr
return True
return False
def visitExternType(self, node):
name = node.children[0].children[0].value
scope_type = "extern_type"
# TODO support external types with bodies.
scope = Scope(node,
scope_type=scope_type,
scope_name=name,
parent=self.scope)
self.scope.scopes.setdefault(name, scope)
sym = Symbol(node, scope, name, scope_type)
type_def = Type_Def(node, name, sym.qualified_name, "scalar_type")
self.type_defs.setdefault(sym.qualified_name, type_def)
sym.type_def = type_def
self.scope.symbols.setdefault(name, sym)
def testGensymConstruction(self):
self.pe("""(define (make-counter prefix)
(let ((i 0))
(lambda ()
(begin
(set! i (+ i 1))
(string->symbol
(string-append (symbol->string prefix)
(number->string i)))))))""")
self.pe("""(define gensym1 (make-counter 'g))""")
self.pe("""(define gensym2 (make-counter 'h))""")
self.assertEquals(Symbol("g1"), self.pe("(gensym1)"))
self.assertEquals(Symbol("g2"), self.pe("(gensym1)"))
self.assertEquals(Symbol("g3"), self.pe("(gensym1)"))
self.assertEquals(Symbol("h1"), self.pe("(gensym2)"))
def __init__(self, grammar):
super(Rule, self).__init__()
self.guard = Symbol.guard(grammar, self)
self.guard.join(self.guard)
self.reference_count = 0
self.unique_number = Rule.unique_rule_number
Rule.unique_rule_number += 1
def op_function(machine):
fun_pad = machine.data_stack.pop()
fun_name = machine.data_stack.pop()
fun_name_w_ctx = '{}.{}'.format(machine.ctx, fun_name)
symbol = Symbol(fun_name_w_ctx, 'function', machine.instruction_ptr, fun_pad)
machine.symbols[fun_name_w_ctx] = symbol
machine.instruction_ptr = machine.instruction_ptr + fun_pad
def fn(*x):
x = list(x)
if isinstance(x, list):
x = x[0]
for i in range(len(arg[0])):
d[Symbol(arg[0][i])] = eval(x[i], ChainMap(d))
return eval(block, ChainMap(d))
def parse_cst(self):
"""Given that the token generator is positioned at the start
of the concrete grammar, read rules. After this routine
completes, each symbol in the GlobalSymbolDict has a set of
productions that contain Tokens, not symbols. Conversion from
tokens to symbols happens in promote_productions."""
stack = []
self.tokenizer.next().must_be('{')
for token in self.tokenizer:
stack += [ token ] # Build a stack to process
if token.text == ".":
# We've got a rule to process. Start by determining correct syntax.
stack[1].must_be(':')
## Name analysis
stack[0].assert_symbol_name()
production_elements = stack[2:-1]
for element in production_elements:
element.assert_symbol_name()
if stack[0].text in self.GlobalSymbolDict: # Redefined lexical sym or add a new production?
existingSymbol = self.GlobalSymbolDict[stack[0].text]
if existingSymbol.is_gla:
raise Exception("Lexical Symbol %s redefined at %d,%d. Originally at %d,%d" % \
(stack[0].text, stack[0].line, stack[0].col, \
existingSymbol.defining_token.line, existingSymbol.defining_token.col))
existingSymbol.productions += [Production(existingSymbol,production_elements)]
else: # Brand new symbol occurrence
s = Symbol(stack[0])
s.is_gla = False
s.productions = [Production(s,production_elements)]
self.GlobalSymbolDict[stack[0].text] = s
stack = []
elif token.text == "{":
raise Exception("Unexpected %s" % token)
elif token.text == "}":
if len(stack) > 1: raise Exception("Unfinished lexical specification beginning with %s" % stack[0])
#pp = pprint.PrettyPrinter()
#pp.pprint(self.GlobalSymbolDict)
return
else: pass
def __init__(self, value, _type = None, lineno = None):
if lineno is None:
raise ValueError # This should be changed to another exception
Symbol.__init__(self, value, 'NUMBER')
if int(value) == value:
value = int(value)
self.value = value
if _type is not None:
self._type = _type
elif isinstance(value, float):
if -32768.0 < value < 32767:
self._type = 'fixed'
else:
self._type = 'float'
elif isinstance(value, int):
if 0 <= value < 256:
self._type = 'u8'
elif -128 <= value < 128:
self._type = 'i8'
elif 0 <= value < 65536:
self._type = 'u16'
elif -32768 <= value < 32768:
self._type = 'i16'
elif value < 0:
self._type = 'i32'
else:
self._type = 'u32'
self.t = value
self.lineno = lineno
def promote_productions(self):
"""Convert all the elements of products from tokens into
symbols, meanwhile checking that all of the elements are
existing symbols. This is name analysis in action: because
symbol names have Algol scoping inside the concrete grammar
portion of the input file, we wait until the whose shebang is
parsed before attempting to promote tokens into symbols."""
for sym in self.GlobalSymbolDict.values():
for production in sym.productions:
elements = production.elements
if len(elements) > 0: # An empty production has no tokens to promote
firstToken = elements[0]
for i in range(0, len(elements)):
if re.compile("^'").match(elements[i].text): # If the element is a literal, no name analysis needs to be done
elements[i] = Symbol(elements[i])
elements[i].is_lit = True
elements[i].regex = Set(re.escape(elements[i].defining_token.text[1:-1]))
self.GlobalSymbolDict[elements[i].defining_token.text]=elements[i]
else: # Do name analysis: check if the symbol is used without being defined.
try:
elements[i] = self.GlobalSymbolDict[elements[i].text]
except KeyError, e:
raise Exception("Production for %s beginning at %d,%d: %s is not a symbol." % \
(sym.defining_token.text, firstToken.line, firstToken.col, elements[i].text))
def __init__(self):
Symbol.__init__(self, None, 'PARAMLIST')
self.size = 0 # Will contain the sum of all the params size (byte params counts as 2 bytes)
self.count = 0 # Counter of number of params
def __init__(self, symbol):
Symbol.__init__(self, symbol._mangled, 'FUNCDECL')
self.fname = symbol.id
self._mangled = symbol._mangled
self.entry = symbol # Symbol table entry
def __init__(self, value, lineno):
Symbol.__init__(self, value, 'STRING')
self._type = 'string'
self.lineno = lineno
self.t = value
def create_root_symbols(self):
"""Insert magical symbols above the root of the grammar in
order to match the beginning and end of the sample."""
RootSymbol = Symbol(Token(None,None,'R00t.Symbol'))
RootSymbol.GlobalSymbolDict=self.GlobalSymbolDict
StartDocSymbol = Symbol(Token(None,None,'%^'))
StartDocSymbol.regex = Set('%^')
StartDocSymbol.is_lit = True
StartDocSymbol.GlobalSymbolDict=self.GlobalSymbolDict
EndDocSymbol = Symbol(Token(None,None,'%$'))
EndDocSymbol.regex = Set('%$')
EndDocSymbol.is_lit = True
EndDocSymbol.GlobalSymbolDict=self.GlobalSymbolDict
RootSymbol.productions = [Production(RootSymbol,[StartDocSymbol]+self.get_roots()+[EndDocSymbol])]
self.GlobalSymbolDict['R00t.Symbol'] = RootSymbol #XXX this is a nasty hack
self.GlobalSymbolDict['%^']=StartDocSymbol
self.GlobalSymbolDict['%$']=EndDocSymbol
def __init__(self, lineno, expr):
Symbol.__init__(self, None, 'CONST')
self.expr = expr
self.lineno = lineno
def __init__(self, new_type):
Symbol.__init__(self, new_type, 'CAST')
self.t = optemps.new_t()
self._type = new_type
def test_create_symbol(self):
symbol = Symbol('a')
self.assertEqual('a', symbol.name())
def test_evaluate_undefined_symbol(self):
context = Context()
symbol = Symbol('a')
self.assertIsNone(symbol.evaluate(context))
from symbol import Symbol
from source import Source
from algos import weaver, fanno, huffman, block
from benchmark import benchmark
a = Symbol('A', 2**-5)
b = Symbol('b', 2**-5)
c = Symbol('c', 2**-5)
d = Symbol('d', 2**-5)
e = Symbol('e', 2**-4)
f = Symbol('f', 2**-4)
g = Symbol('g', 2**-2)
h = Symbol('h', 2**-1)
i = Symbol('i', 0.05)
j = Symbol('j', 0.5)
k = Symbol('k', 0.25)
l = Symbol('l', 0.125)
m = Symbol('m', 0.125)
s = Source(d, b, f, a, e, c, g, h)
c = block(s)
for sym in s.sorted():
print(sym.name(), ' ', c.codeOf(sym))
benchmark(s, c)
def __init__(self, text):
Symbol.__init__(self, text = text)
"OMSTR": parse_omstr,
"OMV": parse_var,
}
OMDICTS = defaultdict(dict)
# logic1 http://www.openmath.org/cd/logic1.xhtml
OMDICTS["logic1"]["true"] = True
OMDICTS["logic1"]["false"] = False
# nums1 http://www.openmath.org/cd/nums1.xhtml1
OMDICTS["nums1"]["rational"] = oms_nums1_rational
# complex1 http://www.openmath.org/cd/complex1.xhtml
OMDICTS["complex1"]["complex_cartesian"] = oms_complex1_complex_cartesian
# interval1 http://www.openmath.org/cd/interval1.xhtml
OMDICTS["interval1"]["integer_interval"] = oms_interval1_integer_interval
# list1 http://www.openmath.org/cd/list1.xhtml
OMDICTS["list1"]["list"] = oms_list1_list
CDs = Symbol.__subclasses__()
for class_ in CDs:
print(class_.name())
OMDICTS[class_.dictionary()][class_.name()] = class_
for subclass in class_.__subclasses__():
print(subclass.name())
OMDICTS[subclass.dictionary()][subclass.name()] = subclass
def __init__(self):
Symbol.__init__(self, None, 'ARGLIST')
self.count = 0 # Number of params
def __init__(self):
Symbol.__init__(self, None, 'BLOCK')
def __init__(self, symbol):
Symbol.__init__(self, symbol._mangled, 'ARRAYDECL')
self._type = symbol._type
self.size = symbol.total_size # Total array cell + index size
self.entry = symbol
self.bounds = symbol.bounds
def __init__(self, symbol):
Symbol.__init__(self, symbol._mangled, 'VARDECL')
self._type = symbol._type
self.size = symbol.size
self.entry = symbol
def __init__(self, asm, lineno):
Symbol.__init__(self, asm, 'ASM')
self.lineno = lineno
def __init__(self, lineno):
Symbol.__init__(self, None, "STRSLICE")
self.lineno = lineno
self._type = "string"
self.t = optemps.new_t()
