def testAnotherQuasiquote(self): self.pe("(define x 42)") self.assertEquals(42, self.pe("x")) self.assertEquals( pair.list(Symbol("x"), pair.list(Symbol("quote"), pair.list(Symbol("x"), 42))), self.pe("`(x '(x ,x))"))
def ReplaceInTree(self, mask, value): """Replace a symbol with the symbol of the value inside the tree hosted by this node""" has_replaced = False print("FOUND:", self.symbol, mask, value) if (self.symbol.mask == mask): print("REPLACING:", self.symbol, mask, value, self.sign) if (value not in ["T", "F"]): sym = Symbol(value) else: if (not self.sign): if (value == "T"): sym = Symbol("F") elif (value == "F"): sym = Symbol("T") self.sign = True else: sym = Symbol(value) has_replaced = True self.symbol = sym print("CHECK REP:", self.symbol, mask, value) if (self.left != None): has_replaced = has_replaced | self.left.ReplaceInTree(mask, value) if (self.right != None): has_replaced = has_replaced | self.right.ReplaceInTree(mask, value) print("has replaced", has_replaced) return has_replaced
def test_equality(): """Two symbols with the same attributes are identical.""" s1 = Symbol(1, 4) s2 = Symbol(2, 2) s3 = Symbol(2, 2) assert s1 != s2 assert s2 == s3
def __init__(self, lineno, byref=False): ''' Initializes the argument data. Byref must be set to True if this Argument is passed by reference. ''' Symbol.__init__(self, None, 'ARGUMENT') self.lineno = lineno self.byref = byref
def __init__(self, symbol, _type): Symbol.__init__(self, symbol._mangled, 'PARAMDECL') self.entry = symbol self.__size = TYPE_SIZES[self._type] self.__size = self.__size + (self.__size % 2) # Make it even-sized (Float and Byte) self.byref = OPTIONS.byref.value # By default all params By value (false) self.offset = None # Set by PARAMLIST, contains positive offset from top of the stack
def __init__(self, value, lineno, offset=None): global SYMBOL_TABLE Symbol.__init__(self, value, 'ID') self.id = value self.filename = gl.FILENAME # In which file was first used self.lineno = lineno # In which line was first used self._class = None self._mangled = '_%s' % value # This value will be overriden later self.t = self._mangled self.declared = False # if declared (DIM var AS <type>) this must be True self._type = None # Unknown type self.offset = offset # For local variables, offset from top of the stack self.default_value = None # If defined, variable will be initialized with this value (Arrays = List of Bytes) self.scope = 'global' # One of 'global', 'parameter', 'local' self.byref = False # By default, it's a global var self.default_value = None # For variables, this is the default initalized value self.__kind = None # If not None, it should be one of 'function' or 'sub' self.addr = None # If not None, the address of this symbol (string) self.alias = None # If not None, this var is an alias of another self.aliased_by = [] # Which variables are an alias of this one self.referenced_by = [ ] # Which objects do use this one (e.g. sentences using this variable) self.references = [ ] # Objects referenced by this one (e.g. variables used in this sentence) self.accessed = False # Where this object has been accessed (if false it might be not compiled, since it is useless) self.caseins = OPTIONS.case_insensitive.value # Whether this ID is case insensitive or not
def __init__(self, value, lineno, offset=None): global SYMBOL_TABLE Symbol.__init__(self, value, "ID") self.id = value self.filename = gl.FILENAME # In which file was first used self.lineno = lineno # In which line was first used self._class = None self._mangled = "_%s" % value # This value will be overriden later self.t = self._mangled self.declared = False # if declared (DIM var AS <type>) this must be True self._type = None # Unknown type self.offset = offset # For local variables, offset from top of the stack self.default_value = None # If defined, variable will be initialized with this value (Arrays = List of Bytes) self.scope = "global" # One of 'global', 'parameter', 'local' self.byref = False # By default, it's a global var self.default_value = None # For variables, this is the default initalized value self.__kind = None # If not None, it should be one of 'function' or 'sub' self.addr = None # If not None, the address of this symbol (string) self.alias = None # If not None, this var is an alias of another self.aliased_by = [] # Which variables are an alias of this one self.referenced_by = [] # Which objects do use this one (e.g. sentences using this variable) self.references = [] # Objects referenced by this one (e.g. variables used in this sentence) self.accessed = ( False ) # Where this object has been accessed (if false it might be not compiled, since it is useless) self.caseins = OPTIONS.case_insensitive.value # Whether this ID is case insensitive or not
def __init__(self, lineno, byref = False): ''' Initializes the argument data. Byref must be set to True if this Argument is passed by reference. ''' Symbol.__init__(self, None, 'ARGUMENT') self.lineno = lineno self.byref = byref
def add_symbol(self): symbol = Symbol() size = len(self.symbols) if size > 0: last = self.symbols[-1] # if the previous tag has started a script or a style, # do not include the contents of those tags as they # are large and unnecessary # NOTE: this does not skip inline tags if last.type == SymbolType.SCRIPT_START or last.type == SymbolType.STYLE_START: symbol = self.tokenizer.get_next_symbol() while (symbol.type == SymbolType.SCRIPT_START or symbol.type == SymbolType.STYLE_START or symbol.type == SymbolType.SCRIPT_END or symbol.type == SymbolType.STYLE_END or symbol.type == SymbolType.PLAIN_TEXT): symbol = self.tokenizer.get_next_symbol() else: symbol = self.tokenizer.get_next_symbol() else: symbol = self.tokenizer.get_next_symbol() # do not add symbols until a start of the document has been reached # the start it represented as DOCTYPE tag if symbol.type != SymbolType.DOCTYPE: symbol = Symbol.empty() if not symbol.is_empty(): # might be empty when the end of the HTML has been reached self.symbols.append(symbol)
def parse_lexical_symbols(self): """Given that the token generator is at the beginning of the lexical symbol specifications, read a series of lexical symbol specifications, doing name and basic type analysis on the fly.""" stack = [] self.tokenizer.next().must_be('{') for token in self.tokenizer: stack += [ token ] if token.text == ".": stack[0].assert_symbol_name() stack[1].must_be(':') stack[2].must_match('^\\$', "regular expression") ## Name analysis if stack[0].text in self.GlobalSymbolDict: originalDef = self.GlobalSymbolDict[stack[0].text].defining_token raise Exception("Symbol %s redefined at %d,%d. Originally at %d,%d" % (stack[0].text, stack[0].line, stack[0].col, \ originalDef.line, originalDef.col)) s = Symbol(stack[0]) s.is_gla = True s.regex = Set(stack[2].text[1:]) self.GlobalSymbolDict[stack[0].text] = s stack = [] elif token.text == "{": raise Exception("Unexpected %s" % token) elif token.text == "}": if len(stack) > 1: raise Exception("Unfinished lexical specification beginning with %s" % stack[0]) return else: pass
def SimpleDistribOR(self, leaf): if (leaf.symbol.code == "OP_AND"): l = leaf.left r = leaf.right if (l.symbol.code == "OP_OR" and r.symbol.code == "IDENTIFIER"): op = l ident = r elif (r.symbol.code == "OP_OR" and l.symbol.code == "IDENTIFIER"): op = r ident = l else: return False leaf.symbol = Symbol("v") opL = op.left opR = op.right ident2, identt = ident.DuplicateTree() self.tree += identt nl = Leaf(Symbol("^"), True, leaf, opL, ident) nr = Leaf(Symbol("^"), True, leaf, opR, ident2) leaf.left = nl leaf.right = nr return True
def __init__(self, coordinator, hparams, shuffle): super(DataFeeder, self).__init__() self._coord = coordinator self._hparams = hparams hp = self._hparams self._offset = 0 self._symbol = Symbol(hp.vocab_path, hp.poly_dict_path) self.input_dim = self._symbol.input_dim self.num_class = self._symbol.num_class self.shuffle = shuffle with open(hp.data_path, "r") as json_file: data = json.load(json_file) self._eval_feature = data["features"][:hp.eval_size] self._eval_label = data["features"][:hp.eval_size] self._metadata = list( zip(data["features"][hp.eval_size:], data["labels"][hp.eval_size:])) self.num_samples = len(self._metadata) self._placeholders = [ tf.placeholder(tf.float32, [None, None, self.input_dim], 'inputs'), tf.placeholder(tf.int32, [None], 'target_lengths'), tf.placeholder(tf.float32, [None, None, self.num_class], 'targets'), tf.placeholder(tf.float32, [None, None, self.num_class], 'poly_mask') ] # Create queue for buffering data: self.queue = tf.FIFOQueue( hp.queue_capacity, [tf.float32, tf.int32, tf.float32, tf.float32], name='input_queue') self._enqueue_op = self.queue.enqueue(self._placeholders)
def __init__(self, symbol, _type): Symbol.__init__(self, symbol._mangled, "PARAMDECL") self.entry = symbol self.__size = TYPE_SIZES[self._type] self.__size = self.__size + (self.__size % 2) # Make it even-sized (Float and Byte) self.byref = OPTIONS.byref.value # By default all params By value (false) self.offset = None # Set by PARAMLIST, contains positive offset from top of the stack
def GetSign(self, leaf): if (not leaf.sign): if (leaf.symbol.mask == "T"): leaf.symbol = Symbol("F") leaf.sign = True elif (leaf.symbol.mask == "F"): leaf.symbol = Symbol("T") leaf.sign = True
def __init__(self, left, treedef_list): ''' Constructor. Requires Symbol S and a list [ ] of TreeDefs objects ''' Symbol.__init__(self) self.left = left # Tree symbol (left side of the rule, S symbol) self.defs = {} self.add_alternatives(treedef_list)
def __init__(self, terminal, arglist = None, generator = None): Symbol.__init__(self) self.first = terminal # 1st symbol in the right side self.arglist = arglist # Optional arglist # If no generator specified, a default one is created # which will just return a string of the terminal self.generator = generator # Optional generator function
def render(self, renderer): for tile in self.body: renderer.render(10 + tile[0], 5 + tile[1], Symbol(' ', blt.color_from_name("white")), 0, colors.WORLD_GEN_HIGHLIGHT) for opening in self.openings: renderer.render(10 + opening[0], 5 + opening[1], Symbol('V', blt.color_from_name("white")), 0, colors.WORLD_GEN_OPENING)
def setupEnvironment(): """Sets up a new environment with a bunch of fairly standard Scheme built-in primitives.""" PRIMITIVE_PROCEDURES = [ ["car", pair.car], ["cdr", pair.cdr], ["cons", pair.cons], ["append", pair.append], ["list", pair.list], ["set-car!", pair.setCarBang], ["set-cdr!", pair.setCdrBang], ["+", schemeAdd], ["-", schemeSubtract], ["*", schemeMultiply], ["/", schemeDivide], ["remainder", schemeRemainder], ["quotient", schemeQuotient], ["sqrt", schemeSqrt], ["floor", schemeFloor], ["ceiling", schemeCeiling], ["=", schemeNumericalEq], ["<", schemeLessThan], ["<=", schemeLessThanOrEquals], [">", schemeGreaterThan], [">=", schemeGreaterThanOrEquals], ["eq?", schemeEqQuestion], ["equal?", schemeEqualQuestion], ["list?", schemeListQuestion], ["pair?", schemePairQuestion], ["null?", schemeNullQuestion], ["display", schemeDisplay], ["write", schemeWrite], ["newline", schemeNewline], ["not", schemeNot], ["string->symbol", schemeStringToSymbol], ["symbol->string", schemeSymbolToString], ["number->string", schemeNumberToString], ["string-append", schemeStringAppend], ["quit", schemeQuit], ["exit", schemeQuit], ["error", schemeError], ["parse", schemeParse], ["pow", schemePower], ] initial_environment = environment.extendEnvironment( pair.NIL, pair.NIL, environment.THE_EMPTY_ENVIRONMENT) for name, proc in PRIMITIVE_PROCEDURES: installPythonFunction(name, proc, initial_environment) ## Finally, put true and false in there. environment.defineVariable(Symbol("#t"), Symbol("#t"), initial_environment) environment.defineVariable(Symbol("#f"), Symbol("#f"), initial_environment) return initial_environment
def __init__(self, _type, lineno, implicit = False): ''' Implicit = True if this type has been "inferred" by default, or by the expression surrounding the ID. ''' Symbol.__init__(self, _type, 'TYPE') self._type = _type self.size = TYPE_SIZES[self._type] self.lineno = lineno self.implicit = implicit
def make_grammar(self): grammar = Grammar() r1 = Rule( Symbol("NP", {"AGR": "?a"}), [ Symbol("ART", {"AGR": "?a"}), Symbol("N", {"AGR": "?a"})]) r1.set_variable_code("?a", -1L) # -1L should be default for any undefined variable # that is referenced while constructing grammar.add_rule(r1) return grammar
def testSchemeEval(self): self.assertEquals(42, self.pe("(+ 2 (eval '(+ 30 10)))")) self.assertEquals(42, self.pe("(+ 2 (eval '(+ 30 ((lambda () 10)))))")) self.assertEquals(Symbol("hello"), self.pe("((eval '(lambda () 'hello)))")) ## Quick test to see that EVAL'ed expressions are also ## expanded, since AND is derived. self.assertEquals(Symbol("foo"), self.pe("(eval '(AND 'this 'is 'a 'foo))"))
def define(self, symbol: Symbol) -> bool: existing = self.findLocal(symbol.name) if existing != None: print(symbol.range_, 'Duplicate symbol:', symbol.name) return False self.symbols.append(symbol) symbol.scope = self symbol.minScopeDepth = self.depth return True
def testCond(self): self.assertEquals( Symbol("ok"), self.pe("""(define (even? x) (= (remainder x 2) 0))""")) self.assertEquals(Symbol("ok"), self.pe("""(define (odd? x) (not (even? x)))""")) self.assertEquals( Symbol("ok"), self.pe("""(cond ((even? 3) "not-ok") ((odd? 2) "still-not-ok") (#t 'ok))"""))
def test_attributes(): """A symbol has two attributes: its index and its weight.""" s1 = Symbol(4, 2) assert s1.index == 4 assert s1.weight == 2 s2 = Symbol(4) assert s2.index == 4 assert s2.weight == 1 # default value s3 = Symbol(weight=2, index=4) assert s3.index == 4 assert s3.weight == 2
def testSetCarCdr(self): self.pe("(define x '(hello world))") self.pe("(define y x)") self.pe("(define z (cons 'hi 'earth))") self.pe("(set-car! x 'hi)") self.pe("(set-cdr! y 'earth)") self.assertEquals(pair.cons(Symbol("hi"), Symbol("earth")), self.pe("x")) self.assert_(self.pe("(eq? x y)")) self.assert_(self.pe("(not (eq? x z))")) self.assert_(self.pe("(equal? x z)"))
def testLoad(self): ## A small test of the stack module in the 't' test directory. self.pe('(load "t/stack.scm")') self.pe("(define s (make-stack))") self.pe("(push s 42)") self.pe("(push s 'foobar)") self.pe("(push s (lambda (x) x))") self.assertEquals(Symbol("muhaha"), self.pe("((pop s) 'muhaha)")) self.assertEquals(Symbol("foobar"), self.pe("(pop s)")) self.assertEquals(42, self.pe("(pop s)")) self.assertRaises(SchemeError, self.pe, "(pop s)")
def test_init(): s1 = Symbol(1, 4) s2 = Symbol(2, 2) s3 = Symbol(3, 7) t = Tile({s1, s2, s3}) assert t.symbols == {s1, s2, s3} assert hasattr(t, "hash") assert t.leaf_symbol == s3 assert t.leaf_index == 3 assert t.weight == 13 assert t.string == "{1, 2, 3}"
def add_sym(self, s_type) -> int: """ Add an empty symbol instance to the syms list. Returns index for new symbol at end of list. This may not be necessary when using python. :param str s_type: """ sym = Symbol() sym.type = s_type self.syms.append(sym) return len(self.syms)
def count_polY_accuracy(model, num_batchs, sess, save_dir): symbol = Symbol(hp.vocab_path, hp.poly_dict_path) with open(hp.poly_dict_path, 'r') as f: POLY_DICT = json.load(f) test_poly_dict = {} for key, value in POLY_DICT.items(): POLY_DICT[key] = sorted(value) test_poly_dict[key] = np.zeros((len(value), len(value))) for i in range(num_batchs): model_inputs, model_pre, mask, poly_mask, model_correct_pred, model_acc, model_outputs, model_targets, model_target_seq \ = sess.run([model.inputs, model.pred, model.mask, model.poly_mask, model.correct_pred, model.accuracy, model.outputs, model.targets, model.target_seq]) print("model_pred", model_pre) print("model_accruacy", model_acc) print("model_outputs", model_outputs) print("model_mask", mask) print("model_poly_mask", poly_mask) print("model_targets", model_targets) print("model_target_seq", model_target_seq) print("model_correct_pred", model_correct_pred) for pred_poly_seq, ta, model_input in zip(model_pre, model_target_seq, model_inputs): pred_poly = symbol.sequence_to_label(pred_poly_seq) target_poly = symbol.sequence_to_label(ta) word_value = symbol.input_to_word_value(model_input) print(pred_poly) print(target_poly) print(word_value) if len(pred_poly_seq) != len(ta): print("length not equal") print("-----------------") continue for p, t, w in zip(pred_poly, target_poly, word_value): if t == "-": continue else: i = POLY_DICT[w].index(p) j = POLY_DICT[w].index(t) test_poly_dict[w][i, j] += 1 print(w) print(test_poly_dict[w]) print("Model Accuracy: {}".format(model_acc)) poly_accuracy_out = open(os.path.join(save_dir, "accuracy.out"), "w") poly_count = open(os.path.join(save_dir, "poly_count.out"), "w") for key, value in test_poly_dict.items(): accuracy = np.trace(value) / np.sum(value) poly_accuracy_out.writelines("{}\t{}\t{}\t{}\n".format( key, int(np.sum(value)), int(np.trace(value)), accuracy)) poly_count.writelines(key + "\n") for i, poly_i in enumerate(POLY_DICT[key]): for j, poly_j in enumerate(POLY_DICT[key]): poly_count.write("{}->{}:{}\n".format(poly_i, poly_j, int(value[i, j])))
def train_string(self, string): """docstring for train_string""" input_sequence = [c for c in string] if (0 < len(input_sequence)): self.root_production.last().insert_after(Symbol.factory(self, input_sequence.pop(0))) while (0 < len(input_sequence)): self.root_production.last().insert_after(Symbol.factory(self, input_sequence.pop(0))) match = self.get_index(self.root_production.last().prev) if not match: self.add_index(self.root_production.last().prev) elif match.next != self.root_production.last().prev: self.root_production.last().prev.process_match(match)
def __init__(self, arglistOrTerminal, terminal = None): ''' Creates an argument list, by concatenaing an arglist + a terminal. If only a terminal is specified, then it is converted to a list. ''' Symbol.__init__(self) self.list = [] if terminal is None: self.list += [arglistOrTerminal] else: self.list = arglistOrTerminal.list + [terminal]
def visitClassDecl(self, node): name = node.children[0].children[0].value scope_type = "type_decl" scope = Scope(node, scope_type=scope_type, scope_name=name, parent=self.scope) self.scope.scopes.setdefault(name, scope) sym = Symbol(node, scope, name, scope_type) type_def = Type_Def(node, name, sym.qualified_name, "scalar_type") self.type_defs.setdefault(sym.qualified_name, type_def) sym.type_def = type_def self.scope.symbols.setdefault(name, sym)
def DistribOR(self, leaf): if (leaf.symbol.code == "OP_OR"): l = leaf.left r = leaf.right if (l.symbol.code == "OP_AND" and r.symbol.code == "OP_AND"): ll1 = l.left lr1 = l.right rl1 = r.left rr1 = r.right leaf.symbol = Symbol("^") ll2, llt = ll1.DuplicateTree() self.tree += llt lr2, lrt = lr1.DuplicateTree() self.tree += lrt rl2, rlt = rl1.DuplicateTree() self.tree += rlt rr2, rrt = rr1.DuplicateTree() self.tree += rrt nsym = Symbol("v") nll = Leaf(nsym, True, l, ll1, rl1) self.tree.append(nll) ll1.upper = nll rl1.upper = nll nlr = Leaf(nsym, True, l, ll2, rr1) self.tree.append(nlr) ll2.upper = nlr rr1.upper = nlr nrl = Leaf(nsym, True, r, lr1, rl2) self.tree.append(nrl) lr1.upper = nrl rl2.upper = nrl nrr = Leaf(nsym, True, r, lr2, rr2) self.tree.append(nrr) lr2.upper = nrr rr2.upper = nrr l.left = nll l.right = nlr r.left = nrl r.right = nrr return True return False
def visitExternType(self, node): name = node.children[0].children[0].value scope_type = "extern_type" # TODO support external types with bodies. scope = Scope(node, scope_type=scope_type, scope_name=name, parent=self.scope) self.scope.scopes.setdefault(name, scope) sym = Symbol(node, scope, name, scope_type) type_def = Type_Def(node, name, sym.qualified_name, "scalar_type") self.type_defs.setdefault(sym.qualified_name, type_def) sym.type_def = type_def self.scope.symbols.setdefault(name, sym)
def testGensymConstruction(self): self.pe("""(define (make-counter prefix) (let ((i 0)) (lambda () (begin (set! i (+ i 1)) (string->symbol (string-append (symbol->string prefix) (number->string i)))))))""") self.pe("""(define gensym1 (make-counter 'g))""") self.pe("""(define gensym2 (make-counter 'h))""") self.assertEquals(Symbol("g1"), self.pe("(gensym1)")) self.assertEquals(Symbol("g2"), self.pe("(gensym1)")) self.assertEquals(Symbol("g3"), self.pe("(gensym1)")) self.assertEquals(Symbol("h1"), self.pe("(gensym2)"))
def __init__(self, grammar): super(Rule, self).__init__() self.guard = Symbol.guard(grammar, self) self.guard.join(self.guard) self.reference_count = 0 self.unique_number = Rule.unique_rule_number Rule.unique_rule_number += 1
def op_function(machine): fun_pad = machine.data_stack.pop() fun_name = machine.data_stack.pop() fun_name_w_ctx = '{}.{}'.format(machine.ctx, fun_name) symbol = Symbol(fun_name_w_ctx, 'function', machine.instruction_ptr, fun_pad) machine.symbols[fun_name_w_ctx] = symbol machine.instruction_ptr = machine.instruction_ptr + fun_pad
def fn(*x): x = list(x) if isinstance(x, list): x = x[0] for i in range(len(arg[0])): d[Symbol(arg[0][i])] = eval(x[i], ChainMap(d)) return eval(block, ChainMap(d))
def parse_cst(self): """Given that the token generator is positioned at the start of the concrete grammar, read rules. After this routine completes, each symbol in the GlobalSymbolDict has a set of productions that contain Tokens, not symbols. Conversion from tokens to symbols happens in promote_productions.""" stack = [] self.tokenizer.next().must_be('{') for token in self.tokenizer: stack += [ token ] # Build a stack to process if token.text == ".": # We've got a rule to process. Start by determining correct syntax. stack[1].must_be(':') ## Name analysis stack[0].assert_symbol_name() production_elements = stack[2:-1] for element in production_elements: element.assert_symbol_name() if stack[0].text in self.GlobalSymbolDict: # Redefined lexical sym or add a new production? existingSymbol = self.GlobalSymbolDict[stack[0].text] if existingSymbol.is_gla: raise Exception("Lexical Symbol %s redefined at %d,%d. Originally at %d,%d" % \ (stack[0].text, stack[0].line, stack[0].col, \ existingSymbol.defining_token.line, existingSymbol.defining_token.col)) existingSymbol.productions += [Production(existingSymbol,production_elements)] else: # Brand new symbol occurrence s = Symbol(stack[0]) s.is_gla = False s.productions = [Production(s,production_elements)] self.GlobalSymbolDict[stack[0].text] = s stack = [] elif token.text == "{": raise Exception("Unexpected %s" % token) elif token.text == "}": if len(stack) > 1: raise Exception("Unfinished lexical specification beginning with %s" % stack[0]) #pp = pprint.PrettyPrinter() #pp.pprint(self.GlobalSymbolDict) return else: pass
def __init__(self, value, _type = None, lineno = None): if lineno is None: raise ValueError # This should be changed to another exception Symbol.__init__(self, value, 'NUMBER') if int(value) == value: value = int(value) self.value = value if _type is not None: self._type = _type elif isinstance(value, float): if -32768.0 < value < 32767: self._type = 'fixed' else: self._type = 'float' elif isinstance(value, int): if 0 <= value < 256: self._type = 'u8' elif -128 <= value < 128: self._type = 'i8' elif 0 <= value < 65536: self._type = 'u16' elif -32768 <= value < 32768: self._type = 'i16' elif value < 0: self._type = 'i32' else: self._type = 'u32' self.t = value self.lineno = lineno
def promote_productions(self): """Convert all the elements of products from tokens into symbols, meanwhile checking that all of the elements are existing symbols. This is name analysis in action: because symbol names have Algol scoping inside the concrete grammar portion of the input file, we wait until the whose shebang is parsed before attempting to promote tokens into symbols.""" for sym in self.GlobalSymbolDict.values(): for production in sym.productions: elements = production.elements if len(elements) > 0: # An empty production has no tokens to promote firstToken = elements[0] for i in range(0, len(elements)): if re.compile("^'").match(elements[i].text): # If the element is a literal, no name analysis needs to be done elements[i] = Symbol(elements[i]) elements[i].is_lit = True elements[i].regex = Set(re.escape(elements[i].defining_token.text[1:-1])) self.GlobalSymbolDict[elements[i].defining_token.text]=elements[i] else: # Do name analysis: check if the symbol is used without being defined. try: elements[i] = self.GlobalSymbolDict[elements[i].text] except KeyError, e: raise Exception("Production for %s beginning at %d,%d: %s is not a symbol." % \ (sym.defining_token.text, firstToken.line, firstToken.col, elements[i].text))
def __init__(self): Symbol.__init__(self, None, 'PARAMLIST') self.size = 0 # Will contain the sum of all the params size (byte params counts as 2 bytes) self.count = 0 # Counter of number of params
def __init__(self, symbol): Symbol.__init__(self, symbol._mangled, 'FUNCDECL') self.fname = symbol.id self._mangled = symbol._mangled self.entry = symbol # Symbol table entry
def __init__(self, value, lineno): Symbol.__init__(self, value, 'STRING') self._type = 'string' self.lineno = lineno self.t = value
def create_root_symbols(self): """Insert magical symbols above the root of the grammar in order to match the beginning and end of the sample.""" RootSymbol = Symbol(Token(None,None,'R00t.Symbol')) RootSymbol.GlobalSymbolDict=self.GlobalSymbolDict StartDocSymbol = Symbol(Token(None,None,'%^')) StartDocSymbol.regex = Set('%^') StartDocSymbol.is_lit = True StartDocSymbol.GlobalSymbolDict=self.GlobalSymbolDict EndDocSymbol = Symbol(Token(None,None,'%$')) EndDocSymbol.regex = Set('%$') EndDocSymbol.is_lit = True EndDocSymbol.GlobalSymbolDict=self.GlobalSymbolDict RootSymbol.productions = [Production(RootSymbol,[StartDocSymbol]+self.get_roots()+[EndDocSymbol])] self.GlobalSymbolDict['R00t.Symbol'] = RootSymbol #XXX this is a nasty hack self.GlobalSymbolDict['%^']=StartDocSymbol self.GlobalSymbolDict['%$']=EndDocSymbol
def __init__(self, lineno, expr): Symbol.__init__(self, None, 'CONST') self.expr = expr self.lineno = lineno
def __init__(self, new_type): Symbol.__init__(self, new_type, 'CAST') self.t = optemps.new_t() self._type = new_type
def test_create_symbol(self): symbol = Symbol('a') self.assertEqual('a', symbol.name())
def test_evaluate_undefined_symbol(self): context = Context() symbol = Symbol('a') self.assertIsNone(symbol.evaluate(context))
from symbol import Symbol from source import Source from algos import weaver, fanno, huffman, block from benchmark import benchmark a = Symbol('A', 2**-5) b = Symbol('b', 2**-5) c = Symbol('c', 2**-5) d = Symbol('d', 2**-5) e = Symbol('e', 2**-4) f = Symbol('f', 2**-4) g = Symbol('g', 2**-2) h = Symbol('h', 2**-1) i = Symbol('i', 0.05) j = Symbol('j', 0.5) k = Symbol('k', 0.25) l = Symbol('l', 0.125) m = Symbol('m', 0.125) s = Source(d, b, f, a, e, c, g, h) c = block(s) for sym in s.sorted(): print(sym.name(), ' ', c.codeOf(sym)) benchmark(s, c)
def __init__(self, text): Symbol.__init__(self, text = text)
"OMSTR": parse_omstr, "OMV": parse_var, } OMDICTS = defaultdict(dict) # logic1 http://www.openmath.org/cd/logic1.xhtml OMDICTS["logic1"]["true"] = True OMDICTS["logic1"]["false"] = False # nums1 http://www.openmath.org/cd/nums1.xhtml1 OMDICTS["nums1"]["rational"] = oms_nums1_rational # complex1 http://www.openmath.org/cd/complex1.xhtml OMDICTS["complex1"]["complex_cartesian"] = oms_complex1_complex_cartesian # interval1 http://www.openmath.org/cd/interval1.xhtml OMDICTS["interval1"]["integer_interval"] = oms_interval1_integer_interval # list1 http://www.openmath.org/cd/list1.xhtml OMDICTS["list1"]["list"] = oms_list1_list CDs = Symbol.__subclasses__() for class_ in CDs: print(class_.name()) OMDICTS[class_.dictionary()][class_.name()] = class_ for subclass in class_.__subclasses__(): print(subclass.name()) OMDICTS[subclass.dictionary()][subclass.name()] = subclass
def __init__(self): Symbol.__init__(self, None, 'ARGLIST') self.count = 0 # Number of params
def __init__(self): Symbol.__init__(self, None, 'BLOCK')
def __init__(self, symbol): Symbol.__init__(self, symbol._mangled, 'ARRAYDECL') self._type = symbol._type self.size = symbol.total_size # Total array cell + index size self.entry = symbol self.bounds = symbol.bounds
def __init__(self, symbol): Symbol.__init__(self, symbol._mangled, 'VARDECL') self._type = symbol._type self.size = symbol.size self.entry = symbol
def __init__(self, asm, lineno): Symbol.__init__(self, asm, 'ASM') self.lineno = lineno
def __init__(self, lineno): Symbol.__init__(self, None, "STRSLICE") self.lineno = lineno self._type = "string" self.t = optemps.new_t()