def test_equality():
"""Two symbols with the same attributes are identical."""
s1 = Symbol(1, 4)
s2 = Symbol(2, 2)
s3 = Symbol(2, 2)
assert s1 != s2
assert s2 == s3
def SimpleDistribOR(self, leaf):
if (leaf.symbol.code == "OP_AND"):
l = leaf.left
r = leaf.right
if (l.symbol.code == "OP_OR" and r.symbol.code == "IDENTIFIER"):
op = l
ident = r
elif (r.symbol.code == "OP_OR" and l.symbol.code == "IDENTIFIER"):
op = r
ident = l
else:
return False
leaf.symbol = Symbol("v")
opL = op.left
opR = op.right
ident2, identt = ident.DuplicateTree()
self.tree += identt
nl = Leaf(Symbol("^"), True, leaf, opL, ident)
nr = Leaf(Symbol("^"), True, leaf, opR, ident2)
leaf.left = nl
leaf.right = nr
return True
def ReplaceInTree(self, mask, value):
"""Replace a symbol with the symbol of the value inside the tree hosted by this node"""
has_replaced = False
print("FOUND:", self.symbol, mask, value)
if (self.symbol.mask == mask):
print("REPLACING:", self.symbol, mask, value, self.sign)
if (value not in ["T", "F"]):
sym = Symbol(value)
else:
if (not self.sign):
if (value == "T"):
sym = Symbol("F")
elif (value == "F"):
sym = Symbol("T")
self.sign = True
else:
sym = Symbol(value)
has_replaced = True
self.symbol = sym
print("CHECK REP:", self.symbol, mask, value)
if (self.left != None):
has_replaced = has_replaced | self.left.ReplaceInTree(mask, value)
if (self.right != None):
has_replaced = has_replaced | self.right.ReplaceInTree(mask, value)
print("has replaced", has_replaced)
return has_replaced
def testAnotherQuasiquote(self):
self.pe("(define x 42)")
self.assertEquals(42, self.pe("x"))
self.assertEquals(
pair.list(Symbol("x"),
pair.list(Symbol("quote"), pair.list(Symbol("x"), 42))),
self.pe("`(x '(x ,x))"))
def GetSign(self, leaf):
if (not leaf.sign):
if (leaf.symbol.mask == "T"):
leaf.symbol = Symbol("F")
leaf.sign = True
elif (leaf.symbol.mask == "F"):
leaf.symbol = Symbol("T")
leaf.sign = True
def render(self, renderer):
for tile in self.body:
renderer.render(10 + tile[0], 5 + tile[1],
Symbol(' ', blt.color_from_name("white")), 0,
colors.WORLD_GEN_HIGHLIGHT)
for opening in self.openings:
renderer.render(10 + opening[0], 5 + opening[1],
Symbol('V', blt.color_from_name("white")), 0,
colors.WORLD_GEN_OPENING)
def setupEnvironment():
"""Sets up a new environment with a bunch of fairly standard
Scheme built-in primitives."""
PRIMITIVE_PROCEDURES = [
["car", pair.car],
["cdr", pair.cdr],
["cons", pair.cons],
["append", pair.append],
["list", pair.list],
["set-car!", pair.setCarBang],
["set-cdr!", pair.setCdrBang],
["+", schemeAdd],
["-", schemeSubtract],
["*", schemeMultiply],
["/", schemeDivide],
["remainder", schemeRemainder],
["quotient", schemeQuotient],
["sqrt", schemeSqrt],
["floor", schemeFloor],
["ceiling", schemeCeiling],
["=", schemeNumericalEq],
["<", schemeLessThan],
["<=", schemeLessThanOrEquals],
[">", schemeGreaterThan],
[">=", schemeGreaterThanOrEquals],
["eq?", schemeEqQuestion],
["equal?", schemeEqualQuestion],
["list?", schemeListQuestion],
["pair?", schemePairQuestion],
["null?", schemeNullQuestion],
["display", schemeDisplay],
["write", schemeWrite],
["newline", schemeNewline],
["not", schemeNot],
["string->symbol", schemeStringToSymbol],
["symbol->string", schemeSymbolToString],
["number->string", schemeNumberToString],
["string-append", schemeStringAppend],
["quit", schemeQuit],
["exit", schemeQuit],
["error", schemeError],
["parse", schemeParse],
["pow", schemePower],
]
initial_environment = environment.extendEnvironment(
pair.NIL, pair.NIL, environment.THE_EMPTY_ENVIRONMENT)
for name, proc in PRIMITIVE_PROCEDURES:
installPythonFunction(name, proc, initial_environment)
## Finally, put true and false in there.
environment.defineVariable(Symbol("#t"), Symbol("#t"), initial_environment)
environment.defineVariable(Symbol("#f"), Symbol("#f"), initial_environment)
return initial_environment
def testSchemeEval(self):
self.assertEquals(42, self.pe("(+ 2 (eval '(+ 30 10)))"))
self.assertEquals(42, self.pe("(+ 2 (eval '(+ 30 ((lambda () 10)))))"))
self.assertEquals(Symbol("hello"),
self.pe("((eval '(lambda () 'hello)))"))
## Quick test to see that EVAL'ed expressions are also
## expanded, since AND is derived.
self.assertEquals(Symbol("foo"),
self.pe("(eval '(AND 'this 'is 'a 'foo))"))
def make_grammar(self):
grammar = Grammar()
r1 = Rule(
Symbol("NP", {"AGR": "?a"}), [
Symbol("ART", {"AGR": "?a"}), Symbol("N", {"AGR": "?a"})])
r1.set_variable_code("?a", -1L)
# -1L should be default for any undefined variable
# that is referenced while constructing
grammar.add_rule(r1)
return grammar
def testLoad(self):
## A small test of the stack module in the 't' test directory.
self.pe('(load "t/stack.scm")')
self.pe("(define s (make-stack))")
self.pe("(push s 42)")
self.pe("(push s 'foobar)")
self.pe("(push s (lambda (x) x))")
self.assertEquals(Symbol("muhaha"), self.pe("((pop s) 'muhaha)"))
self.assertEquals(Symbol("foobar"), self.pe("(pop s)"))
self.assertEquals(42, self.pe("(pop s)"))
self.assertRaises(SchemeError, self.pe, "(pop s)")
def testSetCarCdr(self):
self.pe("(define x '(hello world))")
self.pe("(define y x)")
self.pe("(define z (cons 'hi 'earth))")
self.pe("(set-car! x 'hi)")
self.pe("(set-cdr! y 'earth)")
self.assertEquals(pair.cons(Symbol("hi"), Symbol("earth")),
self.pe("x"))
self.assert_(self.pe("(eq? x y)"))
self.assert_(self.pe("(not (eq? x z))"))
self.assert_(self.pe("(equal? x z)"))
def testCond(self):
self.assertEquals(
Symbol("ok"),
self.pe("""(define (even? x) (= (remainder x 2) 0))"""))
self.assertEquals(Symbol("ok"),
self.pe("""(define (odd? x) (not (even? x)))"""))
self.assertEquals(
Symbol("ok"),
self.pe("""(cond ((even? 3) "not-ok")
((odd? 2) "still-not-ok")
(#t 'ok))"""))
def test_attributes():
"""A symbol has two attributes: its index and its weight."""
s1 = Symbol(4, 2)
assert s1.index == 4
assert s1.weight == 2
s2 = Symbol(4)
assert s2.index == 4
assert s2.weight == 1 # default value
s3 = Symbol(weight=2, index=4)
assert s3.index == 4
assert s3.weight == 2
def test_init():
s1 = Symbol(1, 4)
s2 = Symbol(2, 2)
s3 = Symbol(3, 7)
t = Tile({s1, s2, s3})
assert t.symbols == {s1, s2, s3}
assert hasattr(t, "hash")
assert t.leaf_symbol == s3
assert t.leaf_index == 3
assert t.weight == 13
assert t.string == "{1, 2, 3}"
def DistribOR(self, leaf):
if (leaf.symbol.code == "OP_OR"):
l = leaf.left
r = leaf.right
if (l.symbol.code == "OP_AND" and r.symbol.code == "OP_AND"):
ll1 = l.left
lr1 = l.right
rl1 = r.left
rr1 = r.right
leaf.symbol = Symbol("^")
ll2, llt = ll1.DuplicateTree()
self.tree += llt
lr2, lrt = lr1.DuplicateTree()
self.tree += lrt
rl2, rlt = rl1.DuplicateTree()
self.tree += rlt
rr2, rrt = rr1.DuplicateTree()
self.tree += rrt
nsym = Symbol("v")
nll = Leaf(nsym, True, l, ll1, rl1)
self.tree.append(nll)
ll1.upper = nll
rl1.upper = nll
nlr = Leaf(nsym, True, l, ll2, rr1)
self.tree.append(nlr)
ll2.upper = nlr
rr1.upper = nlr
nrl = Leaf(nsym, True, r, lr1, rl2)
self.tree.append(nrl)
lr1.upper = nrl
rl2.upper = nrl
nrr = Leaf(nsym, True, r, lr2, rr2)
self.tree.append(nrr)
lr2.upper = nrr
rr2.upper = nrr
l.left = nll
l.right = nlr
r.left = nrl
r.right = nrr
return True
return False
def testGensymConstruction(self):
self.pe("""(define (make-counter prefix)
(let ((i 0))
(lambda ()
(begin
(set! i (+ i 1))
(string->symbol
(string-append (symbol->string prefix)
(number->string i)))))))""")
self.pe("""(define gensym1 (make-counter 'g))""")
self.pe("""(define gensym2 (make-counter 'h))""")
self.assertEquals(Symbol("g1"), self.pe("(gensym1)"))
self.assertEquals(Symbol("g2"), self.pe("(gensym1)"))
self.assertEquals(Symbol("g3"), self.pe("(gensym1)"))
self.assertEquals(Symbol("h1"), self.pe("(gensym2)"))
def __init__(self, coordinator, hparams, shuffle):
super(DataFeeder, self).__init__()
self._coord = coordinator
self._hparams = hparams
hp = self._hparams
self._offset = 0
self._symbol = Symbol(hp.vocab_path, hp.poly_dict_path)
self.input_dim = self._symbol.input_dim
self.num_class = self._symbol.num_class
self.shuffle = shuffle
with open(hp.data_path, "r") as json_file:
data = json.load(json_file)
self._eval_feature = data["features"][:hp.eval_size]
self._eval_label = data["features"][:hp.eval_size]
self._metadata = list(
zip(data["features"][hp.eval_size:],
data["labels"][hp.eval_size:]))
self.num_samples = len(self._metadata)
self._placeholders = [
tf.placeholder(tf.float32, [None, None, self.input_dim], 'inputs'),
tf.placeholder(tf.int32, [None], 'target_lengths'),
tf.placeholder(tf.float32, [None, None, self.num_class],
'targets'),
tf.placeholder(tf.float32, [None, None, self.num_class],
'poly_mask')
]
# Create queue for buffering data:
self.queue = tf.FIFOQueue(
hp.queue_capacity, [tf.float32, tf.int32, tf.float32, tf.float32],
name='input_queue')
self._enqueue_op = self.queue.enqueue(self._placeholders)
def fn(*x):
x = list(x)
if isinstance(x, list):
x = x[0]
for i in range(len(arg[0])):
d[Symbol(arg[0][i])] = eval(x[i], ChainMap(d))
return eval(block, ChainMap(d))
def op_function(machine):
fun_pad = machine.data_stack.pop()
fun_name = machine.data_stack.pop()
fun_name_w_ctx = '{}.{}'.format(machine.ctx, fun_name)
symbol = Symbol(fun_name_w_ctx, 'function', machine.instruction_ptr, fun_pad)
machine.symbols[fun_name_w_ctx] = symbol
machine.instruction_ptr = machine.instruction_ptr + fun_pad
def setUp(self):
## We set up a VERY minimal environment here for some tests.
## We also set the recursion limit to something dreadful to see that
## call/cc is doing the right thing.
##
## Note: these tests directly work with analyzer.eval, and not
## through the nicer scheme.AnalyzingInterpreter interface.
self.env = extendEnvironment(pair.list(Symbol('pi')),
pair.list(3.1415926),
THE_EMPTY_ENVIRONMENT)
defineVariable(Symbol("#t"), Symbol("#t"), self.env)
defineVariable(Symbol("#f"), Symbol("#f"), self.env)
self.old_recursion_limit = sys.getrecursionlimit()
sys.setrecursionlimit(100)
def __init__(self, screen, font, direction):
self.screen = screen # screen to draw to
self.font = font # font to draw with
self.WIDTH = self.screen.get_size()[0]
self.visible = True
self.Y = random.randint(
0,
self.WIDTH) # Obtain random start position across width of screen
# which way stream flows - "UP" or "DOWN" screen...
if direction == "DOWN":
self.START_HEIGHT = 0 # start = top
self.END_HEIGHT = self.screen.get_size()[1] # end = bottom
self.velocity = random.randint(
VEL_MIN, VEL_MAX) / VEL_DIV # speed at which the rain drops
else:
self.START_HEIGHT = self.screen.get_size()[1] # start = bottom
self.END_HEIGHT = 0 # end = top
self.velocity = -random.randint(
VEL_MIN, VEL_MAX) / VEL_DIV # speed at which the rain rises
# Obtain random length of stream
self.symbols = []
linesize = self.font.get_linesize()
for i in range(1, random.randint(LEN_MIN, LEN_MAX)):
if direction == "DOWN":
x_pos = self.START_HEIGHT - (
(i - 1) * linesize) # start positions above
else:
x_pos = self.START_HEIGHT + (
(i - 1) * linesize) # start positions below
self.symbols.append(
Symbol(self.screen, self.font, i, x_pos, self.Y,
self.velocity))
def defineNativeTypes(self):
for type_ in VALUE_TYPES + REFERENCE_TYPES + [
BuiltInType.VOID, BuiltInType.TUPLE
]:
is_class = type_ in REFERENCE_TYPES
symbol = Symbol(type_, Type(type_, is_class))
self.global_.scope.define(symbol)
def _render_room(blueprint, openings, room, conjunction_point):
# Draw the currently pending room
blt.clear()
room.render(ANIMATION_RENDERER)
blt.bkcolor(colors.CLEAR)
blt.refresh()
time.sleep(ANIMATION_FRAME_LENGTH / 1000)
# If it was added successfully, show the world with its new addition.
if conjunction_point is not None:
blt.clear()
ANIMATION_CAMERA.move_to(conjunction_point[0], conjunction_point[1])
ANIMATION_RENDERER.transform(ANIMATION_CAMERA)
for y in range(len(blueprint)):
for x in range(len(blueprint[0])):
blueprint[y][x].render(x, y, ANIMATION_RENDERER)
for (x, y) in openings:
ANIMATION_RENDERER.render(
x, y, Symbol(' ', blt.color_from_name("white")), 0,
colors.WORLD_GEN_OPENING)
blt.bkcolor(colors.CLEAR)
blt.refresh()
time.sleep(ANIMATION_FRAME_LENGTH / 1000)
ANIMATION_CAMERA.move_to(0, 0)
ANIMATION_RENDERER.transform(ANIMATION_CAMERA)
def add_symbol(self):
symbol = Symbol()
size = len(self.symbols)
if size > 0:
last = self.symbols[-1]
# if the previous tag has started a script or a style,
# do not include the contents of those tags as they
# are large and unnecessary
# NOTE: this does not skip inline tags
if last.type == SymbolType.SCRIPT_START or last.type == SymbolType.STYLE_START:
symbol = self.tokenizer.get_next_symbol()
while (symbol.type == SymbolType.SCRIPT_START
or symbol.type == SymbolType.STYLE_START
or symbol.type == SymbolType.SCRIPT_END
or symbol.type == SymbolType.STYLE_END
or symbol.type == SymbolType.PLAIN_TEXT):
symbol = self.tokenizer.get_next_symbol()
else:
symbol = self.tokenizer.get_next_symbol()
else:
symbol = self.tokenizer.get_next_symbol()
# do not add symbols until a start of the document has been reached
# the start it represented as DOCTYPE tag
if symbol.type != SymbolType.DOCTYPE:
symbol = Symbol.empty()
if not symbol.is_empty():
# might be empty when the end of the HTML has been reached
self.symbols.append(symbol)
def visit_FuncCall(self, node):
if node.name in self.builtins.names:
self.symbol_tree.insert_child(Symbol(node.name))
self.symbol_tree = self.symbol_tree.children[-1]
visited_params = []
for param in node.params.params:
visited_params.append(self.visit(param))
ret = self.builtins.call(node.name, visited_params)
self.symbol_tree = self.symbol_tree.parent
return ret
func = self.symbol_tree.lookup_function(node.name)
if func:
self.symbol_tree.insert_child(IfStatementSymbol(''))
self.symbol_tree = self.symbol_tree.children[-1]
self.visit(node.params)
self.visit(func.exec_block)
self.symbol_tree = self.symbol_tree.parent
return self.call_stack.pop().members['return']
else:
raise Exception('Function definition not found')
def count_ops(self, symbolic=True):
from symbol import Symbol
if symbolic:
return Add(*[t.count_ops(symbolic) for t in self.args]) + \
Symbol('MUL') * (len(self.args) - 1)
return Add(*[t.count_ops(symbolic) for t in self.args]) + \
(len(self.args) - 1)
def env(*args, **kwargs):
"""
Retorna um ambiente de execução que pode ser aproveitado pela função
eval().
Aceita um dicionário como argumento posicional opcional. Argumentos nomeados
são salvos como atribuições de variáveis.
Ambiente padrão
>>> env()
{...}
Acrescenta algumas variáveis explicitamente
>>> env(x=1, y=2)
{x: 1, y: 2, ...}
Passa um dicionário com variáveis adicionais
>>> d = {Symbol('x'): 1, Symbol('y'): 2}
>>> env(d)
{x: 1, y: 2, ...}
"""
kwargs = {Symbol(k): v for k, v in kwargs.items()}
if len(args) > 1:
raise TypeError('accepts zero or one positional arguments')
elif len(args):
if any(not isinstance(x, Symbol) for x in args[0]):
raise ValueError('keys in a environment must be Symbols')
args[0].update(kwargs)
return ChainMap(args[0], global_env)
return ChainMap(kwargs, global_env)
def decode(hparams):
hp = hparams
# data_feeder
coord = tf.train.Coordinator()
feeder = DataFeeder(coord, hp, False)
symbol = Symbol(hp.vocab_path, hp.poly_dict_path)
# construct model
inputs, target_lengths, targets, poly_mask = feeder.dequeue()
model = Poly_Model(hp, feeder.input_dim, feeder.num_class)
model.initialize(inputs, target_lengths, targets, poly_mask)
test_accuracy = model.accuracy
predict_seq = model.pred
saver = tf.train.Saver()
with tf.Session() as sess:
feeder.start_in_session(sess)
sess.run(tf.global_variables_initializer())
if not restore_from_ckpt(sess, saver, hp.save_dir): sys.exit(-1)
test_acc = 0
num_batchs = int(feeder.num_samples / hp.batch_size)
tmp = 0
correct = 0
total = 0
for i in range(num_batchs):
# print(i)
pred, acc, a, c, tag = sess.run([
predict_seq, test_accuracy, model.outputs, targets,
model.target_seq
])
# print(pred)
# print(a)
# print(b)
# print(c)
# print("Pred Seq: {}".format(pred))
# print(a)
# np.savetxt("{}.out".format(tmp), a[0], fmt='%1.4e')
# print(pred)
for pred_poly_seq in symbol.sequence_to_label(pred):
print("\t".join(pred_poly_seq))
for ta in symbol.sequence_to_label(tag):
print("\t".join(ta))
for pred_poly_seq, ta in zip(symbol.sequence_to_label(pred),
symbol.sequence_to_label(tag)):
if len(pred_poly_seq) != len(ta):
print("length not equal")
print("-----------------")
continue
for p, t in zip(pred_poly_seq, ta):
if t == "-":
continue
if p == t:
correct += 1
total += 1
test_acc += acc
tmp += 1
test_acc /= float(num_batchs)
tf.logging.info("Test Accuracy : {}".format(test_acc))
tf.logging.info("Test Accuracy New: {}".format(
float(correct) / float(total)))
def symbol(self):
"""
:return: Symbol
"""
return Symbol(self).setLine(self.line)
def __init__(self, sym, s_date, e_date, path=None):
self.symbol = sym
self.d_data = {}
self.log = Logging()
if not path:
s = Symbol(sym, s_date, e_date)
self.d_data[s.name] = s
