def help_func_compoundStmt(grammar, tokens):
# Called only by the help_func_manager
# PLACEHOLDER RETURN STATEMENT
print("Compound:", tokens)
tree = Tree()
tree.add_node(Node(tag="Placeholder Compound"), parent=None)
return [tree, 0]
class RegionCollector:
def __init__(self):
self._region_tree = Tree()
"""
GETTER AND SETTER
"""
@property
def region_tree(self):
return self._region_tree
"""
PROPERTY
"""
"""
METHOD
"""
def add_region(self, region, parent=None):
node = Node(identifier=uuid.uuid4(), data=region)
self._region_tree.add_node(node, parent)
def add_regions(self, regions, parent=None):
for region in regions:
if region.bin_pixel('text')[region.bin_pixel('text') == 1].size > 0 \
or len(region.included.text_component().as_list()) > 0:
node = Node(identifier=uuid.uuid4(), data=region)
self._region_tree.add_node(node, parent)
class BacktrackTreelib:
def __init__(self, csp, start_assign, *args, **kwargs):
assert isinstance(csp, CSPSchedule), "problem is not CSP"
assert isinstance(start_assign, Schedule), "assignment is not valid"
self.csp = csp
# self.start_assign = start_assign
self.tree = Tree()
self.root = self.tree("Root", "root", data={})
def solve(self, node_schedule):
if self.csp.satisfied_assignment(node_schedule.assignment):
return node_schedule.assignment
next_var = self.csp.select_next_var(node_schedule.assignment)
if next_var == None:
return None
# here when we expand the childs, we need to fix domain
for child in node_schedule.expand_with_heuristic(
self.csp, node_schedule.assignment, next_var):
self.tree[node_schedule].count += 1
self.tree.add_node(child, node_schedule)
next_node = self.solve(child)
return None
def addNode(tree: Tree, node: Node, parent: Node = None):
"""add existing node to a tree
Args:
tree (Tree): [description]
node (Node): [description]
parent (Node, optional): Defaults to Root.
"""
if parent is None:
parent = tree.root
tree.add_node(node, parent)
def node2TreeOfTwo(self, tuple1, tuple2):
t = Tree()
node1 = Node(tuple1, tuple1[0])
node2 = Node(tuple2, tuple2[0])
freq = tuple1[1] + tuple2[1]
tag = tuple1[0] + tuple2[0]
t.create_node("(" + str(tag) + ", " + str(freq) + ")", tag, None)
if tuple1[1] >= tuple2[1]:
t.add_node(node1, tag)
t.add_node(node2, tag)
else:
t.add_node(node2, tag)
t.add_node(node1, tag)
return t
def node2TreeOfTwo(self,tuple1,tuple2):
t = Tree()
node1 = Node(tuple1,tuple1[0])
node2 = Node(tuple2,tuple2[0])
freq = tuple1[1] + tuple2[1]
tag = tuple1[0] + tuple2[0]
t.create_node("("+str(tag)+", " + str(freq)+")",tag,None)
if tuple1[1] >= tuple2[1]:
t.add_node(node1,tag)
t.add_node(node2,tag)
else:
t.add_node(node2,tag)
t.add_node(node1,tag)
return t
def help_func_return(grammar, tokens, function=None):
tree = Tree()
return_node = Node(tag=tokens[0][1])
tree.add_node(return_node, parent=None)
skip_tokens = 0
if (tokens[1][0] != ';'):
expr_help_out = help_func_expression(grammar,
tokens[1:],
function=function)
tree.paste(return_node.identifier, expr_help_out[0])
skip_tokens = expr_help_out[1]
return [tree, skip_tokens + 2]
def _build_tree(self, scores: ndarray, bin_edges: ndarray) -> Tree:
# Build tree with specified number of children at each level
tree = Tree()
tree.add_node(Node()) # root node
nodes_prev = [tree.get_node(tree.root)]
for level in range(self.depth):
nodes_current = []
for node in nodes_prev:
children = []
for _ in range(self.n_children[level]):
child = Node()
tree.add_node(child, parent=node)
children.append(child)
nodes_current.extend(children)
nodes_prev = nodes_current
assignments = np.digitize(scores, bin_edges) - 1
# Store instance ids in leaves
leaves = tree.leaves()
for k, node in enumerate(leaves):
instance_ids = np.where(assignments == k)[0]
if instance_ids.size == 0:
tree.remove_node(node.identifier)
else:
node.data = instance_ids
# Prune empty leaves
check_for_empty_leaves = True
while check_for_empty_leaves:
check_for_empty_leaves = False
leaves = tree.leaves()
for node in leaves:
if node.data is None and len(node.successors(
tree.identifier)) == 0:
# Node is empty and has no siblings
tree.remove_node(node.identifier)
check_for_empty_leaves = True
# Simplify tree: remove nodes that only have one child
for nid in tree.expand_tree(mode=tree.WIDTH):
children = tree.children(nid)
if len(children) == 1:
tree.link_past_node(nid)
return tree
def run_parser(tokens,
grammar,
look_for_brace=False,
root_name="program",
clear_symbol_table=False):
# Create dictionary of symbol tables
global __symbol_tables
if (clear_symbol_table):
__symbol_tables = {}
# Create base abstract syntax tree
tree = Tree()
# create root node
root = Node(tag=root_name)
tree.add_node(root, parent=None)
num_tokens_to_skip = 0
list_of_tokens = []
for i in range(0, len(tokens)):
if (num_tokens_to_skip > 0):
num_tokens_to_skip -= 1
continue
if (look_for_brace and tokens[i][0] == "}"):
break
list_of_tokens.append(tokens[i]) # append token and metadata
result = check_rules("program", list_of_tokens, grammar)
if (result[0] > 1): #matches more than one possible rule
continue
elif (result[0] == 1): #matches one possible rule
help_fun_tuple = help_func_manager(
result, grammar, tokens[i - len(list_of_tokens) + 1:])
sub_tree = help_fun_tuple[0]
num_tokens_to_skip = help_fun_tuple[1] - len(list_of_tokens)
tree.paste(root.identifier, sub_tree)
#call helper function
list_of_tokens = []
elif (result[0] == 0):
#matches zero rules. parser crash
tree.show(key=lambda x: x.identifier, line_type='ascii')
print("ERRONEOUS RESULT:", result)
print("ERRONEOUS TOKEN LIST:", list_of_tokens)
raise Exception(errors.ERR_NO_RULE + " '" + tokens[i][0] +
"' on line " + str(tokens[i][2]))
return [tree, num_tokens_to_skip, __symbol_tables]
def genTree(node, sList=[]):
nodeType = type(node)
tree = Tree()
if nodeType == pd.core.frame.DataFrame:
df = node
root = tree.create_node(tag='0', identifier=0, data=df)
root.xvar = get_xvar(df)
elif nodeType == Node:
root = deepcopy(node)
root.fpointer = []
root.bpointer = []
df = root.data
tree.add_node(root)
w2 = []
for split in sList:
pid, x, s = split
pNode = tree[pid]
df = pNode.data
dtype = DataTypes[x]
xVal = set(df.iloc[:, x])
if (dtype == 'O') | (dtype == 'u'):
if not (s < xVal): raise IndexError('invalid category split')
S = powerset(xVal)
idx = df.iloc[:, x].isin(s)
elif (dtype == 'i') | (dtype == 'f'):
S = sorted(xVal)[:-1] #leave the last element out
idx = (df.iloc[:, x] <= s)
childDF = [df.loc[idx], df.loc[~idx]]
nodes = [None, None]
cid = array([pid, pid]) * 2 + array([1, 2])
for i in range(2):
if len(childDF[i]) < minObs: raise IndexError('no data in leaf')
nodes[i] = tree.create_node(str(cid[i]),
cid[i],
parent=pid,
data=childDF[i])
nodes[i].var = None
nodes[i].xvar = get_xvar(childDF[i])
pNode.tag = str(pid) + '-' + str(x) + '-' + str(s)
pNode.S = S
pNode.var = x
pNode.split = s
gpid = pNode.bpointer
if gpid in w2: w2.remove(gpid)
w2.append(pid)
tree.w2 = w2
return tree
class BacktrackTreelib:
def __init__(self, csp, start_assign, *args, **kwargs):
assert isinstance(csp, CSPSchedule), "problem is not CSP"
assert isinstance(start_assign, Assignment), "assignment is not valid"
self.csp = csp
self.start_assign = start_assign
self.tree = Tree()
root = NodeSchedule(self.start_assign, tag="Root", identifier="root")
self.tree.add_node(root)
def solve(self, node_schedule, limit=100):
if self.csp.satisfied_assignment(node_schedule.assignment):
return node_schedule
if limit == 0:
return "cutoff"
next_var = self.csp.select_next_var(node_schedule.assignment)
if next_var == None:
return None
cutoff = False
# here when we expand the childs, we need to fix domain
for child in node_schedule.expand_with_heuristic(
self.csp, node_schedule, next_var):
self.tree[node_schedule.identifier].count += 1
if self.tree[node_schedule.identifier].count > 1:
import ipdb
ipdb.set_trace()
# check_feasibility(self.start_assign)
assert False
print("BACKTRACKKKKKKKK")
self.tree.add_node(child, node_schedule)
# print("Depth:{}".format(self.tree.depth()))3
next_node = self.solve(child, limit - 1)
if next_node == "cutoff":
cutoff = True
elif next_node != None:
return next_node
return "cutoff" if cutoff else None
class StateMachine(object):
"""A class to track information about a state machine"""
def __init__(self, name):
self.name = name
self.events = {}
self.effects = {}
self.state_tree = Tree()
self.current_state = None
# Add the Root state automatically
self.add_state('Root')
def add_state(self, name):
assert isinstance(name, str)
state_node = Node(identifier=name, data=State(name))
if self.current_state is None:
self.state_tree.add_node(state_node)
self.current_state = state_node.data
else:
self.state_tree.add_node(state_node, self.current_state.name)
def add_event(self, ev):
assert isinstance(ev, Event)
self.events[ev.name] = ev
def add_effect(self, eff):
assert isinstance(eff, Effect)
self.effects[eff.name] = eff
def enter_state(self, state):
self.current_state = state
def exit_state(self, state):
self.current_state = self.state_tree.parent(state.name).data
def get_state_by_name(self, state_name):
return self.state_tree.get_node(state_name).data
def __init__(self, holes=0):
self.data = np.zeros((3, 3, 3, 3), dtype='int')
element = range(3)
order = direct_product(element, element, element, element)
i = 0
genTree = Tree()
root = Node(i, 'root', data=[order[0], self.data.copy()])
genTree.add_node(root)
currentNode = root
getData = lambda node: node.data[1][tuple(node.data[0])]
while i < len(order):
i += 1
a, b, c, d = order[i - 1]
numPool = pool(self.data, a, b, c, d) - set(
map(getData, genTree.children(currentNode.identifier)))
if numPool:
self.data[a, b, c, d] = np.random.choice(list(numPool))
node = Node(i, data=[order[i - 1], self.data.copy()])
genTree.add_node(node, currentNode)
currentNode = node
else:
prev = genTree.parent(currentNode.identifier)
while len(genTree.children(prev.identifier)) == len(
pool(prev.data[1], *(prev.data[0]))):
currentNode = prev
prev = genTree.parent(currentNode.identifier)
else:
currentNode = prev
self.data = currentNode.data[1].copy()
i = currentNode.tag
continue
h = np.random.choice(len(order), size=holes, replace=False)
self._answer = self.data.copy()
self.holes = np.array(order)[h]
self.data[tuple(self.holes.T.tolist())] = 0
class ZotLib(object):
def __init__(self, library_id, library_type, api_key):
self.z = zotero.Zotero(library_id, library_type, api_key)
self.t = Tree()
self.t.create_node('root', 'root')
self.cols = []
def parse_cols(self):
self.cols = []
top_cols = self.z.collections_top()
for col in top_cols:
col_node = self.make_col_node(col)
self.t.add_node(col_node, 'root')
self.recursive_parse(col_node)
def recursive_parse(self, top_col):
child_cols = self.z.collections_sub(top_col.identifier)
for child in child_cols:
col_node = self.make_col_node(child)
self.t.add_node(col_node, top_col.identifier)
self.recursive_parse(col_node)
def make_col_node(self, col_dict):
col_node = CollectionNode(col_dict)
col_node._items = self.z.collection_items_top(col_node.identifier)
self.cols.append(col_node)
return col_node
def parse_items(self):
for col in self.cols:
for item in col._items:
item_node = ItemNode(item)
try:
self.t.add_node(item_node, parent=col.identifier)
except DuplicatedNodeIdError:
item_node.identifier = str(id(item_node))
if item_node._item_type not in ['attachment', 'note']:
item_node._children = self.z.children(item_node.zot_id)
else:
item_node._children = []
def help_func_funDeclaration(grammar, tokens):
#first token is return type
#the next token is the name
#the third token is the (
#sometime after that should be a
#)
assert (len(tokens) >= 4)
tree = Tree()
# organization nodes
return_node = Node(tag="return_type")
params_node = Node(tag="params")
body_node = Node(tag="func_body")
# create root node
if (tokens[1][0] == "_start"):
raise Exception("Function name _start is reserved for assembly")
elif (tokens[1][0] == "main"):
tokens[1][0] = "_start"
func_name = tokens[1][0]
func_root = Node(tag="func:" + func_name)
return_type = Node(tag=tokens[0][0])
# Create symbol subtable
__symbol_tables[func_name] = {}
# Assemble basic subtree
tree.add_node(func_root, parent=None)
tree.add_node(return_node, parent=func_root)
tree.add_node(params_node, parent=func_root)
tree.add_node(return_type, parent=return_node)
# Create and add params nodes
params = []
var_case = 0 # 0 = empty, 1 = void, 2 = variables
for i in range(3, len(tokens), 3):
if (i == 3 and tokens[i][0] == 'void'):
var_case = 1
break
elif (tokens[i][0] == ")"):
break
else:
try:
params.append((tokens[i][0], tokens[i + 1][0]))
# i+0 = type
# i+1 = name
# i+3 = comma if it exists
var_case = 2
except:
raise Exception(errors.ERR_BAD_FUNC_PAR + " '" + tokens[i][0] +
"' on line " + str(tokens[i][2]))
if (tokens[i + 2][0] != ','):
break
for param in params:
type_node = Node(tag=param[0])
name_node = Node(tag=param[1])
tree.add_node(type_node, parent=params_node)
tree.add_node(name_node, parent=type_node)
#check grammar rules
# Create and add body
body_tokens = []
skip_tokens = 0
if (var_case % 3 == 0):
# Empty parameters
body_tokens = tokens[5:]
skip_tokens = 5
pass
elif (var_case % 3 == 1):
# Void parameter
body_tokens = tokens[6:]
skip_tokens = 6
pass
elif (var_case % 3 == 2):
# Has paremeters
body_tokens = tokens[4 + (3 * (len(params))):]
skip_tokens = 4 + (3 * (len(params)))
pass
#call help_func_block
#parser_out = run_parser(body_tokens, grammar, look_for_brace=True, root_name="func_body") #may be off by one
block_out = help_func_block(grammar,
body_tokens,
root_name="func_body",
function=func_name)
body_tree = block_out[0]
skip_tokens += block_out[1]
tree.paste(func_root.identifier, body_tree)
return [tree, skip_tokens]
class GradientBoostingTree(object):
def __init__(self,
X,
y,
k=4,
epochs=200,
loss='mse',
metrics=['mae'],
learning_rate=0.001,
layers=3,
ending_units=256,
optimizers=[Adam, Nadam, RMSprop],
early_stop=None,
seed=None):
self.X = X
self.y = y
self.k = k
self.epochs = epochs
self.loss = loss
self.metrics = metrics
self.learning_rate = learning_rate
self.layers = layers
self.ending_units = ending_units
self.optimizers = optimizers
self.early_stop = early_stop
self.generator = secrets.SystemRandom(seed)
self.models = []
self.tree = Tree()
def fit(self):
y = self.y
preds = np.zeros(len(y))
def fit_leaf(preds, y, i):
optimizer = self.optimizers[self.generator.randint(
0,
len(self.optimizers) - 1)](learning_rate=self.learning_rate)
model = network_builder(layers=self.layers,
ending_units=self.ending_units)
model.compile(loss=self.loss,
optimizer=optimizer,
metrics=self.metrics)
model.fit(self.X,
y,
epochs=self.epochs * (i + 1),
callbacks=[self.early_stop] if self.early_stop else [])
new_preds = preds + model.predict(self.X).reshape(-1)
new_y = self.y - new_preds
return new_preds, new_y, model
def fit_tree(k, preds, y, i, parent=None):
if k > 0:
l_preds, l_y, l_model = fit_leaf(preds, y, i)
self.tree.add_node(Node(identifier=parent + 'l' + str(k),
data=l_model),
parent=parent)
fit_tree(k - 1, l_preds, l_y, i + 1, parent + 'l' + str(k))
r_preds, r_y, r_model = fit_leaf(preds, y, i)
self.tree.add_node(Node(identifier=parent + 'r' + str(k),
data=r_model),
parent=parent)
fit_tree(k - 1, r_preds, r_y, i + 1, parent + 'r' + str(k))
i = 0
preds, y, root_model = fit_leaf(preds, y, i)
self.tree.add_node(Node(identifier='root', data=root_model))
fit_tree(self.k, preds, y, i + 1, parent='root')
def predict(self, X):
preds = []
leafs = self.tree.leaves('root')
for path in self.tree.paths_to_leaves():
p = .0
for id in path:
p += self.tree.get_node(id).data.predict(X)
preds.append(p)
return sum(preds) / len(leafs)
def main():
""" Solution to Advent of Code Day 7 """
filename = "puzzle_test.txt" if len(sys.argv) < 2 else sys.argv[1]
rules = open(filename).read().splitlines()
# parse input file into a dictionary
# (bag count, bag name)
rules = [
re.sub(r"bags|bag|\.| |contain no other ", "", rule) for rule in rules
]
luggage_dict = dict()
for rule in rules:
relations = rule.split('contain')
if len(relations) > 1:
members = relations[1].replace(',', ' ')
data = re.findall(r"(\d+)(\w+)", members)
luggage_dict[relations[0]] = data
# generate a dictionary of trees mapping each luggage relationship
# (bag count, bag name)
tree_dict = dict()
for data in luggage_dict:
tree_dict[data] = Tree()
root_node = Node(tag=('1', data), data=data)
parent_id = root_node.identifier
tree_dict[data].add_node(root_node)
parent_id = root_node.identifier
for bag in luggage_dict[data]:
tree_dict[data].create_node(tag=bag, data=data, parent=parent_id)
# count every tree that contains a shiny gold bag
print(
"part #1:",
sum([(search_luggage(luggage_dict, key, "shinygold"))
for key in luggage_dict]) - 1)
# build tree of luggage containing shiny gold
items = ['shinygold']
tree = Tree()
node = Node('shinygold', data=int(1))
items = [node]
tree.add_node(node, parent=None)
while items:
parent_node = items.pop()
parent_id = parent_node.identifier
if parent_node.tag not in luggage_dict:
continue
for data in luggage_dict[parent_node.tag]:
node = Node(tag=data[1], data=int(data[0]))
tree.add_node(node, parent=parent_id)
items.append(node)
# preorder traversal to distribute the luggage multiplier
stack = [tree.get_node(tree.root)]
output = []
while stack:
node = stack.pop()
for child_node in tree.children(node.identifier):
child_node.data *= node.data
output.insert(0, node.data)
if tree.children(node.identifier):
stack += reversed(tree.children(node.identifier))
# produce bag total
print("part #2:", sum(output) - 1)
class WarcFileSystem( LoggingMixIn, Operations ):
"""Filesystem built on a WARC's URI paths."""
def __init__( self, warc ):
self.warc = warc
logger.debug( "Mounting %s" % self.warc )
self.fh = WarcRecord.open_archive( warc, gzip="auto", mode="rb" )
self.tree = Tree()
self._get_records()
def _get_records( self ):
"""Parses a WARC, building a hierarchical tree."""
statinfo = os.stat( self.warc )
self.gid = statinfo.st_gid
self.uid = statinfo.st_uid
self.tree.create_node( self.warc, "/" )
self.records = {}
bar = progressbar.ProgressBar( maxval=statinfo.st_size, widgets=[ progressbar.Bar( "=", "[", "]"), " ", progressbar.Percentage() ] )
bar.start()
for( offset, record, errors ) in self.fh.read_records( limit=None ):
if record is not None and record.type != WarcRecord.WARCINFO:
parent = "/"
segments = [ record.type ] + re.split( "/+", record.url )
for e in segments:
identifier = "/".join( [ parent, e ] )
if not self.tree.contains( identifier ):
node = WarcRecordNode( record, offset, tag=e, identifier=identifier )
self.tree.add_node( node, parent=parent )
parent = identifier
self.records[ record.url ] = ( offset, record )
bar.update( offset )
bar.finish()
logger.debug( self.tree.show() )
# def access( self, path, amode ):
# logger.debug( path )
# raise FuseOSError( EPERM )
def chmod( self, path, mode ):
raise FuseOSError( EPERM )
def chown( self, path, uid, gid ):
raise FuseOSError( EPERM )
def create( self, path, mode ):
raise FuseOSError( EPERM )
def destroy( self, path ):
self.fh.close()
# def flush( self, path, fh ):
# raise FuseOSError( EPERM )
def fsync( self, path, datasync, fh ):
raise FuseOSError( EPERM )
def fsyncdir( self, path, datasync, fh ):
raise FuseOSError( EPERM )
def getattr( self, path, fh=None ):
"""Returns stat info for a path in the tree."""
logger.debug( path )
if path == "/":
stat = os.stat( self.warc )
return dict( [
( "st_mode", ( S_IFDIR | 0444 ) ),
( "st_ino", stat.st_ino ),
( "st_dev", stat.st_dev ),
( "st_nlink", stat.st_nlink ),
( "st_uid", stat.st_uid ),
( "st_gid", stat.st_gid ),
( "st_size", stat.st_size ),
( "st_ctime", stat.st_ctime ),
( "st_mtime", stat.st_mtime ),
( "st_atime", stat.st_atime )
] )
else:
return self.name_to_attrs( "/%s" % path )
def getxattr( self, path, name, position=0 ):
"""Returns the value for an extended attribute."""
if path != "/":
path = "/%s" % path
node = self.tree.get_node( path )
if node is None:
raise FuseOSError( ENOENT )
try:
return node.xattrs[ name ]
except KeyError:
raise FuseOSError( ENODATA )
def init( self, path ):
pass
def link( self, target, source ):
raise FuseOSError( EPERM )
def listxattr( self, path ):
"""Returns a list of extended attribute names."""
if path != "/":
path = "/%s" % path
node = self.tree.get_node( path )
if node is None:
raise FuseOSError( ENOENT )
return node.xattrs.keys()
def mkdir( self, path, mode ):
raise FuseOSError( EPERM )
def mknod( self, path, mode, dev ):
raise FuseOSError( EPERM )
def open( self, path, flags ):
"""Should return numeric filehandle; returns file offset for convenience."""
if path != "/":
path = "/%s" % path
node = self.tree.get_node( path )
if node is None:
raise FuseOSError( ENOENT )
return node.offset
# def opendir( self, path ):
# raise FuseOSError( EPERM )
def read( self, path, size, offset, fh ):
"""Reads 'size' data from 'path', starting at 'offset'."""
logger.debug( "read %s from %s at %s " % ( size, path, offset ) )
if path != "/":
path = "/%s" % path
node = self.tree.get_node( path )
if node is None:
raise FuseOSError( ENOENT )
offset += node.payload_offset
mime, data = node.record.content
end = offset + size
return data[ offset:end ]
def name_to_attrs( self, name ):
"""Retrieves attrs for a path name."""
logger.debug( name )
node = self.tree.get_node( name )
if node is None:
raise FuseOSError( ENOENT )
if node.is_leaf():
st_mode = ( S_IFREG | 0444 )
size = node.record.content_length
try:
timestamp = time.mktime( parse( node.record.date ).timetuple() )
except ValueError as v:
logger.warning( "Error parsing time: %s [%s]" % ( node.record.date, str( v ) ) )
timestamp = time.mktime( datetime.fromtimestamp( 0 ).timetuple() )
else:
st_mode = ( S_IFDIR | 0555 )
size = 0
timestamp = time.time()
return dict( [
( "st_mode", st_mode ),
( "st_ino", 0 ),
( "st_dev", 0 ),
( "st_nlink", 0 ),
( "st_uid", self.uid ),
( "st_gid", self.gid ),
( "st_size", size ),
( "st_ctime", timestamp ),
( "st_mtime", timestamp ),
( "st_atime", timestamp )
] )
def readdir( self, path, fh ):
"""Returns a tuple of all files in path."""
logger.debug( path )
if path != "/":
path = "/%s" % path
if self.tree.contains( path ):
names = []
for c in self.tree.get_node( path ).fpointer:
child = self.tree.get_node( c )
names.append( ( child.tag, self.name_to_attrs( child.identifier ), 0 ) )
return names
else:
raise FuseOSError( ENOENT )
def readlink( self, path ):
raise FuseOSError( EPERM )
# def release( self, path, fh ):
# raise FuseOSError( EPERM )
# def releasedir( self, path, fh ):
# raise FuseOSError( EPERM )
def removexattr( self, path, name ):
raise FuseOSError( EPERM )
def rename( self, old, new ):
raise FuseOSError( EPERM )
def rmdir( self, path ):
raise FuseOSError( EPERM )
def setxattr( self, path, name, value, options, position=0 ):
raise FuseOSError( EPERM )
def statfs( self, path ):
raise FuseOSError( EPERM )
def symlink( self, target, source ):
raise FuseOSError( EPERM )
def truncate( self, path, length, fh=None ):
raise FuseOSError( EPERM )
def unlink( self, path ):
raise FuseOSError( EPERM )
def utimens( self, path, times=None ):
raise FuseOSError( EPERM )
def write( self, path, data, offset, fh ):
raise FuseOSError( EPERM )
def node2Tree(self, tuple1, tuple2, tuple3):
t = Tree()
#Id of each node = freq of that node(i[1])
#Node(tag,identifier(ID)) #create_node(tag,identifier(ID),parent)
#Nodes added in decreasing order of Frequency
node1 = Node(tuple1, tuple1[0])
node2 = Node(tuple2, tuple2[0])
node3 = Node(tuple3, tuple3[0])
freq = tuple1[1] + tuple2[1] + tuple3[1]
tag = tuple1[0] + tuple2[0] + tuple3[0]
t.create_node("(" + str(tag) + ", " + str(freq) + ")", tag, None)
#Addded the nodes as left right mid according to there frequency
#so the first node has always the highest frequency
if tuple1[1] >= tuple2[1] and tuple1[1] >= tuple3[1]:
if tuple2[1] >= tuple3[1]:
t.add_node(node1, tag)
t.add_node(node2, tag)
t.add_node(node3, tag)
else:
t.add_node(node1, tag)
t.add_node(node3, tag)
t.add_node(node2, tag)
elif tuple2[1] >= tuple3[1] and tuple2[1] >= tuple1[1]:
if tuple1[1] >= tuple3[1]:
t.add_node(node2, tag)
t.add_node(node1, tag)
t.add_node(node3, tag)
else:
t.add_node(node2, tag)
t.add_node(node3, tag)
t.add_node(node1, tag)
else:
if tuple1[1] >= tuple2[1]:
t.add_node(node3, tag)
t.add_node(node1, tag)
t.add_node(node2, tag)
else:
t.add_node(node3, tag)
t.add_node(node2, tag)
t.add_node(node1, tag)
#t.show()
return t
def node2Tree(self,tuple1,tuple2,tuple3):
t = Tree()
#Id of each node = freq of that node(i[1])
#Node(tag,identifier(ID)) #create_node(tag,identifier(ID),parent)
#Nodes added in decreasing order of Frequency
node1 = Node(tuple1,tuple1[0])
node2 = Node(tuple2,tuple2[0])
node3 = Node(tuple3,tuple3[0])
freq = tuple1[1] + tuple2[1] + tuple3[1]
tag = tuple1[0] + tuple2[0] + tuple3[0]
t.create_node("("+str(tag)+", " + str(freq)+")",tag,None)
#Addded the nodes as left right mid according to there frequency
#so the first node has always the highest frequency
if tuple1[1] >= tuple2[1] and tuple1[1] >= tuple3[1]:
if tuple2[1] >= tuple3[1]:
t.add_node(node1,tag)
t.add_node(node2,tag)
t.add_node(node3,tag)
else:
t.add_node(node1,tag)
t.add_node(node3,tag)
t.add_node(node2,tag)
elif tuple2[1] >= tuple3[1] and tuple2[1] >= tuple1[1]:
if tuple1[1] >= tuple3[1]:
t.add_node(node2,tag)
t.add_node(node1,tag)
t.add_node(node3,tag)
else:
t.add_node(node2,tag)
t.add_node(node3,tag)
t.add_node(node1,tag)
else:
if tuple1[1] >= tuple2[1]:
t.add_node(node3,tag)
t.add_node(node1,tag)
t.add_node(node2,tag)
else:
t.add_node(node3,tag)
t.add_node(node2,tag)
t.add_node(node1,tag)
#t.show()
return t
def hierarchy(fna_mapping, dist):
pending = list(fna_mapping.keys())
node_id = max(pending) + 1
mapping = bidict.bidict()
cls_dist = []
cls_dist_temp = {}
index = 0
pending.sort()
for i in pending:
mapping[i] = index
index += 1
for i in range(0, len(pending)):
temp1 = []
for j in range(0, i):
temp1.append(cls_dist_temp[(mapping[pending[j]],
mapping[pending[i]])])
for j in range(i, len(pending)):
temp = cal_cls_dist(dist, fna_mapping[pending[i]],
fna_mapping[pending[j]])
temp1.append(temp)
cls_dist_temp[(mapping[pending[i]], mapping[pending[j]])] = temp
cls_dist.append([np.array(temp1)])
cls_dist = np.concatenate(cls_dist)
cls_dist_recls = cls_dist.copy()
mapping_recls = mapping.copy()
tree_relationship = {}
pending = set(pending)
while (len(pending) > 1):
(child_a, child_b) = divmod(np.argmax(cls_dist), cls_dist.shape[1])
temp1 = [
np.concatenate([[cls_dist[child_a]],
[cls_dist[child_b]]]).max(axis=0)
]
cls_dist = np.concatenate([cls_dist, temp1], axis=0)
temp1 = np.append(temp1, -1)
temp1 = np.vstack(temp1)
cls_dist = np.concatenate([cls_dist, temp1], axis=1)
cls_dist = np.delete(cls_dist, [child_a, child_b], axis=0)
cls_dist = np.delete(cls_dist, [child_a, child_b], axis=1)
# change mapping
cluster_a = mapping.inv[child_a]
cluster_b = mapping.inv[child_b] # cluster id
tree_relationship[node_id] = (cluster_a, cluster_b)
del mapping[cluster_a], mapping[cluster_b]
pending.remove(cluster_a)
pending.remove(cluster_b)
pending = sorted(list(pending))
for i in pending:
if (mapping[i] > min([child_a, child_b])
and mapping[i] < max([child_a, child_b])):
mapping[i] -= 1
elif (mapping[i] > max([child_a, child_b])):
mapping[i] -= 2
mapping[node_id] = len(cls_dist) - 1
pending = set(pending)
pending.add(node_id)
node_id += 1
# build tree structure
pending = list(pending)
tree = Tree()
T0 = Node(identifier=pending[0])
tree.add_node(T0)
while (len(pending) > 0):
parent = pending[0]
for i in tree_relationship[parent]:
tree.add_node(Node(identifier=i), parent=parent)
if (i in tree_relationship):
pending.append(i)
pending.remove(parent)
# load depth info
depths = {}
depths_mapping = defaultdict(set)
leaves = set(tree.leaves())
for i in tree.all_nodes():
depths[i] = tree.depth(node=i)
if (i in leaves):
depths_mapping[depths[i]].add(i)
return cls_dist_recls, mapping_recls, tree, depths, depths_mapping
next_var = self.csp.select_next_var(node_schedule.assignment)
if next_var == None:
return None
# here when we expand the childs, we need to fix domain
for child in node_schedule.expand_with_heuristic(
self.csp, node_schedule.assignment, next_var):
self.tree[node_schedule].count += 1
self.tree.add_node(child, node_schedule)
next_node = self.solve(child)
return None
# var = self.csp.select_next_var(node_schedule.schedule_assign)
# if not var: return None
# value = self.csp.select_next_value(node_schedule.schedule_assign, var)
if __name__ == "__main__":
# csp = CSPSchedule
# print("put sad wings around me now")
# bb = BacktrackSchedule()
from treelib import Node, Tree
tree = Tree()
root = tree.create_node(1, 1, data={})
node = NodeX("lul", 2, 2)
tree.add_node(node, root)
def init_phase(rubik, steps_required=[]):
tree = Tree()
tree.add_node(Node(identifier=rubik.copy(), tag={'steps': steps_required}))
return tree
def load(path):
gtype = None
graph = None
nodes = []
#TEMP: LOAD FILE FROM PATH#
file = open(path,"r",0)
#TEMP#
gtype = file.next()
gtype = int(gtype)
if (gtype == 0):
graph = Tree()
nodes = []
if (gtype == 1):
graph = digraph()
graph.nodeAttrs = []
graph.edgeAttrs = []
for linenum,line in enumerate(file):
if (linenum == 0):
parts = line.split()
nodeAttrs = parts[1].split(';')
numNodeAttrs = nodeAttrs.pop(0)
graph.nodeAttrs = nodeAttrs
continue
if (linenum == 1):
for node in line.split():
tokens = node.split(';')
index = tokens.pop(0)
if (isInt(index)):
index = int(index)
if (gtype == 0):
nodes.append(Node(identifier=index,data=tokens))
if (index == 0):
graph.add_node(nodes[0])
if (gtype == 1):
graph.add_node(index, attrs=tokens)
continue
if (linenum == 2):
if (gtype == 0):
numEdges = int(line)
if (gtype == 1):
parts = line.split()
lineAttrs = parts[1].split(';')
numLineAttrs = lineAttrs.pop(0)
continue
if (linenum > 2):
#CHECK THAT PARTS ARE INT
parts = line.split()
tail = int(parts[0])
head = int(parts[1])
if (gtype == 0):
graph.add_node(nodes[head],tail)
if (gtype == 1):
attributes = parts[2].split(';')
weight = attributes.pop(0)
graph.add_edge((tail,head),weight,attrs=attributes)
return graph
def help_func_expression(grammar, tokens, function=None):
tokens_skip = 0
# Remove leading and trailing ( and )
while (len(tokens) > 0 and (tokens[0][0] == '(' or tokens[0][0] == ')')):
tokens.pop(0)
tokens_skip += 1
while (len(tokens) > 0 and (tokens[-1][0] == '(' or tokens[-1][0] == ')')):
tokens.pop()
tokens_skip += 1
# Check for subexpression denoted by parentheses
op_depth = []
depth = 0
paren_open = -1
paren_close = -1
for i in range(len(tokens)):
if (tokens[i][0] == ';'):
break # End of expression
elif (tokens[i][0] == '('):
depth += 1
paren_open = i
elif (tokens[i][0] == ')'):
paren_close = i
depth -= 1
op_depth.append(depth)
# Find the lowest precedence operator
lowest_prec_op = []
op_precedence = {
"&&": 50,
"||": 40,
"shiftop": 35,
"mulop": 30,
"sumop": 20,
"relop": 10,
"=": 0,
"+=": 0,
"-=": 0,
"*=": 0,
"\=": 0,
"%=": 0
}
for token in tokens:
if (token[0] == ';'):
break
elif (token[1] in op_precedence):
if (len(lowest_prec_op) == 0):
lowest_prec_op = token
else:
cur_token_depth = op_depth[tokens.index(token)]
lowest_prec_depth = op_depth[tokens.index(lowest_prec_op)]
if (cur_token_depth < lowest_prec_depth):
# Higher depth guarantees replacement
lowest_prec_op = token
elif (cur_token_depth == lowest_prec_depth
and op_precedence[token[1]] <=
op_precedence[lowest_prec_op[1]]):
# To replace, must be on same depth and lower precedence
lowest_prec_op = token
if (len(lowest_prec_op) == 0):
# Check if "expression" is just a single constant
if (len(tokens) > 1 and tokens[0][1] == "ID" and tokens[1][1] == "("):
# Create node with function name
tree = Tree()
if (tokens[0][0] == "_start"):
raise Exception(
"Function name _start is reserved for assembly")
elif (tokens[0][0] == "main"):
tokens[0][0] = "_start"
call_node = Node(tag="func:" + tokens[0][0])
tree.add_node(call_node, parent=None)
tokens_skip += 2
# Children of node are function parameters
# Iterate through tokens to find each parameter
# Parameters split on ',' with depth=0
depth = 0
token_depth = []
end_point = -1
for i in range(2, len(tokens)):
tokens_skip += 1
if (tokens[i][0] == "("):
depth += 1
elif (tokens[i][0] == ")"):
depth -= 1
token_depth.append(depth)
if (depth < 0):
# End ) found
end_point = i
break
if (end_point < 0):
end_point = len(tokens)
#raise Exception("No ending ')' for function call '" +
# tokens[0][0] + "'")
# Find split points for the expressions
split_points = [2]
for i in range(len(token_depth)):
if (token_depth[i] == 0 and tokens[i + 2][0] == ','):
split_points.append(i + 3)
split_points.append(end_point)
func_params = []
for i in range(len(split_points) - 1):
# print([split_points[i], split_points[i+1]])
func_params.append(tokens[split_points[i]:split_points[i + 1]])
# Add parameters to tree
for p in func_params:
# Needs to call expression handler to evaluate parameters
# - Currently, operators in function calls are lower prec than the function for some reason
# - Exceptions caused by nesting function calls
param_node = Node(tag=p[0][0])
tree.add_node(param_node, parent=call_node)
return [tree, tokens_skip]
elif ((tokens[0][1] == "NUMCONST" or tokens[0][1] == "FLOATCONST"
or tokens[0][1] == "CHARCONST" or tokens[0][1] == "STRINGCONST"
or tokens[0][1] == "true" or tokens[0][1] == "false"
or tokens[0][1] == "ID")):
# Check for no-parameter function
if (tokens[0][0] in __symbol_tables.keys()):
# Found a function call without parameters
tokens_skip += 1
tree = Tree()
if (tokens[0][0] == "_start"):
raise Exception(
"Function name _start is reserved for assembly")
elif (tokens[0][0] == "main"):
tokens[0][0] = "_start"
value_node = Node(tag="func:" + tokens[0][0])
tree.add_node(value_node, parent=None)
return [tree, tokens_skip]
else:
# Expression is a constant or named variable
tokens_skip += 1
tree = Tree()
value_node = Node(tag=tokens[0][0])
tree.add_node(value_node, parent=None)
return [tree, tokens_skip]
else:
raise Exception("Unknown token sequence: " + str(tokens))
# Lowest precedence operator found
# Lowest precedence operator is root.
tree = Tree()
op_node = Node(tag=lowest_prec_op[0])
tree.add_node(op_node, parent=None)
# Recursive calls to make left and right subtrees
tokens_skip += 1
tokens_l = tokens[:tokens.index(lowest_prec_op)]
tokens_r = tokens[tokens.index(lowest_prec_op) + 1:]
has_tokens_l = False
for token in tokens_l:
if (token[0] != '(' and token[0] != ')'):
has_tokens_l = True
break
has_tokens_r = False
for token in tokens_r:
if (token[0] != '(' and token[0] != ')'):
has_tokens_r = True
break
if (len(tokens_l) > 0 and has_tokens_l):
expr_l = help_func_expression(grammar, tokens_l, function=function)
tree.paste(op_node.identifier, expr_l[0])
tokens_skip += expr_l[1]
else:
tokens_skip += len(tokens_l)
if (len(tokens_r) > 0 and has_tokens_r):
expr_r = help_func_expression(grammar, tokens_r, function=function)
tree.paste(op_node.identifier, expr_r[0])
tokens_skip += expr_r[1]
else:
tokens_skip += len(tokens_r)
return [tree, tokens_skip]
class Parser:
def __init__(self, file_path):
self.ERROR = 0
self.RIGHT = 1
self.path = file_path
self.lexer = Lexer(file_path)
self.token = Token(Token_Type.ERRTOKEN, "", 0.0, None)
self.state = self.RIGHT
self.count = 0
self.iters = 0
self.origin_x = 0.0
self.origin_y = 0.0
self.rot_ang = 0.0
self.scale_x = 1.0
self.scale_y = 1.0
self.tree = Tree()
self.root = Node()
def typecheck(self, _type):
if (self.token.type != _type):
self.state = self.ERROR
def add_node(self, node_name, parents=None, _data=None):
node = Node(tag=node_name, data=_data)
self.tree.add_node(node, parent=parents)
return node
def getValue(self):
fig = plt.figure()
pic = plt.subplot()
with open(self.path, 'r') as f:
lines = f.readline()
while (lines):
lines = lines.lower()
if (lines.find('pi') != -1):
lines = lines.replace('pi', '3.1415926')
if (lines.find('origin') != -1):
start = lines.find('(')
end = lines.find(',')
endd = lines.find(')')
self.origin_x = eval(lines[start + 1:end])
self.origin_y = eval(lines[end + 1:endd])
elif (lines.find('rot') != -1):
start = lines.find('is')
self.rot_ang = eval(lines[start + 2:-2])
elif (lines.find('scale') != -1):
start = lines.find('(')
end = lines.find(',')
endd = lines.find(')')
self.scale_x = eval(lines[start + 1:end])
self.scale_y = eval(lines[end + 1:endd])
elif (lines.find('for') != -1):
first = lines.find('from')
second = lines.find('to')
third = lines.find('step')
fourth = lines.find('draw')
start = eval(lines[first + 4:second])
end = eval(lines[second + 2:third])
steps = eval(lines[third + 4:fourth])
ax = []
ay = []
l_c = lines.find('(')
comma = lines.find(',')
r_c = lines.rfind(')')
# for iters in range (start, end, steps) :
# t = iters
# ax.append ( eval (lines[l_c + 1 : comma]) )
# ay.append ( eval (lines[comma + 1 : r_c]) )
iters = start
while (iters < end):
t = iters
ax.append(eval(lines[l_c + 1:comma]))
ay.append(eval(lines[comma + 1:r_c]))
iters += steps
ax = np.array(ax)
ay = np.array(ay)
ax = ax * self.scale_x
ay = ay * self.scale_y
temp = ax * np.cos(self.rot_ang) + ay * np.sin(
self.rot_ang)
ay = ay * np.cos(self.rot_ang) - ax * np.sin(self.rot_ang)
ax = temp
ax += self.origin_x
ay += self.origin_y
color = ['blue', 'green', 'yellow', 'red']
ax = ax.tolist()
ay = ay.tolist()
self.count = self.count % 4
pic.scatter(ax, ay, s=2, c=color[self.count])
# plt.show ()
self.count += 1
self.origin_x = 0
self.origin_y = 0
self.scale_x = 1
self.scale_y = 1
self.rot_ang = 0
lines = f.readline()
print(self.origin_x, self.origin_y)
print(self.scale_x, self.scale_y)
print(self.rot_ang)
plt.show()
def program(self):
'''
Program → Statement SEMICO Program |ε
P -> S ; P
'''
# node = Node (tag= 'a')
self.root = Node(tag='Program')
self.token = self.lexer.getToken()
self.tree.add_node(self.root)
node = self.root
while (self.token.type != Token_Type.NONTOKEN):
node1 = Node(tag='Statement')
node2 = Node(tag=';')
node3 = Node(tag='Program')
self.tree.add_node(node1, node)
self.tree.add_node(node2, node)
self.tree.add_node(node3, node)
self.statement(node1)
# self.token = self.lexer.getToken ()
# print (self.token.type)
self.typecheck(Token_Type.SEMICO)
self.token = self.lexer.getToken()
node = node3
if (self.state == self.ERROR):
raise SyntaxError('SyntaxError !')
self.add_node('Empty', node)
print('---------------------Object Tree----------------------')
self.tree.show()
self.getValue()
def statement(self, node):
'''
Statement → OriginStatment | ScaleStatment
| RotStatment | ForStatment
'''
print('--Enter Statement--')
if (self.token.type == Token_Type.ORIGIN):
node_temp = self.add_node('OriginStatment', node)
self.originstatement(node_temp)
elif (self.token.type == Token_Type.SCALE):
node_temp = self.add_node('ScaleStatment', node)
self.scalestatement(node_temp)
elif (self.token.type == Token_Type.ROT):
node_temp = self.add_node('RotStatment', node)
self.rotstatement(node_temp)
elif (self.token.type == Token_Type.FOR):
node_temp = self.add_node('forstatement', node)
self.forstatement(node_temp)
else:
self.state = self.ERROR
print(self.state)
print('--End statement--')
def originstatement(self, node):
'''
OriginStatment → ORIGIN is
L_BRACKET Expression COMMA Expression R_BRACKET
'''
print('--Enter originstatement--')
temp_node = Node(tag=' ')
if (self.token.type == Token_Type.ORIGIN):
temp_node = self.add_node('ORIGIN', node)
self.token = self.lexer.getToken()
if (self.token.type == Token_Type.IS):
self.token = self.lexer.getToken()
temp_node = self.add_node('IS', node)
if (self.token.type == Token_Type.L_BRACKET):
temp_node = self.add_node('L_BRACKET', node)
self.token = self.lexer.getToken()
temp_node = self.add_node('Expression', node)
self.expression(temp_node)
# self.token = self.lexer.getToken ()
self.typecheck(Token_Type.COMMA)
temp_node = self.add_node('COMMA', node)
self.token = self.lexer.getToken()
temp_node = self.add_node('Expression', node)
self.expression(temp_node)
# self.token = self.lexer.getToken ()
if (self.token.type != Token_Type.R_BRACKET):
self.state = self.ERROR
temp_node = self.add_node('R_BRACKET', node)
self.token = self.lexer.getToken()
else:
self.state = self.ERROR
else:
self.state = self.ERROR
print(self.state)
print('--End originstatement--')
def scalestatement(self, node):
'''
ScaleStatment → SCALE IS
L_BRACKET Expression COMMA Expression R_BRACKET
'''
print('--Enter scalestatement--')
temp_node = self.add_node('SCALE', node)
temp_node = self.add_node('IS', node)
temp_node = self.add_node('L_BRACKET', node)
self.token = self.lexer.getToken()
if (self.token.type != Token_Type.IS):
self.state = self.ERROR
self.token = self.lexer.getToken()
if (self.token.type != Token_Type.L_BRACKET):
self.state = self.ERROR
self.token = self.lexer.getToken()
temp_node = self.add_node('Expression', node)
self.expression(temp_node)
temp_node = self.add_node('COMMA', node)
temp_node = self.add_node('Expression', node)
# self.token = self.lexer.getToken ()
self.typecheck(Token_Type.COMMA)
self.token = self.lexer.getToken()
self.expression(temp_node)
# self.token = self.lexer.getToken ()
if (self.token.type != Token_Type.R_BRACKET):
self.state = self.ERROR
temp_node = self.add_node('R_BRACKET', node)
self.token = self.lexer.getToken()
print(self.state)
print('--End scalestatement--')
def rotstatement(self, node):
'''
RotStatment → ROT IS Expression
'''
print('--Enter rotstatement --')
temp_node = self.add_node('ROT', node)
temp_node = self.add_node('IS', node)
temp_node = self.add_node('Expression', node)
self.token = self.lexer.getToken()
if (self.token.type != Token_Type.IS):
self.state = self.ERROR
self.token = self.lexer.getToken()
# print (self.token.type)
self.expression(temp_node)
print(self.state)
print('--End rotstatement--')
def forstatement(self, node):
'''
ForStatment → FOR T
FROM Expression
TO Expression
STEP Expression
DRAW L_BRACKET Expression COMMA Expression R_BRACKET
'''
print('--Enter forstatement--')
temp_node = self.add_node('FOR', node)
temp_node = self.add_node('T', node)
temp_node = self.add_node('FROM', node)
temp_node = self.add_node('Expression', node)
self.token = self.lexer.getToken()
if (self.token.type != Token_Type.T):
self.state = self.ERROR
self.token = self.lexer.getToken()
if (self.token.type != Token_Type.FROM):
self.state = self.ERROR
self.token = self.lexer.getToken()
self.expression(temp_node)
temp_node = self.add_node('TO', node)
temp_node = self.add_node('Expression', node)
# self.token = self.lexer.getToken ()
self.typecheck(Token_Type.TO)
self.token = self.lexer.getToken()
self.expression(temp_node)
temp_node = self.add_node('STEP', node)
temp_node = self.add_node('Expression', node)
# self.token = self.lexer.getToken ()
self.typecheck(Token_Type.STEP)
self.token = self.lexer.getToken()
self.expression(temp_node)
temp_node = self.add_node('DRAW', node)
temp_node = self.add_node('L_BRACKET', node)
temp_node = self.add_node('Expression', node)
# self.token = self.lexer.getToken ()
self.typecheck(Token_Type.DRAW)
self.token = self.lexer.getToken()
self.typecheck(Token_Type.L_BRACKET)
self.token = self.lexer.getToken()
self.expression(temp_node)
temp_node = self.add_node('COMMA', node)
temp_node = self.add_node('Expression', node)
# self.token = self.lexer.getToken ()
self.typecheck(Token_Type.COMMA)
self.token = self.lexer.getToken()
self.expression(temp_node)
temp_node = self.add_node('R_BRACKET', node)
# self.token = self.lexer.getToken ()
self.typecheck(Token_Type.R_BRACKET)
self.token = self.lexer.getToken()
print(self.state)
print('--End forstatement--')
def expression(self, node):
'''
Expression → Term {(PLUS|MINUS)Term }
E -> T {(PLUS | MINUS) T}
'''
print('--Enter expression--')
temp_node = self.add_node('Term', node)
self.term(temp_node)
while (self.token.type == Token_Type.PLUS
or self.token.type == Token_Type.MINUS):
if (token.type == Token_Type.PLUS):
temp_node = self.add_node('PLUS', node, _data='+')
else:
temp_node = self.add_node('MINUS', node, _data='-')
temp_node = self.add_node('Term', node)
self.token = self.lexer.getToken()
self.term(temp_node)
# self.token = self.lexer.getToken ()
print(self.state)
print('--End expression--')
def term(self, node):
'''
Term → Factor { ( MUL | DIV ) Factor }
'''
print('--Enter term--')
temp_node = self.add_node('Factor', node)
self.factor(temp_node)
while (self.token.type == Token_Type.MUL
or self.token.type == Token_Type.DIV):
if (self.token.type == Token_Type.MUL):
temp_node = self.add_node('*', node)
else:
temp_node = self.add_node('/', node)
temp_node = self.add_node('Factor', node)
self.token = self.lexer.getToken()
self.factor(temp_node)
# self.token = self.lexer.getToken ()
print(self.state)
print('--End term--')
def factor(self, node):
'''
Factor → PLUS Factor | MINUS Factor | Component
'''
print('--Enter factor--')
if (self.token.type == Token_Type.PLUS
or self.token.type == Token_Type.MINUS):
if (self.token.type == Token_Type.PLUS):
temp_node = self.add_node('+', node)
else:
temp_node = self.add_node('-', node)
temp_node = self.add_node('Factor', node)
self.token = self.lexer.getToken()
self.factor(temp_node)
else:
# print (self.token.type, self.token.value)
temp_node = self.add_node('Component', node)
self.component(temp_node)
print(self.state)
print('--End Factor--')
def component(self, node):
'''
Component → Atom [POWER Component]
'''
print('--Enter component--')
temp_node = self.add_node('Atom', node)
self.atom(temp_node)
self.token = self.lexer.getToken()
if (self.token.type == Token_Type.POWER):
# self.token = self.lexer.getToken ()
self.token = self.lexer.getToken()
temp_node = self.add_node('POWER', node)
temp_node = self.add_node('Component', node)
self.component(temp_node)
print(self.state)
# print (self.token.type)
print('--End component--')
def atom(self, node):
'''
Atom → CONST_ID
| T
| FUNC L_BRACKET Expression R_BRACKET
| L_BRACKET Expression R_BRACKET
'''
print('--Enter atom--')
if (self.token.type == Token_Type.CONST_ID):
value = self.token.value
print(value)
temp_node = self.add_node('CONST_ID', node, _data=value)
elif (self.token.type == Token_Type.T):
temp_node = self.add_node('T', node)
elif (self.token.type == Token_Type.FUNC):
temp_node = self.add_node('FUNC', node)
temp_node = self.add_node('L_BRACKET', node)
temp_node = self.add_node('Expression', node)
self.token = self.lexer.getToken()
self.typecheck(Token_Type.L_BRACKET)
self.token = self.lexer.getToken()
self.expression(temp_node)
self.typecheck(Token_Type.R_BRACKET)
temp_node = self.add_node('R_BRACKET', node)
elif (self.token.type == Token_Type.L_BRACKET):
temp_node = self.add_node('L_BRACKET', node)
temp_node = self.add_node('Expression', node)
self.token = self.lexer.getToken()
self.typecheck(Token_Type.R_BRACKET)
temp_node = self.add_node('R_BRACKET', node)
else:
self.state = self.ERROR
print(self.state)
print('--End Atom--')
def start(self):
print('Begin !')
self.program()
print('End !')
def help_func_selectionStmt(grammar, tokens):
# Called only by the help_func_manager
# PLACEHOLDER RETURN STATEMENT
tree = Tree()
tree.add_node(Node(tag="Placeholder Selection"), parent=None)
return [tree, 0]
def load(path):
gtype = None
graph = None
nodes = []
#TEMP: LOAD FILE FROM PATH#
file = open(path,"r",0)
#TEMP#
gtype = file.next()
#print isInt(gtype) #ASSERT LINE 0 IS AN INTEGER
gtype = int(gtype)
#print gtype
if (gtype == 0):
graph = Tree()
nodes = []
print 'GRAPH TYPE: TREE'
if (gtype == 1):
graph = digraph()
print 'GRAPH TYPE: EWD'
#print type(graph)
for linenum,line in enumerate(file):
if (linenum == 0):
parts = line.split()
#print parts[0]
nodeAttrs = parts[1].split(';')
numNodeAttrs = nodeAttrs.pop(0)
#print numNodeAttrs
#print isInt(numNodeAttrs)
#print numNodeAttrs, nodeAttrs
continue
if (linenum == 1):
for node in line.split():
tokens = node.split(';')
index = tokens.pop(0)
#print index #in final, assert numNodes == index
#print isInt(index)
if (isInt(index)):
index = int(index)
if (gtype == 0):
nodes.append(Node(identifier=index,data=tokens))
if (index == 0):
#print "ZERO"
#n = nodes[0]
#print type(n)
graph.add_node(nodes[0])
#graph.show()
#print graph.get_node(0)
if (gtype == 1):
graph.add_node(index, attrs=tokens)
#print tokens
continue
if (linenum == 2):
if (gtype == 0):
#print isInt(line) #ASSERT LINE IS A SINGLE INT
numEdges = int(line)
if (gtype == 1):
parts = line.split()
#print parts[0]
lineAttrs = parts[1].split(';')
numLineAttrs = lineAttrs.pop(0)
#print numLineAttrs
#print isInt(numLineAttrs)
#print numLineAttrs, lineAttrs
continue
if (linenum > 2):
#CHECK THAT PARTS ARE INT
parts = line.split()
tail = int(parts[0])
head = int(parts[1])
if (gtype == 0):
graph.add_node(nodes[head],tail)
if (gtype == 1):
attributes = parts[2].split(';')
weight = attributes.pop(0)
#check head and tail are integers
#check number attributes
#print (tail,head), weight, attributes
graph.add_edge((tail,head),weight,attrs=attributes)
if (gtype == 0):
graph.show()
if (gtype == 1):
for node in graph.nodes():
print node, graph.node_attr[node]
for edge in graph.edges():
print edge, graph.edge_weight(edge), graph.edge_attr[edge]
def build_tree(arg):
# read parameters
start = time.time()
dist_matrix_file = arg[0]
cls_file = arg[1]
tree_dir = arg[2]
ksize = arg[3]
params = arg[4]
alpha_ratio = params[0]
minsize = params[1]
maxsize = params[2]
max_cls_size = params[3]
# save genomes info
fna_seq = bidict.bidict() # : 1
fna_path = {}
# read dist matrix (represented by similarity: 1-dist)
# output: dist, fna_path, fna_seq
f = open(dist_matrix_file, "r")
lines = f.readlines()
f.close()
index = 0
d = lines[0].rstrip().split("\t")[1:]
bac_label = 0
for i in lines[0].rstrip().split("\t")[1:]:
temp = i[i.rfind('/') + 1:].split(".")[0]
fna_seq[temp] = index
fna_path[index] = i
index += 1
dist = []
for line in lines[1:]:
dist.append(
[np.array(list(map(float,
line.rstrip().split("\t")[1:])))])
dist = np.concatenate(dist)
# read initial clustering results. fna_mapping, from 1 for indexing
f = open(cls_file, 'r')
lines = f.readlines()
f.close()
fna_mapping = defaultdict(set)
for line in lines:
temp = line.rstrip().split("\t")
for i in temp[2].split(","):
fna_mapping[int(temp[0])].add(fna_seq[i])
if (len(lines) == 1):
tree = Tree()
kmer_sta = defaultdict(int)
T0 = Node(identifier=list(fna_mapping.keys())[0])
tree.add_node(T0)
kmer_sta = defaultdict(int)
kmer_index_dict = bidict.bidict()
kmer_index = 1
alpha_ratio = 1
Lv = set()
for i in fna_mapping[T0.identifier]:
for seq_record in SeqIO.parse(fna_path[i], "fasta"):
temp = str(seq_record.seq)
for k in range(0, len(temp) - ksize):
forward = temp[k:k + ksize]
reverse = seqpy.revcomp(forward)
for kmer in [forward, reverse]:
try:
kmer_sta[kmer_index_dict[kmer]] += 1
except KeyError:
kmer_index_dict[kmer] = kmer_index
kmer_sta[kmer_index] += 1
kmer_index += 1
alpha = len(fna_mapping[T0.identifier]) * alpha_ratio
for x in kmer_sta:
if (kmer_sta[x] >= alpha):
Lv.add(x)
print(T0.identifier, len(Lv))
# save2file
kmerlist = set()
pkl.dump(tree, open(tree_dir + '/tree.pkl', 'wb'))
f = open(tree_dir + "/tree_structure.txt", "w")
os.system("mkdir " + tree_dir + "/kmers")
os.system("mkdir " + tree_dir + "/overlapping_info")
f.write("%d\t" % T0.identifier)
f.close()
os.system(f'cp {cls_file} {tree_dir}/')
f = open(tree_dir + "/reconstructed_nodes.txt", "w")
f.close()
if (len(Lv) > maxsize):
Lv = set(random.sample(Lv, maxsize))
kmerlist = Lv
length = len(Lv)
f = open(tree_dir + "/kmers/" + str(T0.identifier), "w")
for j in Lv:
f.write("%d " % j)
f.close()
f = open(tree_dir + "/node_length.txt", "w")
f.write("%d\t%d\n" % (T0.identifier, length))
kmer_mapping = {}
index = 0
f = open(tree_dir + "/kmer.fa", "w")
for i in kmerlist:
f.write(">1\n")
f.write(kmer_index_dict.inv[i])
kmer_mapping[i] = index
index += 1
f.write("\n")
f.close()
# change index
files = os.listdir(tree_dir + "/kmers")
for i in files:
f = open(tree_dir + "/kmers/" + i, "r")
lines = f.readlines()
if (len(lines) == 0):
continue
d = lines[0].rstrip().split(" ")
d = map(int, d)
f = open(tree_dir + "/kmers/" + i, "w")
for j in d:
f.write("%d " % kmer_mapping[j])
f.close()
end = time.time()
print(
'- The total running time of tree-based indexing struture building is ',
str(end - start), ' s\n')
return
# initially build tree
cls_dist, mapping, tree, depths, depths_mapping = hierarchy(
fna_mapping, dist)
# initially extract k-mers
kmer_index_dict = bidict.bidict()
kmer_index = 1
Lv = defaultdict(set)
spec = defaultdict(set) # k-mers <= alpha
leaves = tree.leaves()
for i in leaves:
kmer_index = extract_kmers(fna_mapping[i.identifier], fna_path, ksize,
kmer_index_dict, kmer_index, Lv, spec,
tree_dir, alpha_ratio, i.identifier)
end = time.time()
print('- The total running time of k-mer extraction is ', str(end - start),
' s\n')
start = time.time()
# leaf nodes check
recls_label = 0
leaves_check = []
check_waitlist = reversed(leaves)
while (True):
if (recls_label):
cls_dist, mapping, tree, depths, depths_mapping = hierarchy(
fna_mapping, dist)
leaves = tree.leaves()
temp = {}
temp2 = []
for i in check_waitlist:
if (i in fna_mapping):
temp2.append(i)
check_waitlist = temp2.copy()
for i in check_waitlist:
temp[tree.get_node(i)] = depths[tree.get_node(i)]
check_waitlist = []
a = sorted(temp.items(), key=lambda x: x[1], reverse=True)
for i in a:
check_waitlist.append(i[0])
for i in fna_mapping:
if (i not in Lv):
kmer_index = extract_kmers(fna_mapping[i], fna_path, ksize,
kmer_index_dict, kmer_index, Lv,
spec, tree_dir, alpha_ratio, i)
higher_union = defaultdict(set)
for i in check_waitlist:
diff, diff_nodes = get_leaf_union(depths[i], higher_union,
depths_mapping, Lv, spec, i)
kmer_t = Lv[i.identifier] - diff
for j in diff_nodes:
kmer_t = kmer_t - Lv[j.identifier]
for j in diff_nodes:
kmer_t = kmer_t - spec[j.identifier]
print(str(i.identifier) + " checking", end="\t")
print(len(kmer_t))
if (len(kmer_t) < minsize):
leaves_check.append(i)
if (len(leaves_check) > 0):
recls_label = 1
else:
break
# re-clustering
check_waitlist = []
while (recls_label == 1):
cluster_id = max(list(fna_mapping.keys())) + 1
check_waitlist.append(cluster_id)
leaf_a = leaves_check[0].identifier
row_index = mapping[leaf_a]
column_index = cls_dist[row_index].argmax()
leaf_b = mapping.inv[column_index] # (leaf_a, leaf_b)
temp2 = fna_mapping[leaf_a] | fna_mapping[leaf_b]
print(cluster_id, leaf_a, leaf_b, temp2)
del fna_mapping[leaf_a], fna_mapping[leaf_b]
if (leaf_a in Lv):
del Lv[leaf_a], spec[leaf_a]
if (leaf_b in Lv):
del Lv[leaf_b], spec[leaf_b]
del leaves_check[0]
if (tree.get_node(leaf_b) in leaves_check):
leaves_check.remove(tree.get_node(leaf_b))
temp1 = [
np.concatenate([[cls_dist[row_index]],
[cls_dist[column_index]]]).max(axis=0)
]
cls_dist = np.concatenate([cls_dist, temp1], axis=0)
temp1 = np.append(temp1, -1)
temp1 = np.vstack(temp1)
cls_dist = np.concatenate([cls_dist, temp1], axis=1)
cls_dist = np.delete(cls_dist, [row_index, column_index], axis=0)
cls_dist = np.delete(cls_dist, [row_index, column_index], axis=1)
# change mapping
del mapping[leaf_a], mapping[leaf_b]
pending = list(fna_mapping.keys())
pending.sort()
for i in pending:
if (mapping[i] > min([row_index, column_index])
and mapping[i] < max([row_index, column_index])):
mapping[i] -= 1
elif (mapping[i] > max([row_index, column_index])):
mapping[i] -= 2
fna_mapping[cluster_id] = temp2
mapping[cluster_id] = len(cls_dist) - 1
if (len(leaves_check) == 0):
break
del higher_union
# rebuild identifiers
all_nodes = tree.all_nodes()
all_leaves_id = set([])
leaves = set(tree.leaves())
for i in leaves:
all_leaves_id.add(i.identifier)
id_mapping = bidict.bidict()
index = 1
index_internal = len(leaves) + 1
for i in all_nodes:
if (recls_label == 0):
id_mapping[i.identifier] = i.identifier
elif (i in leaves):
id_mapping[i.identifier] = index
index += 1
else:
id_mapping[i.identifier] = index_internal
index_internal += 1
leaves_identifier = list(range(1, len(leaves) + 1))
all_identifier = list(id_mapping.values())
all_identifier.sort()
# save2file
f = open(tree_dir + "/tree_structure.txt", "w")
os.system("mkdir " + tree_dir + "/kmers")
os.system("mkdir " + tree_dir + "/overlapping_info")
for nn in all_identifier:
i = id_mapping.inv[nn]
f.write("%d\t" % id_mapping[i])
if (i == all_nodes[0].identifier):
f.write("N\t")
else:
f.write("%d\t" % id_mapping[tree.parent(i).identifier])
if (nn in leaves_identifier):
f.write("N\t")
else:
[child_a, child_b] = tree.children(i)
f.write("%d %d\t" % (id_mapping[child_a.identifier],
id_mapping[child_b.identifier]))
if (len(fna_mapping[i]) == 1):
temp = list(fna_mapping[i])[0]
temp = fna_seq.inv[temp]
f.write("%s" % temp)
f.write("\n")
f.close()
f = open(tree_dir + "/hclsMap_95_recls.txt", "w")
for nn in leaves_identifier:
i = id_mapping.inv[nn]
f.write("%d\t%d\t" % (nn, len(fna_mapping[i])))
temp1 = list(fna_mapping[i])
for j in temp1:
temp = fna_seq.inv[j]
if (j == temp1[-1]):
f.write("%s\n" % temp)
else:
f.write("%s," % temp)
f.close()
end = time.time()
print('- The total running time of re-clustering is ', str(end - start),
' s\n')
start = time.time()
# build indexing structure
kmerlist = set([]) # all kmers used
length = {}
overload_label = 0
if (len(tree.leaves()) > max_cls_size):
overload_label = 1
# from bottom to top (unique k-mers)
uniq_temp = defaultdict(set)
rebuilt_nodes = []
descendant = defaultdict(set) # including itself
ancestor = defaultdict(set)
descendant_leaves = defaultdict(set)
ancestor[all_nodes[0].identifier].add(all_nodes[0].identifier)
for i in all_nodes[1:]:
ancestor[i.identifier] = ancestor[tree.parent(
i.identifier).identifier].copy()
ancestor[i.identifier].add(i.identifier)
for i in reversed(all_nodes):
print(str(id_mapping[i.identifier]) + " k-mer removing...")
if (i in leaves):
uniq_temp[i.identifier] = Lv[i.identifier]
descendant_leaves[i.identifier].add(i.identifier)
else:
(child_a, child_b) = tree.children(i.identifier)
descendant[i.identifier] = descendant[
child_a.identifier] | descendant[child_b.identifier]
descendant_leaves[i.identifier] = descendant_leaves[
child_a.identifier] | descendant_leaves[child_b.identifier]
uniq_temp[i.identifier] = uniq_temp[
child_a.identifier] & uniq_temp[child_b.identifier]
uniq_temp[child_a.identifier] = uniq_temp[
child_a.identifier] - uniq_temp[i.identifier]
uniq_temp[child_b.identifier] = uniq_temp[
child_b.identifier] - uniq_temp[i.identifier]
descendant[i.identifier].add(i.identifier)
all_nodes_id = set(id_mapping.keys())
# remove overlapping
for i in reversed(all_nodes):
print(str(id_mapping[i.identifier]) + " k-mer set building...")
# no difference with sibling, subtree and ancestors
if (i == all_nodes[0]):
kmer_t = uniq_temp[i.identifier]
else:
diff = {}
temp = all_nodes_id - descendant[i.identifier] - set([
tree.siblings(i.identifier)[0].identifier
]) - ancestor[i.identifier]
for j in temp:
diff[j] = len(uniq_temp[j])
a = sorted(diff.items(), key=lambda x: x[1], reverse=True)
kmer_t = uniq_temp[i.identifier]
for j in a:
k = j[0]
kmer_t = kmer_t - uniq_temp[k]
# remove special k-mers
temp = all_leaves_id - descendant_leaves[i.identifier]
diff = {}
for j in temp:
diff[j] = len(spec[j])
a = sorted(diff.items(), key=lambda x: x[1], reverse=True)
for j in a:
k = j[0]
kmer_t = kmer_t - spec[k]
if (len(kmer_t) < minsize and overload_label == 0):
rebuilt_nodes.append(i)
print("%d waiting for reconstruction..." %
id_mapping[i.identifier])
else:
if (len(kmer_t) > maxsize):
kmer_t = set(random.sample(kmer_t, maxsize))
f = open(tree_dir + "/kmers/" + str(id_mapping[i.identifier]), "w")
for j in kmer_t:
f.write("%d " % j)
f.close()
length[i] = len(kmer_t)
kmerlist = kmerlist | kmer_t
del uniq_temp
# rebuild nodes
overlapping = defaultdict(dict)
intersection = defaultdict(set)
higher_union = defaultdict(set)
del_label = {}
for i in leaves:
del_label[i.identifier] = [0, 0]
for i in rebuilt_nodes:
print(str(id_mapping[i.identifier]) + " k-mer set rebuilding...")
kmer_t = get_intersect(intersection, descendant_leaves[i.identifier],
Lv, del_label, i.identifier)
diff = get_diff(higher_union, descendant_leaves, depths, all_nodes, i,
Lv, spec, del_label)
for j in diff:
kmer_t = kmer_t - j
lower_leaves = set([])
for j in leaves:
if (depths[j] < depths[i]):
lower_leaves.add(j)
if (len(kmer_t) > maxsize):
kmer_overlapping_sta = defaultdict(int)
for j in lower_leaves:
kmer_o = Lv[j.identifier] & kmer_t
for k in kmer_o:
kmer_overlapping_sta[k] += 1
temp = sorted(kmer_overlapping_sta.items(),
key=lambda kv: (kv[1], kv[0]))
kmer_t = set([])
for j in range(0, maxsize):
kmer_t.add(temp[j][0])
nkmer = {}
f = open(tree_dir + "/kmers/" + str(id_mapping[i.identifier]), "w")
index = 0
for j in kmer_t:
f.write("%d " % j)
nkmer[j] = index
index += 1
length[i] = len(kmer_t)
kmerlist = kmerlist | kmer_t
# save overlapping info
for j in lower_leaves:
temp = Lv[j.identifier] & kmer_t
if (len(temp) > 0):
ii = id_mapping[i.identifier]
jj = id_mapping[j.identifier]
overlapping[jj][ii] = set([])
for k in temp:
overlapping[jj][ii].add(nkmer[k])
delete(Lv, spec, del_label)
for i in overlapping:
f = open(tree_dir + "/overlapping_info/" + str(i), "w")
f1 = open(tree_dir + "/overlapping_info/" + str(i) + "_supple", "w")
count = -1
for j in overlapping[i]:
if (len(overlapping[i]) != 0):
f.write("%d\n" % j)
for k in overlapping[i][j]:
f.write("%d " % k)
f.write("\n")
count += 2
f1.write("%d %d\n" % (j, count))
f.close()
f1.close()
# final saving
f = open(tree_dir + "/reconstructed_nodes.txt", "w")
for i in rebuilt_nodes:
f.write("%d\n" % id_mapping[i.identifier])
f.close()
f = open(tree_dir + "/node_length.txt", "w")
for nn in all_identifier:
i = id_mapping.inv[nn]
f.write("%d\t%d\n" % (nn, length[tree[i]]))
f.close()
kmer_mapping = {}
index = 0
f = open(tree_dir + "/kmer.fa", "w")
for i in kmerlist:
f.write(">1\n")
f.write(kmer_index_dict.inv[i])
kmer_mapping[i] = index
index += 1
f.write("\n")
f.close()
# change index
files = os.listdir(tree_dir + "/kmers")
for i in files:
f = open(tree_dir + "/kmers/" + i, "r")
lines = f.readlines()
if (len(lines) == 0):
continue
d = lines[0].rstrip().split(" ")
d = map(int, d)
f = open(tree_dir + "/kmers/" + i, "w")
for j in d:
f.write("%d " % kmer_mapping[j])
f.close()
end = time.time()
print(
'- The total running time of tree-based indexing struture building is ',
str(end - start), ' s\n')
def init_phase(rubik, steps_required = []):
tree = Tree()
tree.add_node(Node(identifier = rubik.copy(), tag = {'steps': steps_required}))
return tree
def help_func_block(grammar, tokens, root_name="block", function=None):
#go line by line
#if }
#return tree
#if {
#recursive help_func_block
#grab up to till first ;
#call expression handeler on that sub list
#returns a tree which is appended
tree = Tree()
root_node = Node(tag=root_name)
tree.add_node(root_node, parent=None)
func_flag_no_init = 0
func_flag_init = 0
func_flag = 0
front_index = 0
num_tokens_to_skip = 0
i = 0
while (i < len(tokens)):
if (tokens[i][0] == "}"):
return [tree, num_tokens_to_skip + 1]
elif (tokens[i][0] == "{"):
result = help_func_block(grammar,
tokens[i + 1:],
function=function)
front_index += 1 + result[1]
i += 1 + result[1]
num_tokens_to_skip += 1 + result[1]
tree.paste(root_node.identifier, result[0])
elif (tokens[i][0] in ["if", "while"]):
if_node = Node(tag=tokens[i][0])
tree.add_node(if_node, parent=root_node)
if_cond = Node(tag="condition")
tree.add_node(if_cond, parent=if_node)
first_bracket = -1
for token in tokens[i:]:
if (token[0] == '{'):
first_bracket = tokens.index(token)
break
elif (token[0] == '}'):
# Break to throw error if unmatched
break
if (first_bracket < 0):
raise Exception(tokens[i][0] + " without body '{' on line " +
str(tokens[i][2]))
cond_result = help_func_expression(grammar,
tokens[i + 2:first_bracket - 1],
function=function)
body_result = help_func_block(grammar,
tokens[first_bracket + 1:],
root_name="condition_body",
function=function)
# Increment i, num_tokens_to_skip, and front_index
if_skip = 1 # if/while
if_skip += 1 # opening bracket
if_skip += 2 # parens
if_skip += cond_result[1]
if_skip += body_result[1]
num_tokens_to_skip += if_skip
front_index += if_skip
i += if_skip
tree.paste(if_cond.identifier, cond_result[0])
tree.paste(if_node.identifier, body_result[0])
elif (tokens[i][0] == "return"):
result = help_func_return(grammar, tokens[i:], function=function)
front_index += result[1]
i += result[1]
num_tokens_to_skip += result[1]
tree.paste(root_node.identifier, result[0])
elif (tokens[i][0] == ";"):
back_index = i
expr_tokens = tokens[front_index:back_index]
# Remove leading and trailing ( and )
while (len(expr_tokens) > 0
and (expr_tokens[0][0] == '(' or expr_tokens[0][0] == ')')):
expr_tokens.pop(0)
num_tokens_to_skip += 1
while (len(expr_tokens) > 0 and
(expr_tokens[-1][0] == '(' or expr_tokens[-1][0] == ')')):
expr_tokens.pop(-1)
num_tokens_to_skip += 1
if (len(expr_tokens) > 0):
if (len(expr_tokens) == 2
and expr_tokens[0][1] == 'typeSpecifier'
and expr_tokens[1][1] == 'ID'):
func_flag = 1
func_flag_no_init = 1
# print("This is a variable declaration with no intilization")
var_type = expr_tokens[0][0]
var_name = expr_tokens[1][0]
__symbol_tables[function][var_name] = var_type
expr_tokens.pop(0)
elif (len(expr_tokens) > 2
and expr_tokens[0][1] == 'typeSpecifier'
and expr_tokens[1][1] == 'ID'
and expr_tokens[2][1] == '='):
func_flag = 1
# print("This is a variable declaration with intilization")
var_type = expr_tokens[0][0]
var_name = expr_tokens[1][0]
__symbol_tables[function][var_name] = var_type
expr_tokens.pop(0)
if (func_flag == 1):
tmp_tree = Tree()
tmp_tree_root = Node(tag=var_type)
tmp_tree.add_node(tmp_tree_root, parent=None)
tmp_tree.add_node(Node(tag=var_name), parent=tmp_tree_root)
result = help_func_expression(grammar,
expr_tokens,
function=function)
if (func_flag == 1):
result[1] += 1
front_index = back_index + 1
i += 1
num_tokens_to_skip += 1 + result[1]
if (func_flag_no_init != 1):
tree.paste(root_node.identifier, result[0])
if (func_flag == 1):
# pass
tree.paste(root_node.identifier, tmp_tree)
func_flag_no_init = 0
func_flag_init = 0
func_flag = 0
tmp_tree = None
else:
i += 1
# Iterated through tokens without closing '}'
raise Exception(errors.ERR_NO_BLOCK_END + " on line " +
str(tokens[i - 1][2]))
class MCTS_better():
def __init__(self,
confidence=CONFIDENCE,
time=MAX_CAL_TIME,
max_actions=1000):
self.max_cal_time = float(time)
self.max_actions = max_actions
self.confidence = confidence
self.max_depth = 0
def get_move(self, board, player):
self.board = board
self.player = player # The chess color [BLACK or WHITE] represent the player
empty_set, a, b = self.board.get_board_item()
if len(a) == 0 and len(b) == 0:
return (MIDDLE, MIDDLE)
if len(empty_set) == 1:
return (empty_set[0][0], empty_set[0][1])
if len(empty_set) == 0:
print("No place to play")
return None
self.MCTS_tree = Tree()
self.HeadNode = Node('HeadNode', 0)
self.MCTS_tree.add_node(self.HeadNode)
self.plays = {}
self.wins = {}
simulations = 0
start = time.time()
while time.time() - start < (self.max_cal_time - 0.5):
board_for_MCTS = copy.deepcopy(self.board)
player_for_MCTS = self.player
self.run_simulation(board_for_MCTS, player_for_MCTS)
simulations += 1
print("total simuations = ", simulations)
move = self.select_best_move()
print("MCTS move:", move[0], move[1])
return move
def run_simulation(self, board, player):
tree = self.MCTS_tree
node = self.HeadNode
availables = board.get_k_dist_empty_tuple(2)
visited_states = set()
winner = -1
expand = True
# Simulation Start
for t in range(1, self.max_actions + 1):
availables = board.get_k_dist_empty_tuple(2)
children = tree.children(node.identifier)
self.plays = {}
self.wins = {}
plays = self.plays
wins = self.wins
for n in children:
plays[n.tag] = n.data[0]
wins[n.tag] = n.data[1]
# Selection
noused_set = self.select_noused_node(availables, player)
if len(noused_set) == 0:
#print("ok")
#print(sum(plays[(player,move)] for move in availables))
log_total = log(
sum(plays[(player, move)] for move in availables))
value, move = max(
((wins[(player, move)] / plays[(player, move)]) +
sqrt(self.confidence * log_total / plays[(player, move)]),
move) for move in availables)
#print(move)
for n in children:
if n.tag == (player, move):
node = n
else:
#print('good')
random.shuffle(noused_set)
move = noused_set.pop()
new_node = Node(tag=(player, move), data=[0, 0])
tree.add_node(new_node, parent=node)
node = new_node
board.draw_xy(move[0], move[1], player)
#Expand
# if expand and (player,move,t) not in plays:
# expand = False
# plays[(player,move,t)] = 0
# wins[(player,move,t)] = 0
# if t > self.max_depth:
# self.max_depth = t
#visited_states.add((player,move))
availables = board.get_k_dist_empty_tuple(2)
is_full = not len(availables)
winner = board.anyone_win(move[0], move[1])
if winner is not EMPTY or is_full:
#print(str(move) + '----' + str(winner) + '-----' + str(player))
break
player = self.player_change(player)
while (node.is_root() == False):
if winner == self.player:
node.data[1] += 1
node.data[0] += 1
node = tree.parent(node.identifier)
def select_best_move(self):
empty_set = self.board.get_k_dist_empty_tuple(2)
# for move in empty_set:
# ratio = (self.wins.get((self.player,move),0)/
# self.plays.get((self.player,move),1))
# print(move)
# print(ratio)
#print(empty_set)
self.plays = {}
self.wins = {}
plays = self.plays
wins = self.wins
children = self.MCTS_tree.children(0)
for n in children:
plays[n.tag] = n.data[0]
wins[n.tag] = n.data[1]
# print(plays)
# print(wins)
raio_to_win, move = max(
(self.wins.get((self.player, move), 0) /
self.plays.get((self.player, move), 1) + self.closest_value(move),
move) for move in empty_set)
print(raio_to_win)
return move
def closest_value(self, move):
x = move[0]
y = move[1]
return (abs((x - N_LINE + 1) * x) + abs((y - N_LINE + 1) * y)) * 0.0001
def player_change(self, player):
if player == BLACK:
player = WHITE
elif player == WHITE:
player = BLACK
return player
def select_noused_node(self, availables, player):
noused_set = []
for move in availables:
if not self.plays.get((player, move)):
noused_set.append(move)
return noused_set
def help_func_varDeclaration(grammar, tokens):
# Called only by the help_func_manager
# PLACEHOLDER RETURN STATEMENT
tree = Tree()
tree.add_node(Node(tag="Placeholder Var Dec"), parent=None)
return [tree, 0]
class DMNSPolicy(Policy):
"""
Representation of a deterministic markovian non-stationary policy.
Deterministic markovian means that the the act method returns only an action, and that the relevant history consists
only in the current state.
Attributes
__________
policy: dict
A dictionary with the deterministic markovian policy.
tree: treelib.Tree
a tree object for visualization purposes.
"""
def __init__(self, space):
super().__init__(space)
def __repr__(self):
st = str(self.tree.show())
return st
def _create_tree(self, initial_state):
"""
A tree object for visualization purposes is created.
Parameters
----------
initial_state: state
An initial state
"""
self.tree = Tree()
self.tree.add_node(
Node(f'({0}:{initial_state}:{self.policy[(0, initial_state)]})',
f'({0}:{initial_state}:{self.policy[(0, initial_state)]})'))
def add_sons(s, t):
a = self.policy[(t, s)]
if t == self.T:
for st in self.Q(s, a):
n = Node(f'({t + 1}:{st})', f'({t + 1}:{st})')
self.tree.add_node(node=n, parent=f'({t}:{s}:{a})')
elif t < self.T - 1:
for st in self.Q(s, a):
at = self.policy[(t + 1, st)]
n = Node(f'({t + 1}:{st}:{at})', f'({t + 1}:{st}:{at})')
if n.identifier not in map(lambda x: x.identifier,
self.tree.all_nodes()):
self.tree.add_node(node=n, parent=f'({t}:{s}:{a})')
add_sons(st, t + 1)
add_sons(initial_state, 0)
def act(self, history):
"""
Parameters
----------
history
Returns
-------
"""
time, state = history
return self.policy[(time, state)]
def add_action(self, time, state, action):
"""
Add an action to the policy.
Parameters
----------
time: int
Time
state: State
State
action: Action
Action
"""
# assert state in self.S and action in self.A
self.policy[time, state] = action
def add_policy(self, policy, initial_state=None):
"""
Adds a complete policy
Parameters
----------
policy: dict
Policy to add
initial_state: state
(Optional)
If the initial state is given, a tree for visualization purposes is created.
"""
self.policy = policy
if initial_state is not None:
self._create_tree(initial_state)
def main():
# Create dummy AST
ast_dummy = Tree()
ast_root = Node(tag="program")
ast_dummy.add_node(ast_root, parent=None)
# Create nodes for add_int()
func_root = Node(tag="func:add_int")
func_type = Node(tag="return_type")
func_param = Node(tag="params")
func_body = Node(tag="func_body")
ast_dummy.add_node(func_root, parent=ast_root)
ast_dummy.add_node(func_type, parent=func_root)
ast_dummy.add_node(func_param, parent=func_root)
ast_dummy.add_node(func_body, parent=func_root)
### add_int(): return and params
return_type = Node(tag="int")
ast_dummy.add_node(return_type, parent=func_type)
param_type = Node(tag="int")
ast_dummy.add_node(param_type, parent=func_param)
ast_dummy.add_node(Node(tag="x"), parent=param_type)
param_type = Node(tag="int")
ast_dummy.add_node(param_type, parent=func_param)
ast_dummy.add_node(Node(tag="y"), parent=param_type)
### add_int(): body
body_node_0 = Node(tag="return")
ast_dummy.add_node(body_node_0, parent=func_body)
body_node_1 = Node(tag="+")
ast_dummy.add_node(body_node_1, parent=body_node_0)
body_node_2 = Node(tag="x")
ast_dummy.add_node(body_node_2, parent=body_node_1)
body_node_2 = Node(tag="y")
ast_dummy.add_node(body_node_2, parent=body_node_1)
# Create nodes for main()
func_root = Node(tag="func:main")
func_type = Node(tag="return_type")
func_param = Node(tag="params")
func_body = Node(tag="func_body")
ast_dummy.add_node(func_root, parent=ast_root)
ast_dummy.add_node(func_type, parent=func_root)
ast_dummy.add_node(func_param, parent=func_root)
ast_dummy.add_node(func_body, parent=func_root)
### add_int(): return and params
return_type = Node(tag="int")
ast_dummy.add_node(return_type, parent=func_type)
### add_int(): body
body_node_0 = Node(tag="return")
ast_dummy.add_node(body_node_0, parent=func_body)
body_node_1 = Node(tag="add_int")
ast_dummy.add_node(body_node_1, parent=body_node_0)
body_node_2 = Node(tag="1")
ast_dummy.add_node(body_node_2, parent=body_node_1)
body_node_2 = Node(tag="2")
ast_dummy.add_node(body_node_2, parent=body_node_1)
# Print dummy AST
print("=== DUMMY AST ===")
ast_dummy.show(key=lambda x: x.identifier, line_type='ascii')
# Convert dummy AST into LLVM
llvm_ir = build_llvm(ast_dummy)
# Output result
print("=== LLVM IR OUTPUT ===")
print(llvm_ir)
