def flatten(node, st=None, inPlace=False):
newChildren = []
newInPlace = None
preStmts = []
# Setup flattening for this node's children.
if util.classGuard(node, Assign, BasicBlock, Module):
newInPlace = node.__class__
elif isinstance(node, Function):
st = node.st
# Flatten our children.
if isinstance(node, While):
if not isinstance(node.cond, Symbol):
condBody, node.cond = flatten(node.cond, st)
node.condBody = BasicBlock(condBody)
bodyPreStmts, node.body = flatten(node.body, st)
preStmts.append(bodyPreStmts)
elif isinstance(node, If):
for child in node:
childPreStmts, newChild = flatten(child, st, newInPlace)
preStmts.append(childPreStmts)
newChildren.append(newChild)
# Set our new child nodes.
newChildren = util.flatten(newChildren)
node.setChildren(newChildren)
else:
for child in node:
childPreStmts, newChild = flatten(child, st, newInPlace)
if isinstance(node, BasicBlock):
newChildren.append(childPreStmts)
newChildren.append(newChild)
else:
preStmts.append(childPreStmts)
newChildren.append(newChild)
# Set our new child nodes.
newChildren = util.flatten(newChildren)
node.setChildren(newChildren)
# Flatten the current node.
if classGuard(node, BinOp, FunctionCall, IfExp, UnaryOp) and not inPlace:
sym = st.getTemp()
preStmts.append(Assign(sym, node))
node = sym
# Flatten our list of pre-statements.
preStmts = util.flatten(preStmts)
return node if isinstance(node, Module) else (preStmts, node)
def flatten(node, st = None, inPlace = False): newChildren = [] newInPlace = None preStmts = [] # Setup flattening for this node's children. if util.classGuard(node, Assign, BasicBlock, Module): newInPlace = node.__class__ elif isinstance(node, Function): st = node.st # Flatten our children. if isinstance(node, While): if not isinstance(node.cond, Symbol): condBody, node.cond = flatten(node.cond, st) node.condBody = BasicBlock(condBody) bodyPreStmts, node.body = flatten(node.body, st) preStmts.append(bodyPreStmts) elif isinstance(node, If): for child in node: childPreStmts, newChild = flatten(child, st, newInPlace) preStmts.append(childPreStmts) newChildren.append(newChild) # Set our new child nodes. newChildren = util.flatten(newChildren) node.setChildren(newChildren) else: for child in node: childPreStmts, newChild = flatten(child, st, newInPlace) if isinstance(node, BasicBlock): newChildren.append(childPreStmts) newChildren.append(newChild) else: preStmts.append(childPreStmts) newChildren.append(newChild) # Set our new child nodes. newChildren = util.flatten(newChildren) node.setChildren(newChildren) # Flatten the current node. if classGuard(node, BinOp, FunctionCall, IfExp, UnaryOp) and not inPlace: sym = st.getTemp() preStmts.append(Assign(sym, node)) node = sym # Flatten our list of pre-statements. preStmts = util.flatten(preStmts) return node if isinstance(node, Module) else (preStmts, node)
def get_shifts(grunid, runid2fn_index, args):
tmp_dir = args[2]
runidstr = str(grunid[0])
fn_shifts_runid = "shifts_runid_" + runidstr + ".npy"
fn_shiftimg_runid = "shiftimg_runid_" + runidstr + ".npy"
queried_fns = np.unique(list(flatten([runid2fn_index[runid] for runid in grunid])))
plot_tiles(queried_fns, args)
# Calculate height difference
if fn_shifts_runid not in os.listdir(tmp_dir):
d_shifts_runid, d_shiftimg_runid = calc_height_diff_grunid(grunid, queried_fns, args,
fn_shifts_runid, fn_shiftimg_runid)
else:
d_shifts_runid = np.load(tmp_dir + fn_shifts_runid, allow_pickle=True)[0]
d_shiftimg_runid = np.load(tmp_dir + fn_shiftimg_runid, allow_pickle=True)[0]
d_shifts_runid = remove_nones(d_shifts_runid)
plot_tiles_hdiff(d_shifts_runid, args)
# For the unknow nearby cell, calculate a height change base on the neigbours
d_shifts_runid = assign_unknown_cells(queried_fns, d_shifts_runid, d_shiftimg_runid,
r, res_update, threshold_count = 140)
plot_tiles_hdiff(d_shifts_runid, args)
return queried_fns, d_shifts_runid
def eliminateDeadStores(node, klass=0):
newChildren = []
if isinstance(node, Class):
klass += 1
for child in node:
newChildren.append(eliminateDeadStores(child, klass))
node.setChildren(flatten(newChildren))
if isinstance(node, Assign):
sym = extractSymbol(node.var)
if sym['reads'] == 0 and klass == 0:
if classGuard(node.exp, Class, FunctionCall):
return node
else:
return None
else:
return node
elif isinstance(node, Phi) and node.target['reads'] == 0:
return None
else:
return node
def compute_all_possible_moves(groups: Dict[Tuple[int, int], int],
shape: Tuple[int, int, int]):
if len(groups.keys()) == LIMIT_NB_GROUPS:
possible_moves_per_group = [
get_moves(group_position, size,
get_neighbors(group_position, shape))
for group_position, size in groups.items()
]
possible_moves: List[Tuple[Tuple[int, int, int, int, int],
...]] = list(
product(*possible_moves_per_group))
return possible_moves
# HACK We need to compute moves for one group max to avoid timeouts
biggest_group_position, biggest_size = sorted(groups.items(),
key=lambda kv: kv[1],
reverse=True)[0]
possible_moves_per_group = [
compute_all_possible_moves_for_one_group(biggest_size,
biggest_group_position, shape)
]
possible_moves: List[Tuple[Tuple[int, int, int, int, int],
...]] = list(product(*possible_moves_per_group))
return [flatten(moves) for moves in possible_moves]
def eliminateDeadStores(node, klass = 0): newChildren = [] if isinstance(node, Class): klass += 1 for child in node: newChildren.append(eliminateDeadStores(child, klass)) node.setChildren(flatten(newChildren)) if isinstance(node, Assign): sym = extractSymbol(node.var) if sym['reads'] == 0 and klass == 0: if classGuard(node.exp, Class, FunctionCall): return node else: return None else: return node elif isinstance(node, Phi) and node.target['reads'] == 0: return None else: return node
def extractStmts(exp): stmts = [] if classGuard(exp, BinOp, Dictionary, List, UnaryOp): for child in exp: if classGuard(child, Statement, FunctionCall): stmts.append(child) else: stmts.append(extractStmts(child)) return flatten(stmts)
def eliminateDeadCode(node): newChildren = [] for child in node: newChild = eliminateDeadCode(child) if isinstance(node, BasicBlock): if classGuard(newChild, Class, Function, FunctionCall, SetAttr, Statement): newChildren.append(newChild) elif isinstance(newChild, BasicBlock): newChildren.append(newChild.children) elif isinstance(newChild, Return): newChildren.append(newChild) # All nodes after the Return node will be discarded due to # unreachability. break else: # Anything that reaches here is an expression outside of a # statement, and therefor has no effect on the program. # Therefor we remove any nested statements from the # expression then throw it away. newChildren.append(extractStmts(newChild)) else: newChildren.append(newChild) node.setChildren(flatten(newChildren)) if classGuard(node, If, IfExp) and isinstance(node.cond, Literal): # If statements or expressions with a literal conditional value can # be eliminated and replaced with the appropriate BasicBlock's # children. return node.then if node.cond.value else node.els else: return node
def simplify(node, st=None):
newChildren = []
preStmts = []
st = node.st if isinstance(node, Function) else st
##############################
# Pre-simplification Clauses #
##############################
if isinstance(node, IfExp):
# Simplify he conditional expression.
condPreStmts, cond = simplify(node.cond, st)
preStmts.append(condPreStmts)
# Create the assignment variable for the different clauses.
sym0 = st.getTemp()
sym1 = st.getSymbol(sym0.name, True)
# Build our new nodes.
then = BasicBlock([Assign(sym0, node.then)])
els = BasicBlock([Assign(sym1, node.els)])
_, if_ = simplify(If(cond, then, els, st), st)
# Append this new If node to our pre-statements and then replace the
# node with the target from the join node's only Phi node.
preStmts.append(if_)
node = if_.jn.phis[0].target
# Simplify the children of this node.
for child in node:
childPreStmts, newChild = simplify(child, st)
if isinstance(node, BasicBlock):
newChildren.append(childPreStmts)
newChildren.append(newChild)
else:
preStmts.append(childPreStmts)
newChildren.append(newChild)
# Set the node's new children.
newChildren = util.flatten(newChildren)
node.setChildren(newChildren)
# Post-simplification clauses.
if isinstance(node, Assign) and isinstance(node.var, Subscript):
node = FunctionCall(st.getBIF('set_subscript'), node.var.symbol,
node.var.subscript, node.exp)
elif isinstance(node, Dictionary):
fun = FunctionCall(st.getBIF('create_dict'))
fun.tag = OBJ
sym = st.getTemp()
preStmts.append(Assign(sym, fun))
name = st.getBIF('set_subscript')
for key in node.value:
# Add the key/value pair to the dictionary.
preStmts.append(FunctionCall(name, sym, key, node.value[key]))
# Replace this node with the symbol that now holds the dictionary.
node = sym
elif isinstance(node, Function) and not node['simplified']:
closure = []
# Mark this node as having been simplified.
node['simplified'] = True
for sym in node['free']:
if sym['heapify'] == 'closure':
closure.append(sym)
# Remove the variables that we have put into the closure from the
# list of free variables for this function.
node['free'] -= set(closure)
preStmts.append(node)
if len(closure) > 0:
closureSym = st.getSymbol('!closure', True)
node.argSymbols.insert(0, closureSym)
# Set up our callTest, substituteTest, and replacement lambda.
callTest = lambda node: True
substituteTest = lambda node: isinstance(node, Symbol
) and node in closure
replacement = lambda node: Subscript(closureSym,
Integer(closure.index(node)))
util.substitute(node, callTest, substituteTest, replacement)
node = FunctionCall(st.getBIF('create_closure'), node.name,
List(closure))
node.tag = OBJ
else:
node = node.name
elif isinstance(node, GetAttr):
node = FunctionCall(st.getBIF('get_attr'), node.exp, node.attrName)
elif isinstance(node, List):
fun = FunctionCall(st.getBIF('create_list'), Integer(len(node.value)))
fun.tag = OBJ
sym = st.getTemp()
preStmts.append(Assign(sym, fun))
name = st.getBIF('set_subscript')
for index in range(0, len(node.value)):
preStmts.append(
FunctionCall(name, sym, Integer(index), node.value[index]))
# Replace this node with the symbol that now holds the list.
node = sym
elif isinstance(node, SetAttr):
node = FunctionCall(st.getBIF('set_attr'), node.exp, node.attrName,
node.value)
elif isinstance(node, Subscript):
# If there is a read from a subscript it needs to be replaced with a
# function call.
funName = st.getBIF('get_subscript')
node = FunctionCall(funName, node.symbol, node.subscript)
# Flatten our list of pre-statements.
preStmts = util.flatten(preStmts)
return node if isinstance(node, Module) else (preStmts, node)
def s3_path(*keys):
keys = util.flatten(keys)
return os.path.join("s3://", *keys)
def declassify(node, st = None, strings = None, klass = None):
newChildren = []
preStmts = []
st = node.st if isinstance(node, Function) else st
strings = node.strings if isinstance(node, Module) else strings
if isinstance(node, Class):
sym = st.getTemp()
fun = FunctionCall(st.getBIF('create_class'), List(node.bases))
fun.tag = OBJ
preStmts.append(Assign(sym, fun))
_, body = declassify(node.body, st, strings, sym)
preStmts.append(body)
node = sym
elif isinstance(node, Assign) and klass:
string = node.var.name
string = strings.setdefault(string, String(string))
node = SetAttr(klass, string, node.exp)
elif isinstance(node, FunctionCall):
if isinstance(node.name, Symbol) and node.name.has_key('type') and node.name['type'] == 'class':
sym = st.getTemp()
string = strings.setdefault('__init__', String('__init__'))
base = node.name
fun = FunctionCall(st.getBIF('create_object'), base)
fun.tag = OBJ
preStmts.append(Assign(sym, fun))
cond = FunctionCall(st.getBIF('has_attr'), base, string)
cond.tag = BOOL
then = BasicBlock([FunctionCall(GetAttr(base, string), sym, *node.args)])
els = BasicBlock([])
preStmts.append(If(cond, then, els, st))
node = sym
elif isinstance(node.name, GetAttr) and not node.name.exp.has_key('type'):
node.args.insert(0, node.name.exp)
if not isinstance(node, Class):
# Declassify the children of this node.
for child in node:
childPreStmts, newChild = declassify(child, st, strings, klass)
if isinstance(node, BasicBlock):
newChildren.append(childPreStmts)
newChildren.append(newChild)
else:
preStmts.append(childPreStmts)
newChildren.append(newChild)
# Set the node's new children.
newChildren = util.flatten(newChildren)
node.setChildren(newChildren)
if isinstance(node, Module):
node.strings = strings
return node if isinstance(node, Module) else (preStmts, node)
def simplify(node, st = None):
newChildren = []
preStmts = []
st = node.st if isinstance(node, Function) else st
##############################
# Pre-simplification Clauses #
##############################
if isinstance(node, IfExp):
# Simplify he conditional expression.
condPreStmts, cond = simplify(node.cond, st)
preStmts.append(condPreStmts)
# Create the assignment variable for the different clauses.
sym0 = st.getTemp()
sym1 = st.getSymbol(sym0.name, True)
# Build our new nodes.
then = BasicBlock([Assign(sym0, node.then)])
els = BasicBlock([Assign(sym1, node.els )])
_, if_ = simplify(If(cond, then, els, st), st)
# Append this new If node to our pre-statements and then replace the
# node with the target from the join node's only Phi node.
preStmts.append(if_)
node = if_.jn.phis[0].target
# Simplify the children of this node.
for child in node:
childPreStmts, newChild = simplify(child, st)
if isinstance(node, BasicBlock):
newChildren.append(childPreStmts)
newChildren.append(newChild)
else:
preStmts.append(childPreStmts)
newChildren.append(newChild)
# Set the node's new children.
newChildren = util.flatten(newChildren)
node.setChildren(newChildren)
# Post-simplification clauses.
if isinstance(node, Assign) and isinstance(node.var, Subscript):
node = FunctionCall(st.getBIF('set_subscript'), node.var.symbol, node.var.subscript, node.exp)
elif isinstance(node, Dictionary):
fun = FunctionCall(st.getBIF('create_dict'))
fun.tag = OBJ
sym = st.getTemp()
preStmts.append(Assign(sym, fun))
name = st.getBIF('set_subscript')
for key in node.value:
# Add the key/value pair to the dictionary.
preStmts.append(FunctionCall(name, sym, key, node.value[key]))
# Replace this node with the symbol that now holds the dictionary.
node = sym
elif isinstance(node, Function) and not node['simplified']:
closure = []
# Mark this node as having been simplified.
node['simplified'] = True
for sym in node['free']:
if sym['heapify'] == 'closure':
closure.append(sym)
# Remove the variables that we have put into the closure from the
# list of free variables for this function.
node['free'] -= set(closure)
preStmts.append(node)
if len(closure) > 0:
closureSym = st.getSymbol('!closure', True)
node.argSymbols.insert(0, closureSym)
# Set up our callTest, substituteTest, and replacement lambda.
callTest = lambda node: True
substituteTest = lambda node: isinstance(node, Symbol) and node in closure
replacement = lambda node: Subscript(closureSym, Integer(closure.index(node)))
util.substitute(node, callTest, substituteTest, replacement)
node = FunctionCall(st.getBIF('create_closure'), node.name, List(closure))
node.tag = OBJ
else:
node = node.name
elif isinstance(node, GetAttr):
node = FunctionCall(st.getBIF('get_attr'), node.exp, node.attrName)
elif isinstance(node, List):
fun = FunctionCall(st.getBIF('create_list'), Integer(len(node.value)))
fun.tag = OBJ
sym = st.getTemp()
preStmts.append(Assign(sym, fun))
name = st.getBIF('set_subscript')
for index in range(0, len(node.value)):
preStmts.append(FunctionCall(name, sym, Integer(index), node.value[index]))
# Replace this node with the symbol that now holds the list.
node = sym
elif isinstance(node, SetAttr):
node = FunctionCall(st.getBIF('set_attr'), node.exp, node.attrName, node.value)
elif isinstance(node, Subscript):
# If there is a read from a subscript it needs to be replaced with a
# function call.
funName = st.getBIF('get_subscript')
node = FunctionCall(funName, node.symbol, node.subscript)
# Flatten our list of pre-statements.
preStmts = util.flatten(preStmts)
return node if isinstance(node, Module) else (preStmts, node)