def get_children(_sc, position, graph): """ Returns the children of a position. Arguments: _sc : SparkContext The Spark Context configurations. position : list A position. Example: ('A', 'B', 'C', '-') graph : RDD RDD Object wich represents the graph. """ # print type(position) if "tuple" not in str(type(position)): print "not tuple" children = [Sliding.children(w, h, x) for x in position.collect()] print "start parallelizing" children = _sc.parallelize(sum(children, [])) print "children parallelized" return children children = _sc.parallelize(Sliding.children( w, h, position)) # obtain the children from position obtained return children
def bfs_map(value): """ YOUR CODE HERE """ level += 1 # get children, make tuples, make a list resultList = [] result.append(value) #ensure parent is linked with children below result.append( Sliding.children().flatMap( lambda x: (x,level) ) )#get first item of board
def bfs_map(value): """ YOUR CODE HERE """ while curr level has nodes: if level += 1 RDD = RDD.flatMap(lambda x: (x, level), Sliding.children()) bfs_map(each node)
def slidingBfsSolver(puzzle, width, height, max_level=-1): """ BF visita todo el grafo del puzzle, construye las estructuras: * level_to_pos * pos_to_level """ solution = puzzle # Solucion del puzzle level = 0 # La solucion es level 0 level_to_pos[level] = [solution] # el level 0 solo consiste de la solucion pos_to_level[solution] = level # Mientras existan posiciones en el level de la frontera while level_to_pos[level] and (max_level == -1 or level < max_level): level += 1 # Incrementamos el level level_to_pos[level] = [] # Creamos una lista vacia en el nuevo level # Para cada posicion en el ultimo level (antes de aumentarlo) for position in level_to_pos[level - 1]: # Para cada hijo de cada posicion del for de arriba for child in Sliding.children(width, height, position): # Si es primera vez que miramos al child if child not in pos_to_level: # Actualizamos los mappings para recordarlo, y que van a # ser parte de la nueva frontera pos_to_level[child] = level level_to_pos[level].append(child) del level_to_pos[level] # El ultimo level siempre esta vacio, lo eliminan pprint(level_to_pos)
def bfs_map(value): """ YOUR CODE HERE """ return_list = [value] children = Sliding.children(HEIGHT, WIDTH, Sliding.hash_to_board(WIDTH, HEIGHT, value[0])) for child in children: return_list.append((Sliding.board_to_hash(WIDTH, HEIGHT, child), value[1]+1)) return return_list
def bfs_flat_map(state): """ """ self_list = [state] if get_level(state) == level-1: #expand children if state is on current (highest) level children = Sliding.children(WIDTH, HEIGHT, get_board(state)) return [make_state(level, board) for board in children] + self_list return self_list
def press_map(data): """ YOUR CODE HERE """ # get children output list of them if level == data[1]: # form k,v pairs from children return [data] + [(k, level+1) for k in Sliding.children(WIDTH,HEIGHT,data[0])] else: return [data]
def bfs_flatmap(board): if board[1] == (level - 1): children_list = Sliding.children(WIDTH, HEIGHT, board[0]) level_list = [level for _ in range(len(children_list))] children_list = zip(children_list, level_list) children_list.append(board) return children_list return [board]
def bfs_map(value): # value is a (position, level) pair result = [value] if value[1] == level: # if the node is at frontier for child in Sliding.children(WIDTH, HEIGHT, value[0]): result.append((child, level + 1)) return result
def bfs_map(value): """ YOUR CODE HERE """ result = [] if value[1] == level - 1: result = Sliding.children(WIDTH, HEIGHT, value[0]) for i in range(0, len(result)): result[i] = (result[i], level) result.append(value) return result
def bfs_map(value): #value is the (puzzle, level) tuple """ YOUR CODE HERE """ lst = [(value)] if value[1] == level: children = Sliding.children(WIDTH,HEIGHT,Sliding.hash_to_board(WIDTH, HEIGHT, value[0])) for child in children: lst.append((Sliding.board_to_hash(WIDTH, HEIGHT, child),level+1)) return lst return lst
def bfs_map(value): """ YOUR CODE HERE """ mapped=[value] if level<=value[1] children = Sliding.children(WIDTH, HEIGHT, value[0]) for pos in children: val=value[1]+1 mapped.append([pos,val]) return mapped
def bfs_map(value): """ YOUR CODE HERE """ result = [] if value[1] == level - 1: result = Sliding.children(WIDTH, HEIGHT, Sliding.hash_to_board(WIDTH, HEIGHT, value[0])) for i in range(0, len(result)): result[i] = (Sliding.board_to_hash(WIDTH, HEIGHT, result[i]), level) result.append(value) return result
def bfs_map(value): """ YOUR CODE HERE """ result = [] if value[1] == level - 1: result = Sliding.children(WIDTH, HEIGHT, value[0]) for i in range(0, len(result)): result[i] = (result[i], level) result.append(value) return result
def bfs_map(value): """ YOUR CODE HERE """ # Add board's children to global position list if value[1] == level: children = Sliding.children(WIDTH, HEIGHT, value[0]) for i in range(0, len(children)): positions.append((children[i], level + 1)) # level += 1 return positions
def bfs_map(value): """ YOUR CODE HERE """ # Add board's children to global position list if value[1] == level: children = Sliding.children(WIDTH, HEIGHT, value[0]) for i in range(0, len(children)): positions.append((children[i], level+1)) # level += 1 return positions
def bfs_map(value): """ YOUR CODE HERE """ global level re_list = [] if value[1] == level - 1: childrenlist = Sliding.children(WIDTH,HEIGHT,value[0]) for child in childrenlist: re_list.append([child, level]) return [value] + re_list
def bfs_map(value): """ YOUR CODE HERE """ mapVal = [] mapVal.append((value[0], value[1])) if value[1] == level: pos = Sliding.hash_to_board(WIDTH, HEIGHT, value[0]) for cpos in Sliding.children(WIDTH, HEIGHT, pos): cpos2 = Sliding.board_to_hash(WIDTH, HEIGHT, cpos) mapVal.append((cpos2,level+1)) return mapVal
def bfs_map(value): items = [] if value[1] < level: items.append((value[0],value[1])) if value[1] == level-1: children_board = Sliding.hash_to_board(WIDTH, HEIGHT, value[0]) children = Sliding.children(WIDTH, HEIGHT, children_board) for child in children: items.append((Sliding.board_to_hash(WIDTH, HEIGHT, child), value[1] + 1)) return items
def bfs_map(value): """ YOUR CODE HERE """ if (value[1] != (level - 1)): return [value] else: children = Sliding.children(WIDTH, HEIGHT, Sliding.hash_to_board(WIDTH,HEIGHT,value[0])) childList = [value] for child in children: childList.append((Sliding.board_to_hash(WIDTH,HEIGHT,child), level)) return childList
def bfs_map(value): """ children = Sliding.children(WIDTH, HEIGHT, value[0]) items = [] for child in children: items.append((child, value[1] + 1)) return items """ child_boards = Sliding.children(WIDTH, HEIGHT, value[0]) return [(child, value[1] + 1) for child in child_boards]
def bfs_map(value): """ YOUR CODE HERE """ if (value[1] != (level - 1)): return [value] else: children = Sliding.children(WIDTH, HEIGHT, value[0]) childList = [value] for child in children: childList.append((child, level)) return childList
def bfs_map(arg): """ YOUR CODE HERE """ if arg[1] == level: children = Sliding.children(WIDTH, HEIGHT, arg[0]) toreturn = [arg] for position in children: toreturn.append((position, level + 1)) return toreturn else: return [arg]
def bfs_map(value): """ YOUR CODE HERE """ #value is the tuple lst = [(value)] if value[1] == level: children = Sliding.children(WIDTH,HEIGHT,value[0]) #list of children for child in children: lst.append((child,level+1)) return lst return lst
def bfs_map(value): """ YOUR CODE HERE """ prev = [(value[0], value[1])] if value[1] == level: children = Sliding.children(WIDTH, HEIGHT, value[0]) curr = [] for i in range(0, len(children)): curr.append((children[i], level+1)) return prev + curr return prev
def bfs_map(value): """ YOUR CODE HERE """ prev = [(value[0], value[1])] if value[1] == level: children = Sliding.children(WIDTH, HEIGHT, value[0]) curr = [] for i in range(0, len(children)): curr.append((children[i], level + 1)) return prev + curr return prev
def bfs_map(value): #value is the (puzzle, level) tuple """ YOUR CODE HERE """ lst = [(value)] if value[1] == level: children = Sliding.children( WIDTH, HEIGHT, Sliding.hash_to_board(WIDTH, HEIGHT, value[0])) for child in children: lst.append((Sliding.board_to_hash(WIDTH, HEIGHT, child), level + 1)) return lst return lst
def bfs_flat_map(state): """Takes in a state (level and board) and returns a list containing the original state with the addition of the children if the state is at the current level (in the frontier) """ self_list = (state,) if get_level(state) == level-1: children = Sliding.children(WIDTH, HEIGHT, get_board(state)) return tuple(make_state(level, board) for board in children) + self_list return self_list
def bfs_flat_map(parentValue): mappedList = [] mappedList.append(parentValue) # if the parentValue is from the last level we checked, look for its children and append them if parentValue[1] == level - 1: children = Sliding.children(WIDTH, HEIGHT, parentValue[0]) for element in children: mappedList.append((element, level)) return mappedList
def sol_map(value): lst = [] if (value[1] < level): lst.append(value) return lst lst.append(value) children = Sliding.children(WIDTH, HEIGHT, value[0]) for each in children: lst.append((tuple(each), value[1]+1)) del children return lst
def bfs_flat_map(value): """ YOUR CODE HERE """ re = [] value = (value[0], value[1]) re.append(value) if value[1] == (level - 1): #check if its the previous level children = Sliding.children(WIDTH, HEIGHT, value[0]) #children is a list of children for each in children: re.append(tuple((each, level))) return re
def bfs_map(value): """ YOUR CODE HERE """ result = [] if value[1] == level - 1: result = Sliding.children( WIDTH, HEIGHT, Sliding.hash_to_board(WIDTH, HEIGHT, value[0])) for i in range(0, len(result)): result[i] = (Sliding.board_to_hash(WIDTH, HEIGHT, result[i]), level) result.append(value) return result
def bfs_map(value): """ YOUR CODE HERE """ if (value[1] != (level - 1)): return [value] else: children = Sliding.children( WIDTH, HEIGHT, Sliding.hash_to_board(WIDTH, HEIGHT, value[0])) childList = [value] for child in children: childList.append((Sliding.board_to_hash(WIDTH, HEIGHT, child), level)) return childList
def bfs_flat_map(value): """ YOUR CODE HERE """ re = [] value = (value[0], value[1]) re.append(value) if value[1] == (level-1): #check if its the previous level children = Sliding.children(WIDTH, HEIGHT, value[0]) #children is a list of children for each in children: each = Sliding.board_to_hash(WIDTH, HEIGHT, each) #instead of storing boards as keys, we store the corresponding hashed ints as keys re.append(tuple((each, level))) return re
def bfs_map(value): """ YOUR CODE HERE """ # get children, make tuples, make a list resultList = [] result.append(value) #ensure parent is linked with children below result.append( map(lambda x: (x, level), Sliding.children(WIDTH, HEIGHT, value[0]))) #get first item of board # RDD.flatMap(lambda x: (x, level)) return result # for bfs_reduce
def bfs_map(arg): """ YOUR CODE HERE """ if arg[1] == level: children = Sliding.children( WIDTH, HEIGHT, Sliding.hash_to_board(WIDTH, HEIGHT, arg[0])) toreturn = [arg] for position in children: toreturn.append((Sliding.board_to_hash(WIDTH, HEIGHT, position), level + 1)) return toreturn else: return [arg]
def bfs_flat_map(state): """Takes in a state (level and board) and returns a list containing the original state with the addition of the children if the state is at the current level (in the frontier) """ self_list = (state, ) if get_level(state) == level - 1: children = Sliding.children(WIDTH, HEIGHT, get_board(state)) return tuple(make_state(level, board) for board in children) + self_list return self_list
def bfs_map(value): """ YOUR CODE HERE """ prev = [(value[0], value[1])] if value[1] == level: #convert from int to board # hash_to_board(WIDTH, HEIGHT, ) # ask manny if this is the correct to call this method # also what would number be? children = Sliding.children(WIDTH, HEIGHT, value[0]) curr = [] for i in range(0, len(children)): curr.append((children[i], level + 1)) return prev + curr return prev
def bfs_map(value): """ YOUR CODE HERE """ prev = [(value[0], value[1])] if value[1] == level: #convert from int to board # hash_to_board(WIDTH, HEIGHT, ) # ask manny if this is the correct to call this method # also what would number be? children = Sliding.children(WIDTH, HEIGHT, value[0]) curr = [] for i in range(0, len(children)): curr.append((children[i], level+1)) return prev + curr return prev
def bfs_map(value): """ YOUR CODE HERE """ # This function is the argument of flatMap method, and it should return a list. # format of value: (pos, level) pair rtn = [value] # Should not change the pos in previous level, the rtn list contains value itself if (level <= value[1]): # Check if the value pair exceeds the level. If it is larger than level, we need # append rtn with all of it's children. child = Sliding.children(WIDTH, HEIGHT, value[0]) for _ in child: rtn.append((_, value[1]+1)) return rtn
def bfs_flat_map(value): """ YOUR CODE HERE """ re = [] value = (value[0], value[1]) re.append(value) if value[1] == (level - 1): #check if its the previous level children = Sliding.children(WIDTH, HEIGHT, value[0]) #children is a list of children for each in children: each = Sliding.board_to_hash(WIDTH, HEIGHT, each) #instead of storing boards as keys, we store the corresponding hashed ints as keys re.append(tuple((each, level))) return re
def bfs_map(value): """value is a (key, value) pair where key = a configuration and value = level. Take this (key, value) pair and generate a list of all possible children for this configuration EFFICIENCY IMPROVEMENT: include parent in list to return to be rid of need for union""" """YOUR CODE HERE""" toRtn = list() #our list to return parent = (value[0], value[1]) toRtn.append(parent) #add parent to list if (value[1] == level): #we only want to look at values where its level == global level for child in Sliding.children(WIDTH, HEIGHT, value[0]): #this looks at each child in the list elem = (child, level+1) #create a new (K, V) pair toRtn.append(elem) #append this entry to our return list return toRtn #return this set of (K, V) pairs
def bfs_map(value): """ YOUR CODE HERE """ print "--------Just entered bfs_map---------" print "the value of level in bfs_map is:%d" % level # get children, make tuples, make a list result = [] result.append(value) #ensure parent is linked with children below print "value of value: " # this should print a tuple with ((A,B,C, -), 0) print value result.extend( map(lambda x: (x, level), Sliding.children(WIDTH, HEIGHT, value[0]))) #get first item of board return result # for bfs_reduce
def bfs_map(value): """ YOUR CODE HERE """ prev = [(value[0], value[1])] if value[1] == level: #convert from int to board # do I save it, not sure? ask Manny , using hashID since it was declared down? hashID = Sliding.board_to_hash(WIDTH, HEIGHT, prev[0][0]) #value[0] currBoard = Sliding.hash_to_board(WIDTH, HEIGHT, hashID) # ask manny if this is the correct to call this method # also what would number be? children = Sliding.children(WIDTH, HEIGHT, currBoard) # not sure value[0], currBoard #nextID = Sliding.board_to_hash(WIDTH, HEIGHT, children) curr = [] for i in range(0, len(children)): curr.append((children[i], level+1)) return prev + curr #nextID = Sliding.board_to_hash(WIDTH, HEIGHT, children[0]) #children[0] return prev
def bfs_map(value): """ Takes in a key, value pair of (board state, level), creates all of the children of that board state if is on the same level as the global level, and returns them in a list. """ """ YOUR CODE HERE """ child_list = [] #Check if we are at the right level, so then we can make children of only those boards if value[1] == level: temp = Sliding.hash_to_board(WIDTH, HEIGHT, value[0]) iter_list = Sliding.children(WIDTH, HEIGHT, temp) for child in iter_list: child_list += [(Sliding.board_to_hash(WIDTH,HEIGHT, child), level + 1)] #Spark map only lets us return a list if we want multiple things. #Unlike Hadoop I believe which allows us to emit return child_list
def bfs_flat_map(value): """ YOUR CODE HERE """ # childrenLst=[] # childrenLst.append(value) # for child in Sliding.children(WIDTH,HEIGHT,Sliding.hash_to_board(WIDTH, HEIGHT, value[0])): # pair=[] # pair.append(Sliding.board_to_hash(WIDTH, HEIGHT, child)) # pair.append(level) # childrenLst.append(tuple(pair)) # return childrenLst childrenLst = [] childrenLst.append((value[0], value[1])) if(value[1] == level - 1): for child in Sliding.children(WIDTH, HEIGHT, Sliding.hash_to_board(WIDTH,HEIGHT, value[0])): childrenLst.append((Sliding.board_to_hash(WIDTH,HEIGHT,child), value[1]+1)) return childrenLst
def bfs_map(value): """ value: Taken an element from RDD bfs_map function only applies children() to each element at the last level in RDD; return :If an element is not at the last level, then it will be put in an empty list and return; return :If an element is at the last level, then its children and the element will be put into an empty list and return; """ lst = [] lst.append(value) value = (Sliding.hash_to_board(WIDTH,HEIGHT,value[0]), value[1]) if (value[1] < level): return lst children = Sliding.children(WIDTH, HEIGHT, value[0]) for each in children: lst.append(((Sliding.board_to_hash(WIDTH, HEIGHT, tuple(each))), value[1]+1)) return lst
def bfs_map(value): """ Takes in a key, value pair of (board state, level), creates all of the children of that board state if is on the same level as the global level, and returns them in a list. """ """ YOUR CODE HERE """ child_list = [] #Check if we are at the right level, so then we can make children of only those boards if value[1] == level: temp = Sliding.hash_to_board(WIDTH, HEIGHT, value[0]) iter_list = Sliding.children(WIDTH, HEIGHT, temp) for child in iter_list: child_list += [(Sliding.board_to_hash(WIDTH, HEIGHT, child), level + 1)] #Spark map only lets us return a list if we want multiple things. #Unlike Hadoop I believe which allows us to emit return child_list
def bfs_map(value): """ value: Taken an element from RDD bfs_map function only applies children() to each element at the last level in RDD; return :If an element is not at the last level, then it will be put in an empty list and return; return :If an element is at the last level, then its children and the element will be put into an empty list and return; """ lst = [] lst.append(value) value = (Sliding.hash_to_board(WIDTH, HEIGHT, value[0]), value[1]) if (value[1] < level): return lst children = Sliding.children(WIDTH, HEIGHT, value[0]) for each in children: lst.append(((Sliding.board_to_hash(WIDTH, HEIGHT, tuple(each))), value[1] + 1)) return lst
def bfs_flat_map(value): """ YOUR CODE HERE """ # childrenLst=[] # childrenLst.append(value) # for child in Sliding.children(WIDTH,HEIGHT,Sliding.hash_to_board(WIDTH, HEIGHT, value[0])): # pair=[] # pair.append(Sliding.board_to_hash(WIDTH, HEIGHT, child)) # pair.append(level) # childrenLst.append(tuple(pair)) # return childrenLst childrenLst = [] childrenLst.append((value[0], value[1])) if (value[1] == level - 1): for child in Sliding.children( WIDTH, HEIGHT, Sliding.hash_to_board(WIDTH, HEIGHT, value[0])): childrenLst.append((Sliding.board_to_hash(WIDTH, HEIGHT, child), value[1] + 1)) return childrenLst
def bfs_map(value): """ YOUR CODE HERE """ prev = [(value[0], value[1])] if value[1] == level: #convert from int to board # do I save it, not sure? ask Manny , using hashID since it was declared down? hashID = Sliding.board_to_hash(WIDTH, HEIGHT, prev[0][0]) #value[0] currBoard = Sliding.hash_to_board( WIDTH, HEIGHT, hashID) # ask manny if this is the correct to call this method # also what would number be? children = Sliding.children(WIDTH, HEIGHT, currBoard) # not sure value[0], currBoard #nextID = Sliding.board_to_hash(WIDTH, HEIGHT, children) curr = [] for i in range(0, len(children)): curr.append((children[i], level + 1)) return prev + curr #nextID = Sliding.board_to_hash(WIDTH, HEIGHT, children[0]) #children[0] return prev
def bfs_map(value): """ only looking at board state at a time and making children for it""" children_list = [] children_list.append((value[0], value[1])) #append the key of the tuple and the global variable #if input level is current level, make children #append input and children if value[1] == level: #key,value is just the level ch = Sliding.children(WIDTH, HEIGHT, value[0]) #value[0] is the board if ch: for child in ch: children_list.append((child, level+1)) #append tuple #return (children, level+1) return children_list
def solve_puzzle(master, output, height, width, slaves): global HEIGHT, WIDTH, level HEIGHT=height WIDTH=width level = 0 sc = SparkContext(master, "python") sol = Sliding.solution(WIDTH, HEIGHT) frontierRDD = sc.parallelize([(Sliding.board_to_hash(WIDTH, HEIGHT, sol), 0)]) boardsRDD = sc.parallelize([(Sliding.board_to_hash(WIDTH, HEIGHT, sol), 0)]) #while frontierRDD.count() != 0: while True: level += 1 # get all frontier nodes as a flattened list of ONLY (key), NOT (key, value) frontierRDD = frontierRDD.flatMap(lambda v: Sliding.children(WIDTH, HEIGHT, Sliding.hash_to_board(WIDTH, HEIGHT, v[0]))) # add new (chilq, level) pairs to all boards boardsRDD = boardsRDD + frontierRDD.map(lambda v: (Sliding.board_to_hash(WIDTH, HEIGHT, v), level)) #boardsRDD = boardsRDD.partitionBy(8, partitionFunc) # only keep board seen at lowest level boardsRDD = boardsRDD.reduceByKey(lambda v1, v2: min(v1, v2)) # frontier is only the boards that have the current level frontierRDD = boardsRDD.filter(lambda v: v[1] == level) # magic voodoo that it doesn't work without boardsRDD = boardsRDD.partitionBy(slaves, lambda v: v) frontierRDD = frontierRDD.partitionBy(slaves, lambda v: v) if level % 4 == 0 and frontierRDD.count() == 0: break boardsRDD.coalesce(slaves).saveAsTextFile(output) sc.stop()
def slidingBfsSolver(puzzle, max_level=-1): """BF visit the entire puzzle graph, build level_to_pos, pos_to_level structures.""" solution = puzzle level = 0 level_to_pos[level] = [solution] ### level 0 consists of a single solution pos_to_level[solution] = level ### While there are still positions on the frontier ### (seen for the first time in the last iteration) while level_to_pos[level] and (max_level == -1 or level < max_level): level += 1 level_to_pos[level] = [] ### For every position on the last level (these farthest-away positions are the "frontier") for position in level_to_pos[level - 1]: ### For every child of those frontier positions for child in Sliding.children(W, H, position): ### If it's the first time we've seen this child if child not in pos_to_level: ### Update the mappings to remember it, and it will be part of the new frontier pos_to_level[child] = level level_to_pos[level].append(child) del level_to_pos[level] ### the last level is always empty, so remove it. pprint(level_to_pos)
def slidingBfsSolver(puzzle, max_level=-1): """BF visit the entire puzzle graph, build level_to_pos, pos_to_level structures.""" solution = puzzle level = 0 level_to_pos[level] = [solution] ### level 0 consists of a single solution pos_to_level[solution] = level ### While there are still positions on the frontier ### (seen for the first time in the last iteration) while level_to_pos[level] and (max_level==-1 or level < max_level): level += 1 level_to_pos[level] = [] ### For every position on the last level (these farthest-away positions are the "frontier") for position in level_to_pos[level-1]: ### For every child of those frontier positions for child in Sliding.children(W, H, position): ### If it's the first time we've seen this child if child not in pos_to_level: ### Update the mappings to remember it, and it will be part of the new frontier pos_to_level[child] = level level_to_pos[level].append(child) del level_to_pos[level] ### the last level is always empty, so remove it. pprint(level_to_pos)
def bfs_flatmap(value): if value[1] != level: return [[value[0], value[1]]] return [[value[0], level]] + [[el, level+1] for el in Sliding.children(WIDTH, HEIGHT, value[0])]
def get_children_boards(board): return Sliding.children(width, height, board)