def test_subgraph(): A, expected = config() B_init = np.random.rand(2) graph = mapfission_sdfg() graph.apply_transformations(MapFission) dace.sdfg.propagation.propagate_memlets_sdfg(graph) cgraph = graph.compile() B = dcpy(B_init) cgraph(A=A, B=B) del cgraph assert np.allclose(B, expected) graph.validate() subgraph = SubgraphView(graph.nodes()[0], graph.nodes()[0].nodes()) sf = SubgraphFusion(subgraph) assert sf.can_be_applied(graph, subgraph) fusion(graph, graph.nodes()[0], None) ccgraph = graph.compile() B = dcpy(B_init) ccgraph(A=A, B=B) assert np.allclose(B, expected) graph.validate()
def softimpute(self): it = 0 X_hat = dcpy(self.X) M = dcpy(self.M) start_time = time.time() while it < self.maxiter: it = it + 1 X_hat = self.P_X - M * self.omega + M U, D, VT = np.linalg.svd(X_hat, full_matrices = False) #prt('U', U) #prt('D', D) #prt('V.T', VT) SD = np.diag(np.fmax(D - self._lambda, 0)) M = U @ SD @ VT diff = np.linalg.norm(M - self.M, ord = 'fro') if diff < self.eps: break self.M = dcpy(M) self.time_seq.append( time.time()-start_time ) #self.mse_seq.append( np.sqrt(mse(self.X_truth, M)) ) self.mse_seq.append( mse(self.X_truth, M)) end_time = time.time() print('number of iteration:', it) print('time:', end_time - start_time)
def __deepcopy__(self, memo): node = object.__new__(AccessNode) node._data = self._data node._setzero = self._setzero node._in_connectors = dcpy(self._in_connectors, memo=memo) node._out_connectors = dcpy(self._out_connectors, memo=memo) node._debuginfo = dcpy(self._debuginfo, memo=memo) return node
def coverage_dicts(sdfg, graph, map_entry, outer_range=True): ''' returns a tuple of two dicts: the first dict has as a key all data entering the map and its associated access range the second dict has as a key all data exiting the map and its associated access range if outer_range = True, substitutes outer ranges into min/max of inner access range ''' map_exit = graph.exit_node(map_entry) map = map_entry.map entry_coverage = {} exit_coverage = {} # create dicts with which we can replace all iteration # variable_mapping by their range map_min = {dace.symbol(param): e for param, e in zip(map.params, map.range.min_element())} map_max = {dace.symbol(param): e for param, e in zip(map.params, map.range.max_element())} # look at inner memlets at map entry for e in graph.out_edges(map_entry): if not e.data.subset: continue if outer_range: # get subset min_element = [m.subs(map_min) for m in e.data.subset.min_element()] max_element = [m.subs(map_max) for m in e.data.subset.max_element()] # create range rng = subsets.Range((min_e, max_e, 1) for min_e, max_e in zip(min_element, max_element)) else: rng = dcpy(e.data.subset) if e.data.data not in entry_coverage: entry_coverage[e.data.data] = rng else: old_coverage = entry_coverage[e.data.data] entry_coverage[e.data.data] = subsets.union(old_coverage, rng) # look at inner memlets at map exit for e in graph.in_edges(map_exit): if outer_range: # get subset min_element = [m.subs(map_min) for m in e.data.subset.min_element()] max_element = [m.subs(map_max) for m in e.data.subset.max_element()] # craete range rng = subsets.Range((min_e, max_e, 1) for min_e, max_e in zip(min_element, max_element)) else: rng = dcpy(e.data.subset) if e.data.data not in exit_coverage: exit_coverage[e.data.data] = rng else: old_coverage = exit_coverage[e.data] exit_coverage[e.data.data] = subsets.union(old_coverage, rng) # return both coverages as a tuple return (entry_coverage, exit_coverage)
def can_be_applied(self, sdfg: SDFG, subgraph: SubgraphView) -> bool: # get lowest scope maps of subgraph # grab first node and see whether all nodes are in the same graph # (or nested sdfgs therein) graph = subgraph.graph # next, get all the maps by obtaining a copy (for potential offsets) map_entries = helpers.get_outermost_scope_maps(sdfg, graph, subgraph) ranges = [dcpy(map_entry.range) for map_entry in map_entries] # offset if option is toggled if self.allow_offset == True: for r in ranges: r.offset(r.min_element(), negative=True) brng = helpers.common_map_base_ranges(ranges) # more than one outermost scoped map entry has to be availble if len(map_entries) <= 1: return False # check whether any parameters are in common if len(brng) == 0: return False # if option enabled, return false if any splits are introduced if self.permutation_only == True: for map_entry in map_entries: if len(map_entry.params) != len(brng): return False # if option enabled, check contiguity in the last contiguous dimension # if there is a map split ocurring with the last contiguous dimension being # in the *outer* map, we fail (-> bad data access pattern) if self.check_contiguity == True: reassignment = helpers.find_reassignment(map_entries, brng, offset=self.allow_offset) for map_entry in map_entries: no_common = sum( [1 for j in reassignment[map_entry] if j != -1]) if no_common != len(map_entry.params): # check every memlet for access for e in itertools.chain( graph.out_edges(map_entry), graph.in_edges(graph.exit_node(map_entry))): subset = dcpy(e.data.subset) subset.pop([i for i in range(subset.dims() - 1)]) for s in subset.free_symbols: if reassignment[map_entry][ map_entry.map.params.index(s)] != -1: warnings.warn( "MultiExpansion::Contiguity fusion violation detected" ) return False return True
def differ(old, new): N, M = len(old), len(new) dist_mat = [] path_mat = [] for i in range(N + 1): dist_mat.append([-1 for j in range(M + 1)]) path_mat.append([[] for j in range(M + 1)]) for n in range(N + 1): for m in range(M + 1): if n == 0 and m == 0: dist = 0 path = [] elif n == 0: dist = m path = [Action('insert', 1, m)] elif m == 0: dist = n path = [Action('delete', 1, n)] else: d_keep = dist_mat[n - 1][m - 1] if old[n - 1] == new[m - 1] else 2 + N + M d_del = dist_mat[n - 1][m] + 1 d_ins = dist_mat[n][m - 1] + 1 dist = min(d_keep, d_del, d_ins) if d_keep == dist: path = dcpy(path_mat[n - 1][m - 1]) if path and path[-1].kind == 'keep': path[-1].end += 1 else: path.append(Action('keep', n, n)) elif d_ins == dist: path = dcpy(path_mat[n][m - 1]) if path and path[-1].kind == 'insert': path[-1].end += 1 else: path.append(Action('insert', m, m)) else: # d_del == dist: path = dcpy(path_mat[n - 1][m]) if path and path[-1].kind == 'delete': path[-1].end += 1 else: path.append(Action('delete', n, n)) dist_mat[n][m] = dist path_mat[n][m] = path return dist_mat[N][M], path_mat[N][M]
def apply(self, graph: SDFGState, sdfg: SDFG): import dace.libraries.blas as blas transpose_a = self.transpose_a _at = self.at transpose_b = self.transpose_b _bt = self.bt a_times_b = self.a_times_b for src, src_conn, _, _, memlet in graph.in_edges(transpose_a): graph.add_edge(src, src_conn, a_times_b, '_b', memlet) graph.remove_node(transpose_a) for src, src_conn, _, _, memlet in graph.in_edges(transpose_b): graph.add_edge(src, src_conn, a_times_b, '_a', memlet) graph.remove_node(transpose_b) graph.remove_node(_at) graph.remove_node(_bt) for _, _, dst, dst_conn, memlet in graph.out_edges(a_times_b): subset = dcpy(memlet.subset) subset.squeeze() size = subset.size() shape = [size[1], size[0]] break tmp_name, tmp_arr = sdfg.add_temp_transient(shape, a_times_b.dtype) tmp_acc = graph.add_access(tmp_name) transpose_c = blas.Transpose('_Transpose_', a_times_b.dtype) for edge in graph.out_edges(a_times_b): _, _, dst, dst_conn, memlet = edge graph.remove_edge(edge) graph.add_edge(transpose_c, '_out', dst, dst_conn, memlet) graph.add_edge(a_times_b, '_c', tmp_acc, None, dace.Memlet.from_array(tmp_name, tmp_arr)) graph.add_edge(tmp_acc, None, transpose_c, '_inp', dace.Memlet.from_array(tmp_name, tmp_arr))
def crossover(cls, parent_1, parent_2): """Return a pair of child solutions from parent crossover. Method based on swapping random client data. """ child_1 = dcpy(parent_1) child_2 = dcpy(parent_2) for client in range(Solution.client_nr): # product delivery data swapped with 50% probability # client order will be corrected later if randint(1, 2) == 2: child_1.product_delivery[client] = parent_2.product_delivery[ client][:] child_2.product_delivery[client] = parent_1.product_delivery[ client][:] return child_1, child_2
def generate_files_sony(old_subj_data, savepath): num_subj = len(old_subj_data) num_cond = len(old_subj_data[0]) subj_data = [[dcpy(empty_map) for i in range(num_cond)] for j in range(num_subj)] clip_size = old_subj_data[0][0]['trials'].shape[2] / 2 num_electrodes = old_subj_data[0][0]['trials'].shape[1] for subj_ind in range(num_subj): for cond in range(num_cond): conds = conds_map[cond] old_entry = old_subj_data[subj_ind][cond] for i in range(2): clip = range(clip_size * i, clip_size * (i + 1)) entry = subj_data[subj_ind][conds[i]] entry['trials'] = concat(entry['trials'], old_entry['trials'][:, :, clip]) if not os.path.isdir(savepath): os.makedirs(savepath) dats = [] dats_det = [] for cond in range(num_cond): print 'Generating condition', cond + 1, 'files...' dats.append(np.zeros((0, clip_size, num_electrodes))) dats_det.append(np.zeros((0, clip_size, num_electrodes))) for subj in range(num_subj): trials = subj_data[subj][cond]['trials'] dats[-1] = concat(dats[-1], permute(trials, axes=(0, 2, 1))) savename = os.path.join(savepath, cond_names_map[cond]) savemat(savename, {'dat': dats[-1]}) dats_det[-1] = detrend(dats[-1], axis=1) savename = os.path.join(savepath, cond_names_map[cond] + '_det') savemat(savename, {'dat': dats_det[-1]})
def fuse_nodes(self, sdfg, graph, edge, new_dst, new_dst_conn): """ Fuses two nodes via memlets and possibly transient arrays. """ memlet_path = graph.memlet_path(edge) access_node = memlet_path[-1].dst local_name = "__s%d_n%d%s_n%d%s" % ( self.state_id, graph.node_id(edge.src), edge.src_conn, graph.node_id(edge.dst), edge.dst_conn, ) # Add intermediate memory between subgraphs. If a scalar, # uses direct connection. If an array, adds a transient node if edge.data.subset.num_elements() == 1: sdfg.add_scalar( local_name, dtype=access_node.desc(graph).dtype, transient=True, storage=dtypes.StorageType.Register, ) edge.data.data = local_name edge.data.subset = "0" local_node = edge.src src_connector = edge.src_conn else: sdfg.add_transient(local_name, edge.data.subset.size(), dtype=access_node.desc(graph).dtype) local_node = graph.add_access(local_name) src_connector = None edge.data.data = local_name edge.data.subset = ",".join( ["0:" + str(s) for s in edge.data.subset.size()]) # Add edge that leads to transient node graph.add_edge( edge.src, edge.src_conn, local_node, None, dcpy(edge.data), ) ######## # Add edge that leads to the second node graph.add_edge(local_node, src_connector, new_dst, new_dst_conn, dcpy(edge.data))
def callback_traj(msg): global arr_msg, pose_msg, traj_pub, pose_pub msg.pose.position.z = 4 pose_msg = dcpy(msg.pose) # pose_msg.orientation = msg.pose.orientation # pose_msg.position = msg.pose.position arr_msg.poses.insert(callback_traj.counter, pose_msg) arr_msg.header.frame_id = msg.header.frame_id callback_traj.counter += 1
def data_concatenate(input, grpname='', intersection=False): import numpy as np from copy import deepcopy as dcpy #Test data for equal keys res_description='Concatenation of ' test=[] intsect=[] for k in input: test.extend(k.keys) intsect.append(set(k.keys)) res_description+=' '+k.description if not intersection: assert set(test)==set(input[0].keys), 'Input files have different key variables.' #Concatenate data ind_base=len(input[0].data) group_variable=[input[0].grpname]*ind_base res_dropped=input[0].dropped res_keystype=input[0].keystype res_keys=input[0].keys res_keysdescr = input[0].keysdescr res_filename="Concatenated structure. All variables." res_nodata=input[0].nodata res_data=input[0].data for k in input[1:]: res_dropped+=k.dropped res_data=np.concatenate((res_data,k.data)) for j in k.nodata: res_nodata[j].extend(list(np.array(k.nodata[j])+ind_base)) ind_base+=len(k.data) group_variable.extend([k.grpname]*len(k.data)) res=Csvdata(res_description,res_keys,res_keysdescr,res_data,res_nodata,res_keystype,res_dropped,res_filename,grpname) else: allkeys=set.intersection(*intsect) ind_base=len(input[0].data) group_variable=[input[0].grpname]*ind_base res_data,res_keys,res_keystype,res_keysdescr=input[0].extractdatabykeylist(allkeys) res_dropped=None res_filename="Concatenated structure. Intersected variables." res_nodata=input[0].nodata for k in input[1:]: dta,qq,qq1,qq2=k.extractdatabykeylist(allkeys) group_variable.extend([k.grpname]*len(dta)) res_data=np.concatenate((res_data,dta)) for j in k.nodata: if j not in res_nodata.keys(): res_nodata.update({j:[]}) res_nodata[j].extend(list(np.array(k.nodata[j])+ind_base)) ind_base+=len(k.data) tmpdict=dcpy(res_nodata) for k in tmpdict: if k not in allkeys: res_nodata.pop(k) res=Csvdata(res_description,res_keys,res_keysdescr,res_data,res_nodata,res_keystype,res_dropped,res_filename,grpname) return (res,group_variable)
def Affect(self, Friendly, IncludeSelf=False, All=False): if not IncludeSelf: groups = [_ for _ in self.groups if _ != Friendly] else: groups = dcpy(self.groups) if not All: # Trim to one group groups = self.rng.choices(groups, k=1) # Iterate over all affected groups yield len(groups) for group in groups: yield group
def adjust_arrays_nsdfg(self, sdfg, nsdfg, name, nname): ''' DFS to replace strides and volumes of data that has adjacent nested SDFGs to its access nodes. Needed in a post-processing step during fusion. ''' nsdfg.data(nname).strides = dcpy(sdfg.data(name).strides) nsdfg.data(nname).total_size = dcpy(sdfg.data(name).total_size) # traverse the whole graph and search for arrays for ngraph in nsdfg.nodes(): for nnode in ngraph.nodes(): if isinstance(nnode, nodes.AccessNode) and nnode.label == nname: # trace and recurse if necessary for e in chain(ngraph.out_edges(nnode), ngraph.in_edges(nnode)): for te in ngraph.memlet_tree(e): if isinstance(te.dst, nodes.NestedSDFG): self.adjust_arrays_nsdfg( nsdfg, te.dst.sdfg, nname, te.dst_conn) if isinstance(te.src, nodes.NestedSDFG): self.adjust_arrays_nsdfg( nsdfg, te.src.sdfg, nname, te.src_conn)
def __init__(self, gamestate, chcol: str): Node.__init__(self) self.is_root = True self.gamestate = dcpy(gamestate) # get character index self.id, self.character = self.get_character_id( self.gamestate['characters'], chcol) # Removing current character from options. self.gamestate['options'].pop( next(id for id, ch in enumerate(self.gamestate['options']) if ch['color'] == chcol))
def test_stopword_affection(): import sys import codecs as c sys.stdout = c.open('result.stopword', 'w', 'utf8') T = rw.load_folder() vec.cutT(T) vec.n_gram(T, 2) from copy import deepcopy as dcpy rawT = dcpy(T) nostf, stf = [], [] test_time = 100 for i in range(test_time): T = dcpy(rawT) vec.confuse(T) rw.write_data(T) D,rD = vec.dictify(T) rw.write_dict(D, rD) tsr.tfidf(T) tsr.x2max_filter(T,D,1000,'tfidf') rw.write_libsvm(T,'x2max') nostf.append(knn.test(T, 0.8, warn_on_equidistant=False)) T = rw.read_T() vec.stopwordfilter(T) D,rD = vec.dictify(T) rw.write_dict(D, rD) tsr.tfidf(T) tsr.x2max_filter(T,D,1000,'tfidf') rw.write_libsvm(T,'x2max') stf.append(knn.test(T, 0.8, warn_on_equidistant=False)) print 'no stop word filtering...' knn.score_analysis(nostf) print 'stop word filtering...' knn.score_analysis(stf)
def __init__(self, gamestate: dict, charid: int, pos: int): Node.__init__(self) # Copy the state of the game, then set the original inaccessible self.gamestate = dcpy(gamestate) gamestate = None self.pos = pos self.character = self.gamestate['characters'][charid] self.character['position'] = self.pos self.gain = self.gamestate['compute_gain'].pop(0)(self.gamestate) self.try_debug() if len(self.gamestate['options']) > 0: self.next = self.gamestate['root_node'](self.gamestate)
def __init__(self, gamestate: dict, room: int, charid: int, moves: list): Node.__init__(self) # Copy the state of the game then remove original to avoid using it self.gamestate = dcpy(gamestate) gamestate = None self.pos = room # Moving the blackout to the given room self.gamestate['shadow'] = room # Check every possible move for m in moves: tmp = MoveNode(self.gamestate, charid, m) self.update_best_node(tmp) self.options.append(tmp)
def __deepcopy__(self, memo): n = object.__new__(type(self)) n.name = dcpy(self.name) n.inputs = dcpy(self.inputs) n.outputs = dcpy(self.outputs) n.globals = self.globals n.locals = dcpy(self.locals) n.transients = dcpy(self.transients) n.params = dcpy(self.params) n.parent = None n.children = [] n.is_async = dcpy(self.is_async) return n
def __init__(self, gamestate: dict, charid: int, pos: int, targetid: int): Node.__init__(self) # Copy the state of the game, then set the original inaccessible self.gamestate = dcpy(gamestate) gamestate = None self.pos = pos self.character = {'color': 'brown'} # Moving the persian and one other character to the next room self.gamestate['characters'][charid]['position'] = pos self.gamestate['characters'][targetid]['position'] = pos self.gain = self.gamestate['compute_gain'].pop(0)(self.gamestate) self.try_debug() if len(self.gamestate['options']) > 0: self.next = self.gamestate['root_node'](self.gamestate)
def copy_edge(self, graph, edge, new_src=None, new_src_conn=None, new_dst=None, new_dst_conn=None, new_data=None, remove_old=False): ''' Copies an edge going from source to dst. If no destination is specified, the edge is copied with the same destination and port as the original edge, else the edge is copied with the new destination and the new port. If no source is specified, the edge is copied with the same source and port as the original edge, else the edge is copied with the new source and the new port If remove_old is specified, the old edge is removed immediately If new_data is specified, inserts new_data as a memlet, else else makes a deepcopy of the current edges memlet ''' data = new_data if new_data else dcpy(edge.data) src = edge.src if new_src is None else new_src src_conn = edge.src_conn if new_src is None else new_src_conn dst = edge.dst if new_dst is None else new_dst dst_conn = edge.dst_conn if new_dst is None else new_dst_conn ret = graph.add_edge(src, src_conn, dst, dst_conn, data) if remove_old: graph.remove_edge(edge) ''' if new_src: ret = graph.add_edge(new_src, new_src_conn, edge.dst, edge.dst_conn, data) graph.remove_edge(edge) if new_dst: ret = graph.add_edge(edge.src, edge.src_conn, new_dst, new_dst_conn, data) graph.remove_edge(edge) ''' return ret
def __init__(self, gamestate: dict, charid: int, pos: int, moves: list): Node.__init__(self) # Copy the state of the game, then set the original inaccessible self.gamestate = dcpy(gamestate) gamestate = None self.pos = pos self.character = {'color': 'Christine'} # Drawing everyone around to the given room (including christine) for ch in self.gamestate['characters']: if ch['position'] in moves: ch['position'] = self.pos self.gain = self.gamestate['compute_gain'].pop(0)(self.gamestate) self.try_debug() if len(self.gamestate['options']) > 0: self.next = self.gamestate['root_node'](self.gamestate)
def generate_data(path, savepath): subjs = [item for item in os.listdir(path) if '.' not in item] num_cond = get_num_conds(os.path.join(path, subjs[0])) subj_data = [[dcpy(empty_map) for i in range(num_cond)] for j in range(len(subjs))] for subj_ind in xrange(len(subjs)): subj = subjs[subj_ind] print 'Adding subject', subj subj_path = os.path.join(path, subj) files = [item for item in os.listdir(subj_path) if 'Raw' in item] for data_ind in xrange(len(files)): data_file = files[data_ind] cond = int(data_file[data_file.rfind('c00') + 3]) - 1 data = loadmat(os.path.join(subj_path, data_file)) raw_data = data['RawTrial'][:, electrodes] num_TP = raw_data.shape[0] / data['NmbEpochs'] raw_data = permute(raw_data.reshape(num_TP, data['NmbEpochs'], len(electrodes)), axes=(1, 2, 0)) valid_epochs = [i for i in range(data['NmbEpochs']) if \ np.sum(data['IsEpochOK'][i]) > num_channels/2] raw_data = raw_data[valid_epochs, :, :] num_epochs = raw_data.shape[0] if num_epochs: entry = subj_data[subj_ind][cond] entry['groups'].append(num_epochs) entry['trials'] = concat(entry['trials'], raw_data) raw_data_mean = np.mean(raw_data, axis=0)[None, :] entry['mean_trials'] = concat(entry['mean_trials'], raw_data_mean) for cond in xrange(num_cond): entry = subj_data[subj_ind][cond] entry['groups'] = np.squeeze(entry['groups']) entry['trials'] = np.squeeze(entry['trials']) entry['mean_trials'] = np.squeeze(entry['mean_trials']) if not os.path.isdir(savepath): os.makedirs(savepath) savemat(os.path.join(savepath, 'subj_data'), {'subj_data': subj_data}) return subj_data
def mutation(self): """Perform mutation on solution by applying all or several mutation methods in random order.""" # randomly order possible mutations (1 to 5) muts = range(1, 6) shuffle(muts) # save original self original = dcpy(self) # apply first mutation self.mutation_wrapper(muts[0]) # if no mutation returned, iterate until mutation returned i = 1 while self == original: self.mutation_wrapper(muts[i]) i += 1 # apply remaining mutations with 50% probability for j in range(i, 5): k = randint(1, 2) if k == 1: self.mutation_wrapper(muts[j]) # correct after sequential mutations self.correct()
def _copy_first_map_contents(self, sdfg, graph, first_map_entry): inter_nodes = list( graph.all_nodes_between(first_map_entry, self.first_map_exit) - {first_map_entry}) new_inter_nodes = [dcpy(node) for node in inter_nodes] tmp_map = dict() for node in new_inter_nodes: if isinstance(node, nds.AccessNode): data = sdfg.arrays[node.data] if isinstance(data, dt.Scalar) and data.transient: tmp_name = sdfg.temp_data_name() sdfg.add_scalar(tmp_name, data.dtype, transient=True) tmp_map[node.data] = tmp_name node.data = tmp_name graph.add_node(node) id_map = { graph.node_id(old): graph.node_id(new) for old, new in zip(inter_nodes, new_inter_nodes) } def map_node(node): return graph.node(id_map[graph.node_id(node)]) def map_memlet(memlet): memlet = dcpy(memlet) memlet.data = tmp_map.get(memlet.data, memlet.data) return memlet for edge in graph.edges(): if edge.src in inter_nodes or edge.dst in inter_nodes: src = map_node( edge.src) if edge.src in inter_nodes else edge.src dst = map_node( edge.dst) if edge.dst in inter_nodes else edge.dst edge_data = map_memlet(edge.data) graph.add_edge(src, edge.src_conn, dst, edge.dst_conn, edge_data) return new_inter_nodes
def _replicate_first_map(self, sdfg, graph, first_map_entry, intermediate_dnodes): for dnode in intermediate_dnodes: array_name = dnode.data array = sdfg.arrays[array_name] read_offsets = self._read_offsets(graph, array_name) # Replicate first map tasklets once for each read offset access and # connect them to other tasklets accordingly for offset, edges in read_offsets.items(): new_nodes = self._copy_first_map_contents( sdfg, graph, first_map_entry) tmp_name = "__otf" tmp_name, _ = sdfg.add_scalar(tmp_name, array.dtype, transient=True, find_new_name=True) tmp_access = graph.add_access(tmp_name) for node in new_nodes: for edge in graph.edges_between(node, self.first_map_exit): graph.add_edge(edge.src, edge.src_conn, tmp_access, None, Memlet(tmp_name)) graph.remove_edge(edge) for edge in graph.edges_between(first_map_entry, node): memlet = dcpy(edge.data) memlet.subset.offset(list(offset), negative=False) self.second_map_entry.add_out_connector(edge.src_conn + "_") graph.add_edge(self.second_map_entry, edge.src_conn + "_", node, edge.dst_conn, memlet) graph.remove_edge(edge) for edge in edges: graph.add_edge(tmp_access, None, edge.dst, edge.dst_conn, Memlet(tmp_name))
def generate_rawDict(ports, caltype='solt'): ''' Make a dictionary to store raw signals (e.g. used for VNA calibration) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Usage: ports = [1, 2] CAL_data = generate_CALdict(measPorts) IN: ports -- list of ports to be measured on caltype -- (optional, str) type of structure to create choose from: - 'solt' (short, open, load, through on each port + crossterms) - 'meas' (custom) OUT: CAL_data -- dictionary to store raw signals ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ''' import itertools from copy import deepcopy as dcpy rawDict = {} rawDict['frequencies'] = None if caltype.lower() == 'solt': raw = {'a': None, 'u': None, 'v': None, 'b': None} CAL_port = { 's': dcpy(raw), 'o': dcpy(raw), 'l': dcpy(raw) } # short, open, load CAL_cross = {'t': dcpy(raw)} # through for port in ports: rawDict['p' + str(port)] = dcpy(CAL_port) # s-o-l # crossterms = tuple(itertools.combinations(ports, r=2)) for term in crossterms: rawDict['p' + str(term[0]) + 'p' + str(term[1])] = dcpy(CAL_cross) # else: print('Sorry, caltype "%s" not supported' % caltype) raise ('caltypeError') # return rawDict
def apply(self, sdfg): graph = sdfg.nodes()[self.state_id] transpose_a = graph.nodes()[self.subgraph[ MatrixProductTranspose._transpose_a]] _at = graph.nodes()[self.subgraph[MatrixProductTranspose._at]] transpose_b = graph.nodes()[self.subgraph[ MatrixProductTranspose._transpose_b]] _bt = graph.nodes()[self.subgraph[MatrixProductTranspose._bt]] a_times_b = graph.nodes()[self.subgraph[ MatrixProductTranspose._a_times_b]] for src, src_conn, _, _, memlet in graph.in_edges(transpose_a): graph.add_edge(src, src_conn, a_times_b, '_b', memlet) graph.remove_node(transpose_a) for src, src_conn, _, _, memlet in graph.in_edges(transpose_b): graph.add_edge(src, src_conn, a_times_b, '_a', memlet) graph.remove_node(transpose_b) graph.remove_node(_at) graph.remove_node(_bt) for _, _, dst, dst_conn, memlet in graph.out_edges(a_times_b): subset = dcpy(memlet.subset) subset.squeeze() size = subset.size() shape = [size[1], size[0]] break tmp_name, tmp_arr = sdfg.add_temp_transient(shape, a_times_b.dtype) tmp_acc = graph.add_access(tmp_name) transpose_c = blas.Transpose('_Transpose_', a_times_b.dtype) for edge in graph.out_edges(a_times_b): _, _, dst, dst_conn, memlet = edge graph.remove_edge(edge) graph.add_edge(transpose_c, '_out', dst, dst_conn, memlet) graph.add_edge(a_times_b, '_c', tmp_acc, None, dace.Memlet.from_array(tmp_name, tmp_arr)) graph.add_edge(tmp_acc, None, transpose_c, '_inp', dace.Memlet.from_array(tmp_name, tmp_arr))
def __deepcopy__(self, memo): node = object.__new__(Memlet) # Set properties node._volume = dcpy(self._volume, memo=memo) node._dynamic = self._dynamic node._subset = dcpy(self._subset, memo=memo) node._other_subset = dcpy(self._other_subset, memo=memo) node._data = dcpy(self._data, memo=memo) node._wcr = dcpy(self._wcr, memo=memo) node._wcr_nonatomic = dcpy(self._wcr_nonatomic, memo=memo) node._debuginfo = dcpy(self._debuginfo, memo=memo) node._wcr_nonatomic = self._wcr_nonatomic node._allow_oob = self._allow_oob node._is_data_src = self._is_data_src # Nullify graph references node._sdfg = None node._state = None node._edge = None return node
def generate_rawDict(ports, caltype='solt'): ''' Make a dictionary to store raw signals (e.g. used for VNA calibration) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Usage: ports = [1, 2] CAL_data = generate_CALdict(measPorts) IN: ports -- list of ports to be measured on caltype -- (optional, str) type of structure to create choose from: - 'solt' (short, open, load, through on each port + crossterms) - 'meas' (custom) OUT: CAL_data -- dictionary to store raw signals ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ''' import itertools from copy import deepcopy as dcpy rawDict = {} rawDict['frequencies'] = None if caltype.lower()=='solt': raw = {'a': None, 'u': None, 'v': None, 'b': None} CAL_port = { 's': dcpy(raw), 'o': dcpy(raw), 'l': dcpy(raw) } # short, open, load CAL_cross = {'t': dcpy(raw) } # through for port in ports: rawDict['p'+str(port)] = dcpy(CAL_port) # s-o-l # crossterms = tuple(itertools.combinations(ports, r=2)) for term in crossterms: rawDict['p'+str(term[0])+'p'+str(term[1])] = dcpy(CAL_cross) # else: print('Sorry, caltype "%s" not supported' % caltype) raise('caltypeError') # return rawDict
def apply(self, sdfg): state = sdfg.nodes()[self.state_id] nested_sdfg = state.nodes()[self.subgraph[CopyToDevice._nested_sdfg]] storage = self.storage created_arrays = set() for _, edge in enumerate(state.in_edges(nested_sdfg)): src, src_conn, dst, dst_conn, memlet = edge dataname = memlet.data if dataname is None: continue memdata = sdfg.arrays[dataname] name = 'device_' + dataname + '_in' if name not in created_arrays: if isinstance(memdata, data.Array): name, _ = sdfg.add_array( 'device_' + dataname + '_in', shape=[ symbolic.overapproximate(r) for r in memlet.bounding_box_size() ], dtype=memdata.dtype, transient=True, storage=storage, find_new_name=True) elif isinstance(memdata, data.Scalar): name, _ = sdfg.add_scalar('device_' + dataname + '_in', dtype=memdata.dtype, transient=True, storage=storage, find_new_name=True) else: raise NotImplementedError created_arrays.add(name) data_node = nodes.AccessNode(name) to_data_mm = dcpy(memlet) from_data_mm = dcpy(memlet) from_data_mm.data = name offset = [] for ind, r in enumerate(memlet.subset): offset.append(r[0]) if isinstance(memlet.subset[ind], tuple): begin = memlet.subset[ind][0] - r[0] end = memlet.subset[ind][1] - r[0] step = memlet.subset[ind][2] from_data_mm.subset[ind] = (begin, end, step) else: from_data_mm.subset[ind] -= r[0] state.remove_edge(edge) state.add_edge(src, src_conn, data_node, None, to_data_mm) state.add_edge(data_node, None, dst, dst_conn, from_data_mm) for _, edge in enumerate(state.out_edges(nested_sdfg)): src, src_conn, dst, dst_conn, memlet = edge dataname = memlet.data if dataname is None: continue memdata = sdfg.arrays[dataname] name = 'device_' + dataname + '_out' if name not in created_arrays: if isinstance(memdata, data.Array): name, _ = sdfg.add_array( name, shape=[ symbolic.overapproximate(r) for r in memlet.bounding_box_size() ], dtype=memdata.dtype, transient=True, storage=storage, find_new_name=True) elif isinstance(memdata, data.Scalar): name, _ = sdfg.add_scalar(name, dtype=memdata.dtype, transient=True, storage=storage) else: raise NotImplementedError created_arrays.add(name) data_node = nodes.AccessNode(name) to_data_mm = dcpy(memlet) from_data_mm = dcpy(memlet) to_data_mm.data = name offset = [] for ind, r in enumerate(memlet.subset): offset.append(r[0]) if isinstance(memlet.subset[ind], tuple): begin = memlet.subset[ind][0] - r[0] end = memlet.subset[ind][1] - r[0] step = memlet.subset[ind][2] to_data_mm.subset[ind] = (begin, end, step) else: to_data_mm.subset[ind] -= r[0] state.remove_edge(edge) state.add_edge(src, src_conn, data_node, None, to_data_mm) state.add_edge(data_node, None, dst, dst_conn, from_data_mm) # Change storage for all data inside nested SDFG to device. change_storage(nested_sdfg.sdfg, storage)
def apply(self, sdfg): """ This method applies the mapfusion transformation. Other than the removal of the second map entry node (SME), and the first map exit (FME) node, it has the following side effects: 1. Any transient adjacent to both FME and SME with degree = 2 will be removed. The tasklets that use/produce it shall be connected directly with a scalar/new transient (if the dataflow is more than a single scalar) 2. If this transient is adjacent to FME and SME and has other uses, it will be adjacent to the new map exit post fusion. Tasklet-> Tasklet edges will ALSO be added as mentioned above. 3. If an access node is adjacent to FME but not SME, it will be adjacent to new map exit post fusion. 4. If an access node is adjacent to SME but not FME, it will be adjacent to the new map entry node post fusion. """ graph = sdfg.nodes()[self.state_id] first_exit = graph.nodes()[self.subgraph[MapFusion._first_map_exit]] first_entry = graph.entry_node(first_exit) second_entry = graph.nodes()[self.subgraph[ MapFusion._second_map_entry]] second_exit = graph.exit_nodes(second_entry)[0] intermediate_nodes = set() for _, _, dst, _, _ in graph.out_edges(first_exit): intermediate_nodes.add(dst) assert isinstance(dst, nodes.AccessNode) # Check if an access node refers to non transient memory, or transient # is used at another location (cannot erase) do_not_erase = set() for node in intermediate_nodes: if sdfg.arrays[node.data].transient is False: do_not_erase.add(node) else: for edge in graph.in_edges(node): if edge.src != first_exit: do_not_erase.add(node) break else: for edge in graph.out_edges(node): if edge.dst != second_entry: do_not_erase.add(node) break # Find permutation between first and second scopes perm = MapFusion.find_permutation(first_entry.map, second_entry.map) params_dict = {} for index, param in enumerate(first_entry.map.params): params_dict[param] = second_entry.map.params[perm[index]] # Replaces (in memlets and tasklet) the second scope map # indices with the permuted first map indices. # This works in two passes to avoid problems when e.g., exchanging two # parameters (instead of replacing (j,i) and (i,j) to (j,j) and then # i,i). second_scope = graph.scope_subgraph(second_entry) for firstp, secondp in params_dict.items(): if firstp != secondp: replace(second_scope, secondp, '__' + secondp + '_fused') for firstp, secondp in params_dict.items(): if firstp != secondp: replace(second_scope, '__' + secondp + '_fused', firstp) # Isolate First exit node ############################ edges_to_remove = set() nodes_to_remove = set() for edge in graph.in_edges(first_exit): memlet_path = graph.memlet_path(edge) edge_index = next(i for i, e in enumerate(memlet_path) if e == edge) access_node = memlet_path[-1].dst if access_node not in do_not_erase: out_edges = [ e for e in graph.out_edges(access_node) if e.dst == second_entry ] # In this transformation, there can only be one edge to the # second map assert len(out_edges) == 1 # Get source connector to the second map connector = out_edges[0].dst_conn[3:] new_dst = None new_dst_conn = None # Look at the second map entry out-edges to get the new # destination for _e in graph.out_edges(second_entry): if _e.src_conn[4:] == connector: new_dst = _e.dst new_dst_conn = _e.dst_conn break if new_dst is None: # Access node is not used in the second map nodes_to_remove.add(access_node) continue # If the source is an access node, modify the memlet to point # to it if (isinstance(edge.src, nodes.AccessNode) and edge.data.data != edge.src.data): edge.data.data = edge.src.data edge.data.subset = ("0" if edge.data.other_subset is None else edge.data.other_subset) edge.data.other_subset = None else: # Add a transient scalar/array self.fuse_nodes(sdfg, graph, edge, new_dst, new_dst_conn) edges_to_remove.add(edge) # Remove transient node between the two maps nodes_to_remove.add(access_node) else: # The case where intermediate array node cannot be removed # Node will become an output of the second map exit out_e = memlet_path[edge_index + 1] conn = second_exit.next_connector() graph.add_edge( second_exit, 'OUT_' + conn, out_e.dst, out_e.dst_conn, dcpy(out_e.data), ) second_exit.add_out_connector('OUT_' + conn) graph.add_edge(edge.src, edge.src_conn, second_exit, 'IN_' + conn, dcpy(edge.data)) second_exit.add_in_connector('IN_' + conn) edges_to_remove.add(out_e) # If the second map needs this node, link the connector # that generated this to the place where it is needed, with a # temp transient/scalar for memlet to be generated for out_e in graph.out_edges(second_entry): second_memlet_path = graph.memlet_path(out_e) source_node = second_memlet_path[0].src if source_node == access_node: self.fuse_nodes(sdfg, graph, edge, out_e.dst, out_e.dst_conn) edges_to_remove.add(edge) ### # First scope exit is isolated and can now be safely removed for e in edges_to_remove: graph.remove_edge(e) graph.remove_nodes_from(nodes_to_remove) graph.remove_node(first_exit) # Isolate second_entry node ########################### for edge in graph.in_edges(second_entry): memlet_path = graph.memlet_path(edge) edge_index = next(i for i, e in enumerate(memlet_path) if e == edge) access_node = memlet_path[0].src if access_node in intermediate_nodes: # Already handled above, can be safely removed graph.remove_edge(edge) continue # This is an external input to the second map which will now go # through the first map. conn = first_entry.next_connector() graph.add_edge(edge.src, edge.src_conn, first_entry, 'IN_' + conn, dcpy(edge.data)) first_entry.add_in_connector('IN_' + conn) graph.remove_edge(edge) out_e = memlet_path[edge_index + 1] graph.add_edge( first_entry, 'OUT_' + conn, out_e.dst, out_e.dst_conn, dcpy(out_e.data), ) first_entry.add_out_connector('OUT_' + conn) graph.remove_edge(out_e) ### # Second node is isolated and can now be safely removed graph.remove_node(second_entry) # Fix scope exit to point to the right map second_exit.map = first_entry.map
def can_be_applied(graph, candidate, expr_index, sdfg, strict=False): first_map_exit = graph.nodes()[candidate[MapFusion._first_map_exit]] first_map_entry = graph.entry_node(first_map_exit) second_map_entry = graph.nodes()[candidate[ MapFusion._second_map_entry]] for _in_e in graph.in_edges(first_map_exit): if _in_e.data.wcr is not None: for _out_e in graph.out_edges(second_map_entry): if _out_e.data.data == _in_e.data.data: # wcr is on a node that is used in the second map, quit return False # Check whether there is a pattern map -> access -> map. intermediate_nodes = set() intermediate_data = set() for _, _, dst, _, _ in graph.out_edges(first_map_exit): if isinstance(dst, nodes.AccessNode): intermediate_nodes.add(dst) intermediate_data.add(dst.data) # If array is used anywhere else in this state. num_occurrences = len([ n for n in graph.nodes() if isinstance(n, nodes.AccessNode) and n.data == dst.data ]) if num_occurrences > 1: return False else: return False # Check map ranges perm = MapFusion.find_permutation(first_map_entry.map, second_map_entry.map) if perm is None: return False # Check if any intermediate transient is also going to another location second_inodes = set(e.src for e in graph.in_edges(second_map_entry) if isinstance(e.src, nodes.AccessNode)) transients_to_remove = intermediate_nodes & second_inodes # if any(e.dst != second_map_entry for n in transients_to_remove # for e in graph.out_edges(n)): if any(graph.out_degree(n) > 1 for n in transients_to_remove): return False # Create a dict that maps parameters of the first map to those of the # second map. params_dict = {} for _index, _param in enumerate(first_map_entry.map.params): params_dict[_param] = second_map_entry.map.params[perm[_index]] out_memlets = [e.data for e in graph.in_edges(first_map_exit)] # Check that input set of second map is provided by the output set # of the first map, or other unrelated maps for second_edge in graph.out_edges(second_map_entry): # Memlets that do not come from one of the intermediate arrays if second_edge.data.data not in intermediate_data: # however, if intermediate_data eventually leads to # second_memlet.data, need to fail. for _n in intermediate_nodes: source_node = _n destination_node = graph.memlet_path(second_edge)[0].src # NOTE: Assumes graph has networkx version if destination_node in nx.descendants( graph._nx, source_node): return False continue provided = False # Compute second subset with respect to first subset's symbols sbs_permuted = dcpy(second_edge.data.subset) sbs_permuted.replace({ symbolic.pystr_to_symbolic(k): symbolic.pystr_to_symbolic(v) for k, v in params_dict.items() }) for first_memlet in out_memlets: if first_memlet.data != second_edge.data.data: continue # If there is a covered subset, it is provided if first_memlet.subset.covers(sbs_permuted): provided = True break # If none of the output memlets of the first map provide the info, # fail. if provided is False: return False # Success return True