def test_subgraph_convert(): G = nx.complete_graph(5) edge_index = from_networkx(G).edge_index sub_edge_index_1, _ = subgraph([0, 1, 3, 4], edge_index) sub_edge_index_2 = from_networkx(G.subgraph([0, 1, 3, 4])).edge_index assert sub_edge_index_1.tolist() == sub_edge_index_2.tolist()
def test_from_networkx_subgraph_convert(): import networkx as nx G = nx.complete_graph(5) edge_index = from_networkx(G).edge_index sub_edge_index_1, _ = subgraph([0, 1, 3, 4], edge_index, relabel_nodes=True) sub_edge_index_2 = from_networkx(G.subgraph([0, 1, 3, 4])).edge_index assert sub_edge_index_1.tolist() == sub_edge_index_2.tolist()
def get_data_from_graph(g: nx.DiGraph) -> Data: # graph_spicy = nx.to_scipy_sparse_matrix(g, format='coo') # edge_index = torch.tensor([graph_spicy.row, graph_spicy.col], dtype=torch.long) # x = torch.tensor(graph_spicy.data, dtype=torch.float) # # data = Data(x=x,edge_index=edge_index) return from_networkx(g)
def __init__(self, ckpt_path, newModel=False): self.dualGraph = readGraph() file_name = "dualGraphNodes.pkl" open_file = open(file_name, "wb") pickle.dump(list(self.dualGraph.nodes), open_file) open_file.close() #print(list(self.dualGraph.nodes)) self.data = from_networkx(self.dualGraph) self.device = 'cuda' if torch.cuda.is_available() else 'cpu' open_file = open("edge_index.pkl", "wb") pickle.dump(self.data.edge_index, open_file) open_file.close() open_file = open("edge_index.pkl", "rb") edge_index = pickle.load(open_file) open_file.close() self.model = Node2Vec(edge_index, embedding_dim=32, walk_length=20, context_size=10, walks_per_node=10, num_negative_samples=1, p=1, q=1, sparse=True).to(self.device) self.loader = self.model.loader(batch_size=128, shuffle=True, num_workers=0) self.optimizer = torch.optim.SparseAdam(list(self.model.parameters()), lr=0.01) if newModel: self.train(epochs=20) self.saveTo(ckpt_path) else: self.loadFrom(ckpt_path)
def observation(self) -> Data: """ Returns a pytorch geometric grpah object as observation of the AF where edge_input contains features wheter the attack exists, whether the oppsite attack exists and a self attack exists, and whether an edge have been flipped. Node features are set to indicate which arguments should be enforced """ graph = self.state.enforcement_representation if self.flipped_edges: for u, v in self.flipped_edges: graph.edges[u, v]["edge_flipped"] = 1 data: Data = from_networkx(graph) data["edge_input"] = torch.cat( [ data[key].unsqueeze(1) for key in ["edge_exists", "edge_opposite", "edge_self", "edge_flipped"] ], dim=1, ) data.edge_index, data["edge_input"] = coalesce( data.edge_index, data["edge_input"], data.num_nodes, data.num_nodes ) if self.task in [STRICT, NONSTRICT]: data["node_input"][list(self.problem.desired_extension)] = 1 elif self.task in [CRED, SCEPT]: data["node_input"][list(self.problem.positive)] = 1 data["node_input"][list(self.problem.negative)] = -1 return data
def graph_torch(G_networkx): """Create the Pytorch Geometric graph with 5 simple node features: the node degree, the min and max neighbor degree, the mean and std of the neighbors degree. Parameters: ---------- G_networkx: networkx graph Returns: ------- G_torch: Pytorch Geometric Data object""" # transform the networkx object to pytorch geometric data G_torch = from_networkx(G_networkx).to(device) G_torch.edge_index = G_torch.edge_index.type(torch.LongTensor).to(device) # Add features with torch.no_grad(): add_features = LocalDegreeProfile() G_torch = add_features(G_torch) # if there is a nan, put it to 0 G_torch.x[torch.isnan(G_torch.x)] = 0 # normalize the 4 first features by the max possible degree in the graph n = len(G_networkx) G_torch.x[:, :4] /= n - 1 # normalize the std of degree by the max std possible G_torch.x[:, -1] /= (n - 1) / 2 return G_torch
def test_from_networkx_without_edges(): graph = nx.Graph() graph.add_node(1) graph.add_node(2) data = from_networkx(graph) assert len(data) == 1 assert data.edge_index.size() == (2, 0)
def to_line_graph(data: Data, directed: bool = True) -> Data: """ Convert a graph G to its corresponding line-graph L(G) Args: data: a torch_gemoetric Data object representing representing a graph directed: whether the original graph is directed or undirected """ original_edge_attrs = data.edge_attr original_edge_names = [ (from_.item(), to_.item()) for from_, to_ in zip(data.edge_index[0, :], data.edge_index[1, :]) ] original_edge_to_attr = { e: attr for e, attr in zip(original_edge_names, original_edge_attrs) } ctor = nx.DiGraph if directed else nx.Graph G = to_networkx(data, node_attrs=['x'], edge_attrs=['edge_attr'], to_undirected=not directed) line_graph = nx.line_graph(G, create_using=ctor) res_data = from_networkx(line_graph) # Copy original attribtues res_data.x = torch.stack( [original_edge_to_attr[e] for e in line_graph.nodes]) res_data.y = data.y return data
def _create_pyg_graphs(self): typer.echo("Preprocessing input networks...") # Extend all graphs with nodes in `self.union` and add self-loops # to all nodes. new_graphs = [nx.Graph() for _ in self.graphs] for G, nG in zip(self.graphs, new_graphs): nG.add_nodes_from(self.union) nG.add_weighted_edges_from([(s, t, weight["weight"]) for (s, t, weight) in G.edges(data=True)]) nG.remove_edges_from( nx.selfloop_edges(nG)) # remove existing selfloops first nG.add_weighted_edges_from([(n, n, 1.0) for n in nG.nodes()]) self.graphs = new_graphs pyg_graphs = [from_networkx(G) for G in self.graphs] for G in pyg_graphs: G.edge_weight = G.weight del G.weight to_sparse_tensor = ToSparseTensor(remove_edge_index=False) for G in pyg_graphs: to_sparse_tensor(G) pyg_graphs = [t.to(Device()) for t in pyg_graphs] return pyg_graphs
def preprocess_input_graph(G, labels, normalize_adj=False): """ Load an existing graph to be converted for the experiments. Args: G: Networkx graph to be loaded. labels: Associated node labels. normalize_adj: Should the method return a normalized adjacency matrix. Returns: A dictionary containing adjacency, node features and labels """ # Define adj matrix adj = np.array(nx.to_numpy_matrix(G)) if normalize_adj: sqrt_deg = np.diag(1.0 / np.sqrt(np.sum(adj, axis=0, dtype=float).squeeze())) adj = np.matmul(np.matmul(sqrt_deg, adj), sqrt_deg) # Convert to our adj matrix type pyg_G = from_networkx(G) adj = pyg_G.edge_index f = pyg_G.feat # Define labels labels = torch.tensor(labels) # Add batch dim # adj = np.expand_dims(adj, axis=0) # f = np.expand_dims(f, axis=0) # labels = np.expand_dims(labels, axis=0) return f, adj, labels
def process(self): i = 0 processedValues = self.processed_file_names() classValues = np.loadtxt(self.root + '/dayClasses.txt') # Read data from `raw_path`. for raw_path in self.raw_paths: # if file is already processed, continue. if (f'data_{i}.pt' in processedValues): i = i + 1 print(f"data_{i}.pt already processed.") continue print('raw path', raw_path) print('root', self.root) index = int(raw_path.split('_')[1].split('.')[0]) classVal = int(classValues[index]) print("index", index, '\t Class: ', classVal) data = nx.read_gml(raw_path) data = utils.from_networkx(data) data['y'] = [classVal] print(data, data['y']) torch.save( data, os.path.join(self.processed_dir, 'data_{}.pt'.format(i))) i += 1
def process(self): # Read data into huge `Data` list. data_list = [] for i in range(self.num_samples): y = np.random.randint(len(self.targets)) probs = [ [self.in_probs[0], self.targets[y]], [self.targets[y], self.in_probs[1]], ] block_sizes = self.sizes[np.random.randint(len(self.sizes))] G = nx.stochastic_block_model(block_sizes, probs, seed=i + 1) x = torch.zeros((sum(block_sizes), self.n_features)) for i, partition in enumerate(G.graph["partition"]): partition = np.array(list(partition)) feat = np.random.choice(self.n_features, len(partition), p=self.feat_probs[i]) x[partition, feat] = 1 data = from_networkx(G) data["x"] = x.float() data["y"] = torch.tensor([y]) data_list.append(data) if self.pre_filter is not None: data_list = [data for data in data_list if self.pre_filter(data)] if self.pre_transform is not None: data_list = [self.pre_transform(data) for data in data_list] data, slices = self.collate(data_list) torch.save((data, slices), self.processed_paths[0])
def process(self): # Read data into huge `Data` list. data_list = [] for idx, patient in enumerate(self.patients): y = 1 if "Schizophrenia" in patient else 0 # Load the adjacency matrix into a pandas adj = pd.read_csv(patient, skiprows=1, header=None, index_col=0) G = nx.from_pandas_adjacency(adj) if idx in self.test and y == 0: # Thin the test graphs of one class G.remove_nodes_from( np.random.random_integers(1, len(G), int(self.p * len(G)))) G.remove_nodes_from(list(nx.isolates(G))) G = nx.convert_node_labels_to_integers(G) data = from_networkx(G) data["x"] = torch.ones((len(G), 1)) data["y"] = torch.tensor([y]) data_list.append(data) if self.pre_filter is not None: data_list = [data for data in data_list if self.pre_filter(data)] if self.pre_transform is not None: data_list = [self.pre_transform(data) for data in data_list] data, slices = self.collate(data_list) torch.save((data, slices), self.processed_paths[0])
def test_from_networkx_with_same_node_and_edge_attributes(): G = nx.Graph() G.add_nodes_from([(0, {'age': 1}), (1, {'age': 6}), (2, {'age': 5})]) G.add_edges_from([(0, 1, {'age': 2}), (1, 2, {'age': 7})]) data = from_networkx(G) assert len(data) == 4 assert data.age.tolist() == [1, 6, 5] assert data.num_nodes == 3 assert data.edge_index.tolist() == [[0, 1, 1, 2], [1, 0, 2, 1]] assert data.edge_age.tolist() == [2, 2, 7, 7] data = from_networkx(G, group_node_attrs=all, group_edge_attrs=all) assert len(data) == 3 assert data.x.tolist() == [[1], [6], [5]] assert data.edge_index.tolist() == [[0, 1, 1, 2], [1, 0, 2, 1]] assert data.edge_attr.tolist() == [[2], [2], [7], [7]]
def test_vice_versa_convert(): G = nx.complete_graph(5) assert G.is_directed() is False data = from_networkx(G) assert data.is_directed() is False G = to_networkx(data) assert G.is_directed() is True G = nx.to_undirected(G) assert G.is_directed() is False
def test_from_networkx_without_edges(): import networkx as nx graph = nx.Graph() graph.add_node(1) graph.add_node(2) data = from_networkx(graph) assert len(data) == 2 assert data.edge_index.size() == (2, 0) assert data.num_nodes == 2
def dataloader(): G = nx.read_gpickle("spatial_graph.gpickle") data = from_networkx(G) # print('here') # print(data.num_nodes) # print(data.num_edges) # print(data.num_node_features) # print(data) # print(data['x']) # print(data['y']) return data, G
def test_from_networkx_non_numeric_labels(): graph = nx.Graph() graph.add_node('4') graph.add_node('2') graph.add_edge('4', '2') for node in graph.nodes(): graph.nodes[node]['x'] = node data = from_networkx(graph) assert len(data) == 2 assert data.x == ['4', '2'] assert data.edge_index.tolist() == [[0, 1], [1, 0]]
def test_from_networkx_non_consecutive(): graph = nx.Graph() graph.add_node(4) graph.add_node(2) graph.add_edge(4, 2) for node in graph.nodes(): graph.nodes[node]['x'] = node data = from_networkx(graph) assert len(data) == 2 assert data.x.tolist() == [4, 2] assert data.edge_index.tolist() == [[0, 1], [1, 0]]
def __getitem__(self, idx): if type(idx) is slice: res = [ self.__getitem__(i) for i in range( *list(filter(None, [idx.start, idx.stop, idx.step]))) ] return res fname = os.path.join(self.datadir, self.prefix + '_' + f"{idx:06}" + '.xyz') dat = process(fname) nx_graph = rdkit_process(dat) return from_networkx(nx_graph)
def test_from_networkx_inverse(): graph = nx.Graph() graph.add_node(3) graph.add_node(2) graph.add_node(1) graph.add_node(0) graph.add_edge(3, 1) graph.add_edge(2, 1) graph.add_edge(1, 0) data = from_networkx(graph) assert len(data) == 1 assert data.edge_index.tolist() == [[0, 1, 2, 2, 2, 3], [2, 2, 0, 1, 3, 2]]
def state_to_vector(graph_state: MiniWoBGraphState, prior_actions: dict): e_dom = encode_dom_graph(graph_state.dom_graph) e_fields = encode_fields(graph_state.fields) encode_field_actions_onto_encoded_dom_graph(e_dom, e_fields, graph_state.fields, graph_state.utterance) encode_prior_actions_onto_encoded_dom_graph(e_dom, prior_actions, graph_state.fields) # for node_id in e_dom.nodes: # print(node_id, e_dom.nodes[node_id]) e_dom = nx.convert_node_labels_to_integers(e_dom) d = from_networkx(e_dom) return d
def __getitem__(self, idx) -> Tuple[ArgumentationFramework, Data]: """ Return AF and data representaiton of idx-th AF""" af = self.get_af(idx) graph = af.graph_representation(self.representation) data = from_networkx(graph) data["idx"] = idx data["edge_input"] = torch.cat( [ data[key].unsqueeze(1) for key in ["edge_exists", "edge_opposite", "edge_self"] ], dim=1, ) return af, data
def test_from_networkx(): x = torch.randn(2, 8) pos = torch.randn(2, 3) edge_index = torch.tensor([[0, 1, 0], [1, 0, 0]]) edge_attr = torch.randn(edge_index.size(1)) perm = torch.tensor([0, 2, 1]) data = Data(x=x, pos=pos, edge_index=edge_index, edge_attr=edge_attr) G = to_networkx(data, node_attrs=['x', 'pos'], edge_attrs=['edge_attr']) data = from_networkx(G) assert len(data) == 4 assert data.x.tolist() == x.tolist() assert data.pos.tolist() == pos.tolist() assert data.edge_index.tolist() == edge_index[:, perm].tolist() assert data.edge_attr.tolist() == edge_attr[perm].tolist()
def _edge_removal(graph): actions = set() _, num_edges = graph.edge_index.size() edges = graph.edge_index.t().numpy().tolist() for i in range(num_edges//2): tmp = to_networkx(graph, to_undirected=True) tmp.remove_edge(*edges[i]) tmp = from_networkx(tmp) a = graph.clone() a.edge_index = tmp.edge_index actions.add(a) return actions
def test_from_networkx(): x = torch.Tensor([[1, 2], [3, 4]]) pos = torch.Tensor([[0, 0], [1, 1]]) edge_index = torch.tensor([[0, 1, 0], [1, 0, 0]]) edge_attr = torch.Tensor([1, 2, 3]) data = Data(x=x, pos=pos, edge_index=edge_index, edge_attr=edge_attr) G = to_networkx(data, node_attrs=['x', 'pos'], edge_attrs=['edge_attr']) data = from_networkx(G) assert len(data) == 4 assert data.x.tolist() == x.tolist() assert data.pos.tolist() == pos.tolist() edge_index, edge_attr = coalesce(data.edge_index, data.edge_attr, 2, 2) assert edge_index.tolist() == [[0, 0, 1], [0, 1, 0]] assert edge_attr.tolist() == [3, 1, 2]
def process(self): i = 1 for raw_path in self.raw_paths: nxg = preprocess_qm9.rdkit_process( preprocess_qm9.process(raw_path)) data = from_networkx(nxg) if self.pre_filter is not None and not self.pre_filter(data): continue if self.pre_transform is not None: data = self.pre_transform(data) torch.save(data, os.path.join(self.processed_dir, '{}.pt'.format(i))) i += 1
def process(self): # Read data into huge `Data` list. with open(Path(self.root) / self.raw_file_names[0], 'rb') as f: aspect_to_aspect_graph = pickle.load(f) data = from_networkx(aspect_to_aspect_graph) nodes = aspect_to_aspect_graph.nodes() nodes_mapping = dict(zip(nodes, range(0, aspect_to_aspect_graph.number_of_nodes()))) aspect_embeddings = [self.nlp(aspect).vector for aspect in tqdm(nodes, desc='Generating aspects embeddings...')] node_features = torch.tensor(aspect_embeddings, dtype=torch.float) data.nodes_mapping = nodes_mapping data.x = node_features data_list = [data] data, slices = self.collate(data_list) torch.save((data, slices), self.processed_paths[0])
def process(self): """ Processes two pickle files into a series of graph files. The first pickle file is the simulation/experimental data that contains the light seen by the PMT and the true position of the event (when given a simulation). The second pickle file is the graph structure to use for the GCNN. """ # Graph structure get with open(os.path.join(self.root_dir, "data", self.graph_name), 'rb') as fn: input_graph = pk.load(fn) torch_graph = from_networkx(input_graph) # Starting to process the data files. i = 0 for data_path in self.raw_paths: with open(data_path, 'rb') as fn: data_contents = pk.load(fn) self.content_size += len(raw_contents) for event_data in data_contents: # Light detecting by each PMT x = event_data[ 'area_per_channel'][:127] # Only getting the top PMTs x = np.reshape(x, (len(x), 1)) x = np.hstack( (x, np.array(torch_graph.pos))).astype(np.float32) # True position of this event if self.simulation: y = np.reshape(event_data['true_pos'], (1, len(event_data['true_pos']))) else: # Not a simulation -> no truth y = None processed_data = Data(x=torch.tensor(x), edge_index=torch_graph.edge_index, y=torch.tensor(y), pos=torch_graph.pos) # Saving the contents into the processed directory path_to_proc = os.path.join(self.processed_dir, self.proc_anem) if not os.path.isdir(path_to_proc): os.makedirs(path_to_proc) torch.save(processed_data, os.path.join(path_to_proc, 'graph_{}.pt'.format(i))) i += 1