def transform_setup(graph_u=False,
graph_gcn=False,
rotation=180,
samplePoints=1024,
mesh=False,
node_translation=0.01):
if not graph_u and not graph_gcn:
# Default transformation for scale noralization, centering, point sampling and rotating
pretransform = T.Compose([T.NormalizeScale(), T.Center()])
transform = T.Compose([
T.SamplePoints(samplePoints),
T.RandomRotate(rotation[0], rotation[1])
])
print("pointnet rotation {}".format(rotation))
elif graph_u:
pretransform = T.Compose([T.NormalizeScale(), T.Center()])
transform = T.Compose([
T.NormalizeScale(),
T.Center(),
T.SamplePoints(samplePoints, True, True),
T.RandomRotate(rotation[0], rotation[1]),
T.KNNGraph(k=graph_u)
])
elif graph_gcn:
pretransform = T.Compose([T.NormalizeScale(), T.Center()])
if mesh:
if mesh == "extraFeatures":
transform = T.Compose([
T.RandomRotate(rotation[0], rotation[1]),
T.GenerateMeshNormals(),
T.FaceToEdge(True),
T.Distance(norm=True),
T.TargetIndegree(cat=True)
]) # ,
else:
transform = T.Compose([
T.RandomRotate(rotation[0], rotation[1]),
T.GenerateMeshNormals(),
T.FaceToEdge(True),
T.Distance(norm=True),
T.TargetIndegree(cat=True)
])
else:
transform = T.Compose([
T.SamplePoints(samplePoints, True, True),
T.KNNGraph(k=graph_gcn),
T.Distance(norm=True)
])
print("no mesh")
print("Rotation {}".format(rotation))
print("Meshing {}".format(mesh))
else:
print('no transfom')
return transform, pretransform
def test_data_transforms(self):
"""
export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64:/usr/local/cudnn-10.0-v7.6.5.32
proxychains python -c "from template_lib.examples.DGL.geometric.test_pytorch_geometric import TestingGeometric;\
TestingGeometric().test_data_transforms()"
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'PORT' not in os.environ:
os.environ['PORT'] = '6006'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '1'
# func name
assert sys._getframe().f_code.co_name.startswith('test_')
command = sys._getframe().f_code.co_name[5:]
class_name = self.__class__.__name__[7:] \
if self.__class__.__name__.startswith('Testing') \
else self.__class__.__name__
outdir = f'results/{class_name}/{command}'
from datetime import datetime
TIME_STR = bool(int(os.getenv('TIME_STR', 0)))
time_str = datetime.now().strftime("%Y%m%d-%H_%M_%S_%f")[:-3]
outdir = outdir if not TIME_STR else (outdir + '_' + time_str)
print(outdir)
import collections, shutil
shutil.rmtree(outdir, ignore_errors=True)
os.makedirs(outdir, exist_ok=True)
import torch_geometric.transforms as T
from torch_geometric.datasets import ShapeNet
from template_lib.d2.data.build_points_toy import plot_points
dataset = ShapeNet(root='datasets/shapenet', categories=['Airplane'])
idx = -1
plot_points(dataset[idx].pos)
plot_points(dataset[idx].x)
dataset = ShapeNet(root='datasets/shapenet',
categories=['Airplane'],
pre_transform=T.KNNGraph(k=6))
dataset = ShapeNet(root='datasets/shapenet',
categories=['Airplane'],
pre_transform=T.KNNGraph(k=6),
transform=T.RandomTranslate(0.01))
pass
def process(self):
seq_b = pd.read_csv('data/sequential_background.csv')
scalar_b = pd.read_csv('data/scalar_background.csv')
data_list = []
i = 0
for index, sca in scalar_b.iterrows():
tmp_event = seq_b.loc[seq_b['event_no'] == sca['event_no']]
x = torch.tensor(tmp_event[['dom_charge','dom_time','dom_x','dom_y','dom_z']].to_numpy(),dtype=torch.float) #Features
pos = torch.tensor(tmp_event[['dom_x','dom_y','dom_z']].to_numpy(),dtype=torch.float) #Position
y = torch.tensor(sca[sca.keys()[2:]].to_numpy(),dtype=torch.float) #Target
dat = Data(x=x,edge_index=None,edge_attr=None,y=y,pos=pos)
T.KNNGraph(loop=True)(dat) #defining edges by k-NN with k=6
data_list.append(dat)
if i % 1000 == 0:
print(i)
i += 1
if self.pre_filter is not None:
data_list = [data for data in data_list if self.pre_filter(data)]
if self.pre_transform is not None:
data_list = [self.pre_transform(data) for data in data_list]
# print(data_list)
data, slices = self.collate(data_list)
torch.save((data,slices), self.processed_paths[0])
# MyOwnDataset(root = 'C:/Users/jv97/Desktop/github/Neutrino-Machine-Learning/dataset_background')
#Could be done without all this, but the collate method is nessecary and torch.save
def __init__(self,
descriptor_dim,
sampler=None,
split='train',
transform=DefaultTransform,
build_graph=False,
cls=False):
super(FaustFEPts5k).__init__()
self.name = 'FaustFEPts5k'
self.IDlist = np.arange(10000)
self.split = split
if self.split == 'train':
raise RuntimeError("This dataset is Test Only")
elif self.split == 'test':
self.IDlist = self.IDlist
elif self.split == 'val':
self.IDlist = self.IDlist[:40]
self.file_path = '{}/faust/scans/{{0:03d}}_{{0:03d}}.mat'.format(
PATH_TO_DATA)
self.template_feats = helper.loadSMPLDescriptors()[:, :descriptor_dim]
self.template_points = helper.loadSMPLModels()[0].verts
self.pre_transform = None #T.NormalizeScale()
self.cls = cls
if build_graph:
self.transform = T.Compose(
[transform, T.KNNGraph(k=6),
T.ToDense(5000)])
else:
self.transform = T.Compose([transform, T.ToDense(5000)])
def __init__(self,
descriptor_dim,
sampler=None,
split='train',
transform=DefaultTransform,
cls=False,
build_graph=False):
super(SurrealFEPts5k).__init__()
self.name = 'SurrealFEPts5k'
self.split = split
if self.split == 'train':
self.IDlist = IDlist[:, :-(num_test * num_views)].reshape(-1)
elif self.split == 'test':
self.IDlist = IDlist[:, -(num_test * num_views):].reshape(-1)
elif self.split == 'val':
self.IDlist = IDlist[:, :num_views].reshape(-1)
self.file_path = '{}/scans/{{0:06d}}/{{1:03d}}.mat'.format(
PATH_TO_SURREAL)
self.template_feats = helper.loadSMPLDescriptors()[:, :descriptor_dim]
self.template_points = helper.loadSMPLModels()[0].verts
self.cls = cls
if build_graph:
self.transform = T.Compose(
[transform, T.KNNGraph(k=6),
T.ToDense(5000)])
else:
self.transform = T.Compose([transform, T.ToDense(5000)])
def __init__(self,
n_neigh=9,
rad_neigh=0.1,
knn=None,
self_loop=True,
edge_attr=None,
flow='source_to_target'):
super(GraphReg, self).__init__()
# defining graph transform
graph_transform_list = []
self.del_edge_attr = False
self.knn = knn
self.n_neigh = n_neigh
self.rad_neigh = rad_neigh
self.self_loop = self_loop
self.edge_attr = edge_attr
if self.knn == True:
graph_transform_list.append(
T.KNNGraph(n_neigh, loop=self_loop, flow=flow))
elif self.knn == False:
graph_transform_list.append(
T.RadiusGraph(self.rad_neigh,
loop=self_loop,
max_num_neighbors=n_neigh,
flow=flow))
else:
print("Connectivity of the graph will not be re-generated")
# edge attr
if edge_attr is not None:
self.del_edge_attr = True
if type(edge_attr) == str:
if edge_attr:
edge_attr = [attr.strip() for attr in edge_attr.split('-')]
else:
edge_attr = []
for attr in edge_attr:
attr = attr.strip().lower()
if attr == 'poscart':
graph_transform_list.append(Cartesian(norm=False,
cat=True))
elif attr == 'posspherical':
graph_transform_list.append(Spherical(cat=True))
elif attr == 'featureoffsets':
graph_transform_list.append(
FeatureDistances(metric='offset', cat=True))
elif attr == 'featurel2':
graph_transform_list.append(
FeatureDistances(metric='l2', cat=True))
else:
raise RuntimeError('{} is not supported'.format(attr))
self.graph_transform = T.Compose(graph_transform_list)
def save_graphs(feat_dir, images_list, graph_files_dir):
constructor = T.KNNGraph(k=6, force_undirected=True)
for i in progressbar.progressbar(range(0, len(images_list))):
name = images_list[i]
feat_path = os.path.join(feat_dir, name)
dict_feats = torch.load(feat_path)
data = Data(pos=dict_feats['norm_rois'])
data = constructor(data)
torch.save(data, os.path.join(graph_files_dir, name))
def constr_graph(feat_dir, images_list, graph_files_dir):
constructor = T.KNNGraph(k=6, force_undirected=True)
res = []
for i in progressbar.progressbar(range(0, len(images_list))):
name = images_list[i]
feat_path = os.path.join(feat_dir, name)
dict_feats = torch.load(feat_path)
data = Data(x=dict_feats['pooled_feat'], pos=dict_feats['norm_rois'])
res.append(constructor(data))
return res
def main():
# ------------
# args
# ------------
parser = ArgumentParser()
parser.add_argument("--batch_size", default=64, type=int)
parser.add_argument("--num_workers", default=2, type=int)
parser.add_argument("--task", default="off_center", type=str)
parser = pl.Trainer.add_argparse_args(parser)
args = parser.parse_args()
# ------------
# data
# ------------
data_dir = Path(gvp.__file__).parents[1] / "data/synthetic"
transform = transforms.Compose([transforms.KNNGraph(k=10), ExtendedPPF()])
dm = SyntheticDataModule(
data_dir, args.batch_size, args.task, transform, num_workers=args.num_workers
)
# ------------
# model
# ------------
model = SyntheticGNN(4, 32, 4, 32)
# ------------
# training
# ------------
wandb_logger = WandbLogger(
name=f"SyntheticGNN-{args.task}", project="GVP", reinit=True
)
checkpoint_callback = ModelCheckpoint(
monitor="val_loss",
dirpath="model_checkpoints",
filename=f"SyntheticGNN-{args.task}-" + "{epoch:02d}-{val_loss:.2f}",
save_weights_only=True,
save_top_k=3,
mode="min",
)
trainer = pl.Trainer.from_argparse_args(
args,
logger=wandb_logger,
callbacks=[checkpoint_callback],
)
trainer.fit(model, dm)
# ------------
# testing
# ------------
result = trainer.test(datamodule=dm)
print(result)
wandb.finish()
def __init__(self, root_path, transform=None, pre_transform=None):
super(GFTE_POS_DATASET, self).__init__(root_path, transform,
pre_transform)
self.root_path = root_path
self.json_file_list = os.listdir(os.path.join(root_path, "structure"))
#清洗数据，基于json 开始，通过json文件名判断是否有相应的img文件，再通过json与chunk判断数据的有效性和合法性，实际就是看是否json与chunk数据匹配
self.imglist = os.listdir(os.path.join(root_path, "testtrain")) #img
#此处是使用check后的list
self.imglist = self.check_all()
self.graph_transform = GT.KNNGraph(
k=6
) #使用pos创建k-nn图 Creates a k-NN graph based on node positions :obj:`pos`.
def __init__(self, root_path, transform=None, pre_transform=None):
super(ScitsrDataset, self).__init__(root_path, transform,
pre_transform)
self.root_path = root_path
self.jsonfile = os.path.join(self.root_path, "imglist.json")
if os.path.exists(self.jsonfile): # imglist.json去掉了一些有疑问的文件
with open(self.jsonfile, "r") as read_file:
self.imglist = json.load(read_file)
else:
self.imglist = list(
filter(
lambda fn: fn.lower().endswith('.jpg') or fn.lower().
endswith('.png'),
os.listdir(os.path.join(self.root_path, "img"))))
self.imglist = self.check_all()
with open(self.jsonfile, "w") as write_file:
json.dump(self.imglist, write_file)
self.graph_transform = GT.KNNGraph(k=6)
def __init__(self, root_path, transform=None, pre_transform=None):
super(ScitsrDataset, self).__init__(root_path, transform,
pre_transform)
self.root_path = root_path
self.jsonfile = os.path.join(self.root_path, "imglist.json")
self.img_size = 256
self.kernel = np.ones((3, 3), np.uint8) # 把图像的线变粗一点
if os.path.exists(self.jsonfile): # imglist.json去掉了一些有疑问的文件
with open(self.jsonfile, "r") as read_file:
self.imglist = json.load(read_file)
else:
self.imglist = list(
filter(
lambda fn: fn.lower().endswith('.jpg') or fn.lower().
endswith('.png'),
os.listdir(
os.path.join(self.root_path,
"/content/scitsr_data/train/img"))))
self.imglist = self.check_all()
with open(self.jsonfile, "w") as write_file:
json.dump(self.imglist, write_file)
self.graph_transform = GT.KNNGraph(k=6)
def load(config):
'''
input: data path to vtk simulation files
return: a list includes path to individual path to each file
'''
FORCE_MEAN, FORCE_STD = DatasetLoader.dic_to_np(config.json_force)
DISP_MEAN, DISP_STD = DatasetLoader.dic_to_np(config.json_disp)
print("FORCE_MEAN", FORCE_STD)
print("DISP_STD", DISP_STD)
dataset = []
knn = T.KNNGraph(k=6)
print('[INFO] Loading dataset ...')
for file_path in sorted(os.listdir(config.data_path)):
# print('[INFO] Reading Folder named: {}'.format(folder))
if file_path.split('.')[-1] == 'vtk':
full_path = os.path.join(config.data_path, file_path)
mesh_pv = pv.read(full_path)
force = mesh_pv.point_arrays['externalForce']
disp = mesh_pv.point_arrays['computedDispl']
force_norm = (force - FORCE_MEAN) / FORCE_STD
disp_norm = (disp - DISP_MEAN) / DISP_STD
point_torch = torch.from_numpy(mesh_pv.points)
disp_torch = torch.from_numpy(disp_norm) #labels
force_torch = torch.from_numpy(force_norm) #node features
data = Data(x=force_torch, y=disp_torch, pos=point_torch)
data = knn(data)
dataset.append(data)
return dataset
x = torch.cat([torch.tensor(tmp_event[['charge_log10','time']].values,dtype=torch.float),x_pos,y_pos,z_pos],dim=1)
pos = torch.cat([x_pos,y_pos,z_pos],dim=1)
else:
x = torch.tensor(tmp_event[['charge_log10','time','dom_x','dom_y','dom_z']].values,dtype=torch.float) #Features
pos = torch.tensor(tmp_event[['dom_x','dom_y','dom_z']].values,dtype=torch.float) #Position
query = "SELECT energy_log10, time, position_x, position_y, position_z, direction_x, direction_y, direction_z, azimuth, zenith FROM truth WHERE event_no = {}".format(event_no)
y = pd.read_sql(query,con)
y = torch.tensor(y.values,dtype=torch.float) #Target
dat = Data(x=x,edge_index=None,edge_attr=None,y=y,pos=pos)
# T.KNNGraph(loop=True)(dat) #defining edges by k-NN with k=6 !!! Make sure .pos is not scaled!!! ie. x,y,z -!-> ax,by,cz
T.KNNGraph(k=6, loop=False, force_undirected = False)(dat)
dat.adj_t = None
T.ToUndirected()(dat)
T.AddSelfLoops()(dat)
(row, col) = dat.edge_index
dat.edge_index = torch.stack([col,row],dim=0)
data_list.append(dat)
if (i+1) % subdivides == 0:
data, slices = InMemoryDataset.collate(data_list)
torch.save((data,slices), destination + '/{}k_{}{}.pt'.format(subdivides//1000,save_filename,subset))
subset += 1
data_list = [] #Does this free up the memory?
if i % 500 == 0:
if self.transform is not None:
data_s = self.transform(data_s)
data_t = self.transform(data_t)
data = Data(num_nodes=pos_s.size(0))
for key in data_s.keys:
data['{}_s'.format(key)] = data_s[key]
for key in data_t.keys:
data['{}_t'.format(key)] = data_t[key]
return data
transform = T.Compose([
T.Constant(),
T.KNNGraph(k=8),
T.Cartesian(),
])
train_dataset = RandomGraphDataset(30, 60, 0, 20, transform=transform)
train_loader = DataLoader(train_dataset, args.batch_size, shuffle=True,
follow_batch=['x_s', 'x_t'])
path = osp.join('..', 'data', 'PascalPF')
test_datasets = [PascalPF(path, cat, transform) for cat in PascalPF.categories]
device = 'cuda' if torch.cuda.is_available() else 'cpu'
psi_1 = SplineCNN(1, args.dim, 2, args.num_layers, cat=False, dropout=0.0)
psi_2 = SplineCNN(args.rnd_dim, args.rnd_dim, 2, args.num_layers, cat=True,
dropout=0.0)
model = DGMC_modified_v2(psi_1, psi_2, num_steps=args.num_steps).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
def getKNNConstructor(k=6, force_undirected=True):
return T.KNNGraph(k=k, force_undirected=force_undirected)
import torch_geometric.transforms as T
from torch_geometric.datasets import ShapeNet
dataset = ShapeNet(root='/media/lepoeme20/Data/graphs/ShapeNet',
categories=['Airplane'])
print(dataset[0])
# Convert the point cloud dataset into a graph dataset
# by generating nearest neighbor graphs from the point clouds via transforms:
transformed_dataset = ShapeNet(root='/media/lepoeme20/Data/graphs/ShapeNet',\
categories=['Airplane'], pre_transform=T.KNNGraph(k=6), transform=T.RandomTranslate(0.01))
print(transformed_dataset[0])
dataset = TUDataset(root='./data/ENZYMES', name='ENZYMES')
loader = DataLoader(dataset, batch_size=32, shuffle=True)
for i, data in enumerate(loader):
# In the first batch, there are 1047 nodes (belong to the 32 graphs (i.e.,32 batches)), [1047, 3]
print(data.x.size())
"""
-----------Data Transforms------------
"""
# Only datasets
dataset = ShapeNet(root='./data/ShapeNet', categories=['Airplane'])
print(dataset[0]) # >>> Data(pos=[2518, 3], y=[2518])
# Constucting the Graph with neighbor
dataset = ShapeNet(
root='./data/ShapeNet',
categories=['Airplane'],
pre_transform=T.KNNGraph(
k=6)) # >>> Data(edge_index=[2, 15108], pos=[2518, 3], y=[2518])
# translate each node position by a small number:
dataset = ShapeNet(root='/tmp/ShapeNet',
categories=['Airplane'],
pre_transform=T.KNNGraph(k=6),
transform=T.RandomTranslate(0.01))
"""
-----------Learning Methods on Graphs---------------
Let's implement a two-layer GCN
"""
# Load the datasets: a specific datasets format
dataset = Planetoid(root='./data/Cora', name='Cora') # Only one Graph
# datasets中含有的一些属性: dataset.num_classes, dataset.num_node_features, dataset.num_edge_features, dataset.num_features
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Nov 27 09:35:12 2019
@author: dipu
"""
from torch_geometric.datasets import ShapeNet
dataset = ShapeNet(root='/tmp/ShapeNet', categories=['Airplane'])
print(dataset[0])
import torch_geometric.transforms as T
from torch_geometric.datasets import ShapeNet
dataset = ShapeNet(root='/tmp/ShapeNet',
categories=['Airplane'],
pre_transform=T.KNNGraph(k=6))
data = dataset[0]
dataset = ShapeNet(root='/tmp/ShapeNet',
categories=['Airplane'],
pre_transform=T.KNNGraph(k=6),
transform=T.RandomTranslate(0.01))
data = dataset[0]
from torch_geometric.datasets import ShapeNet
import torch_geometric.transforms as T
dataset = ShapeNet(root='/tmp/ShapeNet', categories=['Airplane'])
print(dataset[0])
# convert point cloud dataset into a graph dataset, create a nearest neighbor graph via transforms
dataset = ShapeNet(root='/tmp/ShapeNet', categories=['Airplane'], pre_transform=T.KNNGraph(k=6))
print(dataset[0])
# add random noise to each node
dataset = ShapeNet(root='/tmp/ShapeNet', categories=['Airplane'],
pre_transform=T.KNNGraph(k=6), transform=T.RandomTranslate(0.01))
print(dataset[0])
def __init__(self, pos_list: list):
super(PRED_DATASET, self).__init__()
self.graph_transform = GT.KNNGraph(k=6)
self.pos_list = pos_list
