def get_data():
dataset = args.name
path = '../data/geometric/ShapeNet-Bag'
trainset = ShapeNet(path, "Bag")
testset = ShapeNet(path, "Bag")
lenTrain = len(trainset)
lenTest = len(testset)
print("Len Dataset:", lenTrain)
trainLoader = DataLoader(trainset[:lenTrain], batch_size=1, shuffle=False)
testloader = DataLoader(trainset[:lenTest], batch_size=1, shuffle=False)
print("Len TrainLoader:", len(trainLoader))
return trainLoader, testloader
def test_data_transforms(self):
"""
export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64:/usr/local/cudnn-10.0-v7.6.5.32
proxychains python -c "from template_lib.examples.DGL.geometric.test_pytorch_geometric import TestingGeometric;\
TestingGeometric().test_data_transforms()"
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'PORT' not in os.environ:
os.environ['PORT'] = '6006'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '1'
# func name
assert sys._getframe().f_code.co_name.startswith('test_')
command = sys._getframe().f_code.co_name[5:]
class_name = self.__class__.__name__[7:] \
if self.__class__.__name__.startswith('Testing') \
else self.__class__.__name__
outdir = f'results/{class_name}/{command}'
from datetime import datetime
TIME_STR = bool(int(os.getenv('TIME_STR', 0)))
time_str = datetime.now().strftime("%Y%m%d-%H_%M_%S_%f")[:-3]
outdir = outdir if not TIME_STR else (outdir + '_' + time_str)
print(outdir)
import collections, shutil
shutil.rmtree(outdir, ignore_errors=True)
os.makedirs(outdir, exist_ok=True)
import torch_geometric.transforms as T
from torch_geometric.datasets import ShapeNet
from template_lib.d2.data.build_points_toy import plot_points
dataset = ShapeNet(root='datasets/shapenet', categories=['Airplane'])
idx = -1
plot_points(dataset[idx].pos)
plot_points(dataset[idx].x)
dataset = ShapeNet(root='datasets/shapenet',
categories=['Airplane'],
pre_transform=T.KNNGraph(k=6))
dataset = ShapeNet(root='datasets/shapenet',
categories=['Airplane'],
pre_transform=T.KNNGraph(k=6),
transform=T.RandomTranslate(0.01))
pass
def load_dataset(path,
transform=None,
pre_transform=None,
pre_filter=None,
category=None,
name='10',
test_area=6):
if path.name == 'ShapeNet':
train_dataset = ShapeNet(path,
category,
split='trainval',
transform=transform,
pre_transform=pre_transform,
pre_filter=pre_filter)
test_dataset = ShapeNet(path,
category,
split='test',
transform=transform,
pre_transform=pre_transform,
pre_filter=pre_filter)
elif path.name == 'ModelNet':
train_dataset = ModelNet(path,
name=name,
train=True,
transform=transform,
pre_transform=pre_transform,
pre_filter=pre_filter)
test_dataset = ModelNet(path,
name=name,
train=False,
transform=transform,
pre_transform=pre_transform,
pre_filter=pre_filter)
elif path.name == 'S3DIS':
train_dataset = S3DIS(path,
test_area=test_area,
train=True,
transform=transform,
pre_transform=pre_transform,
pre_filter=pre_filter)
test_dataset = S3DIS(path,
test_area=test_area,
train=False,
transform=transform,
pre_transform=pre_transform,
pre_filter=pre_filter)
return train_dataset, test_dataset
def data_transforms():
dataset = 'ShapeNet'
path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..',
'data', dataset)
dataset = ShapeNet(root=path, categories=['Airplane'])
print(dataset[0])
dataset = ShapeNet(root=path,
categories=['Airplane'],
pre_transform=T.KNNGraph(k=6))
print(dataset[0])
dataset = ShapeNet(root=dataset,
categories=['Airplane'],
pre_transform=T.KNNGraph(k=6),
transform=T.RandomTranslate(0.01))
print(dataset[0])
pass
def __init__(self, root_dir, category, train=True, transform=None, npoints=2500):
categories = ['Airplane', 'Bag', 'Cap', 'Car', 'Chair', 'Earphone', 'Guitar',
'Knife', 'Lamp', 'Laptop', 'Motorbike', 'Mug', 'Pistol', 'Rocket', 'Skateboard', 'Table']
# assert os.path.exists(root_dir)
assert category in categories
self.npoints = npoints
self.dataset = ShapeNet(root_dir, category, train, transform)
def __init__(self, dataset_opt, training_opt):
super().__init__(dataset_opt, training_opt)
self._data_path = os.path.join(dataset_opt.dataroot, 'ShapeNet')
self._category = dataset_opt.shapenet.category
transform = T.Compose([
T.RandomTranslate(0.01),
T.RandomRotate(15, axis=0),
T.RandomRotate(15, axis=1),
T.RandomRotate(15, axis=2)
])
pre_transform = T.NormalizeScale()
train_dataset = ShapeNet(self._data_path, self._category, train=True, transform=transform,
pre_transform=pre_transform)
test_dataset = ShapeNet( self._data_path, self._category, train=False,
pre_transform=pre_transform)
self.create_dataloaders(train_dataset, test_dataset, validation=None)
def load_dataset(args):
# load ShapeNet dataset
if args.dataset == 'shapenet':
pre_transform, transform = augment_transforms(args)
categories = args.categories.split(',')
train_dataset = ShapeNet('../data_root/ShapeNet_normal', categories, split='trainval', include_normals=False,
pre_transform=pre_transform, transform=transform)
test_dataset = ShapeNet('../data_root/ShapeNet_normal', categories, split='test', include_normals=False,
pre_transform=pre_transform, transform=T.FixedPoints(args.num_pts))
train_dataloader = DataLoader(train_dataset, batch_size=args.bsize, shuffle=True,
num_workers=6, drop_last=True)
test_dataloader = DataLoader(test_dataset, batch_size=args.bsize, shuffle=True,
num_workers=6, drop_last=True)
# load ModelNet dataset
if args.dataset == 'modelnet':
pre_transform, transform = augment_transforms(args)
train_dataset = ModelNet('../data_root/ModelNet40', name='40', train=True,
pre_transform=pre_transform, transform=transform)
test_dataset = ModelNet('../data_root/ModelNet40', name='40', train=False,
pre_transform=pre_transform, transform=T.SamplePoints(args.num_pts))
train_dataloader = DataLoader(train_dataset, batch_size=args.bsize, shuffle=True,
num_workers=6, drop_last=True)
test_dataloader = DataLoader(test_dataset, batch_size=args.bsize, shuffle=True,
num_workers=6, drop_last=True)
# load completion3D dataset
if args.dataset == 'completion3D':
pre_transform, transform = augment_transforms(args)
categories = args.categories.split(',')
train_dataset = completion3D_class('../data_root/completion3D', categories, split='train',
include_normals=False, pre_transform=pre_transform, transform=transform)
test_dataset = completion3D_class('../data_root/completion3D', categories, split='val',
include_normals=False, pre_transform=pre_transform, transform=transform)
train_dataloader = DataLoader(train_dataset, batch_size=args.bsize, shuffle=True,
num_workers=8, drop_last=True)
test_dataloader = DataLoader(test_dataset, batch_size=args.bsize, shuffle=False,
num_workers=8, drop_last=True)
return train_dataloader, test_dataloader
def __init__(self,
n_sweeps=2,
n_points=512,
max_distance=None,
flow_dim=3,
**kwargs):
"""
Args:
kwargs:
"""
self.n_sweeps = n_sweeps
self.n_points = n_points
self.max_distance = max_distance
self.flow_dim = flow_dim
self.occlusion = kwargs.get("occlusion", True)
self.partition = kwargs.get("partition", "train")
self.overfit = kwargs.get("overfit", False)
self.data_size = kwargs.get("data_size", 4242)
if (self.data_size is None) or (self.data_size < 1):
self.data_size = 4242
self.dataset_len = 4242 if self.partition == "train" else 242
self.max_objects = 10
self.odds = 2 if self.partition == "train" else 10
# root_path = "../../../media/deepstorage01/datasets_external/"
root_path = "data/"
self.dataset = ShapeNet(
root=root_path,
# transform=T.SamplePoints(n_sweeps * n_points),
# pre_transform=T.Cartesian(),
split=self.partition)
self.loader = DataLoader_geo(self.dataset, batch_size=1, shuffle=True)
# Initialize kwargs
self.random_transition = kwargs.get("random_transition", False)
self.acceleration = kwargs.get("acceleration", False)
self.rotate = kwargs.get("rotate", True)
# set some needed values:
self.sigma_transition = 0.1 * self.random_transition # defines the variance of the transitions
self.sigma_pc = np.minimum(1.0, np.maximum(
self.n_points / 10.0, 0.1)) # variance to generate the point cloud
self.a = 1 + self.acceleration # defines if speed is constant or if acceleration is constant.
self.cache = {}
import torch_geometric.transforms as T
from torch_geometric.datasets import ShapeNet
from torch_geometric.loader import DataLoader
from torch_geometric.nn import MLP, DynamicEdgeConv
category = 'Airplane' # Pass in `None` to train on all categories.
path = osp.join(osp.dirname(osp.realpath(__file__)), '..', 'data', 'ShapeNet')
transform = T.Compose([
T.RandomTranslate(0.01),
T.RandomRotate(15, axis=0),
T.RandomRotate(15, axis=1),
T.RandomRotate(15, axis=2)
])
pre_transform = T.NormalizeScale()
train_dataset = ShapeNet(path, category, split='trainval', transform=transform,
pre_transform=pre_transform)
test_dataset = ShapeNet(path, category, split='test',
pre_transform=pre_transform)
train_loader = DataLoader(train_dataset, batch_size=10, shuffle=True,
num_workers=6)
test_loader = DataLoader(test_dataset, batch_size=10, shuffle=False,
num_workers=6)
class Net(torch.nn.Module):
def __init__(self, out_channels, k=30, aggr='max'):
super().__init__()
self.conv1 = DynamicEdgeConv(MLP([2 * 6, 64, 64]), k, aggr)
self.conv2 = DynamicEdgeConv(MLP([2 * 64, 64, 64]), k, aggr)
self.conv3 = DynamicEdgeConv(MLP([2 * 64, 64, 64]), k, aggr)
from torch_geometric.datasets import ShapeNet
import torch_geometric.transforms as T
dataset = ShapeNet(root='/tmp/ShapeNet', categories=['Airplane'])
print(dataset[0])
# convert point cloud dataset into a graph dataset, create a nearest neighbor graph via transforms
dataset = ShapeNet(root='/tmp/ShapeNet', categories=['Airplane'], pre_transform=T.KNNGraph(k=6))
print(dataset[0])
# add random noise to each node
dataset = ShapeNet(root='/tmp/ShapeNet', categories=['Airplane'],
pre_transform=T.KNNGraph(k=6), transform=T.RandomTranslate(0.01))
print(dataset[0])
import torch_geometric.transforms as T
from torch_geometric.datasets import ShapeNet
dataset = ShapeNet(root='/media/lepoeme20/Data/graphs/ShapeNet',
categories=['Airplane'])
print(dataset[0])
# Convert the point cloud dataset into a graph dataset
# by generating nearest neighbor graphs from the point clouds via transforms:
transformed_dataset = ShapeNet(root='/media/lepoeme20/Data/graphs/ShapeNet',\
categories=['Airplane'], pre_transform=T.KNNGraph(k=6), transform=T.RandomTranslate(0.01))
print(transformed_dataset[0])
from torch_geometric.utils import mean_iou
from pointnet2_classification import MLP
category = 'Airplane'
path = osp.join(osp.dirname(osp.realpath(__file__)), '..', 'data', 'ShapeNet')
transform = T.Compose([
T.RandomTranslate(0.01),
T.RandomRotate(15, axis=0),
T.RandomRotate(15, axis=1),
T.RandomRotate(15, axis=2)
])
pre_transform = T.NormalizeScale()
train_dataset = ShapeNet(path,
category,
train=True,
transform=transform,
pre_transform=pre_transform)
test_dataset = ShapeNet(path,
category,
train=False,
pre_transform=pre_transform)
train_loader = DataLoader(train_dataset,
batch_size=10,
shuffle=True,
num_workers=6)
test_loader = DataLoader(test_dataset,
batch_size=10,
shuffle=False,
num_workers=6)
# ================================== DATA TRANSFORMS =============================================== # ransforms are a common way in torchvision to transform images and perform augmentation. PyTorch # Geometric comes with its own transforms, which expect a Data object as input and return a new # transformed Data object. Transforms can be chained together using # torch_geometric.transforms.Compose and are applied before saving a processed dataset # (pre_transform) on disk or before accessing a graph in a dataset (transform). # Let’s look at an example, where we apply transforms on the ShapeNet dataset (containing 17,000 # 3D shape point clouds and per point labels from 16 shape categories). from torch_geometric.datasets import ShapeNet dataset = ShapeNet(root='/tmp/ShapeNet', category='Airplane') print(data[0]) # Data(pos=[2518, 3], y=[2518]) # We can convert the point cloud dataset into a graph dataset by generating nearest neighbor graphs # from the point clouds via transforms: import torch_geometric.transforms as T from torch_geometric.datasets import ShapeNet dataset = ShapeNet(root='/tmp/ShapeNet', category='Airplane', pre_transform=T.NNGraph(k=6)) print(data[0]) # Data(edge_index=[2, 17768], pos=[2518, 3], y=[2518]) # In addition, we can use the transform argument to randomly augment a Data object, # e.g. translating each node position by a small number: import torch_geometric.transforms as T
perm = torch.randperm(len(dataset))
dataset = dataset[perm]
dataset = Planetoid(root='./data/Cora', name='Cora')
"""
-----------Mini-batches------------
"""
dataset = TUDataset(root='./data/ENZYMES', name='ENZYMES')
loader = DataLoader(dataset, batch_size=32, shuffle=True)
for i, data in enumerate(loader):
# In the first batch, there are 1047 nodes (belong to the 32 graphs (i.e.,32 batches)), [1047, 3]
print(data.x.size())
"""
-----------Data Transforms------------
"""
# Only datasets
dataset = ShapeNet(root='./data/ShapeNet', categories=['Airplane'])
print(dataset[0]) # >>> Data(pos=[2518, 3], y=[2518])
# Constucting the Graph with neighbor
dataset = ShapeNet(
root='./data/ShapeNet',
categories=['Airplane'],
pre_transform=T.KNNGraph(
k=6)) # >>> Data(edge_index=[2, 15108], pos=[2518, 3], y=[2518])
# translate each node position by a small number:
dataset = ShapeNet(root='/tmp/ShapeNet',
categories=['Airplane'],
pre_transform=T.KNNGraph(k=6),
transform=T.RandomTranslate(0.01))
"""
-----------Learning Methods on Graphs---------------
Let's implement a two-layer GCN
from torch_geometric.datasets import ShapeNet
from torch_geometric.transforms import KNNGraph
# # --------读取的是点云数据------------
# point_dataset = ShapeNet(root='../datasets/ShapeNet', categories=['Airplane'])
# print(point_dataset)
#
# point_data = point_dataset[0]
# print(point_data)
#
# print('\n')
# --------利用transform转换成图数据-----------
graph_dataset = ShapeNet(root='../datasets/ShapeNet',
categories=['Airplane'],
pre_transform=KNNGraph(k=6))
print(graph_dataset)
graph_data = graph_dataset[0]
print(graph_data)
# Tips: 两个数据集读取，都会在数据集文件夹下生成processed文件，所以每次换读取方式记得删除文件
def LoadDataset(dataset_train,
dataset_test,
pre_transform_train=None,
transform_train=None,
pre_transform_test=None,
transform_test=None,
category=None):
path_train = osp.join(DATASET_PATH, dataset_test)
path_test = osp.join(DATASET_PATH, dataset_train)
if not osp.exists(path_train):
os.makedirs(path_train)
if not osp.exists(path_test):
os.makedirs(path_test)
_train_dataset = None
if (dataset_train == 'ModelNet10'):
_train_dataset = ModelNet(path_train, '10', True, transform_train,
pre_transform_train)
elif (dataset_train == 'ModelNet40'):
_train_dataset = ModelNet(path_train, '40', True, transform_train,
pre_transform_train)
elif (dataset_train == 'PoissonModelNet40'):
_train_dataset = PoissonModelNet(path_train, '40', True)
elif (dataset_train == 'PoissonModelNet10'):
_train_dataset = PoissonModelNet(path_train, '10', True)
elif (dataset_train == 'ShapeNet'):
_train_dataset = ShapeNet(path_train,
category,
train=True,
transform=transform_train,
pre_transform=pre_transform_train)
_test_dataset = None
if (dataset_test == 'ModelNet10'):
_test_dataset = ModelNet(path_test, '10', False, transform_test,
pre_transform_test)
elif (dataset_test == 'ModelNet40'):
_test_dataset = ModelNet(path_test, '40', False, transform_test,
pre_transform_test)
elif (dataset_test == 'PoissonModelNet40'):
_test_dataset = PoissonModelNet(path_test, '40', False)
elif (dataset_test == 'PoissonModelNet10'):
_test_dataset = PoissonModelNet(path_test, '10', False)
elif (dataset_test == 'ShapeNet'):
_test_dataset = ShapeNet(path_test,
category,
train=False,
pre_transform=pre_transform_test)
if _train_dataset is None or _test_dataset is None:
print("Invalid dataset requested!")
return (None, None)
else:
print('Dataset Train: {}'.format(dataset_train))
logging.info('Dataset Train: {}'.format(dataset_train))
print('Dataset Test: {}'.format(dataset_test))
logging.info('Dataset Test: {}'.format(dataset_test))
return (_train_dataset, _test_dataset)
