def __init__(self, model, datagen, config, tester):
self.model = model
self.config = config
self.datagen = datagen
self.tester = tester
self.visualizer = Visualizer(config, 'train')
self.storage_client = dl.Storage()
def __init__(self, classifier):
self.classifier = classifier
self.visualizer = Visualizer()
self.test = ClassificationTest(self.classifier.dataset,
self.classifier)
self.loss_interval = 0
self.print_loss = False
self.precision_interval = 0
self.print_precision = False
self.trainset_precision_interval = 0
self.print_trainset_precision = False
def visualize_region_growing(pc):
""" Visualization of region growing on a given set of points """
pc_sub = pc[np.isin(pc[:, -1], [0, 1, 4]), :]
pc_stem = pc[np.isin(pc[:, -1], [2, 5])]
points, colors = pc_sub[:, :3], pc_sub[:, 3:6]
# region growing
# curvatures, normals = estimate_curvature_and_normals(points, n_neighbors=50)
cluster_mask = region_growing(points, pc_sub[:, 6:9], pc_sub[:, 9:10], min_points=1000)
# visualize clusters
vis = Visualizer(background_color=(1, 1, 1))
# color each region in a random color
colors = np.empty_like(points)
for i in np.unique(cluster_mask):
colors[(cluster_mask==i), :] = np.random.uniform(0, 1, size=3)
# vis.add_by_features(points[(cluster_mask==i)], colors[(cluster_mask==i)])
# ignore outliers
colors[(cluster_mask==-1), :] = 1
vis.add_by_features(points, colors)
vis.n -= 1
vis.add_by_features(pc_stem[:, :3], np.zeros_like(pc_stem[:, :3]))
vis.run()
def visualize_pointcloud_breeding(pc):
# get normals and curvature
normals, curvature = pc[:, 6:9], pc[:, 9]
# augment from each point
points = pc[:, :3]
# generate one vector on each plane spanned by a
# point and its normal in the pointcloud
r = np.random.uniform(-1, 1, size=points.shape)
n = np.cross(normals, r)
n /= np.linalg.norm(n, axis=-1, keepdims=True)
# create new points
d = curvature / curvature.max() * 1e-2
new_points = points + n * d.reshape(-1, 1)
# find the nearest neighbors of each new point
tree = NearestNeighbors(n_neighbors=5, algorithm='ball_tree', n_jobs=-1).fit(points)
nearest_idx = tree.kneighbors(new_points, return_distance=False)
# interpolate color from the n nearest points
colors = pc[nearest_idx, 3:6].mean(axis=1)
# stack all together
full_points = np.concatenate((points, new_points), axis=0)
full_colors = np.concatenate((pc[:, 3:6], colors), axis=0)
# show
vis = Visualizer()
vis.add_by_features(points, pc[:, 3:6]/255)
vis.add_by_features(new_points, colors/255)
vis.add_by_features(full_points, full_colors/255)
vis.run(True)
class ClassificationTrainer:
def __init__(self, classifier):
self.classifier = classifier
self.visualizer = Visualizer()
self.test = ClassificationTest(self.classifier.dataset,
self.classifier)
self.loss_interval = 0
self.print_loss = False
self.precision_interval = 0
self.print_precision = False
self.trainset_precision_interval = 0
self.print_trainset_precision = False
# decide how often loss should be logged, and if it should be printed to console
def enable_loss(self, interval=1, print=False):
self.loss_interval = interval
self.print_loss = print
# decide how often precision should be logged, and if it should be printed to console
def enable_precision(self, interval=5, print=False):
self.precision_interval = interval
self.print_precision = print
# decide how often loss shall be should, and if it should be printed to console
def enable_trainset_precision(self, interval=10, print=False):
self.trainset_precision_interval = interval
self.print_trainset_precision = print
# trains the classifier and prints info as configured
def train(self, epochs=10):
for _ in range(epochs):
loss = self.classifier.train_epoch()
epoch = self.classifier.epoch
if self.loss_interval != 0:
if epoch % self.loss_interval == 0:
self.visualizer.add_loss(epoch, loss)
if self.print_loss:
print("[" + str(epoch) + "] " + "Loss: " + str(loss))
if self.precision_interval != 0:
if epoch % self.precision_interval == 0:
precision = self.test.test()
self.visualizer.add_prec(epoch, precision)
if self.print_precision:
print("[" + str(epoch) + "] " + "Precision: " +
str(precision))
if self.trainset_precision_interval != 0:
if epoch % self.trainset_precision_interval == 0:
trainset_precision = self.test.test_trainset()
self.visualizer.add_train_prec(epoch, trainset_precision)
if self.print_trainset_precision:
print("[" + str(epoch) + "] " +
"Trainset Precision: " + str(trainset_precision))
# shows a diagram of the training progress
def show(self):
self.visualizer.show()
def __init__(self, config, model_path):
# initialize visualizer
Visualizer.__init__(self, background_color=(1, 1, 1))
# save configurations
self.class_bins = OrderedDict(config['data']['classes'])
self.classes = list(self.class_bins.keys())
# get features used by model
self.features = config['data']['features']
feature_dim = config['data']['feature_dim']
# create model
self.model = Model_SEG(K=len(self.classes), feat_dim=feature_dim)
# load parameters
self.model.load_encoder(os.path.join(model_path, 'encoder.model'))
self.model.load_segmentater(
os.path.join(model_path, 'segmentater.model'))
# evaluate model
self.model.eval()
def basic_visualization(pc):
# get points and colors
points = raw[:, :3]
colors = raw[:, 3:6] / 255
# get label-colors
y = raw[:, -1:]
classes = list(class2color.keys())
get_color = lambda i: class2color[classes[int(i)]]
label_colors = np.apply_along_axis(get_color, axis=-1, arr=y.reshape(-1, 1))
# visualize
vis = Visualizer(background_color=(1, 1, 1))
vis.add_by_features(points, colors)
vis.add_by_features(points, label_colors)
vis.run()
def __init__(self, config, model_path):
# initialize visualizer
Visualizer.__init__(self, background_color=(1, 1, 1))
# data preparation
self.align_pointclouds = config['preparation']['align_pointclouds']
# get classes predicted by model
self.hierarchical_classes = [
OrderedDict(classes)
for classes in config['data']['hierarchy_classes']
]
K = len(self.hierarchical_classes[0])
# get features and feature dimension
self.features = config['data']['features']
feat_dim = config['data']['feature_dim']
# create model
self.model = Model_SEG(K=K, feat_dim=feat_dim)
# load parameters
self.model.load_encoder(os.path.join(model_path, "encoder.model"))
self.model.load_segmentater(
os.path.join(model_path, "segmentater.model"))
# evaluate
self.model.eval()
def voxel_down_sample_octree(pc, n_points):
points = pc[:, :3]
print(points.shape[0])
voxels = group_points_by_octree(points, 1_000)
vis = Visualizer(background_color=(1, 1, 1))
colors = np.empty_like(points)
for vox in voxels:
colors[vox, :] = np.random.uniform(0, 1, size=3)
vis.add_by_features(points, colors)
samples_points = []
# get the number of points selected from each voxel
weights = np.asarray([len(voxel)/points.shape[0] for voxel in voxels])
points_per_voxel = np.ceil(weights * n_points).astype(np.int32)
# select random points from each voxel and add to list
idx = sum([sample(list(v), min(n, len(v))) for v, n in zip(voxels, points_per_voxel)], [])
vis.add_by_features(points[idx, :], colors[idx, :])
vis.run()
def visualize_graph_partition(points, k=-1):
""" Separate visualization of k normalized clusters created by region growing algorithm
where the colors show the best partition of the nearest neighbor graph in each cluster
"""
# import graph and partition
import networkx as nx
import community
# region growing
curvatures, normals = estimate_curvature_and_normals(points, n_neighbors=50)
cluster_mask = region_growing(points, normals, curvatures, min_points=100)
# get cluster indices of interest ignoreing outliers
cluster_idx = np.unique(cluster_mask)
cluster_idx = cluster_idx[(cluster_idx!=-1)]
cluster_idx = cluster_idx[:k] if k > -1 else cluster_idx
# create visualizer
vis = Visualizer(background=(1, 1, 1))
# visualize each cluster
for i in cluster_idx:
# get points of current cluster
cluster = normalize_pc(points[(cluster_mask==i)])
# build ajacency matrix for neighbor graph
tree = NearestNeighbors(algorithm='ball_tree', radius=0.03, n_jobs=-1).fit(cluster)
graph_matrix = tree.radius_neighbors_graph(cluster, mode='distance')
# build graph from sparse adjacency matrix and compute best partition
G = nx.from_scipy_sparse_matrix(graph_matrix)
partition = community.best_partition(G)
# color each partition in a random color
community_colors, colors = {}, np.empty_like(cluster)
for i, c in partition.items():
# get random color
if c not in community_colors:
community_colors[c] = np.random.uniform(0, 1, size=3)
# set color
colors[i, :] = community_colors[c]
# add to visualizer
vis.add_by_features(cluster, colors)
# show
vis.run()
def visualize_aligned_region_growing_and_breeding_clusters(pc, k=-1):
""" Separate visualization of k normalized, aligned and breeded
clusters created by region growing algorithm
"""
# region growing
points, curvatures, normals = pc[:, :3], pc[:, 9], pc[:, 6:9]
cluster_mask = region_growing(points, normals, curvatures, min_points=1000)
# get cluster indices of interest ignoreing outliers
cluster_idx = np.unique(cluster_mask)
cluster_idx = cluster_idx[(cluster_idx!=-1)]
cluster_idx = cluster_idx[:k] if k > -1 else cluster_idx
# create visualizer
vis = Visualizer()
# visualize each cluster
for i in cluster_idx:
# get points of current cluster
mask = (cluster_mask==i)
cluster = points[mask]
# make colors from classes
get_color = lambda i: list(class2color.values())[int(i)]
colors = np.apply_along_axis(get_color, axis=-1, arr=pc[mask, -1:])
# principle component anaylsis
pca = PCA(n_components=1).fit(cluster)
# print(i, pca.explained_variance_, pca.explained_variance_ratio_)
a = pca.components_[0].reshape(-1, 1)
a /= np.linalg.norm(a)
# priciple component must show away from origin such that the aligned custer/pointcloud is not upside down
mean = cluster.mean(axis=0).reshape(-1, 1)
b = mean / np.linalg.norm(mean)
# compute angle between principle component and position of pointcloud and inverse direction if needed
a *= 1 if (a.T @ b < 0) else -1
# create rotation matrix to align principle component to (0, 1, 0)
c = a + np.asarray([0, 1, 0]).reshape(-1, 1) # b = (0, 1, 0).T
R = 2 * (c @ c.T) / (c.T @ c) - np.eye(3)
# rotate points
rotated_points = cluster @ R.T
# normalize points
rotated_points = normalize_pc(rotated_points)
# get normals and curvature of current cluster
cur_normals, cur_curvature = normals[mask], curvatures[mask]
l = max(ceil(10_000/cluster.shape[0]) - 1, 0)
# generate one vector on each plane spanned by a
# point and its normal in the pointcloud
r = np.random.uniform(-1, 1, size=(cluster.shape[0] * l, 3))
n = np.cross(cur_normals.repeat(l, axis=0), r)
n /= np.linalg.norm(n, axis=-1, keepdims=True)
# create new points
d = np.random.uniform(-5e-2, 5e-2, size=n.shape[0])
new_points = rotated_points.repeat(l, axis=0) + n * d.reshape(-1, 1)
# stack points together
all_points = np.concatenate((rotated_points, new_points), axis=0)
all_colors = colors.repeat(l+1, axis=0)
# visualize
vis.add_by_features(rotated_points, colors/255, normalize=False)
vis.add_by_features(all_points, all_colors/255, normalize=False)
# increase distance between pointclouds
vis.n += 1
vis.run(show_coordinate_frame=True)
class Trainer():
def __init__(self, model, datagen, config, tester):
self.model = model
self.config = config
self.datagen = datagen
self.tester = tester
self.visualizer = Visualizer(config, 'train')
self.storage_client = dl.Storage()
def plotGraphs(self, folder, results):
for result in results:
epoch_list = list(range(1, len(results[result]) + 1))
new_list = [float(x) for x in results[result]]
plt.plot(epoch_list, new_list)
# print('result :', result)
# print('results[result]:',results[result])
plt.xlabel('epoch')
plt.ylabel(result)
plt.savefig(os.path.join(folder, result + '.png'))
self.storage_client.set_file(self.config.exp_name + "_trainviz_" +
result,
os.path.join(folder, result + '.png'),
storage_type='data')
plt.close()
plt.clf()
def train(self):
accumulated_results = dict()
for cur_epoch in range(self.config.num_epochs):
# print()
# print()
# print()
# print('Epoch ',cur_epoch)
# print(' -------------------------------')
results_epoch = self.train_epoch(cur_epoch)
if (not (self.tester is None)):
self.tester.test(cur_epoch)
for current_result in results_epoch:
if current_result in accumulated_results:
accumulated_results[current_result].append(
results_epoch[current_result])
else:
accumulated_results[current_result] = [
results_epoch[current_result]
]
self.plotGraphs(self.config.summary_dir, accumulated_results)
def train_one_epoch(self, accumulated_results, cur_epoch):
results_epoch = self.train_epoch(cur_epoch)
for current_result in results_epoch:
if current_result in accumulated_results:
accumulated_results[current_result].append(
str(results_epoch[current_result]))
else:
accumulated_results[current_result] = [
str(results_epoch[current_result])
]
self.plotGraphs(self.config.summary_dir, accumulated_results)
return accumulated_results
def train_epoch(self, cur_epoch):
num_steps = self.datagen.get_num_batches()
accumulated_results = dict()
start_time_epoch = time.time()
fo = open('/tmp/step.txt', 'w')
fo.close()
for i in range(num_steps):
start_time = time.time()
results_step, data_batch = self.train_step(i)
if ((((i + 1) % self.config.step_result_print_frequency) == 0)):
# print(' Step : ',i+1 , " of total : ",num_steps)
fo = open('/tmp/step.txt', 'w')
fo.write('\nStep : ' + str(i + 1) + ", epoch : " +
str(cur_epoch))
# fo.close()
# self.storage_client.set_file(self.config.exp_name+'_steplog','/tmp/step.txt', storage_type='data')
for current_result in results_step:
if current_result in accumulated_results:
accumulated_results[current_result].append(
results_step[current_result])
else:
accumulated_results[current_result] = [
results_step[current_result]
]
# if ( (((i+1)%self.config.step_result_print_frequency)==0) ):
# print(current_result, " : ", results_step[current_result])
if ((i % self.config.visualization_frequency) == 0):
start_t = time.time()
order = data_batch['order']
type_list = data_batch['type_list']
self.visualizer.Visualize(data_batch,
order,
type_list,
cur_epoch,
accumulate=False)
shutil.make_archive(self.config.exp_name + "_viz", 'zip',
self.config.visualization_dir)
self.storage_client.set_file(self.config.exp_name + "_viz",
self.config.exp_name + "_viz.zip",
storage_type='data')
emptyInsideFolder(self.config.visualization_dir)
# print('viz time : ',time.time()-start_t)
sys.stdout.flush()
if ((((i + 1) % self.config.step_result_print_frequency) == 0)):
# fo = open('/tmp/step.txt','a+')
fo.write('\nstep took : ' + str(time.time() - start_time))
hdd = psutil.disk_usage('/home/')
fo.write("\nTotal: " + str(hdd.total * 1.0 / (2**30)) + " GiB")
fo.write("\nUsed: " + str(hdd.used * 1.0 / (2**30)) + " GiB")
fo.write("\nFree: " + str(hdd.free * 1.0 / (2**30)) + " GiB")
hdd = psutil.disk_usage('/')
fo.write("\nTotal: " + str(hdd.total * 1.0 / (2**30)) + " GiB")
fo.write("\nUsed: " + str(hdd.used * 1.0 / (2**30)) + " GiB")
fo.write("\nFree: " + str(hdd.free * 1.0 / (2**30)) + " GiB")
fo.close()
self.storage_client.set_file(self.config.exp_name + '_steplog',
'/tmp/step.txt',
storage_type='data')
os.remove('/tmp/step.txt')
# print('step took : ',time.time()-start_time)
# print('Overall train results of epoch ',cur_epoch,' : ')
for current_result in accumulated_results:
accumulated_results[current_result] = np.mean(
accumulated_results[current_result])
# print(current_result,' : ',accumulated_results[current_result])
if (((cur_epoch % self.config.model_save_frequency) == 0)):
# print('saving')
self.model.save(cur_epoch)
self.visualizer.reset()
# print('epoch took : ',time.time()-start_time_epoch)
# print('-------------------------------')
return accumulated_results
def train_step(self, cur_step):
start_time = time.time()
data_train_step = self.datagen.get_batch()
data_time = time.time()
# print('data_time ',data_time-start_time)
results_step, data_batch = self.model.run_batch(
data_train_step, "train", cur_step)
run_time = time.time()
# print('run_time',run_time-data_time)
return results_step, data_batch
class Tester():
def __init__(self, model, datagen, config):
self.model = model
self.config = config
self.datagen = datagen
self.visualizer = Visualizer(config, 'test')
def test(self, cur_epoch):
accumulated_results = dict()
print()
print()
print()
print('Testing now')
print('Epoch ', cur_epoch)
print(' -------------------------------')
results_epoch = self.test_epoch(cur_epoch)
for current_result in results_epoch:
if current_result in accumulated_results:
accumulated_results[current_result].append(
results_epoch[current_result])
else:
accumulated_results[current_result] = [
results_epoch[current_result]
]
plotGraphs(self.config.summary_dir, accumulated_results)
def test_epoch(self, cur_epoch):
num_steps = self.datagen.get_num_batches()
accumulated_results = dict()
start_time_epoch = time.time()
for i in range(num_steps):
start_time = time.time()
results_step, data_batch = self.test_step(i)
if ((((i + 1) % self.config.step_result_print_frequency) == 0)):
print(' Step : ', i + 1, " of total : ", num_steps)
for current_result in results_step:
if current_result in accumulated_results:
accumulated_results[current_result].append(
results_step[current_result])
else:
accumulated_results[current_result] = [
results_step[current_result]
]
if ((((i + 1) %
self.config.step_result_print_frequency) == 0)):
print(current_result, " : ", results_step[current_result])
if ((i % self.config.visualization_frequency) == 0):
start_t = time.time()
order = data_batch['order']
type_list = data_batch['type_list']
self.visualizer.Visualize(data_batch,
order,
type_list,
cur_epoch,
accumulate=True)
sys.stdout.flush()
if ((((i + 1) % self.config.step_result_print_frequency) == 0)):
print('step took : ', time.time() - start_time)
print('Overall test results of epoch ', cur_epoch, ' : ')
for current_result in accumulated_results:
accumulated_results[current_result] = np.mean(
accumulated_results[current_result])
print(current_result, ' : ', accumulated_results[current_result])
self.visualizer.reset()
print('test took : ', time.time() - start_time_epoch)
print('-------------------------------')
return accumulated_results
def test_step(self, cur_step):
start_time = time.time()
data_train_step = self.datagen.get_batch()
data_time = time.time()
# print('data_time ',data_time-start_time)
results_step, data_batch = self.model.run_batch(
data_train_step, "test", cur_step)
run_time = time.time()
# print('run_time',run_time-data_time)
return results_step, data_batch
def __init__(self, model, datagen, config):
self.model = model
self.config = config
self.datagen = datagen
self.visualizer = Visualizer(config, 'test')
def __init__(self, args):
super(ActorLearner, self).__init__()
self.summ_base_dir = args.summ_base_dir
self.local_step = 0
self.global_step = args.global_step
self.local_episode = 0
self.last_saving_step = 0
self.filename = str(args.game)+"_"+str(args.alg_type)+"_"+str(args.actor_id)+"_minimize_local"
self.saver = None
self.actor_id = args.actor_id
self.visdom = args.visdom
# launch python -m visdom.server
self.vis = Visualizer(self.actor_id,self.visdom,args.num_actions) ## default port is 8098 http://localhost:8097. actor_id
self.alg_type = args.alg_type
#print("self.alg.type is: {}".format(self.alg_type))
self.use_monitor = args.use_monitor
self.max_local_steps = args.max_local_steps
self.optimizer_type = args.opt_type
self.optimizer_mode = args.opt_mode
self.num_actions = args.num_actions
self.initial_lr = args.initial_lr
self.lr_annealing_steps = args.lr_annealing_steps
self.num_actor_learners = args.num_actor_learners
self.is_train = args.is_train
self.input_shape = args.input_shape
self.reward_clip_val = args.reward_clip_val
self.q_update_interval = args.q_update_interval
self.restore_checkpoint = args.restore_checkpoint
self.random_seed = args.random_seed
# Shared mem vars
if self.alg_type != "AE":
self.learning_vars = args.learning_vars
else:
self.learning_vars_lower = args.learning_vars_lower
self.learning_vars_upper = args.learning_vars_upper
if self.optimizer_mode == 'local':
if self.alg_type != "AE":
if self.optimizer_type == 'rmsprop':
self.opt_st = np.ones(self.learning_vars.size, dtype=ctypes.c_float)
else:
self.opt_st = np.zeros(self.learning_vars.size, dtype=ctypes.c_float)
else:
if self.optimizer_type == 'rmsprop':
self.opt_st = np.ones(self.learning_vars_lower.size, dtype=ctypes.c_float)
else:
self.opt_st = np.zeros(self.learning_vars_lower.size, dtype=ctypes.c_float)
elif self.optimizer_mode == 'shared':
self.opt_st = args.opt_state_lower
# rmsprop/momentum
self.alpha = args.momentum
# adam
self.b1 = args.b1
self.b2 = args.b2
self.e = args.e
if args.env == 'GYM':
from environments.atari_environment import AtariEnvironment
self.emulator = AtariEnvironment(
args.game,
self.random_seed,
args.visualize,
use_rgb=args.use_rgb,
frame_skip=args.frame_skip,
agent_history_length=args.history_length,
max_episode_steps=args.max_episode_steps,
single_life_episodes=args.single_life_episodes,
)
elif args.env == 'ALE':
from environments.emulator import Emulator
self.emulator = Emulator(
args.rom_path,
args.game,
args.visualize,
self.actor_id,
self.random_seed,
args.single_life_episodes)
else:
raise Exception('Invalid environment `{}`'.format(args.env))
self.grads_update_steps = args.grads_update_steps
self.max_global_steps = args.max_global_steps
self.gamma = args.gamma
self.rescale_rewards = args.rescale_rewards
self.max_achieved_reward = -float('inf')
if self.rescale_rewards:
self.thread_max_reward = 1.0
# Barrier to synchronize all actors after initialization is done
self.barrier = args.barrier
self.game = args.game
# Initizlize Tensorboard summaries
self.summary_ph, self.update_ops, self.summary_ops = self.setup_summaries()
self.summary_op = tf.summary.merge_all()
# open log file each agent
#with open(str(self.filename), 'w') as file_name:
file_name = open(str(self.filename), 'w')
file_name.seek(0)
file_name.truncate()
self.opened_log_file = file_name
self.wr = csv.writer(file_name, quoting=csv.QUOTE_ALL)
if AUC >= 0.75:
torch.save(myModel, save_path)
if __name__ == '__main__':
os.environ["CUDA_VISIBLE_DEVICES"] = Config.gpu
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
save_dir = Config.savePath
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
if Config.isOri:
in_channel = 2
else:
in_channel = 1
myModel = ResNet(in_ch=in_channel, num_classes=2).to(device)
myModel = torch.nn.DataParallel(myModel).cuda()
optimizer = optim.Adam(myModel.parameters(),
lr=Config.base_lr,
weight_decay=Config.weight_decay)
criterion = nn.CrossEntropyLoss()
best = {'loss': 0.0, 'save': ''}
vis = Visualizer(env=Config.env)
train(myModel, optimizer, criterion, Config.num_epochs, Config.batch_size)
def visualize_aligned_region_growing_clusters(pc, k=-1):
""" Separate visualization of k normalized and aligned clusters created by region growing algorithm
where the colors for each point come from the distances to the nearest neighbors in a given radius
"""
points, colors = pc[:, :3], pc[:, 3:6]
# region growing
curvatures, normals = estimate_curvature_and_normals(points, n_neighbors=250)
cluster_mask = region_growing(points, normals, curvatures, min_points=1000)
# get cluster indices of interest ignoreing outliers
cluster_idx = np.unique(cluster_mask)
cluster_idx = cluster_idx[(cluster_idx!=-1)]
cluster_idx = cluster_idx[:k] if k > -1 else cluster_idx
# create visualizer
vis = Visualizer()
# visualize each cluster
for i in cluster_idx:
# # get points of current cluster
cluster = points[(cluster_mask==i)]
cluster_colors = colors[(cluster_mask==i)]
# # compute colors by nearest neighbors
# tree = NearestNeighbors(algorithm='ball_tree', radius=0.03, n_jobs=-1).fit(normalize_pc(cluster))
# distances, _ = tree.radius_neighbors(cluster)
# # compute colors from distances
# colors = np.asarray([[sum(ds)] for ds in distances]) + 0.5
# colors = (colors / colors.max()).repeat(3, axis=-1)
# principle component anaylsis
pca = PCA(n_components=1).fit(cluster)
# print(i, pca.explained_variance_, pca.explained_variance_ratio_)
a = pca.components_[0].reshape(-1, 1)
a /= np.linalg.norm(a)
# priciple component must show away from origin such that the aligned custer/pointcloud is not upside down
mean = cluster.mean(axis=0).reshape(-1, 1)
b = mean / np.linalg.norm(mean)
# compute angle between principle component and position of pointcloud and inverse direction if needed
a *= 1 if (a.T @ b < 0) else -1
# create rotation matrix to align principle component to (0, 1, 0)
c = a + np.asarray([0, 1, 0]).reshape(-1, 1) # b = (0, 1, 0).T
R = 2 * (c @ c.T) / (c.T @ c) - np.eye(3)
# add to visualizer
# pc_orig = vis.add_by_features(cluster, colors)
pc_rot = vis.add_by_features(cluster @ R.T, cluster_colors)
# visualize princple components
# origin_orig, origin_rot = np.asarray(pc_orig.points).mean(axis=0).reshape(-1, 1), np.asarray(pc_rot.points).mean(axis=0).reshape(-1, 1)
# line_points = [a * 0.5 + origin_orig, -a * 0.5 + origin_orig, R @ a * 0.5 + origin_rot, -R @ a * 0.5 + origin_rot]
# create lineset of principle components
# lineset = open3d.geometry.LineSet()
# lineset.points = open3d.utility.Vector3dVector(line_points)
# lineset.lines = open3d.utility.Vector2iVector([[0, 1], [2, 3]])
# lineset.colors = open3d.utility.Vector3dVector([(0, 0, 1), (0, 0, 1)])
# add to visualizer
# vis.add_geometry(lineset)
# increase distance between clusters
vis.n += 1
vis.run(show_coordinate_frame=True)
def train(**kwargs):
opt.parse(kwargs)
vis = Visualizer(opt.env)
# step1: configure model
# model = getattr(models, opt.model)() # opt.model = ResNet34 模块内的文件可以看做是它的属性
model = Mymodel(pretrained=True)
model = model.model
# 直接调用torchvision中的resnet34
# 预训练里面是包括最后全连接层的，所以直接调用会报错： While copying the parameter named fc.weight, whose dimensions in the model are torch.Size([120, 512])
# and whose dimensions in the checkpoint are torch.Size([1000, 512]).
# pre_model = resnet34(pretrained=True)
# Linear = t.nn.Linear(1000, opt.num_classes)
# model = t.nn.Sequential(
# pre_model,
# Linear
# )
if opt.load_model_path:
model.load(opt.load_model_path)
if opt.use_gpu: model.cuda()
# step2: data
train_data = DogBreedData(opt.train_data_root, train=True)
val_data = DogBreedData(opt.train_data_root, train=False)
train_dataloader = DataLoader(train_data,
opt.batch_size,
shuffle=True,
num_workers=opt.num_workers)
val_dataloader = DataLoader(val_data,
opt.batch_size,
shuffle=True,
num_workers=opt.num_workers)
# setp3: loss and optimizer
loss = t.nn.CrossEntropyLoss()
optimizer = t.optim.Adam(model.parameters(),
lr=opt.lr,
weight_decay=opt.weight_decay)
# step4: meters
loss_meter = meter.AverageValueMeter() # 一个类，计算平均值，方差的
# confusion_matrix = meter.ConfusionMeter(120)
previous_loss = 1e100
# train
for epoch in range(opt.max_epoch): # one epoch 表示遍历整个数据集
loss_meter.reset()
# confusion_matrix.reset()
# loss = 0 # 损失值清零，计算每一个epoch的损失值的平均值
for ii, (data, label) in tqdm(enumerate(train_dataloader)):
input = Variable(data)
target = Variable(label.type(t.LongTensor))
if opt.use_gpu:
input = input.cuda()
target = target.cuda()
# 梯度清零
optimizer.zero_grad()
# 前向传播
score = model(input)
# 计算交叉熵损失
loss = loss(
score, target
) # loss = t.nn.CrossEntropyLoss(_WeightedLoss)也是Module类，这里是forward()函数
# 反向传播，梯度下降
loss.backward()
optimizer.step()
# loss update and visualize
# 两种方法对比以下。。
loss_meter.add(
loss.data[0]) # def value(self): return self.mean, self.std
# confusion_matrix.add(score.data, target.data)
# 更新loss
# loss.data[0] += previous_loss
# loss_old = loss.data[0]
# loss_mean = loss_old/float(ii)
if ii % opt.print_freq == opt.print_freq - 1:
# vis.plot('loss', loss_mean)
vis.plot('loss', loss_meter.value()[0])
model.save()
# validate and visualize
# val_cm, val_accuracy = val(model, val_dataloader)
# vis.plot('val_accuracy', val_accuracy)
# vis.log("epoch:{epoch},lr:{lr},loss:{loss},train_cm:{train_cm},val_cm:{val_cm}".format(
# epoch=epoch, loss=loss_meter.value()[0], val_cm=str(val_cm.value()),
# train_cm=str(confusion_matrix.value()), lr=opt.lr))
# update learning rate
if loss_meter.value()[0] > previous_loss:
lr = opt.lr * opt.lr_decay
# 第二种降低学习率的方法:不会有moment等信息的丢失
for param_group in optimizer.param_groups:
param_group['lr'] = lr
previous_loss = loss_meter.value()[0]
def voxel_down_sample_random(pc, voxel_grid_size=0.2, n_points=35_000):
# get points
points = pc[:, :3]
colors = np.ones_like(points)
# get boundings of points
max_ = np.max(points, axis=0)
min_ = np.min(points, axis=0)
# move bounding box anchor to origin
bbox = max_ - min_
# compute number of voxels in each dimesion
n_voxels = np.ceil(bbox/voxel_grid_size).astype(np.int32)
voxel_points = []
# loop over all voxels
for index in tqdm(np.ndindex(*n_voxels), total=np.product(n_voxels)):
# build anchors of current voxels
anchorA = np.asarray(index) * voxel_grid_size + min_
anchorB = anchorA + voxel_grid_size
# get points in current voxel
point_idx = get_points_in_bbox(points, anchorA, anchorB)
if len(point_idx) > 0:
voxel_points.append(point_idx)
sampled_point_idx = []
# compute weight of first voxel
weights = np.asarray([len(idx) / points.shape[0] for idx in voxel_points])
# weights = 1 - np.asarray([len(idx) / points.shape[0] for idx in voxel_points])
# weight = weights[0] / np.sum(weights)
# sample random points from each voxel
for i, idx in enumerate(voxel_points):
# get number of points to sample from current voxel
n_points_from_current = min(ceil(weights[i] * n_points), len(idx))
# sample points random
sampled_point_idx += sample(list(idx), n_points_from_current)
# if i+1 < len(voxel_points):
# # update weight
# weights = 1 - np.asarray([len(idx) / (points.shape[0] - len(sampled_point_idx)) for idx in voxel_points[i+1:]])
# weight = weights[0] / np.sum(weights)
# color points of current voxel
colors[idx] = np.random.uniform(0, 1, size=3)
print(len(sampled_point_idx), n_points, n_voxels)
# get sampled points
sampled_point_idx = sample(sampled_point_idx, n_points)
sampled_points = points[sampled_point_idx, :]
sampled_colors = colors[sampled_point_idx, :]
random_point_idx = sample(range(points.shape[0]), n_points)
random_points = points[random_point_idx, :]
random_colors = colors[random_point_idx, :]
# visualize
vis = Visualizer(background_color=(1, 1, 1))
vis.add_by_features(points, colors)
# vis.add_by_features(random_points, random_colors)
vis.add_by_features(sampled_points, sampled_colors)
vis.run()
def train(**kwargs):
opt.parse(kwargs)
vis = Visualizer(opt.env)
# step1: configure model
# model = getattr(models, opt.model)() # opt.model = ResNet34 模块内的文件可以看做是它的属性
model = Mymodel(pretrained=True)
model = model.model
# 直接调用torchvision中的resnet34
# 预训练里面是包括最后全连接层的，所以直接调用会报错： While copying the parameter named fc.weight, whose dimensions in the model are torch.Size([120, 512])
# and whose dimensions in the checkpoint are torch.Size([1000, 512]).
# pre_model = resnet34(pretrained=True)
# Linear = t.nn.Linear(1000, opt.num_classes)
# model = t.nn.Sequential(
# pre_model,
# Linear
# )
if opt.load_model_path:
model.load(opt.load_model_path)
if opt.use_gpu: model.cuda()
# step2: data
train_data = DogBreedData(opt.train_data_root, train=True)
val_data = DogBreedData(opt.train_data_root, train=False)
train_dataloader = DataLoader(train_data, opt.batch_size,
shuffle=True, num_workers=opt.num_workers)
val_dataloader = DataLoader(val_data, opt.batch_size,
shuffle=True, num_workers=opt.num_workers)
# setp3: loss and optimizer
loss = t.nn.CrossEntropyLoss()
optimizer = t.optim.Adam(model.parameters(), lr=opt.lr, weight_decay=opt.weight_decay)
# step4: meters
loss_meter = meter.AverageValueMeter() # 一个类，计算平均值，方差的
# confusion_matrix = meter.ConfusionMeter(120)
previous_loss = 1e100
# train
for epoch in range(opt.max_epoch): # one epoch 表示遍历整个数据集
loss_meter.reset()
# confusion_matrix.reset()
# loss = 0 # 损失值清零，计算每一个epoch的损失值的平均值
for ii,(data, label) in tqdm(enumerate(train_dataloader)):
input = Variable(data)
target = Variable(label.type(t.LongTensor))
if opt.use_gpu:
input = input.cuda()
target = target.cuda()
# 梯度清零
optimizer.zero_grad()
# 前向传播
score = model(input)
# 计算交叉熵损失
loss = loss(score, target) # loss = t.nn.CrossEntropyLoss(_WeightedLoss)也是Module类，这里是forward()函数
# 反向传播，梯度下降
loss.backward()
optimizer.step()
# loss update and visualize
# 两种方法对比以下。。
loss_meter.add(loss.data[0]) # def value(self): return self.mean, self.std
# confusion_matrix.add(score.data, target.data)
# 更新loss
# loss.data[0] += previous_loss
# loss_old = loss.data[0]
# loss_mean = loss_old/float(ii)
if ii%opt.print_freq == opt.print_freq-1:
# vis.plot('loss', loss_mean)
vis.plot('loss', loss_meter.value()[0])
model.save()
# validate and visualize
# val_cm, val_accuracy = val(model, val_dataloader)
# vis.plot('val_accuracy', val_accuracy)
# vis.log("epoch:{epoch},lr:{lr},loss:{loss},train_cm:{train_cm},val_cm:{val_cm}".format(
# epoch=epoch, loss=loss_meter.value()[0], val_cm=str(val_cm.value()),
# train_cm=str(confusion_matrix.value()), lr=opt.lr))
# update learning rate
if loss_meter.value()[0] > previous_loss:
lr = opt.lr * opt.lr_decay
# 第二种降低学习率的方法:不会有moment等信息的丢失
for param_group in optimizer.param_groups:
param_group['lr'] = lr
previous_loss = loss_meter.value()[0]
def train(dataroot):
#ipdb.set_trace()
#============================================
#============ setup visdom ============
#============================================
vis = Visualizer(opt.trainenv) # opt.env
#vis = visdom.Visdom(env='train')
#============================================
#============= load model ==============
#============================================
anetwork = audioNetwork().double()
vnetwork = videoNetwork().double()
if opt.load_amodel_path:
anetwork.load(opt.load_amodel_path)
if opt.load_vmodel_path:
vnetwork.load(opt.load_vmodel_path)
anetwork.to(opt.device)
vnetwork.to(opt.device)
#============================================
#============ load data ===============
#============================================
trainData = lipDataset(dataroot, True, False, False, opt.augment)
#ipdb.set_trace() #len(trainData)
trainDataLoader = DataLoader(trainData,
batch_size=opt.batch_size,
num_workers=opt.num_workers,
shuffle=opt.shuffle,
drop_last=opt.drop_last)
#============================================
#====== optimizer and loss =============
#============================================
audioOptimizer = optim.SGD(anetwork.parameters(), opt.audiolr,
opt.audioMomentum)
videoOptimizer = optim.SGD(vnetwork.parameters(), opt.videolr,
opt.videoMomentum)
criterion = ContrastiveLoss()
index = 1
# start training
for epoch in range(opt.max_epoch):
for idx, (vinput, ainput, label) in enumerate(trainDataLoader):
#ipdb.set_trace()
vinput = vinput.to(opt.device)
ainput = ainput.to(opt.device)
label = label.to(opt.device)
audioOptimizer.zero_grad()
videoOptimizer.zero_grad()
vfeat = vnetwork.forward(vinput)
afeat = anetwork.forward(ainput)
#ipdb.set_trace()
loss = criterion.forward(vfeat, afeat, label)
#vis.plot(idx, loss, opt.trainwin)
print('loss: ', loss)
loss.backward()
audioOptimizer.step()
videoOptimizer.step()
if (idx + 1) % opt.print_freq == 0:
print('---epoch---:', epoch + 1, ' loss:', loss)
vis.plot(index, loss, opt.trainwin)
index = index + 1
anetwork.save(opt.save_model_path + '/anetwork' + str(epoch + 1) +
'.pth')
vnetwork.save(opt.save_model_path + '/vnetwork' + str(epoch + 1) +
'.pth')
os.makedirs(save_dir)
save_path = os.path.join(save_dir, 'state-{}-{}-AUC-{}.pth'.format(epoch + 1, i + 1,AUC))
if AUC >= 0.75:
torch.save(network, save_path)
if __name__ == '__main__':
os.environ["CUDA_VISIBLE_DEVICES"] = Config_graph.gpu
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
save_dir = Config_graph.savePath
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
vis = Visualizer(env=Config_graph.env)
adjMetrix = np.load('utils/adjMetrix.npy')
adjMetrix = torch.from_numpy(adjMetrix)
adjMetrix = sp.coo_matrix(adjMetrix)
adjMetrix = normalize(adjMetrix + sp.eye(adjMetrix.shape[0]))
adjMetrix = adjMetrix.todense()
adjMetrix = torch.from_numpy(adjMetrix)
adjMetrix = adjMetrix.float()
adjMetrix = adjMetrix.to(device)
model = UGGAT(nfeat=Config_graph.feat_in,
nhid=Config_graph.hidden,
nclass=Config_graph.nclass,
dropout=Config_graph.dropout,
nheads=Config_graph.nb_heads,