def merge(self, bond1, bond2, fragment, freeze=False):
'''Merges two bonds. Bond1 is the bond being bonded to.'''
# bond1 <= (bond2 from frag)
# find the part to change
if bond1.atoms[0].element[0] in CONNECTIONS:
R1, C1 = bond1.atoms
elif bond1.atoms[1].element[0] in CONNECTIONS:
C1, R1 = bond1.atoms
else:
raise Exception(5, "bad bond")
if bond2.atoms[0].element[0] in CONNECTIONS:
R2, C2 = bond2.atoms
elif bond2.atoms[1].element[0] in CONNECTIONS:
C2, R2 = bond2.atoms
else:
raise Exception(6, "bad bond")
# saved to prevent overwriting them
R2xyz = R2.xyz.copy()
C1xyz = C1.xyz.copy()
vec1 = R1.xyz - C1.xyz
vec2 = C2.xyz - R2.xyz
# diff = [azimuth, altitude]
vec1_angles = numpy.array(get_angles(vec1))
vec2_angles = numpy.array(get_angles(vec2))
diff = vec1_angles - vec2_angles
# angle of 1 - angle of 2 = angle to rotate
rot = get_full_rotation_matrix(vec2, -diff[0], -diff[1])
fragment.rotate_3d(rot, R2xyz, C1xyz)
if bond1.atoms[0].element[0] in CONNECTIONS:
bond1.atoms = (C2, C1)
else:
bond1.atoms = (C1, C2)
# remove the extension parts
for x in (bond2, R1, R2):
x.remove()
if freeze:
self.frozen.append(bond1.atoms)
def _get_angles(self, index, ad_indexes) -> list:
v_a = np.array(self.graph.vertices[index])
vert_arr = np.array([self.graph.vertices[i] for i in ad_indexes])
l = len(vert_arr)
degrees = get_angles(v_a, vert_arr)
tuple_list = [((index, ad_indexes[i], ad_indexes[j]), degrees[i, j])
for i in range(l) for j in range(l) if i != j]
arcs, multidict = gurobipy.multidict(tuple_list) if tuple_list else (
None, None)
return arcs, multidict
def simulate(net, config):
robot, ball, goal = reset()
global fitness, total_steps, MAX_STEPS, reset_sim
for step in range(MAX_STEPS):
# calculate new net inputs
rotated_center = Vec2d(10.5, 0)
rotated_center.rotate(robot.angle)
rotated_center += robot.position
goal_pos = utils.avg(goal.a, goal.b)
ball_dist = utils.dist(rotated_center,
ball.position) # robot -> ball dist
goal_dist = utils.dist(ball.position, goal_pos) # ball -> goal dist
ball_dir, goal_dir = utils.get_angles(
rotated_center, ball.position, goal_pos) # inputs for neural net
fitness = utils.calculate_fitness(ball_dist, goal_dist,
robot_touched_ball)
# scale values for nn
#ball_dir = np.interp(ball_dir, [0, 360], [0.0, 1.0])
#goal_dir = np.interp(goal_dir, [0, 360], [0.0, 1.0])
#ball_dist = np.interp(ball_dist, [0, 303.6], [0.0, 1.0])
# get input from neural net here, need to calculate balldir and goaldir though
# NEW INPJTS SHOULD BE: fixed ball dir, fixed ball dist, fix goal direction, fixed goal distance
# need to subtract curent heading from the ball dir
# goal dir should be robot to goal not bloody ball to goal
# remove int touched ball
# something else here?
rotation, speed = net.activate(
[ball_dir, ball_dist, goal_dir,
int(robot_touched_ball)])
rotation = utils.clamp(rotation, -1.0, 1.0)
speed = utils.clamp(speed, -1.0, 1.0)
rotation *= 10 # rotation will be in degrees
speed *= 50 # max speed = 60
robot.angle += math.radians(rotation)
robot.velocity = (speed * math.cos(robot.angle - 1.5708),
speed * math.sin(robot.angle - 1.5708))
# step sim based on input
robot.angular_velocity = 0
robot.center_of_gravity = (10.5, 10.5)
space.step(1.0 / 60.0)
total_steps += 1
# session was ended from one of the callback listeners, so we know it's got the bonuses already
if reset_sim:
reset_sim = False
return fitness
# test failed to complete, still subtract total steps
return fitness - (total_steps / 1.5) + total_steps_touching_ball
def merge(self, bond1, bond2, fragment, freeze=False):
'''Merges two bonds. Bond1 is the bond being bonded to.'''
# bond1 <= (bond2 from frag)
# find the part to change
if bond1.atoms[0].element[0] in CONNECTIONS:
R1, C1 = bond1.atoms
elif bond1.atoms[1].element[0] in CONNECTIONS:
C1, R1 = bond1.atoms
else:
raise Exception(5, "bad bond")
if bond2.atoms[0].element[0] in CONNECTIONS:
R2, C2 = bond2.atoms
elif bond2.atoms[1].element[0] in CONNECTIONS:
C2, R2 = bond2.atoms
else:
raise Exception(6, "bad bond")
# saved to prevent overwriting them
R2xyz = R2.xyz.copy()
C1xyz = C1.xyz.copy()
vec1 = R1.xyz - C1.xyz
vec2 = C2.xyz - R2.xyz
# diff = [azimuth, altitude]
vec1_angles = numpy.matrix(get_angles(vec1))
vec2_angles = numpy.matrix(get_angles(vec2))
diff = vec1_angles - vec2_angles
# angle of 1 - angle of 2 = angle to rotate
rot = get_full_rotation_matrix(vec2, -diff[0, 0], -diff[0, 1])
fragment.rotate_3d(rot, R2xyz, C1xyz)
if bond1.atoms[0].element[0] in CONNECTIONS:
bond1.atoms = (C2, C1)
else:
bond1.atoms = (C1, C2)
# remove the extension parts
[x.remove() for x in (bond2, R1, R2)]
if freeze:
self.frozen.append(bond1.atoms)
def _get_angles(self, index) -> (grb.tuplelist, grb.tupledict):
v_a = np.array(self.graph.vertices[index])
ad_indexes = [
j for j in range(len(self.graph.vertices))
if self.graph.ad_matrix[index, j] > 0
]
vert_arr = np.array([self.graph.vertices[i] for i in ad_indexes])
l = len(vert_arr)
degrees = get_angles(v_a, vert_arr)
tuple_list = [((index, ad_indexes[i], ad_indexes[j]), degrees[i, j])
for i in range(l) for j in range(i + 1, l)]
arcs, multidict = grb.multidict(tuple_list) if tuple_list else (None,
None)
return arcs, multidict
def _get_angles(self, index) -> (List[tuple], dict):
v_a = np.array(self.graph.vertices[index])
ad_indexes = [j for j in range(len(self.graph.vertices)) if self.graph.ad_matrix[index, j] > 0]
vert_arr = np.array([self.graph.vertices[i] for i in ad_indexes])
l = len(vert_arr)
angles = get_angles(v_a, vert_arr, in_degree=False)
pi_angle = angles / np.pi
pi_angle = np.round(pi_angle, 8) # round to get rid of some rounding errors before
tuple_dict = {(index, ad_indexes[i], ad_indexes[j]): pi_angle[i, j]
for i in range(l) for j in range(i+1, l)}
#arcs, multidict = grb.multidict(tuple_list) if tuple_list else (None, None)
return tuple_dict.keys(), tuple_dict
def _get_angles(self, index) -> (grb.tuplelist, grb.tupledict):
v_a = np.array(self.graph.vertices[index])
ad_indexes = [
j for j in range(len(self.graph.vertices))
if self.graph.ad_matrix[index, j] > 0
]
vert_arr = np.array([self.graph.vertices[i] for i in ad_indexes])
l = len(vert_arr)
degrees = get_angles(v_a, vert_arr)
tuple_list = [((index, ad_indexes[i], ad_indexes[j]), degrees[i, j])
for i in range(l) for j in range(l) if i != j]
# Correct entries with NaN
for i in range(len(tuple_list)):
if np.isnan(tuple_list[i][-1]):
tuple_list[i] = (tuple_list[i][0], 0)
arcs, multidict = grb.multidict(tuple_list) if tuple_list else (None,
None)
return arcs, multidict
def train(self):
writer = SummaryWriter(logdir=self.log_dir)
device = "cuda:0"
wu_cfg = self.cfg.fe.trainer
model = UNet2D(**self.cfg.fe.backbone)
model.cuda(device)
train_set = SpgDset(self.cfg.gen.data_dir_raw_train, reorder_sp=False)
val_set = SpgDset(self.cfg.gen.data_dir_raw_val, reorder_sp=False)
# pm = StridedPatches2D(wu_cfg.patch_stride, wu_cfg.patch_shape, train_set.image_shape)
pm = NoPatches2D()
train_set.length = len(train_set.graph_file_names) * np.prod(pm.n_patch_per_dim)
train_set.n_patch_per_dim = pm.n_patch_per_dim
val_set.length = len(val_set.graph_file_names)
# dset = LeptinDset(self.cfg.gen.data_dir_raw, self.cfg.gen.data_dir_affs, wu_cfg.patch_manager, wu_cfg.patch_stride, wu_cfg.patch_shape, wu_cfg.reorder_sp)
train_loader = DataLoader(train_set, batch_size=wu_cfg.batch_size, shuffle=True, pin_memory=True,
num_workers=0)
val_loader = DataLoader(val_set, batch_size=wu_cfg.batch_size, shuffle=True, pin_memory=True,
num_workers=0)
gauss_kernel = GaussianSmoothing(1, 5, 3, device=device)
optimizer = torch.optim.Adam(model.parameters(), lr=self.cfg.fe.lr)
sheduler = ReduceLROnPlateau(optimizer,
patience=20,
threshold=1e-4,
min_lr=1e-5,
factor=0.1)
criterion = RagContrastiveWeights(delta_var=0.1, delta_dist=0.4)
acc_loss = 0
valit = 0
iteration = 0
best_loss = np.inf
while iteration <= wu_cfg.n_iterations:
for it, (raw, gt, sp_seg, affinities, offs, indices) in enumerate(train_loader):
raw, gt, sp_seg, affinities = raw.to(device), gt.to(device), sp_seg.to(device), affinities.to(device)
# edge_img = F.pad(get_contour_from_2d_binary(sp_seg), (2, 2, 2, 2), mode='constant')
# edge_img = gauss_kernel(edge_img.float())
# input = torch.cat([raw, edge_img], dim=1)
offs = offs.numpy().tolist()
loss_embeds = model(raw[:, :, None]).squeeze(2)
edge_feat, edges = tuple(zip(*[get_edge_features_1d(seg.squeeze().cpu().numpy(), os, affs.squeeze().cpu().numpy()) for seg, os, affs in zip(sp_seg, offs, affinities)]))
edges = [torch.from_numpy(e.astype(np.long)).to(device).T for e in edges]
edge_weights = [torch.from_numpy(ew.astype(np.float32)).to(device)[:, 0][None] for ew in edge_feat]
# put embeddings on unit sphere so we can use cosine distance
loss_embeds = loss_embeds / (torch.norm(loss_embeds, dim=1, keepdim=True) + 1e-9)
loss = criterion(loss_embeds, sp_seg.long(), edges, edge_weights,
chunks=int(sp_seg.max().item()//self.cfg.gen.train_chunk_size),
sigm_factor=self.cfg.gen.sigm_factor, pull_factor=self.cfg.gen.pull_factor)
optimizer.zero_grad()
loss.backward()
optimizer.step()
print(loss.item())
writer.add_scalar("fe_train/lr", optimizer.param_groups[0]['lr'], iteration)
writer.add_scalar("fe_train/loss", loss.item(), iteration)
if (iteration) % 100 == 0:
with torch.set_grad_enabled(False):
for it, (raw, gt, sp_seg, affinities, offs, indices) in enumerate(val_loader):
raw, gt, sp_seg, affinities = raw.to(device), gt.to(device), sp_seg.to(device), affinities.to(device)
offs = offs.numpy().tolist()
embeddings = model(raw[:, :, None]).squeeze(2)
# relabel to consecutive ints starting at 0
edge_feat, edges = tuple(zip(
*[get_edge_features_1d(seg.squeeze().cpu().numpy(), os, affs.squeeze().cpu().numpy())
for seg, os, affs in zip(sp_seg, offs, affinities)]))
edges = [torch.from_numpy(e.astype(np.long)).to(device).T for e in edges]
edge_weights = [torch.from_numpy(ew.astype(np.float32)).to(device)[:, 0][None] for ew in edge_feat]
# put embeddings on unit sphere so we can use cosine distance
embeddings = embeddings / (torch.norm(embeddings, dim=1, keepdim=True) + 1e-9)
ls = criterion(embeddings, sp_seg.long(), edges, edge_weights,
chunks=int(sp_seg.max().item()//self.cfg.gen.train_chunk_size),
sigm_factor=self.cfg.gen.sigm_factor, pull_factor=self.cfg.gen.pull_factor)
# ls = 0
acc_loss += ls
writer.add_scalar("fe_val/loss", ls, valit)
valit += 1
acc_loss = acc_loss / len(val_loader)
if acc_loss < best_loss:
print(self.save_dir)
torch.save(model.state_dict(), os.path.join(self.save_dir, "best_val_model.pth"))
best_loss = acc_loss
sheduler.step(acc_loss)
acc_loss = 0
fig, ((a1, a2), (a3, a4)) = plt.subplots(2, 2, sharex='col', sharey='row', gridspec_kw={'hspace': 0, 'wspace': 0})
a1.imshow(raw[0].cpu().permute(1, 2, 0)[..., 0].squeeze())
a1.set_title('raw')
a2.imshow(cm.prism(sp_seg[0, 0].cpu().squeeze() / sp_seg[0, 0].cpu().squeeze().max()))
a2.set_title('sp')
a3.imshow(pca_project(get_angles(embeddings)[0].detach().cpu()))
a3.set_title('angle_embed')
a4.imshow(pca_project(embeddings[0].detach().cpu()))
a4.set_title('embed')
# plt.show()
writer.add_figure("examples", fig, iteration//100)
iteration += 1
print(iteration)
if iteration > wu_cfg.n_iterations:
print(self.save_dir)
torch.save(model.state_dict(), os.path.join(self.save_dir, "last_model.pth"))
break
return
def train(self):
writer = SummaryWriter(logdir=self.log_dir)
writer.add_text("conf", self.cfg.pretty())
device = "cuda:0"
wu_cfg = self.cfg.fe.trainer
model = UNet2D(self.cfg.fe.n_raw_channels,
self.cfg.fe.n_embedding_features,
final_sigmoid=False,
num_levels=5)
momentum_model = UNet2D(self.cfg.fe.n_raw_channels,
self.cfg.fe.n_embedding_features,
final_sigmoid=False,
num_levels=5)
if wu_cfg.identical_initialization:
soft_update_params(model, momentum_model, 1)
momentum_model.cuda(device)
for param in momentum_model.parameters():
param.requires_grad = False
model.cuda(device)
dset = SpgDset(self.cfg.gen.data_dir, wu_cfg.patch_manager,
wu_cfg.patch_stride, wu_cfg.patch_shape,
wu_cfg.reorder_sp)
dloader = DataLoader(dset,
batch_size=wu_cfg.batch_size,
shuffle=True,
pin_memory=True,
num_workers=0)
optimizer = torch.optim.Adam(model.parameters(), lr=self.cfg.fe.lr)
sheduler = ReduceLROnPlateau(optimizer,
patience=100,
threshold=1e-3,
min_lr=1e-6,
factor=0.1)
criterion = EntrInfoNCE(alpha=self.cfg.fe.alpha,
beta=self.cfg.fe.beta,
lbd=self.cfg.fe.lbd,
tau=self.cfg.fe.tau,
gamma=self.cfg.fe.gamma,
num_neg=self.cfg.fe.num_neg,
subs_size=self.cfg.fe.subs_size)
tfs = RndAugmentationTfs(wu_cfg.patch_shape)
acc_loss = 0
iteration = 0
k_step = math.ceil((wu_cfg.n_iterations - wu_cfg.n_k_stop_it) /
(wu_cfg.k_start - wu_cfg.k_stop))
k = wu_cfg.k_start
psi_step = (wu_cfg.psi_start - wu_cfg.psi_stop) / (
wu_cfg.n_iterations - wu_cfg.n_k_stop_it)
psi = wu_cfg.psi_start
while iteration <= wu_cfg.n_iterations:
for it, (raw, gt, sp_seg, indices) in enumerate(dloader):
inp, gt, sp_seg = raw.to(device), gt.to(device), sp_seg.to(
device)
mask = torch.ones((
inp.shape[0],
1,
) + inp.shape[2:],
device=device).float()
# get transforms
spat_tf, int_tf = tfs.sample(1, 1)
_, _int_tf = tfs.sample(1, 1)
# add noise to intensity tf of input for momentum network
mom_inp = add_sp_gauss_noise(_int_tf(inp), 0.2, 0.1, 0.3)
# get momentum prediction
embeddings_mom = momentum_model(
mom_inp.unsqueeze(2)).squeeze(2)
# do the same spatial tf for input, mask and momentum prediction
paired = spat_tf(torch.cat((mask, inp, embeddings_mom), -3))
embeddings_mom, mask = paired[..., inp.shape[1] +
1:, :, :], paired[...,
0, :, :][:,
None]
# do intensity transform for spatial transformed input
aug_inp = int_tf(paired[..., 1:inp.shape[1] + 1, :, :])
# and add some noise
aug_inp = add_sp_gauss_noise(aug_inp, 0.2, 0.1, 0.3)
# get prediction of the augmented input
embeddings = model(aug_inp.unsqueeze(2)).squeeze(2)
# put embeddings on unit sphere so we can use cosine distance
embeddings = embeddings / torch.norm(
embeddings, dim=1, keepdim=True)
embeddings_mom = embeddings_mom + (
mask == 0) # set the void of the image to the 1-vector
embeddings_mom = embeddings_mom / torch.norm(
embeddings_mom, dim=1, keepdim=True)
loss = criterion(embeddings.squeeze(0),
embeddings_mom.squeeze(0),
k,
mask.squeeze(0),
whiten=wu_cfg.whitened_embeddings,
warmup=iteration < wu_cfg.n_warmup_it,
psi=psi)
optimizer.zero_grad()
loss.backward()
optimizer.step()
acc_loss += loss.item()
print(loss.item())
writer.add_scalar("fe_warm_start/loss", loss.item(), iteration)
writer.add_scalar("fe_warm_start/lr",
optimizer.param_groups[0]['lr'], iteration)
if (iteration) % 50 == 0:
sheduler.step(acc_loss / 10)
acc_loss = 0
fig, (a1, a2, a3, a4) = plt.subplots(1,
4,
sharex='col',
sharey='row',
gridspec_kw={
'hspace': 0,
'wspace': 0
})
a1.imshow(inp[0].cpu().permute(1, 2, 0).squeeze())
a1.set_title('raw')
a2.imshow(aug_inp[0].cpu().permute(1, 2, 0))
a2.set_title('augment')
a3.imshow(
pca_project(
get_angles(embeddings).squeeze(0).detach().cpu()))
a3.set_title('embed')
a4.imshow(
pca_project(
get_angles(embeddings_mom).squeeze(
0).detach().cpu()))
a4.set_title('mom_embed')
writer.add_figure("examples", fig, iteration // 100)
iteration += 1
psi = max(psi - psi_step, wu_cfg.psi_stop)
if iteration % k_step == 0:
k = max(k - 1, wu_cfg.k_stop)
if iteration > wu_cfg.n_iterations:
break
if iteration % wu_cfg.momentum == 0:
soft_update_params(model, momentum_model,
wu_cfg.momentum_tau)
return
if approx_5 is None:
peri = cv2.arcLength(cnts[0], True)
frame = cv2.approxPolyDP(cnts[0], 0.05 * peri, True)
else:
frame = approx_5
# Calculate convex hull of the frame
frame = cv2.convexHull(frame)
# Get the rectangle with the min area
rect = cv2.minAreaRect(frame)
frame = cv2.boxPoints(rect)
frame = np.int0(frame)
# Get the 2 main angles of the frame
angles = ut.get_angles(frame)
rect2 = (rect[0], rect[1], angles[-1])
img_croped = ut.crop_min_area_rect(color, rect2)
cv2.imshow("Cropped", img_croped)
cv2.waitKey(0)
# As we scale the image, we need to scale back the contour to the original image
frame_orig = list(
map(lambda x: ut.rescale(x, size_ini, size_end), frame))
# Save data to export
frames.append([ut.rad2deg(angles[-1]), frame_orig])
# Save the data in a pickle file
ut.bbox_to_pkl(frames, fname='frames', folder='pkl')
def train(self):
writer = SummaryWriter(logdir=self.log_dir)
writer.add_text("conf", self.cfg.pretty())
device = "cuda:0"
wu_cfg = self.cfg.fe.trainer
model = UNet2D(self.cfg.fe.n_raw_channels,
self.cfg.fe.n_embedding_features,
final_sigmoid=False,
num_levels=5)
model.cuda(device)
dset = SpgDset(self.cfg.gen.data_dir, wu_cfg.patch_manager,
wu_cfg.patch_stride, wu_cfg.patch_shape,
wu_cfg.reorder_sp)
dloader = DataLoader(dset,
batch_size=wu_cfg.batch_size,
shuffle=True,
pin_memory=True,
num_workers=0)
optimizer = torch.optim.Adam(model.parameters(), lr=self.cfg.fe.lr)
sheduler = ReduceLROnPlateau(optimizer,
patience=100,
threshold=1e-3,
min_lr=1e-6,
factor=0.1)
criterion = RagInfoNCE(tau=self.cfg.fe.tau)
acc_loss = 0
iteration = 0
while iteration <= wu_cfg.n_iterations:
for it, (raw, gt, sp_seg, indices) in enumerate(dloader):
inp, gt, sp_seg = raw.to(device), gt.to(device), sp_seg.to(
device)
edges = dloader.dataset.get_graphs(indices, sp_seg, device)[0]
off = 0
for i in range(len(edges)):
sp_seg[i] += off
edges[i] += off
off = sp_seg[i].max() + 1
edges = torch.cat(edges, 1)
embeddings = model(inp.unsqueeze(2)).squeeze(2)
# put embeddings on unit sphere so we can use cosine distance
embeddings = embeddings / torch.norm(
embeddings, dim=1, keepdim=True)
loss = criterion(embeddings, sp_seg, edges)
optimizer.zero_grad()
loss.backward()
optimizer.step()
acc_loss += loss.item()
print(loss.item())
writer.add_scalar("fe_warm_start/loss", loss.item(), iteration)
writer.add_scalar("fe_warm_start/lr",
optimizer.param_groups[0]['lr'], iteration)
if (iteration) % 50 == 0:
sheduler.step(acc_loss / 10)
acc_loss = 0
fig, (a1, a2) = plt.subplots(1,
2,
sharex='col',
sharey='row',
gridspec_kw={
'hspace': 0,
'wspace': 0
})
a1.imshow(inp[0].cpu().permute(1, 2, 0).squeeze())
a1.set_title('raw')
a2.imshow(
pca_project(
get_angles(embeddings).squeeze(0).detach().cpu()))
a2.set_title('embed')
writer.add_figure("examples", fig, iteration // 100)
iteration += 1
if iteration > wu_cfg.n_iterations:
break
return
rotate_speed = math.radians(5)
clock.tick(60)
screen.fill((230, 230, 230))
# calculate inputs and fitness for NN
rotated_center = Vec2d(10.5, 0)
rotated_center.rotate(robot.angle)
rotated_center += robot.position
draw_options.draw_dot(2, rotated_center, (0, 255, 0))
goal_pos = utils.avg(goal.a, goal.b)
ball_dist = utils.dist(rotated_center,
ball.position) # robot -> ball dist
goal_dist = utils.dist(ball.position, goal_pos) # ball -> goal dist
ball_dir, goal_dir = utils.get_angles(
rotated_center, ball.position, goal_pos) # inputs for neural net
# scale values for nn
#ball_dir = np.interp(ball_dir, [0, 360], [0.0, 1.0])
#goal_dir = np.interp(goal_dir, [0, 360], [0.0, 1.0])
#ball_dist = np.interp(ball_dist, [0, 303.6], [0.0, 1.0])
# get input from neural net here
inputs = [ball_dir, ball_dist, goal_dir, int(robot_touched_ball)]
rotation, speed = net.activate(inputs)
rotation = utils.clamp(rotation, -1.0, 1.0)
speed = utils.clamp(speed, -1.0, 1.0)
rotation *= 10 # rotation will be in degrees
speed *= 50 # max speed = 60
robot.angle += math.radians(rotation)
def train(self):
writer = SummaryWriter(logdir=self.log_dir)
writer.add_text("conf", self.cfg.pretty())
device = "cuda:0"
wu_cfg = self.cfg.fe.trainer
model = UNet2D(self.cfg.fe.n_raw_channels,
self.cfg.fe.n_embedding_features,
final_sigmoid=False,
num_levels=5)
model.cuda(device)
dset = SpgDset(self.cfg.gen.data_dir, wu_cfg.patch_manager,
wu_cfg.patch_stride, wu_cfg.patch_shape,
wu_cfg.reorder_sp)
dloader = DataLoader(dset,
batch_size=wu_cfg.batch_size,
shuffle=True,
pin_memory=True,
num_workers=0)
optimizer = torch.optim.Adam(model.parameters(), lr=self.cfg.fe.lr)
tfs = RndAugmentationTfs(wu_cfg.patch_shape)
criterion = AugmentedAffinityContrastive(delta_var=0.1, delta_dist=0.3)
acc_loss = 0
iteration = 0
while iteration <= wu_cfg.n_iterations:
for it, (raw, gt, sp_seg, indices) in enumerate(dloader):
raw, gt, sp_seg = raw.to(device), gt.to(device), sp_seg.to(
device)
# this is still not the correct mask calculation as the affinity offsets go in no tf offset direction
mask = torch.from_numpy(
get_valid_edges([len(criterion.offs)] +
list(raw.shape[-2:]),
criterion.offs)).to(device)[None]
# _, _, _, _, affs = dset.get_graphs(indices, sp_seg, device)
spat_tf, int_tf = tfs.sample(1, 1)
_, _int_tf = tfs.sample(1, 1)
inp = add_sp_gauss_noise(_int_tf(raw), 0.2, 0.1, 0.3)
embeddings = model(inp.unsqueeze(2)).squeeze(2)
paired = spat_tf(torch.cat((mask, raw, embeddings), -3))
embeddings_0, mask = paired[
..., inp.shape[1] + len(criterion.offs):, :, :], paired[
..., :len(criterion.offs), :, :].detach()
# do intensity transform for spatial transformed input
aug_inp = int_tf(paired[...,
len(criterion.offs):inp.shape[1] +
len(criterion.offs), :, :]).detach()
# get prediction of the augmented input
embeddings_1 = model(
add_sp_gauss_noise(aug_inp, 0.2, 0.1,
0.3).unsqueeze(2)).squeeze(2)
# put embeddings on unit sphere so we can use cosine distance
embeddings_0 = embeddings_0 / (
torch.norm(embeddings_0, dim=1, keepdim=True) + 1e-6)
embeddings_1 = embeddings_1 / (
torch.norm(embeddings_1, dim=1, keepdim=True) + 1e-6)
loss = criterion(embeddings_0, embeddings_1, aug_inp, mask)
optimizer.zero_grad()
loss.backward()
optimizer.step()
acc_loss += loss.item()
print(loss.item())
writer.add_scalar("fe_warm_start/loss", loss.item(), iteration)
writer.add_scalar("fe_warm_start/lr",
optimizer.param_groups[0]['lr'], iteration)
if (iteration) % 50 == 0:
acc_loss = 0
fig, ((a1, a2), (a3, a4)) = plt.subplots(2, 2)
a1.imshow(aug_inp[0].cpu().permute(1, 2, 0).squeeze())
a1.set_title('tf_raw')
a3.imshow(
pca_project(
get_angles(embeddings_0).squeeze(
0).detach().cpu()))
a3.set_title('tf_embed')
a4.imshow(
pca_project(
get_angles(embeddings_1).squeeze(
0).detach().cpu()))
a4.set_title('embed')
a2.imshow(raw[0].cpu().permute(1, 2, 0).squeeze())
a2.set_title('raw')
plt.show()
# writer.add_figure("examples", fig, iteration//100)
iteration += 1
if iteration > wu_cfg.n_iterations:
break
return