def load_models(files, batch_size=1):
transformations = list(
itertools.product(range(0, 360, 90), range(0, 360, 90)))
size = len(transformations) * len(files)
yield int(np.ceil(size / batch_size))
images = np.zeros((batch_size, SIZE, SIZE, SIZE, 1), dtype="float32")
masks = np.zeros((batch_size, SIZE, SIZE, SIZE, 1), dtype="float32")
ip = 0
while True:
for image_filename, mask_filename in files:
image = nb.load(str(image_filename)).get_fdata()
mask = nb.load(str(mask_filename)).get_fdata()
image = resize(image, (SIZE, SIZE, SIZE),
mode="constant",
anti_aliasing=True)
mask = resize(mask, (SIZE, SIZE, SIZE),
mode="constant",
anti_aliasing=True)
image = image_normalize(image)
mask = image_normalize(mask)
for rot1, rot2 in transformations:
t_image = apply_transform(image, rot1, rot2)
t_mask = apply_transform(mask, rot1, rot2)
print(image_filename, rot1, rot2)
images[ip] = t_image.reshape(SIZE, SIZE, SIZE, 1)
masks[ip] = t_mask.reshape(SIZE, SIZE, SIZE, 1)
ip += 1
if ip == batch_size:
yield (images, masks)
ip = 0
def scale_factors_from_result(T, frame1scaled, frame2scaled):
Tinv = utils.transform_inv(T)
scale_factors = [utils.apply_transform(Tinv, f2)/f1
for f1, f2 in frame1scaled]
scale_factors += [utils.apply_transform(T, f1)/f2
for f1, f2 in frame2scaled]
return scale_factors
def run_tests():
#get the screenshots
screenshots = []
for i in range(4):
img = cv2.imread('screenshot{}.png'.format(i))
#img = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
screenshots.append(img)
#gets the templates to do template matching with
template = cv2.cvtColor(cv2.imread('template.jpg'), cv2.COLOR_BGR2GRAY)
sample = screenshots[0]
bird = apply_transform(template, sample, ((399, 350), (580, 500), (189,73), (206, 96)))
bird = cv2.resize(bird, (0,0), fx=RESIZE, fy=RESIZE)
sample = screenshots[1]
pipe = apply_transform(template, sample, ((564, 32), (837, 157), (214,139), (230, 176)))
pipe = cv2.resize(pipe, (0,0), fx=RESIZE, fy=RESIZE)
# Does the processing, currently draws rectangles around where the program things the
# object is. Takes ~.6ms/image -> 24ms/image on the PI
# Note can use for performance testing or sanity test.
# for performance, comment out display stuff and increase num_trials
# for sanity test, set num_trials=1 and uncomment display stuff
total_times = [0] * len(screenshots)
num_trials = 1000
for i in range(num_trials):
for j in range(len(screenshots)):
background = screenshots[j]
t0 = time.time()
#hardcoded these values to make the program only focus on what I think is important
img = background[70:200, 60:285]
img = cv2.resize(img, (0,0), fx=RESIZE, fy=RESIZE)
img = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
#background_copy = background.copy()
bird_corners = find_bird(bird, img)
top_left_bird, bottom_right_bird = scale_corners(bird_corners)
pipe_corners = find_pipes(pipe, img)
p0, p1, p2, p3 = scale_corners(pipe_corners)
total_times[j] += time.time() - t0
'''
# display stuff. If you use this, please set num_trials to 1
cv2.rectangle(background, top_left_bird, bottom_right_bird, (0,0,0), 1)
cv2.rectangle(background, p0, p1, (0,0,0), 1)
cv2.rectangle(background, p2, p3, (0,0,0), 1)
show(background)
'''
for i in range(len(screenshots)):
print 'running {} trials for image {} took {} seconds'\
.format(num_trials, i, total_times[i])
def _plot_cam(img, K, markers, truem, T, origin=False):
_plot_coordinate_transforms(T, origin=origin)
_plot_img(img, K, T)
target_loc = projectionto3d(K, markers)
target_loc = utils.apply_transform(T, target_loc)
origin = np.zeros(3)
origin = utils.apply_transform(T, origin)
_plot_lines(origin, target_loc, 2)
truem = utils.apply_transform(T, truem)
return mlab.points3d(truem[:,0], truem[:,1], truem[:, 2], scale_factor=0.01)
def _compute_err(T, frame0scaled, frame1scaled):
Tinv = transform_inv(T)
errs0 = [np.linalg.norm(
projection(apply_transform(Tinv, f1)) - projection(f0))
for f0, f1 in frame0scaled]
errs1 = [np.linalg.norm(
projection(apply_transform(T, f0)) - projection(f1))
for f0, f1 in frame1scaled]
tot_err = sum(errs0 + errs1)
return tot_err
def train_cl_epoch(epoch,
model,
train_dl,
criterion,
optimizer,
_get_w,
bar_label_prefix=''):
torch.cuda.reset_peak_memory_stats()
epoch_loss = 0.
train_corrects = 0
model.train()
bar_label = f'{bar_label_prefix}{"Training":10s}'
with tqdm(train_dl, desc=bar_label, total=len(train_dl)) as bar:
for i, (x, y) in enumerate(bar):
w = _get_w(epoch * len(train_dl) + i)
x_aug, _ = apply_transform(x, None, tf=aug_tf)
x_aug = x_aug.cuda()
x, y = apply_transform(x, y, tf=base_tf)
x = x.cuda()
y = y.cuda()
x_all = torch.cat((x, x_aug), dim=0)
out = model(x_all)
out_x = out[:len(x)]
out_aug = out[len(x):]
supervised_loss = criterion(out_x, y)
consistency_loss = F.mse_loss(
out_x.detach().softmax(dim=-1), out_aug.softmax(dim=-1))
loss = supervised_loss + w * consistency_loss
optimizer.zero_grad()
loss.backward()
optimizer.step()
epoch_loss += loss.detach().item()
preds = out_x.detach().argmax(dim=-1)
train_corrects += (preds == y).cpu().float().sum()
bar.set_postfix({
'w': f'{w:.03f}',
'sup_loss': f'{supervised_loss.item():.03f}',
'unsup_loss': f'{consistency_loss.item():.03f}',
})
train_acc = train_corrects / len(train_dl.sampler)
return epoch_loss, train_acc
def error(x):
quat = x[:4]
trans = np.array(x[4:7])
T = transform_from_quat(quat, trans)
Tinv = transform_inv(T)
err0 = np.hstack([projection(f0) - projection(apply_transform(Tinv, f1))
for f0, f1 in frame1scaled])
err1 = np.hstack([projection(apply_transform(T, f0)) - projection(f1)
for f0, f1 in frame2scaled])
err3 = (np.linalg.norm(quat) - 1)
return np.hstack((err0, err1, err3))
def predict(self, history):
# scale input data
history = apply_transform(history, self.scaler.transform)
# predict
history = np.reshape(
history, (history.shape[0], 1,
self.nps)) # [samples, time steps, features] for model
bws = self.model.predict(history)
# inverse scale predicted data
bws = apply_transform(bws, self.scaler.inverse_transform)
return bws.flatten()
def update(self, new_april_msg):
if len(new_april_msg.detections) > 1:
detected_tag = sort_l2(new_april_msg.detections)
else:
detected_tag = new_april_msg.detections[0]
for detect in [detected_tag]:
tag_id = str(detect.id[0])
if self.prev_tag == tag_id:
continue
else:
self.cam2id_tf = self.buf.lookup_transform(self.cur_frame_id, "tag_"+tag_id, rospy.Time(0), rospy.Duration(0.1))
self.id2org_tf = self.buf.lookup_transform(tag_id+"_cur"+self.frame_suffix, "tag_world_cur"+self.frame_suffix, rospy.Time(0), rospy.Duration(0.3))
self.cam2id_tf.child_frame_id = tag_id+"_cur"+self.frame_suffix
#self.prev_tag = tag_id
print("------")
print("Tag ID "+tag_id+" seen.")
print("Got TF")
if self.id2org_tf != None and self.cam2id_tf != None:
self.cur_tf = apply_transform(self.cam2id_tf, self.id2org_tf)
self.cam2id_tf = None
self.id2org_tf = None
self.can_update = True
def prepare_image(dbs, args): im_slices = list() labels = list() keys = list() for slice_idx, entry in enumerate(dbs): env, txn, cursor = entry im_slice, label_slice = get_image(cursor.value(), slice_idx, args) im_slice = apply_transform(im_slice, args.transform) im_slice = scale_shift_im(im_slice, slice_idx, args) im_slices.append(im_slice) labels.append(label_slice) keys.append(cursor.key()) # check that all keys match key = keys[0] for slice_idx, _key in enumerate(keys): if _key != key: log(args, "WARNING!, keys differ %s vs %s for slices %d and %d" % (key, _key, 0, slice_idx)) # check that all labels match label = labels[0] for slice_idx, _label in enumerate(labels): if _label != label: log(args, "WARNING!, key %s has differing labels: %d vs %d for slices %d and %d" % (key, label, _label, 0, slice_idx)) whole_im = np.concatenate(im_slices, axis=2) # append along channels return whole_im, label
def load_models_patches(files,
transformations,
patch_size=SIZE,
batch_size=BATCH_SIZE):
for image_filename, mask_filename in files:
image = nb.load(str(image_filename)).get_fdata()
mask = nb.load(str(mask_filename)).get_fdata()
image = image_normalize(image)
mask = image_normalize(mask)
rot1, rot2 = random.choice(transformations)
t_image = apply_transform(image, rot1, rot2)
t_mask = apply_transform(mask, rot1, rot2)
print(image_filename, mask_filename, rot1, rot2, t_image.min(),
t_image.max(), t_mask.min(), t_mask.max())
for sub_image, sub_mask in gen_patches(t_image, t_mask, patch_size):
yield (sub_image, sub_mask)
def gen_train_arrays(files, patch_size=SIZE, batch_size=BATCH_SIZE):
transformations = list(
itertools.product(range(0, 360, 15), range(0, 360, 15)))
files_transforms_patches = gen_all_patches(files, transformations,
patch_size, batch_size,
NUM_PATCHES)
size = len(files_transforms_patches)
print(size)
yield int(np.ceil(size / batch_size))
images = np.zeros(shape=(batch_size, patch_size, patch_size, patch_size,
1),
dtype=np.float32)
masks = np.zeros_like(images)
yield get_proportion(files_transforms_patches, patch_size)
ip = 0
last_filename = ""
for image_filename, mask_filename, rot1, rot2, patch in itertools.cycle(
files_transforms_patches):
if image_filename != last_filename:
image = nb.load(str(image_filename)).get_fdata()
mask = nb.load(str(mask_filename)).get_fdata()
image = image_normalize(image)
mask = image_normalize(mask)
image = apply_transform(image, rot1, rot2)
mask = apply_transform(mask, rot1, rot2)
last_filename = image_filename
print(last_filename)
images[ip] = get_image_patch(image, patch, patch_size).reshape(
1, patch_size, patch_size, patch_size, 1)
masks[ip] = get_image_patch(mask, patch, patch_size).reshape(
1, patch_size, patch_size, patch_size, 1)
ip += 1
if ip == batch_size:
yield images, masks
ip = 0
def get_proportion(files_transforms_patches, patch_size=SIZE):
sum_bg = 0.0
sum_fg = 0.0
last_filename = ""
for image_filename, mask_filename, rot1, rot2, patch in files_transforms_patches:
if image_filename != last_filename:
mask = nb.load(str(mask_filename)).get_fdata()
mask = image_normalize(mask)
mask = apply_transform(mask, rot1, rot2)
last_filename = image_filename
_mask = get_image_patch(mask, patch, patch_size)
sum_bg += (_mask < 0.5).sum()
sum_fg += (_mask >= 0.5).sum()
return sum_bg / (sum_fg + sum_bg), sum_fg / (sum_bg + sum_fg)
def _plot_img(img, K, T, z=0.1):
obj = mlab.imshow(img.T)
quat = tf.quaternion_from_matrix(T)
angle, axis = angle_axis_from_quaternion(quat)
obj.actor.rotate_wxyz(angle * 180 / np.pi, *axis)
h, w = img.shape
xmax_pixel, ymax_pixel = w, h
point3d = projectionto3d(K, (xmax_pixel, ymax_pixel))[0]
origin3d = projectionto3d(K, (0, 0))[0]
point3d = point3d * z / point3d[2]
origin3d = origin3d * z / origin3d[2]
center3d = (point3d + origin3d) / 2.
xmax, ymax, _ = point3d - origin3d
obj.actor.scale = np.array([xmax / xmax_pixel, ymax / ymax_pixel, 1.0])
obj.actor.position = utils.apply_transform(T, center3d)
mlab.view(distance=20 * z)
return obj
def publish_map(m2r, t2m, pub):
pub_msg = Odometry()
pub_msg.header = t2m.header
pub_msg.child_frame_id = m2r.child_frame_id
trfm_msg = apply_transform(t2m, m2r)
[
pub_msg.pose.pose.position.x, pub_msg.pose.pose.position.y,
pub_msg.pose.pose.position.z
] = [
trfm_msg.transform.translation.x, trfm_msg.transform.translation.y,
trfm_msg.transform.translation.z
]
[
pub_msg.pose.pose.orientation.x, pub_msg.pose.pose.orientation.y,
pub_msg.pose.pose.orientation.z, pub_msg.pose.pose.orientation.w
] = [
trfm_msg.transform.rotation.x, trfm_msg.transform.rotation.y,
trfm_msg.transform.rotation.z, trfm_msg.transform.rotation.w
]
pub.publish(pub_msg)
def train_epoch_fixmatch(epoch,
model,
ema_model,
train_dl_l,
train_dl_ul,
optimizer,
sched,
conf_thresh,
w_u,
bar_label_prefix=''):
torch.cuda.reset_peak_memory_stats()
epoch_loss = 0.
train_corrects = 0
total = 0
bar_label = f'{bar_label_prefix}{"Training":10s}'
model.train()
it = zip(cycle(train_dl_l), train_dl_ul)
with tqdm(it, desc=bar_label, total=len(train_dl_ul)) as bar:
for i, ((x_l, y_l), (x_ul, _)) in enumerate(bar):
x_l, y_l = apply_transform(x_l, y_l, tf=base_tf)
x_l = x_l.cuda()
y_l = y_l.cuda()
out_l = model(x_l)
supervised_loss = F.cross_entropy(out_l, y_l)
loss = 0.
loss += supervised_loss
x_weak_aug, _ = apply_transform(x_ul, y=None, tf=weak_aug_tf)
x_weak_aug = x_weak_aug.cuda()
with torch.no_grad():
# model.eval()
out_weak_aug = model(x_weak_aug).detach().softmax(dim=-1)
# model.train()
# out_weak_aug = ema_model(x_weak_aug).detach().softmax(dim=-1)
max_vals, max_inds = out_weak_aug.cpu().max(dim=-1)
high_confidence_mask = max_vals >= conf_thresh
unsupervised_loss = None
if high_confidence_mask.any():
y_ul = max_inds[high_confidence_mask]
x_ul = x_ul[high_confidence_mask]
x_strong_aug, y_ul = apply_transform(x_ul,
y=y_ul,
tf=strong_aug_tf)
x_strong_aug = x_strong_aug.cuda()
y_ul = y_ul.cuda()
out_ul = model(x_strong_aug)
unsupervised_loss = F.cross_entropy(out_ul, y_ul)
loss += w_u * unsupervised_loss
optimizer.zero_grad()
loss.backward()
optimizer.step()
ema_model.update(model)
sched.step()
epoch_loss += loss.detach().item()
preds = out_l.detach().argmax(dim=-1)
train_corrects += (preds == y_l).detach().cpu().float().sum()
total += len(y_l)
bar.set_postfix({
'sup_loss':
f'{supervised_loss.item():.03f}',
'unsup_loss':
f'{unsupervised_loss.item():.03f}'
if unsupervised_loss is not None else '0.000',
'pseudo_used':
f'{high_confidence_mask.float().mean().item():.03f}'
})
train_acc = train_corrects / total
return epoch_loss, train_acc
def preprocess_cifar100(config_updates, config):
cfg = get_cfg_defaults()
cfg.merge_from_file(config)
cfg.merge_from_list(config_updates)
cfg_dict = utils.cfg_to_dict(cfg)
trainset = torchvision.datasets.CIFAR100('./data',
train=True,
transform=None,
target_transform=None,
download=True)
testset = torchvision.datasets.CIFAR100('./data',
train=False,
transform=None,
target_transform=None,
download=True)
X_train = trainset.data
X_test = testset.data
N_train = len(trainset.data)
N_test = len(testset.data)
train_idxs = np.arange(N_train)
test_idxs = np.arange(N_test)
X_train = X_train[train_idxs]
X_test = X_test[test_idxs]
y_train = np.array(trainset.targets)[train_idxs]
y_test = np.array(testset.targets)[test_idxs]
X_train_full = [X_train]
y_train_full = [y_train]
for i, (aug_name, mag) in enumerate(cfg.DATASET.AUGMENTATIONS):
print(f"Applying augmentation: {aug_name}, {mag}")
if aug_name == "Random":
aug_transformer = random_aug.randaugment(
cfg.DATASET.RAND_AUGMENT_N_MAX,
mag,
seed=cfg.DATASET.RAND_SEED + i,
transforms=cfg.DATASET.RAND_AUGMENT_AUGS,
replace=cfg.DATASET.RAND_AUGMENT_REPLACE,
random_n=cfg.DATASET.RAND_AUGMENT_RAND_N)
else:
aug_transformer = augmentation_transforms.NAME_TO_TRANSFORM[
aug_name].pil_transformer(1.0, mag, X_train[0].shape)
X_train_aug = utils.apply_transform(aug_transformer, X_train)
X_train_full.append(X_train_aug)
y_train_full.append(y_train)
X_train_full = np.vstack(X_train_full)
y_train_full = np.hstack(y_train_full)
(X_train, X_test), global_ZCA = utils.preprocess(
(X_train_full / 255).astype('float32'),
(X_test / 255).astype('float32'),
min_divisor=1e-8,
zca_bias=1e-4,
return_weights=True)
print('Got Data')
train_bio = io.BytesIO()
test_bio = io.BytesIO()
np.savez("cifar100_train.npz",
X_train=X_train,
y_train=y_train,
global_ZCA=global_ZCA)
np.savez("cifar100_test.npz",
X_test=X_test,
y_test=y_test,
global_ZCA=global_ZCA)
def gen_image_patches(files, patch_size=SIZE, num_patches=NUM_PATCHES):
image_filename, mask_filename = files
original_image = nb.load(str(image_filename)).get_fdata()
original_mask = nb.load(str(mask_filename)).get_fdata()
transformations = list(
itertools.product(range(0, 360, STEP_ROT), range(0, 360, STEP_ROT)))
patches_files = []
# Mirroring
for m in range(4):
print("m", m)
if m == 0:
image = original_image[:].copy()
mask = original_mask[:].copy()
if m == 1:
image = original_image[::-1].copy()
mask = original_mask[::-1].copy()
elif m == 2:
image = original_image[:, ::-1, :].copy()
mask = original_mask[:, ::-1, :].copy()
elif m == 3:
image = original_image[:, :, ::-1].copy()
mask = original_mask[:, :, ::-1].copy()
for n in range(4):
print("r", n)
if n == 0:
rot1, rot2 = 0, 0
else:
rot1 = random.randint(1, 359)
rot2 = random.randint(1, 359)
patches_added = 0
_image = apply_transform(image, rot1, rot2)
_image = image_normalize(_image)
_mask = apply_transform(mask, rot1, rot2)
_mask = image_normalize(_mask)
sz, sy, sx = _image.shape
patches = list(
itertools.product(
range(0, sz, patch_size - OVERLAP),
range(0, sy, patch_size - OVERLAP),
range(0, sx, patch_size - OVERLAP),
))
random.shuffle(patches)
for patch in patches:
sub_image = get_image_patch(_image, patch, patch_size)
sub_mask = get_image_patch(_mask, patch, patch_size)
if sub_mask.any():
tmp_filename = mktemp(suffix=".npz")
np.savez(tmp_filename, image=sub_image, mask=sub_mask)
del sub_image
del sub_mask
patches_files.append(tmp_filename)
patches_added += 1
if patches_added == num_patches:
break
return patches_files
def train_epoch_mixmatch(epoch,
model,
ema_model,
train_dl_l,
train_dl_ul,
optimizer,
_get_w,
num_augs,
T,
α,
bar_label_prefix=''):
torch.cuda.reset_peak_memory_stats()
epoch_loss = 0.
bar_label = f'{bar_label_prefix}{"Training":10s}'
model.train()
with tqdm(zip(cycle(train_dl_l), train_dl_ul),
desc=bar_label,
total=len(train_dl_ul)) as bar:
for i, ((x_l, y_l), (x_ul, _)) in enumerate(bar):
w_u = _get_w(epoch * len(train_dl_ul) + i)
# --------------
# create pseudo labels
# --------------
# (num_augs * batch_size, C, H, W)
x_ul_aug = torch.cat(
[apply_transform([_x, _x], None, tf=aug_tf)[0] for _x in x_ul],
dim=0)
x_ul_aug = x_ul_aug.cuda()
with torch.no_grad():
model.eval()
# (batch_size * num_augs, num_classes)
out_ul_aug = model(x_ul_aug).detach()
model.train()
# (batch_size, num_augs, num_classes)
out_ul_aug = out_ul_aug.reshape(-1, num_augs, out_ul_aug.shape[-1])
# (batch_size, num_classes)
out_ul_aug = out_ul_aug.mean(dim=1).softmax(dim=-1)
out_ul_aug_sharp = sharpen(out_ul_aug, T=T, dim=-1)
out_ul_aug_sharp = out_ul_aug_sharp.cuda()
y_ul = out_ul_aug_sharp
# --------------
# mixup and forward pass
# --------------
x_ul, _ = apply_transform(x_ul, None, tf=aug_tf)
x_ul = x_ul.cuda()
x_l, y_l = apply_transform(x_l, y_l, tf=base_tf)
x_l = x_l.cuda()
x = torch.cat([x_l, x_ul], dim=0).cuda()
inds = torch.randperm(len(x)).cuda()
x = x[inds]
L, λ_l_1, λ_l_2 = mixup(α, α, x_l, x[:len(x_l)])
U, λ_ul_1, λ_ul_2 = mixup(α, α, x_ul, x[len(x_l):])
out_l = model(L)
out_ul = model(U).softmax(dim=-1)
# --------------
# loss and backward pass
# --------------
y_l = y_l.cuda()
y_l_new = torch.zeros_like(out_l)
y_l_new[torch.arange(len(y_l)), y_l] = 1.
y = torch.cat([y_l_new, y_ul], dim=0).contiguous()
y = y[inds]
y_l_1 = y_l_new
y_l_2 = y[:len(x_l)]
y_ul_1 = y_ul
y_ul_2 = y[len(x_l):]
y_sup = λ_l_1.unsqueeze(-1) * y_l_1 + λ_l_2.unsqueeze(-1) * y_l_2
supervised_loss = -torch.log_softmax(out_l, dim=-1)
supervised_loss *= y_sup
supervised_loss = supervised_loss.sum(dim=-1).mean()
y_unsup = λ_ul_1.unsqueeze(-1) * y_ul_1 + λ_ul_2.unsqueeze(
-1) * y_ul_2
unsupervised_loss = F.mse_loss(out_ul, y_unsup)
loss = supervised_loss + w_u * unsupervised_loss
optimizer.zero_grad()
loss.backward()
optimizer.step()
ema_model.update(model)
epoch_loss += loss.detach().item()
bar.set_postfix({
'w': f'{w_u:.03f}',
'sup_loss': supervised_loss.item(),
'unsup_loss': unsupervised_loss.item(),
})
return epoch_loss, -1
plot_triangulation_lines=True,
target_img_patch=None):
obj = mlab.gcf()
obj.scene.disable_render = True
_plot_coordinate_transforms(T0, T1)
_plot_img(img0, K0, T0)
_plot_img(img1, K1, T1)
# plot at most 3 lines otherwise it is a mess
if plot_triangulation_lines:
max_lines = 4
if len(img_pos_target0) > max_lines:
img_pos_target0 = img_pos_target0[:max_lines]
img_pos_target1 = img_pos_target1[:max_lines]
target_loc0 = projectionto3d(K0, img_pos_target0)
target_loc0 = utils.apply_transform(T0, target_loc0)
origin0 = utils.apply_transform(T0, np.zeros(3))
_plot_lines(origin0, target_loc0, 10)
target_loc1 = projectionto3d(K1, img_pos_target1)
target_loc1 = utils.apply_transform(T1, target_loc1)
origin1 = utils.apply_transform(T1, np.zeros(3))
_plot_lines(origin1, target_loc1, 10)
if target_img_patch is not None and target_loc3d is not None:
pass
# add _plot_img for the patch at target_loc3d with normal along z-axis
elif target_loc3d is not None:
target_loc3d = np.asarray(target_loc3d).reshape(-1, 3)
if len(target_loc3d) > 4:
venlens = matmlab.vector_lengths(target_loc3d)
filtered = venlens < 10
def _distance_of_other_marker(T, m1):
return np.linalg.norm(apply_transform(transform_inv(T), m1))
# create and fit the LSTM network
model = Sequential()
model.add(LSTM(4, input_shape=(1, look_back)))
model.add(Dense(look_forward))
model.compile(loss='mean_squared_error', optimizer='adam')
model.fit(trainX, trainY, epochs=100, batch_size=1, verbose=2)
# model.fit(trainX, trainY, epochs=5, batch_size=1, verbose=2) # for debugging
model.save('lstm_model.h5') # save for later use
# make predictions
trainPredict = model.predict(trainX)
testPredict = model.predict(testX)
# invert predictions
trainPredict = apply_transform(trainPredict, scaler.inverse_transform)
trainY = apply_transform(trainY, scaler.inverse_transform)
testPredict = apply_transform(testPredict, scaler.inverse_transform)
testY = apply_transform(testY, scaler.inverse_transform)
# calculate root mean squared error
trainScore = math.sqrt(mean_squared_error(trainY, trainPredict))
print('Train Score: %.2f RMSE' % (trainScore))
testScore = math.sqrt(mean_squared_error(testY, testPredict))
print('Test Score: %.2f RMSE' % (testScore))
# shift train predictions for plotting
trainPredictPlot = np.empty_like(dataset)
trainPredictPlot[:, :] = np.nan
tmp = trainPredict[:, 0].reshape(
(len(trainPredict), 1)) # only the first predictions
def preprocess_cifar(config_updates, config):
cfg = get_cfg_defaults()
cfg.merge_from_file(config)
cfg.merge_from_list(config_updates)
cfg_dict = utils.cfg_to_dict(cfg)
# load cifar-10 from db
data = utils.load_dataset("cifar-10-raw")
y_test_full = data["y_test"]
X_test_raw = data["X_test"]
y_train_full = data["y_train"]
X_train_raw = data["X_train"]
# load cifar-10.1
X_test_2 = np.load(
io.BytesIO(
urllib.request.urlopen(
"https://github.com/modestyachts/CIFAR-10.1/raw/master/datasets/cifar10.1_v6_data.npy"
).read()))
y_test_2_full = np.load(
io.BytesIO(
urllib.request.urlopen(
"https://github.com/modestyachts/CIFAR-10.1/raw/master/datasets/cifar10.1_v6_labels.npy"
).read()))
cfg.SYSTEM.NUM_GPUS = torch.cuda.device_count()
N_train = cfg.DATASET.TRAIN_SUBSET
N_test = cfg.DATASET.TEST_SUBSET
np.random.seed(cfg.DATASET.RAND_SEED)
if cfg.DATASET.RANDOM_TRAIN_SUBSET:
train_idxs = np.random.choice(X_train_raw.shape[0],
N_train,
replace=False)
else:
train_idxs = np.arange(N_train)
if cfg.DATASET.RANDOM_TEST_SUBSET:
test_idxs = np.random.choice(X_test_raw.shape[0],
N_test,
replace=False)
else:
test_idxs = np.arange(N_test)
print(f"train_idxs: {train_idxs}")
print(f"test_idxs: {test_idxs}")
X_train = X_train_raw[train_idxs]
X_test = X_test_raw[test_idxs]
y_train = y_train_full[train_idxs]
y_test = y_test_full[test_idxs]
X_test_full = np.vstack((X_test, X_test_2))
X_train_full = [X_train]
y_train_full = [y_train]
for i, (aug_name, mag) in enumerate(cfg.DATASET.AUGMENTATIONS):
print(f"Applying augmentation: {aug_name}, {mag}")
if aug_name == "Random":
aug_transformer = random_aug.randaugment(
cfg.DATASET.RAND_AUGMENT_N_MAX,
mag,
seed=cfg.DATASET.RAND_SEED + i,
transforms=cfg.DATASET.RAND_AUGMENT_AUGS,
replace=cfg.DATASET.RAND_AUGMENT_REPLACE,
random_n=cfg.DATASET.RAND_AUGMENT_RAND_N)
else:
aug_transformer = augmentation_transforms.NAME_TO_TRANSFORM[
aug_name].pil_transformer(1.0, mag, X_train[0].shape)
X_train_aug = utils.apply_transform(aug_transformer, X_train)
X_train_full.append(X_train_aug)
y_train_full.append(y_train)
X_train_full = np.vstack(X_train_full)
y_train_full = np.hstack(y_train_full)
X_train_full_no_corners = X_train_full
if cfg.DATASET.CORNERS:
print("Extracting corners...")
X_train_full = utils.corners_full(X_train_full)
X_test = utils.corners_full(X_test)
if cfg.DATASET.ZCA:
print("Performing ZCA...")
if cfg.DATASET.ZCA_FULL:
(X_train_pp, X_test_pp), global_ZCA = utils.preprocess(
np.vstack((X_train_full, X_train_raw[N_train:])),
X_test,
zca_bias=cfg.DATASET.ZCA_BIAS,
return_weights=True)
X_train_pp = X_train_pp[:N_train]
elif cfg.DATASET.ZCA_EXTRA_AUGMENT:
_X_train_full = [X_train_full]
for i in range(cfg.DATASET.NUM_ZCA_EXTRA_AUGMENT):
aug_transformer = random_aug.randaugment(
cfg.DATASET.RAND_AUGMENT_N_MAX,
4,
seed=cfg.DATASET.RAND_SEED + i,
transforms=cfg.DATASET.RAND_AUGMENT_AUGS,
replace=cfg.DATASET.RAND_AUGMENT_REPLACE,
random_n=cfg.DATASET.RAND_AUGMENT_RAND_N)
X_train_aug = utils.apply_transform(aug_transformer,
X_train_full_no_corners)
if cfg.DATASET.CORNERS:
print(f"Extracting corners for extra augment index: {i}")
X_train_aug = utils.corners_full(X_train_aug)
_X_train_full.append(X_train_aug)
_X_train_full = np.vstack(_X_train_full)
(X_train_pp, X_test_pp_full), global_ZCA = utils.preprocess(
_X_train_full,
X_test_full,
zca_bias=cfg.DATASET.ZCA_BIAS,
return_weights=True)
X_train_pp = X_train_pp[:X_train_full.shape[0]]
else:
(X_train_pp, X_test_pp_full), global_ZCA = utils.preprocess(
X_train_full,
X_test_full,
zca_bias=cfg.DATASET.ZCA_BIAS,
return_weights=True)
X_test_pp = X_test_pp_full[:X_test.shape[0]]
X_test_2_pp = X_test_pp_full[X_test.shape[0]:]
elif cfg.DATASET.STANDARD_PREPROCESS:
print("standard preprocess")
X_train_pp, X_test_pp = (X_train / 255.0).astype('float32'), (
X_test / 255.0).astype('float32')
X_test_2_pp = (X_test_2 / 255.0).astype('float32')
X_train_pp = (
X_train_pp - CIFAR_MEAN[np.newaxis, np.newaxis, np.newaxis, :]
) / CIFAR_STD[np.newaxis, np.newaxis, np.newaxis, :].astype('float32')
X_test_pp = (
X_test_pp - CIFAR_MEAN[np.newaxis, np.newaxis, np.newaxis, :]
) / CIFAR_STD[np.newaxis, np.newaxis, np.newaxis, :].astype('float32')
X_test_2_pp = (
X_test_2_pp - CIFAR_MEAN[np.newaxis, np.newaxis, np.newaxis, :]
) / CIFAR_STD[np.newaxis, np.newaxis, np.newaxis, :].astype('float32')
global_ZCA = 0
else:
X_train_pp, X_test_pp = (X_train / 255.0).astype('float32'), (
X_test / 255.0).astype('float32')
X_test_2_pp = (X_test_2 / 255.0).astype('float32')
global_ZCA = 0
np.savez("cifar_10_train.npz",
X_train=X_train_pp,
y_train=y_train,
global_ZCA=global_ZCA)
np.savez("cifar_10_test.npz",
X_test=X_test_pp,
y_test=y_test,
global_ZCA=global_ZCA)
np.savez("cifar_10.1.npz",
X_test=X_test_2_pp,
y_test=y_test_2_full,
global_ZCA=global_ZCA)
meta_data = {
"classes": 10,
"d": 1024,
"height": 32,
"width": 32,
"channels": 3,
"cfg": cfg_dict
}
