def __call__(self, images, target_layers, target_inds=None, metric=""):
masks_map, pred = self.gcam(images, target_layers, target_inds)
for i in range(min(len(images),5)):
img = images[i]
results_data = [{
"img": denormalize(img),
"label": "Result:"
}]
heatmaps_data = [{
"img": denormalize(img),
"label": "Heatmap:"
}]
for layer in target_layers:
mask = masks_map[layer][i]
heatmap, result = self.visualize_cam(mask, img)
results_data.append({
"img": result,
"label": layer
})
heatmaps_data.append({
"img": heatmap,
"label": layer
})
pred_class = self.classes[pred[i][0]]
fname = "gradcam_%s_%s_%s.png" % (metric, i, pred_class)
self.plot_heatmaps(results_data+heatmaps_data, pred_class, fname)
def predict(self, enc_input, dec_input, actual_output, prev_output, plot=True):
"""
Make a prediction and plot the result
:param enc_input: Input for the encoder
:param dec_input: Input for the decoder
:param actual_output: The actual output (during decoding time)
:param prev_output: The previous output (during encoding time)
:param plot: Boolean to indicate if the prediction should be plotted
:return: Made normalized_predictions.
"""
# Make a prediction on the given data
normalized_predictions = self.make_prediction(enc_input[0, :self.seq_len_in], dec_input[0, 0:1])
# Concat the normalized_ys so we get a smooth line for the normalized_ys
normalized_ys = np.concatenate([prev_output, actual_output[0]])[-self.plot_time_steps_view:]
ys = denormalize(normalized_ys, self.output_std, self.output_mean)
predictions = denormalize(normalized_predictions, self.output_std, self.output_mean)
if plot:
# Plot them
plt.plot(range(0, self.plot_time_steps_view), ys, label="real")
plt.plot(range(self.plot_time_steps_view - self.seq_len_out, self.plot_time_steps_view), predictions,
label="predicted")
plt.legend()
plt.title(label="seq2seq_1dconv")
plt.show()
return normalized_predictions
def random_samples(self):
filenames = os.listdir(self.DBpath)
random_pics_idx = np.random.randint(low=0, high=len(filenames), size=10)
rows = []
for e in random_pics_idx:
img = np.stack([normalize(sci.imread(os.path.join(self.DBpath, filenames[e])))])
splits = filenames[e].split('_')
labels = literal_eval(splits[1].split('.')[0])
row_images = []
row_images.append(denormalize(np.squeeze(img)))
for j in range(0,len(labels)):
fakeLabels = np.copy(labels)
if j < 3: # hair label
if fakeLabels[j] == 0:
fakeLabels[0:3] = [0]*3
fakeLabels[j] = 1
else:
fakeLabels[j] = 0 if fakeLabels[j]==1 else 1
generatedImage = np.squeeze(self.sess.run([self.fakeX], feed_dict={self.realX: img,self.fakeLabelsOneHot: stackLabelsOnly([fakeLabels])}), axis=0)
row_images.append(denormalize(generatedImage))
row = np.concatenate(row_images, axis=1)
rows.append(row)
samples = np.concatenate(rows, axis=0)
sci.imsave('D:/GANSProject/samples/random_samples.jpg', samples)
def recognize(self, path):
figure = parse(path, '*')
res = self.netw.recognize(normalize(figure))
if res[0]:
print "%s[ok] in %d iters" % (path, res[2])
pretty_print(denormalize(res[1], self.m, self.n))
else:
print "%s[fail]" % path
pretty_print(denormalize(res[1], self.m, self.n))
def __call__(self,
images,
truth_inds,
target_layers,
mean,
std,
target_inds=None,
metric="",
per_image=True,
path=None):
masks_map, pred = self.gcam(images, target_inds, target_layers)
if per_image:
for i in range(min(len(images), 5)):
img = images[i]
results_data = [{
"img": denormalize(img, mean, std),
"label": "Result:"
}]
heatmaps_data = [{
"img": denormalize(img, mean, std),
"label": "Heatmap:"
}]
for layer in target_layers:
mask = masks_map[layer][i]
heatmap, result = self.visualize_cam(mask, img)
results_data.append({"img": result, "label": layer})
heatmaps_data.append({"img": heatmap, "label": layer})
pred_class = self.classes[pred[i][0]]
truth_class = self.classes[truth_inds[i]]
fname = path + "gradcam_%s_%s_t%s_p%s.png" % (
metric, i, truth_class, pred_class)
self.plot_heatmaps_indvidual(results_data + heatmaps_data,
truth_class, pred_class, fname)
else:
img_data = []
for i in range(len(images)):
img = images[i]
pred_class = self.classes[pred[i][0]]
truth_class = self.classes[truth_inds[i]]
results_data = [{
"img":
denormalize(img, mean, std),
"label":
"A:%s P:%s" % (truth_class, pred_class)
}]
for layer in masks_map.keys():
mask = masks_map[layer][i]
heatmap, result = self.visualize_cam(
mask, denormalize(img, mean, std))
results_data.append({
"img": result,
"label": "%s" % (layer)
})
img_data.append(results_data)
fname = path + "gradcam_%s.png" % (metric)
self.plot_heatmaps(img_data, fname)
def calculate_accuracy_per_time_step(models, plot=True, save=True):
"""
Plot the accuracy of each model for each timestep and plot it.
:param models: The models
:return Dict containing the average RMSE of each model for each timestep.
"""
slicing_point = 1500
test_xe_batches, test_xd_batches, test_y_batches, test_y_batches_prev = models[0].create_validation_data_with_prev_y_steps(slice_point=slicing_point)
rmse_dict = {}
# Initialize dict containing the rmses
for model in models:
rmse_dict[model.name] = []
print("Total datapoints to process with slicing point {}:".format(slicing_point), len(test_xe_batches))
update_point = int(len(test_xe_batches) / 10)
for i in range(len(test_xe_batches)):
if i % update_point == 0:
print("Processing datapoint", i)
# Reshape into batch
test_xe_batches_inf = np.reshape(test_xe_batches[i], newshape=(1, seq_len_in, input_feature_amount))
test_xd_batches_inf = np.reshape(test_xd_batches[i], newshape=(1, seq_len_in, output_feature_amount))
test_y_batches_inf = np.reshape(test_y_batches[i], newshape=(1, seq_len_out, output_feature_amount))
normalized_ys = test_y_batches[i]
ys = denormalize(normalized_ys, data_dict['output_std'], data_dict['output_mean'])
for model in models:
prediction = model.predict(test_xe_batches_inf, test_xd_batches_inf, test_y_batches_inf,
test_y_batches_prev[i], plot=False)
denormalized_prediction = denormalize(prediction, data_dict['output_std'], data_dict['output_mean'])
rmse_result = []
rmse_result.append(np.sqrt(np.square(ys - denormalized_prediction)))
rmse_dict[model.name].append(rmse_result)
for model in rmse_dict.keys():
rmse_dict[model] = np.average(rmse_dict[model], axis=0)
if save:
out_file = open("avg_rmse_timesteps-agg{}-sp{}.pkl".format(agg_level, slicing_point), "wb")
pickle.dump(rmse_dict, out_file)
if plot:
plt.title("RMSE for {}".format(agg_level))
plt.xlabel("Timestep (15 min)")
plt.ylabel("Average RMSE")
for model in rmse_dict.keys():
plt.plot(rmse_dict[model][0], label=model)
plt.legend()
plt.show()
return rmse_dict
def __call__(self,
images,
truth_inds,
target_layers,
target_inds=None,
metric="",
per_image=True):
masks_map, pred = self.gcam(images, target_layers, target_inds)
if per_image:
for i in range(len(images)):
img = images[i]
results_data = [{"img": denormalize(img), "label": "Result:"}]
heatmaps_data = [{
"img": denormalize(img),
"label": "Heatmap:"
}]
for layer in target_layers:
mask = masks_map[layer][i]
heatmap, result = self.visualize_cam(
mask, denormalize(img))
results_data.append({"img": result, "label": layer})
heatmaps_data.append({"img": heatmap, "label": layer})
pred_class = self.classes[pred[i][0]]
truth_class = self.classes[truth_inds[i]]
fname = "gradcam_%s_%s_t%s_p%s.png" % (metric, i, truth_class,
pred_class)
self.plot_heatmaps_indvidual(results_data + heatmaps_data,
truth_class, pred_class, fname)
else:
img_data = []
for i in range(len(images)):
img = images[i]
pred_class = self.classes[pred[i][0]]
truth_class = self.classes[truth_inds[i]]
results_data = [{
"img":
denormalize(img),
"label":
"Actual: %s\nPredicted: %s" % (truth_class, pred_class)
}]
layer = "layer4"
mask = masks_map[layer][i]
heatmap, result = self.visualize_cam(mask, denormalize(img))
results_data.append({
"img": result,
"label": "GradCAM: %s" % (layer)
})
results_data.append({
"img": heatmap,
"label": "Heatmap: %s" % (layer)
})
img_data.append(results_data)
fname = "gradcam_%s.png" % (metric)
self.plot_heatmaps(img_data, fname)
def process_frame(img):
img = cv2.resize(img, (H, W))
frame = Frame(m, img, K, H, W)
if frame.id == 0:
return
f1 = m.frames[-1]
f2 = m.frames[-2]
# match detection
idx1, idx2, Rt = frame_matches(f1, f2)
# rotation and translation estimation
f1.pose = np.dot(Rt, f2.pose)
# points triangulation
pts4 = triangulate(f1.pose, f2.pose, f1.kps[idx1], f2.kps[idx2])
for i, idx in enumerate(idx2):
if f2.pts[idx] is not None and f1.pts[idx1[i]] is None:
f2.pts[idx].add_observation(f1, idx1[i])
# points filtering
unmatched = np.array([f1.pts[i] is None for i in idx1]).astype(np.bool)
_filter = (np.abs(pts4[:, 3]) > .005) & (pts4[:, 2] > 0) & unmatched
# _filter = np.array([f1.kps[i] is None for i in idx1])
#_filter &= np.abs(pts4[:, 3]) != 0
print('%d new points' % len(unmatched))
pts4 /= pts4[:, 3:]
for i, p in enumerate(pts4):
if not _filter[i]:
continue
u, v = int(round(f1.kpss[idx1[i], 0])), int(round(f1.kpss[idx1[i], 1]))
pt = Point(m, p, img[v, u])
pt.add_observation(f1, idx1[i])
pt.add_observation(f2, idx2[i])
for (p1, p2) in zip(f1.kps[idx1], f2.kps[idx2]):
# draw features
u1, v1 = denormalize(K, p1)
u2, v2 = denormalize(K, p2)
cv2.circle(img, (u1, v1), color=(0, 255, 0), radius=3)
cv2.circle(img, (u2, v2), color=(0, 255, 0), radius=3)
cv2.line(img, (u1, v1), (u2, v2), color=(255, 0, 0))
if frame.id >= 4:
m.optimize()
if os.getenv('d2d'):
cv2.imshow('SLAM', img)
m.display()
if cv2.waitKey(1) == 27:
exit(-1)
def test(self, epoch=0, step=0, save_pic=True):
# load test sample from testloader
test_batch = iter(self.testloader).next()
for key in test_batch['input'].keys():
test_batch['input'][key] = normalize(test_batch['input'][key])
test_img = test_batch['input']['image']
test_img = test_img.type(torch.FloatTensor).to(self.device)
test_pose = test_batch['input']['t_pose']
test_pose = test_pose.type(torch.FloatTensor).to(self.device)
test_style = test_batch['input']['n_img']
test_style = test_style.type(torch.FloatTensor).to(self.device)
# test_map = test_batch['input']['s_map']
# test_map = test_map.type(torch.FloatTensor).to(self.device)
# test_t_map = test_batch['input']['t_map']
# test_t_map = test_t_map.type(torch.FloatTensor).to(self.device)
with torch.no_grad():
source = (test_batch['input']['image']).detach().cpu()
poses = (test_batch['input']['t_pose']).detach().cpu()
poses = denormalize(poses)
poses = torch.max(poses, 1).values
poses = torch.unsqueeze(poses, dim=1)
target_ = (test_batch['input']['target']).detach().cpu()
fake_ = self.pix2pix(test_img, test_pose, test_style)
fake_ = fake_.detach().cpu()
fake_ = denormalize(fake_)
target_ = denormalize(target_)
source = denormalize(source)
sample_source = vutils.make_grid(source, padding=2, normalize=False)
sample_pose = vutils.make_grid(poses, padding=2, normalize=False)
sample_target = vutils.make_grid(target_, padding=2, normalize=False)
sample_fake = vutils.make_grid(fake_, padding=2, normalize=False)
plt.figure(figsize=(32, 16))
plt.axis('off')
plt.title('fake image')
plt.subplot(4, 1, 1)
plt.imshow(np.transpose(sample_source, (1, 2, 0)))
plt.subplot(4, 1, 2)
plt.imshow(np.transpose(sample_pose, (1, 2, 0)))
plt.subplot(4, 1, 3)
plt.imshow(np.transpose(sample_fake, (1, 2, 0)))
plt.subplot(4, 1, 4)
plt.imshow(np.transpose(sample_target, (1, 2, 0)))
if save_pic:
plt.savefig("./output/epoch_{}_iter_{}.png".format(epoch, step))
plt.close()
def val(data, model, y_min, y_max, verbose=True):
mae = 0
for i, x in enumerate(data):
y = x[-1]
y_pred = model(x[:-1])[0]
y_denorm = denormalize(y, y_min, y_max)
y_pred_denorm = denormalize(y_pred, y_min, y_max)
if verbose:
print('[{:3d}] y, y_pred = {:6.3f} {:6.3f}; y, y_pred = {:6.3f} {:6.3f}'.format(
i, y, y_pred, y_denorm, y_pred_denorm))
mae += abs(y_denorm - y_pred_denorm)
mae /= len(data)
if verbose:
print('MAE = {}'.format(mae))
return mae
def plot_gradcam_1(gcam_layers, images, target_classes, target_layers, classes,
predicted_classes, channel_means, channel_stdevs):
i_index = len(images)
l_index = len(target_layers)
for j in range(0, i_index):
image_actual_class = classes[target_classes[j]]
image_predicted_class = classes[predicted_classes[j][0]]
fig = plt.figure(figsize=(12, 8))
plt.subplot(1, l_index + 1, 1)
plt_image = denormalize(images[j].cpu(), channel_means, channel_stdevs)
img = np.transpose(plt_image, (1, 2, 0))
img = np.uint8(255 * img)
plt.tight_layout()
plt.axis('off')
plt.title("Actual %s \n Predicted %s" %
(image_actual_class, image_predicted_class))
plt.imshow(img, interpolation='bilinear')
for i in range(0, l_index):
plt.subplot(1, l_index + 1, i + 2)
heatmap = 1 - gcam_layers[i][j].cpu().numpy()[
0] # reverse the color map
heatmap = np.uint8(255 * heatmap)
heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
superimposed_img = cv2.resize(
cv2.addWeighted(img, 0.5, heatmap, 0.5, 0), (128, 128))
plt.tight_layout()
plt.title(target_layers[i])
plt.axis('off')
plt.imshow(superimposed_img, interpolation='bilinear')
def getPrediction(values):
'''
Uses the model to predict values and returns a JSON
'''
try:
values = np.array(
[values['light'], values['moisture'], values['temperature']],
dtype=np.float32)
values = normalize(values)
values = values.transpose([1, 0])
values = values.reshape([1, 6, 3])
print('Input:', values)
result = model.predict(values)
result = result.reshape([6, 3])
result = result.transpose([1, 0])
result = denormalize(result)
result = result[:, ::-1]
result = result.tolist()
print('Result:', result)
result = {
'light': result[0],
'moisture': result[1],
'temperature': result[2]
}
json_string = json.dumps(result)
except Exception as e:
json_string = json.dumps({'result': str(e)})
return json_string
def get_locations_from_model(trainer, img):
'''
Forward trainer to get locations at each glimpse
Args:
- trainer: Trainer object, whose model has been trained
- img: 4D tensor, [B, C, H, W]
Returns:
- loc: list of (x,y) tuples, locations from raw image
(num_glimpses, num_stacks, 2)
'''
img_size = img.shape[-1]
if trainer.use_gpu:
img = img.cuda()
img = Variable(img, volatile=True)
h_t, l_t = trainer.reset()
loc = [[l.squeeze().data.cpu().numpy() for l in l_t]]
for t in range(trainer.num_glimpses - 1):
h_t, l_t, b_t, p = trainer.model(img, l_t, h_t)
loc.append([l.squeeze().data.cpu().numpy() for l in l_t])
h_t, l_t, b_t, log_p, p = trainer.model(img, l_t, h_t, True)
loc.append([l.squeeze().data.cpu().numpy() for l in l_t])
for i in range(len(loc)):
for j in range(trainer.num_stacks):
size = img_size // 2**j
loc[i][j] = denormalize(size, loc[i][j])
return loc
def test(self, labels=None, img=None):
if not os.path.exists(self.modelPath):
print("The model does not exit")
return
with tf.Session() as sess:
# Restore model weights from previously saved model
folder = os.path.dirname(os.path.normpath(self.modelPath))
saver = tf.train.import_meta_graph(os.path.join(folder,'model.ckpt.meta'))
saver.restore(sess, tf.train.latest_checkpoint(folder))
if img is None:
img = sci.imread(os.path.join(self.DBpath, "000001_[0, 0, 1, 0, 1].jpg"))
if labels is None:
labels = np.random.randint(2, size=(1, 5))
# Get tensors
recov_fakeX = sess.graph.get_tensor_by_name("fakeX:0")
recov_realX = sess.graph.get_tensor_by_name("realX:0")
recov_fakeLabelsOneHot = sess.graph.get_tensor_by_name("fakeLabelsOneHot:0")
img = np.reshape(img, (1, 128, 128, 3))
generatedImage = np.squeeze(sess.run([recov_fakeX], feed_dict={recov_realX: np.stack(img),
recov_fakeLabelsOneHot: stackLabelsOnly(
labels)}), axis=0)
# sci.imsave('img.jpg', img)
# img = normalize(img)
# sci.imsave('out.jpg', denormalize(img))
sci.imsave('outfile1.jpg', denormalize(generatedImage))
def show_samples(self):
# get some random training images
samples = next(iter(self.train_loader))
print("Dimensions:")
print("bg :", list(samples["bg"].shape)[1:])
print("fg_bg :", list(samples["fg_bg"].shape)[1:])
print("fg_bg_mask :", list(samples["fg_bg_mask"].shape)[1:])
print("fg_bg_depth:", list(samples["fg_bg_depth"].shape)[1:], "\n")
num_img = 4
images = []
keys = ("bg", "fg_bg", "fg_bg_mask", "fg_bg_depth")
for i in range(num_img):
for k in keys:
# if k in ("bg", "fg_bg"):
images.append(
denormalize(samples[k][i],
mean=self.stats[k]["mean"],
std=self.stats[k]["std"]))
# else:
# images.append(samples[k][i])
show_images(images,
cols=num_img,
figsize=(6, 6),
show=True,
titles=keys)
def Predict(self, img_path, vis=True):
'''
User function: Run inference on image and visualize it. Output mask saved as output_mask.npy
Args:
img_path (str): Relative path to the image file
vis (bool): If True, predicted mask is displayed.
Returns:
list: List of bounding box locations of predicted objects along with classes.
'''
dirPath = "tmp_test"
if (os.path.isdir(dirPath)):
shutil.rmtree(dirPath)
os.mkdir(dirPath)
os.mkdir(dirPath + "/img_dir")
os.mkdir(dirPath + "/gt_dir")
os.system("cp " + img_path + " " + dirPath + "/img_dir")
os.system("cp " + img_path + " " + dirPath + "/gt_dir")
x_test_dir = dirPath + "/img_dir"
y_test_dir = dirPath + "/gt_dir"
if (self.system_dict["params"]["image_shape"][0] % 32 != 0):
self.system_dict["params"]["image_shape"][0] += (
32 - self.system_dict["params"]["image_shape"][0] % 32)
if (self.system_dict["params"]["image_shape"][1] % 32 != 0):
self.system_dict["params"]["image_shape"][1] += (
32 - self.system_dict["params"]["image_shape"][1] % 32)
preprocess_input = sm.get_preprocessing(
self.system_dict["params"]["backbone"])
test_dataset = Dataset(
x_test_dir,
y_test_dir,
self.system_dict["params"]["classes_dict"],
classes_to_train=self.system_dict["params"]["classes_to_train"],
augmentation=get_validation_augmentation(
self.system_dict["params"]["image_shape"][0],
self.system_dict["params"]["image_shape"][1]),
preprocessing=get_preprocessing(preprocess_input),
)
test_dataloader = Dataloder(test_dataset, batch_size=1, shuffle=False)
image, gt_mask = test_dataset[0]
image = np.expand_dims(image, axis=0)
pr_mask = self.system_dict["local"]["model"].predict(image).round()
np.save("output_mask.npy", pr_mask)
if (vis):
visualize(
image=denormalize(image.squeeze()),
pr_mask=pr_mask[..., 0].squeeze(),
)
def kde(self):
epoch = 5
print("plotting kde of trained model with ", len(self.test_loader),
" examples")
self.load_checkpoint(epoch=epoch)
fig, ax = plt.subplots()
# for key, value in model_preds[model].items():
# fly_kde = value[fly_idx, :, :2]
# t_5_x.append(fly_kde[timestep, 0])
# t_5_y.append(fly_kde[timestep, 1])
img_min = 0
img_max = self.image_size
# m1 = np.array(t_5_x)
# m2 = np.array(t_5_y)
X, Y = np.mgrid[img_min:img_max:100j, img_min:img_max:100j]
positions = np.vstack([X.ravel(), Y.ravel()])
all_locations = torch.Tensor([])
for i, (x, y) in enumerate(self.test_loader):
with torch.no_grad():
if self.use_gpu:
x, y = x.cuda(), y.cuda()
try:
x, y = Variable(x), Variable(y.squeeze(1))
except:
x, y = Variable(x), Variable(y)
# duplicate 10 times
# x = x.repeat(self.M, 1, 1, 1)
# initialize location vector and hidden state
self.batch_size = x.shape[0]
h_t, l_t = self.reset()
# extract the glimpses
for t in range(self.num_glimpses - 1):
# forward pass through model
h_t, l_t, b_t, p = self.model(x, l_t, h_t)
# last iteration
h_t, l_t, b_t, log_probas, p = self.model(x,
l_t,
h_t,
last=True)
all_locations = torch.cat((all_locations, l_t))
coords = denormalize(self.image_size, all_locations)
coords = coords + (self.patch_size / 2)
values = torch.stack((coords[:, 0], (self.image_size - coords[:, 1])))
kernel = stats.gaussian_kde(values)
Z = np.reshape(kernel(positions).T, X.shape)
im = ax.imshow(np.rot90(Z),
cmap=plt.cm.gist_earth_r,
extent=[0, 256, 0, 256])
plt.show()
def inverse(self, mel):
if len(mel.size()) == 2:
mel = mel.unsqueeze(0)
if self.is_normalize:
mel = denormalize(mel)
wav = self.vocoder.inverse(mel).detach().cpu()
# wav: torch.Tensor (1, L)
return wav
def Predict_Frame(self, path, vis=True):
'''
User function: Run inference on image and visualize it. Output mask saved as output_mask.npy
Args:
img_path (str): Relative path to the image file
vis (bool): If True, predicted mask is displayed.
Returns:
list: List of bounding box locations of predicted objects along with classes.
'''
preprocess_input = sm.get_preprocessing(
self.system_dict["params"]["backbone"])
image_normalize = preprocess_input(path)
print(image_normalize)
image = cv2.resize(image_normalize, (int(
len(image_normalize[0]) * 1024 / len(image_normalize[1])), 1024),
interpolation=cv2.INTER_LINEAR)
image = np.expand_dims(image, axis=0)
print("bla bla bla bla ")
print()
print()
print()
print(len(image[0]))
pr_mask = self.system_dict["local"]["model"].predict(image).round()
np.save("output_mask.npy", pr_mask)
'''
if(vis):
visualize(
image=denormalize(image.squeeze()),
pr_mask=pr_mask[..., 0].squeeze(),
)
'''
img_list = [denormalize(image.squeeze())]
label_list = ["image"]
print(img_list)
for i in range(len(self.system_dict["params"]["classes_to_train"])):
img_list.append(pr_mask[..., i].squeeze())
label_list.append(
self.system_dict["params"]["classes_to_train"][i])
print(img_list)
if (vis):
visualize2(label_list, img_list)
return (pr_mask, img_list)
def main(plot_dir, epoch):
# read in pickle files
glimpses = pickle.load(
open(plot_dir + "g_{}.p".format(epoch), "rb")
)
locations = pickle.load(
open(plot_dir + "l_{}.p".format(epoch), "rb")
)
glimpses = np.concatenate(glimpses)
# grab useful params
size = int(plot_dir.split('_')[2][0])
num_anims = len(locations)
num_cols = glimpses.shape[0]
img_shape = glimpses.shape[1:3]#3X32X128
img_shape = img_shape[::-1]
print(glimpses.shape,num_anims)
print(img_shape)
# denormalize coordinates
#print(locations)
coords = [denormalize(img_shape, l) for l in locations]
#coords = [(c[1], c[0]) for c in coords]
#print(coords)
fig, axs = plt.subplots(nrows=1, ncols=num_cols)
# fig.set_dpi(100)
# plot base image
for j, ax in enumerate(axs.flat):
ax.imshow(glimpses[j], cmap="Greys_r")
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
def updateData(i):
#print("i",i)
color = 'r'
co = coords[i]
#print(size, co)
for j, ax in enumerate(axs.flat):
for p in ax.patches:
p.remove()
c = co[j]
rect = bounding_box(
c[0], c[1], size, color
)
ax.add_patch(rect)
# animate
anim = animation.FuncAnimation(
fig, updateData, frames=num_anims, interval=500, repeat=True
)
# save as mp4
name = plot_dir + 'epoch_{}.mp4'.format(epoch)
anim.save(name, extra_args=['-vcodec', 'h264', '-pix_fmt', 'yuv420p'])
def main(plot_dir, epoch):
# read in pickle files
glimpses = pickle.load(
open(plot_dir + "g_{}.p".format(epoch), "rb")
)
locations = pickle.load(
open(plot_dir + "l_{}.p".format(epoch), "rb")
)
probas = pickle.load(
open(plot_dir + "lg_{}.p".format(epoch), "rb")
)
glimpses = np.concatenate(glimpses)
# grab useful params
size = 8
num_cols = glimpses.shape[0]
img_shape = glimpses.shape[1]
# denormalize coordinates
coords = [denormalize(img_shape, l) for l in locations]
fig, axs = plt.subplots(nrows=num_cols // 3, ncols=num_cols // 3)
# fig.set_dpi(100)
# plot base image
for j, ax in enumerate(axs.flat):
ax.imshow(glimpses[j], cmap="Greys_r")
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
names = []
for i in range(len(coords)):
fig.suptitle('{} epoch, {} time'.format(epoch, i + 1))
color = 'r'
co = coords[i]
pr = probas[i].max(axis=1)
for j, ax in enumerate(axs.flat):
for p in ax.patches:
p.remove()
c = co[j]
rect = bounding_box(
c[0], c[1], size, color
)
ax.set_title('Proba: {:.2f}%'.format(pr[j] * 100.),fontsize=10)
ax.add_patch(rect)
name = os.path.join(plot_dir, '{}_epoch{}.png'.format(i + 1, epoch))
plt.savefig(name)
names.append(name)
frames = []
for name in names:
frames.append(imageio.imread(name))
os.remove(name)
imageio.mimsave(os.path.join(plot_dir + 'epoch{}.gif'.format(epoch)), frames, 'GIF', duration=0.5)
def predict_all_validation_data(models, save=True):
"""
Make predictions over the validation data, saving the predicted value and actual value
:param models: The models
:return Dict containing the predicted and actual values of the all datapoints in the validation data
"""
slicing_point = 1500
test_xe_batches, test_xd_batches, test_y_batches, test_y_batches_prev = models[0].create_validation_data_with_prev_y_steps(slice_point=slicing_point)
values_dict = dict()
# Initialize dict
for model in models:
values_dict[model.name] = []
print("Total datapoints to process with slicing point {}:".format(slicing_point), len(test_xe_batches))
update_point = int(len(test_xe_batches) / 10)
for i in range(len(test_xe_batches)):
if i % update_point == 0:
print("Processing datapoint", i)
# Reshape into batch
test_xe_batches_inf = np.reshape(test_xe_batches[i], newshape=(1, seq_len_in, input_feature_amount))
test_xd_batches_inf = np.reshape(test_xd_batches[i], newshape=(1, seq_len_in, output_feature_amount))
test_y_batches_inf = np.reshape(test_y_batches[i], newshape=(1, seq_len_out, output_feature_amount))
normalized_ys = test_y_batches[i]
ys = denormalize(normalized_ys, data_dict['output_std'], data_dict['output_mean'])
for model in models:
prediction = model.predict(test_xe_batches_inf, test_xd_batches_inf, test_y_batches_inf,
test_y_batches_prev[i], plot=False)
denormalized_prediction = denormalize(prediction, data_dict['output_std'], data_dict['output_mean'])
result = (denormalized_prediction, ys)
values_dict[model.name].append(result)
if save:
out_file = open("predicted_and_actuals-agg{}-sp{}.pkl".format(agg_level, slicing_point), "wb")
pickle.dump(values_dict, out_file)
return values_dict
def val(self, train=True, verbose=False):
data = self.train_data if train else self.test_data
self.model.eval()
with torch.no_grad():
mae = 0
for i, (x, y) in enumerate(data):
y_pred = self.model(x)
y_denorm = denormalize(y, self.minimums[-1],
self.maximums[-1]).item()
y_pred_denorm = denormalize(y_pred, self.minimums[-1],
self.maximums[-1]).item()
if verbose:
print(
'[{:3d}] y, y_pred = {:6.3f} {:6.3f}; y, y_pred = {:6.3f} {:6.3f}'
.format(i, y.item(), y_pred.item(), y_denorm,
y_pred_denorm))
mae += abs(y_denorm - y_pred_denorm)
mae /= len(data)
if verbose:
print('MAE = {}'.format(mae))
return mae
def plot_rollout_test(self, rollout):
# Note: A_gt was passed to this class during "SIMILE_PREDICT" initialization if present
if (self.op_param['normalize_output'] == 'True'):
Y_norm_param = pickle.load(
open(
self.data_param['model_dir'] +
self.data_param['model_label'] + '_Ynorm.p', 'rb'))
if (self.op_param['normalize_output'] == 'True'):
if (self.op_param['include_gt'] == True):
A_gt_denorm = utils.denormalize(self.A_gt,
Y_norm_param['min_val'],
Y_norm_param['max_val'])
else:
A_gt_denorm = None
if (self.op_param['only_env_feat'] == 'True'):
self.plot_results(rollout['y_hat_raw'], self.A_gt, 'test',
self.pol_load - 1, 'check_env_feat')
if (self.op_param['normalize_output'] == 'True'):
rollout_raw_denorm = utils.denormalize(rollout['y_hat_raw'],
Y_norm_param['min_val'],
Y_norm_param['max_val'])
self.plot_results(rollout_raw_denorm, A_gt_denorm, 'denorm',
self.pol_load - 1, 'check_env_feat')
else:
self.plot_results(rollout['y_hat'], self.A_gt, 'test',
self.pol_load - 1, 'rollout')
self.plot_autoregressor(rollout['y_hat'], rollout['a_h'],
rollout['y_hat_raw'], self.A_gt, 'test',
self.pol_load - 1)
if (self.op_param['normalize_output'] == 'True'):
rollout_denorm = utils.denormalize(rollout['y_hat'],
Y_norm_param['min_val'],
Y_norm_param['max_val'])
self.plot_results(rollout_denorm, A_gt_denorm, 'denorm',
self.pol_load - 1, 'rollout')
def plot_images(img_data, classes, img_name):
figure = plt.figure(figsize=(10, 10))
num_of_images = len(img_data)
for index in range(1, num_of_images + 1):
img = denormalize(img_data[index-1]["img"]) # unnormalize
plt.subplot(5, 5, index)
plt.axis('off')
plt.imshow(np.transpose(img.cpu().numpy(), (1, 2, 0)))
plt.title("Predicted: %s\nActual: %s" % (classes[img_data[index-1]["pred"]], classes[img_data[index-1]["target"]]))
plt.tight_layout()
plt.savefig(img_name)
def plot_random_day_sample(models):
"""
Plot a random sample, from the start until the end of the day, with all the models predictions
:param models: The models
"""
predict_x_batches, predict_y_batches, predict_y_batches_prev = seq2seq.create_validation_sample(is_start_of_day=True)
predictions = {}
for model in models:
prediction = model.predict(predict_x_batches[0], predict_x_batches[1], predict_y_batches,
predict_y_batches_prev)
predictions[model.name] = denormalize(prediction, data_dict['output_std'], data_dict['output_mean'])
normalized_ys = np.concatenate([predict_y_batches_prev, predict_y_batches[0]])[-plot_time_steps_view:]
ys = denormalize(normalized_ys, data_dict['output_std'], data_dict['output_mean'])
ax = plt.subplot()
# removing top and right borders
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
# adds major gridlines
ax.grid(color='grey', linestyle='-', linewidth=0.25, alpha=0.5)
# Set labels
ax.set_xlabel("Time (15-min interval)")
ax.set_ylabel("Energy use [kWh]")
ax.plot(range(0, plot_time_steps_view), ys, label="real", linewidth=0.8)
for model_name in predictions.keys():
ax.plot(range(plot_time_steps_view - seq_len_out, plot_time_steps_view), predictions[model_name],
label=model_name + " prediction", linewidth=0.8)
ax.legend()
plt.show()
def test_denormalize(self):
job = {
"website": "https://www.elli.eco/de/startseite",
"rating": 3.3,
"remote": False,
"name": "Elli",
"speculative": True,
"field": "Electric Mobility",
"jobs": "https://elli.jobs.personio.de/",
"geo": [
{
"lat": 52.526286,
"country": "Germany",
"long": 13.4153968
},
{
"lat": 52.4210151,
"country": "Germany",
"long": 10.7433627
}
],
"review": "https://www.kununu.com/de/elli",
"description": "Charging solutions by Volkswagen"
}
assert denormalize(job) == [
{'country': 'Germany',
'description': 'Charging solutions by Volkswagen',
'field': 'Electric Mobility',
'geo': {'lat': 52.526286,
'long': 13.4153968},
'jobs': 'https://elli.jobs.personio.de/',
'name': 'Elli',
'rating': 3.3,
'remote': False,
'review': 'https://www.kununu.com/de/elli',
'speculative': True,
'website': 'https://www.elli.eco/de/startseite'},
{'country': 'Germany',
'description': 'Charging solutions by Volkswagen',
'field': 'Electric Mobility',
'geo': {'lat': 52.4210151,
'long': 10.7433627},
'jobs': 'https://elli.jobs.personio.de/',
'name': 'Elli',
'rating': 3.3,
'remote': False,
'review': 'https://www.kununu.com/de/elli',
'speculative': True,
'website': 'https://www.elli.eco/de/startseite'},
]
def main(plot_dir, epoch):
# read in pickle files
glimpses = pickle.load(open(plot_dir + "g_{}.p".format(epoch), "rb"))
locations = pickle.load(open(plot_dir + "l_{}.p".format(epoch), "rb"))
# from ipdb import set_trace
#
# set_trace()
glimpses = np.concatenate(glimpses)
# grab useful params
size = int(plot_dir.split("_")[2][0])
num_anims = len(locations)
num_cols = glimpses.shape[0]
img_shape = glimpses.shape[1]
# denormalize coordinates
coords = [denormalize(img_shape, l) for l in locations]
fig, axs = plt.subplots(nrows=1, ncols=num_cols)
# fig.set_dpi(100)
# plot base image
for j, ax in enumerate(axs.flat):
ax.imshow(glimpses[j], cmap="Greys_r")
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
def updateData(i):
color = "r"
co = coords[i]
for j, ax in enumerate(axs.flat):
for p in ax.patches:
p.remove()
c = co[j]
rect = bounding_box(c[0], c[1], size, color)
ax.add_patch(rect)
# animate
anim = animation.FuncAnimation(fig,
updateData,
frames=num_anims,
interval=500,
repeat=True)
# save as mp4
name = plot_dir + "epoch_{}.mp4".format(epoch)
anim.save(name, extra_args=["-vcodec", "h264", "-pix_fmt", "yuv420p"])
def plot_random_sample(models):
"""
Plot a random sample with all the models predictions
:param models: The models
"""
predict_x_batches, predict_y_batches, predict_y_batches_prev = seq2seq.create_validation_sample()
predictions = {}
for model in models:
prediction = model.predict(predict_x_batches[0], predict_x_batches[1], predict_y_batches, predict_y_batches_prev)
predictions[model.name] = denormalize(prediction, data_dict['output_std'], data_dict['output_mean'])
normalized_ys = np.concatenate([predict_y_batches_prev, predict_y_batches[0]])[-plot_time_steps_view:]
ys = denormalize(normalized_ys, data_dict['output_std'], data_dict['output_mean'])
plt.plot(range(0, plot_time_steps_view), ys, label="real")
for model_name in predictions.keys():
plt.plot(range(plot_time_steps_view - seq_len_out, plot_time_steps_view), predictions[model_name],
label=model_name + " prediction")
plt.legend()
plt.title(label="predictions")
plt.show()
def main():
# Data loading
# train_dataset = CustomDataset(root=config.root_train, annFile=config.annFile_train, transforms=config.train_transforms, catagory=config.CATEGORY_FILTER)
val_dataset = CustomDataset(root=config.root_val,
annFile=config.annFile_val,
transforms=config.val_transforms,
catagory=config.CATEGORY_FILTER)
# train_loader = DataLoader(dataset=train_dataset, batch_size=16, num_workers=2, pin_memory=True, shuffle=True, drop_last=True)
val_loader = DataLoader(dataset=val_dataset,
batch_size=16,
num_workers=2,
pin_memory=True,
shuffle=False,
drop_last=True)
# Model
model = YoloV3(num_classes=config.C).to(device=config.DEVICE)
optimizer = optim.Adam(model.parameters(),
lr=config.LEARNING_RATE,
weight_decay=config.WEIGHT_DECAY)
# Miscellaneous
scaled_anchors = (torch.tensor(config.anchors) * torch.tensor(
config.Scale).unsqueeze(1).unsqueeze(1).repeat(1, 3, 2)).to(
config.DEVICE)
# Loading previously saved model weights
load_checkpoint("res50_35k.pth.tar", model, optimizer,
config.LEARNING_RATE)
# Rendering loop
model.eval()
for cycle, (x, y) in enumerate(val_loader):
with torch.no_grad():
x_gpu = x.to(config.DEVICE)
yp = model(x_gpu)
yp = [yp[0].to('cpu'), yp[1].to('cpu'), yp[2].to('cpu')]
x = denormalize(x) * 255
draw_y_on_x(x, y)
draw_yp_on_x(x, yp, probability_threshold=0.5, anchors=config.anchors)
# Save batch grid as image
image_dir = "./batch_dir"
image_dir_exists = os.path.exists(image_dir)
if not image_dir_exists:
os.makedirs(image_dir)
img_name = str(image_dir) + "/batch_" + str(cycle) + ".png"
save_image(x / 255, img_name)
def fitDecay(self):
self.widget_fitting.clear()
self.widget_fitting.plot(self.dataLoadedX, self.dataLoadedY)
if self.radioButton_decay1.isChecked():
decay = decay1
bounds = [[0, 0, -np.inf], np.inf]
elif self.radioButton_decay2.isChecked():
decay = decay2
bounds = [[0, 0, 0, 0, -np.inf], np.inf]
elif self.radioButton_decay3.isChecked():
decay = decay3
bounds = [[0, 0, 0, 0, 0, 0, -np.inf], np.inf]
starting = self.doubleSpinBox_starting.value()
ending = self.doubleSpinBox_ending.value()
selectedIndices = np.where(
np.logical_and(self.dataLoadedX > starting,
self.dataLoadedX < ending))[0]
xSelected = self.dataLoadedX[selectedIndices]
ySelected = self.dataLoadedY[selectedIndices]
xFit = normalize(xSelected)
params, corrs = curve_fit(decay, xFit, ySelected, bounds=bounds)
self.yFitted = decay(xFit, *params)
self.xFitReal = np.round(denormalize(xFit, xSelected), 2)
self.widget_fitting.plot(self.xFitReal,
self.yFitted,
pen=mkPen(color='r'))
if decay == decay1:
self.textBrowser_result.setText(
'A=' + str(np.round(params[0], 5)) + ', t=' +
str(np.round(denormalize(params[1], xSelected), 5)) + ', y0=' +
str(np.round(params[2], 5)))
elif decay == decay2:
# paramIndices = np.argsort([params[1],params[2]])
self.textBrowser_result.setText(
'A1=' + str(np.round(params[0], 5)) + ', A2=' +
str(np.round(params[1], 5)) + ', t1=' +
str(np.round(denormalize(params[2], xSelected), 5)) + ', t2=' +
str(np.round(denormalize(params[3], xSelected), 5)) + ', y0=' +
str(np.round(params[4], 5)))
elif decay == decay3:
self.textBrowser_result.setText(
'A1=' + str(np.round(params[0], 5)) + ', A2=' +
str(np.round(params[1], 5)) + ', A3=' +
str(np.round(params[2], 5)) + ', t1=' +
str(np.round(denormalize(params[3], xSelected), 5)) + ', t2=' +
str(np.round(denormalize(params[4], xSelected), 5)) + ', t3=' +
str(np.round(denormalize(params[5], xSelected), 5)) + ', y0=' +
str(np.round(params[6], 5)))
# target_values = target_values[indices]
predictions = zeros((len(verificationset), ), dtype=[('dateAndTime', datetime), ('lokrr', float)])
#l = lokrr(trainingset, testingset, 1)
# with open('lokrr_object2.pkl', 'wb') as output:
# pickle.dump(l, output, pickle.HIGHEST_PROTOCOL)
#load lokrr object from file
file = open('lokrr_object2.pkl', 'rb')
file.seek(0)
l = pickle.load(file)
h = []
for i in range(0, len(verificationset)):
curr = get_data_point(verificationset, i)
while(len(h) > 0 and h[0][0] < curr[0]): #new travel times are observed prior to the current time
index = heapq.heappop(h)[1]
observation = get_data_point(verificationset, index)
l.update(observation)
predictions[i] = (curr[0], l.predict(curr[0:3]))
print predictions[i]
heapq.heappush(h, (curr[0] + timedelta(seconds=curr[3]), i))
predictions['lokrr'] = denormalize(predictions['lokrr'], min_travel_time, max_travel_time)
save_path = "../../Data/Autopassdata/Singledatefiles/Dataset/predictions/"
saveDataSet(save_path, "_lokrr.csv", predictions, ('%s;%f'), 'dateAndTime;lokrr')
