def np_tensor_np():
image = loadtiff3d(
'/Users/wonh/Desktop/flyJanelia/images/1.tif') # x, y, z
image = ToTensor()(image) # z, x, y
image = image.numpy() * 255
image = image.astype('uint8') # important
image = np.transpose(image, (1, 2, 0)) # z, x, y => x, y, z
def callback(self, oimg):
try:
#if you want to save images and labels ,please uncomment following codes(No.1 to No.4).
#NO.1
write_image_name = "image_" + str(self.count) + ".jpg"
#No.2
write_label_name = "label_" + str(self.count) + ".jpg"
oimg_b = bytes(oimg.data)
np_arr = np.fromstring(oimg_b, np.uint8)
img = cv2.imdecode(np_arr, cv2.IMREAD_COLOR)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
#No.3
#cv2.imwrite("/home/amsl/images/output/seg_pub_2/image/" + write_image_name, img)
image = PIL_Image.fromarray(img)
#image = image.crop((0, 120, 640, 480))
image = image.crop((0, 10, 640, 330))
#image = image.resize((1024,512),PIL_Image.NEAREST)
image = image.resize((1024,512),PIL_Image.NEAREST)
#img_size = image.shape
image = ToTensor()(image)
image = torch.Tensor(np.array([image.numpy()]))
image = image.cuda()
input_image = Variable(image)
with torch.no_grad():
output_image = self.model(input_image)
label = output_image[0].max(0)[1].byte().cpu().data
label_color = Colorize()(label.unsqueeze(0))
label_pub = ToPILImage()(label_color)
#label_pub = label_pub.resize((1024, 512),PIL_Image.NEAREST)
#label_pub = label_pub.resize((1024, 512),PIL_Image.LANCZOS)
label_pub = np.asarray(label_pub)
#show label.
#plt.imshow(label_pub)
#plt.pause(0.001)
#No.4
#cv2.imwrite("/home/amsl/images/output/seg_pub_2/label/" + write_label_name, label_pub)
self.pub_seg.publish(self.bridge.cv2_to_imgmsg(label_pub, "bgr8"))
print("published")
self.count += 1
except CvBridgeError as e:
print(e)
def readRGBData(self, folderName):
path = os.path.join(self.rootDir, folderName)
print(path)
bunch_of_frames = []
for index in range(0, self.perVideo):
frames = torch.FloatTensor(3, self.nfra, self.numpixels)
end = len(
[name for name in sorted(os.listdir(path)) if ".jpg" in name])
offset = [
name for name in sorted(os.listdir(path)) if ".jpg" in name
]
# print(offset[0])
# print("offset: ", str(int(offset[0].replace('.jpg',''))))
offset = int(offset[0].replace('.jpg', ''))
# input()
offset = random.randint(offset, (end - self.nfra) - 2)
# offset = 1;
### NEED N + 1 frames when starting with raw frames
frames = torch.zeros(3, self.nfra + 1, 240, 320)
i = 0
for framenum in range(offset, 2 * (self.nfra) + offset, 2):
# print("reading from frame " + str('%04d' % framenum) + "...")
# print("For rfolder: ", folderName)
img_path1 = os.path.join(path, str('%07d' % framenum) + '.jpg')
image1 = Image.open(img_path1)
image1 = ToTensor()(image1)
image1 = image1.float()
# print(image1.shape)
data = np.transpose(image1.numpy(),
(1, 2, 0)) # put height and width in front
data = skimage.transform.resize(data, (240, 320))
image1 = torch.from_numpy(np.transpose(data,
(2, 0, 1))) # move back
# print(image1.shape)
frames[:, i, :] = image1 # .view(3, 480 * 640)
i = i + 1
bunch_of_frames = bunch_of_frames + [frames]
return bunch_of_frames
def upscale_tile(self, idx):
x_idx = idx % len(self.tiles[0])
y_idx = idx // len(self.tiles[0])
tile_to_upscale = self.tiles[y_idx][x_idx]
upscaled_tile = tile_to_upscale.resize(
(self.big_grid_size, self.big_grid_size))
tensor = ToTensor()(upscaled_tile).unsqueeze(0)
inputs = {self.model.get_inputs()[0].name: tensor.numpy()}
img_hr = torch.tensor(self.model.run(None, inputs)[0].squeeze(0))
img_hr = to_image(img_hr.clip(0, 1))
self.result.paste(img_hr,
(y_idx * (self.big_grid_size -
(self.overlap * self.scale)), x_idx *
(self.big_grid_size - (self.overlap * self.scale))),
self.alpha_circle)
def train():
#cross validation
order = np.random.RandomState().permutation(len(labeled_dataset))
train_dataset, val_dataset = split_dataset(labeled_dataset,
int(training_size * 0.9), order)
train_loader = DataLoader(train_dataset, batch_size=batch_size, \
num_workers=4, shuffle=True)
valid_loader = DataLoader(val_dataset, batch_size=batch_size, \
num_workers=4)
max_total_acc_x = 0
max_euclidean_distance = 99999
for epoch in range(num_epochs):
dataloader_iterator = iter(train_loader)
try:
sample_batched = next(dataloader_iterator)
except StopIteration:
order = np.random.RandomState().permutation(len(labeled_dataset))
train_dataset, val_dataset = split_dataset(
labeled_dataset, int(training_size * 0.9), order)
train_loader = DataLoader(train_dataset, batch_size=batch_size, \
num_workers=4, shuffle=True)
valid_loader = DataLoader(val_dataset, batch_size=batch_size, \
num_workers=4)
dataloader_iterator = iter(train_loader)
sample_batched = next(dataloader_iterator)
inputs = sample_batched['image'].cuda()
labels = sample_batched['hmap'].cuda()
coors_bc = sample_batched['coor_1'].cpu().detach().numpy()
# img_names = sample_batched['img_name']
origin_imgs = sample_batched['origin_img']
optimizer.zero_grad()
outputs = model(inputs)
loss = mse(outputs.float(), labels.float())
# loss_bce = nn.functional.binary_cross_entropy(class_pred, class_real)
# loss = (1-class_real) * loss_bce + class_real * (l * loss_bce + (1-l)*loss_mse)
# torch.mean(loss).backward()
loss.backward()
optimizer.step()
############################ 896 block ############
empty_batch = True
for index, out in enumerate(outputs.cpu().detach().numpy()):
w_center, h_center = heatmap_to_coor(out.squeeze())
cropped_image, cropped_hmap = crop(origin_imgs[index], w_center,
h_center, coors_bc[index],
224 * 4)
cropped_image = ToTensor()(Resize(
(224,
224))(Image.fromarray(cropped_image.numpy()))).unsqueeze(0)
cropped_hmap = cropped_hmap.unsqueeze(dim=0).unsqueeze(0)
if empty_batch:
cropped_image_batch = cropped_image
cropped_hmap_batch = cropped_hmap
empty_batch = False
else:
cropped_image_batch = torch.cat(
[cropped_image_batch, cropped_image])
cropped_hmap_batch = torch.cat(
[cropped_hmap_batch, cropped_hmap])
optimizer.zero_grad()
outputs = model(cropped_image_batch.cuda())
loss = mse(outputs, cropped_hmap_batch.cuda())
torch.mean(loss).backward()
# loss.backward()
optimizer.step()
############################ 448 block ############
empty_batch = True
for index, out in enumerate(outputs.cpu().detach().numpy()):
w_center, h_center = heatmap_to_coor(out.squeeze())
cropped_image, cropped_hmap = crop(origin_imgs[index], w_center,
h_center, coors_bc[index],
224 * 2)
cropped_image = ToTensor()(Resize(
(224,
224))(Image.fromarray(cropped_image.numpy()))).unsqueeze(0)
cropped_hmap = cropped_hmap.unsqueeze(dim=0).unsqueeze(0)
if empty_batch:
cropped_image_batch = cropped_image
cropped_hmap_batch = cropped_hmap
empty_batch = False
else:
cropped_image_batch = torch.cat(
[cropped_image_batch, cropped_image])
cropped_hmap_batch = torch.cat(
[cropped_hmap_batch, cropped_hmap])
optimizer.zero_grad()
outputs = model(cropped_image_batch.cuda())
loss = mse(outputs, cropped_hmap_batch.cuda())
torch.mean(loss).backward()
# loss.backward()
optimizer.step()
############################ 224 block ############
empty_batch = True
for index, out in enumerate(outputs.cpu().detach().numpy()):
w_center, h_center = heatmap_to_coor(out.squeeze())
cropped_image, cropped_hmap = crop(origin_imgs[index], w_center,
h_center, coors_bc[index], 224)
cropped_image = ToTensor()(
cropped_image.unsqueeze(dim=-1).numpy()).unsqueeze(0)
cropped_hmap = cropped_hmap.unsqueeze(dim=0).unsqueeze(0)
if empty_batch:
cropped_image_batch = cropped_image
cropped_hmap_batch = cropped_hmap
empty_batch = False
else:
cropped_image_batch = torch.cat(
[cropped_image_batch, cropped_image])
cropped_hmap_batch = torch.cat(
[cropped_hmap_batch, cropped_hmap])
optimizer.zero_grad()
outputs = model(cropped_image_batch.cuda())
loss = mse(outputs, cropped_hmap_batch.cuda())
# torch.mean(loss).backward()
loss.backward()
optimizer.step()
############################ 112 block ############
empty_batch = True
for index, out in enumerate(outputs.cpu().detach().numpy()):
w_center, h_center = heatmap_to_coor(out.squeeze())
cropped_image, cropped_hmap = crop(origin_imgs[index], w_center,
h_center, coors_bc[index],
int(224 / 2))
cropped_image = ToTensor()(Resize(
(224,
224))(Image.fromarray(cropped_image.numpy()))).unsqueeze(0)
cropped_hmap = cropped_hmap.unsqueeze(dim=0).unsqueeze(0)
if empty_batch:
cropped_image_batch = cropped_image
cropped_hmap_batch = cropped_hmap
empty_batch = False
else:
cropped_image_batch = torch.cat(
[cropped_image_batch, cropped_image])
cropped_hmap_batch = torch.cat(
[cropped_hmap_batch, cropped_hmap])
optimizer.zero_grad()
outputs = model(cropped_image_batch.cuda())
loss = mse(outputs, cropped_hmap_batch.cuda())
torch.mean(loss).backward()
optimizer.step()
############################ 56 block ############
empty_batch = True
for index, out in enumerate(outputs.cpu().detach().numpy()):
w_center, h_center = heatmap_to_coor(out.squeeze())
cropped_image, cropped_hmap = crop(origin_imgs[index], w_center,
h_center, coors_bc[index],
int(224 / 4))
cropped_image = ToTensor()(Resize(
(224,
224))(Image.fromarray(cropped_image.numpy()))).unsqueeze(0)
cropped_hmap = cropped_hmap.unsqueeze(dim=0).unsqueeze(0)
if empty_batch:
cropped_image_batch = cropped_image
cropped_hmap_batch = cropped_hmap
empty_batch = False
else:
cropped_image_batch = torch.cat(
[cropped_image_batch, cropped_image])
cropped_hmap_batch = torch.cat(
[cropped_hmap_batch, cropped_hmap])
optimizer.zero_grad()
outputs = model(cropped_image_batch.cuda())
loss = mse(outputs, cropped_hmap_batch.cuda())
torch.mean(loss).backward()
optimizer.step()
if (epoch + 1) % 5 == 0: # every 20 mini-batches...
e_distance = 0
for index, out in enumerate(outputs):
x, y = heatmap_to_coor(
out.reshape(224, 224).cpu().detach().numpy())
e_distance += ((int(x/224*1280)-coors_bc[index][0])**2 + \
(int(y/224*1024)-coors_bc[index][1])**2)**0.5
print('Train epoch: {}\tLoss: {:.30f}'.format(
epoch + 1,
# i_batch * len(inputs),
# len(train_loader.dataset), 100. * i_batch / len(train_loader),
torch.mean(loss).item())) #/ len(inputs)))
# writer.add_scalar("cascade4_training_loss", \
# torch.mean(loss).item(), #/ len(inputs), \
# epoch + epoch * math.ceil(len(train_loader) / batch_size) \
# )
# writer.add_scalar("cascade4_training_Euclidean_Distance", \
# e_distance,
# epoch + epoch * math.ceil(len(train_loader) / batch_size) \
# )
if (epoch + 1) % 50 == 0: # every 20 mini-batches...
# model.eval()
with torch.no_grad():
valid_loss = 0
total_acc_x = 0
total_acc_y = 0
e_distance = 0
for i, batch in enumerate(valid_loader):
inputs = batch['image'].float().cuda()
labels = batch['hmap'].float().cuda()
coors_bc = batch['coor_1'].cpu().detach().numpy()
outputs = model(inputs)
loss = mse(outputs, labels)
outputs = outputs.cpu().detach().numpy()
labels = labels.cpu().detach().numpy()
sum_acc_x, sum_acc_y, list_acc_x, list_acc_y = accuracy_sum(
outputs, coors_bc)
total_acc_x += sum_acc_x
total_acc_y += sum_acc_y
# all_acc_x.extend(list_acc_x)
# all_acc_y.extend(list_acc_y)
for index, out in enumerate(outputs):
x, y = heatmap_to_coor(out.reshape(224, 224))
e_distance += ((int(x/224*1280)-coors_bc[index][0])**2 + \
(int(y/224*1024)-coors_bc[index][1])**2)**0.5
valid_loss = valid_loss / len(valid_loader)
print('Valid loss {}'.format(valid_loss))
# writer.add_scalar("Valid_loss_adbc", valid_loss, epoch)
# writer.add_scalar("Valid_adbc_Euclidean_Distance", e_distance/len(valid_loader.dataset), epoch)
print("=" * 30)
print("total acc_x = {:.10f}".format(
total_acc_x / len(valid_loader.dataset)))
print("total acc_y = {:.10f}".format(
total_acc_y / len(valid_loader.dataset)))
print("Euclidean Distance: {}".format(
e_distance / len(valid_loader.dataset)))
print("=" * 30)
# if total_acc_x > max_total_acc_x:
# max_total_acc_x = total_acc_x
if e_distance / len(
valid_loader.dataset) < max_euclidean_distance:
max_euclidean_distance = e_distance / len(
valid_loader.dataset)
torch.save(model.state_dict(), saved_weight_dir)
print('model saved to ' + saved_weight_dir)
