def np_tensor_np(): image = loadtiff3d( '/Users/wonh/Desktop/flyJanelia/images/1.tif') # x, y, z image = ToTensor()(image) # z, x, y image = image.numpy() * 255 image = image.astype('uint8') # important image = np.transpose(image, (1, 2, 0)) # z, x, y => x, y, z
def callback(self, oimg): try: #if you want to save images and labels ,please uncomment following codes(No.1 to No.4). #NO.1 write_image_name = "image_" + str(self.count) + ".jpg" #No.2 write_label_name = "label_" + str(self.count) + ".jpg" oimg_b = bytes(oimg.data) np_arr = np.fromstring(oimg_b, np.uint8) img = cv2.imdecode(np_arr, cv2.IMREAD_COLOR) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) #No.3 #cv2.imwrite("/home/amsl/images/output/seg_pub_2/image/" + write_image_name, img) image = PIL_Image.fromarray(img) #image = image.crop((0, 120, 640, 480)) image = image.crop((0, 10, 640, 330)) #image = image.resize((1024,512),PIL_Image.NEAREST) image = image.resize((1024,512),PIL_Image.NEAREST) #img_size = image.shape image = ToTensor()(image) image = torch.Tensor(np.array([image.numpy()])) image = image.cuda() input_image = Variable(image) with torch.no_grad(): output_image = self.model(input_image) label = output_image[0].max(0)[1].byte().cpu().data label_color = Colorize()(label.unsqueeze(0)) label_pub = ToPILImage()(label_color) #label_pub = label_pub.resize((1024, 512),PIL_Image.NEAREST) #label_pub = label_pub.resize((1024, 512),PIL_Image.LANCZOS) label_pub = np.asarray(label_pub) #show label. #plt.imshow(label_pub) #plt.pause(0.001) #No.4 #cv2.imwrite("/home/amsl/images/output/seg_pub_2/label/" + write_label_name, label_pub) self.pub_seg.publish(self.bridge.cv2_to_imgmsg(label_pub, "bgr8")) print("published") self.count += 1 except CvBridgeError as e: print(e)
def readRGBData(self, folderName): path = os.path.join(self.rootDir, folderName) print(path) bunch_of_frames = [] for index in range(0, self.perVideo): frames = torch.FloatTensor(3, self.nfra, self.numpixels) end = len( [name for name in sorted(os.listdir(path)) if ".jpg" in name]) offset = [ name for name in sorted(os.listdir(path)) if ".jpg" in name ] # print(offset[0]) # print("offset: ", str(int(offset[0].replace('.jpg','')))) offset = int(offset[0].replace('.jpg', '')) # input() offset = random.randint(offset, (end - self.nfra) - 2) # offset = 1; ### NEED N + 1 frames when starting with raw frames frames = torch.zeros(3, self.nfra + 1, 240, 320) i = 0 for framenum in range(offset, 2 * (self.nfra) + offset, 2): # print("reading from frame " + str('%04d' % framenum) + "...") # print("For rfolder: ", folderName) img_path1 = os.path.join(path, str('%07d' % framenum) + '.jpg') image1 = Image.open(img_path1) image1 = ToTensor()(image1) image1 = image1.float() # print(image1.shape) data = np.transpose(image1.numpy(), (1, 2, 0)) # put height and width in front data = skimage.transform.resize(data, (240, 320)) image1 = torch.from_numpy(np.transpose(data, (2, 0, 1))) # move back # print(image1.shape) frames[:, i, :] = image1 # .view(3, 480 * 640) i = i + 1 bunch_of_frames = bunch_of_frames + [frames] return bunch_of_frames
def upscale_tile(self, idx): x_idx = idx % len(self.tiles[0]) y_idx = idx // len(self.tiles[0]) tile_to_upscale = self.tiles[y_idx][x_idx] upscaled_tile = tile_to_upscale.resize( (self.big_grid_size, self.big_grid_size)) tensor = ToTensor()(upscaled_tile).unsqueeze(0) inputs = {self.model.get_inputs()[0].name: tensor.numpy()} img_hr = torch.tensor(self.model.run(None, inputs)[0].squeeze(0)) img_hr = to_image(img_hr.clip(0, 1)) self.result.paste(img_hr, (y_idx * (self.big_grid_size - (self.overlap * self.scale)), x_idx * (self.big_grid_size - (self.overlap * self.scale))), self.alpha_circle)
def train(): #cross validation order = np.random.RandomState().permutation(len(labeled_dataset)) train_dataset, val_dataset = split_dataset(labeled_dataset, int(training_size * 0.9), order) train_loader = DataLoader(train_dataset, batch_size=batch_size, \ num_workers=4, shuffle=True) valid_loader = DataLoader(val_dataset, batch_size=batch_size, \ num_workers=4) max_total_acc_x = 0 max_euclidean_distance = 99999 for epoch in range(num_epochs): dataloader_iterator = iter(train_loader) try: sample_batched = next(dataloader_iterator) except StopIteration: order = np.random.RandomState().permutation(len(labeled_dataset)) train_dataset, val_dataset = split_dataset( labeled_dataset, int(training_size * 0.9), order) train_loader = DataLoader(train_dataset, batch_size=batch_size, \ num_workers=4, shuffle=True) valid_loader = DataLoader(val_dataset, batch_size=batch_size, \ num_workers=4) dataloader_iterator = iter(train_loader) sample_batched = next(dataloader_iterator) inputs = sample_batched['image'].cuda() labels = sample_batched['hmap'].cuda() coors_bc = sample_batched['coor_1'].cpu().detach().numpy() # img_names = sample_batched['img_name'] origin_imgs = sample_batched['origin_img'] optimizer.zero_grad() outputs = model(inputs) loss = mse(outputs.float(), labels.float()) # loss_bce = nn.functional.binary_cross_entropy(class_pred, class_real) # loss = (1-class_real) * loss_bce + class_real * (l * loss_bce + (1-l)*loss_mse) # torch.mean(loss).backward() loss.backward() optimizer.step() ############################ 896 block ############ empty_batch = True for index, out in enumerate(outputs.cpu().detach().numpy()): w_center, h_center = heatmap_to_coor(out.squeeze()) cropped_image, cropped_hmap = crop(origin_imgs[index], w_center, h_center, coors_bc[index], 224 * 4) cropped_image = ToTensor()(Resize( (224, 224))(Image.fromarray(cropped_image.numpy()))).unsqueeze(0) cropped_hmap = cropped_hmap.unsqueeze(dim=0).unsqueeze(0) if empty_batch: cropped_image_batch = cropped_image cropped_hmap_batch = cropped_hmap empty_batch = False else: cropped_image_batch = torch.cat( [cropped_image_batch, cropped_image]) cropped_hmap_batch = torch.cat( [cropped_hmap_batch, cropped_hmap]) optimizer.zero_grad() outputs = model(cropped_image_batch.cuda()) loss = mse(outputs, cropped_hmap_batch.cuda()) torch.mean(loss).backward() # loss.backward() optimizer.step() ############################ 448 block ############ empty_batch = True for index, out in enumerate(outputs.cpu().detach().numpy()): w_center, h_center = heatmap_to_coor(out.squeeze()) cropped_image, cropped_hmap = crop(origin_imgs[index], w_center, h_center, coors_bc[index], 224 * 2) cropped_image = ToTensor()(Resize( (224, 224))(Image.fromarray(cropped_image.numpy()))).unsqueeze(0) cropped_hmap = cropped_hmap.unsqueeze(dim=0).unsqueeze(0) if empty_batch: cropped_image_batch = cropped_image cropped_hmap_batch = cropped_hmap empty_batch = False else: cropped_image_batch = torch.cat( [cropped_image_batch, cropped_image]) cropped_hmap_batch = torch.cat( [cropped_hmap_batch, cropped_hmap]) optimizer.zero_grad() outputs = model(cropped_image_batch.cuda()) loss = mse(outputs, cropped_hmap_batch.cuda()) torch.mean(loss).backward() # loss.backward() optimizer.step() ############################ 224 block ############ empty_batch = True for index, out in enumerate(outputs.cpu().detach().numpy()): w_center, h_center = heatmap_to_coor(out.squeeze()) cropped_image, cropped_hmap = crop(origin_imgs[index], w_center, h_center, coors_bc[index], 224) cropped_image = ToTensor()( cropped_image.unsqueeze(dim=-1).numpy()).unsqueeze(0) cropped_hmap = cropped_hmap.unsqueeze(dim=0).unsqueeze(0) if empty_batch: cropped_image_batch = cropped_image cropped_hmap_batch = cropped_hmap empty_batch = False else: cropped_image_batch = torch.cat( [cropped_image_batch, cropped_image]) cropped_hmap_batch = torch.cat( [cropped_hmap_batch, cropped_hmap]) optimizer.zero_grad() outputs = model(cropped_image_batch.cuda()) loss = mse(outputs, cropped_hmap_batch.cuda()) # torch.mean(loss).backward() loss.backward() optimizer.step() ############################ 112 block ############ empty_batch = True for index, out in enumerate(outputs.cpu().detach().numpy()): w_center, h_center = heatmap_to_coor(out.squeeze()) cropped_image, cropped_hmap = crop(origin_imgs[index], w_center, h_center, coors_bc[index], int(224 / 2)) cropped_image = ToTensor()(Resize( (224, 224))(Image.fromarray(cropped_image.numpy()))).unsqueeze(0) cropped_hmap = cropped_hmap.unsqueeze(dim=0).unsqueeze(0) if empty_batch: cropped_image_batch = cropped_image cropped_hmap_batch = cropped_hmap empty_batch = False else: cropped_image_batch = torch.cat( [cropped_image_batch, cropped_image]) cropped_hmap_batch = torch.cat( [cropped_hmap_batch, cropped_hmap]) optimizer.zero_grad() outputs = model(cropped_image_batch.cuda()) loss = mse(outputs, cropped_hmap_batch.cuda()) torch.mean(loss).backward() optimizer.step() ############################ 56 block ############ empty_batch = True for index, out in enumerate(outputs.cpu().detach().numpy()): w_center, h_center = heatmap_to_coor(out.squeeze()) cropped_image, cropped_hmap = crop(origin_imgs[index], w_center, h_center, coors_bc[index], int(224 / 4)) cropped_image = ToTensor()(Resize( (224, 224))(Image.fromarray(cropped_image.numpy()))).unsqueeze(0) cropped_hmap = cropped_hmap.unsqueeze(dim=0).unsqueeze(0) if empty_batch: cropped_image_batch = cropped_image cropped_hmap_batch = cropped_hmap empty_batch = False else: cropped_image_batch = torch.cat( [cropped_image_batch, cropped_image]) cropped_hmap_batch = torch.cat( [cropped_hmap_batch, cropped_hmap]) optimizer.zero_grad() outputs = model(cropped_image_batch.cuda()) loss = mse(outputs, cropped_hmap_batch.cuda()) torch.mean(loss).backward() optimizer.step() if (epoch + 1) % 5 == 0: # every 20 mini-batches... e_distance = 0 for index, out in enumerate(outputs): x, y = heatmap_to_coor( out.reshape(224, 224).cpu().detach().numpy()) e_distance += ((int(x/224*1280)-coors_bc[index][0])**2 + \ (int(y/224*1024)-coors_bc[index][1])**2)**0.5 print('Train epoch: {}\tLoss: {:.30f}'.format( epoch + 1, # i_batch * len(inputs), # len(train_loader.dataset), 100. * i_batch / len(train_loader), torch.mean(loss).item())) #/ len(inputs))) # writer.add_scalar("cascade4_training_loss", \ # torch.mean(loss).item(), #/ len(inputs), \ # epoch + epoch * math.ceil(len(train_loader) / batch_size) \ # ) # writer.add_scalar("cascade4_training_Euclidean_Distance", \ # e_distance, # epoch + epoch * math.ceil(len(train_loader) / batch_size) \ # ) if (epoch + 1) % 50 == 0: # every 20 mini-batches... # model.eval() with torch.no_grad(): valid_loss = 0 total_acc_x = 0 total_acc_y = 0 e_distance = 0 for i, batch in enumerate(valid_loader): inputs = batch['image'].float().cuda() labels = batch['hmap'].float().cuda() coors_bc = batch['coor_1'].cpu().detach().numpy() outputs = model(inputs) loss = mse(outputs, labels) outputs = outputs.cpu().detach().numpy() labels = labels.cpu().detach().numpy() sum_acc_x, sum_acc_y, list_acc_x, list_acc_y = accuracy_sum( outputs, coors_bc) total_acc_x += sum_acc_x total_acc_y += sum_acc_y # all_acc_x.extend(list_acc_x) # all_acc_y.extend(list_acc_y) for index, out in enumerate(outputs): x, y = heatmap_to_coor(out.reshape(224, 224)) e_distance += ((int(x/224*1280)-coors_bc[index][0])**2 + \ (int(y/224*1024)-coors_bc[index][1])**2)**0.5 valid_loss = valid_loss / len(valid_loader) print('Valid loss {}'.format(valid_loss)) # writer.add_scalar("Valid_loss_adbc", valid_loss, epoch) # writer.add_scalar("Valid_adbc_Euclidean_Distance", e_distance/len(valid_loader.dataset), epoch) print("=" * 30) print("total acc_x = {:.10f}".format( total_acc_x / len(valid_loader.dataset))) print("total acc_y = {:.10f}".format( total_acc_y / len(valid_loader.dataset))) print("Euclidean Distance: {}".format( e_distance / len(valid_loader.dataset))) print("=" * 30) # if total_acc_x > max_total_acc_x: # max_total_acc_x = total_acc_x if e_distance / len( valid_loader.dataset) < max_euclidean_distance: max_euclidean_distance = e_distance / len( valid_loader.dataset) torch.save(model.state_dict(), saved_weight_dir) print('model saved to ' + saved_weight_dir)