def __call(self, sample):
transform = self.train_transform
xi = transforms(sample)
xj = transforms(sample)
return xi, xj
def process_results(results, transforms=None):
if transforms is not None:
results["inputs"] = transforms(results["inputs"])
results["targets"] = transforms(results["targets"])
results['predictions'] = transforms(results['predictions'])
results['hw_validation'] = results.copy()
return results
def draw_data(self):
transforms = self.raw_data[0]
disp_ref_image = self.raw_data[1].clone()
ref_gt = self.raw_data[2].clone()
disp_cur_image = self.raw_data[3].clone()
cur_gt = self.raw_data[4].clone()
pred_box = self.raw_data[5].clone()
disp_ref_image = transforms(disp_ref_image)
disp_cur_image = transforms(disp_cur_image)
c_ref_gt_rect = ref_gt
c_cur_gt_rect = cur_gt
c_pred_rect = pred_box
# --------------------- convert to numpy type --------------------
disp_ref_image = np.array(torchvision.transforms.ToPILImage()(
disp_ref_image.cpu()))
disp_cur_image = np.array(torchvision.transforms.ToPILImage()(
disp_cur_image.cpu()))
# ---------------------- draw rectangles -------------------------
# for ref
cv2.rectangle(disp_ref_image,
(int(c_ref_gt_rect[0]), int(c_ref_gt_rect[1])),
(int(c_ref_gt_rect[2]), int(c_ref_gt_rect[3])),
(0, 255, 0), 1)
# for cur
cv2.rectangle(disp_cur_image,
(int(c_cur_gt_rect[0]), int(c_cur_gt_rect[1])),
(int(c_cur_gt_rect[2]), int(c_cur_gt_rect[3])),
(0, 255, 0), 1)
#cv2.rectangle(disp_cur_image,
# (int(c_pred_rect[0]), int(c_pred_rect[1])),
# (int(c_pred_rect[2]),
# int(c_pred_rect[3])), (255, 0, 0), 1)
# ----------------- convert to tensor ---------------
disp_ref_image = numpy_to_torch(disp_ref_image).squeeze(0)
disp_ref_image = disp_ref_image.float()
disp_cur_image = numpy_to_torch(disp_cur_image).squeeze(0)
disp_cur_image = disp_cur_image.float()
# ----------------- visdom image show ------------------
self.visdom.image(disp_ref_image,
opts={'title': self.title + '_ref'},
win=self.title + '_ref')
self.visdom.image(disp_cur_image,
opts={'title': self.title + '_cur'},
win=self.title + '_cur')
def fetch_all_external_targets(target_label,
root_path,
subset,
start_idx,
end_idx,
num,
transforms,
device='cpu'):
import math
from PIL import Image
target_list = []
indices = []
# target_label = [int(i == target_label) for i in range(10)]
if start_idx == -1 and end_idx == -1:
print(
"No specific indices are determined, so try to include whatever we find"
)
idx = 1
while True:
path = '{}_{}_{}.jpg'.format(root_path, '%.2d' % subset,
'%.3d' % idx)
if os.path.exists(path):
img = Image.open(path)
target_list.append([
transforms(img)[None, :, :, :].to(device),
torch.tensor([target_label]).to(device)
])
indices.append(idx)
idx += 1
else:
print("In total, we found {} images of target {}".format(
len(indices), subset))
break
else:
assert start_idx != -1
assert end_idx != -1
for target_index in range(start_idx, end_idx + 1):
indices.append(target_index)
path = '{}_{}_{}.jpg'.format(root_path, '%.2d' % subset,
'%.3d' % target_index)
assert os.path.exists(path), "external target couldn't find"
img = Image.open(path)
target_list.append([
transforms(img)[None, :, :, :].to(device),
torch.tensor([target_label]).to(device)
])
i = math.ceil(len(target_list) / num)
return [t for j, t in enumerate(target_list) if j % i == 0], [t for j, t in enumerate(indices) if j % i == 0], \
[t for j, t in enumerate(target_list) if j % i != 0], [t for j, t in enumerate(indices) if j % i != 0]
def __getitem__(self, index):
labels = {}
line = self.dataset[index]
strs = line.split()
# print(os.path.join(IMG_BASE_DIR, strs[0]))
_img_data = Image.open(os.path.join(IMG_BASE_DIR, strs[0]))
img_data = transforms(_img_data)
# img_data = transforms(_img_data)
# _boxes = np.array(float(x) for x in strs[1:])
_boxes = np.array(list(map(float, strs[1:])))
boxes = np.split(_boxes, len(_boxes) // 5)
for feature_size, anchors in cfg.ANCHORS_GROUP.items():
labels[feature_size] = np.zeros(shape=(feature_size, feature_size,
3, 5 + cfg.CLASS_NUM))
for box in boxes:
cls, cx, cy, w, h = box
cx_offset, cx_index = math.modf(cx * feature_size /
cfg.IMG_WIDTH)
cy_offset, cy_index = math.modf(cy * feature_size /
cfg.IMG_WIDTH)
for i, anchor in enumerate(anchors):
anchor_area = cfg.ANCHORS_GROUP_AREA[feature_size][i]
p_w, p_h = w / anchor[0], h / anchor[1]
p_area = w * h
iou = min(p_area, anchor_area) / max(p_area, anchor_area)
labels[feature_size][int(cy_index),
int(cx_index), i] = np.array([
iou, cx_offset, cy_offset,
np.log(p_w),
np.log(p_h),
*one_hot(cfg.CLASS_NUM, int(cls))
]) #4,i
return labels[13], labels[26], labels[52], img_data
def get_pose_estimation_prediction(cfg, model, image, vis_thre, transforms):
# size at scale 1.0
base_size, center, scale = get_multi_scale_size(
image, cfg.DATASET.INPUT_SIZE, 1.0, 1.0
)
parser = HeatmapRegParser(cfg)
with torch.no_grad():
heatmap_fuse = 0
final_heatmaps = None
final_kpts = None
input_size = cfg.DATASET.INPUT_SIZE
for idx, s in enumerate(sorted(cfg.TEST.SCALE_FACTOR, reverse=True)):
#joints, mask do not use in demo mode
joints = np.zeros((0, cfg.DATASET.NUM_JOINTS, 3))
mask = np.zeros((image.shape[0], image.shape[1]))
image_resized, _, _, center, scale = resize_align_multi_scale(
image, joints, mask, input_size, s, 1.0
)
image_resized = transforms(image_resized)
image_resized = image_resized.unsqueeze(0).cuda()
outputs, heatmaps, kpts = get_multi_stage_outputs(
cfg, model, image_resized, cfg.TEST.FLIP_TEST
)
final_heatmaps, final_kpts = aggregate_results(
cfg, final_heatmaps, final_kpts, heatmaps, kpts
)
for heatmap in final_heatmaps:
heatmap_fuse += up_interpolate(
heatmap,
size=(base_size[1], base_size[0]),
mode='bilinear'
)
heatmap_fuse = heatmap_fuse/float(len(final_heatmaps))
# for only pred kpts
grouped, scores = parser.parse(
final_heatmaps, final_kpts, heatmap_fuse[0], use_heatmap=False
)
if len(scores) == 0:
return []
results = get_final_preds(
grouped, center, scale,
[heatmap_fuse.size(-1), heatmap_fuse.size(-2)]
)
final_results = []
for i in range(len(scores)):
if scores[i] > vis_thre:
final_results.append(results[i])
if len(final_results) == 0:
return []
return final_results
def create_train_data():
# Target: ([isCat, isDog]
train_data_list = []
target_list = []
train_data = []
files = listdir("data/catsanddogs/train/")
for i in range(len(listdir("data/catsanddogs/train/"))):
f = random.choice(files)
files.remove(f)
img = Image.open("data/catsanddogs/train/" + f)
img_tensor = transforms(img)
train_data_list.append(img_tensor)
is_cat = 1 if 'cat' in f else 0
is_dog = 1 if 'dog' in f else 0
target = [is_cat, is_dog]
target_list.append(target)
if len(train_data_list) >= 64:
train_data.append((torch.stack(train_data_list), target_list))
train_data_list = []
target_list = []
print("Loaded batch {} of {}".format(len(train_data), int(len(listdir("data/catsanddogs/train/")) / 64)))
print("Percentage done: {}%".format(
100 * len(train_data) / int(len(listdir("data/catsanddogs/train/")) / 64)))
return train_data
def collate(data, transforms=None):
xs, y = zip(*data)
x1, x2 = zip(*xs)
if transforms: x1 = transforms(x1)
return (torch.stack(x1), torch.stack(x2)), torch.stack(y)
def demo(image_lists):
classes = ["gangjin"]
model = "./best_models/model.pt"
retinanet = torch.load(model)
retinanet = retinanet.cuda()
retinanet.eval()
#detect
transforms = T.Compose([
Normalizer(),
Resizer()
])
for filename in image_lists:
image = skimage.io.imread(filename)
sampler = {"img":image.astype(np.float32)/255.0,"annot":np.empty(shape=(5,5))}
image_tf = transforms(sampler)
scale = image_tf["scale"]
new_shape = image_tf['img'].shape
x = torch.autograd.Variable(image_tf['img'].unsqueeze(0).transpose(1,3), volatile=True)
with torch.no_grad():
scores,_,bboxes = retinanet(x.cuda().float())
bboxes /= scale
scores = scores.cpu().data.numpy()
bboxes = bboxes.cpu().data.numpy()
# select threshold
idxs = np.where(scores > threshold)[0]
scores = scores[idxs]
bboxes = bboxes[idxs]
#embed()
for i,box in enumerate(bboxes):
cv2.rectangle(image,(int(box[1]),int(box[0])),(int(box[3]),int(box[2])),color=(0,0,255),thickness=2 )
results_file.write(filename.split("/")[-1] +","+ str(int(box[1])) + " " + str(int(box[0])) + " " + str(int(box[3])) + " " +str(int(box[2])) + "\n")
print("Predicting image: %s "%filename)
cv2.imwrite("./outputs/%s"%filename.split("/")[-1],image)
def infer(model, image, transforms, device="cpu"):
"""
"""
model.to(device)
model.eval()
since = time.time()
# inference
predictions = pd.DataFrame(columns=AVADataset._label_key_ratings)
# instances
image = transforms(image).type(TORCH_FLOAT).to(device)
# forward
with torch.set_grad_enabled(mode=(phase == TRAIN)):
batch_predicitions = model(batch_images).type(TORCH_FLOAT)
batch_predicitions = pd.DataFrame(
data=batch_predicitions.cpu().numpy(),
columns=AVADataset._label_key_ratings)
prediction = prediction.append(batch_predicitions, ignore_index=True)
predictions.index.name = AVADataset._label_key_id
time_elapsed = time.time() - since
print("training complete in {:.0f}m, {:.0f}s" \
.format(time_elapsed // 60, time_elapsed % 60))
return prediction
def process_image(image):
''' Scales, crops, and normalizes a PIL image for a PyTorch model,
returns an Numpy array
'''
transforms = get_prediction_data_transforms()
image = transforms(image)
return image
def center_by_face(image: torch.Tensor, landmarks: torch.Tensor):
image, landmarks = np.transpose(image.numpy(),
(1, 2, 0)), landmarks.numpy()
# y_center = int(landmarks[36][0] + landmarks[45][0]) // 2
# x_center = int(landmarks[:,1].mean().item())
y, x = landmarks[:, 0], landmarks[:, 1]
keypoints_landmarks = [x, y, 0, 1]
# H, W, C = image.shape
# W_max = min(x_center, W - x_center)
# H_max = min(y_center, H - y_center)
# radius = min(W_max, H_max)
# y_max, y_min = min(H, y_center + H//2), max(0, y_center - H//2)
# x_max, x_min = min(W, x_center + W//2), max(0, x_center - W//2)
H, W, C = image.shape
H09 = int(H * 0.9)
rh = max(int(270 * H09 / W), 270)
rw = max(int(270 * W / H09), 270)
transforms = albumentations.Compose([
albumentations.Crop(x_min=0, y_min=0, x_max=W, y_max=H09),
albumentations.Resize(rh, rw),
albumentations.CenterCrop(256, 256),
# albumentations.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
])
data_dict = transforms(image=image, keypoints=[keypoints_landmarks])
image_new = torch.tensor(np.transpose(data_dict['image'], (2, 0, 1)))
kp_x, kp_y = data_dict['keypoints'][0][0], data_dict['keypoints'][0][1]
keypoints_new = torch.cat(
[torch.tensor(kp_x)[..., None],
torch.tensor(kp_y)[..., None]], dim=1)
return image_new, keypoints_new
def get_vgg_tensor(video_id, transforms, model):
vgg_features = []
path = './frames/' + video_id + '/'
frames = os.listdir(path)
for frame in frames:
img = Image.open(path + frame)
img = transforms(img)
#features = model(img.unsqueeze(0))
vgg_features.append(img)
feats = torch.stack(vgg_features, dim=0)
feats = feats.cuda()
model = model.cuda()
with torch.no_grad():
vgg_tensor = model(feats)
del feats
gc.collect()
model = model.cpu()
vgg_tensor = vgg_tensor.cpu()
return vgg_tensor
def predict_image(net, image):
image_tensor = transforms(image).float()
image_tensor = image_tensor.unsqueeze_(0)
input_tensor = Variable(image_tensor)
output = net(input_tensor)
index = output.data.cpu().numpy().argmax()
return index
def process_image(img, transforms):
image = Image.open(img)
image_tensor = transforms(image)
# Add an extra dimension to image tensor representing batch size
image_tensor = image_tensor.unsqueeze_(0)
return image_tensor
def __getitem__(self, index):
# 1. Image
# -----------------------------------------------------------------------------------
image_path = self.image_files[index].rstrip()
# Apply augmentations
if self.augment:
transforms = torchvision.transforms.Compose([
torchvision.transforms.ColorJitter(brightness=1.5, saturation=1.5, hue=0.1),
torchvision.transforms.ToTensor()
])
else:
transforms = torchvision.transforms.ToTensor()
# Extract image as PyTorch tensor
image = transforms(Image.open(image_path).convert('RGB'))
_, h, w = image.shape
h_factor, w_factor = (h, w) if self.normalized_labels else (1, 1)
# Pad to square resolution
image, pad = pad_to_square(image)
_, padded_h, padded_w = image.shape
# 2. Label
# -----------------------------------------------------------------------------------
label_path = self.label_files[index].rstrip()
targets = None
if os.path.exists(label_path):
boxes = torch.from_numpy(np.loadtxt(label_path).reshape(-1, 5))
# Extract coordinates for unpadded + unscaled image
x1 = w_factor * (boxes[:, 1] - boxes[:, 3] / 2)
y1 = h_factor * (boxes[:, 2] - boxes[:, 4] / 2)
x2 = w_factor * (boxes[:, 1] + boxes[:, 3] / 2)
y2 = h_factor * (boxes[:, 2] + boxes[:, 4] / 2)
# Adjust for added padding
x1 += pad[0]
y1 += pad[2]
x2 += pad[1]
y2 += pad[3]
# Returns (x, y, w, h)
boxes[:, 1] = ((x1 + x2) / 2) / padded_w
boxes[:, 2] = ((y1 + y2) / 2) / padded_h
boxes[:, 3] *= w_factor / padded_w
boxes[:, 4] *= h_factor / padded_h
targets = torch.zeros((len(boxes), 6))
targets[:, 1:] = boxes
# Apply augmentations
if self.augment:
if np.random.random() < 0.5:
image, targets = horisontal_flip(image, targets)
return image_path, image, targets
def picture2(self):
"""create train and test data from pictures of type 2"""
#print(1)
for path in [self.path_train, self.path_test]:
batch_size = self.batch_size
transforms = self.transforms
data = []
data_list = []
target_list = []
files = listdir(path)
#print(2)
for i in range(len(listdir(path))):
#print(2.1)
f = random.choice(files)
#print(2.2)
files.remove(f)
#print(2.3)
#print(path+f)
img = Image.open(path + "/" + f)
#print(2.4)
img_tensor = transforms(img) # (3,256,256)
#print(2.5)
data_list.append(img_tensor)
#print(3)
target = []
#print(f)
for cat in self.categories:
if cat in f:
target.append(1)
else:
target.append(0)
#print(target)
target_list.append(target)
#print(4)
if len(data_list) >= batch_size:
data.append((torch.stack(data_list), target_list))
data_list = []
target_list = []
#print(5)
self.signal.emit('Loaded batch ' + str(len(data)) +
' of ' +
str(int(len(listdir(path)) / batch_size)))
#print('Loaded batch ', len(data), 'of ', int(len(listdir(path)) / batch_size))
#print(6)
self.signal.emit('Percentage Done: ' + str(
round(
100 * len(data) /
int(len(listdir(path)) / batch_size), 2)) + '%')
#print('Percentage Done: ', 100 * len(data) / int(len(listdir(path)) / batch_size), '%')
#print(7)
if (path == self.path_train):
self.train_data = data
else:
self.test_data = data
if (self.path_train == self.path_test):
self.test_data = data
def picture1(self):
"""create train and test data from pictures of type 1"""
for path in [self.path_train, self.path_test]:
# path = self.path_train
batch_size = self.batch_size
transforms = self.transforms
data = []
files = []
data_list = []
target_list = []
categories_for_choice = listdir(path)
categories_for_index = listdir(path)
category_size = len(categories_for_choice)
file_number = 0
for cat in listdir(path):
files.append(listdir(path + "/" + cat))
file_number += len(listdir(path + "/" + cat))
for i in range(file_number):
cat_index = random.randint(0, len(categories_for_index) - 1)
file = random.choice(files[cat_index])
img = Image.open(path + "/" + categories_for_index[cat_index] +
"/" + file)
img_tensor = transforms(img)
data_list.append(img_tensor)
target = [0] * category_size
target[categories_for_choice.index(
categories_for_index[cat_index])] = 1
target_list.append(target)
if (len(data_list) >= batch_size):
data.append((torch.stack(data_list), target_list))
data_list = []
target_list = []
self.signal.emit('Loaded batch ' + str(len(data)) +
' of ' + str(file_number // batch_size))
self.signal.emit('Percentage Done: ' + str(
round(100 * len(data) /
(file_number // batch_size), 2)) + ' %')
# self.signal.emit(str(i))
#print('Loaded batch ', len(data), 'of ', file_number // batch_size)
#print('Percentage Done: ', round(100 * len(data) / (file_number // batch_size), 2), '%')
files[cat_index].remove(file)
if (len(files[cat_index]) == 0):
files.pop(cat_index)
categories_for_index.pop(cat_index)
if (path == self.path_train):
self.train_data = data
else:
self.test_data = data
if (self.path_train == self.path_test):
self.test_data = data
self.categories = listdir(self.path_train)
def get_training_augmentation(img):
#print(type(img))
train_transform = [
albumentations.Resize(32 , 32),
AT.ToTensor()
]
transforms = albumentations.Compose(train_transform)
return lambda img:transforms(image=np.array(img))['image']
def __getitem__(self, idx):
img_name = self.img_dir[idx]
image = Image.open(img_name)
label = self.labels[idx]
label = torch.from_numpy(np.array(label))
transformed_image = transforms(image)
return transformed_image, label
def get_img_embedding(cover, model):
# 处理图片数据， 传入的不是图片则 生成（1， 1000）的0向量
transforms = get_transforms()
if str(cover)[-3:] != 'jpg':
return np.zeros((1, 1000))[0]
image = Image.open(cover).convert("RGB")
image = transforms(image).to(config.device)
return model(image.unsqueeze(0)).detach().cpu().numpy()[0]
def read_image(path_to_image):
"""Reads an image into memory and transform to a tensor."""
img: Image = Image.open(path_to_image)
transforms = default_transforms()
img_tensor: torch.Tensor = transforms(img)
return img_tensor
def batch_X_transform(Xs,transforms):
imgs=[]
for x in Xs:
img=Image.fromarray(x)
img=transforms(img)
imgs.append(img)
imgs=torch.stack(imgs,dim=0)
return imgs
def get_inception_probs(npy_img):
"""
Given a numpy array (corresponding to an image) returns the class probabilities output by inception
"""
img_tensor = transforms(npy_img).unsqueeze(0)
output = model(img_tensor).detach().numpy()[0]
probs = softmax(output)
return probs
def __getitem__(self, idx):
filename = self.filenames[idx]
img = Image.open(filename).convert(self.space)
transforms = self.transform()
tensor = transforms(img)
basename = os.path.basename(filename).split('.')[0].split('_')[-1]
label = self.synset2label[self.validation_synset_labels[int(basename) - 1]]
return tensor, label
def process_image(image_path, transforms):
''' Scales, crops, and normalizes a PIL image for a PyTorch model,
returns an Numpy array
'''
im = Image.open(image_path)
im = transforms(im)
return im
def transform_batch(datas, transforms):
trans_datas = []
for i in range(datas.shape[0]):
trans_data = transforms(datas[i])
trans_datas.append(trans_data)
trans_datas = torch.stack(trans_datas)
return trans_datas
def model10_resnet_train_transforms():
transforms = C.Compose([
A.HorizontalFlip(),
#A.RandomCrop(height=30, width=30, p=5.0),
A.Cutout(num_holes=1, max_h_size=16, max_w_size=16),
P.ToTensor(dict (mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)))
])
return lambda img: transforms(image = np.array(img))["image"]
def generate_images_post():
if "image" not in request.form:
return "image must be provided", status.HTTP_400_BAD_REQUEST
base64Img = request.form['image']
print("POST base64Img :", type(base64Img))
print("POST base64Img :", base64Img)
base64Img = base64Img.replace(" ", "+")
ext = guess_extension(guess_type(base64Img)[0])
with tempfile.TemporaryDirectory() as folder:
local_full_path_file = os.path.join(
folder, "{0}{1}".format(Helper.generate_name(), ext))
exts = ["jpeg", "jpg", "png", "gif", "tiff"]
with open(local_full_path_file, 'wb') as f:
content = base64Img.split(',')[1]
encoded_content = content.encode()
decoded = Helper.decode_base64(encoded_content)
f.write(decoded)
_, hed = edge_transformer.transform(local_full_path_file)
filename_hed = os.path.join(
folder, "{0}_hed_{1}".format(Helper.generate_name(), ext))
cv2.imwrite(filename_hed, hed)
with open(filename_hed, "rb") as image_file:
encoded_string = base64.b64encode(
image_file.read()).decode('ascii')
hed_to_return = np.invert(hed) #inverse color
filename_hed_to_return = os.path.join(
folder, "{0}_hed_to_return_{1}".format(Helper.generate_name(),
ext))
cv2.imwrite(filename_hed_to_return, hed_to_return)
with open(filename_hed_to_return, "rb") as image_file:
hed_to_return_encoded_string = base64.b64encode(
image_file.read()).decode('ascii')
hed_image_base64 = "data:image/png;base64," + hed_to_return_encoded_string
mask = Image.open(filename_hed)
mask = transforms(mask)
mask = mask.unsqueeze(0)
mask = nn.functional.interpolate(mask, size=image_shape)
mask = mask.to(device)
with torch.no_grad():
generated_image = gen(mask)
generated_image_file = os.path.join(folder,
Helper.generate_name() + ".png")
save_image(generated_image[0], generated_image_file)
with open(generated_image_file, "rb") as image_file:
encoded_string = base64.b64encode(
image_file.read()).decode('ascii')
generated_image_base64 = "data:image/png;base64," + encoded_string
return jsonify({
"hed": hed_image_base64,
"generated": generated_image_base64
})
def get_augmentation(img):
train_transform = [
albu.Resize(height=IMG_SIZE, width=IMG_SIZE, p=1),
# albu.GaussNoise(p=1),
# albu.Blur(blur_limit=3, p=1),
albu.GaussianBlur(blur_limit=3, p=1)
]
transforms = albu.Compose(train_transform) # <- Compose
return transforms(image=img)['image'], transforms
