def process(self, frame):
if self.display.get():
input_image = cv2.resize(frame,
None,
fx=self.scale_factor,
fy=self.scale_factor)
input_image = Image.fromarray(input_image)
mask = self.get_mask(input_image)
mask = cv2.resize((mask * 255).astype(np.uint8),
frame.shape[:2][::-1],
interpolation=cv2.INTER_CUBIC)
kernel = np.ones((7, 7), np.uint8)
mask = cv2.erode(mask, kernel, iterations=1)
kernel = kernel / kernel.sum()
mask = cv2.filter2D(mask, -1, kernel)
mask = Image.fromarray(mask)
background = self.get_background_frame()[:, ::-1, ::-1]
if background.shape != frame.shape:
background = np.array(
crop_center(Image.fromarray(background),
*frame.shape[:2][::-1]))
frame = np.array(
Image.composite(Image.fromarray(frame),
Image.fromarray(background), mask))
return frame
def eco_preprocessing(frames, batch_size, time_step, image_size,
input_channels, is_training):
res = []
selected_frames = []
chunks = np.array_split(frames, time_step)
for c in chunks:
selected_idx = np.random.randint(c.shape[0])
frame_sample = c[selected_idx]
selected_frames.append(frame_sample)
is_flip = random.randint(1, 10) % 2 == 0
for f in selected_frames:
if is_training:
resized = utils.random_crop(f, image_size[0], image_size[1])
if is_flip:
resized = np.fliplr(resized)
else:
resized = utils.crop_center(f, image_size[0], image_size[1])
resized = resized - [104, 117, 123]
resized = resized.astype(np.float32)
res.append(resized)
return np.array(res)
def load_images_and_labels(self,
imgs_names,
image_dir,
n_class,
file_names_dict,
num_channel=3,
do_center_crop=False):
imgs = np.zeros((imgs_names.shape[0], self.input_size, self.input_size,
num_channel),
dtype=np.float32)
labels = np.zeros((imgs_names.shape[0], n_class), dtype=np.float32)
for i, img_name in tqdm(enumerate(imgs_names)):
img = imread(os.path.join(image_dir, img_name))
if do_center_crop and self.input_size == 128:
img = crop_center(img, 150, 150)
img = np.array(
Image.fromarray(img).resize(
(self.input_size, self.input_size)))
# img = scm.imresize(img, [self.input_size, self.input_size, num_channel]) # not supported by scipy>=1.4
img = np.reshape(img,
[self.input_size, self.input_size, num_channel])
img = img / 255.0
img = img - 0.5
img = img * 2.0
imgs[i] = img
try:
labels[i] = file_names_dict[img_name]
except:
print(img_name)
labels[np.where(labels == -1)] = 0
return imgs, labels
def __getitem__(self, index):
if isinstance(index, ImagePatchIndex):
patch_index = index.patch_index
index = index.image_index
else:
patch_index = 0 #FIXME sort this out
input_id = self.input_index[index % len(self)]
input_img = self._load_input(input_id)
#print(input_id, index, patch_index)
h, w = input_img.shape[:2]
if self.train:
if self.generate_target:
target_arr = self.data_by_id[input_id]['coords']
else:
target_arr = self._load_target(input_id)
#print(target_arr.shape)
attempts = 2
for i in range(attempts):
pw, ph = self.patch_size
cx, cy = self._random_patch_center(input_id, w, h)
input_patch, target_patch = self._crop_and_transform(
cx, cy, input_img, target_arr, randomize=True)
# check centre of chosen patch_index for valid pixels
if np.any(
utils.crop_center(input_patch, pw // 2, ph // 2,
pw // 4, ph // 4)):
break
input_tile_tensor = self.transform(input_patch)
target_tile_tensor = to_tensor(target_patch)
else:
target_arr = None
if self.has_targets:
if self.generate_target:
target_arr = self.data_by_id[input_id]['coords']
else:
target_arr = self._load_target(input_id)
cx, cy = self._indexed_patch_center(input_id, patch_index)
input_patch, target_patch = self._crop_and_transform(
cx, cy, input_img, target_arr, randomize=False)
input_tile_tensor = self.transform(input_patch)
if target_patch is None:
target_tile_tensor = torch.zeros(1)
else:
target_tile_tensor = to_tensor(target_patch)
#print(input_tile_tensor.size(), target_tile_tensor)
#cv2.imwrite('test-scaled-input-%d.png' % index, input_patch)
#cv2.imwrite('test-scaled-target-%d.png' % index, 4096*target_tile[:, :, :3])
index_tensor = torch.LongTensor([input_id, index, patch_index])
return input_tile_tensor, target_tile_tensor, index_tensor
def remove_ramp(arr):
# pad zeros around arr, to the size of 3 times (ups = 3) of arr size
padded = ut.zero_pad(arr, arr.shape)
data = np.fft.fftshift(np.fft.fftn(np.fft.fftshift(padded)))
com = center_of_mass(np.power(np.absolute(data), 2)) - .5
sub_pixel_shifted = sub_pixel_shift(data, (com[1], com[0], com[2]))
ramp_removed_padded = np.fft.fftshift(np.fft.ifftn(np.fft.fftshift(sub_pixel_shifted)))
ramp_removed = ut.crop_center(ramp_removed_padded, arr.shape)
return ramp_removed
def inception_preprocessing(img, threshold=0.875):
import cv2
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = utils.crop_center(img, threshold=threshold)
img = cv2.resize(img, dsize=(224, 224))
img = np.expand_dims(img, axis=0)
img = img / 256
img = (img - 0.5) * 2
img = img.astype('float32')
return img
def _centre_crop_and_transform(self, input_img, scale=1.0, trans=False, vflip=False, hflip=False):
h, w = input_img.shape[:2]
cx = w // 2
cy = h // 2
crop_w, crop_h = utils.calc_crop_size(self.img_size[0], self.img_size[1], scale=scale)
input_img = utils.crop_center(input_img, cx, cy, crop_w, crop_h)
if trans:
input_img = cv2.transpose(input_img)
if hflip or vflip:
if hflip and vflip:
c = -1
else:
c = 0 if vflip else 1
input_img = cv2.flip(input_img, flipCode=c)
if scale != 1.0:
input_img = cv2.resize(input_img, self.img_size, interpolation=cv2.INTER_LINEAR)
return input_img
def _crop_and_transform(self,
cx,
cy,
input_img,
target_arr,
randomize=False):
target_tile = None
transform_target = False if target_arr is None else True
target_is_coords = True if transform_target and target_arr.shape[
1] == 3 else False
if randomize:
angle = 0.
hflip = random.random() < 0.5
vflip = random.random() < 0.5
do_rotate = random.random(
) < 0.25 if not hflip and not vflip else False
if do_rotate:
angle = random.random() * 360
scale = random.uniform(0.5, 1.125)
#print('hflip: %d, vflip: %d, angle: %f, scale: %f' % (hflip, vflip, angle, scale))
else:
angle = 0.
scale = 1.
hflip = False
vflip = False
crop_w, crop_h = utils.calc_crop_size(self.patch_size[0],
self.patch_size[1], angle, scale)
input_tile = utils.crop_center(input_img, cx, cy, crop_w, crop_h)
if transform_target:
if target_is_coords:
target_points = target_arr.copy()
target_points = utils.crop_points_center(
target_points, cx, cy, crop_w, crop_h)
#print(cx, cy, crop_w, crop_h, angle, scale, hflip, vflip)
#print(target_points)
target_points[:, :2] = target_points[:, :2] - [cx, cy]
else:
target_tile = utils.crop_center(target_arr, cx, cy, crop_w,
crop_h)
# Perform tile geometry transforms if needed
if angle or scale != 1. or hflip or vflip:
Mtrans = np.identity(3)
Mtrans[0, 2] = (self.patch_size[0] - crop_w) // 2
Mtrans[1, 2] = (self.patch_size[1] - crop_h) // 2
if hflip:
Mtrans[0, 0] *= -1
Mtrans[0, 2] = self.patch_size[0] - Mtrans[0, 2]
if vflip:
Mtrans[1, 1] *= -1
Mtrans[1, 2] = self.patch_size[1] - Mtrans[1, 2]
if angle or scale != 1.:
Mrot = cv2.getRotationMatrix2D((crop_w // 2, crop_h // 2),
angle, scale)
Mfinal = np.dot(Mtrans, np.vstack([Mrot, [0, 0, 1]]))
else:
Mfinal = Mtrans
input_tile = cv2.warpAffine(input_tile, Mfinal[:2, :],
tuple(self.patch_size))
if transform_target:
if target_is_coords:
if len(target_points):
target_cats = target_points[:, 2].copy()
target_points[:, 2] = np.ones(len(target_points))
target_points = np.dot(target_points, Mfinal)
#print(target_points)
target_points[:, 2] = target_cats
else:
tt64 = target_tile.astype(np.float64)
tt64 = cv2.warpAffine(tt64, Mfinal[:2, :],
tuple(self.patch_size))
if scale != 1.:
tt64 /= scale**2
target_tile = tt64.astype(np.float32)
if target_is_coords:
target_points = np.rint(target_points).astype(np.int)
target_points[:, :2] = target_points[:, :2] + [
self.patch_size[0] // 2, self.patch_size[1] // 2
]
target_points = utils.crop_points(target_points, 0, 0,
self.patch_size[0],
self.patch_size[1])
#print(target_points)
if self.target_type == 'countception':
dtype = np.uint8 if self.num_logits else np.float32
max_count = self.num_logits - 1 if self.num_logits else 0
target_tile = gen_target_countception(target_points,
self.patch_size,
max_count=max_count,
dtype=dtype)
else:
target_tile = gen_target_gauss(target_points,
self.patch_size,
factor=1024.)
return input_tile, target_tile
def train(self, config):
# NOTE : if train, the nx, ny are ingnored
#print("config.is_train:", config.is_train)
nx, ny, original_shape = input_setup(config)
#print("nx, ny, original_shape:", nx, ny, original_shape)
data_dir = checkpoint_dir(config)
print("reading data..")
input_, label_ = read_data(data_dir)
print("input_", input_.shape)
merged_summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter("./log/train_300") #, self.sess.graph)
#self.summary_writer = tf.summary.FileWriter("./log/", tf.get_default_graph())
# Stochastic gradient descent with the standard backpropagation
#self.train_op = tf.train.GradientDescentOptimizer(config.learning_rate).minimize(self.loss)
self.optimizer = tf.train.AdamOptimizer(learning_rate=config.learning_rate)
self.train_op = self.optimizer.minimize(self.loss)
#self.train_op = tf.train.AdamOptimizer(learning_rate=config.learning_rate).minimize(self.loss)
tf.initialize_all_variables().run()
counter = 0
time_ = time.time()
self.load(config.checkpoint_dir)
# Train
if config.is_train:
print("Now Start Training...")
#for ep in range(config.epoch):
for ep in range(300, 1000+1, 1):
#print("ep:", ep)
#sys.exit()
loss_summary_per_batch = []
# Run by batch images
batch_idxs = len(input_) // config.batch_size
for idx in range(0, batch_idxs):
batch_images = input_[idx * config.batch_size : (idx + 1) * config.batch_size]
batch_labels = label_[idx * config.batch_size : (idx + 1) * config.batch_size]
counter += 1
_, err, summary = self.sess.run([self.train_op, self.loss, merged_summary_op], feed_dict={self.images: batch_images, self.labels: batch_labels})
summary_pb = tf.summary.Summary()
summary_pb.ParseFromString(summary)
summaries = {}
for val in summary_pb.value:
summaries[val.tag] = val.simple_value
#print("summaries:", summaries)
loss_summary_per_batch.append(summaries['loss'])
summary_writer.add_summary(summary, (ep) * counter)
#self.summary_writer.add_summary(summary, (ep+1) * counter)
if counter % 1000 == 0:
print("Epoch: [%2d], step: [%2d], time: [%4.4f], loss: [%.8f]" % ((ep), counter, time.time()-time_, err))
#print(label_[1] - self.pred.eval({self.images: input_})[1],'loss:]',err)
#print("Epoch: [%2d], loss: [%.8f]", (ep+1), tf.reduce_mean(tf.square(label_ - self.pred.eval({self.images: input_}))))
#if counter % 500 == 0:
#if counter % 20 == 0:
# self.save(config.checkpoint_dir, counter)
if ep ==0 or ep % 10 == 0:
self.save(config.checkpoint_dir, ep)
###
'''
try:
config.is_train = False
nx_, ny_, original_shape_ = input_setup(config)
data_dir_ = checkpoint_dir(config)
input__, label__ = read_data(data_dir_)
print("Now Start Testing...")
result_ = self.pred.eval({self.images: input__})
image_ = merge(result_, [nx_, ny_], self.c_dim)
print("image after merge:", image_.shape)
print("[nx_, ny_]:", [nx_, ny_])
print("original_shape:", original_shape_)
print(type(image__), type(original_shape_[0]), type(original_shape_[1]))
cropped_img_ = crop_center(image, original_shape_[0], original_shape_[1])
print("cropped_img_:", cropped_img_.shape)
imsave(image_, config.result_dir + '/result-' + ep + '.png', config)
imsave(cropped_img_, config.result_dir + '/result_crop-' + ep + '.png', config)
except:
print("Unexpected error while evaluating image:", sys.exc_info()[0])
config.is_train = True
'''
###
print("loss per epoch[%d] loss: [%.8f]" % ((ep), np.mean(loss_summary_per_batch)))
summary_writer.add_summary(tf.Summary(value=[tf.Summary.Value(tag="loss per epoch", simple_value=np.mean(loss_summary_per_batch)),]), ((ep)))
summary_writer.add_summary(tf.Summary(value=[tf.Summary.Value(tag="learning rate", simple_value=self.optimizer._lr),]), ((ep)))
#print("learning rate:", self.optimizer._lr)
# Test
else:
print("Now Start Testing...")
#print("nx","ny",nx,ny)
result = self.pred.eval({self.images: input_})
print("result:", result.shape)
#print(label_[1] - result[1])
image = merge(result, [nx, ny], self.c_dim)
print("image after merge:", image.shape)
print("[nx, ny]:", [nx, ny])
print("original_shape:", original_shape)
print(type(image), type(original_shape[0]), type(original_shape[1]))
cropped_img = crop_center(image, original_shape[0], original_shape[1])
print("cropped_img:", cropped_img.shape)
#image_LR = merge(input_, [nx, ny], self.c_dim)
#checkimage(image_LR)
imsave(image, config.result_dir+'/result.png', config)
imsave(cropped_img, config.result_dir+'/result_crop.png', config)
def _random_crop_and_transform(self, input_img, scale_range=(1.0, 1.0), rot=0.0):
angle = 0.
hflip = random.random() < 0.5
vflip = random.random() < 0.5
trans = random.random() < 0.5
do_rotate = (rot > 0 and random.random() < 0.2) if not hflip and not vflip else False
h, w = input_img.shape[:2]
# Favour rotation/scale choices that involve cropping within image bounds
attempts = 0
while attempts < 3:
if do_rotate:
angle = random.uniform(-rot, rot)
scale = random.uniform(*scale_range)
crop_w, crop_h = utils.calc_crop_size(self.img_size[0], self.img_size[1], angle, scale)
if crop_w <= w and crop_h <= h:
break
attempts += 1
if crop_w > w or crop_h > h:
# We can still handle crops larger than the source, add a border
angle = 0.0
#scale = 1.0
border_w = crop_w - w
border_h = crop_h - h
input_img = cv2.copyMakeBorder(
input_img,
border_h//2, border_h - border_h//2,
border_w//2, border_w - border_w//2,
cv2.BORDER_REFLECT_101)
input_img = np.ascontiguousarray(input_img) # trying to hunt a pytorch/cuda crash, is was this necessary?
assert input_img.shape[:2] == (crop_h, crop_w)
else:
hd = max(0, h - crop_h)
wd = max(0, w - crop_w)
ho = random.randint(0, hd) - math.ceil(hd / 2)
wo = random.randint(0, wd) - math.ceil(wd / 2)
cx = w // 2 + wo
cy = h // 2 + ho
input_img = utils.crop_center(input_img, cx, cy, crop_w, crop_h)
#print('hflip: %d, vflip: %d, angle: %f, scale: %f' % (hflip, vflip, angle, scale))
if angle:
if trans:
input_img = cv2.transpose(input_img)
m_translate = np.identity(3)
if hflip:
m_translate[0, 0] *= -1
m_translate[0, 2] = (self.img_size[0] + crop_w) / 2 - 1
else:
m_translate[0, 2] = (self.img_size[0] - crop_w) / 2
if vflip:
m_translate[1, 1] *= -1
m_translate[1, 2] = (self.img_size[1] + crop_h) / 2 - 1
else:
m_translate[1, 2] = (self.img_size[1] - crop_h) / 2
if angle or scale != 1.:
m_rotate = cv2.getRotationMatrix2D((crop_w / 2, crop_h / 2), angle, scale)
m_final = np.dot(m_translate, np.vstack([m_rotate, [0, 0, 1]]))
else:
m_final = m_translate
input_img = cv2.warpAffine(input_img, m_final[:2, :], self.img_size, borderMode=cv2.BORDER_REFLECT_101)
else:
if trans:
input_img = cv2.transpose(input_img)
if hflip or vflip:
if hflip and vflip:
c = -1
else:
c = 0 if vflip else 1
input_img = cv2.flip(input_img, flipCode=c)
input_img = cv2.resize(input_img, self.img_size, interpolation=cv2.INTER_LINEAR)
return input_img
