def postConversionForOpenDingux(self):
self.logger.log(" opendingux post-conversion")
openDinguxPicDir = '.previews' if self.conversionType == util.esoteric else '.media'
# Copy image to opendingux img folder for game.pc
distPicPath = os.path.join(self.getLocalParentOutputDir(),
openDinguxPicDir)
if not os.path.exists(distPicPath):
os.mkdir(distPicPath)
shutil.copy2(self.metadata.frontPic,
os.path.join(distPicPath, self.game + '.pc.png'))
# Resize image
util.resize(os.path.join(distPicPath, self.game + '.pc.png'))
# Copy image to opendingux img folder for game.pc/dosbox.bat
dosboxBatPicPath = os.path.join(self.getLocalGameOutputDir(),
openDinguxPicDir)
if not os.path.exists(dosboxBatPicPath):
os.mkdir(dosboxBatPicPath)
shutil.copy2(os.path.join(distPicPath, self.game + '.pc.png'),
os.path.join(dosboxBatPicPath, 'dosbox.png'))
# Generate RG350 mapper
mapper = open(os.path.join(self.getLocalGameOutputDir(), "mapper.map"),
'w')
mapper.write('key_space "key 308"\n')
mapper.write('key_lshift "key 32"\n')
mapper.write('key_lctrl "key 304"\n')
mapper.write('key_lalt "key 306"\n')
mapper.write('key_esc "key 27"\n')
mapper.write('key_enter "key 13"\n')
mapper.write('key_up "key 273"\n')
mapper.write('key_down "key 274"\n')
mapper.write('key_right "key 275"\n')
mapper.write('key_left "key 276"\n')
mapper.write('key_n "key 9"\n')
mapper.write('key_y "key 8"\n')
mapper.close()
def resize( size, rows=0, cols=0 ): if ( g_subplots or rows == 0 or cols == 0 ): util.resize( size ) else: size2 = [ size[0]/cols, size[1]/rows ] for index in range( 1, rows*cols+1 ): subplot( rows, cols, index ) util.resize( g_figsize2 )
def cover(self, fea, cover_mask, mid=False):
if mid:
mid_list = []
b, c, h, w = fea.size()
cover_resize = resize(cover_mask, [h, w]).expand(b, c, h, w)
fea_max = F.max_pool2d(fea, (h, w), (h, w),
padding=0).expand(b, c, h, w)
fea_min = -F.max_pool2d(-fea, (h, w),
(h, w), padding=0).expand(b, c, h, w)
fea = (fea - fea_min) / (fea_max - fea_min + 1e-8)
if mid:
mid_list.append(fea * 1)
noise = torch.rand(fea.size()).cuda()
if mid:
mid_list.append(noise * 1)
mid_list.append(noise * (1 - cover_resize))
fea = fea * cover_resize + noise * (1 - cover_resize)
if mid:
mid_list.append(fea * 1)
fea = fea * (fea_max - fea_min + 1e-8) + fea_min
if mid:
mid_list.append(fea * 1)
return fea, mid_list
else:
return fea
def mark_all(model, data_dir, out_dir='marked_predictions', backbone='resnet34',
input_size=None, preprocessing_fcn=None):
if preprocessing_fcn is None:
preprocessing_fcn = get_preprocessing(backbone)
if not os.path.isdir(out_dir):
os.mkdir(out_dir)
images = []
for directory, _, files in os.walk(data_dir):
for fn in files:
if is_image(fn):
images.append(os.path.join(directory, fn))
s = 'filename,x1,y1,x2,y2,x3,y3,x4,y4,x5,y5,x6,y6,x7,y7,x8,y8,x9,y9,x10,y10\n'
for fn_full in images:
name = os.path.splitext(os.path.basename(fn_full))[0]
mask = predict(model=model,
image=fn_full,
out_path=None,
preprocessing_fcn=preprocessing_fcn,
input_size=input_size)
pts = pmask2points(np.squeeze(mask))
s += (os.path.basename(fn_full) + ',' +
','.join('%s,%s' % (x, y) for x, y in pts) + '\n')
image = resize(cv.imread(fn_full), dsize=input_size)
cv.imwrite(os.path.join(out_dir, name + '.jpg'),
mark_image(image, pts))
cv.imwrite(os.path.join(out_dir, name + '.png'),
(np.squeeze(mask).sum(axis=2)*255).astype('uint8'))
with open(os.path.join(out_dir, 'points.csv'), 'w+') as csv_out:
csv_out.write(s)
def next_batch(self):
start = self.index
end = self.index + self.batch_size
if end >= self.volumn:
end = self.volumn
self.index = 0
else:
self.index = end
data_batch = self.data[start:end]
imgs = []
dmaps_micro = []
dmaps_tiny = []
dmaps_mid = []
dmaps_large = []
names = []
for piece in data_batch:
tmp = misc.imread(self.img_dir + piece['image_id'] + '.jpg')
names.append(piece['image_id'])
# resize to 368x368
tmp, rate, left, top = \
util.resize(tmp, self.large)
imgs.append(tmp)
annos = np.array(list(piece['keypoint_annotations'].values()),
dtype=np.float16)
annos[:, ::3] = annos[:, ::3] * rate - left
annos[:, 1::3] = annos[:, 1::3] * rate - top
annos = annos.astype(np.int16)
self.append_dmap(annos, dmaps_large, self.large)
annos = annos.astype(np.float32)
annos[:, ::3] = annos[:, ::3] * 0.5
annos[:, 1::3] = annos[:, 1::3] * 0.5
annos = np.round(annos).astype(np.int16)
self.append_dmap(annos, dmaps_mid, self.mid)
annos = annos.astype(np.float32)
annos[:, ::3] = annos[:, ::3] * 0.5
annos[:, 1::3] = annos[:, 1::3] * 0.5
annos = np.round(annos).astype(np.int16)
self.append_dmap(annos, dmaps_tiny, self.tiny)
annos = annos.astype(np.float32)
annos[:, ::3] = annos[:, ::3] * 0.5
annos[:, 1::3] = annos[:, 1::3] * 0.5
annos = np.round(annos).astype(np.int16)
self.append_dmap(annos, dmaps_micro, self.micro)
imgs = np.array(imgs, dtype=np.float32)
imgs -= self.mean
imgs /= self.var
dmaps_micro = np.array(dmaps_micro)
dmaps_tiny = np.array(dmaps_tiny)
dmaps_mid = np.array(dmaps_mid)
dmaps_large = np.array(dmaps_large)
return imgs, dmaps_micro, dmaps_tiny, dmaps_mid, dmaps_large, names
def get_card_shape(card, training_set, thresh=150):
# binary = get_binary(card, thresh=thresh)
binary = get_canny(card)
contours = find_contours(binary)
# if canny doesn't give enough contours, fallback to binary
if len(contours) < 2:
binary = get_binary(card)
contours = find_contours(binary)
# if still not enough contours, consider invalid
if len(contours) < 2:
return None
poly = get_approx_poly(contours[1], do_rectify=False)
# for each card in trainings set, find one with most similarity
diffs = []
this_shape = get_shape_contour(card)
for i, that_shape in training_set.items():
# resize image
this_shape_res = util.resize(this_shape, that_shape.shape)
# find diff and its sum
d = cv2.absdiff(this_shape_res, that_shape)
sum_diff = np.sum(d)
diffs.append(sum_diff)
# return index of shape that has minimum difference
return diffs.index(min(diffs)) + 1
def get_shape_from_contour(contour, contour_box):
# uppack bounding box of contour
x, y, w, h = contour_box
# list of contours (only contains the first contour, really) shifted so
# that they can be drawn in a box the size of the bounding box
contours_shifted = [np.array([[[j[0] - x, j[1] - y] for j in i] for i in contour])]
# create image with filled contour
shape_img = util.draw_contour(contours_shifted, 0, h, w)
# for each card in trainings set, find one with most similarity
diffs = []
training_set = get_training_set()
for i, shape_ref in training_set.items():
# resize image
shape_img_res = util.resize(shape_img, shape_ref.shape)
# find diff and its sum
d = cv2.absdiff(shape_img_res, shape_ref)
sum_diff = np.sum(d)
diffs.append(sum_diff)
return diffs.index(min(diffs)) + 1
def find_components(edges, max_components=10):
# Dilate the image until there are just a few connected components.
count = max_components + 1
n = 1
components = None
while count > max_components:
n += 1
dilated_image = cv2.dilate(edges / 255, np.ones((3, 3)), iterations=n)
components = cv2.connectedComponentsWithStats(dilated_image)
count = components[0]
if imshow:
cv2.imshow("Edged", util.resize(edges, height=650))
cv2.imshow("Edged dilated", util.resize(255 * dilated_image, height=650))
cv2.waitKey(0)
cv2.destroyAllWindows()
return components
def _extract_heat_map_and_paf(self, image):
"""
:param image: target image
:return: 18 layers heat map for body parts and 1 layer for background, paf,
detail to see OpenPose, https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/doc/output.md
"""
height, width = self.im_height, self.im_width
multiplier = [
x * model_params['boxsize'] / height
for x in params['scale_search']
]
multiplier_len = len(multiplier)
heat_map_average = np.zeros((height, width, 19))
paf_average = np.zeros((height, width, 38))
for scale in multiplier:
image2test = resize(image, fx=scale, fy=scale)
padded_image, pad = pad_right_down_corner(image2test,
model_params['stride'],
model_params['padValue'])
input_img = np.transpose(
np.float32(padded_image[:, :, :, np.newaxis]), (3, 0, 1, 2))
results = self.model.predict(input_img)
heat_map = np.squeeze(results[1])
heat_map = resize(heat_map,
fx=model_params['stride'],
fy=model_params['stride'])
heat_map = heat_map[:padded_image.shape[0] -
pad[2], :padded_image.shape[1] - pad[3], :]
heat_map = resize(heat_map, output_size=(width, height))
heat_map_average = heat_map_average + heat_map / multiplier_len
paf = np.squeeze(results[0]) # output 0 is PAFs
paf = resize(paf,
fx=model_params['stride'],
fy=model_params['stride'])
paf = paf[:padded_image.shape[0] - pad[2], :padded_image.shape[1] -
pad[3], :]
paf = resize(paf, output_size=(width, height))
paf_average = paf_average + paf / multiplier_len
return heat_map_average, paf_average
def showImage(self, path):
print(path)
image = cv2.imread(path)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
self.ori_h, self.ori_w = image.shape[:2]
self.image = resize(image, self.SIZE)
h, w = self.image.shape[:2]
qimg = QImage(self.image.flatten(), w, h, QImage.Format_RGB888)
self.le.setPixmap(QPixmap.fromImage(qimg))
def next_batch(self):
start = self.index
end = self.index + self.batch_size
if end >= self.volumn:
end = self.volumn
self.index = 0
else:
self.index = end
data_batch = self.data[start:end]
img = []
keypoint_hmap = []
direction_hmap = []
for piece in data_batch:
tmp = misc.imread(self.img_dir + piece['image_id'] + '.jpg')
tmp, rate, left, top = \
util.resize(tmp, self.length)
annos = np.array(list(piece['keypoint_annotations'].values()),
dtype=np.float16)
annos[:, ::3] = annos[:, ::3] * rate - left
annos[:, 1::3] = annos[:, 1::3] * rate - top
annos = annos.astype(np.int16)
img.append(tmp)
tmp = misc.imresize(tmp, (self.short, self.short))
# annos = np.round(annos * self.short / self.length).astype(np.int8)
rate = self.short / self.length
annos[:, ::3] = annos[:, ::3] * rate
annos[:, 1::3] = annos[:, 1::3] * rate
annos = np.round(annos).astype(np.int8)
# print(annos)
kmap = util.get_key_hmap(\
tmp.shape, annos, self.patch, self.patch_l//2)
keypoint_hmap.append(kmap)
tmp_dmap, tmp_dmap_re = util.get_dir_hmap(\
tmp.shape, annos, self.ones, self.limbs, self.patch_l//2)
dmap = util.weight_dir_hmap(kmap, tmp_dmap, tmp_dmap_re,
self.limbs)
# use forward only
dmap = dmap[:, :, len(self.limbs) * 2:]
direction_hmap.append(dmap)
img = np.array(img, dtype=np.float32)
img -= self.mean
img /= self.var
keypoint_hmap = np.array(keypoint_hmap, dtype=np.float32)
direction_hmap = np.array(direction_hmap, dtype=np.float32)
return img, keypoint_hmap, direction_hmap
def preprocess(image, input_size=None, backbone='resnet34', preprocessing_fcn=None):
if isinstance(image, str):
image = imread(image)
if preprocessing_fcn is None:
preprocessing_fcn = get_preprocessing(backbone)
if input_size is not None:
image = resize(image, input_size)
image = image / 255
image = preprocessing_fcn(image)
return np.expand_dims(image, 0)
def next_batch(self):
start = self.index
end = self.index + self.batch_size
if end >= self.volumn:
end = self.volumn
self.index = 0
else:
self.index = end
data_batch = self.data[start:end]
img = []
keypoint_hmap = []
affinity_hmap = []
names = []
for piece in data_batch:
tmp = misc.imread(self.img_dir + piece['image_id'] + '.jpg')
names.append(piece['image_id'])
tmp, rate, left, top = \
util.resize(tmp, self.length)
annos = np.array(list(piece['keypoint_annotations'].values()),
dtype=np.float16)
annos[:, ::3] = annos[:, ::3] * rate - left
annos[:, 1::3] = annos[:, 1::3] * rate - top
annos = annos.astype(np.int16)
img.append(tmp)
tmp = misc.imresize(tmp, (self.short, self.short))
rate = self.short / self.length
annos[:, ::3] = annos[:, ::3] * rate
annos[:, 1::3] = annos[:, 1::3] * rate
annos = np.round(annos).astype(np.int8)
keypoint_hmap.append(
util.get_key_hmap(tmp.shape,
annos,
self.patch,
self.patch_l // 2,
strict=True))
affinity_hmap.append(
util.get_aff_hmap(tmp.shape, annos, self.limbs, strict=True))
img = np.array(img, dtype=np.float32)
img -= self.mean
img /= self.var
keypoint_hmap = np.array(keypoint_hmap, dtype=np.float32)
affinity_hmap = np.array(affinity_hmap, dtype=np.float32)
return img, keypoint_hmap, affinity_hmap, names
def compute_gradcam_gif(cam, x, x_un_normalized):
gradcam_output_buffer = BytesIO()
new_cam = util.resize(cam, x[0])
input_np = np.transpose(x_un_normalized[0], (1, 2, 3, 0))
input_normed = np.float32(input_np) / 255
cam_frames = list(util.add_heat_map(input_normed, new_cam))
cam_frames = [Image.fromarray(frame) for frame in cam_frames]
cam_frames[0].save(
gradcam_output_buffer,
save_all=True,
append_images=cam_frames[1:] if len(cam_frames) > 1 else [],
format="GIF",
)
return gradcam_output_buffer
def crop(path, out_path, show=False):
global imshow
imshow = show
orig_im = cv2.imread(path)
downscaled_height = 600.0
scale = orig_im.shape[0] / downscaled_height
im = util.resize(orig_im, height=int(downscaled_height))
edges = cv2.Canny(im, 30, 200)
edges = 255 * (edges > 0).astype(np.uint8)
edges_blurred = cv2.medianBlur(edges, 5)
components = find_components(edges_blurred)
if components[0] == 0:
print '%s -> (no text!)' % path
return
crop = find_optimal_components_subset(components, edges)
crop = [int(x * scale) for x in crop] # upscale to the original image size.
orig_im = orig_im[crop[1]:crop[3], crop[0]: crop[2]]
cv2.imwrite(out_path, orig_im)
def transform_card(card, image):
# find out if card is rotated
x, y, w, h = cv2.boundingRect(card)
card_shape = [[0, 0], [sc.SIZE_CARD_W, 0], [sc.SIZE_CARD_W, sc.SIZE_CARD_H], [0, sc.SIZE_CARD_H]]
# get poly of contour
approximated_poly = get_approx_poly(card, do_rectify=True)
if approximated_poly is None:
# could not find card poly
return None
dest = np.array(card_shape, np.float32)
# do transformatiom
transformation = cv2.getPerspectiveTransform(approximated_poly, dest)
warp = cv2.warpPerspective(image, transformation, sc.SIZE_CARD)
# rotate card back up
if w > h:
return util.resize(np.rot90(warp), (sc.SIZE_CARD_H, sc.SIZE_CARD_W))
return warp
def get_obs(self):
# Show window with environment
self.env.render()
if self.img_mode:
# Get matrix of pixels of environment
env_pxls = self.env.render(mode='rgb_array')
# Convert to grayscale
env_pxls = util.rgb2gray(env_pxls)
# Cut target area
env_pxls = env_pxls[ENV_HEIGHT_SLICE, ENV_WIDTH_SLICE]
# Reduce image size
env_pxls = util.resize(env_pxls, (self.img_size))
# Normalize values
env_pxls = env_pxls.astype('float32') / MAX_IMAGE_BRIGHT
# Expand tensor (height, width, 1)
tensor = np.expand_dims(env_pxls, axis=2)
# Add previous states (height, width, num_prev_states + 1)
for i in range(self.num_prev_states):
tensor = np.append(tensor,
self.prev_states[:, :, i:i + 1],
axis=2)
# Expand tensor (1, height, width, num_prev_states + 1)
tensor = np.expand_dims(tensor, axis=0)
# Save last observation
self.last_obs = tensor
else:
# Normalize parameters
self.last_obs[0] = (self.last_obs[0] + MAX_SHIFT) / (2 * MAX_SHIFT)
self.last_obs[1] = (self.last_obs[1] + 2) / 4
self.last_obs[2] = (self.last_obs[2] + MAX_ANGLE) / (2 * MAX_ANGLE)
self.last_obs[3] = (self.last_obs[3] + 2) / 4
return self.last_obs
else:
predict(model=m,
image=args.input,
out_path=args.output,
backbone=args.backbone,
preprocessing_fcn=get_preprocessing(args.backbone),
input_size=args.input_size)
elif args.command == 'mark':
metrics = get_keypoint_metrics((args.batch_size,) + args.input_size[::-1] + (10,))
custom_objects = dict((m.__name__, m) for m in metrics)
m = load_model(args.checkpoint_path, custom_objects=custom_objects)
if os.path.isdir(args.input):
mark_all(model=m,
data_dir=args.input,
out_dir=args.output,
input_size=args.input_size,
backbone=args.backbone,
preprocessing_fcn=get_preprocessing(args.backbone))
else:
mask = predict(model=m,
image=args.input,
out_path=None,
preprocessing_fcn=get_preprocessing(args.backbone),
input_size=args.input_size)
pts = pmask2points(np.squeeze(mask))
image = resize(cv.imread(args.input), dsize=args.input_size)
marked = mark_image(image, pts)
cv.imwrite(args.output, marked)
else:
raise ValueError('`command` = "%s" not understood.' % args.command)
def scan(imgname="chom4.jpg", show=True):
path = imgname
image = cv2.imread(path)
ratio = image.shape[0] / 700.0
orig = image.copy()
image = util.resize(image, height=700)
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
gray = cv2.medianBlur(gray, 5)
edged = cv2.Canny(gray, 40, 150)
edged_copy = edged.copy()
edged_copy = cv2.GaussianBlur(edged_copy, (3, 3), 0)
cv2.imwrite('edged.jpg', edged)
if show:
cv2.imshow("Edged", edged)
cv2.imshow("Edged blurred", edged_copy)
cv2.waitKey(0)
cv2.destroyAllWindows()
(_, cnts, _) = cv2.findContours(edged_copy, cv2.RETR_LIST,
cv2.CHAIN_APPROX_SIMPLE)
cnts = sorted(cnts, key=cv2.contourArea, reverse=True)[:4]
screenCnt = []
for c in cnts:
# approximate the contour
peri = cv2.arcLength(c, True)
approx = cv2.approxPolyDP(c, 0.015 * peri, True)
# approx = np.array(cv2.boundingRect(c))
# if our approximated contour has four points, then we
# can assume that we have found our screen
debugging = False
if debugging:
cv2.drawContours(image, [approx], -1, (0, 255, 0), 2)
cv2.imshow("Outline", image)
cv2.waitKey(0)
if len(approx) == 4:
screenCnt = approx
break
if screenCnt.__len__() != 0:
if show:
cv2.drawContours(image, [screenCnt], -1, (0, 255, 0), 2)
cv2.imwrite('outlined.jpg', image)
cv2.imshow("Outline", image)
cv2.waitKey(0)
cv2.destroyAllWindows()
warped = four_point_transform(orig, screenCnt.reshape(4, 2) * ratio)
else:
warped = orig
warped = cv2.cvtColor(warped, cv2.COLOR_BGR2GRAY)
warped = threshold_adaptive(warped, 251, offset=10)
warped = warped.astype("uint8") * 255
if show:
cv2.imshow("Original", util.resize(orig, height=650))
cv2.imshow("Scanned", util.resize(warped, height=650))
cv2.waitKey(0)
cv2.imwrite('deskewed.jpg', warped)
def grid_net_forward(self, input0, name):
conv_out = []
for i in range(self.height):
conv_out.append([])
for j in range(0, self.width):
conv_out[i].append(0)
conv_out[0][0] = self.lrelu(
getattr(self, name + '_' + 'input')(rp(input0)))
for i in range(self.height):
for j in range(0, self.width / 2):
# 来自左
if j - 1 >= 0:
input_0 = conv_out[i][j - 1]
input_1 = getattr(
self, name + '_norm0_' + str(i) + '_' + str(j - 1) +
'_' + str(i) + '_' + str(j))(input_0)
input_1 = rp(self.lrelu(input_1))
conv_out[i][j] = conv_out[i][j] +\
getattr(self, name + '_conv0_' + str(i) + '_' +
str(j - 1) + '_' + str(i) + '_' + str(j))(input_1) + input_0
# 来自上
if i - 1 >= 0:
input_0 = conv_out[i - 1][j]
input_1 = getattr(
self, name + '_norm0_' + str(i - 1) + '_' + str(j) +
'_' + str(i) + '_' + str(j))(input_0)
input_1 = rp(self.lrelu(input_1))
input_1 = getattr(
self, name + '_conv0_' + str(i - 1) + '_' + str(j) +
'_' + str(i) + '_' + str(j))(input_1)
conv_out[i][j] = conv_out[i][j] + input_1
for i in range(self.height - 1, -1, -1):
for j in range(self.width / 2, self.width):
# 来自左
input_0 = conv_out[i][j - 1]
input_1 = getattr(
self, name + '_norm0_' + str(i) + '_' + str(j - 1) + '_' +
str(i) + '_' + str(j))(input_0)
input_1 = rp(self.lrelu(input_1))
conv_out[i][j] = conv_out[i][j] + \
getattr(self, name + '_conv0_' + str(i) + '_' +
str(j - 1) + '_' + str(i) + '_' + str(j))(input_1) + input_0
# 来自下
if i + 1 < self.height:
h = conv_out[i][j].size(2)
w = conv_out[i][j].size(3)
re_input = resize(conv_out[i + 1][j], [h, w])
re_input = getattr(
self, name + '_norm0_' + str(i + 1) + '_' + str(j) +
'_' + str(i) + '_' + str(j))(re_input)
re_input = rp(self.lrelu(re_input))
conv = getattr(
self, name + '_conv0_' + str(i + 1) + '_' + str(j) +
'_' + str(i) + '_' + str(j))(re_input)
conv_out[i][j] += conv
output0 = getattr(self, name + '_' + 'output')(rp(
conv_out[0][self.width - 1]))
return output0
#g=model.Generator(s_size=s_size)
#generated=g.__call__(z,training=True) #shape=(64, 512, 512, 3)
### get train_op and loss
wgan = model.WGAN(batch_size,
d_depths=d_depths,
g_depths=g_depths,
s_size=s_size)
losses = wgan.loss(x) #果然是要generated的图片与真是的对应问题
train_op = wgan.train(losses=losses, d_lr=d_lr, g_lr=g_lr)
d_clip_variable = [
p.assign(tf.clip_by_value(p, -1.0, 1.0)) for p in wgan.d.variables
]
crop = util.resize(res_img_h=img_h, res_img_w=img_w)
image_files, image_labels = util.get_files(train_dir)
#training
sess = tf.Session()
sess.run(tf.global_variables_initializer())
#image_batch=np.array(
# map(crop.crop_resize,image_files[:32])).astype(np.float32)
#
#_, g_loss_value, d_loss_value = sess.run([train_op, losses[wgan.g], losses[wgan.d]],
# feed_dict={x:image_batch})
#
#print "g_loss: ------ %f\td_loss: -------%f"%(g_loss_value,d_loss_value)
iteration = 0
def data_generator(data_dir,
batch_size,
input_size=None,
keras_augmentations=None,
preprocessing_function_x=None,
preprocessing_function_y=None,
preload=False,
cached_preloading=False,
verbosity=True,
mode=None,
random_crops=0.0):
if keras_augmentations is None:
keras_augmentations = dict()
if random_crops:
print(
'\n\nWARNING: Crop size hard coded as 25%-100% of image size.\n\n')
# mask and image generators must use same seed
keras_seed = np.random.randint(314)
# note: preprocessing_fcn will run after image is resized and augmented
image_datagen = ImageDataGenerator(
preprocessing_function=preprocessing_function_x, **keras_augmentations)
mask_datagen = ImageDataGenerator(
preprocessing_function=preprocessing_function_y, **keras_augmentations)
if preload:
# from tempfile import tempdir
# x = preloader(dataset_dir=data_dir,
# img_shape=tuple(list(input_size)[::-1] + [3]),
# subset='images',
# cached_preloading=cached_preloading,
# storage_dir=os.path.join(tempdir, 'unet-preloader'),
# verbose=verbosity)
# y = preloader(dataset_dir=data_dir,
# img_shape=tuple(list(input_size)[::-1] + [3]),
# subset='segmentations',
# cached_preloading=cached_preloading,
# storage_dir=os.path.join(tempdir, 'unet-preloader'),
# verbose=verbosity)
#
# pair_generator = image_datagen.flow(
# x=(x, y),
# y=None,
# batch_size=32,
# shuffle=True,
# seed=keras_seed,
# subset=mode)
raise NotImplementedError
else:
image_generator = image_datagen.flow_from_directory(
directory=data_dir,
classes=['images'],
class_mode=None,
color_mode='rgb',
target_size=input_size,
batch_size=batch_size,
seed=keras_seed,
subset=mode)
mask_generator = mask_datagen.flow_from_directory(
directory=data_dir,
classes=['segmentations'],
class_mode=None,
color_mode='grayscale',
target_size=input_size,
batch_size=batch_size,
seed=keras_seed,
subset=mode)
pair_generator = zip(image_generator, mask_generator)
for (unaugmented_image_batch, unaugmented_mask_batch) in pair_generator:
if random_crops:
image_batch = np.empty((batch_size, ) + input_size[::-1] +
unaugmented_image_batch.shape[3:])
# mask_batch = np.empty((batch_size,) + input_size[::-1] + mb.shape[3:])
mask_batch = np.empty((batch_size, ) + input_size[::-1] + (1, ))
for k in range(batch_size):
cropped = crop(
**{
'image': unaugmented_image_batch[k],
'mask': unaugmented_mask_batch[k]
})
if (cropped['image'].shape[0] != input_size[1]
or cropped['image'].shape[1] != input_size[0]):
image_batch[k] = resize(cropped['image'], dsize=input_size)
mask_batch[k] = resize(np.squeeze(cropped['mask']),
dsize=input_size)[:, :, None]
else:
image_batch[k], mask_batch[k] = cropped['image'], cropped[
'mask']
else:
image_batch, mask_batch = unaugmented_image_batch, unaugmented_mask_batch
image_batch = image_batch.astype('float32') / 255
mask_batch = mask_batch.astype('float32') / 255
# print('\nimage', image_batch.shape, image_batch.dtype, image_batch.min(), image_batch.max())
# print('mask', mask_batch.shape, mask_batch.dtype, mask_batch.min(), mask_batch.max())
yield image_batch, mask_batch
data = util.train_sample(0xFFFFFFFF) + util.test_sample(0xFFFFFFF)
resolution_list = [4, 8, 16]
lim_list = [200, 2000, 20000]
for c_res in resolution_list:
pr = []
e_in = []
e_out = []
for sz in lim_list:
t_set = util.train_sample(sz)
l = len(t_set)
f = open(str(c_res) + str(sz) + "vectors.dat","w")
for i in range(0, l):
t_set[i] = (t_set[i][0], util.resize(t_set[i][1], c_res))
img = t_set[i]
if img[0] == 1:
f.write("-1 ")
else:
f.write("1 ")
for k in range(0, len(img[1])):
if (img[1][k] > 0):
f.write(str(k + 1) + ":" + str(img[1][k]) + " ")
f.write("\n")
f.close()
f = open(str(c_res)+str(sz)+"test.dat","w")
t_set = util.test_sample(sz)
def load_test_image(image_file):
input_image, real_image = load_dataset_image(image_file)
input_image, real_image = resize(input_image, real_image, IMG_HEIGHT,
IMG_WIDTH)
input_image, real_image = normalize(input_image, real_image)
return input_image, real_image
def get_cams(args):
print('Loading model...')
model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids)
model = model.to(args.device)
args.start_epoch = ckpt_info['epoch'] + 1
print('Last layer in model.features is named "{}"...'.format([k for k in model.module.encoders._modules.keys()][-1]))
print('Extracting feature maps from layer named "{}"...'.format(args.target_layer))
grad_cam = GradCAM(model, args.device, is_binary=True, is_3d=True)
print(grad_cam)
gbp = GuidedBackPropagation(model, args.device, is_binary=True, is_3d=True)
print(gbp)
num_generated = 0
data_loader = CTDataLoader(args, phase=args.phase, is_training=False)
print(data_loader)
study_idx_dict = {}
study_count = 1
for inputs, target_dict in data_loader:
#print(inputs, target_dict)
#print('target_dict dir={}'.format(dir(target_dict)))
#print('\ntarget_dict[study_num]={}'.format(target_dict['study_num']))
probs, idx = grad_cam.forward(inputs)
grad_cam.backward(idx=idx[0]) # Just take top prediction
cam = grad_cam.get_cam(args.target_layer)
labels = target_dict['is_abnormal']
if labels.item() == 0:
# Keep going until we get an aneurysm study
print('Skipping a normal example...')
continue
print('Generating CAM...')
study_num = 1
with torch.set_grad_enabled(True):
probs, idx = grad_cam.forward(inputs)
print(probs, idx)
grad_cam.backward(idx=idx[0]) # Just take top prediction
cam = grad_cam.get_cam(args.target_layer)
guided_backprop = None
if args.use_gbp:
inputs2 = torch.autograd.Variable(inputs, requires_grad=True)
probs2, idx2 = gbp.forward(inputs2)
gbp.backward(idx=idx2[0])
guided_backprop = np.squeeze(gbp.generate())
print('Overlaying CAM...')
print(cam.shape)
new_cam = util.resize(cam, inputs[0])
print(new_cam.shape)
input_np = util.un_normalize(inputs[0], args.img_format, data_loader.dataset.pixel_dict)
input_np = np.transpose(input_np, (1, 2, 3, 0))
input_frames = list(input_np)
input_normed = np.float32(input_np) / 255
cam_frames = list(util.add_heat_map(input_normed, new_cam))
gbp_frames = None
if args.use_gbp:
gbp_np = util.normalize_to_image(guided_backprop * new_cam)
gbp_frames = []
for dim in range(gbp_np.shape[0]):
slice_ = gbp_np[dim, :, :]
gbp_frames.append(slice_[..., None])
# Write to a GIF file
output_path_input = os.path.join(os.path.join(args.cam_dir, '{}_{}_input_fn.gif'.format(target_dict['study_num'], study_count)))
output_path_cam = os.path.join(args.cam_dir, '{}_{}_cam_fn.gif'.format(target_dict['study_num'], study_count))
output_path_combined = os.path.join(args.cam_dir, '{}_{}_combined_fn.gif'.format(target_dict['study_num'], study_count))
print('Writing set {}/{} of CAMs to {}...'.format(num_generated + 1, args.num_cams, args.cam_dir))
input_clip = mpy.ImageSequenceClip(input_frames, fps=4)
input_clip.write_gif(output_path_input, verbose=False)
cam_clip = mpy.ImageSequenceClip(cam_frames, fps=4)
cam_clip.write_gif(output_path_cam, verbose=False)
combined_clip = mpy.clips_array([[input_clip, cam_clip]])
combined_clip.write_gif(output_path_combined, verbose=False)
if args.use_gbp:
output_path_gcam = os.path.join(args.cam_dir, 'gbp_{}.gif'.format(num_generated + 1))
gbp_clip = mpy.ImageSequenceClip(gbp_frames, fps=4)
gbp_clip.write_gif(output_path_gcam, verbose=False)
study_count += 1
num_generated += 1
if num_generated == args.num_cams:
return
__author__ = 'minya'
import Image
import sys
import util
import os
i_dir = sys.argv[1]
o_dir = sys.argv[2]
for file in os.listdir(i_dir):
i_fname = os.path.join(i_dir, file)
img = Image.open(i_fname, "r")
box = int(640), int(480)
o_fname = os.path.join(o_dir, file)
print "converting %s out to %s ..." % (i_fname, o_fname)
util.resize(img, box, False, o_fname)
print "done."
def preprocess(x, opt):
x = x.astype('float32')
if opt.resize:
x = resize(x, size=(opt.width, opt.height))
x = map_range(x)
return x
def scan(im_path, show=True):
im = cv2.imread(im_path)
orig = im.copy()
downscaled_height = 700.0
im, scale = util.downscale(im, downscaled_height)
gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
kern_size = 5
gray_blurred = cv2.medianBlur(gray, kern_size)
threshold_lower = 40
threshold_upper = 150
edged = cv2.Canny(gray_blurred, threshold_lower, threshold_upper)
edged_copy = edged.copy()
edged_copy = cv2.GaussianBlur(edged_copy, (3, 3), 0)
cv2.imwrite('edged.jpg', edged)
if show:
cv2.imshow('Edged', edged)
cv2.imshow('Edged blurred', edged_copy)
cv2.waitKey(0)
cv2.destroyAllWindows()
(_, cnts, _) = cv2.findContours(edged_copy, cv2.RETR_LIST,
cv2.CHAIN_APPROX_SIMPLE)
cnts = sorted(cnts, key=cv2.contourArea, reverse=True)[:4]
screenCnt = []
for c in cnts:
# approximate the contour
peri = cv2.arcLength(c, True)
approx = cv2.approxPolyDP(c, 0.015 * peri, True)
# approx = np.array(cv2.boundingRect(c))
# if our approximated contour has four points, then we
# can assume that we have found our target
debugging = False
if debugging:
cv2.drawContours(im, [approx], -1, (0, 255, 0), 2)
cv2.imshow('Outline', im)
cv2.waitKey(0)
if len(approx) == 4:
screenCnt = approx
break
if screenCnt.__len__() != 0:
if show:
cv2.drawContours(im, [screenCnt], -1, (0, 255, 0), 2)
cv2.imwrite('outlined.jpg', im)
cv2.imshow('Outline', im)
cv2.waitKey(0)
cv2.destroyAllWindows()
warped = four_point_transform(orig, screenCnt.reshape(4, 2) * scale)
else:
warped = orig
warped = cv2.cvtColor(warped, cv2.COLOR_BGR2GRAY)
warped = warped > threshold_local(warped, 251, offset=10)
warped = warped.astype('uint8') * 255
if show:
cv2.imshow('Original', util.resize(orig, height=650))
cv2.imshow('Scanned', util.resize(warped, height=650))
cv2.waitKey(0)
cv2.imwrite('deskewed.jpg', warped)
def next_batch(self):
start = self.index
end = self.index + self.batch_size
if end >= self.volumn:
end = self.volumn
self.index = 0
else:
self.index = end
data_batch = self.data[start:end]
img = []
keypoint_hmap = []
direction_hmap = []
names = []
lables = []
link_kmaps = []
for piece in data_batch:
tmp = misc.imread(self.img_dir + piece['image_id'] + '.jpg')
names.append(piece['image_id'])
# resize the img and annotation to 368x368
tmp, rate, left, top = \
util.resize(tmp, self.length)
annos = np.array(list(piece['keypoint_annotations'].values()),
dtype=np.float16)
annos[:, ::3] = annos[:, ::3] * rate - left
annos[:, 1::3] = annos[:, 1::3] * rate - top
annos = annos.astype(np.int16)
img.append(tmp)
# resize the img and annotations to 46x46
tmp = misc.imresize(tmp, (self.short, self.short))
rate = self.short / self.length
annos[:, ::3] = annos[:, ::3] * rate
annos[:, 1::3] = annos[:, 1::3] * rate
annos = np.round(annos).astype(np.int8)
kmap = util.get_key_hmap(\
tmp.shape, annos, self.patch, self.patch_l//2)
keypoint_hmap.append(kmap)
human_num = len(annos)
failed = True
# since many img contains only one people, we manually
# reduce the possibility to get the positive sample
random.shuffle(self.limb_index)
if np.random.random_sample() < 0.15 or human_num == 1:
# extract from the same people
for limb in self.limb_index:
start = self.limbs[limb][0]
end = self.limbs[limb][1]
human = annos[np.random.randint(human_num)]
start_x = human[start * 3]
start_y = human[start * 3 + 1]
start_v = human[start * 3 + 2]
end_x = human[end * 3]
end_y = human[end * 3 + 1]
end_v = human[end * 3 + 2]
if util.validate(start_x, start_y, start_v, self.short, self.short) \
and util.validate(end_x, end_y, end_v, self.short, self.short):
kmap_start = util.get_single_kmap(
(self.short, self.short), start_x, start_y,
self.patch_l // 2, self.patch)
kmap_end = util.get_single_kmap(
(self.short, self.short), end_x, end_y,
self.patch_l // 2, self.patch)
link_kmaps.append(
np.stack((kmap_start, kmap_end), axis=-1))
failed = False
break
if failed:
link_kmaps.append(np.zeros((self.short, self.short, 2)))
print('no validate limb in %s (same)' % piece['image_id'])
lables.append(1)
else:
# extract from two different humen
for limb in self.limb_index:
start = self.limbs[limb][0]
end = self.limbs[limb][1]
index1 = np.random.randint(human_num)
human1 = annos[index1]
start_x = human1[start * 3]
start_y = human1[start * 3 + 1]
start_v = human1[start * 3 + 2]
index2 = np.random.randint(human_num)
human2 = annos[index2]
end_x = human2[end * 3]
end_y = human2[end * 3 + 1]
end_v = human2[end * 3 + 2]
if util.validate(start_x, start_y, start_v, self.short, self.short) \
and util.validate(end_x, end_y, end_v, self.short, self.short) \
and index1 != index2:
kmap_start = util.get_single_kmap(
(self.short, self.short), start_x, start_y,
self.patch_l // 2, self.patch)
kmap_end = util.get_single_kmap(
(self.short, self.short), end_x, end_y,
self.patch_l // 2, self.patch)
link_kmaps.append(
np.stack((kmap_start, kmap_end), axis=-1))
lables.append(0)
failed = False
break
if failed:
link_kmaps.append(np.zeros((self.short, self.short, 2)))
print('no validate limb in %s (differen)' %
piece['image_id'])
lables.append(0)
tmp_dmap, tmp_dmap_re = util.get_dir_hmap(\
tmp.shape, annos, self.ones, self.limbs, self.patch_l//2)
dmap = util.weight_dir_hmap(kmap, tmp_dmap, tmp_dmap_re,
self.limbs)
direction_hmap.append(dmap)
img = np.array(img, dtype=np.float32)
img -= self.mean
img /= self.var
keypoint_hmap = np.array(keypoint_hmap, dtype=np.float32)
direction_hmap = np.array(direction_hmap, dtype=np.float32)
lables = np.array(lables, dtype=np.float32)
link_kmaps = np.array(link_kmaps, dtype=np.float32)
return img, keypoint_hmap, direction_hmap, names, lables, link_kmaps
from feature import *
import util
import numpy as np
X, y, X_test, Xname, Xname_test = util.load_image()
y = np.array(y)
Theta = [0, np.pi / 6, np.pi / 3, np.pi / 2, 2 * np.pi / 3, 5 * np.pi / 6]
Sigma = [2, 3, 4, 5]
X = util.resize(X, 11)
X_test = util.resize(X_test, 11)
A, B = feature_gabor_list(X, Theta, Sigma)
T = gen_energy_array(A, B)
A_test, B_test = feature_gabor_list(X_test, Theta, Sigma)
T_test = gen_energy_array(A_test, B_test)
s = np.std(T, 0)
T1 = T / s
T_test1 = T_test / s
def svm(T, y, T_test):
from sklearn.svm import SVC
clf = SVC()
clf.fit(T, y)
result = clf.predict(T_test)
return result
import os
import sys
import util
import math
import random
import functools
import numpy as np
from PIL import Image, ImageEnhance
dir = sys.argv[1]
size = sys.argv[2]
for img_path in util.imgList(dir):
util.resize(img_path, int(size))