def extract_features(self, image, tensor_name='InceptionV3/Logits/AvgPool_1a_8x8/AvgPool:0'):
"""Extract image feature from image (numpy array) or from jpeg file."""
sess = self.session
feat_tensor = sess.graph.get_tensor_by_name(tensor_name)
# image is a path to an jpeg file
assert os.path.exists(image), 'File does not exist %s' % image
image_loader = ImageLoader()
features = sess.run(feat_tensor, {'input:0': image_loader.load_imgs([image])})
return list(np.squeeze(features))
def __init__(self, bg_code):
Scene.__init__(self)
self.menuentries = {}
self.menu_size = 0
self.background = ImageLoader.get_image(bg_code)
self.background = pygame.transform.scale(self.background, (self.screen.get_width(), self.screen.get_height()))
self.cursor = ImageLoader.get_single_sprite("SPRITE_HAND")
self.cursor_rect = self.cursor.get_rect()
self.left = self.screen.get_width()/8
self.right = self.screen.get_width()/2.1
self.cursor_rect.left = self.left
self.cursor_rect.top = self.screen.get_height()*2/3 + self.cursor.get_height()/3
self.selection = 0
def __init__(self, width = None, height=None):
if (width is None) | (height is None):
self.screen = pygame.display.set_mode(( self.scene_default_width , self.scene_default_height ))
else:
self.screen = pygame.display.set_mode(( width , height ))
pygame.display.set_caption(self.base_title)
pygame.display.set_icon(ImageLoader.get_single_sprite("SPRITE_PUMPKIN_ICO"))
def func(content_path, style_path):
imgloader = ImageLoader()
content_img, style_image = imgloader.loadimages(content_path, style_path)
cnn = models.vgg19(pretrained=True).features.cuda().eval()
cnn_normalization_mean = torch.tensor([0.485, 0.456, 0.406]).cuda()
cnn_normalization_std = torch.tensor([0.229, 0.224, 0.225]).cuda()
input_img = content_img.clone()
model = Model()
output = model.run_style_transfer(cnn, cnn_normalization_mean,
cnn_normalization_std, content_img,
style_image, input_img)
unloader = transforms.ToPILImage()
image = output.cpu().clone()
image = image.squeeze(0)
image = unloader(image)
image.save('./outputs/out1.jpg')
def __init__(self, params, cfg):
self.params = params
self.mode = params.phase
self.is_train = (self.mode == 'train')
self.batch_size = params.batch_size
self.stage = params.stage
self.cfg = {}
self.cfg = cfg
self.image_loader = ImageLoader(cfg)
self.tf_placeholders = {}
self.create_tf_placeholders()
self.global_step = tf.Variable(0, name='global_step', trainable=False)
self.train_op, self.loss_op = None, None
self.eval_op = None
self.mask_logits = self.build_net()
def __init__(self, cfg):
self.cfg = cfg
self.tf_placeholders = {}
self.create_tf_placeholders()
self.global_step = tf.Variable(0, name='global_step', trainable=False)
self.d_train_op, self.g_train_op = None, None
self.ema_op, self.ema_vars = None, {}
self.d_loss, self.g_loss = None, None
self.gen_images, self.eval_op = None, None
self.image_loader = ImageLoader(self.cfg)
def __init__(self, surface):
pygame.sprite.Sprite.__init__(self)
self.surface = surface
self.image = ImageLoader.get_single_sprite("SPRITE_GROUND")
self.move_width = self.surface.get_width() / self.tile_size
self.rect = self.image.get_rect()
self.rect.top = self.surface.get_height() - self.tile_size
self.group = pygame.sprite.GroupSingle(self)
def build_feature(filelist, dest_prefix, area, days=4):
features = []
image_loader = ImageLoader(cache_size=10)
idx = 0
for i in range(len(filelist) - days):
data = [filelist[i + j] for j in range(days + 1)]
data.pop(1)
# check images is continuous
tstamps = [
re.search('CV1_3600_([0-9]{12})\.(jpg|png)', data[j]).group(1)
for j in range(len(data))
]
if not is_complete(tstamps):
continue
feature = [image_loader.read(image, area) for image in data]
label = get_classification(feature[0])
feature = np.array(feature[1:])
label = np.array(label)
ex1, label, ex2 = np.split(label, [4, 76], axis=0)
ex1, label, ex2 = np.split(label, [4, 76], axis=1)
features.append({'x': feature, 'label': label})
if len(features) % 1000 == 0:
np.save('%s.%d.npy' % (dest_prefix, idx), np.stack(features,
axis=0))
print(idx)
print(label.shape)
idx += 1
features = []
np.save('%s.%d.npy' % (dest_prefix, idx), np.stack(features, axis=0))
def __init__(self, surface, life=5):
pygame.sprite.Sprite.__init__(self)
self.surface = surface
self.image = ImageLoader.get_single_sprite("SPRITE_GRAVE")
self.rect = self.image.get_rect()
self.rect.top = self.image.get_height()
self.rect.left = self.image.get_width()
self.group = pygame.sprite.GroupSingle(self)
self.font = pygame.font.Font(None, 36)
self.update_text(life)
def main():
"""Main pipeline of Image Similarity using Deep Ranking."""
net = TripletNet(resnet50(True))
# For training on GPU, we need to transfer net and data onto the GPU
# http://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html#training-on-gpu
if args.is_gpu:
print("==> Initialize CUDA support for TripletNet model ...")
net = torch.nn.DataParallel(net).cuda()
cudnn.benchmark = True
# resume training from the last time
if args.resume:
# Load checkpoint
print('==> Resuming training from checkpoint ...')
checkpoint = torch.load(args.ckptroot)
args.start_epoch = checkpoint['epoch']
net.load_state_dict(checkpoint['state_dict'])
print("==> Loaded checkpoint '{}' (epoch {})".format(
args.ckptroot, checkpoint['epoch']))
# Loss function, optimizer and scheduler
criterion = nn.TripletMarginLoss(margin=args.g, p=args.p)
# optimizer = torch.optim.SGD(net.parameters(),
# lr=args.lr,
# momentum=args.momentum,
# weight_decay=args.weight_decay,
# nesterov=args.nesterov)
optimizer = torch.optim.Adam(net.parameters())
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='min',
factor=0.1,
patience=10,
verbose=True)
# load triplet dataset
trainloader = ImageLoader(batch_size=args.batch_size_train, triplets_name='../triplets.txt', train_flag=True)
# train model
train(net, criterion, optimizer, scheduler, trainloader,
args.start_epoch, args.epochs, args.is_gpu, batch_size=args.batch_size_train)
def __init__(self, surface, left=None, difficulty=1):
pygame.sprite.Sprite.__init__(self)
self.surface = surface
self.image = ImageLoader.get_single_sprite("SPRITE_PUMPKIN")
self.move_width = self.surface.get_width() / self.tile_size
self.move_height = self.surface.get_height() / self.tile_size * 0.75
self.rect = self.image.get_rect()
if left is None:
self.rect.left = random.randint(
1, self.move_width - 1) * self.tile_size
else:
self.rect.left = left * self.tile_size
self.group = pygame.sprite.GroupSingle(self)
self.move_difficulty = difficulty
def __init__(self):
Scene.__init__(self)
self.background = ImageLoader.get_image("IMG_GAME_BG")
self.background = pygame.transform.scale(self.background, (self.screen.get_width(), self.screen.get_height()))
width = self.screen.get_width() / 6
height = self.screen.get_height()
self.info_zone = pygame.Surface((width, height))
self.info_zone.fill((0, 0, 0))
self.info_zone.set_colorkey((0,0,0))
self.rect = self.info_zone.get_rect()
self.rect.left = self.screen.get_width() - width
self.score_area = area.ScoreArea(self.info_zone)
self.life_area = area.LiveArea(self.info_zone)
self.notif_area = area.NotificationArea(self.screen)
self.update_score_area(0)
self.update_life_area(5)
self.update_notification_area("Catch a max of pumpkins !")
def __init__(self, img_code): Scene.__init__(self) self.image = ImageLoader.get_image(img_code) self.image = pygame.transform.scale(self.image, (self.screen.get_width(), self.screen.get_height()))
class Model(object):
def __init__(self, params, cfg):
self.params = params
self.mode = params.phase
self.is_train = (self.mode == 'train')
self.batch_size = params.batch_size
self.stage = params.stage
self.cfg = {}
self.cfg = cfg
self.image_loader = ImageLoader(cfg)
self.tf_placeholders = {}
self.create_tf_placeholders()
self.global_step = tf.Variable(0, name='global_step', trainable=False)
self.train_op, self.loss_op = None, None
self.eval_op = None
self.mask_logits = self.build_net()
def create_tf_placeholders(self):
roi_h, roi_w = self.cfg['scaled_img_shape'] #self.cfg['roi_shape']
roi_images = tf.placeholder(tf.float32,
[self.batch_size, roi_h, roi_w, 3])
roi_masks = tf.placeholder(tf.float32,
[self.batch_size, roi_h, roi_w, 2])
roi_weights = tf.placeholder(tf.float32,
[self.batch_size, roi_h, roi_w])
self.tf_placeholders = {
'images': roi_images,
'masks': roi_masks,
'weights': roi_weights
}
def build_net(self):
batch_norm = self.params.batch_norm
# tf.reset_default_graph()
roi_images = self.tf_placeholders["images"]
net = self.params.net
if net == 'unet':
mask_logits = unet(roi_images,
num_classes=2,
training=self.is_train,
init_channels=8,
n_layers=6,
batch_norm=batch_norm)
else:
mask_logits = fcn_gcn_net(roi_images,
num_classes=2,
k_gcn=11,
training=self.is_train,
init_channels=8,
n_layers=7,
batch_norm=True)
return mask_logits
def make_train_op(self):
learning_rate = self.params.learning_rate
roi_masks = self.tf_placeholders["masks"]
roi_masks_pos = tf.slice(roi_masks, [0, 0, 0, 1], [-1, -1, -1, 1])
roi_masks_pos = tf.squeeze(roi_masks_pos, [-1])
roi_weights = self.tf_placeholders["weights"]
_, tf_mask = mask_prediction(self.mask_logits)
loss0 = dice_coef_loss(roi_masks_pos, tf_mask)
loss1 = pixel_wise_loss(self.mask_logits,
roi_masks,
pixel_weights=roi_weights)
loss = loss0 + self.params.sce_weight * loss1
solver = tf.train.AdamOptimizer(learning_rate, epsilon=1e-8)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
self.train_op = solver.minimize(loss, global_step=self.global_step)
self.loss_op = [loss0, loss1]
def make_eval_op(self):
pred_probs, pred_masks = mask_prediction(self.mask_logits)
self.eval_op = [pred_probs, pred_masks]
def get_feed_dict(self, batch, perturb=True):
if self.stage == 1:
roi_images, roi_masks, roi_weights = \
self.image_loader.load_img_batch(batch, edge_factor=self.params.edge_factor)
else:
roi_images, roi_masks, roi_weights = \
self.image_loader.load_roi_batch(batch, perturb=perturb,
edge_factor=self.params.edge_factor)
tf_roi_images = self.tf_placeholders["images"]
if roi_masks is None:
return {tf_roi_images: roi_images}
tf_roi_masks = self.tf_placeholders["masks"]
tf_roi_weights = self.tf_placeholders["weights"]
return {
tf_roi_images: roi_images,
tf_roi_masks: roi_masks,
tf_roi_weights: roi_weights
}
def train(self, data):
""" Train the model. """
params = self.params
save_dir = os.path.join(params.save_dir,
str(params.set).zfill(2),
'stage_' + str(self.stage))
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_dir = os.path.join(save_dir, 'model')
self.make_train_op()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
if params.load:
self.load(sess, saver)
n_display = params.display_period
for i_epoch in tqdm(list(range(params.num_epochs)), desc='epoch'):
dice_loss, sce_loss, n_steps = 0, 0, 0
for _ in tqdm(list(range(0, data.count, self.batch_size)),
desc='batch'):
batch = data.next_batch()
if len(batch[0]) < self.batch_size:
continue
ops = [self.train_op, self.global_step] + self.loss_op
feed_dict = self.get_feed_dict(batch, perturb=True)
_, global_step, loss0, loss1 = sess.run(
ops, feed_dict=feed_dict)
if n_steps + 1 == n_display:
print(
"Dice coeff : {}, Cross entropy loss : {}".format(
-dice_loss / n_steps, sce_loss / n_steps))
dice_loss, sce_loss, n_steps = 0, 0, 0
else:
dice_loss += loss0
sce_loss += loss1
n_steps += 1
if (global_step + 1) % params.save_period == 0:
print("Saving model in {}".format(save_dir))
saver.save(sess, save_dir, global_step)
data.reset()
print("{} epochs finished.".format(i_epoch))
def validate(self, data):
""" Test the model. """
# params = self.params
self.make_eval_op()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
self.load(sess, saver)
for _ in tqdm(list(range(data.count)), desc='batch'):
batch = data.next_batch()
img_file, mask_file = batch[0][0], batch[1][0]
gt_bbox = self.image_loader.generate_rois([mask_file],
perturb=False)[0]
feed_dict = self.get_feed_dict(batch, perturb=False)
pred_probs, _ = sess.run(self.eval_op, feed_dict=feed_dict)
# pred_mask = np.zeros_like(pred_probs, dtype=np.uint8)
# pred_mask[np.where(pred_mask > 0.5)] = 1
# print(np.where(pred_mask > 0.5))
mask_pred = pred_probs[0, :, :, 1]
mask_pred[mask_pred > 0.5] = 1
mask_pred[mask_pred <= 0.5] = 0
if True:
img = cv2.imread(img_file)
real_mask = np.zeros_like(img, dtype=np.uint8)
if self.stage == 1:
img_h, img_w = self.cfg['image_shape']
l, r, t, b = self.cfg['crops']
pred_mask = imresize(
mask_pred, (img_h - t - b, img_w - l - r)) / 255
real_mask[t:img_h - b, l:img_w - r,
0] = np.uint8(np.round(pred_mask))
else:
y, x, h, w = gt_bbox
pred_mask = cv2.resize(mask_pred, (w, h))
real_mask[y:y + h, x:x + w, 0] = np.uint8(pred_mask)
winname = 'Image %s' % (img_file)
img = cv2.resize(img, (1438, 960))
img_mask = cv2.resize(real_mask * 255, (1438, 960),
interpolation=cv2.INTER_CUBIC)
display_img = cv2.addWeighted(img, 0.2, img_mask, 0.8, 0)
cv2.imshow(winname, display_img)
cv2.moveWindow(winname, 100, 100)
cv2.waitKey(1000)
gt_mask = self.image_loader.load_mask(mask_file)
print("Dice coefficient : ",
dice_coef(gt_mask, real_mask[:, :, 0]))
def test(self, data):
""" Test the model. """
params = self.params
self.make_eval_op()
res_dir = params.test_results_dir
res_dir = os.path.join(res_dir,
str(params.set).zfill(2),
'stage_' + str(params.stage))
if not os.path.exists(res_dir):
os.makedirs(res_dir)
img_names = []
rle_strings = []
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
self.load(sess, saver)
for _ in tqdm(list(range(data.count)), desc='batch'):
batch = data.next_batch()
img_file = batch[0][0]
feed_dict = self.get_feed_dict(batch, perturb=False)
pred_probs, _ = sess.run(self.eval_op, feed_dict=feed_dict)
# pred_mask = np.zeros_like(pred_probs, dtype=np.uint8)
# pred_mask[np.where(pred_mask > 0.5)] = 1
# print(np.where(pred_mask > 0.5))
mask_pred = pred_probs[0, :, :]
#mask_pred[mask_pred > 0.5] = 1
#mask_pred[mask_pred <= 0.5] = 0
real_mask = self.image_loader.postprocess(mask_pred)
rle = rle_encode(real_mask)
rle_strings.append(rle_to_string(rle))
if 1:
img = cv2.imread(img_file)
img_mask = np.zeros_like(img)
img_mask[:, :, 0] = real_mask * 255
# y, x, h, w = gt_bbox
# print(gt_bbox)
winname = 'Image %s' % (img_file)
img = cv2.resize(img, (1438, 960))
img_mask = cv2.resize(img_mask, (1438, 960))
display_img = cv2.addWeighted(img, 0.4, img_mask, 0.6, 0)
cv2.imshow(winname, display_img)
cv2.moveWindow(winname, 100, 100)
cv2.waitKey(1000)
img_name = os.path.basename(img_file)
img_names.append(img_name)
#outfile = os.path.join(res_dir, str(img_name) + '.npy')
#np.save(outfile, mask_pred)
df = {'img': img_names, 'rle_mask': rle_strings}
df = pd.DataFrame(df)
outfile = os.path.join(res_dir, 'results.csv')
df.to_csv(outfile)
def load(self, sess, saver):
""" Load the trained model. """
params = self.params
print("Loading model...")
load_dir = os.path.join(params.save_dir,
str(params.set).zfill(2),
'stage_' + str(params.stage), 'model')
checkpoint = tf.train.get_checkpoint_state(os.path.dirname(load_dir))
if checkpoint is None:
print("Error: No saved model found. Please train first.")
sys.exit(0)
saver.restore(sess, checkpoint.model_checkpoint_path)