def generate_heatmaps(dataset, network_type):
segm = Segmenter(network_type)
loader = data.Loader()
images = loader.load_original_images(dataset=dataset)
if dataset == 'train':
outdir = settings.TRAIN_HEATMAP_DIR
elif dataset == 'test_st1':
outdir = settings.TEST_HEATMAP_DIR
if network_type == 'region':
netname = settings.REGION_HEATMAP_NETWORK_WEIGHT_NAME
elif network_type == 'individual':
netname = settings.INDIVIDUAL_HEATMAP_NETWORK_WEIGHT_NAME
outdir = os.path.join(outdir, netname)
if not os.path.exists(outdir):
os.makedirs(outdir)
for img in images:
filename = img['m']['filename']
print('Generating heatmap for image: ' + filename)
image = img['x']()
if network_type == 'region':
zoom = 224 / 400
image = scipy.ndimage.zoom(image, [zoom, zoom, 1])
heatmap = segm.heatmap(image)
scipy.misc.imsave(os.path.join(outdir, filename + '.jpg'),
(255 * heatmap).astype('uint8'))
def run_feature_generation(dataset,
start=0,
end=-1,
patches=False,
ignore_pups=False):
loader = data.Loader()
images = loader.load_original_images(dataset=dataset)
if dataset == 'train':
if patches:
outdir = settings.DENSITY_MAP_FEATURE_CROPS_DIR
else:
outdir = settings.TRAIN_FEATURES_DIR
indir = settings.TRAIN_ORIGINAL_IMAGES_DIR
elif dataset == 'test_st1':
outdir = settings.TEST_FEATURES_DIR
indir = settings.TEST_ORIGINAL_IMAGES_DIR
else:
raise Exception('Data set not implemented: ' + dataset)
if not os.path.exists(outdir):
os.makedirs(outdir)
if end == -1:
end = len(images)
for idx in range(start, end):
imageid = images[idx]['m']['filename']
settings.logger.info('Generating crops image %d (%s.jpg)...' %
(idx, imageid))
image = vigra.readImage(os.path.join(indir, imageid + '.jpg'))
image = vigra.sampling.resampleImage(image, factor=RESAMPLE_FACTOR)
if patches:
sea_lions = [((round(float(coord['x_coord']) * RESAMPLE_FACTOR),
round(float(coord['y_coord']) * RESAMPLE_FACTOR)),
coord['category'])
for coord in images[idx]['m']['coordinates']]
crops = sliding_window_crop_generation(sea_lions, image.shape,
ignore_pups)
generate_features(image,
os.path.join(outdir, imageid),
patches=crops)
else:
generate_features(image, os.path.join(outdir, imageid))
def from_coordinates():
d = data.Loader()
avg_counts = stats.prior_counts('average')
total = (
avg_counts['adult_males']
+ avg_counts['subadult_males']
+ avg_counts['adult_females']
+ avg_counts['juveniles']
+ avg_counts['pups'])
prior = {
'adult_males': avg_counts['adult_males'] / total,
'subadult_males': avg_counts['subadult_males'] / total,
'adult_females': avg_counts['adult_females'] / total,
'juveniles': avg_counts['juveniles'] / total,
'pups': avg_counts['pups'] / total,
}
logger.info("Loading test coordinates (this may take a long time)")
coords = d.load_test_coordinates()
logger.info("Loading test image information")
images = d.load_original_images('test_st1')
logger.info("Starting output writing")
with open(settings.TEST_COUNTS_CSV, 'w', newline="\n", encoding="utf-8") as file:
writer = csv.DictWriter(file, fieldnames=['test_id', 'adult_males', 'subadult_males', 'adult_females', 'juveniles', 'pups'])
writer.writeheader()
for img in images:
id = img['m']['filename']
#n = len(coords[id]) / 74 * 85 if id in coords else 0
n = len(coords[id]) if id in coords else 0
counts = {
'test_id': id,
'adult_males': round(n * prior['adult_males']),
'subadult_males': round(n * prior['subadult_males']),
'adult_females': round(n * prior['adult_females']),
'juveniles': round(n * prior['juveniles']),
'pups': round(n * prior['pups']),
}
writer.writerow(counts)
def __init__(self, crop_size, attention, diameter=None, output_size=None):
"""
:param crop_size: Int, the size of the desired crops
:param attention: Boolean, whether to apply blackout attention to the crops
or not
"""
self.crop_size = crop_size
self.attention = attention
self.loader = data.Loader()
self.window_coords = {'row': 0, 'column': 0}
self.current_image_id = None
self.current_image_size = {'x': None, 'y': None}
self.current_image = None
self.current_image_dotted = None
self.candidate_matrix = None
self.candidate_current_ix = {'row': 0, 'column': 0}
self.blackout_masks = []
self.diameter = diameter
self.output_size = output_size
0:1]),
opt.outH, opt.outW),
util.imageSummary(opt, "image_mask/GT", maskGT[:, 0, :, :, 0:1], opt.H,
opt.W)
]
summaryImage = tf.summary.merge(summaryImage)
summaryLoss = [
tf.summary.scalar("loss_total", loss),
tf.summary.scalar("loss_mask", loss_mask),
tf.summary.scalar("loss_depth", loss_depth)
]
summaryLoss = tf.summary.merge(summaryLoss)
# load data
print(util.toMagenta("loading dataset..."))
dataloader = data.Loader(opt)
dataloader.loadChunk(opt)
# prepare model saver/summary writer
saver = tf.train.Saver()
summaryWriter = tf.summary.FileWriter("summary_{0}/{1}".format(
opt.group, opt.model))
print(util.toYellow("======= TRAINING START ======="))
timeStart = time.time()
# start session
tfConfig = tf.ConfigProto(allow_soft_placement=True)
tfConfig.gpu_options.allow_growth = True
with tf.Session(config=tfConfig) as sess:
sess.run(tf.global_variables_initializer())
if opt.fromIt != 0:
def __init__(self,
data_type="original",
input_shape=(300, 300, 3),
prediction_class_type="multi",
class_balancing=True,
mini_batch_size=32,
tensor_board=False,
validate=True):
"""
:param prediction_class_type: "single" or "multi"
"""
self.model = None
self.input_shape = input_shape
self.prediction_class_type = prediction_class_type
self.mini_batch_size = mini_batch_size
self.tensor_board = tensor_board
self.validate = validate
self.data_type = data_type
settings.logger.info("Starting...")
loader = data.Loader()
transform = self.data_transformer()
if data_type == 'original_train':
train_data = loader.load_original_images(dataset='train')
elif data_type == 'original_test':
train_data = loader.load_original_images(dataset='test_st1')
elif data_type == "density_map_feature_crops":
train_data = loader.load_density_map_feature_crops()
else:
train_data = loader.load_crop_images(data_type=data_type)
if data_type == 'heatmap_crops' and class_balancing:
settings.logger.error(
'Class balancing can\'t be activated while data_type=' +
data_type)
if validate:
train_val_split = loader.train_val_split(train_data)
self.iterator = data.DataIterator(
train_val_split['train'],
transform,
batch_size=mini_batch_size,
shuffle=True,
seed=42,
class_balancing=class_balancing,
class_transformation=self.data_class_transform())
# No class balancing for validation
self.val_iterator = data.DataIterator(
train_val_split['validate'],
transform,
batch_size=mini_batch_size,
shuffle=True,
seed=42,
class_balancing=False,
class_transformation=self.data_class_transform())
else:
self.iterator = data.DataIterator(
train_data,
transform,
batch_size=mini_batch_size,
shuffle=True,
seed=42,
class_balancing=class_balancing,
class_transformation=self.data_class_transform())
def generate_individual_crops(sea_lion_size,
num_negative_crops,
ignore_pups=False,
blackout=False,
blackout_diameter=100):
"""
:param sea_lion_size: The width/height (in the actual image) of a sea lion crop (default 100 by 100)
:param num_negative_crops: The total number of negative crops generated, over all images
:param ignore_pups: If true, pups will not result in positive crops
:param blackout: If true, the crops' corners will be made black and only a circle will remain
:param blackout_diameter: The diameter of the blackout mask
"""
loader = data.Loader()
images = loader.load_original_images()
dir = settings.CROPS_OUTPUT_DIR # with timestamp, so makedirs is needed every time
os.makedirs(dir)
num_images = len(images)
num_neg_crops_per_image = int(num_negative_crops / num_images)
num_neg_crops_remainder = num_neg_crops_per_image % num_images
n = 0
for image in images:
n += 1
logger.info('Cropping image ' + str(n) + '...')
num_neg_crops_this_image = (
num_neg_crops_per_image +
1) if n <= num_neg_crops_remainder else num_neg_crops_per_image
img = image['x']()
filename = image['m']['filename']
img_height, img_width, _ = img.shape
coords = [(max(
0,
min(img_width - sea_lion_size,
int(float(coord['x_coord']) - sea_lion_size / 2))),
max(
0,
min(img_height - sea_lion_size,
int(float(coord['y_coord']) - sea_lion_size / 2))),
coord['category']) for coord in image['m']['coordinates']]
maskname = os.path.join(settings.OVERLAP_MASKS_DIR, filename + '.mask')
if not os.path.exists(maskname):
raise AssertionError(
'Overlap mask not found! Generate overlap masks before crops: '
+ maskname)
with gzip.open(maskname, 'rb') as maskfile:
mask = pickle.load(maskfile)
num_neg = 0
negcoords = []
while num_neg < num_neg_crops_this_image:
x_coord = random.randint(0, img_width - sea_lion_size)
y_coord = random.randint(0, img_height - sea_lion_size)
# Check for overlap with any sea lions (also slightly outside of the bounding box, to avoid matching bigger sea lions' parts)
if any(
coords_overlap((sea_lion[0], sea_lion[1]), (
x_coord, y_coord), 1.25 * sea_lion_size) > 0
for sea_lion in coords):
continue
negcoords.append((x_coord, y_coord, 'negative'))
num_neg += 1
for x_coord, y_coord, category in negcoords + coords:
if ignore_pups and coord['category'] == 'pups':
continue
# Check for black markings
mask_crop = utils.crop_image(mask, (x_coord, y_coord),
sea_lion_size)
if is_blacked_out(mask_crop):
continue
# Crop
crop_img = utils.crop_image(img, (x_coord, y_coord), sea_lion_size)
if blackout:
blackout_coord = (sea_lion_size - blackout_diameter) / 2
crop_img = utils.blackout(crop_img,
[(blackout_coord, blackout_coord)],
blackout_diameter)
cropname = category + '_id' + filename + '_' + str(
1 * (num_neg <= 0)) + 'clions_at_' + str(x_coord) + '-' + str(
y_coord) + '_' + str(sea_lion_size) + 'px.jpg'
num_neg -= 1
scipy.misc.imsave(os.path.join(dir, cropname), crop_img)
0:1]),
opt.outH, opt.outW),
util.imageSummary(opt, "image_mask/GT", maskGT[:, :, :, 0:1], opt.outH,
opt.outW)
]
summaryImage = tf.summary.merge(summaryImage)
summaryLoss = [
tf.summary.scalar("loss_total", loss),
tf.summary.scalar("loss_mask", loss_mask),
tf.summary.scalar("loss_XYZ", loss_XYZ)
]
summaryLoss = tf.summary.merge(summaryLoss)
# load data
print(util.toMagenta("loading dataset..."))
dataloader = data.Loader(opt, loadNovel=False, loadFixedOut=True)
dataloader.loadChunk(opt)
# prepare model saver/summary writer
saver = tf.train.Saver(max_to_keep=50)
summaryWriter = tf.summary.FileWriter("summary_{0}/{1}".format(
opt.group, opt.model))
print(util.toYellow("======= TRAINING START ======="))
timeStart = time.time()
# start session
tfConfig = tf.ConfigProto(allow_soft_placement=True)
tfConfig.gpu_options.allow_growth = True
with tf.Session(config=tfConfig) as sess:
sess.run(tf.global_variables_initializer())
if opt.fromIt != 0:
PH = [inputImage,renderTrans,depthGT,maskGT]
# ------ build encoder-decoder ------
encoder = graph.encoder if opt.arch=="original" else \
graph.encoder_resnet if opt.arch=="resnet" else None
decoder = graph.decoder if opt.arch=="original" else \
graph.decoder_resnet if opt.arch=="resnet" else None
latent = encoder(opt,inputImage)
XYZ,maskLogit = decoder(opt,latent) # [B,H,W,3V],[B,H,W,V]
mask = tf.to_float(maskLogit>0)
# ------ build transformer ------
fuseTrans = tf.nn.l2_normalize(opt.fuseTrans,dim=1)
XYZid,ML = transform.fuse3D(opt,XYZ,maskLogit,fuseTrans) # [B,1,VHW]
# load data
print(util.toMagenta("loading dataset..."))
dataloader = data.Loader(opt,loadNovel=False,loadTest=True)
CADN = len(dataloader.CADs)
chunkN = int(np.ceil(CADN/opt.chunkSize))
dataloader.loadChunk(opt,loadRange=[0,opt.chunkSize])
# prepare model saver/summary writer
saver = tf.train.Saver()
print(util.toYellow("======= EVALUATION START ======="))
timeStart = time.time()
# start session
tfConfig = tf.ConfigProto(allow_soft_placement=True)
tfConfig.gpu_options.allow_growth = True
with tf.Session(config=tfConfig) as sess:
util.restoreModel(opt,sess,saver)
print(util.toMagenta("loading pretrained ({0})...".format(opt.load)))
def train_ld(): ds = data.Dataset() ds.datas = data.read_train() ld = data.Loader(ds, 1) return ld
def test_ld(): ds = data.Dataset() ds.datas = data.read_test() ld = data.Loader(ds, 1) return ld