def get_positives(mixcomp, settings, indices, files, crop=False):
im_size = settings['detector']['image_size']
# Use the same seed for all mixture components! That will make them easier to compare,
# without having to sample to infinity.
# HERE: Make it possible to input data directly!
descriptor = gv.load_descriptor(settings)
radii = settings['detector']['spread_radii']
psize = settings['detector']['subsample_size']
rotspread = settings['detector'].get('rotation_spreading_radius', 0)
cb = settings['detector'].get('crop_border')
all_feats = []
for index in indices:
ag.info("Fetching positives from image of index {0} and mixture component {1}".format(index, mixcomp))
gray_im = gv.img.asgray(gv.img.load_image(files[index]))
if crop:
gray_im = gv.img.crop(gray_im, im_size)
else:
gray_im = gv.img.resize(gray_im, im_size)
#gray_im = gv.img.resize(gray_im, im_size)
feats = descriptor.extract_features(gray_im, settings=dict(spread_radii=radii, subsample_size=psize, rotation_spreading_radius=rotspread, crop_border=cb))
all_feats.append(feats)
return mixcomp, np.asarray(all_feats)
def _extract_patches(self,X):
ag.info('Extract Patches: Seperate coded regions into groups')
trainingDataNum = X.shape[0]
totalRange = X.shape[1]
frame = (self._part_shape[0] - self._lowerLayerShape[0]) / 2
frame = int(frame)
partsRegion = [[] for x in range(self._num_lower_parts//self._rotation)]
for i in range(trainingDataNum):
for m in range(totalRange)[frame:totalRange - frame]:
for n in range(totalRange)[frame:totalRange - frame]:
if(X[i,m,n]!=-1):
partsGrid = X[i,m-frame:m+frame+1,n-frame:n+frame+1]
detectedIndex = X[i,m,n]
uprightIndex = (detectedIndex - detectedIndex % self._rotation)//self._rotation
partsRegion[uprightIndex].append(partsGrid)
finalResultRegion = []
for i in range(self._num_lower_parts//self._rotation):
blocks = []
for ori in range(self._rotation):
angle = ori / self._rotation * 360
from pnet.cyfuncs import rotate_index_map_pooling
yy1 = rotate_index_map_pooling(np.asarray(partsRegion[i]),angle, 0, self._rotation, self._num_lower_parts, (self._part_shape[0] - self._lowerLayerShape[0] + 1,self._part_shape[1] - self._lowerLayerShape[1] + 1))
#yy.shape = (numOfPatches, 1, 1, rotation, trueParts)
yy = yy1.reshape(yy1.shape[:3] + (self._rotation, self._num_lower_parts // self._rotation))
blocks.append(yy)
#blocks.shape = (numofPatches, numOfRotations, 1, 1, rotation, trueParts)
blocks = np.asarray(blocks).transpose((1,0,2,3,4,5))
shape = blocks.shape[2:4] + (np.prod(blocks.shape[4:]),)
## Flatten
blocks = blocks.reshape(blocks.shape[:2] + (-1,))
finalResultRegion.append(blocks)
return (finalResultRegion, shape)
def _get_patches(self, image):
samples_per_image = self.settings['samples_per_image']
fr = self.settings['patch_frame']
the_patches = []
the_originals = []
if isinstance(image, str):
ag.info("Extracting patches from", filename)
img = load_image(filename)
else:
ag.info("Extracting patches from image of size", image.shape)
img = image
#edges, img = ag.features.bedges_from_image(f, k=5, radius=1, minimum_contrast=0.05, contrast_insensitive=False, return_original=True, lastaxis=True)
# Convert to grayscale if necessary
if img.ndim == 3:
img = img[...,:3].mean(axis=-1)
# LEAVE-BEHIND
if 1:
edges = ag.features.bedges(img, **self.settings['bedges'])
else:
edges, img = ag.features.bedges_from_image(filename, return_original=True, **self.settings['bedges'])
#s = self.settings['bedges'].copy()
#if 'radius' in s:
# del s['radius']
#edges_nospread = ag.features.bedges_from_image(filename, radius=0, **s)
# How many patches could we extract?
w, h = [edges.shape[i]-self.patch_size[i]+1 for i in range(2)]
# TODO: Maybe shuffle an iterator of the indices?
for sample in range(samples_per_image):
for tries in range(20):
x, y = random.randint(0, w-1), random.randint(0, h-1)
selection = [slice(x, x+self.patch_size[0]), slice(y, y+self.patch_size[1])]
# Return grayscale patch and edges patch
edgepatch = edges[selection]
#edgepatch_nospread = edges_nospread[selection]
num = edgepatch[fr:-fr,fr:-fr].sum()
if num >= self.settings['threshold']:
the_patches.append(edgepatch)
#the_patches.append(edgepatch_nospread)
# The following is only for clearer visualization of the
# patches. However, normalizing like this might be misleading
# in other ways.
vispatch = img[selection]
if 1:
pass
else:
vispatch = vispatch[...,:3].mean(axis=vispatch.ndim-1)
span = vispatch.min(), vispatch.max()
if span[1] - span[0] > 0:
vispatch = (vispatch-span[0])/(span[1]-span[0])
the_originals.append(vispatch)
break
return the_patches, the_originals
def train(self, X, Y=None):
curX = X
shapes = []
for l, layer in enumerate(self._layers):
#print("part-net")
#print(layer)
#print(np.array(curX).shape)
#if isinstance(curX,tuple):
# print(curX[0].shape)
# print(curX[1])
if not (not layer.supervised or (layer.supervised and Y is not None)):
break
print(layer)
print(layer.trained)
if not layer.trained:
ag.info('Training layer {}...'.format(l))
layer.train(curX, Y=Y, OriginalX=X)
ag.info('Done.')
print(layer)
if layer.name == 'raondom-partition-svm-layer':
curX = layer.extract(curX, Y = Y)
else:
curX = layer.extract(curX)
if isinstance(curX, tuple):
sh = curX[0].shape[1:-1] + (curX[1],)
else:
sh = curX.shape[1:]
print(sh)
shapes.append(sh)
self._train_info['shapes'] = shapes
def train_from_samples(self, patches, patches_label, patches_original):
ag.info("training from samples starts")
self.random_partition()
import time
p = Pool(4, initializer=init, initargs=(patches,))
patches_label_list = [patches_label for i in range(4)]
args = ((patches_label_list[i], self._partition, i, self._num_parts // 4, ) for i in range(4))
start_time = time.time()
svm_objects = p.map(task_spread, args)
allObjects = []
for objects in svm_objects:
allObjects+=objects
p.terminate()
print("--- %s seconds ---" % (time.time() - start_time))
print len(allObjects)
self._svms = allObjects
self._parts = []
self._coef = []
self._intercept = []
for i in range(self._num_parts):
svm_coef = allObjects[i].coef_
svm_intercept = allObjects[i].intercept_
self._parts.append((svm_coef, svm_intercept))
self._coef.append(svm_coef)
self._intercept.append(svm_intercept)
self._coef = np.vstack(self._coef)
self._intercept = np.vstack(self._intercept)
def calc_bbs(detector):
psize = detector.settings['subsample_size']
def get_full_size_bb(k):
bb = detector.bounding_box_for_mix_comp(k)
return tuple(bb[i] * psize[i%2] for i in xrange(4))
def iround(x):
return int(round(x))
def make_bb(bb, max_bb):
# First, make it integral
bb = (iround(bb[0]), iround(bb[1]), iround(bb[2]), iround(bb[3]))
bb = gv.bb.inflate(bb, detector.settings.get('inflate_feature_frame', 4))
bb = gv.bb.intersection(bb, max_bb)
return bb
ag.info("Checkpoint 5")
max_bb = (0, 0) + detector.settings['image_size']
if 'bbs' in detector.extra:
bbs = [make_bb(detector.extra['bbs'][k], max_bb) for k in xrange(detector.num_mixtures)]
else:
bbs = [make_bb(get_full_size_bb(k), max_bb) for k in xrange(detector.num_mixtures)]
return bbs
def extract(self, X_F):
X = X_F[0]
F = X_F[1]
#X = X.astype(np.float16)
#if X.ndim == 3:
#from pnet.cyfuncs import index_map_pooling as poolf
#else:
#print(np.any(np.isnan(X)))
#print(np.all(np.isfinite(X)))
print ("inside extrating")
print(X.shape)
print(X.dtype)
support_mask = self._settings.get('support_mask')
relu = self._settings.get("relu", True)
if support_mask is not None:
from pnet.cyfuncs import activation_map_pooling as poolf
#feature_map = poolf(X, support_mask.astype(np.uint8), F, self._shape, self._strides)
else:
from pnet.cyfuncs import activation_map_pooling as poolf
feature_map_list = []
batch_number = 500
batch_size = X.shape[0] // batch_number
for i in range(batch_number):
#print i
feature_map_list.append(poolf(X[i * batch_size : min((i+1) * batch_size, X.shape[0])].astype(np.float32), F, self._shape, self._strides, relu).astype(np.float16))
feature_map = np.vstack(feature_map_list)
feature_map.astype(np.float16)
self._extract_info['concentration'] = np.apply_over_axes(np.mean, feature_map, [0, 1, 2])[0,0,0]
ag.info("finish pooling")
#print(feature_map.shape)
print(np.any(np.isnan(feature_map)))
print(np.all(np.isfinite(feature_map)))
return feature_map
def run_EM(self, tol, min_probability=0.05, debug_plot=False):
"""
Run the EM algorithm to specified convergence.
Parameters
----------
tol : float
The tolerance gives the stopping condition for convergence.
If the loglikelihood decreased with less than ``tol``, then it will break the loop.
min_probability : float
Disallow probabilities to fall below this value, and extend below one minus this value.
"""
#self._preload_log_templates()
all_llhs = []
all_templates = []
all_weights = []
for loop in range(self.n_init):
# initializing weights
self.weights = 1/self.num_mix * np.ones(self.num_mix, dtype=self.float_type)
#self.opt_type=opt_type TODO: Not used yet.
self.init_affinities_templates(self.init_type)
self.min_probability = min_probability
loglikelihood = -np.inf
# First E step plus likelihood computation
self.M_step()
new_loglikelihood = self._compute_loglikelihoods()
if debug_plot:
plw = ag.plot.PlottingWindow(subplots=(1, self.num_mix), figsize=(self.num_mix*3, 3))
self.iterations = 0
while np.isinf(loglikelihood) or np.fabs((new_loglikelihood - loglikelihood)/loglikelihood) > tol:
if self.iterations >= self.max_iter:
break
ag.info("Iteration {0}: loglikelihood {1}".format(self.iterations, loglikelihood))
loglikelihood = new_loglikelihood
# M-step
self.M_step()
# E-step
new_loglikelihood = self._compute_loglikelihoods()
self.iterations += 1
if debug_plot and not self._plot(plw):
raise ag.AbortException
self.loglikelihood = loglikelihood
self.set_templates()
all_llhs.append(loglikelihood)
all_templates.append(self.templates)
all_weights.append(self.weights)
best_i = np.argmax(all_llhs)
self.templates = all_templates[best_i]
self.weights = all_weights[best_i]
def _process_file_full(fn, sh, descriptor, detector):
counts = np.zeros((1, sh[0], sh[1], descriptor.num_parts))
ag.info("Processing file", fn)
# Which mixture component does this image belong to?
# TODO: Temporary until multicomp
mixcomp = 0#np.argmax(detector.affinities
# Binarize support and Extract alpha
color_img, alpha = gv.img.load_image_binarized_alpha(fn)
img = gv.img.asgray(color_img)
alpha_pad = ag.util.zeropad(alpha, padding)
inv_alpha_pad_expanded = np.expand_dims(~alpha_pad, -1)
# Iterate every duplicate
for loop in xrange(num_duplicates):
ag.info("Iteration {0}/{1}".format(loop+1, num_duplicates))
# Superimpose onto gray background
graymap = create_graymap(img.shape, loop / (num_duplicates - 1), prnds[0])
# Composite
img_with_gray = composite(img, graymap, alpha)
# Retrieve unspread edges (with a given background gray level)
edges = ag.features.bedges(img_with_gray, **bsettings)
# Pad the edges
edges_pad = ag.util.zeropad(edges, (padding, padding, 0))
for i, j in product(locations0, locations1):
selection = [slice(i, i+X_pad_size[0]), slice(j, j+X_pad_size[1])]
X_pad = edges_pad[selection].copy()
nA_pad = inv_alpha_pad_expanded[selection]
# Draw background part from categorical distribution
f_bkg = weighted_choice_unit(bkg, prnds[1])
probs_bkg = get_probs(padded_theta, f_bkg)
probs = nA_pad * probs_bkg
# Iterate over all locations
# Draw from background edge probability over ~alpha
X_pad |= (prnds[2].rand(*probs.shape) < probs)
# Do spreading
X_pad_spread = ag.features.bspread(X_pad, spread=bsettings['spread'], radius=radius)
# De-pad
X_spread = X_pad_spread[padding:-padding,padding:-padding]
# Code parts
parts = descriptor.extract_parts(X_spread.astype(np.uint8))
# Accumulate and return
counts[mixcomp,i,j] += parts[0,0]
return counts
def _extract_patches(self,X):
ag.info('Extract Patches: Seperate coded regions into groups')
trainingDataNum = X.shape[0]
totalRange = X.shape[1]
frame = (self._part_shape[0] - self._lowerLayerShape[0]) / 2
frame = int(frame)
partsRegion = [[] for x in range(self._num_lower_parts),[] for x in range(self._num_lower_parts),[] for x in range(self._num_lower_parts),[] for x in range(self._num_lower_parts)]
def train_from_images(self, images):
"""
Given a list of filenames, it will draw random patches from these images,
and then train the parts model using those patches.
Settings (such as patch size) are specified when constructing the class.
Parameters
----------
images : list or ndarray
This can be:
* A list of filenames.
* A list of images (ndarray).
* An ndarray of images of the same size.
"""
# TODO: Allow filenames to be a list of actual image objects too?
raw_patches, raw_originals = self._random_patches_from_images(images)
if len(raw_patches) == 0:
raise Exception(
"No patches found, maybe your thresholds are too strict?")
mixture = ag.stats.BernoulliMixture(self.num_parts,
raw_patches,
init_seed=0)
# Also store these in "settings"
mixture.run_EM(1e-8, min_probability=self.settings['min_probability'])
ag.info("Done.")
# Reject weak parts
scores = np.empty(self.num_parts)
for i in range(self.num_parts):
part = mixture.templates[i]
sh = part.shape
p = part.reshape((sh[0] * sh[1], sh[2]))
#import ipdb; ipdb.set_trace()
pec = p.mean(axis=0)
N = np.sum(p * np.log(p / pec) + (1 - p) * np.log((1 - p) /
(1 - pec)))
D = np.sqrt(np.sum(np.log(p / (1 - p))**2 * p * (1 - p)))
scores[i] = N / D
# Only keep with a certain score
visparts = mixture.remix(raw_originals)
self.parts = mixture.templates[scores > 1]
self.visparts = visparts[scores > 1]
self.num_parts = self.parts.shape[0]
# Update num_parts
# Store the stuff in the instance
#self.parts = mixture.templates
#self.visparts = mixture.remix(raw_originals)
self._preprocess_logs()
def _get_color_patches(self, X, Y, OriginalX):
assert X.ndim == 4
assert OriginalX.ndim == 4
channel = X.shape[-2]
samples_per_image = self._settings.get('samples_per_image', 20)
fr = self._settings['outer_frame']
patches = []
patches_label = []
patches_original = []
rs = np.random.RandomState(self._settings.get('patch_extraction_seed', 0))
th = self._settings['threshold']
for i in range(X.shape[0]):
Xi = X[i]
OriginalXi = OriginalX[i]
# How many patches could we extract?
w, h = [Xi.shape[j]-self._part_shape[j]+1 for j in range(2)]
# TODO: Maybe shuffle an iterator of the indices?
indices = list(itr.product(range(w-1), range(h-1)))
rs.shuffle(indices)
i_iter = itr.cycle(iter(indices))
for sample in range(samples_per_image):
N = 200
for tries in range(N):
x, y = next(i_iter)
selection = [slice(x, x+self._part_shape[0]), slice(y, y+self._part_shape[1])]
patch = Xi[selection]
#edgepatch_nospread = edges_nospread[selection]
if fr == 0:
tot = patch.sum()
else:
tot = abs(patch[fr:-fr,fr:-fr]).sum()
if th <= tot * channel:
patches.append(patch)
patches_label.append(Y[i])
vispatch = OriginalXi[selection]
span = vispatch.min(),vispatch.max()
if span[1]-span[0] > 0:
vispatch = (vispatch - span[0])/(span[1] - span[0])
patches_original.append(vispatch)
if len(patches) >= self._settings.get('max_samples', np.inf):
return np.asarray(patches),np.asarray(patches_label).astype(np.uint8),np.asarray(patches_original)
break
if tries == N-1:
ag.info('WARNING: {} tries'.format(N))
return np.asarray(patches),np.asarray(patches_label).astype(np.uint8),np.asarray(patches_original)
def _get_patches(self, X):
assert X.ndim == 4
samples_per_image = self._settings.get('samples_per_image', 20)
fr = self._settings.get('outer_frame', 0)
patches = []
rs = np.random.RandomState(self._settings.get('patch_extraction_seed', 0))
th = self._settings['threshold']
support_mask = self._settings.get('support_mask')
consecutive_failures = 0
for Xi in X:
# How many patches could we extract?
w, h = [Xi.shape[i]-self._part_shape[i]+1 for i in xrange(2)]
# TODO: Maybe shuffle an iterator of the indices?
indices = list(itr.product(xrange(w-1), xrange(h-1)))
rs.shuffle(indices)
i_iter = itr.cycle(iter(indices))
for sample in xrange(samples_per_image):
N = 200
for tries in xrange(N):
x, y = i_iter.next()
selection = [slice(x, x+self._part_shape[0]), slice(y, y+self._part_shape[1])]
patch = Xi[selection]
#edgepatch_nospread = edges_nospread[selection]
if support_mask is not None:
tot = patch[support_mask].sum()
elif fr == 0:
tot = patch.sum()
else:
tot = patch[fr:-fr,fr:-fr].sum()
if th <= tot:
patches.append(patch)
if len(patches) >= self._settings.get('max_samples', np.inf):
return np.asarray(patches)
consecutive_failures = 0
break
if tries == N-1:
ag.info('WARNING: {} tries'.format(N))
ag.info('cons', consecutive_failures)
consecutive_failures += 1
if consecutive_failures >= 10:
# Just give up.
raise ValueError("FATAL ERROR: Threshold is probably too high.")
return np.asarray(patches)
def split_train_svms(self, X, Y):
ag.info("Split training SVMs")
import time, random
num_of_processor = 6
class1, class2 = split_2vs2(10)
round_each_proc = self._num_parts // 4 // num_of_processor
rs = np.random.RandomState(self._settings.get("random_seeds", 0))
# four quadrants
p = Pool(num_of_processor, initializer=init_Image, initargs=(X, Y))
allObjects = []
for i in range(4):
args = [
(
i,
j,
round_each_proc,
self._part_shape,
(3, 3),
class1[round_each_proc * j : round_each_proc * (j + 1)],
class2[round_each_proc * j : round_each_proc * (j + 1)],
self._settings.get("random_seeds", 0),
)
for j in range(num_of_processor)
]
svm_objects = p.map(task_spread_svms, args)
for objects in svm_objects:
allObjects += objects
self._svms = allObjects
self._parts = []
self._coef = []
self._intercept = []
for i in range(self._num_parts):
print (i, self._num_parts)
svm_coef = allObjects[i].coef_
svm_intercept = allObjects[i].intercept_
allParam = np.zeros(svm_coef.shape[1] + 1)
allParam[: svm_coef.shape[1]] = svm_coef[0, :]
allParam[-1] = svm_intercept
std = np.std(allParam) * 20
# std = 1
print (np.std(allParam / std), np.max(allParam / std))
self._parts.append((svm_coef / std, svm_intercept / std))
self._coef.append(svm_coef / std)
self._intercept.append(svm_intercept / std)
self._coef = np.vstack(self._coef)
self._intercept = np.vstack(self._intercept)
print (np.max(self._coef))
print (np.mean(self._coef))
print (np.max(self._intercept))
print (np.mean(self._intercept))
def __process_bkg(fn, descriptor, sett, factor):
im = gv.img.asgray(gv.img.load_image(fn))
im = gv.img.resize_with_factor_new(im, factor)
ag.info("Processing image for background model:", fn)
feats = descriptor.extract_features(im, settings=sett)
#count += np.prod(feats.shape[:2])
#bkg_counts += np.apply_over_axes(np.sum, feats, [0, 1]).ravel()
return np.apply_over_axes(np.sum, feats, [0, 1]).ravel(), np.prod(feats.shape[:2])
def train_model(self, filenames):
data = []
for f in filenames[:50]:
ag.info("Preparing", f)
# edges, img = ag.features.bedges_from_image(f, k=5, radius=1, minimum_contrast=0.05, contrast_insensitive=True, return_original=True)
edges, img = self.features_from_image(f)
print "edges.shape =", edges.shape
data.append(edges)
data = np.asarray(data)
print "data.shape =", data.shape
def train_model(self, filenames):
data = []
for f in filenames[:50]:
ag.info("Preparing", f)
#edges, img = ag.features.bedges_from_image(f, k=5, radius=1, minimum_contrast=0.05, contrast_insensitive=True, return_original=True)
edges, img = self.features_from_image(f)
print "edges.shape =", edges.shape
data.append(edges)
data = np.asarray(data)
print "data.shape =", data.shape
def train_from_images(self, images):
"""
Given a list of filenames, it will draw random patches from these images,
and then train the parts model using those patches.
Settings (such as patch size) are specified when constructing the class.
Parameters
----------
images : list or ndarray
This can be:
* A list of filenames.
* A list of images (ndarray).
* An ndarray of images of the same size.
"""
# TODO: Allow filenames to be a list of actual image objects too?
raw_patches, raw_originals = self._random_patches_from_images(images)
if len(raw_patches) == 0:
raise Exception("No patches found, maybe your thresholds are too strict?")
mixture = ag.stats.BernoulliMixture(self.num_parts, raw_patches, init_seed=0)
# Also store these in "settings"
mixture.run_EM(1e-8, min_probability=self.settings['min_probability'])
ag.info("Done.")
# Reject weak parts
scores = np.empty(self.num_parts)
for i in range(self.num_parts):
part = mixture.templates[i]
sh = part.shape
p = part.reshape((sh[0]*sh[1], sh[2]))
#import ipdb; ipdb.set_trace()
pec = p.mean(axis=0)
N = np.sum(p * np.log(p/pec) + (1-p)*np.log((1-p)/(1-pec)))
D = np.sqrt(np.sum(np.log(p/(1-p))**2 * p * (1-p)))
scores[i] = N/D
# Only keep with a certain score
visparts = mixture.remix(raw_originals)
self.parts = mixture.templates[scores > 1]
self.visparts = visparts[scores > 1]
self.num_parts = self.parts.shape[0]
# Update num_parts
# Store the stuff in the instance
#self.parts = mixture.templates
#self.visparts = mixture.remix(raw_originals)
self._preprocess_logs()
def train_from_images(self, filenames):
raw_patches, raw_originals = self.random_patches_from_images(filenames)
mixture = ag.stats.BernoulliMixture(self.num_patches, raw_patches, init_seed=0)
# Also store these in "settings"
mixture.run_EM(1e-8, min_probability=self.settings['min_probability'])
ag.info("Done.")
# Store the stuff in the instance
self.patches = mixture.templates
self.vispatches = mixture.remix(raw_originals)
self._preprocess_logs()
def get_strong_fps_single(detector, i, fileobj, threshold, mixcomp):
topsy = []
ag.info('{0} Getting strong FPs from {1}'.format(i, fileobj.img_id))
img = gv.img.load_image(fileobj.path)
grayscale_img = gv.img.asgray(img)
bbobjs = detector.detect_coarse(grayscale_img, fileobj=fileobj, mixcomps=[mixcomp], use_padding=False, use_scale_prior=False, cascade=True, discard_weak=True, more_detections=True, farming=True)
for bbobj in bbobjs:
bbobj.img_id = fileobj.img_id
if bbobj.confidence > threshold:
topsy.append(bbobj)
return topsy
def get_strong_fps_single(detector, i, fileobj, threshold, mixcomp):
topsy = []
ag.info('{0} Farming {1}'.format(i, fileobj.img_id))
img = gv.img.load_image(fileobj.path)
grayscale_img = gv.img.asgray(img)
bbobjs = detector.detect_coarse(grayscale_img, fileobj=fileobj, mixcomps=[mixcomp], use_padding=False, use_scale_prior=False, cascade=True, discard_weak=True, more_detections=True, farming=True, save_samples=True)
for bbobj in bbobjs:
bbobj.img_id = fileobj.img_id
if bbobj.confidence > threshold:
topsy.append(bbobj)
return topsy
def theano_train_logistic(self, X, Y):
ag.info("Using theano to train splitting SVM/Logistics")
import time, random
class1, class2 = split_1vs2(10)
combined = zip(class1, class2)
rs = np.random.RandomState(self._settings.get('random_seeds', 0))
#sample_per_model = 360 * 30 * 27 / (self._part_shape[0] * self._part_shape[1] * X.shape[3])
print(X.shape)
time1 = time.time()
batch_size = np.sqrt(10 ** 9 / self._num_parts / 4 / 9 / 0.3 / self._part_shape[0] / self._part_shape[1] / X.shape[3]) * 3
sample_per_model = int(9 * batch_size * 0.3 * 0.7 // 100) * 100
num_batch = max(int(np.floor(X.shape[0] / batch_size)), 1)
print(sample_per_model, "sample_per_model")
print(batch_size, "batch_size")
print(num_batch, "num_batch")
clf = multiLogisticRegression(n_feature = self._part_shape[0] * self._part_shape[1] * X.shape[3], n_model = self._num_parts, sample_per_model=sample_per_model, trainingSteps = 1000)
for i in range(num_batch):
print(i, num_batch)
allPatches = ()
allPatches_label = ()
for j in range(self._num_parts):
patches, patches_label = get_color_patches_location(X[i * batch_size: min((i + 1) * batch_size, X.shape[0])], Y[i * batch_size : (i + 1) * batch_size], class1[j], class2[j],
locations_per_try=1, part_shape=self._part_shape,
sample_shape=(3, 3),fr = 1, randomseed=j + self._settings.get('random_seeds', 0),
threshold=0, max_samples=1000000)
#print(patches.shape[0])
print(patches.shape[0], sample_per_model)
assert patches.shape[0] >= sample_per_model
patches = patches[:sample_per_model].reshape((1,) + patches[:sample_per_model].shape)
patches_label = patches_label[:sample_per_model].reshape((1,) + patches_label[:sample_per_model].shape)
allPatches += ((patches,))
allPatches_label += ((patches_label,))
allPatches = np.vstack(allPatches)
allPatches_label = np.vstack(allPatches_label)
print(allPatches.shape)
#print(allPatches_label.shape)
clf.train(input_data=allPatches, input_label=allPatches_label)
self._coef = clf.W
self._intercept = clf.bias
time2 = time.time()
print 'theano_train function took %0.3f ms' % ((time2-time1)*1000.0)
print(np.max(self._coef))
print(np.mean(self._coef))
print(np.max(self._intercept))
print(np.mean(self._intercept))
def get_patches(args):
f, patch_size, samples_per_image, fr = args
the_patches = []
the_originals = []
ag.info("File", f)
edges, img = ag.features.bedges_from_image(
f, k=5, radius=1, minimum_contrast=0.05, contrast_insensitive=False, return_original=True
)
edges_nospread = ag.features.bedges_from_image(f, k=5, radius=0, minimum_contrast=0.05, contrast_insensitive=False)
# How many patches could we extract?
w, h = [edges.shape[i] - patch_size[i] + 1 for i in xrange(2)]
# if samples_per_image is None:
## Get all of them
# indices = range(w * h)
# else:
## Get samples from this
# indices = random.sample(xrange(w * h), samples_per_image)
# indices = range(w * h)
# random.shuffle(indices)
# positions = map(lambda index: (index%w, index/w), indices)
# ag.plot.images([img])
# for x, y in positions:
for sample in xrange(samples_per_image):
for tries in xrange(20):
x, y = random.randint(0, w - 1), random.randint(0, h - 1)
selection = [slice(x, x + patch_size[0]), slice(y, y + patch_size[1])]
# Return grayscale patch and edges patch
edgepatch = edges[selection]
edgepatch_nospread = edges_nospread[selection]
num = edgepatch[fr:-fr, fr:-fr].sum()
# num_edges.append(num)
if num >= 4:
the_patches.append(edgepatch_nospread)
vispatch = img[selection]
vispatch = vispatch[..., :3].mean(axis=vispatch.ndim - 1)
span = vispatch.min(), vispatch.max()
if span[1] - span[0] > 0:
vispatch = (vispatch - span[0]) / (span[1] - span[0])
the_originals.append(vispatch)
break
return the_patches, the_originals
def train_from_images(self, filenames):
raw_patches, raw_originals = self.random_patches_from_images(filenames)
mixture = ag.stats.BernoulliMixture(self.num_patches,
raw_patches,
init_seed=0)
# Also store these in "settings"
mixture.run_EM(1e-8, min_probability=self.settings['min_probability'])
ag.info("Done.")
# Store the stuff in the instance
self.patches = mixture.templates
self.vispatches = mixture.remix(raw_originals)
self._preprocess_logs()
def extract(self, X_all, Y=None, test_accuracy=False):
ag.info("randomPartition SVM start extracting")
outer_frame = self._settings.get("outer_frame", 0)
dim = (X_all.shape[1], X_all.shape[2])
if outer_frame != 0:
XX = np.zeros(
(X_all.shape[0], X_all.shape[1] + 2 * outer_frame, X_all.shape[2] + 2 * outer_frame, X_all.shape[3]),
dtype=np.float16,
)
XX[:, outer_frame : X_all.shape[1] + outer_frame, outer_frame : X_all.shape[2] + outer_frame, :] = X_all
X_all = XX
numcl = 2
if numcl == 2:
feature_map = np.zeros((X_all.shape[0],) + dim + (self._num_parts,), dtype=np.float16)
else:
feature_map = np.zeros((X_all.shape[0],) + dim + (numcl, self._num_parts), dtype=np.float16)
print ("Before blowing up the memory")
roundNumber = 100
numEachRound = X_all.shape[0] // roundNumber
for round in range(roundNumber):
print (round)
X = np.array(X_all[numEachRound * round : numEachRound * (round + 1)], dtype=np.float16)
X_num = X.shape[0]
import itertools
argList = list(itertools.product(range(dim[0]), range(dim[1])))
p = Pool(4)
args = (
(
x,
y,
self._coef,
self._intercept,
X[:, x : x + self._part_shape[0], y : y + self._part_shape[1], :]
.reshape(X_num, -1)
.astype(np.float16),
)
for (x, y) in argList
)
count = 0
for x, y, score in p.imap(calcScore, args):
feature_map[numEachRound * round : numEachRound * (round + 1), x, y, :] = score
count += 1
p.terminate()
print (feature_map[0, 0, 0, 0])
return (feature_map, self._num_parts)
def train(self, X, Y, OriginalX = None):
assert Y is not None
ag.info('Extracting patches')
print(np.any(np.isnan(X)))
print(np.all(np.isfinite(X)))
if(self._settings.get("all_locations",True)):
patches, patches_label, patches_original = self._get_color_patches(X,Y,OriginalX)
ag.info('Done extracting patches')
ag.info('Training patches', patches.shape)
ag.info('Training patches labels', patches_label.shape)
return self.train_from_samples(patches,patches_label, patches_original)
elif self._settings.get("use_theano", False):
return self.theano_train_logistic(X, Y)
else:
ag.info('Start multiple svm training: each svm is trained using patches from a specific location and a certain partition')
return self.split_train_svms(X,Y)
def get_patches(args):
f, patch_size, samples_per_image, fr = args
the_patches = []
the_originals = []
ag.info("File", f)
edges, img = ag.features.bedges_from_image(f, k=5, radius=1, minimum_contrast=0.05, contrast_insensitive=False, return_original=True)
edges_nospread = ag.features.bedges_from_image(f, k=5, radius=0, minimum_contrast=0.05, contrast_insensitive=False)
# How many patches could we extract?
w, h = [edges.shape[i]-patch_size[i]+1 for i in xrange(2)]
#if samples_per_image is None:
## Get all of them
#indices = range(w * h)
#else:
## Get samples from this
#indices = random.sample(xrange(w * h), samples_per_image)
#indices = range(w * h)
#random.shuffle(indices)
#positions = map(lambda index: (index%w, index/w), indices)
#ag.plot.images([img])
#for x, y in positions:
for sample in xrange(samples_per_image):
for tries in xrange(20):
x, y = random.randint(0, w-1), random.randint(0, h-1)
selection = [slice(x, x+patch_size[0]), slice(y, y+patch_size[1])]
# Return grayscale patch and edges patch
edgepatch = edges[selection]
edgepatch_nospread = edges_nospread[selection]
num = edgepatch[fr:-fr,fr:-fr].sum()
#num_edges.append(num)
if num >= 4:
the_patches.append(edgepatch_nospread)
vispatch = img[selection]
vispatch = vispatch[...,:3].mean(axis=vispatch.ndim-1)
span = vispatch.min(), vispatch.max()
if span[1] - span[0] > 0:
vispatch = (vispatch-span[0])/(span[1]-span[0])
the_originals.append(vispatch)
break
return the_patches, the_originals
def _get_patches(self, filename):
samples_per_image = self.settings['samples_per_image']
fr = self.settings['patch_frame']
the_patches = []
the_originals = []
ag.info("Extracting patches from", filename)
#edges, img = ag.features.bedges_from_image(f, k=5, radius=1, minimum_contrast=0.05, contrast_insensitive=False, return_original=True, lastaxis=True)
edges, img = ag.features.bedges_from_image(filename,
return_original=True,
**self.settings['bedges'])
s = self.settings['bedges'].copy()
if 'radius' in s:
del s['radius']
edges_nospread = ag.features.bedges_from_image(filename, radius=0, **s)
# How many patches could we extract?
w, h = [edges.shape[i] - self.patch_size[i] + 1 for i in range(2)]
# TODO: Maybe shuffle an iterator of the indices?
for sample in range(samples_per_image):
for tries in range(20):
x, y = random.randint(0, w - 1), random.randint(0, h - 1)
selection = [
slice(x, x + self.patch_size[0]),
slice(y, y + self.patch_size[1])
]
# Return grayscale patch and edges patch
edgepatch = edges[selection]
edgepatch_nospread = edges_nospread[selection]
num = edgepatch[fr:-fr, fr:-fr].sum()
if num >= self.settings['threshold']:
the_patches.append(edgepatch_nospread)
# The following is only for clearer visualization of the
# patches. However, normalizing like this might be misleading
# in other ways.
vispatch = img[selection]
vispatch = vispatch[..., :3].mean(axis=vispatch.ndim - 1)
span = vispatch.min(), vispatch.max()
if span[1] - span[0] > 0:
vispatch = (vispatch - span[0]) / (span[1] - span[0])
the_originals.append(vispatch)
break
return the_patches, the_originals
def get_fps(detector, i, fileobj, size):
ag.info('{0} Initial processing {1}'.format(i, fileobj.img_id))
gen = generate_random_patches([fileobj.path], size, per_image=PER_IMAGE)
neg_feats = []
radii = detector.settings['spread_radii']
psize = detector.settings['subsample_size']
rotspread = detector.settings.get('rotation_spreading_radius', 0)
cb = detector.settings.get('crop_border')
setts = dict(spread_radii=radii, subsample_size=psize, rotation_spreading_radius=rotspread, crop_border=cb)
for neg in itr.islice(gen, PER_IMAGE):
#images.append(neg)
feat = detector.descriptor.extract_features(neg, settings=setts)
neg_feats.append(feat)
return neg_feats
def extract(self,X_all, Y = None, test_accuracy = False):
ag.info("randomPartition SVM start extracting")
outer_frame = self._settings.get('outer_frame', 0)
dim = (X_all.shape[1],X_all.shape[2])
if outer_frame != 0:
XX = np.zeros((X_all.shape[0], X_all.shape[1] + 2 * outer_frame, X_all.shape[2] + 2 * outer_frame, X_all.shape[3]), dtype=np.float16)
XX[:, outer_frame:X_all.shape[1] + outer_frame, outer_frame:X_all.shape[2] + outer_frame,:] = X_all
X_all = XX
numcl = 2
print("Before blowing up the memory")
roundNumber = 100
numEachRound = X_all.shape[0] // roundNumber
feature_map_list = []
for round in range(roundNumber):
print(round)
X = np.array(X_all[numEachRound * round : numEachRound * (round + 1)], dtype = np.float16)
X_num = X.shape[0]
if(numcl == 2):
feature_map = np.zeros((X.shape[0],) + dim + (self._num_parts,),dtype=np.float16)
else:
feature_map = np.zeros((X.shape[0],) + dim + (numcl, self._num_parts,),dtype=np.float16)
import itertools
argList = list(itertools.product(range(dim[0]),range(dim[1])))
p = Pool(4)
args = ((x, y, self._coef,self._intercept,
X[:,x:x+self._part_shape[0],
y:y+self._part_shape[1],:].reshape(X_num,-1).astype(np.float16)) for(x,y) in argList
)
count = 0
for x, y, score in p.imap(calcScore, args):
feature_map[:,x, y, :] = score
count+=1
p.terminate()
# Start to do pooling
relu = self._settings.get("relu", True)
from pnet.cyfuncs import activation_map_pooling as poolf
feature_map_list.append(poolf(feature_map.astype(np.float32), self._num_parts, self._shape, self._strides, relu).astype(np.float16))
result_map = np.vstack(feature_map_list)
result_map.astype(np.float16)
print(np.any(np.isnan(result_map)))
print(np.all(np.isfinite(result_map)))
return result_map
def _get_patches(self, X):
#assert X.ndim == 3
samples_per_image = self._settings.get('samples_per_image', 20)
patches = []
rs = np.random.RandomState(self._settings.get('patch_extraction_seed', 0))
consecutive_failures = 0
for Xi in X:
# How many patches could we extract?
w, h = [Xi.shape[i]-self._part_shape[i]+1 for i in range(2)]
# TODO: Maybe shuffle an iterator of the indices?
indices = list(itr.product(range(w-1), range(h-1)))
rs.shuffle(indices)
i_iter = itr.cycle(iter(indices))
for sample in range(samples_per_image):
N = 200
for tries in range(N):
x, y = next(i_iter)
selection = [slice(x, x+self._part_shape[0]), slice(y, y+self._part_shape[1])]
patch = Xi[selection]
if patch.sum() > self._settings.get('threshold', 5):
patches.append(patch)
if len(patches) >= self._settings.get('max_samples', np.inf):
return np.asarray(patches)
consecutive_failures = 0
break
if tries == N-1:
ag.info('WARNING: {} tries'.format(N))
ag.info('cons', consecutive_failures)
consecutive_failures += 1
if consecutive_failures >= 200:
# Just give up.
raise ValueError("FATAL ERROR: Threshold is probably too high.")
return np.asarray(patches)
def train(self, X_n, Y=None, OriginalX = None):
assert Y is None
X = X_n[0]
num_parts = X_n[1]
num_orientations = X_n[2]
num_true_parts = num_parts // num_orientations
assert num_parts * self. _num_components == self._num_parts
assert num_orientations == self._rotation
assert X.ndim == 4
X = X.reshape((X.shape[0],X.shape[1],X.shape[2]))
ag.info('Extracting patches')
partsRegion, shape = self._extract_patches(X)
ag.info('Done extracting patches')
return self._train_patches(partsRegion,shape)
def _extract_patches(self, X):
ag.info("Extract Patches: Seperate coded regions into groups")
trainingDataNum = X.shape[0]
totalRange = X.shape[1]
frame = (self._part_shape[0] - self._lowerLayerShape[0]) / 2
frame = int(frame)
partsRegion = [[] for x in range(self._num_lower_parts)]
for i in range(trainingDataNum):
for m in range(totalRange)[frame : totalRange - frame]:
for n in range(totalRange)[frame : totalRange - frame]:
if X[i, m, n] != -1:
partsGrid = self.partsPool(
X[i, m - frame : m + frame + 1, n - frame : n + frame + 1], self._num_lower_parts
)
partsRegion[X[i, m, n]].append(partsGrid)
return partsRegion
def get_pos_and_neg(mixcomp, settings, bb, indices, files, neg_files, duplicates_mult=1):
im_size = settings['detector']['image_size']
size = gv.bb.size(bb)
# Use the same seed for all mixture components! That will make them easier to compare,
# without having to sample to infinity.
# HERE: Make it possible to input data directly!
descriptor = gv.load_descriptor(settings)
all_pos_feats = []
all_neg_feats = []
radii = settings['detector']['spread_radii']
psize = settings['detector']['subsample_size']
rotspread = settings['detector'].get('rotation_spreading_radius', 0)
duplicates = settings['detector'].get('duplicates', 1) * duplicates_mult
cb = settings['detector'].get('crop_border')
crop_image = settings['detector'].get('crop_image')
sett = dict(spread_radii=radii, subsample_size=psize, rotation_spreading_radius=rotspread, crop_border=cb, crop_image=crop_image)
extra = {}
if settings['detector'].get('selective_bkg'):
ag.info("SELECTIVE!")
extra['selective'] = True
extra['concentrations'] = _get_avg_positives(mixcomp, settings, bb, indices, files, neg_files, descriptor, sett)
alpha_maps = []
args = [(index, mixcomp, files, im_size, bb, duplicates, neg_files, descriptor, sett, extra) for index in indices]
#index, mixcomp, files, im_size, bb, duplicates):
for pos_feats, neg_feats, alpha in gv.parallel.starmap_unordered(__process_one, args):
all_pos_feats.extend(pos_feats)
all_neg_feats.extend(neg_feats)
alpha_maps.append(alpha)
all_neg_feats = np.asarray(all_neg_feats)
all_pos_feats = np.asarray(all_pos_feats)
#alpha_maps = np.asarray(alpha_maps)
#support = alpha_maps.mean(axis=0)
return mixcomp, all_neg_feats, all_pos_feats, alpha_maps, extra
def train(self, X, Y=None, OriginalX=None):
if len(X) == 2:
X_data, num_lower_parts = X
else:
X_data, num_real_part, orientation = X
num_lower_parts = num_real_part
assert Y is None
assert X_data.ndim == 4
assert num_lower_parts == self._num_lower_parts
X_data = X_data.reshape((X_data.shape[0], X_data.shape[1], X_data.shape[2]))
# Extract Patches
partsRegion = self._extract_patches(X_data)
# Train Patches
ag.info("Done extracting patches")
# ag.info('Training patches', patches.shape)
return self._train_patches(partsRegion)
def run_EM(self, tol, min_probability=0.05, debug_plot=False):
"""
Run the EM algorithm to specified convergence.
Parameters
----------
tol : float
The tolerance gives the stopping condition for convergence.
If the loglikelihood decreased with less than ``tol``, then it will break the loop.
min_probability : float
Disallow probabilities to fall below this value, and extend below one minus this value.
"""
#self._preload_log_templates()
self.min_probability = min_probability
loglikelihood = -np.inf
# First E step plus likelihood computation
self.M_step()
new_loglikelihood = self._compute_loglikelihoods()
if debug_plot:
plw = ag.plot.PlottingWindow(subplots=(1, self.num_mix),
figsize=(self.num_mix * 3, 3))
self.iterations = 0
while np.isinf(loglikelihood) or np.fabs(
(new_loglikelihood - loglikelihood) / loglikelihood) > tol:
ag.info("Iteration {0}: loglikelihood {1}".format(
self.iterations, loglikelihood))
loglikelihood = new_loglikelihood
# M-step
self.M_step()
# E-step
new_loglikelihood = self._compute_loglikelihoods()
self.iterations += 1
if debug_plot and not self._plot(plw):
raise ag.AbortException
self.set_templates()
def train(self, X, Y=None, OriginalX = None):
X_data, num_lower_parts = X
assert Y is None
assert X_data.ndim == 4
assert num_lower_parts == self._num_lower_parts
X_data = X_data.reshape((X_data.shape[0],X_data.shape[1],X_data.shape[2]))
#Extract Patches
partsRegion = self._extract_patches(X_data)
#Train Patches
ag.info('Done extracting patches')
#ag.info('Training patches', patches.shape)
self._classificationLayers = [[],[],[],[]]
for i in range(4):
self._train_patches(partsRegion[i],i)
def train_patches_from_image_files(self, filenames, samples_per_image=1000):
random.seed(0)
filenames_copy = filenames[:]
random.shuffle(filenames_copy)
raw_patches = []
raw_originals = []
fr = self.big_patch_frame
for file_i, f in enumerate(filenames_copy):
ag.info(file_i, "File", f)
edges, img = ag.features.bedges_from_image(f, k=5, radius=1, minimum_contrast=0.05, contrast_insensitive=True, return_original=True)
# Make grayscale
imggray = img[...,:3].mean(axis=2)
for impatch, edgepatch in self._gen_patches(imggray, edges, self.patch_size):
if edgepatch[fr:-fr,fr:-fr].sum() >= 20:
raw_originals.append(impatch)
raw_patches.append(edgepatch)
raw_patches = np.asarray(raw_patches)
raw_originals = np.asarray(raw_originals)
mixture = ag.stats.BernoulliMixture(self.K, raw_patches, init_seed=0)
mixture.run_EM(1e-4, min_probability=0.01, debug_plot=False)
# Store the stuff in the instance
self.patches = mixture.templates
self.vispatches = mixture.remix(raw_originals)
self.vispatches /= self.vispatches.max()
#self.info = {
# 'K': self.K,
# 'patch_size': self.patch_size,
# 'big_patch_frame': self.big_patch_frame,
#}
return self.patches
def get_strong_fps(detector, i, fileobj, threshold, mixcomp):
topsy = [[] for k in xrange(detector.num_mixtures)]
#for i, fileobj in enumerate(itr.islice(gen, COUNT)):
ag.info('{0} Farming {1}'.format(i, fileobj.img_id))
#img = gv.img.load_image(img_fn)
img = gv.img.load_image(fileobj.path)
grayscale_img = gv.img.asgray(img)
for m in xrange(detector.num_mixtures):
bbobjs = detector.detect_coarse(grayscale_img, fileobj=fileobj, mixcomps=[m], use_padding=False, use_scale_prior=False, cascade=True, discard_weak=True, more_detections=True)
# Add in img_id
for bbobj in bbobjs:
bbobj.img_id = fileobj.img_id
#array.append(bbobj.X)
if bbobj.confidence > threshold:
#if len(topsy[m]) < TOP_N:
# heapq.heappush(topsy[m], bbobj)
#else:
# heapq.heappushpop(topsy[m], bbobj)
topsy[m].append(bbobj)
return topsy
def train_from_images(self, filenames):
ag.info('Fetching training images for learning parts')
def size_ok(im):
return np.min(im.shape[:2]) >= 300
def crop(im):
return gv.img.crop(im, (300, 300))
ims = list(
map(
crop,
list(
filter(size_ok, [
gv.img.asgray(gv.img.load_image(fn))
for fn in filenames
]))))
ims = np.asarray(ims)
ag.info('Training images shape', ims.shape)
ag.info('Training parts')
self._net.train(ims)
self._num_parts = self._net.layers[1].num_parts
def fromarray(cls, *args, **kwargs):
ag.info('Deprecation warning: Use ImageGrid(...) instead of '
'ImageGrid.fromarray(...)')
return cls(*args, **kwargs)
def fit(self, X, tol=0.00001, min_probability=0.01, min_q=0.01):
Xsafe = np.clip(X, min_probability, 1 - min_probability)
self._init(X, seed=0)
M = self._n_clusters
N = X.shape[0]
D = X.shape[1]
qlogs = np.zeros((M, N))
q = np.zeros((M, N))
v = np.zeros(N)
loglikelihood = -np.inf
new_loglikelihood = self._compute_loglikelihoods(Xsafe)
self.iterations = 0
while self.iterations < 200 and (
np.isinf(loglikelihood) or self.iterations < 2 or np.fabs(
(new_loglikelihood - loglikelihood) / loglikelihood) > tol
):
loglikelihood = new_loglikelihood
ag.info("Iteration {0}: loglikelihood {1}".format(
self.iterations, loglikelihood))
for m in range(M):
v = qlogs[m]
v[:] = 0.0
for d in range(D):
#print beta.logpdf(Xsafe[:,d], self.theta_[m,d,0], self.theta_[m,d,1])
v += beta.logpdf(Xsafe[:, d], self.theta_[m, d, 0],
self.theta_[m, d, 1])
qlogs[m] = v
#print v.min(), v.max()
#try:
try:
q[:] = np.exp(
np.maximum(np.log(min_q),
qlogs - logsumexp(qlogs, axis=0)))
except:
pass
#except:
# pass
# Clip it, for saftey
#print q.min(), q.max()
q[:] = np.clip(q, min_q, 1 - min_q)
# Update labels from these responsibilities
self.labels_ = q.argmax(axis=0)
# Update thetas with the new labels
if 0:
for m in range(M):
for d in range(D):
Xsafem = Xsafe[self.labels_ == m, d]
sm, sv = weighted_avg_and_var(Xsafe[:, d], q[m])
#sm = np.mean(Xsafem)
#sv = np.var(Xsafem)
self.theta_[m, d, 0] = sm * (sm * (1 - sm) / sv - 1)
self.theta_[m, d,
1] = (1 - sm) * (sm * (1 - sm) / sv - 1)
if np.isnan(self.theta_[m, d, 0]) or np.isnan(
self.theta_[m, d, 1]):
import pdb
pdb.set_trace()
raise Exception()
else:
for m in range(M):
for d in range(D):
#from scipy.optimize import newton_krylov, nonlin
from scipy.special import psi
Ca = np.average(np.log(Xsafe[:, d]), weights=q[m])
Cb = np.average(np.log(1 - Xsafe[:, d]), weights=q[m])
a, b = self.theta_[m, d]
#self.theta_[m,d,0] = newton_krylov(lambda x: (psi(x) - psi(x+b)) - Ca, 1.0)
self.theta_[m, d, 0] = binary_search(
lambda x: (psi(x) - psi(x + b)) - Ca,
0.0001,
10000.0,
maxiter=20)
self.theta_[m, d, 1] = binary_search(
lambda x: (psi(x) - psi(x + a)) - Cb,
0.0001,
10000.0,
maxiter=20)
# Make sure the alpha and the beta don't get too extreme. If one needs adjusting, we need
# adjust both, to preserve its mean
#C = np.average(
#self.theta_ = np.clip(self.theta_, 0.1, 100.0)
# Calculate log-likelihood
new_loglikelihood = self._compute_loglikelihoods(Xsafe)
self.iterations += 1
ag.info("Iteration DONE: loglikelihood {}".format(new_loglikelihood))
if 1:
# Now fit constrained betas using this distribution
#params = np.asarray([(b+a, b+c-a) for b in np.linspace(1, 2, 5) for c in np.linspace(1, 50, 10) for a in np.linspace(0, c, P)])
#def fit_beta(cls, X):
#import pdb; pdb.set_trace()
for m in range(M):
Xm = Xsafe[self.labels_ == m]
self.theta_[m] = self.fit_beta_atleast_std(Xm, 0.225)
#for m in xrange(M):
# for d in xrange(D):
if 0:
for m in range(M):
for d in range(D):
if self.theta_[m, d].max() > 50:
self.theta_[m, d] /= self.theta_[m, d].max() / 50
def fit(self, X):
N, P, F = X.shape
assert P == len(self.permutations)
K = self.n_components
eps = self.min_probability
#theta = self.random_state.uniform(eps, 1 - eps, size=(K, P, F))
all_pi = [] # np.zeros((self.n_init, K, P))
all_mu = []
all_loglikelihoods = []
#import scipy.sparse
#X = scipy.sparse.csr_matrix(X)
for trial in range(self.n_init):
pi = np.ones((K, P)) / (K * P)
# Initialize
clusters = self.random_state.randint(K, size=N)
theta = np.asarray(
[np.mean(X[clusters == k], axis=0) for k in range(K)])
theta[:] = np.clip(theta, eps, 1 - eps)
# TEMP
#nice_theta = theta.reshape((theta.shape[0]*theta.shape[1], 6, 6, 8))
#nice_theta = np.rollaxis(nice_theta, 3, 1)
#vz.image_grid(nice_theta, vmin=0, vmax=1, cmap=vz.cm.RdBu_r, name='iter0', scale=4)
calc_loglikelihood = True
self.q = np.empty((N, K, P))
logq = np.empty((N, K, P))
loglikelihood = None
for loop in range(self.n_iter):
if calc_loglikelihood:
ag.info("Iteration ", loop + 1, 'log-likelihood',
loglikelihood)
else:
ag.info("Iteration ", loop + 1)
#logq[:] = np.log(pi)[np.newaxis,:,np.newaxis]
#for k, p in itr.product(xrange(K), xrange(P)):
# logq[:,k,p] = np.log(pi[k,p])
logq[:] = np.log(pi[np.newaxis])
for p in range(P):
for shift in range(P):
#p0_ = (p + shift)%P
p0 = self.permutations[shift, p]
#assert p0 == p0_, self.permutations
#import pdb; pdb.set_trace()
logq[:, :, p] += np.dot(
X[:, p0],
logit(theta[:, shift]).T) + np.log(
1 - theta[:, shift]).sum(axis=1)[np.newaxis]
#self.q[:] = np.exp(logq)
#normq = self.q / np.apply_over_axes(np.sum, self.q, [1, 2])
#self.q /= np.apply_over_axes(np.sum, self.q, [1, 2])
#q2 = np.exp(logq - logsumexp(logq.reshape((-1, logq.shape[-1])), axis=0)[...,np.newaxis,np.newaxis])
norm_logq = (logq - logsumexp(
logq.reshape((logq.shape[0], -1)),
axis=-1)[..., np.newaxis, np.newaxis]).clip(min=-200)
q2 = np.exp(norm_logq)
self.q[:] = q2
# Try regularizing a bit
#self.q[:] = self.q.clip(min=1e-4)
#self.q[:] /= np.apply_over_axes(np.sum, self.q, [1, 2])
#norm_logq = logq - logsumexp(logq.reshape((logq.shape[0], -1)), axis=-1)[...,np.newaxis,np.newaxis]
#self.q[:] = np.exp(norm_logq)
#dens = np.apply_over_axes(np.sum, self.q, [0, 2])
log_dens = logsumexp(np.rollaxis(norm_logq, 2, 1).reshape(
(-1, norm_logq.shape[1])),
axis=0)[np.newaxis, :, np.newaxis]
dens = np.exp(log_dens)
for p in range(P):
v = 0 #np.dot(self.q[:,:,0].T, X[:,0]) + np.dot(self.q[:,:,1].T, X[:,1])
for shift in range(P):
#p0_ = (p + shift)%P
p0 = self.permutations[shift, p]
#assert p0 == p0_, self.permutations
v += np.dot(self.q[:, :, shift].T, X[:, p0])
theta[:, p, :] = v
np.seterr(all='raise')
#try:
#vz.image_grid(self.q[:100], vmin=0, vmax=1, cmap=vz.cm.RdBu_r, name='plot')
#import IPython; IPython.embed()
#import pdb; pdb.set_trace()
theta = theta / dens.ravel()[:, np.newaxis, np.newaxis]
# TODO: Shape is hard coded here
#nice_theta = theta.reshape((theta.shape[0]*theta.shape[1], 6, 6, 8))
#nice_theta = np.rollaxis(nice_theta, 3, 1)
#vz.image_grid(nice_theta, vmin=0, vmax=1, cmap=vz.cm.RdBu_r, name='iter{}'.format(loop+1), scale=4)
#new parts
#pi[:] = np.apply_over_axes(np.sum, self.q, [0, 2])[0,:,0] / N
pi[:] = np.apply_over_axes(np.sum, self.q, [0])[0, :, :] / N
pi[:] = np.clip(pi, 0.0001, 1 - 0.0001)
# TODO: KEEP THIS?
#pi[:] = np.ones(pi.shape) / pi.shape
theta[:] = np.clip(theta, eps, 1 - eps)
#pi = np.clip(pi, eps, 1 - eps)
# Calculate log likelihood
if calc_loglikelihood:
loglikelihood = logsumexp(logq)
#self.weights_ = pi
#self.means_ = theta
all_pi.append(pi)
all_mu.append(theta)
all_loglikelihoods.append(loglikelihood)
print(all_loglikelihoods)
best_i = np.argmax(all_loglikelihoods)
self.weights_ = all_pi[best_i]
self.means_ = all_mu[best_i]
def _process_file(settings, bkg_stack, bkg_stack_num, fn, mixcomp):
ag.info("Processing file", fn)
seed = np.abs(hash(fn) % 123124)
descriptor_name = settings['detector']['descriptor']
img_size = settings['detector']['image_size']
part_size = settings[descriptor_name]['part_size']
psize = settings['detector']['subsample_size']
# The 4 is for the edge border that falls off
#orig_sh = (img_size[0] - part_size[0] - 4 + 1, img_size[1] - part_size[1] - 4 + 1)
orig_sh = img_size
sh = gv.sub.subsample_size(np.ones(orig_sh), psize)
# We need the descriptor to generate and manipulate images
descriptor = gv.load_descriptor(settings)
counts = np.zeros((settings['detector']['num_mixtures'], sh[0], sh[1],
descriptor.num_parts + 1, descriptor.num_parts),
dtype=np.uint16)
prnds = [np.random.RandomState(seed + i) for i in xrange(5)]
# Binarize support and Extract alpha
#color_img, alpha = gv.img.load_image_binarized_alpha(fn)
color_img = gv.img.load_image(fn)
from skimage.transform import pyramid_reduce, pyramid_expand
f = color_img.shape[0] / settings['detector']['image_size'][0]
if f > 1:
color_img = pyramid_reduce(color_img, downscale=f)
elif f < 1:
color_img = pyramid_expand(color_img, upscale=1 / f)
alpha = color_img[..., 3]
img = gv.img.asgray(color_img)
# Resize it
# TODO: This only looks at the first axis
assert img.shape == settings['detector'][
'image_size'], "Target size not achieved: {0} != {1}".format(
img.shape, settings['detector']['image_size'])
# Settings
bsettings = settings['edges'].copy()
radius = bsettings['radius']
bsettings['radius'] = 0
#offsets = gv.sub.subsample_offset_shape(sh, psize)
#locations0 = xrange(offsets[0], sh[0], psize[0])
#locations1 = xrange(offsets[1], sh[1], psize[1])
locations0 = xrange(sh[0])
locations1 = xrange(sh[1])
#locations0 = xrange(10-4, 10+5)
#locations1 = xrange(10-4, 10+5)
#locations0 = xrange(10, 11)
#locations1 = xrange(10, 11)
#padded_theta = descriptor.unspread_parts_padded
#pad = 10
pad = 5
size = settings[descriptor_name]['part_size']
X_pad_size = (size[0] + pad * 2, size[1] + pad * 2)
img_pad = ag.util.zeropad(img, pad)
alpha_pad = ag.util.zeropad(alpha, pad)
# Iterate every duplicate
dups = settings['detector'].get('duplicates', 1)
bkgs = np.empty(((descriptor.num_parts + 1) * dups, ) + X_pad_size)
#cads = np.empty((descriptor.num_parts,) + X_pad_size)
#alphas = np.empty((descriptor.num_parts,) + X_pad_size, dtype=np.bool)
radii = settings['detector']['spread_radii']
psize = settings['detector']['subsample_size']
cb = settings['detector'].get('crop_border')
sett = dict(spread_radii=radii, subsample_size=psize, crop_border=cb)
plt.clf()
plt.imshow(img)
plt.savefig('output/img.png')
if 0:
# NEW{
totfeats = np.zeros(sh + (descriptor.num_parts, ) * 2)
for f in xrange(descriptor.num_parts):
num = bkg_stack_num[f]
for d in xrange(dups):
feats = np.zeros(sh + (descriptor.num_parts, ), dtype=np.uint8)
for i, j in itr.product(locations0, locations1):
x = i * psize[0]
y = i * psize[1]
bkg_i = prnds[4].randint(num)
bkg = bkg_stack[f, bkg_i]
selection = [
slice(x, x + X_pad_size[0]),
slice(y, y + X_pad_size[1])
]
#X_pad = edges_pad[selection].copy()
patch = img_pad[selection]
alpha_patch = alpha_pad[selection]
#patch = np.expand_dims(patch, 0)
#alpha_patch = np.expand_dims(alpha_patch, 0)
# TODO: Which one?
#img_with_bkg = patch + bkg * (1 - alpha_patch)
img_with_bkg = patch * alpha_patch + bkg * (1 -
alpha_patch)
edges_pads = ag.features.bedges(img_with_bkg, **bsettings)
X_pad_spreads = ag.features.bspread(
edges_pads, spread=bsettings['spread'], radius=radius)
padding = pad - 2
X_spreads = X_pad_spreads[padding:-padding:,
padding:-padding]
partprobs = ag.features.code_parts(
X_spreads, descriptor._log_parts,
descriptor._log_invparts,
descriptor.settings['threshold'],
descriptor.settings['patch_frame'])
part = partprobs.argmax()
if part > 0:
feats[i, j, part - 1] = 1
# Now spread the parts
feats = ag.features.bspread(feats, spread='box', radius=2)
totfeats[:, :, f] += feats
# }
kernels = totfeats[:, :, 0].astype(
np.float32) / (descriptor.num_parts * dups)
# Subsample kernels
sub_kernels = gv.sub.subsample(kernels, psize, skip_first_axis=False)
np.save('tmp2.npy', sub_kernels)
print 'saved tmp2.npy'
import sys
sys.exit(0)
#ag.info("Iteration {0}/{1}".format(loop+1, num_duplicates))
#ag.info("Iteration")
for i, j in itr.product(locations0, locations1):
x = i * psize[0]
y = i * psize[1]
print 'processing', i, j
selection = [slice(x, x + X_pad_size[0]), slice(y, y + X_pad_size[1])]
#X_pad = edges_pad[selection].copy()
patch = img_pad[selection]
alpha_patch = alpha_pad[selection]
patch = np.expand_dims(patch, 0)
alpha_patch = np.expand_dims(alpha_patch, 0)
for f in xrange(descriptor.num_parts + 1):
num = bkg_stack_num[f]
for d in xrange(dups):
bkg_i = prnds[4].randint(num)
bkgs[f * dups + d] = bkg_stack[f, bkg_i]
img_with_bkgs = patch * alpha_patch + bkgs * (1 - alpha_patch)
if 0:
edges_pads = ag.features.bedges(img_with_bkgs, **bsettings)
X_pad_spreads = ag.features.bspread(edges_pads,
spread=bsettings['spread'],
radius=radius)
padding = pad - 2
X_spreads = X_pad_spreads[:, padding:-padding:, padding:-padding]
#partprobs = ag.features.code_parts_many(X_spreads, descriptor._log_parts, descriptor._log_invparts,
#descriptor.settings['threshold'], descriptor.settings['patch_frame'])
#parts = partprobs.argmax(axis=-1)
parts = np.asarray([
descriptor.extract_features(im, settings=sett)[0, 0]
for im in img_with_bkgs
])
for f in xrange(descriptor.num_parts + 1):
hist = np.bincount(parts[f * dups:(f + 1) * dups].ravel(),
minlength=descriptor.num_parts + 1)
counts[mixcomp, i, j, f] += hist[1:]
#import pdb; pdb.set_trace()
#for f in xrange(descriptor.num_parts):
# for d in xrange(dups):
# # Code parts
# #parts = descriptor.extract_parts(X_spreads[f*dups+d].astype(np.uint8))
#
# f_plus = parts[f*dups+d]
# if f_plus > 0:
#tau = self.settings.get('tau')
#if self.settings.get('tau'):
#parts = partprobs.argmax(axis=-1)
# Accumulate and return
# counts[mixcomp,i,j,f,f_plus-1] += 1#parts[0,0]
if 0:
kernels = counts[:, :, :, 0].astype(
np.float32) / (descriptor.num_parts * dups)
import pdb
pdb.set_trace()
radii = (2, 2)
aa_log = np.log(1 - kernels)
aa_log = ag.util.zeropad(aa_log, (0, radii[0], radii[1], 0))
integral_aa_log = aa_log.cumsum(1).cumsum(2)
offsets = gv.sub.subsample_offset(kernels[0], psize)
if 1:
# Fix kernels
istep = 2 * radii[0]
jstep = 2 * radii[1]
sh = kernels.shape[1:3]
for mixcomp in xrange(1):
# Note, we are going in strides of psize, given a certain offset, since
# we will be subsampling anyway, so we don't need to do the rest.
for i in xrange(offsets[0], sh[0], psize[0]):
for j in xrange(offsets[1], sh[1], psize[1]):
p = gv.img.integrate(integral_aa_log[mixcomp], i, j,
i + istep, j + jstep)
kernels[mixcomp, i, j] = 1 - np.exp(p)
# Subsample kernels
sub_kernels = gv.sub.subsample(kernels, psize, skip_first_axis=True)
np.save('tmp.npy', sub_kernels)
print 'saved tmp.npy'
import sys
sys.exit(0)
if 0:
for f in xrange(descriptor.num_parts):
# Pick only one background for this part and file
num = bkg_stack_num[f]
# Assumes num > 0
bkg_i = prnds[4].randint(num)
bkgmap = bkg_stack[f, bkg_i]
# Composite
img_with_bkg = gv.img.composite(patch, bkgmap, alpha_patch)
# Retrieve unspread edges (with a given background gray level)
edges_pad = ag.features.bedges(img_with_bkg, **bsettings)
# Pad the edges
#edges_pad = ag.util.zeropad(edges, (pad, pad, 0))
# Do spreading
X_pad_spread = ag.features.bspread(edges_pad,
spread=bsettings['spread'],
radius=radius)
# De-pad
padding = pad - 2
X_spread = X_pad_spread[padding:-padding, padding:-padding]
# Code parts
parts = descriptor.extract_parts(X_spread.astype(np.uint8))
# Accumulate and return
counts[mixcomp, i, j, f] += parts[0, 0]
# Translate counts to spread counts (since we're assuming independence of samples within one CAD image)
return counts
def bernoulli_deformation(F,
I,
last_level=None,
penalty=1.0,
tol=0.001,
rho=2.0,
wavelet='db2',
maxiter=50,
start_level=1,
means=None,
variances=None,
debug_plot=False):
assert F.ndim == 3, "F must have 3 axes"
assert F.shape == I.shape, "F and I shape mismatch {0} {1}".format(
F.shape, I.shape)
assert F.shape[
0] == 8, "Currently only supports 8 features (not {0})".format(
F.shape[0])
#from scipy.optimize import fmin_bfgs
from amitgroup.stats.fmin_bfgs_tol import fmin_bfgs_tol as fmin_bfgs
# This, or an assert
X = I.astype(float)
all_js = range(8)
# We don't need more capacity for coefficients than the last_level
level_capacity = last_level
delFjs = []
for j in all_js:
delF = np.gradient(F[j], 1 / F[j].shape[0], 1 / F[j].shape[1])
# Normalize since the image covers the square around [0, 1].
delFjs.append(delF)
delFjs = np.rollaxis(np.asarray(delFjs), 1)
settings = dict(
penalty=penalty,
wavelet=wavelet,
rho=rho,
level_capacity=level_capacity,
means=means,
variances=variances,
)
imdef = ag.util.DisplacementFieldWavelet(F.shape[1:], **settings)
x, y = imdef.meshgrid()
if debug_plot:
plw = ag.plot.PlottingWindow(figsize=(8, 8), subplots=(4, 4))
def cb(uk):
if not plw.tick(1):
raise ag.AbortException()
for j in range(8):
plw.imshow(imdef.deform(F[j]), subplot=j * 2)
plw.imshow(I[j], subplot=j * 2 + 1)
else:
cb = None
min_cost = np.inf
for level in range(start_level, last_level + 1):
ag.info("Running coarse-to-fine level", level)
imdef.reset(level)
u = imdef.u
args = (imdef, F, X, 1 - X, delFjs, x, y, level, all_js)
try:
new_u, cost, min_deriv, Bopt, func_calls, grad_calls, warnflag = \
fmin_bfgs(_cost, u, _cost_deriv, args=args, callback=cb, tol=tol, maxiter=maxiter, full_output=True, disp=False)
except ag.AbortException:
return None, {}
if cost < min_cost:
# If the algorithm makes mistakes and returns a really high cost, don't use it.
min_cost = cost
#imdef.u[:,:u.shape[1],:u.shape[2]] = new_u.reshape(u.shape)
imdef.u = new_u.reshape(imdef.u.shape)
#if debug_plot:
# plw.mainloop()
return imdef, {'cost': min_cost}
def image_deformation(F, I, last_level=3, penalty=1.0, rho=2.0, wavelet='db2', tol=1e-5, \
maxiter=5, start_level=1, means=None, variances=None, llh_variances=1.0, debug_plot=False):
"""
Deforms an a prototype image `F` into a data image `I` using a Daubechies wavelet basis and maximum a posteriori.
Parameters
----------
F : ndarray
Prototype image. Array of shape ``(W, H)`` with normalized intensitites. Both `W` and `H` must be powers of two.
I : ndarray
Data image that the prototype will try to deform into. Array of shape ``(W, H)``.
last_level : int, optional
Coefficient depth limit. If `None`, then naturally bounded by image size.
A higher level will allow for finer deformations, but incur a computational overhead.
penalty : float, optional
Determines the weight of the prior as opposed to the likelihood. (arbitrarily proportional to the ratio of the inverse variance of the gaussian deformations of the prior and the likelihood). Reduce this value if you want more deformations.
rho : float, optional
Determines the penalty of more granular coefficients. Increase to smoothen. Must be strictly positive.
wavelet : str
Wavelet type. See :class:`DisplacementFieldWavelet` for more information.
maxiter : int, optional
Maximum number of iterations per level.
tol : float, optional
Cost change must be less than `tol` before succesful termination at each coarse-to-fine level.
first_level : int, optional
First coarse-to-fine coefficient level.
means : ndarray or None, optional
Manually specify the means of the prior coefficients. If this and `variances` are set, then `penalty` and `rho` are unused. Must be of size ``(2, C)``, where `C` is the number of coefficients for the wavelet.
variances : ndarray or None, optional
Analagous to `means`, for specifying variances of the prior coefficients. Size should be the same as for `means`.
llh_variances : ndarray or scalar, optional
Specify log-likelihood variances per-pixel as a ``(W, H)`` array, or overall as a scalar. Default is 1.0, which can be seen as the variance being absorbed by the variance of the prior.
debug_plot : bool, optional
Output deformation progress live using :class:`PlottingWindow`.
Returns
-------
imdef : DisplacementFieldWavelet
The deformation in the form of a :class:`DisplacementField`.
info : dict
Dictionary with info:
* `cost`: The final cost value.
Examples
--------
Deform an image into a prototype image:
>>> import amitgroup as ag
>>> import numpy as np
>>> import matplotlib.pylab as plt
Load two example faces and perform the deformation:
>>> F, I = ag.io.load_example('two-faces')
>>> imdef, info = ag.stats.image_deformation(F, I, penalty=0.1)
>>> Fdef = imdef.deform(F)
Output the results:
>>> ag.plot.deformation(F, I, imdef)
>>> plt.show()
"""
assert rho > 0, "Parameter rho must be strictly positive"
assert len(F.shape) == 2 and len(
I.shape) == 2, "Images must be 2D ndarrays"
assert _powerof2(F.shape[0]) and _powerof2(
I.shape[1]), "Image sides must be powers of 2"
assert F.shape == I.shape, "Images must have the same shape"
#from scipy.optimize import fmin_bfgs
from amitgroup.stats.fmin_bfgs_tol import fmin_bfgs_tol as fmin_bfgs
level_capacity = last_level
delF = np.asarray(np.gradient(F, 1 / F.shape[0], 1 / F.shape[1]))
dx = 1 / np.prod(F.shape)
settings = dict(
penalty=penalty,
rho=rho,
wavelet=wavelet,
level_capacity=level_capacity,
means=means,
variances=variances,
)
imdef = ag.util.DisplacementFieldWavelet(F.shape, **settings)
x, y = imdef.meshgrid()
if debug_plot:
plw = ag.plot.PlottingWindow(figsize=(8, 4), subplots=(1, 2))
def cb(uk):
if not plw.tick(1):
raise ag.AbortException()
plw.imshow(imdef.deform(F), subplot=0)
plw.imshow(I, subplot=1)
else:
cb = None
min_cost = np.inf
for level in range(start_level, last_level + 1):
ag.info("Running coarse-to-fine level", level)
u = imdef.abridged_u(level)
args = (imdef, F, I, delF, x, y, level, llh_variances)
try:
new_u, cost, min_deriv, Bopt, func_calls, grad_calls, warnflag = \
fmin_bfgs(_cost, u, _cost_deriv, args=args, callback=cb, tol=tol, maxiter=maxiter, full_output=True, disp=False)
except ag.AbortException:
return None, {}
if cost < min_cost:
# If the algorithm makes mistakes and returns a really high cost, don't use it.
min_cost = cost
imdef.u[:, :u.shape[1], :u.shape[2]] = new_u.reshape(u.shape)
#if debug_plot:
# plw.mainloop()
return imdef, {'cost': min_cost}
def train_from_images(self, filenames):
raw_patches, raw_unspread_patches, raw_unspread_patches_padded, raw_originals = self.random_patches_from_images(
filenames)
if len(raw_patches) == 0:
raise Exception(
"No patches found, maybe your thresholds are too strict?")
# Also store these in "settings"
mixtures = []
llhs = []
for i in range(1):
mixture = ag.stats.BernoulliMixture(self.num_parts,
raw_patches,
init_seed=0 + i)
mixture.run_EM(1e-8,
min_probability=self.settings['min_probability'])
mixtures.append(mixture)
llhs.append(mixture.loglikelihood)
best_i = np.argmax(llhs)
mixture = mixtures[best_i]
ag.info("Done.")
counts = np.bincount(mixture.mixture_components(),
minlength=self.num_parts)
print(counts)
print('Total', np.sum(counts))
from scipy.stats.mstats import mquantiles
print(mquantiles(counts))
# Reject weak parts
scores = np.empty(self.num_parts)
for i in range(self.num_parts):
part = mixture.templates[i]
sh = part.shape
p = part.reshape((sh[0] * sh[1], sh[2]))
pec = p.mean(axis=0)
N = np.sum(p * np.log(p / pec) + (1 - p) * np.log((1 - p) /
(1 - pec)))
D = np.sqrt(
np.sum(np.log(p / pec * (1 - pec) / (1 - p))**2 * p * (1 - p)))
# Old:
#D = np.sqrt(np.sum(np.log(p/(1-p))**2 * p * (1-p)))
scores[i] = N / D
# Require at least 20 occurrences
#if counts[i] < 5:
#scores[i] = 0
# Only keep with a certain score
if not self.settings['bedges']['contrast_insensitive']:
visparts = mixture.remix(raw_originals)
else:
visparts = np.empty((self.num_parts, ) + raw_originals.shape[1:])
#self.extra['originals'] = []
# Improved visparts
comps = mixture.mixture_components()
for i in range(self.num_parts):
ims = raw_originals[comps == i].copy()
#self.extra['originals'].append(ims)
# Stretch them all out
#for j in xrange(len(ims)):
#ims[j] = (ims[j] - ims[j].min()) / (ims[j].max() - ims[j].min())
# Now, run a GMM with NM components on this and take the most common
NM = 2
from sklearn.mixture import GMM
gmix = GMM(n_components=NM)
gmix.fit(ims.reshape((ims.shape[0], -1)))
visparts[i] = gmix.means_[gmix.weights_.argmax()].reshape(
ims.shape[1:])
# Unspread parts
unspread_parts_all = mixture.remix(raw_unspread_patches)
unspread_parts_padded_all = mixture.remix(raw_unspread_patches_padded)
# The parts to keep
ok = (scores > 1) & (counts >= 10)
#if 'originals' in self.extra:
#self.extra['originals'] = list(itr.compress(self.extra['originals'], ok))
scores = scores[ok]
counts = counts[ok]
self.parts = mixture.templates[ok]
self.unspread_parts = unspread_parts_all[ok]
self.unspread_parts_padded = unspread_parts_padded_all[ok]
self.visparts = visparts[ok]
self.num_parts = self.parts.shape[0]
# Update num_parts
# Store the stuff in the instance
#self.parts = mixture.templates
#self.visparts = mixture.remix(raw_originals)
# Sort the parts according to orientation, for better diagonistics
if 1:
E = self.parts.shape[-1]
E = self.parts.shape[-1]
ang = np.array([[0, -1], [1, -1], [1, 0], [1, 1], [0, 1], [-1, 1],
[-1, 0], [-1, 1]])
nang = ang / np.expand_dims(np.sqrt(ang[:, 0]**2 + ang[:, 1]**2),
1)
orrs = np.apply_over_axes(np.mean, self.parts, [1, 2]).reshape(
(self.num_parts, -1))
if E == 8:
orrs = orrs[..., :4] + orrs[..., 4:]
nang = nang[:4]
norrs = orrs / np.expand_dims(orrs.sum(axis=1), 1)
dirs = (np.expand_dims(norrs, -1) * nang).sum(axis=1)
self.orientations = np.asarray(
[math.atan2(x[1], x[0]) for x in dirs])
II = np.argsort(self.orientations)
II = np.argsort(scores)
scores = scores[II]
counts = counts[II]
self.extra['scores'] = scores
self.extra['counts'] = counts
#self.extra['originals'] = [self.extra['originals'][ii] for ii in II]
# Now resort the parts according to this sorting
self.orientations = self.orientations[II]
self.parts = self.parts[II]
self.unspread_parts = self.unspread_parts[II]
self.unspread_parts_padded = self.unspread_parts_padded[II]
self.visparts = self.visparts[II]
self._preprocess_logs()
def _get_patches(self, filename):
samples_per_image = self.settings['samples_per_image']
fr = self.settings['patch_frame']
the_patches = []
the_unspread_patches = []
the_unspread_patches_padded = []
the_originals = []
ag.info("Extracting patches from", filename)
#edges, img = ag.features.bedges_from_image(f, k=5, radius=1, minimum_contrast=0.05, contrast_insensitive=False, return_original=True, lastaxis=True)
setts = self.settings['bedges'].copy()
radius = setts['radius']
setts['radius'] = 0
setts['preserve_size'] = True
# LEAVE-BEHIND
if 1:
img = gv.img.load_image(filename)
img = gv.img.asgray(img)
unspread_edges = ag.features.bedges(img, **setts)
else:
unspread_edges, img = ag.features.bedges_from_image(
filename, return_original=True, **self.settings['bedges'])
unspread_edges_padded = ag.util.zeropad(unspread_edges,
(radius, radius, 0))
# Spread the edges
edges = ag.features.bspread(unspread_edges,
spread=setts['spread'],
radius=radius)
#s = self.settings['bedges'].copy()
#if 'radius' in s:
# del s['radius']
#edges_nospread = ag.features.bedges_from_image(filename, radius=0, **s)
# How many patches could we extract?
w, h = [edges.shape[i] - self.patch_size[i] + 1 for i in range(2)]
# TODO: Maybe shuffle an iterator of the indices?
indices = list(itr.product(range(w - 1), range(h - 1)))
random.shuffle(indices)
i_iter = iter(indices)
E = edges.shape[-1]
th = _threshold_in_counts(self.settings, 8)
print(('th', th))
for sample in range(samples_per_image):
for tries in range(20):
#x, y = random.randint(0, w-1), random.randint(0, h-1)
x, y = next(i_iter)
selection = [
slice(x, x + self.patch_size[0]),
slice(y, y + self.patch_size[1])
]
selection_padded = [
slice(x, x + radius * 2 + self.patch_size[0]),
slice(y, y + radius * 2 + self.patch_size[1])
]
# Return grayscale patch and edges patch
unspread_edgepatch = unspread_edges[selection]
edgepatch = edges[selection]
#edgepatch_nospread = edges_nospread[selection]
if fr == 0:
tot = unspread_edgepatch.sum()
else:
tot = unspread_edgepatch[fr:-fr, fr:-fr].sum()
print(('tot', tot))
if th <= tot:
the_patches.append(edgepatch)
#the_patches.append(edgepatch_nospread)
the_unspread_patch = unspread_edges[selection]
the_unspread_patches.append(the_unspread_patch)
the_unspread_patch_padded = unspread_edges_padded[
selection_padded]
the_unspread_patches_padded.append(
the_unspread_patch_padded)
# The following is only for clearer visualization of the
# patches. However, normalizing like this might be misleading
# in other ways.
vispatch = img[selection]
if 1:
pass
else:
vispatch = vispatch[..., :3].mean(axis=-1)
span = vispatch.min(), vispatch.max()
if span[1] - span[0] > 0:
vispatch = (vispatch - span[0]) / (span[1] - span[0])
the_originals.append(vispatch)
break
return the_patches, the_unspread_patches, the_unspread_patches_padded, the_originals
def background_adjust_model(settings, bkg, seed=0):
offset = settings['detector'].get('train_offset', 0)
limit = settings['detector'].get('train_limit')
files = sorted(glob.glob(settings['detector']['train_dir']))[
offset:limit] # * settings['detector'].get('duplicate', 1)
try:
detector = gv.Detector.load(settings['detector']['file'])
except KeyError:
raise Exception("Need to train the model first")
# We need the descriptor to generate and manipulate images
descriptor = gv.load_descriptor(settings)
sh = (28, 88)
# Create accumulates for each mixture component
# TODO: Temporary until multicomp
#counts = np.zeros_like(detector.kernel_templates)
counts = np.zeros((1, sh[0], sh[1], descriptor.num_parts))
num_files = len(files)
num_duplicates = settings['detector'].get('duplicate', 1)
# Create several random states, so it's easier to measure
# the influence of certain features
prnds = [np.random.RandomState(seed + i) for i in xrange(10)]
# Setup unspread bedges settings
bsettings = settings['edges'].copy()
radius = bsettings['radius']
bsettings['radius'] = 0
padding = radius
locations0 = xrange(sh[0])
locations1 = xrange(sh[1])
padded_theta = descriptor.unspread_parts_padded
X_pad_size = padded_theta.shape[1:3]
for fn in files:
ag.info("Processing file", fn)
# Which mixture component does this image belong to?
# TODO: Temporary until multicomp
mixcomp = 0 #np.argmax(detector.affinities
# Binarize support and Extract alpha
color_img, alpha = gv.img.load_image_binarized_alpha(fn)
img = gv.img.asgray(color_img)
alpha_pad = ag.util.zeropad(alpha, padding)
inv_alpha_pad_expanded = np.expand_dims(~alpha_pad, -1)
# Iterate every duplicate
for loop in xrange(num_duplicates):
ag.info("Iteration {0}/{1}".format(loop + 1, num_duplicates))
# Superimpose onto gray background
graymap = create_graymap(img.shape, loop / (num_duplicates - 1),
prnds[0])
# Composite
img_with_gray = composite(img, graymap, alpha)
# Retrieve unspread edges (with a given background gray level)
edges = ag.features.bedges(img_with_gray, **bsettings)
# Pad the edges
edges_pad = ag.util.zeropad(edges, (padding, padding, 0))
for i, j in product(locations0, locations1):
selection = [
slice(i, i + X_pad_size[0]),
slice(j, j + X_pad_size[1])
]
X_pad = edges_pad[selection].copy()
nA_pad = inv_alpha_pad_expanded[selection]
# Draw background part from categorical distribution
f_bkg = weighted_choice_unit(bkg, prnds[1])
probs_bkg = get_probs(padded_theta, f_bkg)
probs = nA_pad * probs_bkg
# Iterate over all locations
# Draw from background edge probability over ~alpha
X_pad |= (prnds[2].rand(*probs.shape) < probs)
# Do spreading
X_pad_spread = ag.features.bspread(X_pad,
spread=bsettings['spread'],
radius=radius)
# De-pad
X_spread = X_pad_spread[padding:-padding, padding:-padding]
# Code parts
parts = descriptor.extract_parts(X_spread.astype(np.uint8))
# Accumulate and return
counts[mixcomp, i, j] += parts[0, 0]
"""
if 0:
from multiprocessing import Pool
p = Pool(7)
mapf = p.map
else:
mapf = map
def _process_file(fn):
return _process_file_full(fn, sh, descriptor, detector)
# Iterate images
all_counts = mapf(_process_file, files)
for counti in all_counts:
counts += counti
"""
# Divide accmulate to get new distribution
counts /= num_files * num_duplicates
# Create a new model, with this distribution
new_detector = detector.copy()
new_detector.kernel_templates = counts
new_detector.support = None
new_detector.use_alpha = False
# Return model
return new_detector
random.seed(0)
random.shuffle(files)
validate_files = files[-50:]
raw_patches, raw_originals = random_patches(files[:50],
patch_size,
samples_per_image=500)
#import matplotlib.pylab as plt
#plt.hist(num_edges, 30)
#plt.show()
#print raw_originals.shape
# Make grayscale
#raw_originals = raw_originals[...,:3].mean(axis=raw_originals.ndim-1)
ag.info("Training...")
mixture = ag.stats.BernoulliMixture(K, raw_patches, init_seed=0)
mixture.run_EM(1e-4, min_probability=0.01, debug_plot=False)
ag.info("Done.")
# Store the stuff in the instance
patches = mixture.templates
mix0 = mixture.indices_lists()[0]
info = {
'K': K,
'patch_size': patch_size,
}
vispatches = mixture.remix(raw_originals)
def fit(self, X):
N, P, F = X.shape
assert P == 2, "Only works for P == 2"
K = self.n_components
eps = self.min_probability
pi = np.ones((K, P)) / (K * P)
#theta = self.random_state.uniform(eps, 1 - eps, size=(K, P, F))
# Initialize
clusters = np.random.randint(K, size=N)
theta = np.asarray([np.mean(X[clusters == k], axis=0) for k in range(K)])
theta[:] = np.clip(theta, eps, 1 - eps)
self.q = np.empty((N, K, P))
logq = np.empty((N, K, P))
for i in range(self.n_iter):
ag.info("Iteration ", i+1)
#logq[:] = np.log(pi)[np.newaxis,:,np.newaxis]
#for k, p in itr.product(xrange(K), xrange(P)):
# logq[:,k,p] = np.log(pi[k,p])
logq[:] = np.log(pi[np.newaxis])
for p in range(P):
p0, p1 = p, (p+1)%2
logq[:,:,p] += np.dot(X[:,p0], np.log(theta[:,0] / (1 - theta[:,0])).T) + np.log(1 - theta[:,0]).sum(axis=1)[np.newaxis] +\
np.dot(X[:,p1], np.log(theta[:,1] / (1 - theta[:,1])).T) + np.log(1 - theta[:,1]).sum(axis=1)[np.newaxis]
import scipy.misc
#
#self.q[:] = np.exp(logq)
#normq = self.q / np.apply_over_axes(np.sum, self.q, [1, 2])
#self.q /= np.apply_over_axes(np.sum, self.q, [1, 2])
#q2 = np.exp(logq - scipy.misc.logsumexp(logq.reshape((-1, logq.shape[-1])), axis=0)[...,np.newaxis,np.newaxis])
norm_logq = logq - scipy.misc.logsumexp(logq.reshape((logq.shape[0], -1)), axis=-1)[...,np.newaxis,np.newaxis]
q2 = np.exp(norm_logq)
self.q[:] = q2
if 0:
try:
np.testing.assert_array_almost_equal(self.q, q2)
except:
import pdb; pdb.set_trace()
#norm_logq = logq - scipy.misc.logsumexp(logq.reshape((logq.shape[0], -1)), axis=-1)[...,np.newaxis,np.newaxis]
#self.q[:] = np.exp(norm_logq)
#dens = np.apply_over_axes(np.sum, self.q, [0, 2])
log_dens = scipy.misc.logsumexp(np.rollaxis(norm_logq, 2, 1).reshape((-1, norm_logq.shape[1])), axis=0)[np.newaxis,:,np.newaxis]
dens = np.exp(log_dens)
if 0:
try:
np.testing.assert_array_almost_equal(dens, dens2)
except:
import pdb; pdb.set_trace()
theta[:,0,:] = np.dot(self.q[:,:,0].T, X[:,0]) + np.dot(self.q[:,:,1].T, X[:,1])
theta[:,1,:] = np.dot(self.q[:,:,0].T, X[:,1]) + np.dot(self.q[:,:,1].T, X[:,0])
theta /= dens.flatten()[:,np.newaxis,np.newaxis]
#new parts
#pi[:] = np.apply_over_axes(np.sum, self.q, [0, 2])[0,:,0] / N
pi[:] = np.apply_over_axes(np.sum, self.q, [0])[0,:,:] / N
pi[:] = np.clip(pi, 0.0001, 1 - 0.0001)
# TODO: KEEP THIS?
#pi[:] = np.ones(pi.shape) / pi.shape
theta[:] = np.clip(theta, eps, 1 - eps)
#pi = np.clip(pi, eps, 1 - eps)
# Calculate the log likelihood
if 0:
llh = 0.0
for n in range(N):
v = 0.0
for k, p in itr.product(range(K), range(P)):
v += pi[k,p] * self.q[n,k,p]
#llh += pi[k]/2 * q[n,k,p] * np.sum(X[n,p0] * np.log(theta[k,0]) + (1 - X[n,p0]) * np.log(1 - theta[k,0]) + \
# X[n,p1] * np.log(theta[k,1]) + (1 - X[n,p1]) * np.log(1 - theta[k,1]))
llh += np.log(v)
print(llh)
self.weights_ = pi
self.means_ = theta
def background_adjust_model(settings, bkg, seed=0):
offset = settings['detector'].get('train_offset', 0)
limit = settings['detector'].get('train_limit')
files = sorted(glob.glob(settings['detector']['train_dir']))[
offset:limit] # * settings['detector'].get('duplicate', 1)
try:
detector = gv.Detector.load(settings['detector']['file'])
except KeyError:
raise Exception("Need to train the model first")
# We need the descriptor to generate and manipulate images
descriptor = gv.load_descriptor(settings)
sh = (28, 88)
# Create accumulates for each mixture component
# TODO: Temporary until multicomp
#counts = np.zeros_like(detector.kernel_templates)
counts = np.zeros((1, sh[0], sh[1], descriptor.num_parts))
num_files = len(files)
num_duplicates = settings['detector'].get('duplicate', 1)
# Create several random states, so it's easier to measure
# the influence of certain features
prnds = [np.random.RandomState(seed + i) for i in xrange(10)]
# Setup unspread bedges settings
bsettings = settings['edges'].copy()
radius = bsettings['radius']
bsettings['radius'] = 0
locations0 = xrange(sh[0])
locations1 = xrange(sh[1])
padded_theta = descriptor.unspread_parts_padded
#pad = 10
pad = 5
X_pad_size = (9 + pad * 2, ) * 2
#X_pad_size = padded_theta.shape[1:3]
neg_filenames = sorted(
glob.glob(
os.path.join(os.environ['UIUC_DIR'], 'TrainImages', 'neg-*.pgm')))
gen_raw = generate_random_patches(neg_filenames, X_pad_size, seed)
# Pre-generate a bunch of background patches and loop them.
bkgs = [gen_raw.next() for i in xrange(2000)]
def new_gen():
i = 0
while True:
yield bkgs[i]
i += 1
if i == 2000:
i = 0
gen = new_gen()
for seed, fn in enumerate(files):
ag.info("Processing file", fn)
# Which mixture component does this image belong to?
# TODO: Temporary until multicomp
mixcomp = 0 #np.argmax(detector.affinities
# Binarize support and Extract alpha
color_img, alpha = gv.img.load_image_binarized_alpha(fn)
img = gv.img.asgray(color_img)
img_pad = ag.util.zeropad(img, pad)
alpha_pad = ag.util.zeropad(alpha, pad)
inv_alpha_pad_expanded = np.expand_dims(~alpha_pad, -1)
# Iterate every duplicate
#ag.info("Iteration {0}/{1}".format(loop+1, num_duplicates))
#ag.info("Iteration")
for i, j in product(locations0, locations1):
selection = [
slice(i, i + X_pad_size[0]),
slice(j, j + X_pad_size[1])
]
#X_pad = edges_pad[selection].copy()
patch = img_pad[selection]
alpha_patch = alpha_pad[selection]
#ag.info("Position {0} {1}".format(i, j))
for loop in xrange(num_duplicates):
bkgmap = gen.next()
# Composite
img_with_bkg = composite(patch, bkgmap, alpha_patch)
# Retrieve unspread edges (with a given background gray level)
edges_pad = ag.features.bedges(img_with_bkg, **bsettings)
# Pad the edges
#edges_pad = ag.util.zeropad(edges, (pad, pad, 0))
# Do spreading
X_pad_spread = ag.features.bspread(edges_pad,
spread=bsettings['spread'],
radius=radius)
# De-pad
padding = pad - 2
X_spread = X_pad_spread[padding:-padding, padding:-padding]
# Code parts
parts = descriptor.extract_parts(X_spread.astype(np.uint8))
# Accumulate and return
counts[mixcomp, i, j] += parts[0, 0]
"""
if 0:
from multiprocessing import Pool
p = Pool(7)
mapf = p.map
else:
mapf = map
def _process_file(fn):
return _process_file_full(fn, sh, descriptor, detector)
# Iterate images
all_counts = mapf(_process_file, files)
for counti in all_counts:
counts += counti
"""
# Divide accmulate to get new distribution
counts /= num_files * num_duplicates
# Create a new model, with this distribution
new_detector = detector.copy()
new_detector.kernel_templates = counts
new_detector.support = None
new_detector.use_alpha = False
# Return model
return new_detector
def _get_patches(bedges_settings, settings, filename):
samples_per_image = settings['samples_per_image']
fr = settings['patch_frame']
the_patches = []
the_originals = []
ag.info("Extracting patches from", filename)
#edges, img = ag.features.bedges_from_image(f, k=5, radius=1, minimum_contrast=0.05, contrast_insensitive=False, return_original=True, lastaxis=True)
setts = settings['bedges'].copy()
radius = setts['radius']
setts2 = setts.copy()
setts['radius'] = 0
ps = settings['part_size']
ORI = settings.get('orientations', 1)
POL = settings.get('polarities', 1)
assert POL in (1, 2), "Polarities must be 1 or 2"
#assert ORI%2 == 0, "Orientations must be even, so that opposite can be collapsed"
# LEAVE-BEHIND
# Fetch all rotations of this image
img = gv.img.load_image(filename)
img = gv.img.asgray(img)
from skimage import transform
size = img.shape[:2]
# Make it square, to accommodate all types of rotations
new_size = int(np.max(size) * np.sqrt(2))
img_padded = ag.util.zeropad_to_shape(img, (new_size, new_size))
pad = [(new_size - size[i]) // 2 for i in range(2)]
angles = np.arange(0, 360, 360 / ORI)
radians = angles * np.pi / 180
all_img = np.asarray([
transform.rotate(img_padded, angle, resize=False) for angle in angles
])
# Add inverted polarity too
if POL == 2:
all_img = np.concatenate([all_img, 1 - all_img])
# Set up matrices that will translate a position in the canonical image to
# the rotated iamges. This way, we're not rotating each patch on demand, which
# will end up slower.
matrices = [
gv.matrix.translation(new_size / 2, new_size / 2) *
gv.matrix.rotation(a) *
gv.matrix.translation(-new_size / 2, -new_size / 2) for a in radians
]
# Add matrices for the polarity flips too, if applicable
matrices *= POL
#inv_img = 1 - img
all_unspread_edges = _extract_many_edges(bedges_settings,
settings,
all_img,
must_preserve_size=True)
#unspread_edges_padded = ag.util.zeropad(unspread_edges, (radius, radius, 0))
#inv_unspread_edges_padded = ag.util.zeropad(inv_unspread_edges, (radius, radius, 0))
# Spread the edges
all_edges = ag.features.bspread(all_unspread_edges,
spread=setts['spread'],
radius=radius)
#inv_edges = ag.features.bspread(inv_unspread_edges, spread=setts['spread'], radius=radius)
#setts2['minimum_contrast'] *= 2
#edges2 = ag.features.bedges(img, **setts2)
#inv_edges2 = ag.features.bedges(inv_img, **setts2)
#s = self.settings['bedges'].copy()
#if 'radius' in s:
# del s['radius']
#edges_nospread = ag.features.bedges_from_image(filename, radius=0, **s)
# How many patches could we extract?
# This avoids hitting the outside of patches, even after rotating.
# The 15 here is fairly arbitrary
avoid_edge = int(15 + np.max(ps) * np.sqrt(2))
# This step assumes that the img -> edge process does not down-sample any
# TODO: Maybe shuffle an iterator of the indices?
# These indices represent the center of patches
indices = list(
itr.product(
range(pad[0] + avoid_edge, pad[0] + img.shape[0] - avoid_edge),
range(pad[1] + avoid_edge, pad[1] + img.shape[1] - avoid_edge)))
random.shuffle(indices)
i_iter = itr.cycle(iter(indices))
minus_ps = [-ps[i] // 2 for i in range(2)]
plus_ps = [minus_ps[i] + ps[i] for i in range(2)]
E = all_edges.shape[-1]
th = _threshold_in_counts(settings, E,
bedges_settings['contrast_insensitive'])
rs = np.random.RandomState(0)
for sample in range(samples_per_image):
for tries in range(100):
#selection = [slice(x, x+self.patch_size[0]), slice(y, y+self.patch_size[1])]
#x, y = random.randint(0, w-1), random.randint(0, h-1)
x, y = next(i_iter)
selection0 = [
0,
slice(x + minus_ps[0], x + plus_ps[0]),
slice(y + minus_ps[1], y + plus_ps[1])
]
# Return grayscale patch and edges patch
unspread_edgepatch = all_unspread_edges[selection0]
edgepatch = all_edges[selection0]
#inv_edgepatch = inv_edges[selection]
#amppatch = amps[selection]
#edgepatch2 = edges2[selection]
#inv_edgepatch2 = inv_edges2[selection]
#edgepatch_nospread = edges_nospread[selection]
# The inv edges could be incorproated here, but it shouldn't be that different.
if fr == 0:
tot = unspread_edgepatch.sum()
else:
tot = unspread_edgepatch[fr:-fr, fr:-fr].sum()
#if self.settings['threshold'] <= avg <= self.settings.get('max_threshold', np.inf):
if th <= tot:
XY = np.matrix([x, y, 1]).T
# Now, let's explore all orientations
patch = np.zeros((ORI * POL, ) + ps + (E, ))
vispatch = np.zeros((ORI * POL, ) + ps)
for ori in range(ORI * POL):
p = matrices[ori] * XY
ip = [int(round(p[i])) for i in range(2)]
selection = [
ori,
slice(ip[0] + minus_ps[0], ip[0] + plus_ps[0]),
slice(ip[1] + minus_ps[1], ip[1] + plus_ps[1])
]
patch[ori] = all_edges[selection]
orig = all_img[selection]
span = orig.min(), orig.max()
if span[1] - span[0] > 0:
orig = (orig - span[0]) / (span[1] - span[0])
vispatch[ori] = orig
# Randomly rotate this patch, so that we don't bias
# the unrotated (and possibly unblurred) image
shift = rs.randint(ORI)
patch[:ORI] = np.roll(patch[:ORI], shift, axis=0)
vispatch[:ORI] = np.roll(vispatch[:ORI], shift, axis=0)
if POL == 2:
patch[ORI:] = np.roll(patch[ORI:], shift, axis=0)
vispatch[ORI:] = np.roll(vispatch[ORI:], shift, axis=0)
the_patches.append(patch)
the_originals.append(vispatch)
break
if tries == 99:
print("100 tries!")
return the_patches, the_originals
def classify(features,
all_templates,
means,
variances,
llh_variances=None,
graylevels=None,
graylevel_templates=None,
samples=None,
deformation='edges',
correct_label=None,
threshold_multiple=1.2,
b0=None,
eta=None,
rho=None,
debug_plot=False):
# min loglikelihood
min_cost = None
min_which = None
costs = []
shape = features[-2:]
for digit, templates in enumerate(all_templates):
# Clip them, to avoid 0 probabilities
for mix_component, template in enumerate(templates):
#assert features.shape == template.shape
# Compare mixture with features
cost = -np.sum(features * np.log(template) +
(1 - features) * np.log(1 - template))
#print("Cost {0} digit {1} comp {2}".format(cost, digit, mix_component))
costs.append((cost, digit, mix_component))
if min_cost is None or cost < min_cost:
min_cost = cost
min_which = (digit, mix_component)
info = {}
if correct_label is not None:
info['mixture_correct'] = correct_label == min_which[0]
#info['contendors'] = len(checked)
info['surplus_change'] = 0.0
info['deformation'] = False
info['num_contendors'] = 0
if deformation:
costs = filter(lambda t: t[0] < min_cost * threshold_multiple, costs)
costs.sort()
# If all the costs left are the same digit, don't bother doing the deformation
checked = []
for t in costs:
if t[1] not in checked:
checked.append(t[1])
if correct_label is not None:
info['surplus_before'] = surplus(costs, correct_label)
if len(checked) != 1:
# Filter so that we have only one of each mixture (Not necessary, could even damage results!)
if 0:
for i, t in enumerate(costs):
if t[1] not in checked:
checked.append(t[1])
else:
del [costs[i]]
# Do the deformation
additional = {}
new_costs = []
info['deformation'] = True
info['num_contendors'] = len(costs)
for t in costs:
cost, digit, mix_component = t
var = variances[digit, mix_component]
me = means[digit, mix_component]
samp = samples[digit, mix_component]
if llh_variances is not None:
additional['llh_variances'] = llh_variances[digit,
mix_component]
penalty = None
levels = 3
# Calculate the posterior variance
# TODO: Maybe remove, this is usually done when arranging the coefficient file, and not here.
if b0 and eta and rho and samples is not None:
raise Exception("Are you sure you want to do this?")
var = add_prior(var, shape, eta, rho, b0, samp)
if deformation == 'edges':
F = all_templates[digit, mix_component]
I = features
imdef, information = ag.stats.bernoulli_deformation(
F,
I,
wavelet='db4',
penalty=penalty,
means=me,
variances=var,
start_level=1,
last_level=levels,
debug_plot=debug_plot,
tol=0.1,
maxiter=200,
**additional)
elif deformation == 'intensity':
#assert originals is not None, "Intensity deformation requires originals"
F = graylevel_templates[digit, mix_component]
I = graylevels
imdef, information = ag.stats.image_deformation(
F,
I,
wavelet='db4',
penalty=penalty,
means=me,
variances=var,
start_level=1,
last_level=levels,
debug_plot=debug_plot,
tol=0.00001,
maxiter=50,
**additional)
# Kill if cancelled
imdef or sys.exit(0)
new_cost = information['cost']
# Update the cost
ag.info("{0:4.1f} --> {1:4.1f}".format(cost, new_cost))
new_costs.append((new_cost, digit, mix_component))
if new_cost < min_cost:
min_cost = new_cost
min_which = (digit, mix_component)
# Reevaluate the surplus!
if correct_label is not None:
info['surplus_after'] = surplus(new_costs, correct_label)
info['surplus_change'] = info['surplus_after'] - info[
'surplus_before']
if correct_label is not None:
info['turned_correct'] = False
info['turned_incorrect'] = False
if (correct_label == min_which[0]) != info['mixture_correct']:
if correct_label == min_which[0]:
info['turned_correct'] = True
else:
info['turned_incorrect'] = True
info['mixture_correct'] = correct_label == min_which[0]
info['comp'] = min_which[1]
return min_which[0], info
def train_from_images(self, filenames):
raw_patches, raw_originals = self.random_patches_from_images(filenames)
if len(raw_patches) == 0:
raise Exception(
"No patches found, maybe your thresholds are too strict?")
# Also store these in "settings"
mixtures = []
llhs = []
from gv.latent_bernoulli_mm import LatentBernoulliMM
ORI = self.settings.get('orientations', 1)
POL = self.settings.get('polarities', 1)
#P = self.settings['orientations'] * self.settings['polarities']
P = ORI * POL
def cycles(X):
return np.asarray(
[np.concatenate([X[i:], X[:i]]) for i in range(len(X))])
# Arrange permutations (probably a roundabout way of doing it)
RR = np.arange(ORI)
PP = np.arange(POL)
II = [
list(itr.product(PPi, RRi)) for PPi in cycles(PP)
for RRi in cycles(RR)
]
lookup = dict(list(zip(itr.product(PP, RR), itr.count())))
permutations = np.asarray([[lookup[ii] for ii in rows] for rows in II])
n_init = self.settings.get('n_init', 1)
n_iter = self.settings.get('n_iter', 10)
seed = self.settings.get('seed', 0)
mixture = LatentBernoulliMM(
n_components=self._num_parts,
permutations=permutations,
n_init=n_init,
n_iter=n_iter,
random_state=seed,
min_probability=self.settings['min_probability'])
mixture.fit(raw_patches.reshape(raw_patches.shape[:2] + (-1, )))
ag.info("Done.")
comps = mixture.mixture_components()
counts = np.bincount(comps[:, 0], minlength=self.num_features)
self.extra['counts'] = counts
print(counts)
print('Total', np.sum(counts))
from scipy.stats.mstats import mquantiles
print(mquantiles(counts))
llhs = np.zeros(len(raw_patches))
# Now, sort out low-likelihood patches
for i, patch in enumerate(raw_patches):
patch0 = patch[0]
model = mixture.means_[tuple(comps[i])].reshape(patch0.shape)
llh = (patch0 * np.log(model)).sum() + (
(1 - patch0) * np.log(1 - model)).sum()
llhs[i] = llh
means = []
sigmas = []
for f in range(self.num_features):
model = mixture.means_[f, 0]
mu = np.sum((model * np.log(model)) +
(1 - model) * np.log(1 - model))
a = np.log(model / (1 - model))
sigma = np.sqrt((a**2 * model * (1 - model)).sum())
means.append(mu)
sigmas.append(sigma)
means = np.asarray(means)
sigmas = np.asarray(sigmas)
#self.extra['means'] = np.asarray(means)
#self.extra['stds'] = np.asarray(sigmas)
self.extra['means'] = scipy.ndimage.zoom(means, P, order=0)
self.extra['stds'] = scipy.ndimage.zoom(sigmas, P, order=0)
means2 = []
sigmas2 = []
II = comps[:, 0]
for f in range(self.num_features):
X = llhs[II == f]
if len(X) > 0:
mu = X.mean()
sigma = X.std()
else:
# This one will guaranteed be sorted out anyway
mu = 0.0
sigma = 1.0
means2.append(mu)
sigmas2.append(sigma)
#self.extra['means_emp'] = np.asarray(means2)
#self.extra['stds_emp'] = np.asarray(sigmas2)
self.extra['means_emp'] = scipy.ndimage.zoom(means2, P, order=0)
self.extra['stds_emp'] = scipy.ndimage.zoom(sigmas2, P, order=0)
II = comps[:, 0]
#self.extra['comps'] = comps
#self.extra['llhs'] = llhs
if 0:
keepers = np.zeros(llhs.shape, dtype=np.bool)
for f in range(self.num_features):
llhs_f = llhs[II == f]
th_f = scipy.stats.scoreatpercentile(llhs_f, 80)
keepers[(II == f) & (llhs >= th_f)] = True
else:
keepers = np.ones(llhs.shape, dtype=np.bool)
#th = scipy.stats.scoreatpercentile(llhs, 75)
#keepers = llhs >= th
raw_patches = raw_patches[keepers]
raw_originals = raw_originals[keepers]
#raw_originals = list(itr.compress(raw_originals, keepers))
comps = comps[keepers]
ag.info("Pruned")
self.parts = mixture.means_.reshape((mixture.n_components, P) +
raw_patches.shape[2:])
self.orientations = None
#self.unspread_parts = self.unspread_parts[II]
#self.unspread_parts_padded = self.unspread_parts_padded[II]
self.unspread_parts = None
self.unspread_parts_padded = None
#self.visparts = None#self.visparts[II]
firsts = np.zeros(self.parts.shape[0], dtype=int)
for f in range(self.parts.shape[0]):
avgs = np.apply_over_axes(np.mean, self.parts[f, :ORI, ..., 0],
[1, 2]).squeeze()
firsts[f] = np.argmax(avgs)
self.parts[f, :ORI] = np.roll(self.parts[f, :ORI],
-firsts[f],
axis=0)
if POL == 2:
self.parts[f, ORI:] = np.roll(self.parts[f, ORI:],
-firsts[f],
axis=0)
# Rotate the parts, so that the first orientation is dominating in the first edge feature.
#visparts = mixture.remix(raw_originals)
#aff = np.asarray(self.affinities)
#self.visparts = np.asarray([np.average(data, axis=0, weights=aff[:,m]) for m in xrange(self.num_mix)])
self.visparts = np.zeros((mixture.n_components, ) +
raw_originals.shape[2:])
counts = np.zeros(mixture.n_components)
for n in range(raw_originals.shape[0]):
f, polarity = comps[
n] # np.unravel_index(np.argmax(mixture.q[n]), mixture.q.shape[1:])
self.visparts[f] += raw_originals[n, (polarity + firsts[f]) % P]
counts[f] += 1
self.visparts /= counts[:, np.newaxis, np.newaxis].clip(min=1)
#self.visparts = np.asarray([self.visparts, 1 - self.visparts])
#self.visparts = np.rollaxis(self.visparts, 1)
#self.visparts = self.visparts.reshape((self.visparts.shape[0] * 2,) + self.visparts.shape[2:])
#self.extra['originals'] = [self.extra['originals'][ii] for ii in II]
#self.weights = mixture.weights_
#self.extra['originals'] = originals
weights = mixture.weights_
#II = np.argsort(weights.max(axis=1))
#II = np.arange(self.parts.shape[0])
#self.extra['originals'] = [originals[ii] for ii in II]
#self.extra['originals'] = [self.extra['originals'][ii] for ii in II]
self._num_parts = self.parts.shape[0]
self.parts = self.parts.reshape((self.parts.shape[0] * P, ) +
self.parts.shape[2:])
#order_single = np.argsort(means / sigmas)
order_single = np.argsort(means)
new_order_single = []
for f in order_single:
part = self.parts[P * f]
sh = part.shape
p = part.reshape((sh[0] * sh[1], sh[2]))
pec = p.mean(axis=0)
N = np.sum(p * np.log(p / pec) + (1 - p) * np.log((1 - p) /
(1 - pec)))
D = np.sqrt(
np.sum(np.log(p / pec * (1 - pec) / (1 - p))**2 * p * (1 - p)))
ok = (N / D > 1) and counts[f] > 100
if ok:
new_order_single.append(f)
order_single = np.asarray(new_order_single)
assert len(
order_single) > 0, "No edges kept! Something probably went wrong"
#self._num_parts = len(order_single)
self._num_parts = len(order_single) * ORI
self._num_true_parts = len(order_single)
print('num_parts', self._num_parts)
print('num_true_parts', self._num_true_parts)
order = np.zeros(len(order_single) * P, dtype=int)
for f in range(len(order_single)):
for p in range(P):
order[P * f + p] = order_single[f] * P + p
II = order
# Require at least 20 originals
#II = filter(lambda ii: counts[ii] >= 20, II)
# Resort everything
self.extra['means'] = self.extra['means'][II]
self.extra['stds'] = self.extra['stds'][II]
self.extra['means_emp'] = self.extra['means_emp'][II]
self.extra['stds_emp'] = self.extra['stds_emp'][II]
#self.extra['means_double']
self.parts = self.parts[II]
self.visparts = self.visparts[order_single]
originals = []
for i in order_single:
j = weights[i].argmax()
# You were here: This line is not correct
II0 = np.where(comps[:, 0] == i)[0]
II1 = comps[II0, 1]
ims = np.asarray(
[raw_originals[tuple(ij)] for ij in itr.izip(II0, II1)])
#ims = raw_originals[comps[:,0] == i,comps[:,1]].copy()
originals.append(ims[:50])
self.extra['originals'] = originals
self._preprocess_logs()
#bkg = np.random.uniform(0.05, 0.80, detector.num_features)
B = 100
#bkgs = [np.random.uniform(0.05, 0.60, detector.num_features) for i in xrange(B)]
bkg = np.random.uniform(0.05, 0.40, detector.num_features)
bkgs = [bkg]
# Preprocess kernel
sub_kernels_all = [detector.prepare_kernels(bkg) for bkg in bkgs]
#llhs = [[[] for j in xrange(B)] for i in xrange(detector.num_mixtures)]
llhs = [[] for i in xrange(detector.num_mixtures)]
for i, edges in enumerate(detector.gen_img(files, actual=True)):
# Which mixture?
comp = np.argmax(detector.mixture.affinities[i])
ag.info('Processing', i)
# Check log likelihood of this against the model
# and add to list
#llh = np.sum(edges * np.log(kern) + (1-edges) * np.log(1-kern))
for j, bkg in enumerate(bkgs):
kern = sub_kernels_all[j][comp]
a = np.log(kern / (1 - kern) * ((1 - bkg) / bkg))
llh = np.sum((edges - bkg) * a)
summand = a**2 * bkg * (1 - bkg)
llh /= np.sqrt(summand.sum())
#llhs[comp][j].append(llh)
llhs[comp].append(llh)
mn = 0
def _process_file_full(fn, sh, descriptor, detector):
counts = np.zeros((1, sh[0], sh[1], descriptor.num_parts))
ag.info("Processing file", fn)
# Which mixture component does this image belong to?
# TODO: Temporary until multicomp
mixcomp = 0 #np.argmax(detector.affinities
# Binarize support and Extract alpha
color_img, alpha = gv.img.load_image_binarized_alpha(fn)
img = gv.img.asgray(color_img)
alpha_pad = ag.util.zeropad(alpha, padding)
inv_alpha_pad_expanded = np.expand_dims(~alpha_pad, -1)
# Iterate every duplicate
for loop in xrange(num_duplicates):
ag.info("Iteration {0}/{1}".format(loop + 1, num_duplicates))
# Superimpose onto gray background
graymap = create_graymap(img.shape, loop / (num_duplicates - 1),
prnds[0])
# Composite
img_with_gray = composite(img, graymap, alpha)
# Retrieve unspread edges (with a given background gray level)
edges = ag.features.bedges(img_with_gray, **bsettings)
# Pad the edges
edges_pad = ag.util.zeropad(edges, (padding, padding, 0))
for i, j in product(locations0, locations1):
selection = [
slice(i, i + X_pad_size[0]),
slice(j, j + X_pad_size[1])
]
X_pad = edges_pad[selection].copy()
nA_pad = inv_alpha_pad_expanded[selection]
# Draw background part from categorical distribution
f_bkg = weighted_choice_unit(bkg, prnds[1])
probs_bkg = get_probs(padded_theta, f_bkg)
probs = nA_pad * probs_bkg
# Iterate over all locations
# Draw from background edge probability over ~alpha
X_pad |= (prnds[2].rand(*probs.shape) < probs)
# Do spreading
X_pad_spread = ag.features.bspread(X_pad,
spread=bsettings['spread'],
radius=radius)
# De-pad
X_spread = X_pad_spread[padding:-padding, padding:-padding]
# Code parts
parts = descriptor.extract_parts(X_spread.astype(np.uint8))
# Accumulate and return
counts[mixcomp, i, j] += parts[0, 0]
return counts
