def __init__(self, model=None):
self.file_def = None
self.file_def_train = None
self.file_solver = None
self.file_weights = None
self.file_mean = None
self.net = None
self.mu = None
# paths to dbs with training in validation patches
self.db_images = None
self.db_train = None
self.db_val = None
self.db_test = None
self.db_keys = None
self.sz_key = 256 # gob idx is resource_uuid + sub_id + augmented_id, something like: 00-79Hfk6V5wPQZwuWMDcQxH5-650-9 with 32 chars
self.sz_keyenc = 128 # 64 # sha256 hashed key
self.goodness_function = None
devices = self.get_device_info()
if len(devices) < 1:
raise ConnoisseurException(
responses.INTERNAL_SERVER_ERROR,
'No Caffe available devices were detected...')
super(self.__class__, self).__init__(model)
def classify(self, image, model, args):
''' segment image produicing a semantic mask
'''
num_points = int(args.get('points', 10))
border = int(args.get('border', 0))
my_goodness = float(args.get('goodness', model.minimum_goodness*100))/100.0
my_accuracy = float(args.get('accuracy', model.minimum_accuracy*100))
my_confidence = float(args.get('confidence', 0))
# color output mode
color_mode = args.get('colors', 'ids')
if color_mode not in self.color_modes:
raise ConnoisseurException(responses.BAD_REQUEST, 'Requested color mode "%s" is not supported'%color_mode)
# compute output file name and test cached result
workdir = args['_workdir']
_mkdir (workdir)
filename = '%s_%s_conf%.2f_a%s_c%s_n%s_b%s.png'%(image.uniq, color_mode, my_goodness, my_accuracy, my_confidence, num_points, border)
output_file = os.path.join(workdir, filename)
with Locks(None, output_file, failonexist=True) as l:
if l.locked: # the file is not being currently written by another process
self.do_classify(image, model, args, output_file, color_mode)
# return results
if os.path.exists(output_file):
with Locks(output_file):
pass
return DataToken(data=output_file, mime='image/png', name='Segments', filename=filename)
def get_adapted_image_size(self):
""" returns (H,W) of the whole as seen by this adapter """
if self.pix is None:
raise ConnoisseurException(
responses.INTERNAL_SERVER_ERROR,
'AdapterRGB2DResized: image was not properly initialized')
return self.pix.shape[0:2]
def get_adapted_image_patch(self, gob, **kw):
""" return image pixels for a given gobject as a numpy array """
if self.pix is None:
raise ConnoisseurException(
responses.INTERNAL_SERVER_ERROR,
'AdapterRGB2DResized: image was not properly initialized')
c = gob.centroid()
i = int(c[1] * self.si)
j = int(c[0] * self.sj)
# print 'Requested gob: %s'%(gob)
# print 'Scaled i,j: %s,%s'%(i,j)
# print 'Patch size: %s'%(self.model.db_patch_size)
# print 'Pix size: %s'%(str(self.pix.shape))
if self.patch_width == 0 and self.patch_height == 0 or self.patch_width == self.model.db_patch_size[
0] and self.patch_height == self.model.db_patch_size[1]:
return image_patch(self.pix, i, j, self.model.db_patch_size[1],
self.model.db_patch_size[0])
else:
p = image_patch(self.pix, i, j, self.patch_height,
self.patch_width)
return scipy.misc.imresize(
p, (self.model.db_patch_size[1], self.model.db_patch_size[0]),
interp='bilinear')
def classify(self, image, model, args):
"""returns classified results
image: instance of XImage
model: instance of ClassifierModel
args: dict with arguments parsed from URL
"""
raise ConnoisseurException(responses.NOT_IMPLEMENTED, 'Class adapter must implement "classify" method')
def classify(self, image, model, args):
''' uniformly sampled points '''
Worig, Horig = image.size()
num_points = int(args.get('points', 100))
border = float(args.get('border', 5))
border = int(round(border * np.mean([Worig, Horig]) / 100.0))
pts, num_points_x, num_points_y, sw, sh = distribute_points(
num_points, Worig, Horig, border, equal=False, return_all=True)
# estimate superpixel size
W = self.side
sx = 1.0
if max(Worig, Horig) > W:
sx = float(W) / max(Worig, Horig)
sw = int(
round(sw * sx)
) # this provides an approximate number of centroids as requested
# load SLIC super-pixel segmented image
log.debug('Requesting superpixels of size: %s', sw)
pix = image.pixels(
operations=
'slice=,,1,1&resize=%s,%s,BC,MX&depth=8,d,u&transform=superpixels,%s,0.8&format=tiff'
% (self.side, self.side, sw))
W, H = pix.shape[0:2]
# get regional centroids
label_offset = 1 # region prop function uses 0 as background, we bump the class number and later subtract
segments = skimage.measure.regionprops(pix + label_offset, cache=True)
num_segs = len(segments)
log.debug('Segmented the image into %s segments', num_segs)
if num_segs < 1:
raise ConnoisseurException(responses.NO_CONTENT,
'Centroids classifier: no results')
# compute scaling factor
sx = 1.0
sy = 1.0
if Worig != W or Horig != H:
sx = float(W) / Worig
sy = float(H) / Horig
log.debug(
'Classify: Original image is larger, use scaling factors: %s,%s',
sx, sy)
# scale params to resized image
border = int(round(border * sx))
points = []
for i, s in enumerate(segments):
x, y = s.centroid
if x > border and y > border and x <= W - border and y <= H - border:
points.append((x / sx, y / sy))
return classify_points(image, model, args, points,
'Regional centroids')
def validate(self, classes, my_goodness):
''' classes is a dictionary similar to model classes dict with inited and added variables:
v['true_positive'] = 0
v['false_negative'] = 0
v['false_positive'] = 0
v['false_positive_classes'] = [] # we need to compute weighted false positive later based on number of tested samples in each set
v['n'] = 0
v['discarded'] = 0
'''
# classify samples from the test set
ldb = lmdb.open(self.db_test, readonly=True)
try: # protect the writing process
with ldb.begin(buffers=True) as txn:
cursor = txn.cursor()
for key, value in cursor:
#pylint: disable=no-member
datum = caffe.proto.caffe_pb2.Datum()
datum.ParseFromString(value)
# convert datum data into a 2D image like array WxHxC
x = np.fromstring(datum.data, dtype=np.uint8)
x = x.reshape(1, datum.channels, datum.height,
datum.width).squeeze().transpose((1, 2, 0))
# mean subtraction
if self.mu is not None:
x = x - self.mu
out_class, goodness = self.classify(x)
classes[datum.label]['n'] += 1
if classes[out_class].get('ignored', False) is True:
classes[datum.label]['discarded'] += 1
continue
if goodness < my_goodness:
classes[datum.label]['discarded'] += 1
continue
# update class list
if datum.label == out_class:
classes[datum.label]['true_positive'] += 1
else:
classes[datum.label]['false_negative'] += 1
classes[out_class]['false_positive'] += 1
classes[out_class]['false_positive_classes'].append(
datum.label)
except Exception:
log.exception('Problem validating the model')
raise ConnoisseurException(responses.INTERNAL_SERVER_ERROR,
'Problem validating the model')
finally:
ldb.close()
return classes
def classify_points(image, model, args, pts, name='Points'):
''' classify a list of points'''
adapter = model.create_adapter_pixels(model=model, args=args, image=image)
my_goodness = float(args.get('goodness',
model.minimum_goodness * 100)) / 100.0
my_accuracy = float(args.get('accuracy', model.minimum_accuracy * 100))
my_confidence = float(args.get('confidence', 0))
results = []
for x, y in pts:
try:
pix = adapter.get_adapted_image_patch(gob=point(x=y, y=x))
out_class, goodness = model.classify(pix)
label, accuracy, ignored = get_class_info(model,
out_class,
at_goodness=my_goodness)
confidence = get_sample_confidence(accuracy, goodness)
if goodness >= my_goodness and accuracy >= my_accuracy and confidence >= my_confidence and ignored is False:
#print '%s,%s classified as "%s" with %s goodness score'%(x, y, label, goodness)
#log.debug('%s,%s classified as "%s" with %s goodness score'%(x, y, label, goodness))
results.append({
'gob': 'point',
'vertex': [(x, y)],
'label': label,
'id': out_class,
'accuracy': accuracy,
'goodness': goodness,
'confidence': confidence,
})
except Exception:
log.debug('x: %s y: %s', x, y)
log.exception('Exception in image_patch/model.classify')
if len(results) < 1:
raise ConnoisseurException(responses.NO_CONTENT,
'Point classifier: no results')
# hierarchical passes over samples whose classes have child models
# return results
return DataToken(data=results, mime='table/gobs', name=name)
def format(self, data, args):
""" converts table to CSV """
df = pd.read_csv(StringIO(data))
out = []
# test headers for formatting
h = df.columns.values.tolist() # pylint: disable=no-member
if validate_header(h, hxyz) is True:
# read xyz
for i in range(df.shape[0]):
out.append((df['y'][i], df['x'][i]))
elif validate_header(h, hgobs) is True:
# read gobs
for i in range(df.shape[0]):
v = df['vertices'][i].split(',')
out.append((v[1], v[0]))
else:
raise ConnoisseurException(
responses.UNSUPPORTED_MEDIA_TYPE,
'Importer %s: unsupported format' % (self.name))
# produce [(x,y), (x,y), ...]
args['_points'] = out
def cache_sample_preview(self, class_id, sample_id, filename):
x = None
cur_sample = -1
rdb = lmdb.open(self.db_images, readonly=True)
with rdb.begin() as rtxn:
cursor = rtxn.cursor()
for key, value in cursor:
#pylint: disable=no-member
datum = caffe.proto.caffe_pb2.Datum()
datum.ParseFromString(value)
if datum.label != class_id:
continue
cur_sample += 1
if cur_sample < sample_id:
continue
x = np.fromstring(datum.data, dtype=np.uint8)
x = x.reshape(1, datum.channels, datum.height,
datum.width).squeeze().transpose((1, 2, 0))
break
rdb.close()
if x is None:
raise ConnoisseurException(responses.NOT_FOUND,
'Requested sample was not found')
skimage.io.imsave(filename, x)
def train(self, method='finetune'):
args = [
'train', '-gpu', 'all', '-solver',
self.fix_path(self.file_solver)
]
# fine-tuning
if method == 'finetune':
args.extend(['-weights', self.fix_path(self.file_weights)])
# recovery from snapshot
if method == 'snapshot':
# find the latest snapshot
files = [
f for f in os.listdir(self.path)
if f.endswith('.solverstate') and f.startswith('weights_iter_')
]
files.sort(key=lambda x: int(
x.replace('weights_iter_', '').replace('.solverstate', '')))
latest_snapshot = os.path.join(self.path, files[-1])
args.extend(['--snapshot=%s' % self.fix_path(latest_snapshot)])
# capture output to update the loss/accuracy plot,
# we need to parse: training loss, testing loss and testing accuracy for n iterations
# right now we capture after run finished
#args.extend(['>', 'out.log'])
o = self.run_caffe_bin(self.COMMAND_CAFFE, args)
if o is None:
raise ConnoisseurException(
responses.INTERNAL_SERVER_ERROR,
'Error while running Caffe, intermediate states were preserved...'
)
# parse output
# o
# rename final snapshot to weights file
try:
files = [
f for f in os.listdir(self.path)
if f.endswith('.caffemodel') and f.startswith('weights_iter_')
]
files.sort(key=lambda x: int(
x.replace('weights_iter_', '').replace('.caffemodel', '')))
latest = os.path.join(self.path, files[-1])
safe_remove_dir_file(self.file_weights)
log.debug('Moving %s to %s', latest, self.file_weights)
os.rename(latest, self.file_weights)
except Exception:
raise ConnoisseurException(
responses.INTERNAL_SERVER_ERROR,
'Error while updating weights, intermediate states were preserved...'
)
# remove temporary snapshots
files = [
f for f in os.listdir(self.path) if f.startswith('weights_iter_')
]
for f in files:
log.debug('Removing %s', f)
safe_remove_dir_file(os.path.join(self.path, f))
def split_samples_training_testing(self, sample_preview_paths=None):
# testing_percent indicates the size of the testing set in percent
# the validation set will be the same size as the testing set
# the training set will include everything else
# by default we'll use the 60%/20%/20% split for Training/Validation/Test
total_samples_test = 0
total_samples_val = 0
total_samples_train = 0
for k, c in self.model.classes_model.iteritems():
v = c['samples']
n_val = (v * self.model.testing_percent) / 100
n_test = n_val
n_train = v - n_val - n_test
c['samples_training'] = n_train
c['samples_validation'] = n_val
c['samples_testing'] = n_test
total_samples_test += n_test
total_samples_val += n_val
total_samples_train += n_train
classes = copy.deepcopy(self.model.classes_model)
classes_by_id = dict((v['id'], v) for k, v in classes.iteritems())
for k, c in classes.iteritems():
c['preview'] = 0
random.seed()
# clear dbs
safe_remove_dir_file(self.db_train)
safe_remove_dir_file(self.db_val)
safe_remove_dir_file(self.db_test)
# estimate db sizes
if os.name == 'nt' or sys.platform == 'darwin':
el_sz = self.get_db_element_size()
db_train_sz = self.get_lmdb_size(total_samples_train,
self.sz_keyenc, el_sz)
db_val_sz = self.get_lmdb_size(total_samples_val, self.sz_keyenc,
el_sz)
db_test_sz = self.get_lmdb_size(total_samples_test, self.sz_keyenc,
el_sz)
else:
db_train_sz = int(1e12)
db_val_sz = int(1e12)
db_test_sz = int(1e12)
# create split dbs
error = None
dbs = ['samples_training', 'samples_validation', 'samples_testing']
rdb = lmdb.open(self.db_images, readonly=True)
tdb = lmdb.open(self.db_train, map_size=db_train_sz, lock=True)
vdb = lmdb.open(self.db_val, map_size=db_val_sz, lock=True)
tedb = lmdb.open(self.db_test, map_size=db_test_sz, lock=True)
try: # protect the writing process
with rdb.begin(buffers=True) as rtxn:
cursor = rtxn.cursor()
with vdb.begin(write=True) as vtxn:
with tdb.begin(write=True) as ttxn:
with tedb.begin(write=True) as tetxn:
for key, value in cursor:
#pylint: disable=no-member
datum = caffe.proto.caffe_pb2.Datum()
datum.ParseFromString(value)
c = classes_by_id[datum.label]
# randomly decide which set to write the sample into
out_set = random.randint(0, 2)
c[dbs[out_set]] -= 1
if out_set == 2 and c[dbs[out_set]] >= 0:
tetxn.put(key, datum.SerializeToString())
elif out_set == 1 and c[dbs[out_set]] >= 0:
vtxn.put(key, datum.SerializeToString())
else:
ttxn.put(key, datum.SerializeToString())
precache_key = '{0}.{1}'.format(
c['id'], c['preview'])
if sample_preview_paths and precache_key in sample_preview_paths:
filename = sample_preview_paths.get(
precache_key)
x = np.fromstring(datum.data,
dtype=np.uint8)
x = x.reshape(
1, datum.channels, datum.height,
datum.width).squeeze().transpose(
(1, 2, 0))
skimage.io.imsave(filename, x)
c['preview'] = c['preview'] + 1
except (lmdb.MapFullError, lmdb.DbsFullError, lmdb.BadTxnError):
log.exception('LMDB split file size exhausted')
error = responses.INSUFFICIENT_STORAGE
except Exception:
log.exception('Problem splitting LMDB files')
error = responses.INTERNAL_SERVER_ERROR
finally:
rdb.close()
tdb.close()
vdb.close()
tedb.close()
if error is not None:
safe_remove_dir_file(self.db_train)
safe_remove_dir_file(self.db_val)
safe_remove_dir_file(self.db_test)
raise ConnoisseurException(error,
'Problem splitting sample database')
# compute mean of the dataset
safe_remove_dir_file(self.file_mean)
args = [self.fix_path(self.db_images), self.fix_path(self.file_mean)]
o = self.run_caffe_bin(self.COMMAND_MEAN, args)
if o is None:
raise ConnoisseurException(responses.INTERNAL_SERVER_ERROR,
'Error while running Caffe')
def update_sample_db(self, image):
if isinstance(image, (str, unicode)) is True:
image = XImage(base_url=image)
adapter_pixels = self.model.create_adapter_pixels(model=self.model,
image=image)
adapter_gobs = self.model.create_adapter_gobs(model=self.model,
image=image)
# get gobjects
gobs = adapter_gobs.get_gobjects()
if len(gobs) < 1:
return
log.debug('number gobs: %s', len(gobs))
log.debug('total_samples: %s', self.model.total_samples)
# estimate db sizes
if os.name == 'nt' or sys.platform == 'darwin':
db_img_sz = self.get_lmdb_size(self.model.total_samples,
self.sz_keyenc,
self.get_db_element_size())
db_key_sz = self.get_lmdb_size(self.model.total_samples,
self.sz_keyenc * 2, self.sz_key)
else:
db_img_sz = int(1e12)
db_key_sz = int(1e12)
log.debug('db_img_sz %s', db_img_sz)
log.debug('db_key_sz %s', db_key_sz)
# write samples with randomized keys
keys = {}
error = None
ldb = lmdb.open(self.db_images, map_size=db_img_sz, lock=True)
try:
with ldb.begin(write=True) as txn:
for g in gobs:
c = g.type
if c not in self.model.classes_model and not self.model.use_background_class:
continue
elif c not in self.model.classes_model and self.model.use_background_class is True:
c = self.model.background_class_name
#log.debug('Processing sample from class %s', c)
cid = self.model.classes_model[c][
'id'] # get numerical value for class name
#pix = adapter_pixels.get_adapted_image_patch(gob=g)
# test if augmentation is used
if 'samples_actual' in self.model.classes_model[c]:
#log.debug('Using augmentation')
pixs = adapter_pixels.get_augmented_image_patches(
gob=g)
else:
#log.debug('Using single patch')
pixs = [adapter_pixels.get_adapted_image_patch(gob=g)]
for i, pix in enumerate(pixs):
#log.debug('Pix shape: %s', pix.shape)
if self.patch_shape_matches_db(pix) is not True:
log.warning(
'Patch size does not match, requires %s but has %s, skipping...'
% (str(self.model.db_patch_size), str(
pix.shape)))
# dima: we could reshape the patch to match here
continue
#pix = pix[:,:,::-1] # RGB -> BGR
pix = pix.transpose((2, 0, 1)) # HWC -> CHW
#pylint: disable=no-member
datum = caffe.io.array_to_datum(pix)
datum.label = cid
#print 'datum: %sx%sx%s, len: %s, label: %s'%(datum.width, datum.height, datum.channels, len(datum.data), datum.label)
# key is composed of resource uniq + object id + augmentation id
key = ('%s-%s' % (g.idx, i)).encode('ascii')
# use hashed key to randomize sample order
enc_key = hashlib.sha256(key).hexdigest()
txn.put(enc_key, datum.SerializeToString())
# store the mapping from encoded key to key
# using the same hashing algorithm we can quickly test presence of real key in the db
# likewise it's quick to find the real key for the encoded one if needed
keys[enc_key] = key
# update number of samples
self.model.classes_model[c]['samples'] += 1
except (lmdb.MapFullError, lmdb.DbsFullError, lmdb.BadTxnError):
log.exception('LMDB image file size exhausted')
error = responses.INSUFFICIENT_STORAGE
except Exception:
log.exception('Problem writing data to LMDB file')
error = responses.INTERNAL_SERVER_ERROR
finally:
ldb.close()
# if error is not None:
# safe_remove_dir_file(self.db_images)
# raise ConnoisseurException(error, 'Problem creating sample database')
# store the mapping from encoded key to key
# using the same hashing algorithm we can quickly test presence of real key in the db
# likewise it's quick to find the real key for the encoded one if needed
kdb = lmdb.open(self.db_keys, map_size=db_key_sz, lock=True)
try:
with kdb.begin(write=True) as kxn:
for enc_key, key in keys.iteritems():
kxn.put(enc_key, key)
except (lmdb.MapFullError, lmdb.DbsFullError, lmdb.BadTxnError):
log.exception('LMDB key file size exhausted')
error = responses.INSUFFICIENT_STORAGE
except Exception:
log.exception('Problem writing keys to LMDB file')
error = responses.INTERNAL_SERVER_ERROR
finally:
kdb.close()
if error is not None:
raise ConnoisseurException(error,
'Problem creating sample database')
def classify(self, x):
''' x - 3D numpy array representing RGB image in np.float32
'''
if self.goodness_function is None:
self.load(self.model)
if self.goodness_function is None:
raise ConnoisseurException(
responses.BAD_REQUEST,
'Caffe classify: model is not loaded properly')
if self.net is None:
self.activate(training=False)
if self.net is None:
raise ConnoisseurException(
responses.BAD_REQUEST,
'Caffe classify: network is not activated')
# mean subtraction
if self.mu is not None:
try:
x = x - self.mu
except ValueError:
log.debug(
'Classify: failed to subtract mean, adjusting the size of the patch %s to %s',
str(x.shape), str(self.mu.shape))
x = scipy.misc.imresize(x, self.mu.shape)
log.debug(
'Classify: attempting mean subtraction with shape %s',
str(x.shape))
x = x - self.mu
log.debug('Classify: final subtracted patch with shape %s',
str(x.shape))
#caffe.set_mode_cpu()
#caffe.set_mode_gpu()
samples = caffe.io.oversample(
[x],
(self.model.model_patch_size[0], self.model.model_patch_size[1]))
for i in range(samples.shape[0]):
self.net.blobs['data'].data[i] = samples[i, :, :, :].transpose(
2, 0, 1)
# copy the image data into the memory allocated for the net
#self.net.blobs['data'].data[...] = samples[0,:,:,:].transpose(2,0,1)
#self.net.blobs['data'].data[...] = x
#self.net.blobs['data'].data[...] = transformer.preprocess('data', image)
### perform classification
output = self.net.forward()
#log.debug('Net output: %s', output)
classes = []
for output_prob in output['prob']:
out_class = output_prob.argmax()
#print 'for label %s predicted class is: %s with probability %s'%(datum.label, out_class, output_prob[out_class])
classes.append(out_class)
#log.debug('Net output classes: %s', classes)
# majority vote for class output
out_class = np.bincount(classes).argmax()
#log.debug('out class: %s', out_class)
#log.debug('len of classes: %s', len(classes))
#log.debug('number_classes: %s', self.number_classes_in_model)
probs = np.zeros(self.model.number_classes_in_model)
C = 0
for output_prob in output['prob']:
predicted_class = output_prob.argmax()
#log.debug('probs: %s', probs)
#log.debug('output_prob: %s', output_prob)
if predicted_class == out_class:
probs = probs + np.sort(output_prob)[::-1]
C += 1
probs = probs / C
#log.debug('probabilities: %s', probs)
p = probs[0]
g = self.goodness_function(probs)
agreement = float(C) / float(self.model.batch_size)
goodness = p * g * agreement
#log.debug('goodness: %s', goodness)
return (out_class, goodness)
def do_classify(self, image, model, args, output_file, color_mode):
''' segment image producing a semantic mask
'''
Worig,Horig = image.size()
num_points = int(args.get('points', 100))
border = float(args.get('border', 5))
border = int(round(border*np.mean([Worig,Horig])/100.0))
my_goodness = float(args.get('goodness', model.minimum_goodness*100))/100.0
my_accuracy = float(args.get('accuracy', model.minimum_accuracy*100))
my_confidence = float(args.get('confidence', 0))
# get uniformly distributed points over the original image
pts, num_points_x, num_points_y, sw, sh = distribute_points(num_points, Worig, Horig, border, equal=False, return_all=True)
# estimate superpixel size of the scaled down image
W=self.side; sx=1.0
if max(Worig, Horig)>W:
sx = float(W) / max(Worig, Horig)
sw = int(round(sw*sx)) # this provides an approximate number of centroids as requested
# load SLIC super-pixel segmented image
log.debug('Requesting superpixels of size: %s', int(sw))
seg = image.pixels(operations='slice=,,1,1&resize=%s,%s,BC,MX&depth=8,d,u&transform=superpixels,%s,0.1&format=tiff'%(self.side, self.side, sw))
W,H = seg.shape[0:2]
# compute scaling factors
sx=1.0; sy=1.0
if Worig!=W or Horig!=H:
sx = float(W) / Worig
sy = float(H) / Horig
log.debug('Classify: Original image is larger, use scaling factors: %s,%s', sx, sy)
# find segments to classify
label_offset = 1 # region prop function uses 0 as background, we bump the class number and later subtract
segments = skimage.measure.regionprops(seg+label_offset, cache=True)
num_segs = len(segments)
log.debug('Segmented the image into %s segments', num_segs)
if num_segs<1:
raise ConnoisseurException(responses.NO_CONTENT, 'Segmentation classifier: no results')
# get uniformly distributed points per segment in full image space
points_per_segment = [None]*num_segs
for i,p in enumerate(points_per_segment):
points_per_segment[i] = []
for x,y in pts:
v = seg[int(round(x*sx)), int(round(y*sx))]
points_per_segment[v].append((x,y))
# augment point list with segment centroids in full image space
for i,s in enumerate(segments):
x,y = s.centroid
x,y = (x/sx, y/sy)
if x>border and y>border and x<=Worig-border and y<=Horig-border:
v = s.label-label_offset
points_per_segment[v].append((x,y))
# classify segments based on their points
adapter = model.create_adapter_pixels(model=model, args=args, image=image)
segment_colors = [0]*num_segs
for i,s in enumerate(segments):
#log.debug('i: %i, label: %s', i, s.label)
samples = points_per_segment[s.label-label_offset]
output_classes = []
for p in samples:
x = int(round(p[0]))
y = int(round(p[1]))
try:
#pix = image_patch(pim, x, y, model.db_patch_size[1], model.db_patch_size[0])
pix = adapter.get_adapted_image_patch(gob=point(x=y, y=x))
out_class,goodness = model.classify(pix)
label, accuracy, ignored = get_class_info(model, out_class, at_goodness=my_goodness)
confidence = get_sample_confidence(accuracy, goodness)
if goodness >= my_goodness and accuracy >= my_accuracy and confidence >= my_confidence and ignored is False:
output_classes.append(out_class)
except Exception:
log.debug('x: %s y: %s', x,y)
log.debug('Pix shape: %s', pix.shape)
log.exception('Exception in image_patch/model.classify')
if len(output_classes)>0:
out_class = np.bincount(output_classes).argmax()
#log.debug('%s detected %s from classes: %s', i, out_class, output_classes)
segment_colors[i] = out_class+label_offset
I,J = seg.shape[0:2]
if color_mode == 'ids':
for j in range(J):
for i in range(I):
v = segment_colors[seg[i,j]]
seg[i,j] = v
seg = seg.astype(np.uint16)
skimage.io.imsave(output_file, seg)
#seg = seg.astype(np.uint16)
#seg = median(seg, disk(35))
#skimage.io.imsave('%s_colors_med_goodness_%.2f.tif'%(output_file, my_goodness), seg)
elif color_mode == 'colors':
img = np.zeros((I, J, 3), dtype=np.uint8)
for j in range(J):
for i in range(I):
v = segment_colors[seg[i,j]]
out_class = v-label_offset
#img[i,j,:] = get_color_tuple(out_class)
try:
label = model.classes_model_by_id[out_class].get('label')
img[i,j,:] = get_color_tuple_by_label(label)
except Exception:
pass
skimage.io.imsave(output_file, img)
def get_device_info(self):
''' returns a list of deice info dictionaries
I0615 11:40:41.533318 40528 caffe.cpp:138] Querying GPUs all
I0615 11:40:41.964962 40528 common.cpp:186] Device id: 0
I0615 11:40:41.964962 40528 common.cpp:187] Major revision number: 6
I0615 11:40:41.964962 40528 common.cpp:188] Minor revision number: 1
I0615 11:40:41.964962 40528 common.cpp:189] Name: GeForce GTX 1070
I0615 11:40:41.964962 40528 common.cpp:190] Total global memory: 8589934592
I0615 11:40:41.964962 40528 common.cpp:191] Total shared memory per block: 49152
I0615 11:40:41.964962 40528 common.cpp:192] Total registers per block: 65536
I0615 11:40:41.964962 40528 common.cpp:193] Warp size: 32
I0615 11:40:41.964962 40528 common.cpp:194] Maximum memory pitch: 2147483647
I0615 11:40:41.964962 40528 common.cpp:195] Maximum threads per block: 1024
I0615 11:40:41.964962 40528 common.cpp:196] Maximum dimension of block: 1024, 1024, 64
I0615 11:40:41.964962 40528 common.cpp:199] Maximum dimension of grid: 2147483647, 65535, 65535
I0615 11:40:41.964962 40528 common.cpp:202] Clock rate: 1771500
I0615 11:40:41.964962 40528 common.cpp:203] Total constant memory: 65536
I0615 11:40:41.964962 40528 common.cpp:204] Texture alignment: 512
I0615 11:40:41.964962 40528 common.cpp:205] Concurrent copy and execution: Yes
I0615 11:40:41.964962 40528 common.cpp:207] Number of multiprocessors: 15
I0615 11:40:41.964962 40528 common.cpp:208] Kernel execution timeout: Yes
I0615 01:10:55.654201 1904 common.cpp:193] Device id: 1
I0615 01:10:55.654223 1904 common.cpp:194] Major revision number: 5
I0615 01:10:55.654238 1904 common.cpp:195] Minor revision number: 2
I0615 01:10:55.654240 1904 common.cpp:196] Name: GeForce GTX TITAN X
I0615 01:10:55.654243 1904 common.cpp:197] Total global memory: 12800163840
I0615 01:10:55.654247 1904 common.cpp:198] Total shared memory per block: 49152
I0615 01:10:55.654249 1904 common.cpp:199] Total registers per block: 65536
I0615 01:10:55.654254 1904 common.cpp:200] Warp size: 32
I0615 01:10:55.654258 1904 common.cpp:201] Maximum memory pitch: 2147483647
I0615 01:10:55.654273 1904 common.cpp:202] Maximum threads per block: 1024
I0615 01:10:55.654278 1904 common.cpp:203] Maximum dimension of block: 1024, 1024, 64
I0615 01:10:55.654294 1904 common.cpp:206] Maximum dimension of grid: 2147483647, 65535, 65535
I0615 01:10:55.654299 1904 common.cpp:209] Clock rate: 1215500
I0615 01:10:55.654302 1904 common.cpp:210] Total constant memory: 65536
I0615 01:10:55.654306 1904 common.cpp:211] Texture alignment: 512
I0615 01:10:55.654310 1904 common.cpp:212] Concurrent copy and execution: Yes
I0615 01:10:55.654315 1904 common.cpp:214] Number of multiprocessors: 24
I0615 01:10:55.654319 1904 common.cpp:215] Kernel execution timeout: No
...
'''
command = [self.COMMAND_CAFFE, 'device_query', '-gpu', 'all']
log.debug('Running binary: %s', command)
o = misc.run_command(command) #, shell=True )
if o is None:
raise ConnoisseurException(responses.INTERNAL_SERVER_ERROR,
'Error while running Caffe...')
# parse the output
devices = []
d = None
for l in o.splitlines():
try:
_, l = l.split('] ', 1)
k, v = [s.strip() for s in l.split(':', 1)]
if k == 'Device id':
if d is not None:
devices.append(d)
d = {}
d[k] = v
except Exception:
pass
if d is not None:
devices.append(d)
return devices
def get_adapted_image_size(self):
""" returns (H,W) of the whole as seen by this adapter """
raise ConnoisseurException(responses.NOT_IMPLEMENTED, 'Class adapter must implement "get_adapted_image_size" method')
def get_polygons(self, image, model, args):
''' partition image into regions producing low-res polygons
This method does not guarantee full image partitioning by removing the uncertain regions
'''
Worig, Horig = image.size()
num_points = int(args.get('points', 100))
border = float(args.get('border', 5))
border = int(round(border * np.mean([Worig, Horig]) / 100.0))
pts, num_points_x, num_points_y, sw, sh = distribute_points(
num_points, Worig, Horig, border, equal=False, return_all=True)
# estimate superpixel size
W = self.side
sx = 1.0
if max(Worig, Horig) > W:
sx = float(W) / max(Worig, Horig)
sw = int(
round(sw * sx)
) # this provides an approximate number of centroids as requested
# load SLIC super-pixel segmented image
log.debug('Regions classifier: requesting superpixels of size: %s', sw)
pix = image.pixels(
operations=
'slice=,,1,1&resize=%s,%s,BC,MX&depth=8,d,u&transform=superpixels,%s,0.8&format=tiff'
% (self.side, self.side, sw))
W, H = pix.shape[0:2]
# get regional centroids
label_offset = 1 # region prop function uses 0 as background, we bump the class number and later subtract
segments = skimage.measure.regionprops(pix + label_offset, cache=True)
num_segs = len(segments)
log.debug('Regions classifier: segmented the image into %s segments',
num_segs)
if num_segs < 1:
raise ConnoisseurException(responses.NO_CONTENT,
'Regions classifier: no results')
# compute scaling factors
sx = 1.0
sy = 1.0
if Worig != W or Horig != H:
sx = float(W) / Worig
sy = float(H) / Horig
log.debug(
'Regions classifier: Original image is larger, use scaling factors: %s,%s',
sx, sy)
# scale params to resized image
border = int(round(border * sx))
# get regional centroids in original image space
points = []
for i, s in enumerate(segments):
x, y = s.centroid
if x > border and y > border and x <= W - border and y <= H - border:
points.append((x / sx, y / sy))
# classify regional centroids
# load image
#pim = image.pixels(operations='slice=,,1,1&resize=6000,6000,BC,MX&depth=8,d,u&format=png')
#W,H = pim.shape[0:2]
adapter = model.create_adapter_pixels(model=model,
args=args,
image=image)
# classify points
results = []
for x, y in points:
#pix = image_patch(pim, int(x), int(y), model.db_patch_width, model.db_patch_height)
pix = adapter.get_adapted_image_patch(gob=point(x=y, y=x))
out_class, goodness = model.classify(pix)
label, accuracy, ignored = get_class_info(model,
out_class,
at_goodness=0)
confidence = get_sample_confidence(accuracy, goodness)
results.append({
'x': x,
'y': y,
'id': out_class,
'label': label,
'ignored': ignored,
'accuracy': accuracy,
'goodness': goodness,
'confidence': confidence,
})
if len(results) < 1:
raise ConnoisseurException(responses.NO_CONTENT,
'Regions classifier: no results')
#------------------------------------------------------
# compute Voronoi polygons
#------------------------------------------------------
log.debug('Regions classifier: computing Voronoi')
points = [[r['x'], r['y']] for r in results]
bounds = [[0.0, Worig], [0.0, Horig]]
#radii = [r['w'] for r in results]
# log.debug('Regions classifier: bounds %s', bounds)
# log.debug('Regions classifier: points %s', points)
cells = pyvoro.compute_2d_voronoi(
points,
bounds,
2.0, # block size
#radii = radii
)
log.debug('Regions classifier: producing polygons')
for i in range(len(results)):
results[i]['gob'] = 'polygon'
results[i]['vertex'] = [(p[0], p[1]) for p in cells[i]['vertices']]
results[i]['area'] = cells[i]['volume']
return results
def classify(self, image, model, args):
''' partition image into regions producing low-res polygons
This method guarantees full image partitioning covering the area of the uncertain samples
'''
Worig, Horig = image.size()
num_points = int(args.get('points', 100))
border = float(args.get('border', 5))
border = int(round(border * np.mean([Worig, Horig]) / 100.0))
my_goodness = float(args.get('goodness',
model.minimum_goodness * 100)) / 100.0
my_accuracy = float(args.get('accuracy', model.minimum_accuracy * 100))
my_confidence = float(args.get('confidence', 0))
# uniform sampling
pts, num_points_x, num_points_y, sw, sh = distribute_points(
num_points, Worig, Horig, border, equal=False, return_all=True)
# load image
# pim = image.pixels(operations='slice=,,1,1&resize=6000,6000,BC,MX&depth=8,d,u&format=png')
# W,H = pim.shape[0:2]
# # compute scaling factor
# sx=1.0; sy=1.0
# if Worig>W or Horig>H:
# sx = float(W) / Worig
# sy = float(H) / Horig
# log.debug('Classify: Original image is larger, use scaling factors: %s,%s', sx, sy)
adapter = model.create_adapter_pixels(model=model,
args=args,
image=image)
# classify points
results = []
for x, y in pts:
#pix = image_patch(pim, int(x*sx), int(y*sy), model.db_patch_width, model.db_patch_height)
pix = adapter.get_adapted_image_patch(gob=point(x=y, y=x))
out_class, goodness = model.classify(pix)
label, accuracy, ignored = get_class_info(model,
out_class,
at_goodness=my_goodness)
confidence = get_sample_confidence(accuracy, goodness)
if goodness >= my_goodness and accuracy >= my_accuracy and confidence >= my_confidence and ignored is False:
#print '%s,%s classified as "%s" with %s goodness score'%(x, y, predicted_class, goodness)
#log.debug('%s,%s classified as "%s" with %s goodness score'%(x, y, predicted_class, goodness))
results.append({
'x': x,
'y': y,
'id': out_class,
'goodness': goodness,
})
if len(results) < 1:
raise ConnoisseurException(responses.NO_CONTENT,
'Regions classifier: no results')
#------------------------------------------------------
# init dense grid with measurements
#------------------------------------------------------
label_offset = 1 # region prop function uses 0 as background, we bump the class number and later subtract
def img_to_pp(p):
return (int(round(
(p[0] - border) / sw)), int(round((p[1] - border) / sh)))
def pp_to_img(p):
return (int(round(p[0] * sw + border)),
int(round(p[1] * sh + border)))
# create a grid with class labels
Gi = num_points_y
Gj = num_points_x + 1
G = np.zeros((Gi, Gj), dtype=np.uint8)
C = np.zeros((Gi, Gj), dtype=np.double)
for r in results:
x, y = img_to_pp((r['x'], r['y']))
idx = r['id']
goodness = r['goodness']
G[y, x] = idx + label_offset
C[y, x] = goodness
#skimage.io.imsave(grid_file, G)
#------------------------------------------------------
# compute quasi-convex region centroids
#------------------------------------------------------
results = []
for idx in range(model.number_classes_in_model):
GG = np.zeros((Gi, Gj), dtype=np.uint8)
for i in range(0, Gi):
for j in range(0, Gj):
if G[i, j] == idx + label_offset:
GG[i, j] = 255
else:
GG[i, j] = 0
#grid_flt_file = 'D:\\develop\\caffe\\regions\\grid_%s.tif'%(idx)
#skimage.io.imsave(grid_flt_file, GG)
# ensure boundary is background
GGG = np.zeros((Gi + 2, Gj + 2), dtype=np.uint8)
GGG[1:Gi + 1, 1:Gj + 1] = GG
distance = distance_transform_edt(GGG)
distance = distance[1:Gi + 1, 1:Gj + 1]
#grid_flt_file = 'D:\\develop\\caffe\\regions\\grid_%s_distance.tif'%(idx)
#skimage.io.imsave(grid_flt_file, distance)
#local_maxi = peak_local_max(distance, indices=False, footprint=np.ones((3, 3)), labels=GG)
local_maxi = corner_peaks(distance,
indices=False,
footprint=np.ones((3, 3)),
labels=GG)
markers = ndi.label(local_maxi)[0]
labeled = watershed(-distance, markers, mask=GG)
segments = skimage.measure.regionprops(labeled,
intensity_image=C,
cache=True)
for i, r in enumerate(segments):
c = r.centroid
w = r.area
#w = r.convex_area
#goodness = r.max_intensity
goodness = r.mean_intensity
label, accuracy, ignored = get_class_info(model,
out_class,
at_goodness=0)
confidence = get_sample_confidence(accuracy, goodness)
y, x = pp_to_img(c)
out_class = idx
results.append({
'x': x,
'y': y,
'w': w,
'area': w,
'area_seg': w,
'id': out_class,
#'color': get_color_html(out_class),
'label': label,
'accuracy': accuracy,
'goodness': goodness,
'confidence': confidence,
})
if len(results) < 1:
raise ConnoisseurException(responses.NO_CONTENT,
'Region classifier: no results')
#------------------------------------------------------
# compute Voronoi polygons
#------------------------------------------------------
points = [[r['x'], r['y']] for r in results]
bounds = [[0.0, Worig], [0.0, Horig]]
radii = [r['w'] for r in results]
cells = pyvoro.compute_2d_voronoi(
points,
bounds,
2.0, # block size
radii=radii)
for i in range(len(results)):
results[i]['gob'] = 'polygon'
results[i]['vertex'] = [(p[0], p[1]) for p in cells[i]['vertices']]
results[i]['area'] = cells[i]['volume']
# hierarchical passes over samples whose classes have child models
# return results
return DataToken(data=results, mime='table/gobs', name='Substrate')
def get_class_name(self, node, **kw):
""" must implement: return a class name for a given node """
raise ConnoisseurException(responses.NOT_IMPLEMENTED, 'Class adapter must implement "get_class_name" method')
def get_adapted_image_patch(self, gob, **kw):
""" return image pixels for a given gobject as a numpy array """
raise ConnoisseurException(responses.NOT_IMPLEMENTED, 'Class adapter must implement "get_adapted_image_patch" method')