def init_ftext(minCompSize=5):
scaleFactor = 1.6
nleves = -1
edgeThreshold = 13
keypointTypes = 3
#charClsModelFile = '/tmp/cvBoostChar.xml'
edgeThreshold = 14
fastex = FASTex(edgeThreshold=edgeThreshold)
def draw_missed_letters_tile(
input_dir='/datagrid/personal/TextSpotter/FastTextEval/ICDAR-Train',
color=0,
edgeThreshold=13,
inter=True,
scalingFactor=1.6,
segmList=[]):
ft = FASTex(process_color=color, edgeThreshold=edgeThreshold)
d = input_dir
subdirs = [
os.path.join(d, o) for o in os.listdir(d)
if os.path.isdir(os.path.join(d, o))
]
subdirs = np.sort(subdirs)
lastDir = ''
for dir_name in subdirs:
file_name = '{0}/evaluation.npz'.format(dir_name)
if not os.path.exists(file_name):
continue
vars_dict = np.load(file_name)
inputDir = vars_dict['inputDir']
lastDir = dir_name
if 'letterKeypointHistogram' in vars_dict.keys():
letterKeypointHistogram = vars_dict['letterKeypointHistogram']
letterKeypointHistogram = dict(letterKeypointHistogram.tolist())
print(lastDir)
missing_letters = vars_dict['missing_letters']
missing_letters = dict(missing_letters.tolist())
segmDir = '{0}/segmentations'.format(inputDir)
segmDir = '/datagrid/personal/TextSpotter/evaluation-sets/icdar2013-Test/segmentations'
keys = []
ticks = []
values = []
values.append([])
values.append([])
values.append([])
values.append([])
ticks.append([])
ticks.append([])
ticks.append([])
ticks.append([])
listlen = 0
for letter in letterKeypointHistogram.keys():
keys.append(letter)
values[0].append(0)
ticks[0].append(listlen)
values[1].append(0)
ticks[1].append(listlen + 0.2)
values[2].append(0)
ticks[2].append(listlen + 0.4)
values[3].append(0)
ticks[3].append(listlen + 0.6)
for num in letterKeypointHistogram[letter].keys():
values[num][listlen] = letterKeypointHistogram[letter][num]
listlen += 1
indices = sorted(range(len(values[0])), key=lambda x: values[0][x])
indices.reverse()
border = 15
missLetter = []
imagesMiss = {}
for letter in np.asarray(keys)[np.asarray(indices)]:
if not missing_letters.has_key(letter):
continue
arr = missing_letters[letter]
for i in range(len(arr)):
miss = arr[i]
if len(segmList) > 0:
base = os.path.basename(miss[0])
if not base in segmList:
continue
missLetter.append(miss)
if imagesMiss.has_key(miss[0]):
imagesMiss[miss[0]].append(miss[1])
else:
imagesMiss[miss[0]] = []
imagesMiss[miss[0]].append(miss[1])
rowSize = len(imagesMiss.keys())
f, axes = plt.subplots(2, len(imagesMiss.keys()))
plt.subplots_adjust(left=None,
bottom=None,
right=None,
top=None,
wspace=None,
hspace=None)
figNo = 0
for image in imagesMiss.keys():
if len(imagesMiss.keys()) > 1:
ax0 = axes[0][figNo]
ax = axes[1][figNo]
else:
ax0 = axes[figNo]
ax = axes[figNo]
figNo += 1
if color == 1:
img = cv2.imread(image)
else:
img = cv2.imread(image, 0)
baseName = os.path.basename(image)
baseName = baseName[:-4]
segmImg = '{0}/{1}_GT.bmp'.format(segmDir, baseName)
if not os.path.exists(segmImg):
segmImg = '{0}/gt_{1}.png'.format(segmDir, baseName)
segmImg = cv2.imread(segmImg)
segmentations = ft.getCharSegmentations(img)
keypoints = ft.getLastDetectionKeypoints()
if color == 1:
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
for i in range(len(imagesMiss[image])):
if i == 0:
orBox = imagesMiss[image][0]
else:
orBox = utils.union(orBox, imagesMiss[image][i])
gt0 = orBox
gt = [
gt0[0] - border, gt0[1] - border, gt0[2] + border, gt0[3] + border
]
gt[0] = max(0, gt[0])
gt[1] = max(0, gt[1])
gt[2] = min(img.shape[1], gt[2])
gt[3] = min(img.shape[0], gt[3])
zoom = img[gt[1]:gt[3], gt[0]:gt[2]]
ax.imshow(zoom, cmap=pylab.gray(), interpolation='nearest')
ax0.imshow(zoom, cmap=pylab.gray(), interpolation='nearest')
centers = segmImg[keypoints[:, 1].astype(int),
keypoints[:, 0].astype(int)]
keypointsInsideMask = centers == (255, 255, 255)
keypointsInsideMask = np.invert(
np.bitwise_and(
np.bitwise_and(keypointsInsideMask[:, 0],
keypointsInsideMask[:, 1]),
keypointsInsideMask[:, 2]))
keypointsInside = keypoints[keypointsInsideMask, :]
mask = (keypoints[:, 0] > gt[0]) * (keypoints[:, 0] < gt[2]) * (
keypoints[:, 1] > gt[1]) * (keypoints[:, 1] < gt[3])
kpMask = keypoints[mask]
kpMask[:, 0] = kpMask[:, 0] - gt[0]
kpMask[:, 1] = kpMask[:, 1] - gt[1]
kpMask[:, 7] = kpMask[:, 7] - gt[0]
kpMask[:, 8] = kpMask[:, 8] - gt[1]
ax.plot(kpMask[:, 0], kpMask[:, 1], 'ro')
ax.xaxis.set_ticklabels([])
ax.yaxis.set_ticklabels([])
ax0.xaxis.set_ticklabels([])
ax0.yaxis.set_ticklabels([])
for k in range(kpMask.shape[0]):
ax.plot([kpMask[k, 0], kpMask[k, 7]], [kpMask[k, 1], kpMask[k, 8]],
'r-')
style = 'rx'
if kpMask.shape[1] > 9:
for k in range(3):
maski = kpMask[:, 9] == k + 1
if k == 1:
style = "rv"
if k == 2:
style = "rs"
if k == 4:
style = "bo"
if k == 5:
style = "yo"
ax.plot([kpMask[maski, 7]], [kpMask[maski, 8]], style)
for i in range(len(imagesMiss[image])):
gt0 = imagesMiss[image][i]
mask = (keypointsInside[:, 0] >
gt[0]) * (keypointsInside[:, 0] < gt[2]) * (
keypointsInside[:, 1] > gt[1]) * (keypointsInside[:, 1]
< gt[3])
kpMask = keypointsInside[mask]
keypointsInside[:, 0] = keypointsInside[:, 0] - gt[0]
keypointsInside[:, 1] = keypointsInside[:, 1] - gt[1]
keypointsInside[:, 7] = keypointsInside[:, 7] - gt[0]
keypointsInside[:, 8] = keypointsInside[:, 8] - gt[1]
ax.plot(keypointsInside[:, 0], keypointsInside[:, 1], 'go')
for k in range(keypointsInside.shape[0]):
ax.plot([keypointsInside[k, 0], keypointsInside[k, 7]],
[keypointsInside[k, 1], keypointsInside[k, 8]], 'g-')
ax.set_xlim(0, gt[2] - max(0, gt[0]))
ax.set_ylim((gt[3] - max(0, gt[1]), 0))
line = mlines.Line2D(np.array([
gt0[0] - gt[0], gt0[2] - gt[0], gt0[2] - gt[0], gt0[0] - gt[0],
gt0[0] - gt[0]
]),
np.array([
gt0[1] - gt[1], gt0[1] - gt[1],
gt0[3] - gt[1], gt0[3] - gt[1],
gt0[1] - gt[1]
]),
lw=5.,
alpha=0.6,
color='r')
ax0.add_line(line)
plt.show()
def draw_missed_segm(
input_dir='/datagrid/personal/TextSpotter/FastTextEval/ICDAR-Train',
color=0,
edgeThreshold=12,
inter=True,
scalingFactor=0.5):
ft = FASTex(process_color=color, edgeThreshold=edgeThreshold)
d = input_dir
subdirs = [
os.path.join(d, o) for o in os.listdir(d)
if os.path.isdir(os.path.join(d, o))
]
subdirs = np.sort(subdirs)
for dir_name in subdirs:
file_name = '{0}/evaluation.npz'.format(dir_name)
if not os.path.exists(file_name):
continue
vars_dict = np.load(file_name)
inputDir = vars_dict['inputDir']
missing_segm = vars['missing_segm']
missing_segm = dict(missing_segm.tolist())
segmDir = '{0}/segmentations'.format(inputDir)
for image in missing_segm.keys():
arr = missing_segm[image]
if color == 1:
img = cv2.imread(image)
else:
img = cv2.imread(image, 0)
segmentations = ft.getCharSegmentations(img)
keypoints = ft.getLastDetectionKeypoints()
baseName = os.path.basename(image)
baseName = baseName[:-4]
segmImg = '{0}/{1}_GT.bmp'.format(segmDir, baseName)
if not os.path.exists(segmImg):
segmImg = '{0}/gt_{1}.png'.format(segmDir, baseName)
segmImg = cv2.imread(segmImg)
centers = segmImg[keypoints[:, 1].astype(int),
keypoints[:, 0].astype(int)]
keypointsInsideMask = centers == (255, 255, 255)
keypointsInsideMask = np.invert(
np.bitwise_and(
np.bitwise_and(keypointsInsideMask[:, 0],
keypointsInsideMask[:, 1]),
keypointsInsideMask[:, 2]))
keypointsInside = keypoints[keypointsInsideMask, :]
f = plt.figure(num=110)
ax = f.add_subplot(111)
ax.imshow(img, cmap=pylab.gray(), interpolation='nearest')
style = "rx"
for k in range(6):
maski = keypoints[:, 9] == k + 1
if k == 1:
style = "rv"
if k == 2:
style = "ro"
if k == 4:
style = "bo"
if k == 5:
style = "yo"
ax.plot(keypoints[maski, 0], keypoints[maski, 1], style)
ax.plot(keypointsInside[:, 0], keypointsInside[:, 1], 'go')
ax.set_xlim(0, img.shape[1])
ax.set_ylim(img.shape[0], 0)
for i in range(len(arr)):
miss_gt = arr[i]
line = mlines.Line2D(np.array(
[miss_gt[0], miss_gt[2], miss_gt[2], miss_gt[0], miss_gt[0]]),
np.array([
miss_gt[1], miss_gt[1], miss_gt[3],
miss_gt[3], miss_gt[1]
]),
lw=5.,
alpha=0.4,
color='b')
ax.add_line(line)
ax.set_title('Missing segmentation: {0}'.format(image))
plt.show()
def draw_missed_letters_figure(
input_dir='/datagrid/personal/TextSpotter/FastTextEval/ICDAR-Train',
color=0,
edgeThreshold=13,
inter=True,
scalingFactor=0.5,
segmList=[]):
ft = FASTex(process_color=color, edgeThreshold=edgeThreshold)
d = input_dir
subdirs = [
os.path.join(d, o) for o in os.listdir(d)
if os.path.isdir(os.path.join(d, o))
]
subdirs = np.sort(subdirs)
lastDir = ''
for dir_name in subdirs:
file_name = '{0}/evaluation.npz'.format(dir_name)
if not os.path.exists(file_name):
continue
vars_dict = np.load(file_name)
inputDir = vars_dict['inputDir']
lastDir = dir_name
if 'letterKeypointHistogram' in vars_dict.keys():
letterKeypointHistogram = vars_dict['letterKeypointHistogram']
letterKeypointHistogram = dict(letterKeypointHistogram.tolist())
print(lastDir)
missing_letters = vars_dict['missing_letters']
missing_letters = dict(missing_letters.tolist())
keys = []
ticks = []
values = []
values.append([])
values.append([])
values.append([])
values.append([])
ticks.append([])
ticks.append([])
ticks.append([])
ticks.append([])
listlen = 0
for letter in letterKeypointHistogram.keys():
keys.append(letter)
values[0].append(0)
ticks[0].append(listlen)
values[1].append(0)
ticks[1].append(listlen + 0.2)
values[2].append(0)
ticks[2].append(listlen + 0.4)
values[3].append(0)
ticks[3].append(listlen + 0.6)
for num in letterKeypointHistogram[letter].keys():
values[num][listlen] = letterKeypointHistogram[letter][num]
listlen += 1
indices = sorted(range(len(values[0])), key=lambda x: values[0][x])
indices.reverse()
missLetter = []
imagesMiss = {}
for letter in np.asarray(keys)[np.asarray(indices)]:
if not missing_letters.has_key(letter):
continue
arr = missing_letters[letter]
for i in range(len(arr)):
miss = arr[i]
if len(segmList) > 0:
base = os.path.basename(miss[0])
if not base in segmList:
continue
missLetter.append(miss)
if imagesMiss.has_key(miss[0]):
imagesMiss[miss[0]].append(miss[1])
else:
imagesMiss[miss[0]] = []
imagesMiss[miss[0]].append(miss[1])
for image in imagesMiss.keys():
f = plt.figure(num=250)
ax = f.add_subplot(111)
imgc2 = cv2.imread(image)
img = cv2.imread(image, cv2.IMREAD_GRAYSCALE)
imgc2 = cv2.cvtColor(imgc2, cv2.COLOR_BGR2RGB)
ax.imshow(imgc2)
segmentations = ft.getCharSegmentations(img)
keypoints = ft.getLastDetectionKeypoints()
octaves = np.unique(keypoints[:, 2])
maxOctave = np.max(octaves)
scales = ft.getImageScales()
for i in range(int(maxOctave) + 1):
octavePoints = keypoints[keypoints[:, 2] == i, :]
c = 'red'
if len(octavePoints) > 0 and octavePoints.shape[1] > 9:
for k in range(6):
maski = octavePoints[:, 9] == k + 1
if k == 1:
style = "rv"
if k == 2:
style = "ro"
if k == 4:
style = "bo"
c = 'blue'
if k == 5:
style = "yo"
continue
s = 10 / scales[i]
ax.scatter(octavePoints[maski, 0],
octavePoints[maski, 1],
c=c,
s=s)
for i in range(len(imagesMiss[image])):
gt0 = imagesMiss[image][i]
line = mlines.Line2D(
np.array([gt0[0], gt0[2], gt0[2], gt0[0], gt0[0]]),
np.array([gt0[1], gt0[1], gt0[3], gt0[3], gt0[1]]),
lw=5.,
alpha=0.6,
color='r')
ax.add_line(line)
plt.subplots_adjust(top=1,
bottom=0,
right=1,
left=0,
hspace=0,
wspace=0)
plt.margins(0, 0)
ax.set_xlim([0, imgc2.shape[1]])
ax.set_ylim([imgc2.shape[0], 0])
ax.axes.get_xaxis().set_ticks([])
ax.axes.get_yaxis().set_ticks([])
plt.show()
def compare_missed_segm(
input_dir='/datagrid/personal/TextSpotter/FastTextEval/experiments/segmentation',
input_dir2='/datagrid/personal/TextSpotter/FastTextEval/experiments/segmentationg',
showPictures=False):
ft = FASTex()
(ms, dirs) = read_segm_data(input_dir)
(ms2, dirs2) = read_segm_data(input_dir2, 'g')
ms.extend(ms2)
dirs.extend(dirs2)
sumHash = {}
for j in np.arange(0, len(ms)):
missing_segm = ms[j]
for image in missing_segm.keys():
arr = missing_segm[image]
if not sumHash.has_key(image):
sumHash[image] = arr
continue
for i in range(len(arr)):
miss_gt = arr[i]
check = sumHash[image]
hasGt = False
for k in range(len(check)):
miss_gt2 = check[k]
if miss_gt == miss_gt2:
hasGt = True
if not hasGt:
sumHash[image].append(miss_gt)
missing_segm = ms[0]
data = []
dataf = []
gt_id = 0
columns = ['Img', 'GT Id']
for image in sumHash.keys():
arr = sumHash[image]
f = None
for i in range(len(arr)):
orValue = False
miss_gt = arr[i]
row = []
row.append(os.path.basename(image))
row.append(gt_id)
gt_id += 1
rowf = []
for j in np.arange(0, len(ms)):
if gt_id == 1:
columns.append(dirs[j])
msj = ms[j]
hasSegmj = True
val = 1
if msj.has_key(image):
arrj = msj[image]
for k in range(len(arrj)):
miss_gtj = arrj[k]
if miss_gtj == miss_gt:
hasSegmj = False
val = 0
break
row.append(hasSegmj)
rowf.append(val)
orValue = orValue or hasSegmj
if orValue:
rowf.append(1)
else:
rowf.append(0)
if showPictures:
img = cv2.imread(image)
imgg = cv2.imread(image, cv2.IMREAD_GRAYSCALE)
if f == None:
f, axes = plt.subplots(1, 2, figsize=(16, 3))
f.suptitle('Missing segmentation: {0}'.format(image))
ax = axes[0]
ax.imshow(img,
cmap=pylab.gray(),
interpolation='nearest')
ax = axes[1]
ax.imshow(imgg,
cmap=pylab.gray(),
interpolation='nearest')
orBox = miss_gt
segmentations = ft.getCharSegmentations(imgg)
keypoints = ft.getLastDetectionKeypoints()
style = 'rx'
for k in range(5):
maski = keypoints[:, 9] == k + 1
if k == 1:
style = "rv"
if k == 2:
style = "ro"
if k == 4:
style = "bo"
ax.plot(keypoints[maski, 0], keypoints[maski, 1],
style)
for k in range(keypoints.shape[0]):
ax.plot([keypoints[k, 0], keypoints[k, 7]],
[keypoints[k, 1], keypoints[k, 8]], 'r-')
ax = axes[0]
else:
orBox = utils.union(orBox, miss_gt)
line = mlines.Line2D(np.array([
miss_gt[0], miss_gt[2], miss_gt[2], miss_gt[0],
miss_gt[0]
]),
np.array([
miss_gt[1], miss_gt[1],
miss_gt[3], miss_gt[3], miss_gt[1]
]),
lw=5.,
alpha=0.6,
color='r')
ax.add_line(line)
row.append(orValue)
data.append(row)
dataf.append(rowf)
if f != None:
ax = axes[0]
ax.set_xlim(orBox[0] - 20, orBox[2] + 20)
ax.set_ylim(orBox[3] + 20, orBox[1] - 20)
ax = axes[1]
ax.set_xlim(orBox[0] - 20, orBox[2] + 20)
ax.set_ylim(orBox[3] + 20, orBox[1] - 20)
plt.show()
columns.append("OR")
data = np.array(data)
dataf = np.array(dataf)
df = pandas.DataFrame(data=data, columns=columns)
#print(df)
sumCols = dataf.sum(0)
sumCols = dataf.shape[0] - sumCols
print("Missing Segmentations:")
print(sumCols)
indices = np.argsort(sumCols)
bestFactor = indices[1]
missing_segm = ms[bestFactor]
print("Best factor: {0}".format(dirs[bestFactor]))
maskBest = dataf[:, bestFactor] == 0
datafSec = dataf[maskBest, :]
sumCols = datafSec.sum(0)
sumCols = datafSec.shape[0] - sumCols
print("Missing Segmentations 2 best:")
print(sumCols)
indices = np.argsort(sumCols)
bestFactor2 = indices[1]
print("Best factor 2: {0}, missing segmentations: {1} -> {2}".format(
dirs[bestFactor2], datafSec.shape[0], sumCols[indices[1]]))
maskBest = datafSec[:, bestFactor2] == 0
dataf3 = datafSec[maskBest, :]
sumCols = dataf3.sum(0)
sumCols = dataf3.shape[0] - sumCols
indices = np.argsort(sumCols)
bestFactor2 = indices[1]
print("Best factor 3: {0}, missing segmentations: {1} -> {2}".format(
dirs[bestFactor2], dataf3.shape[0], sumCols[indices[1]]))
def draw_missed_letters(
input_dir='/datagrid/personal/TextSpotter/FastTextEval/ICDAR-Train',
color=0,
edgeThreshold=12,
inter=True,
scalingFactor=0.5):
ft = FASTex(process_color=color, edgeThreshold=edgeThreshold)
d = input_dir
subdirs = [
os.path.join(d, o) for o in os.listdir(d)
if os.path.isdir(os.path.join(d, o))
]
subdirs = np.sort(subdirs)
lastDir = ''
for dir_name in subdirs:
file_name = '{0}/evaluation.npz'.format(dir_name)
if not os.path.exists(file_name):
continue
vars_dict = np.load(file_name)
inputDir = vars_dict['inputDir']
lastDir = dir_name
if 'letterKeypointHistogram' in vars.keys():
letterKeypointHistogram = vars_dict['letterKeypointHistogram']
letterKeypointHistogram = dict(letterKeypointHistogram.tolist())
print(lastDir)
missing_letters = vars['missing_letters']
missing_letters = dict(missing_letters.tolist())
segmDir = '{0}/segmentations'.format(inputDir)
keys = []
ticks = []
values = []
values.append([])
values.append([])
values.append([])
values.append([])
ticks.append([])
ticks.append([])
ticks.append([])
ticks.append([])
listlen = 0
for letter in letterKeypointHistogram.keys():
keys.append(letter)
values[0].append(0)
ticks[0].append(listlen)
values[1].append(0)
ticks[1].append(listlen + 0.2)
values[2].append(0)
ticks[2].append(listlen + 0.4)
values[3].append(0)
ticks[3].append(listlen + 0.6)
for num in letterKeypointHistogram[letter].keys():
values[num][listlen] = letterKeypointHistogram[letter][num]
listlen += 1
indices = sorted(range(len(values[0])), key=lambda x: values[0][x])
indices.reverse()
border = 25
for letter in np.asarray(keys)[np.asarray(indices)]:
if not missing_letters.has_key(letter):
continue
arr = missing_letters[letter]
for i in range(len(arr)):
miss = arr[i]
gt0 = miss[1]
gt = [
gt0[0] - border, gt0[1] - border, gt0[2] + border,
gt0[3] + border
]
gt[0] = max(0, gt[0])
gt[1] = max(0, gt[1])
if color == 1:
img = cv2.imread(miss[0])
else:
img = cv2.imread(miss[0], 0)
gt[2] = min(img.shape[1], gt[2])
gt[3] = min(img.shape[0], gt[3])
baseName = os.path.basename(miss[0])
baseName = baseName[:-4]
segmImg = '{0}/{1}_GT.bmp'.format(segmDir, baseName)
segmImg = '{0}/{1}_GT.bmp'.format(segmDir, baseName)
if not os.path.exists(segmImg):
segmImg = '{0}/gt_{1}.png'.format(segmDir, baseName)
segmImg = cv2.imread(segmImg)
segmentations = ft.getCharSegmentations(img)
keypoints = ft.getLastDetectionKeypoints()
scales = ft.getImageScales()
centers = segmImg[keypoints[:, 1].astype(int),
keypoints[:, 0].astype(int)]
keypointsInsideMask = centers == (255, 255, 255)
keypointsInsideMask = np.invert(
np.bitwise_and(
np.bitwise_and(keypointsInsideMask[:, 0],
keypointsInsideMask[:, 1]),
keypointsInsideMask[:, 2]))
keypointsInside = keypoints[keypointsInsideMask, :]
octaves = np.unique(keypointsInside[:, 2])
maxOctave = 0
if octaves.shape[0] > 0:
maxOctave = np.max(octaves)
mask = (keypoints[:, 0] > gt[0]) * (keypoints[:, 0] < gt[2]) * (
keypoints[:, 1] > gt[1]) * (keypoints[:, 1] < gt[3])
images = []
octPoints = []
octScales = []
keypointsInRect = keypoints[mask, :]
for i in range(int(maxOctave) + 1):
scale = scales[i]
ft = FASTex(process_color=color, edgeThreshold=edgeThreshold)
octavePoints = keypointsInRect[keypointsInRect[:, 2] ==
i, :].copy()
if octavePoints.shape[0] > 0:
dst = ft.getImageAtScale(i)
images.append(dst)
octavePoints[:, 0] *= scales[i]
octavePoints[:, 1] *= scales[i]
octavePoints[:, 5] *= scales[i]
octavePoints[:, 6] *= scales[i]
octavePoints[:, 7] *= scales[i]
octavePoints[:, 8] *= scales[i]
octPoints.append(octavePoints)
octScales.append(scale)
f, axes = plt.subplots(1, 1 + len(images), figsize=(16, 3))
if len(images) > 0:
ax = axes[0]
else:
ax = axes
if color == 1:
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
zoom = img[gt[1]:gt[3], gt[0]:gt[2]]
ax.imshow(zoom, cmap=pylab.gray(), interpolation='nearest')
kpMask = keypoints[mask]
kpMask[:, 0] = kpMask[:, 0] - gt[0]
kpMask[:, 1] = kpMask[:, 1] - gt[1]
kpMask[:, 7] = kpMask[:, 7] - gt[0]
kpMask[:, 8] = kpMask[:, 8] - gt[1]
ax.plot(kpMask[:, 0], kpMask[:, 1], 'ro')
for k in range(kpMask.shape[0]):
ax.plot([kpMask[k, 0], kpMask[k, 7]],
[kpMask[k, 1], kpMask[k, 8]], 'r-')
style = 'rx'
if kpMask.shape[1] > 9:
for k in range(3):
maski = kpMask[:, 9] == k + 1
if k == 1:
style = "rv"
if k == 2:
style = "rs"
if k == 4:
style = "bo"
if k == 5:
style = "yo"
ax.plot([kpMask[maski, 7]], [kpMask[maski, 8]], style)
mask = (keypointsInside[:, 0] >
gt[0]) * (keypointsInside[:, 0] < gt[2]) * (
keypointsInside[:, 1] > gt[1]) * (keypointsInside[:, 1]
< gt[3])
kpMask = keypointsInside[mask]
keypointsInside[:, 0] = keypointsInside[:, 0] - gt[0]
keypointsInside[:, 1] = keypointsInside[:, 1] - gt[1]
keypointsInside[:, 7] = keypointsInside[:, 7] - gt[0]
keypointsInside[:, 8] = keypointsInside[:, 8] - gt[1]
ax.plot(keypointsInside[:, 0], keypointsInside[:, 1], 'go')
for k in range(keypointsInside.shape[0]):
ax.plot([keypointsInside[k, 0], keypointsInside[k, 7]],
[keypointsInside[k, 1], keypointsInside[k, 8]], 'g-')
ax.set_xlim(0, gt[2] - max(0, gt[0]))
ax.set_ylim((gt[3] - max(0, gt[1]), 0))
line = mlines.Line2D(np.array([
gt0[0] - gt[0], gt0[2] - gt[0], gt0[2] - gt[0], gt0[0] - gt[0],
gt0[0] - gt[0]
]),
np.array([
gt0[1] - gt[1], gt0[1] - gt[1],
gt0[3] - gt[1], gt0[3] - gt[1],
gt0[1] - gt[1]
]),
lw=5.,
alpha=0.4,
color='b')
ax.add_line(line)
f.suptitle('Missing letter: {0} ({1})'.format(gt0[4], miss[0]))
for ai in range(len(images)):
ax = axes[ai + 1]
scale = octScales[ai]
gts = (gt[0] * scale, gt[1] * scale, gt[2] * scale,
gt[3] * scale)
ax.plot(octPoints[ai][:, 0] - gts[0],
octPoints[ai][:, 1] - gts[1], 'ro')
zoom = images[ai][int(gt[1] * scale):int(gt[3] * scale),
int(gt[0] * scale):int(gt[2] * scale)]
ax.imshow(zoom, cmap=pylab.gray(), interpolation='nearest')
ax.set_xlim(0, gts[2] - max(0, gts[0]))
ax.set_ylim((gts[3] - max(0, gts[1]), 0))
plt.show()
@author: busta
'''
import numpy as np
import cv2
import sys
from ft import FASTex
from vis import draw_keypoints
if __name__ == '__main__':
outputDir = '/tmp'
edgeThreshold = 13
ft = FASTex(edgeThreshold=edgeThreshold, nlevels=-1, minCompSize=4)
imgName = '/datagrid/personal/TextSpotter/evaluation-sets/bornDigital/img_100.png'
if len(sys.argv) > 1:
if sys.argv[1].endswith(".png") or sys.argv[1].endswith(".jpg"):
imgName = sys.argv[1]
img = cv2.imread(imgName, 0)
imgc = cv2.imread(imgName)
#print(out)
segmentations = ft.getCharSegmentations(img, outputDir, 'base')
print segmentations
for i in range(segmentations.shape[0]):
rectn = segmentations[i, :]
def draw_missed_letters(input_dir='/datagrid/personal/TextSpotter/FastTextEval/ICDAR-Train', color = 0, edgeThreshold = 12, inter = True, scalingFactor=0.5):
ft = FASTex(process_color = color, edgeThreshold = edgeThreshold)
d=input_dir
subdirs = [os.path.join(d,o) for o in os.listdir(d) if os.path.isdir(os.path.join(d,o))]
subdirs = np.sort(subdirs)
lastDir = ''
for dir_name in subdirs:
file_name = '{0}/evaluation.npz'.format(dir_name)
if not os.path.exists(file_name):
continue
vars_dict = np.load(file_name)
inputDir = vars_dict['inputDir']
lastDir = dir_name
if 'letterKeypointHistogram' in vars.keys():
letterKeypointHistogram = vars_dict['letterKeypointHistogram']
letterKeypointHistogram = dict(letterKeypointHistogram.tolist())
print(lastDir)
missing_letters = vars['missing_letters']
missing_letters = dict(missing_letters.tolist())
segmDir = '{0}/segmentations'.format(inputDir)
keys = []
ticks = []
values = []
values.append([])
values.append([])
values.append([])
values.append([])
ticks.append([])
ticks.append([])
ticks.append([])
ticks.append([])
listlen = 0
for letter in letterKeypointHistogram.keys():
keys.append(letter)
values[0].append(0)
ticks[0].append(listlen)
values[1].append(0)
ticks[1].append(listlen + 0.2)
values[2].append(0)
ticks[2].append(listlen + 0.4)
values[3].append(0)
ticks[3].append(listlen + 0.6)
for num in letterKeypointHistogram[letter].keys():
values[num][listlen] = letterKeypointHistogram[letter][num]
listlen += 1
indices = sorted(range(len(values[0])),key=lambda x:values[0][x])
indices.reverse()
border = 25
for letter in np.asarray(keys)[np.asarray(indices)]:
if not missing_letters.has_key(letter):
continue
arr = missing_letters[letter]
for i in range(len(arr)):
miss = arr[i]
gt0 = miss[1]
gt = [gt0[0] - border, gt0[1] - border, gt0[2] + border, gt0[3] + border ]
gt[0] = max(0, gt[0])
gt[1] = max(0, gt[1])
if color == 1:
img = cv2.imread(miss[0])
else:
img = cv2.imread(miss[0], 0)
gt[2] = min(img.shape[1], gt[2])
gt[3] = min(img.shape[0], gt[3])
baseName = os.path.basename(miss[0])
baseName = baseName[:-4]
segmImg = '{0}/{1}_GT.bmp'.format(segmDir, baseName)
segmImg = '{0}/{1}_GT.bmp'.format(segmDir, baseName)
if not os.path.exists(segmImg):
segmImg = '{0}/gt_{1}.png'.format(segmDir, baseName)
segmImg = cv2.imread(segmImg)
segmentations = ft.getCharSegmentations(img)
keypoints = ft.getLastDetectionKeypoints()
scales = ft.getImageScales()
centers = segmImg[keypoints[:, 1].astype(int), keypoints[:, 0].astype(int)]
keypointsInsideMask = centers == (255, 255, 255)
keypointsInsideMask = np.invert(np.bitwise_and(np.bitwise_and(keypointsInsideMask[:, 0], keypointsInsideMask[:, 1]), keypointsInsideMask[:, 2]))
keypointsInside = keypoints[keypointsInsideMask, :]
octaves = np.unique( keypointsInside[:, 2])
maxOctave = 0
if octaves.shape[0] > 0:
maxOctave = np.max(octaves)
mask = (keypoints[:, 0] > gt[0]) * (keypoints[:, 0] < gt[2]) * (keypoints[:, 1] > gt[1]) * (keypoints[:, 1] < gt[3])
images = []
octPoints = []
octScales = []
keypointsInRect = keypoints[mask, :]
for i in range(int(maxOctave) + 1):
scale = scales[i]
ft = FASTex(process_color = color, edgeThreshold = edgeThreshold)
octavePoints = keypointsInRect[keypointsInRect[:, 2] == i, :].copy()
if octavePoints.shape[0] > 0:
dst = ft.getImageAtScale(i)
images.append(dst)
octavePoints[:, 0] *= scales[i]
octavePoints[:, 1] *= scales[i]
octavePoints[:, 5] *= scales[i]
octavePoints[:, 6] *= scales[i]
octavePoints[:, 7] *= scales[i]
octavePoints[:, 8] *= scales[i]
octPoints.append(octavePoints)
octScales.append(scale)
f, axes = plt.subplots(1, 1 + len(images), figsize=(16, 3))
if len(images) > 0:
ax = axes[0]
else:
ax = axes
if color == 1:
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
zoom = img[gt[1]:gt[3], gt[0]:gt[2]]
ax.imshow(zoom, cmap=pylab.gray(), interpolation='nearest')
kpMask = keypoints[mask]
kpMask[:, 0] = kpMask[:, 0] - gt[0]
kpMask[:, 1] = kpMask[:, 1] - gt[1]
kpMask[:, 7] = kpMask[:, 7] - gt[0]
kpMask[:, 8] = kpMask[:, 8] - gt[1]
ax.plot(kpMask[:, 0], kpMask[:, 1], 'ro')
for k in range(kpMask.shape[0]):
ax.plot([kpMask[k,0], kpMask[k,7]], [kpMask[k,1], kpMask[k,8]], 'r-')
style = 'rx'
if kpMask.shape[1] > 9:
for k in range(3):
maski = kpMask[:, 9] == k + 1
if k == 1:
style = "rv"
if k == 2:
style = "rs"
if k == 4:
style = "bo"
if k == 5:
style = "yo"
ax.plot([kpMask[maski,7]], [kpMask[maski,8]], style)
mask = (keypointsInside[:, 0] > gt[0]) * (keypointsInside[:, 0] < gt[2]) * (keypointsInside[:, 1] > gt[1]) * (keypointsInside[:, 1] < gt[3])
kpMask = keypointsInside[mask]
keypointsInside[:, 0] = keypointsInside[:, 0] - gt[0]
keypointsInside[:, 1] = keypointsInside[:, 1] - gt[1]
keypointsInside[:, 7] = keypointsInside[:, 7] - gt[0]
keypointsInside[:, 8] = keypointsInside[:, 8] - gt[1]
ax.plot(keypointsInside[:, 0], keypointsInside[:, 1], 'go')
for k in range(keypointsInside.shape[0]):
ax.plot([keypointsInside[k,0], keypointsInside[k,7]], [keypointsInside[k,1], keypointsInside[k,8]], 'g-')
ax.set_xlim(0, gt[2] - max(0, gt[0]))
ax.set_ylim((gt[3] - max(0, gt[1]), 0))
line = mlines.Line2D(np.array([gt0[0] - gt[0], gt0[2] - gt[0], gt0[2] - gt[0], gt0[0] - gt[0], gt0[0] - gt[0]]), np.array([gt0[1] - gt[1], gt0[1] - gt[1], gt0[3] - gt[1], gt0[3] - gt[1], gt0[1] - gt[1]]), lw=5., alpha=0.4, color='b')
ax.add_line(line)
f.suptitle('Missing letter: {0} ({1})'.format(gt0[4], miss[0]))
for ai in range(len(images)):
ax = axes[ai + 1]
scale = octScales[ai]
gts = (gt[0] * scale, gt[1] * scale, gt[2] * scale, gt[3] * scale)
ax.plot(octPoints[ai][:, 0] - gts[0], octPoints[ai][:, 1] - gts[1], 'ro')
zoom = images[ai][int(gt[1] * scale):int(gt[3] * scale), int(gt[0] * scale):int(gt[2] * scale)]
ax.imshow(zoom, cmap=pylab.gray(), interpolation='nearest')
ax.set_xlim(0, gts[2] - max(0, gts[0]))
ax.set_ylim((gts[3] - max(0, gts[1]), 0))
plt.show()
def compare_missed_segm(input_dir='/datagrid/personal/TextSpotter/FastTextEval/experiments/segmentation', input_dir2='/datagrid/personal/TextSpotter/FastTextEval/experiments/segmentationg', showPictures = False):
ft = FASTex()
(ms, dirs) = read_segm_data(input_dir)
(ms2, dirs2) = read_segm_data(input_dir2, 'g')
ms.extend(ms2)
dirs.extend(dirs2)
sumHash = {}
for j in np.arange(0, len(ms)):
missing_segm = ms[j]
for image in missing_segm.keys():
arr = missing_segm[image]
if not sumHash.has_key(image):
sumHash[image] = arr
continue
for i in range(len(arr)):
miss_gt = arr[i]
check = sumHash[image]
hasGt = False
for k in range(len(check)):
miss_gt2 = check[k]
if miss_gt == miss_gt2:
hasGt = True
if not hasGt:
sumHash[image].append(miss_gt)
missing_segm = ms[0]
data = []
dataf = []
gt_id = 0
columns = ['Img', 'GT Id']
for image in sumHash.keys():
arr = sumHash[image]
f = None
for i in range(len(arr)):
orValue = False
miss_gt = arr[i]
row = []
row.append(os.path.basename(image))
row.append(gt_id)
gt_id += 1
rowf = []
for j in np.arange(0, len(ms)):
if gt_id == 1:
columns.append(dirs[j])
msj = ms[j]
hasSegmj = True
val = 1
if msj.has_key(image):
arrj = msj[image]
for k in range(len(arrj)):
miss_gtj = arrj[k]
if miss_gtj == miss_gt:
hasSegmj = False
val = 0
break
row.append(hasSegmj)
rowf.append(val)
orValue = orValue or hasSegmj
if orValue:
rowf.append(1)
else:
rowf.append(0)
if showPictures:
img = cv2.imread(image)
imgg = cv2.imread(image, cv2.IMREAD_GRAYSCALE)
if f == None:
f, axes = plt.subplots(1, 2, figsize=(16, 3))
f.suptitle('Missing segmentation: {0}'.format(image))
ax = axes[0]
ax.imshow(img, cmap=pylab.gray(), interpolation='nearest')
ax = axes[1]
ax.imshow(imgg, cmap=pylab.gray(), interpolation='nearest')
orBox = miss_gt
segmentations = ft.getCharSegmentations(imgg)
keypoints = ft.getLastDetectionKeypoints()
style = 'rx'
for k in range(5):
maski = keypoints[:, 9] == k + 1
if k == 1:
style = "rv"
if k == 2:
style = "ro"
if k == 4:
style = "bo"
ax.plot(keypoints[maski, 0], keypoints[maski, 1], style)
for k in range(keypoints.shape[0]):
ax.plot([keypoints[k,0], keypoints[k,7]], [keypoints[k,1], keypoints[k,8]], 'r-')
ax = axes[0]
else:
orBox = utils.union(orBox, miss_gt)
line = mlines.Line2D(np.array([miss_gt[0], miss_gt[2], miss_gt[2], miss_gt[0], miss_gt[0]]), np.array([miss_gt[1], miss_gt[1], miss_gt[3], miss_gt[3], miss_gt[1]]), lw=5., alpha=0.6, color='r')
ax.add_line(line)
row.append(orValue)
data.append(row)
dataf.append(rowf)
if f != None:
ax = axes[0]
ax.set_xlim(orBox[0] - 20, orBox[2] + 20)
ax.set_ylim(orBox[3] + 20, orBox[1] - 20)
ax = axes[1]
ax.set_xlim(orBox[0] - 20, orBox[2] + 20)
ax.set_ylim(orBox[3] + 20, orBox[1] - 20)
plt.show()
columns.append("OR")
data = np.array(data)
dataf = np.array(dataf)
df = pandas.DataFrame(data = data, columns=columns)
#print(df)
sumCols = dataf.sum(0)
sumCols = dataf.shape[0] - sumCols
print("Missing Segmentations:")
print(sumCols)
indices = np.argsort(sumCols)
bestFactor = indices[1]
missing_segm = ms[bestFactor]
print( "Best factor: {0}".format(dirs[bestFactor]) )
maskBest = dataf[:, bestFactor] == 0
datafSec = dataf[maskBest, :]
sumCols = datafSec.sum(0)
sumCols = datafSec.shape[0] - sumCols
print("Missing Segmentations 2 best:")
print(sumCols)
indices = np.argsort(sumCols)
bestFactor2 = indices[1]
print( "Best factor 2: {0}, missing segmentations: {1} -> {2}".format(dirs[bestFactor2], datafSec.shape[0], sumCols[indices[1]]) )
maskBest = datafSec[:, bestFactor2] == 0
dataf3 = datafSec[maskBest, :]
sumCols = dataf3.sum(0)
sumCols = dataf3.shape[0] - sumCols
indices = np.argsort(sumCols)
bestFactor2 = indices[1]
print( "Best factor 3: {0}, missing segmentations: {1} -> {2}".format(dirs[bestFactor2], dataf3.shape[0], sumCols[indices[1]]) )
def run_evaluation(inputDir, outputDir, process_color = 0, processTest = 0):
if not os.path.exists(outputDir):
os.mkdir(outputDir)
edgeThreshold = 14
fastex = FASTex(edgeThreshold = edgeThreshold)
modelFile = '/home/busta/outModel.boost'
model = cv2.Boost()
model.load(modelFile)
images = glob.glob('{0}/*.jpg'.format(inputDir))
segmDir = '{0}/segmentations'.format(inputDir)
precision = 0;
precisionDen = 0
recall = 0
recall05 = 0
recallNonMax = 0
recallDen = 0
wordRecall = 0
wordRecallDen = 0
segm2chars = 0
regionsCount = 0
regionsCountNonMax = 0
missing_segmNonMaxCount = 0
letterKeypointHistogram = defaultdict(lambda : defaultdict(float))
octaveLetterKeypointHistogram = defaultdict(lambda : defaultdict(float))
missing_letters = {}
letterHistogram = defaultdict(int)
missing_segm = {}
missing_segm2 = {}
missing_segmNonMax = {}
diffMaxOctavesMap = {}
diffScoreOctavesMap = {}
segmHistogram = []
segmWordHistogram = []
results = []
hist = None
histFp = None
histDist = None
histDistFp = None
histDistMax = None
histDistMaxWhite = None
histDistMaxFp = None
hist2dDist =None
hist2dDistFp = None
hist2dDistScore = None
hist2dDistScoreFp = None
histDistMaxWhiteFp = None
histSegm = np.zeros((256), dtype = np.float)
histSegmCount = np.zeros((256), dtype = np.int)
stat = np.asarray([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], dtype=np.float)
times = []
gtSegmCount = 0
wordsOk = []
wordsFp = []
keypointsTotal = 0
keypointsTotalInside = 0
orbTime = 0
lineNo = 0
perfectWords = 0;
perfectWordsNS = 0;
hasSegm = False
for image in images:
print('Processing {0}'.format(image))
img = cv2.imread(image, 0)
imgc = cv2.imread(image)
imgcO = cv2.imread(image)
if process_color == 1:
imgproc = imgc
else:
imgproc = img
baseName = os.path.basename(image)
baseName = baseName[:-4]
workPoint = 0.3
segmentations = fastex.getCharSegmentations(imgproc, outputDir, baseName)
segmentations = segmentations[:, 0:10]
segmentations = np.column_stack( [ segmentations , np.zeros( (segmentations.shape[0], 2), dtype = np.float ) ] )
maskDuplicates = segmentations[:, 8] == -1
segmentationsDuplicates = segmentations[maskDuplicates, :]
maskNoNei = segmentationsDuplicates[:, 9] > workPoint
segmentationsNoNei = segmentationsDuplicates[maskNoNei, :]
if segmentations.shape[0] > 0:
print( 'Dupl ratio: {0} - {1}/ {2} - {3}'.format(segmentationsDuplicates.shape[0] / float(segmentations.shape[0]), segmentationsDuplicates.shape[0], segmentations.shape[0], segmentationsNoNei.shape[0] ) )
keypoints = fastex.getLastDetectionKeypoints()
keypointsTotal += keypoints.shape[0]
statc = fastex.getDetectionStat()
times.append([ statc[1], statc[2], statc[3], statc[4], statc[5], statc[6], statc[7], statc[8], statc[9], statc[10]])
stat += statc
values = img[ keypoints[:, 1].astype(int), keypoints[:, 0].astype(int) ]
valuesMax = img[keypoints[:, 6].astype(int), keypoints[:, 5].astype(int)]
diffValMax = np.abs(values - valuesMax)
regionsCount += segmentations.shape[0]
regionsCountNonMax += segmentationsNoNei.shape[0]
segmentations[:, 2] += segmentations[:, 0]
segmentations[:, 3] += segmentations[:, 1]
keypointsOrb = fastex.getLastDetectionOrbKeypoints()
orbTime += keypointsOrb[0][9]
segmGt = '{0}/{1}_GT.txt'.format(segmDir, baseName)
pden = 0
rden = 0
if os.path.exists(segmGt):
hasSegm = True
(gt_rects, groups) = utls.read_icdar2013_segm_gt(segmGt)
segmImg = '{0}/{1}_GT.bmp'.format(segmDir, baseName)
if not os.path.exists(segmImg):
segmImg = '{0}/gt_{1}.png'.format(segmDir, baseName)
segmImg = cv2.imread(segmImg)
try:
(hist, histFp, histDist, histDistMax, histDistMaxWhite, hist2dDist, hist2dDistScore, histDistFp, histDistMaxFp, histDistMaxWhiteFp, hist2dDistFp, hist2dDistScoreFp, keypointsInside) = collect_histograms(img, segmImg, keypoints, values, diffValMax, keypointsTotalInside, diffMaxOctavesMap, diffScoreOctavesMap, hist, histFp, histDist, histDistMax, histDistMaxWhite, hist2dDist, hist2dDistScore, histDistFp, histDistMaxFp, histDistMaxWhiteFp, hist2dDistFp, hist2dDistScoreFp)
except:
pass
rcurrent = 0
rcurrent05 = 0
rcurrentNonMax = 0
for k in range(len(gt_rects)):
gt_rect = gt_rects[k]
best_match = 0
best_match_line = 0
if (gt_rect[4] == ',' or gt_rect[4] == '.' or gt_rect[4] == '\'' or gt_rect[4] == ':' or gt_rect[4] == '-') and not evalPunctuation:
continue
gtSegmCount += 1
rectMask = np.bitwise_and(np.bitwise_and( keypointsInside[:, 0] >= gt_rect[0], keypointsInside[:, 0] <= gt_rect[2]), np.bitwise_and(keypointsInside[:, 1] >= gt_rect[1], keypointsInside[:, 1] <= gt_rect[3]))
letterInside = keypointsInside[rectMask, :]
#make keypoints histogram
if letterInside.shape[0] > 0:
octaves = np.unique( letterInside[:, 2])
maxOctave = np.max(octaves)
maxOctavePoints = 0
for i in range(int(maxOctave) + 1):
octavePoints = letterInside[letterInside[:, 2] == i, :]
maxOctavePoints = max(maxOctavePoints, octavePoints.shape[0])
if maxOctavePoints > 0:
octaveLetterKeypointHistogram[gt_rect[4]][0] += 1
if maxOctavePoints > 1:
octaveLetterKeypointHistogram[gt_rect[4]][1] += 1
if maxOctavePoints > 2:
octaveLetterKeypointHistogram[gt_rect[4]][2] += 1
if maxOctavePoints > 3:
octaveLetterKeypointHistogram[gt_rect[4]][3] += 1
if letterInside.shape[0] == 0:
if not missing_letters.has_key(gt_rect[4]):
missing_letters[gt_rect[4]] = []
missing_letters[gt_rect[4]].append( (image, gt_rect) )
if letterInside.shape[0] > 0:
letterKeypointHistogram[gt_rect[4]][0] += 1
if letterInside.shape[0] > 1:
letterKeypointHistogram[gt_rect[4]][1] += 1
if letterInside.shape[0] > 2:
letterKeypointHistogram[gt_rect[4]][2] += 1
if letterInside.shape[0] > 3:
letterKeypointHistogram[gt_rect[4]][3] += 1
letterHistogram[gt_rect[4]] += 1
best_match2 = 0
minSingleOverlap = MIN_SEGM_OVRLAP
if gt_rect[4] == 'i' or gt_rect[4] == '!':
minSingleOverlap = 0.5
for detId in range(segmentations.shape[0]):
rectn = segmentations[detId, :]
rect_int = utils.intersect( rectn, gt_rect )
int_area = utils.area(rect_int)
union_area = utils.area(utils.union(rectn, gt_rect))
ratio = int_area / float(union_area)
rectn[10] = max(ratio, rectn[10])
if rectn[9] > workPoint:
gt_rect[6] = max(ratio, gt_rect[6])
if ratio > best_match:
best_match = ratio
best_segm = segmentations[detId, :]
if ratio > best_match_line and rectn[7] == 1.0 :
best_match_line = ratio
if best_match < minSingleOverlap:
if k < len(gt_rects) - 1:
gt_rect2 = gt_rects[k + 1]
chars2Rect = utils.union(gt_rect2, gt_rect)
rect_int = utils.intersect( rectn, chars2Rect )
int_area = utils.area(rect_int)
union_area = utils.area(utils.union(rectn, chars2Rect))
ratio = int_area / float(union_area)
rectn[10] = max(ratio, rectn[10])
if ratio > best_match2:
if ratio > MIN_SEGM_OVRLAP:
segm2chars += 1
best_match2 = ratio
gt_rect[5] = ratio
gt_rect2[5] = ratio
thickness = 1
color = (255, 0, 255)
if best_match >= minSingleOverlap:
color = (0, 255, 0)
if best_match > 0.7:
thickness = 2
cv2.rectangle(imgc, (gt_rect[0], gt_rect[1]), (gt_rect[2], gt_rect[3]), color, thickness)
recall += best_match
recallNonMax += gt_rect[6]
if best_match >= minSingleOverlap:
recall05 += best_match
rcurrent05 += best_match
else:
if not missing_segm.has_key(image):
missing_segm[image] = []
missing_segm[image].append(gt_rect)
if gt_rect[5] < MIN_SEGM_OVRLAP:
if not missing_segm2.has_key(image):
missing_segm2[image] = []
missing_segm2[image].append(gt_rect)
segm2chars += 1
if gt_rect[6] < minSingleOverlap:
if not missing_segmNonMax.has_key(image):
missing_segmNonMax[image] = []
missing_segmNonMax[image].append(gt_rect)
missing_segmNonMaxCount += 1
rcurrent += best_match
rcurrentNonMax += gt_rect[6]
recallDen += 1
rden += 1
if best_match > 0 and process_color != 1:
val = img[best_segm[5], best_segm[4]]
histSegm[val] += best_match
histSegmCount[val] += 1
pcurrent = 0
for detId in range(segmentations.shape[0]):
best_match = 0
rectn = segmentations[detId, :]
for gt_rect in gt_rects:
rect_int = utils.intersect( rectn, gt_rect )
int_area = utils.area(rect_int)
union_area = utils.area(utils.union(rectn, gt_rect))
ratio = int_area / float(union_area)
if ratio > best_match:
best_match = ratio
precision += best_match
pcurrent += best_match
precisionDen += 1
pden += 1
if pden == 0:
pcurrent = 0
else:
pcurrent = pcurrent / pden
if rden == 0:
rcurrent = 0
rcurrent05 = 0
rcurrentNonMax = 0
else:
rcurrent = rcurrent / rden
rcurrent05 = rcurrent05 / rden
rcurrentNonMax = rcurrentNonMax / rden
segmHistogram.append([ segmentations.shape[0], segmentations[segmentations[:, 10] > 0.4].shape[0], segmentations[segmentations[:, 10] > 0.5].shape[0], segmentations[segmentations[:, 10] > 0.6].shape[0], segmentations[segmentations[:, 10] > 0.7].shape[0] ])
segmWordHistogram.append([segmentations.shape[0], segmentations[np.bitwise_or(segmentations[:, 10] > 0.5, segmentations[:, 11] > 0.5 )].shape[0]])
results.append((baseName, rcurrent, pcurrent, rcurrent05))
if precisionDen == 0:
pcurrent = 0
else:
precision = precision / precisionDen
if recallDen == 0:
rcurrent = 0
else:
recall = recall / recallDen
recall05 = recall05 / recallDen
recallNonMax = recallNonMax / recallDen
wordRecall = wordRecall / max(1, wordRecallDen)
try:
histSegm = histSegm / max(1, histSegmCount)
except ValueError:
pass
print('Evalation Results:')
print( 'recall: {0}, precision: {1}, recall 0.5: {2}, recall NonMax: {3}'.format(recall, precision, recall05, recallNonMax) )
kpTimes = np.histogram(np.asarray(times)[:, 0], bins=20)
print('Keypoint Time Histogram: {0}'.format(kpTimes))
print('Detection statistics:')
print(stat)
for letter in letterKeypointHistogram.keys():
for num in letterKeypointHistogram[letter].keys():
letterKeypointHistogram[letter][num] = letterKeypointHistogram[letter][num] / float(letterHistogram[letter])
for num in octaveLetterKeypointHistogram[letter].keys():
octaveLetterKeypointHistogram[letter][num] = octaveLetterKeypointHistogram[letter][num] / float(letterHistogram[letter])
letterKeypointHistogram[letter] = dict(letterKeypointHistogram[letter])
octaveLetterKeypointHistogram[letter] = dict(octaveLetterKeypointHistogram[letter])
print('Perfect words: {0}'.format(perfectWords))
eval_date = datetime.date.today()
np.savez('{0}/evaluation'.format(outputDir), recall=recall, recall05 = recall05, recallNonMax=recallNonMax, precision=precision, eval_date=eval_date, regionsCount=regionsCount, inputDir = inputDir, hist = hist, histSegm = histSegm, stat=stat, letterKeypointHistogram = dict(letterKeypointHistogram), missing_letters=missing_letters, octaveLetterKeypointHistogram=dict(octaveLetterKeypointHistogram), missing_segm=missing_segm,
times=np.asarray(times), histFp = histFp, gtSegmCount = gtSegmCount, wordRecall=wordRecall, histDist=histDist, histDistFp = histDistFp, histDistMax=histDistMax, histDistMaxFp=histDistMaxFp, hist2dDist=hist2dDist, hist2dDistFp=hist2dDistFp, hist2dDistScore=hist2dDistScore, hist2dDistScoreFp=hist2dDistScoreFp, histDistMaxWhite=histDistMaxWhite, histDistMaxWhiteFp=histDistMaxWhiteFp, wordsOk=wordsOk, wordsFp=wordsFp, diffMaxOctavesMap = diffMaxOctavesMap, diffScoreOctavesMap = diffScoreOctavesMap,
missing_segm2=missing_segm2, segmHistogram=segmHistogram, segmWordHistogram=segmWordHistogram, regionsCountNonMax=regionsCountNonMax, missing_segmNonMax=missing_segmNonMax)
print( "GT segmentations count {0}".format(gtSegmCount) )
print('FasTex Inside {0}/{1} ({2})'.format(keypointsTotalInside, keypointsTotal, keypointsTotalInside / float(keypointsTotal) ))
print('FasText time: {0}, Orb time: {1} '.format( np.sum(times, 0)[0], orbTime))
print('2 Chars Segmentation: {0}'.format(segm2chars) )
print('NonMax Regions Count: {0}/{1}'.format(regionsCountNonMax, missing_segmNonMaxCount))
def draw_missed_letters_tile(input_dir='/datagrid/personal/TextSpotter/FastTextEval/ICDAR-Train', color = 0, edgeThreshold = 13, inter = True, scalingFactor=1.6, segmList=[]):
ft = FASTex(process_color = color, edgeThreshold = edgeThreshold)
d=input_dir
subdirs = [os.path.join(d,o) for o in os.listdir(d) if os.path.isdir(os.path.join(d,o))]
subdirs = np.sort(subdirs)
lastDir = ''
for dir_name in subdirs:
file_name = '{0}/evaluation.npz'.format(dir_name)
if not os.path.exists(file_name):
continue
vars_dict = np.load(file_name)
inputDir = vars_dict['inputDir']
lastDir = dir_name
if 'letterKeypointHistogram' in vars_dict.keys():
letterKeypointHistogram = vars_dict['letterKeypointHistogram']
letterKeypointHistogram = dict(letterKeypointHistogram.tolist())
print(lastDir)
missing_letters = vars_dict['missing_letters']
missing_letters = dict(missing_letters.tolist())
segmDir = '{0}/segmentations'.format(inputDir)
segmDir = '/datagrid/personal/TextSpotter/evaluation-sets/icdar2013-Test/segmentations'
keys = []
ticks = []
values = []
values.append([])
values.append([])
values.append([])
values.append([])
ticks.append([])
ticks.append([])
ticks.append([])
ticks.append([])
listlen = 0
for letter in letterKeypointHistogram.keys():
keys.append(letter)
values[0].append(0)
ticks[0].append(listlen)
values[1].append(0)
ticks[1].append(listlen + 0.2)
values[2].append(0)
ticks[2].append(listlen + 0.4)
values[3].append(0)
ticks[3].append(listlen + 0.6)
for num in letterKeypointHistogram[letter].keys():
values[num][listlen] = letterKeypointHistogram[letter][num]
listlen += 1
indices = sorted(range(len(values[0])),key=lambda x:values[0][x])
indices.reverse()
border = 15
missLetter = []
imagesMiss = {}
for letter in np.asarray(keys)[np.asarray(indices)]:
if not missing_letters.has_key(letter):
continue
arr = missing_letters[letter]
for i in range(len(arr)):
miss = arr[i]
if len(segmList) > 0:
base = os.path.basename(miss[0])
if not base in segmList:
continue
missLetter.append(miss)
if imagesMiss.has_key(miss[0]):
imagesMiss[miss[0]].append( miss[1] )
else:
imagesMiss[miss[0]] = []
imagesMiss[miss[0]].append( miss[1] )
rowSize = len(imagesMiss.keys())
f, axes = plt.subplots(2, len(imagesMiss.keys()))
plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=None, hspace=None)
figNo = 0
for image in imagesMiss.keys():
if len(imagesMiss.keys()) > 1:
ax0 = axes[0][figNo]
ax = axes[1][figNo]
else:
ax0 = axes[figNo]
ax = axes[figNo]
figNo += 1
if color == 1:
img = cv2.imread(image)
else:
img = cv2.imread(image, 0)
baseName = os.path.basename(image)
baseName = baseName[:-4]
segmImg = '{0}/{1}_GT.bmp'.format(segmDir, baseName)
if not os.path.exists(segmImg):
segmImg = '{0}/gt_{1}.png'.format(segmDir, baseName)
segmImg = cv2.imread(segmImg)
segmentations = ft.getCharSegmentations(img)
keypoints = ft.getLastDetectionKeypoints()
if color == 1:
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
for i in range(len(imagesMiss[image])):
if i == 0:
orBox = imagesMiss[image][0]
else:
orBox = utils.union(orBox, imagesMiss[image][i])
gt0 = orBox
gt = [gt0[0] - border, gt0[1] - border, gt0[2] + border, gt0[3] + border ]
gt[0] = max(0, gt[0])
gt[1] = max(0, gt[1])
gt[2] = min(img.shape[1], gt[2])
gt[3] = min(img.shape[0], gt[3])
zoom = img[gt[1]:gt[3], gt[0]:gt[2]]
ax.imshow(zoom, cmap=pylab.gray(), interpolation='nearest')
ax0.imshow(zoom, cmap=pylab.gray(), interpolation='nearest')
centers = segmImg[keypoints[:, 1].astype(int), keypoints[:, 0].astype(int)]
keypointsInsideMask = centers == (255, 255, 255)
keypointsInsideMask = np.invert(np.bitwise_and(np.bitwise_and(keypointsInsideMask[:, 0], keypointsInsideMask[:, 1]), keypointsInsideMask[:, 2]))
keypointsInside = keypoints[keypointsInsideMask, :]
mask = (keypoints[:, 0] > gt[0]) * (keypoints[:, 0] < gt[2]) * (keypoints[:, 1] > gt[1]) * (keypoints[:, 1] < gt[3])
kpMask = keypoints[mask]
kpMask[:, 0] = kpMask[:, 0] - gt[0]
kpMask[:, 1] = kpMask[:, 1] - gt[1]
kpMask[:, 7] = kpMask[:, 7] - gt[0]
kpMask[:, 8] = kpMask[:, 8] - gt[1]
ax.plot(kpMask[:, 0], kpMask[:, 1], 'ro')
ax.xaxis.set_ticklabels([])
ax.yaxis.set_ticklabels([])
ax0.xaxis.set_ticklabels([])
ax0.yaxis.set_ticklabels([])
for k in range(kpMask.shape[0]):
ax.plot([kpMask[k,0], kpMask[k,7]], [kpMask[k,1], kpMask[k,8]], 'r-')
style = 'rx'
if kpMask.shape[1] > 9:
for k in range(3):
maski = kpMask[:, 9] == k + 1
if k == 1:
style = "rv"
if k == 2:
style = "rs"
if k == 4:
style = "bo"
if k == 5:
style = "yo"
ax.plot([kpMask[maski,7]], [kpMask[maski,8]], style)
for i in range(len(imagesMiss[image])):
gt0 = imagesMiss[image][i]
mask = (keypointsInside[:, 0] > gt[0]) * (keypointsInside[:, 0] < gt[2]) * (keypointsInside[:, 1] > gt[1]) * (keypointsInside[:, 1] < gt[3])
kpMask = keypointsInside[mask]
keypointsInside[:, 0] = keypointsInside[:, 0] - gt[0]
keypointsInside[:, 1] = keypointsInside[:, 1] - gt[1]
keypointsInside[:, 7] = keypointsInside[:, 7] - gt[0]
keypointsInside[:, 8] = keypointsInside[:, 8] - gt[1]
ax.plot(keypointsInside[:, 0], keypointsInside[:, 1], 'go')
for k in range(keypointsInside.shape[0]):
ax.plot([keypointsInside[k,0], keypointsInside[k,7]], [keypointsInside[k,1], keypointsInside[k,8]], 'g-')
ax.set_xlim(0, gt[2] - max(0, gt[0]))
ax.set_ylim((gt[3] - max(0, gt[1]), 0))
line = mlines.Line2D(np.array([gt0[0] - gt[0], gt0[2] - gt[0], gt0[2] - gt[0], gt0[0] - gt[0], gt0[0] - gt[0]]), np.array([gt0[1] - gt[1], gt0[1] - gt[1], gt0[3] - gt[1], gt0[3] - gt[1], gt0[1] - gt[1]]), lw=5., alpha=0.6, color='r')
ax0.add_line(line)
plt.show()
def draw_missed_letters_figure(input_dir='/datagrid/personal/TextSpotter/FastTextEval/ICDAR-Train', color = 0, edgeThreshold = 13, inter = True, scalingFactor=0.5, segmList=[]):
ft = FASTex(process_color = color, edgeThreshold = edgeThreshold)
d=input_dir
subdirs = [os.path.join(d,o) for o in os.listdir(d) if os.path.isdir(os.path.join(d,o))]
subdirs = np.sort(subdirs)
lastDir = ''
for dir_name in subdirs:
file_name = '{0}/evaluation.npz'.format(dir_name)
if not os.path.exists(file_name):
continue
vars_dict = np.load(file_name)
inputDir = vars_dict['inputDir']
lastDir = dir_name
if 'letterKeypointHistogram' in vars_dict.keys():
letterKeypointHistogram = vars_dict['letterKeypointHistogram']
letterKeypointHistogram = dict(letterKeypointHistogram.tolist())
print(lastDir)
missing_letters = vars_dict['missing_letters']
missing_letters = dict(missing_letters.tolist())
keys = []
ticks = []
values = []
values.append([])
values.append([])
values.append([])
values.append([])
ticks.append([])
ticks.append([])
ticks.append([])
ticks.append([])
listlen = 0
for letter in letterKeypointHistogram.keys():
keys.append(letter)
values[0].append(0)
ticks[0].append(listlen)
values[1].append(0)
ticks[1].append(listlen + 0.2)
values[2].append(0)
ticks[2].append(listlen + 0.4)
values[3].append(0)
ticks[3].append(listlen + 0.6)
for num in letterKeypointHistogram[letter].keys():
values[num][listlen] = letterKeypointHistogram[letter][num]
listlen += 1
indices = sorted(range(len(values[0])),key=lambda x:values[0][x])
indices.reverse()
missLetter = []
imagesMiss = {}
for letter in np.asarray(keys)[np.asarray(indices)]:
if not missing_letters.has_key(letter):
continue
arr = missing_letters[letter]
for i in range(len(arr)):
miss = arr[i]
if len(segmList) > 0:
base = os.path.basename(miss[0])
if not base in segmList:
continue
missLetter.append(miss)
if imagesMiss.has_key(miss[0]):
imagesMiss[miss[0]].append( miss[1] )
else:
imagesMiss[miss[0]] = []
imagesMiss[miss[0]].append( miss[1] )
for image in imagesMiss.keys():
f = plt.figure(num = 250)
ax = f.add_subplot(111)
imgc2 = cv2.imread(image)
img = cv2.imread(image, cv2.IMREAD_GRAYSCALE)
imgc2 = cv2.cvtColor(imgc2, cv2.COLOR_BGR2RGB)
ax.imshow(imgc2)
segmentations = ft.getCharSegmentations(img)
keypoints = ft.getLastDetectionKeypoints()
octaves = np.unique( keypoints[:, 2])
maxOctave = np.max(octaves)
scales = ft.getImageScales()
for i in range(int(maxOctave) + 1):
octavePoints = keypoints[keypoints[:, 2] == i, :]
c = 'red'
if len(octavePoints) > 0 and octavePoints.shape[1] > 9:
for k in range(6):
maski = octavePoints[:, 9] == k + 1
if k == 1:
style = "rv"
if k == 2:
style = "ro"
if k == 4:
style = "bo"
c = 'blue'
if k == 5:
style = "yo"
continue
s = 10 / scales[i]
ax.scatter(octavePoints[maski, 0], octavePoints[maski, 1],c=c, s=s )
for i in range(len(imagesMiss[image])):
gt0 = imagesMiss[image][i]
line = mlines.Line2D(np.array([gt0[0], gt0[2], gt0[2], gt0[0], gt0[0]]), np.array([gt0[1], gt0[1], gt0[3], gt0[3], gt0[1]]), lw=5., alpha=0.6, color='r')
ax.add_line(line)
plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, hspace = 0, wspace = 0)
plt.margins(0,0)
ax.set_xlim([0, imgc2.shape[1]])
ax.set_ylim([imgc2.shape[0], 0])
ax.axes.get_xaxis().set_ticks([])
ax.axes.get_yaxis().set_ticks([])
plt.show()
def draw_missed_segm(input_dir='/datagrid/personal/TextSpotter/FastTextEval/ICDAR-Train', color = 0, edgeThreshold = 12, inter = True, scalingFactor=0.5):
ft = FASTex(process_color = color, edgeThreshold = edgeThreshold)
d=input_dir
subdirs = [os.path.join(d,o) for o in os.listdir(d) if os.path.isdir(os.path.join(d,o))]
subdirs = np.sort(subdirs)
for dir_name in subdirs:
file_name = '{0}/evaluation.npz'.format(dir_name)
if not os.path.exists(file_name):
continue
vars_dict = np.load(file_name)
inputDir = vars_dict['inputDir']
missing_segm = vars['missing_segm']
missing_segm = dict(missing_segm.tolist())
segmDir = '{0}/segmentations'.format(inputDir)
for image in missing_segm.keys():
arr = missing_segm[image]
if color == 1:
img = cv2.imread(image)
else:
img = cv2.imread(image, 0)
segmentations = ft.getCharSegmentations(img)
keypoints = ft.getLastDetectionKeypoints()
baseName = os.path.basename(image)
baseName = baseName[:-4]
segmImg = '{0}/{1}_GT.bmp'.format(segmDir, baseName)
if not os.path.exists(segmImg):
segmImg = '{0}/gt_{1}.png'.format(segmDir, baseName)
segmImg = cv2.imread(segmImg)
centers = segmImg[keypoints[:, 1].astype(int), keypoints[:, 0].astype(int)]
keypointsInsideMask = centers == (255, 255, 255)
keypointsInsideMask = np.invert(np.bitwise_and(np.bitwise_and(keypointsInsideMask[:, 0], keypointsInsideMask[:, 1]), keypointsInsideMask[:, 2]))
keypointsInside = keypoints[keypointsInsideMask, :]
f = plt.figure(num = 110)
ax = f.add_subplot(111)
ax.imshow(img, cmap=pylab.gray(), interpolation='nearest')
style = "rx"
for k in range(6):
maski = keypoints[:, 9] == k + 1
if k == 1:
style = "rv"
if k == 2:
style = "ro"
if k == 4:
style = "bo"
if k == 5:
style = "yo"
ax.plot(keypoints[maski, 0], keypoints[maski, 1], style)
ax.plot(keypointsInside[:, 0], keypointsInside[:, 1], 'go')
ax.set_xlim(0, img.shape[1])
ax.set_ylim(img.shape[0], 0)
for i in range(len(arr)):
miss_gt = arr[i]
line = mlines.Line2D(np.array([miss_gt[0], miss_gt[2], miss_gt[2], miss_gt[0], miss_gt[0]]), np.array([miss_gt[1], miss_gt[1], miss_gt[3], miss_gt[3], miss_gt[1]]), lw=5., alpha=0.4, color='b')
ax.add_line(line)
ax.set_title('Missing segmentation: {0}'.format(image))
plt.show()
ss = np.zeros(gray.shape,np.uint8)
cv2.drawContours(ss,contours,-1,255,-1)
cv2.imwrite(p2_out_dir+'/' + os.path.basename(imgd).split('.')[0]+'mask.png',ss)
# cv2.imwrite(dstFile+'/'+pic[:-9]+'fuse0.png',img)
if __name__ == '__main__':
outputDir = './tmp'
edgeThreshold = 13
T = 0.8
gray_threshold = 180
ft = FASTex(edgeThreshold=edgeThreshold, nlevels=-1, minCompSize=4)
# imgName = '/datagrid/personal/TextSpotter/evaluation-sets/bornDigital/img_100.png'
# print('='*10 + 'stroke detecting' + '='*10)
ori_dir = 'ori'
p1_out_dir = 'p1_out'
p2_out_dir = 'p2_out'
if len(sys.argv) == 2:
if sys.argv[1].endswith(".png") or sys.argv[1].endswith(".jpg"):
imgName = sys.argv[1]
else:
print('please input image file name.')
starttime = time.time()
img_id = os.path.basename(imgName).split('.')[0]
# print(os.path.join(p1_out_dir, '{}fuse0.png'.format(img_id)))
@author: busta
'''
import numpy as np
import cv2
import sys
from ft import FASTex
from vis import draw_keypoints
if __name__ == '__main__':
outputDir = '/tmp'
edgeThreshold = 13
ft = FASTex(edgeThreshold=edgeThreshold, nlevels=-1, minCompSize=4)
imgName = '/home/busta/Desktop/cutr.png'
if len(sys.argv) > 1:
if sys.argv[1].endswith(".png") or sys.argv[1].endswith(".jpg"):
imgName = sys.argv[1]
img = cv2.imread(imgName, 0)
imgc = cv2.imread(imgName)
#print(out)
segmentations = ft.getCharSegmentations(img, outputDir, 'base')
print segmentations
for i in range(segmentations.shape[0]):
rectn = segmentations[i, :]
@author: busta
'''
import numpy as np
import cv2
import sys
from ft import FASTex
from vis import draw_keypoints
if __name__ == '__main__':
outputDir = '/tmp'
edgeThreshold = 13
ft = FASTex(edgeThreshold= edgeThreshold, nlevels=-1, minCompSize = 4)
imgName = '/datagrid/personal/TextSpotter/evaluation-sets/bornDigital/img_100.png'
if len(sys.argv) > 1:
if sys.argv[1].endswith(".png") or sys.argv[1].endswith(".jpg"):
imgName = sys.argv[1]
img = cv2.imread(imgName, 0)
imgc = cv2.imread(imgName)
#print(out)
segmentations = ft.getCharSegmentations(img, outputDir, 'base')
print segmentations
for i in range(segmentations.shape[0]):
rectn = segmentations[i, :]
default=default_img,
type=str,
help='Image to parse')
# parser.add_argument('--file_save', help='Save results of search to this json')
args = parser.parse_args()
# Verify image
if not os.path.isfile(args.image):
sys.exit("Image not found")
if not args.image.endswith(".png") and not args.image.endswith(".jpg"):
sys.exit("Not valid image")
settings = settings("config.json")
output_dir = settings['debug_dir']
ft = FASTex(edgeThreshold=13, nlevels=-1, minCompSize=4)
# Read image as numpy array
img = cv2.imread(args.image, 0) # load image in grayscale, i.e. 1 channel
img_o = cv2.imread(args.image)
img3 = cv2.imread(args.image)
# Get boxes of characters
# Elem in rows:: [bbox.x, bbox.y, bbox.width, bbox.height, keyPoint.pt.x, keyPoint.pt.y, octave, ?, duplicate, quality, [keypointsIds]]
segmentations = ft.getCharSegmentations(img, output_dir, 'base')
viz_location(img3, ft, segmentations, cumulative=True, max_chars=2000)
# drawCharMasks(img3, ft, segmentations, 100)
# drawCharBoxes(img3, segmentations, 100)
# cv2.imwrite(output_dir + "/mask.jpg", img3)