def main():
try:
instanceSize = 10
step = 1
numOfClasses = 100
filename = 'detector_7_no_5_angle_2.jpg'
mode = 'PRO'
image_files, bubble_num, bubble_regions = getinfo()
data_set = ds.read_data_sets(
instanceSize = instanceSize,
stride = step,
instanceMode = 'test',
labelMode = mode,
imageName = filename,
dtype = tf.float32,
plot_show = 0,
label_mutiplier = 1.0)
print data_set.labels.shape[-1]
print 'Sum of labels: ' + str(np.sum(data_set.labels))
print 'Number bubbles: ' + str(bubble_num[image_files.index(filename)])
except KeyboardInterrupt:
print "Shutdown requested... exiting"
except Exception:
traceback.print_exc(file=sys.stdout)
sys.exit(0)
def plotresultfromfile(ImagePatchWidth = 100,
ImagePatchStep = 10,
trainBatchNum = 10000,
trainBatchSize = 2000,
label_mutiplier = 1.0,
hidden_units = [200, 400, 200],
trainSteps = 200,
optimizer = 'Adagrad', #"SGD", "Adam", "Adagrad"
learning_rate = 0.01,
label_mode = 'PRO',
clip_gradients = 5.0,
config = None,
verbose = 1,
dropout = None):
"""
load data from file and show the data
"""
image_files, bubble_num, bubble_regions = getinfo()
directory = gv.__DIR__ + gv.dp__tuningPar_dir
filename = (
str(ImagePatchWidth) + '_' +
str(ImagePatchStep) + '_' +
str(trainBatchNum) + '_' +
str(trainBatchSize) + '_' +
str(label_mutiplier) + '_' +
str(hidden_units) + '_' +
str(trainSteps) + '_' +
optimizer + '_' +
str(learning_rate) + '_' +
label_mode + '_' +
str(clip_gradients) + '_' +
str(config) + '_' +
str(verbose) + '_' +
str(dropout) + '.dat')
result = np.fromfile(directory + filename)
slope, intercept, r_value, p_value, std_err = linregress(
bubble_num, result.T)
yfit = slope * bubble_num + intercept
fig, ax = plt.subplots(1,1)
ax.set_title('Linearity of DNN Regressor')
ax.set_xlabel('Bubble number counted manually')
ax.set_ylabel('Bubble number labeled by DNN')
ax.scatter(bubble_num, result, color = 'blue', label = 'DNN result')
ax.plot(bubble_num, yfit, color = 'red', label = 'linear fit')
handles, labels = ax.get_legend_handles_labels()
ax.legend(handles, labels, loc="upper left")
text_x = np.amax(bubble_num)*0.6
text_y = np.amax(result)*0.2
text = "R_SQUARED: %.5f\nSTD_ERR: %.5f" % (r_value**2, std_err)
ax.annotate(text, xy = (text_x, text_y), xytext = (text_x, text_y))
plt.show()
def labellinearity(patch_size,
stride,
labelMode = 'PRO',
label_mutiplier = 1,
progress_show = 1,
plot_show = 1):
gv.ds_show_filename = False
image_files, bubble_num, bubble_regions = getinfo()
bubble_num_afterlabel = np.zeros(len(image_files))
probar = progress.progress(0, len(image_files))
for i, image in enumerate(image_files):
probar.setCurrentIteration(i+1)
probar.setInfo(prefix_info = 'dataset linearity ...',
suffix_info = image)
probar.printProgress()
image_ds = ds.read_data_sets(
instanceSize = patch_size,
stride = stride,
instanceMode = 'test',
labelMode = labelMode,
imageName = image,
label_mutiplier = label_mutiplier)
labels = image_ds.labels
bubble_num_afterlabel[i] = np.sum(labels)
slope, intercept, r_value, p_value, std_err = (linregress(bubble_num,
bubble_num_afterlabel))
if(plot_show == 1):
yfit = slope * bubble_num + intercept
fig, ax = plt.subplots(1,1)
ax.set_title('Linearity of Labeling Methods')
ax.set_xlabel('Bubble number counted manually')
ax.set_ylabel('Bubble number after labeling')
ax.scatter(bubble_num, bubble_num_afterlabel,
color = 'blue', label = 'bubble number')
ax.plot(bubble_num, yfit, color = 'red', label = 'linear regression')
handles, labels = ax.get_legend_handles_labels()
ax.legend(handles, labels, loc="upper left")
text_x = np.amax(bubble_num)*0.6
text_y = np.amax(bubble_num_afterlabel)*0.1
text = "r_squared: %.5f\nstd_err: %.5f" % (r_value**2, std_err)
ax.annotate(text, xy = (text_x, text_y), xytext = (text_x, text_y))
plt.show()
return ([[slope, intercept, r_value, p_value, std_err],
bubble_num, bubble_num_afterlabel])
def testall(classifier,
ImagePatchWidth = 20,
ImagePatchStep = 4,
label_mutiplier = 1.0,
labelMode = 'PRO',
image_show = 0, # show labeling result of each image
save_image = True, # save each labeled image
save_result = True): # save labeling result and accuracy
""" labeling all avilable images
"""
# get all image information
image_files, bubble_num, bubble_regions = getinfo()
# files to be saved
result_filename = gv.dp__result_filename
accuracy_filename = gv.dp__accuracy_filename
# initialize benchmark variables
labeling_result = np.zeros((len(image_files),1))
correct = np.zeros(4)
accuracy = np.zeros(4)
totalnum_instances = 0
for i, image_file in enumerate(image_files):
# labeling each image
bubbleNum, correctNumber, totalInstanceNumber = test(
classifier,
image_file,
ImagePatchWidth = ImagePatchWidth,
ImagePatchStep = ImagePatchStep,
labelMode = labelMode,
label_mutiplier = label_mutiplier,
plot_show = image_show,
save_image = save_image)
labeling_result[i] = bubbleNum
correct = np.add(correct, correctNumber)
totalnum_instances += totalInstanceNumber
# calculate total labeling accuracy
accuracy = np.true_divide(correct, totalnum_instances)
# save data
if(gv.dp_test_save_data and save_result):
directory = gv.__dir__ + gv.dp__tuningPar_dir
if not os.path.exists(directory):
os.makedirs(directory)
accuracy.tofile(directory + accuracy_filename, sep = " ")
labeling_result.tofile(result_filename, sep = " ")
return [labeling_result, accuracy]
def main():
try:
data_filename = gv.cu__result_filename
image_files, bubble_num, bubble_regions = getinfo()
if not os.path.isfile(data_filename):
number = linearity(image_files)
number = np.loadtxt(data_filename)
number = np.reshape(number, (len(number)/2,2))
fig, ax = plt.subplots(1,1)
ax.scatter(bubble_num, number[:, 0], c = 'r')
k, b = np.polyfit(bubble_num, number[:, 0], 1)
ax.plot(bubble_num, bubble_num*k+b, 'r', linewidth = 2)
ax.scatter(bubble_num, number[:, 1], c = 'b')
k, b = np.polyfit(bubble_num, number[:, 1], 1)
ax.plot(bubble_num, bubble_num*k+b, 'b', linewidth = 2)
plt.show()
except KeyboardInterrupt:
print "Shutdown requested... exiting"
except Exception:
traceback.print_exc(file=sys.stdout)
sys.exit(0)
def gaussianDatatoFile(instanceSize, stride, labelMode):
""" Calculation of Gaussian value of every point is time consuming.
For accelarate the trainning and test process, this function generate
Gaussian data for each image and save to file. When training and
testing, only need to load data from file.
"""
image_files, bubble_num, bubble_regions = getinfo()
PROGRESS = progress.progress(0, len(image_files))
for i, imageFilename in enumerate(image_files):
PROGRESS.setCurrentIteration(i+1)
PROGRESS.setInfo(prefix_info = 'Gaussian data generating ...',
suffix_info = imageFilename)
PROGRESS.printProgress()
filename = gv.__DIR__ + gv.__TrainImageDir__ + imageFilename
imageData = io.imread(filename)
positiveLabels = bubble_regions[image_files.index(imageFilename)]
[m, n, c] = imageData.shape
Y = np.arange(0, (m-instanceSize + 1), stride)
X = np.arange(0, (n-instanceSize + 1), stride)
instances, labels = patchlabel(Y,X,positiveLabels,
patchsize = instanceSize,
stride = stride,
mode = labelMode)
directory = gv.__DIR__ + gv.dp__Gaussia_data_dir
if not os.path.exists(directory):
os.makedirs(directory)
filename = (directory +
str(instanceSize) + '_' +
str(stride) + '_' +
labelMode +
imageFilename[0:-3]) + 'npy'
np.save(filename, np.array(labels))
def performance(ImagePatchWidth = 100,
ImagePatchStep = 10,
trainBatchNum = 10000,
trainBatchSize = 2000,
label_mutiplier = 1.0,
hidden_units = [200, 400, 200],
trainSteps = 200,
optimizer = 'Adagrad', #"SGD", "Adam", "Adagrad"
learning_rate = 0.01,
label_mode = 'PRO',
clip_gradients = 5.0,
config = None,
verbose = 1,
dropout = None,
plot_show = 1,
save_result = True):
""" performance benchmark through train nand labeling
"""
# information of all aviliable images
image_files, bubble_num, bubble_regions = getinfo()
ds.gaussianDatatoFile(ImagePatchWidth, ImagePatchStep, label_mode)
# train DNN
classifier, _ = train(
batchNum = trainBatchNum,
batchSize = trainBatchSize,
ImagePatchWidth = ImagePatchWidth,
ImagePatchStep = ImagePatchStep,
label_mutiplier = label_mutiplier,
labelMode = label_mode,
hidden_units = hidden_units,
steps = trainSteps,
optimizer = optimizer,
learning_rate = learning_rate,
clip_gradients = clip_gradients,
config = config,
verbose = verbose,
dropout = dropout)
# labeling all images
result, accuracy = testall(
classifier,
ImagePatchWidth = ImagePatchWidth,
ImagePatchStep = ImagePatchStep,
labelMode = label_mode,
label_mutiplier = label_mutiplier,
image_show = 0,
save_image = False,
save_result = False) # Save in this function, no need to save twice
# linear regression
slope, intercept, r_value, p_value, std_err = linregress(
bubble_num, result.T)
status = np.append(
np.array([slope, intercept, r_value, p_value, std_err]), accuracy)
# save data
if(save_result):
directory = gv.__DIR__ + gv.dp__tuningPar_dir
if not os.path.exists(directory):
os.makedirs(directory)
filename = (
str(ImagePatchWidth) + '_' +
str(ImagePatchStep) + '_' +
str(trainBatchNum) + '_' +
str(trainBatchSize) + '_' +
str(label_mutiplier) + '_' +
str(hidden_units) + '_' +
str(trainSteps) + '_' +
optimizer + '_' +
str(learning_rate) + '_' +
label_mode + '_' +
str(clip_gradients) + '_' +
str(config) + '_' +
str(verbose) + '_' +
str(dropout) + '.dat')
result.tofile(directory + filename)
# show linearity
if(plot_show == 1):
print accuracy
yfit = slope * bubble_num + intercept
fig, ax = plt.subplots(1,1)
ax.set_title('Linearity of DNN Regressor')
ax.set_xlabel('Bubble number counted manually')
ax.set_ylabel('Bubble number labeled by DNN')
ax.scatter(bubble_num, result, color = 'blue', label = 'DNN result')
ax.plot(bubble_num, yfit, color = 'red', label = 'linear fit')
handles, labels = ax.get_legend_handles_labels()
ax.legend(handles, labels, loc="upper left")
text_x = np.amax(bubble_num)*0.6
text_y = np.amax(result)*0.1
text = "R_SQUARED: %.5f\nSTD_ERR: %.5f" % (r_value**2, std_err)
ax.annotate(text, xy = (text_x, text_y), xytext = (text_x, text_y))
plt.show()
return [result, status]
def main():
try:
echos_tune = 10
batchNum = 2000
batchSize = 2000
ImagePatchWidth = 40
ImagePatchStep = 1
labelMode = "NUM"
label_mutiplier = 1.0
hidden_units = [200, 400, 200]
steps = 200
optimizer = "Adagrad" # "SGD", "Adam", "Adagrad"
learning_rate = 0.1
clip_gradients = 5.0
config = None
verbose = 1
dropout = None
filename = "detector_3_no_5_angle_2.jpg"
image_files, bubble_num, bubble_regions = getinfo()
benchmark_tune = np.zeros((echos_tune, 2))
# generate data:
# ds.gaussianDatatoFile(ImagePatchWidth, ImagePatchStep, labelMode)
classifier, trainDataset = dl.train(
batchNum=0,
batchSize=1,
ImagePatchWidth=ImagePatchWidth,
ImagePatchStep=ImagePatchStep,
labelMode=labelMode,
hidden_units=hidden_units,
steps=steps,
optimizer=optimizer,
learning_rate=learning_rate,
clip_gradients=clip_gradients,
config=config,
verbose=verbose,
dropout=dropout,
)
PROGRESS = progress.progress(0, echos_tune)
for i in range(echos_tune):
PROGRESS.setCurrentIteration(i + 1)
PROGRESS.setInfo(prefix_info="Batch Number Tune ...", suffix_info=str(i + 1) + "/" + str(echos_tune))
classifier = dl.continuetrain(
classifier=classifier, trainDataset=trainDataset, batchNum=batchNum, batchSize=batchSize
)
result, accuracy = dl.testall(
classifier,
ImagePatchWidth=ImagePatchWidth,
ImagePatchStep=ImagePatchStep,
label_mutiplier=label_mutiplier,
labelMode=labelMode,
image_show=0, # show labeling result of each image
save_image=False, # save each labeled image
save_result=False,
)
slope, intercept, r_value, p_value, std_err = linregress(bubble_num, result.T)
benchmark_tune[i, 0] = r_value
benchmark_tune[i, 1] = std_err
PROGRESS.printProgress()
print "\n"
filename = gv.__DIR__ + gv.dp__tuningPar_dir + "batchnumberresult.npy"
np.save(filename, np.array(benchmark_tune))
fig, ax = plt.subplots(1)
ax.plot(np.arange(echos_tune), benchmark_tune[:, 0], color="r", label="r_value")
ax.plot(np.arange(echos_tune), benchmark_tune[:, 1], color="b", label="std_err")
handles, labels = ax.get_legend_handles_labels()
ax.legend(handles, labels, loc="upper left")
plt.show()
except KeyboardInterrupt:
print "Shutdown requested... exiting"
except Exception:
traceback.print_exc(file=sys.stdout)
sys.exit(0)
def generateInstancesNN(instanceSize,
stride,
filenameList,
mode, plot_show = 1,
label_mutiplier = 1.0):
allInstances = np.array([])
allLabels = np.array([])
allImages = []
xlen = 0
ylen = 0
image_files, bubble_num, bubble_regions = getinfo()
for i, imageFilename in enumerate(filenameList):
if(gv.ds_show_filename):
print ('Generating instances from [' + imageFilename +
']... { patch width: ' + str(instanceSize) + ' stride: ' +
str(stride) + ' }')
filename = gv.__DIR__ + gv.__TrainImageDir__ + imageFilename
imageData = io.imread(filename)
positiveLabels = bubble_regions[image_files.index(imageFilename)]
[m, n, c] = imageData.shape
Y = np.arange(0, (m-instanceSize + 1), stride)
X = np.arange(0, (n-instanceSize + 1), stride)
ylen = len(Y)
xlen = len(X)
fname = (gv.__DIR__ +
gv.dp__Gaussia_data_dir +
str(instanceSize) + '_' +
str(stride) + '_' +
mode +
imageFilename[0:-3] + 'npy')
if(os.path.isfile(fname)):
instances, labels = gaussianDataFromFile(fname, X, Y)
else:
instances, labels = patchlabel(
Y, X, positiveLabels,patchsize = instanceSize,
stride = stride, mode = mode)
labels = labels*label_mutiplier
instances = np.append(instances, i*np.ones((instances.shape[0], 1)),
axis = 1)
allImages.append(imageData)
# append current instance data to all instance data
if (allInstances.size == 0):
allInstances = instances
allLabels = labels
else:
allInstances = np.append(allInstances, instances, axis=0)
allLabels = np.append(allLabels, labels, axis=0)
if(plot_show == 1):
image_show = np.reshape(labels, (ylen, xlen))
fig, ax = plt.subplots(2)
ax[0].imshow(imageData)
ax[0].set_title('Original Image')
img = ax[1].imshow(np.sqrt(image_show))
ax[1].set_title('Labels')
plt.colorbar(img)
plt.show()
return [allInstances, allLabels, ylen, xlen, allImages]