def extract_features(fhs,config,convolve_func):
image_config = config[0]
model_config = config[1]['model']
image_fh = fhs[0]
filter_fh = fhs[1]
conv_mode = model_config['conv_mode']
#preprocessing
array = image2array(model_config,image_fh)
preprocessed,orig_imga = preprocess(array,model_config)
#input normalization
norm_in = norm(preprocessed,conv_mode,model_config.get('normin'))
#filtering
filtered = convolve(norm_in, filter_fh, model_config, convolve_func)
#nonlinear activation
activ = activate(filtered,model_config.get('activ'))
#output normalization
norm_out = norm(activ,conv_mode,model_config.get('normout'))
#pooling
pooled = pool(norm_out,conv_mode,model_config.get('pool'))
#postprocessing
fvector_l = postprocess(norm_in,filtered,activ,norm_out,pooled,orig_imga,model_config.get('featsel'))
output = sp.concatenate(fvector_l).ravel()
return cPickle.dumps(output)
def get_data(simple=False, test_split=0.3, count=1):
"""Load data and format it into folds of (train, test, val) for use in model
Arguments:
simple -- whether or not to use a simple split rather than cross-validation (default False)
test_split -- test set percentage if using simple split (default 0.25)
count -- number of folds if using simple split (default 1)
"""
filtered, filenames, max_size, starts, ends = pr.preprocess(
'./source', './metadata/lift_times_complete.csv', pad=False)
filenames = np.array(filenames)
class_labels = getclasses()
features = createFeatures(filtered, starts)
features = features.assign(person=getpeople())
features = features.assign(filename=getfilenames())
# Remove Person 4 P2 because of incorrect sensors
features = features[~features['filename'].str.contains('04_P2')]
if not simple:
folds = k_fold(features, 1, 10)
else:
folds = simple_split(features, test_split, count)
return folds
def kmeans_skl(cube):
nonzero_cube = remove_blanks(cube)
img_data = preprocess(nonzero_cube)
pooled = pooled_img(img_data)
rxvals = rx_detector(pooled)
rx_reshaped = rxvals.reshape(rxvals.shape[0]*rxvals.shape[1]).reshape(-1, 1)
# smushed_data = data.reshape(data.shape[0]*data.shape[1],
# data.shape[2])
time1 = timer()
km = KMeans(n_clusters=5, max_iter=10, n_jobs=-1)
m = km.fit_predict(rx_reshaped)
m = m.reshape(rxvals.shape[0], rxvals.shape[1])
time2 = timer()
print('kmeans time: ', time2 - time1)
return m
def load_images(grayScale=True):
# Read images and convert to row format
x = []
y = []
for directory, subdirs, files in os.walk(path):
for file in files:
if (file.endswith(".png")):
index = int(file[3:6]) - 1
im = p.preprocess(cv2.imread("%s/%s" % (directory, file)),
grayScale=grayScale) # Grayscale
im = im / 255 # pixels [0, 1] range for faster convergence
x.append(im)
y.append(index) # keras only uses numerical labels
x = np.asarray(x, dtype=np.float32)
y = np.asarray(y, dtype=np.float32)
return x, y
def prediction():
data = {'success': False}
if request.method == 'POST':
if request.files.get('image'):
image = request.files['image'].read()
image = Image.open(io.BytesIO(image))
image = preprocess(image, target=(64, 64))
preds = model.predict(image)
preds = [np.argmax(i) for i in preds]
data['prediction'] = preds
data['success'] = True
return jsonify(data)
def compute_features(model_config, filters, array):
m_config = model_config['config']['model']
convolve_func = c_numpy_mixed
if isinstance(m_config,list):
reslist = []
for (filt,m) in zip(filters,m_config):
image_fh.seek(0)
res = compute_features_core(image_fh,filt,{'config':{'model':m}},convolve_func)
reslist.append(res)
return reslist
else:
conv_mode = m_config['conv_mode']
layers = m_config['layers']
feed_up = m_config.get('feed_up',False)
array,orig_imga = preprocess(array,m_config)
# for k in array:
# array[k] = array[k].reshape(array[k].shape + (1,))
assert len(filters) == len(layers)
dtype = array[0].dtype
array_dict = {}
for (ind,(filter,layer)) in enumerate(zip(filters,layers)):
if feed_up:
array_dict[ind-1] = array
print(array[0].shape,'filter')
if filter is not None:
array = fbcorr(array, filter, layer , convolve_func)
print(array[0].shape,'pool')
if layer.get('lpool'):
array = lpool(array,conv_mode,layer['lpool'])
print(array[0].shape,'lnorm')
if layer.get('lnorm'):
if layer['lnorm'].get('use_old',False):
array = old_norm(array,conv_mode,layer['lnorm'])
else:
array = lnorm(array,conv_mode,layer['lnorm'])
print(array[0].shape)
array_dict[len(layers)-1] = array
return array_dict
def buildRows(path, n, corner):
rows = []
for directory, subdirs, files in os.walk(path):
#print(directory)
for file in files:
if(file.endswith(".png")):
index = int(file[3:6]) - 1 # Images are in the format img0XX-00011.png. The XX tells us the label
#print("\t%s" % file)
image = p.preprocess(cv2.imread("%s/%s" % (directory, file)))
if(corner):
image = p.orb(image)
else:
image = p.canny(image)
image_row = p.convert_to_array(image, n).astype(object)
image_row = np.append(image_row, label_arr[index]) #use vstack. Resulting df is 3 x 7k
rows.append(image_row)
return rows
def center_surround_orth(model_config):
conv_mode = model_config['conv_mode']
L = np.empty((2*len( model_config['filter']['base_images'] ),) + tuple(model_config['filter']['kshape']) )
for (ind,image_config) in enumerate(model_config['filter']['base_images']):
image_fh = rendering.cairo_render(image_config,returnfh=True)
#preprocessing
array = processing.image2array(model_config ,image_fh)
preprocessed,orig_imga = processing.preprocess(array,model_config )
norm_in = norm(preprocessed,conv_mode,model_config.get('normin'))
array = make_array(norm_in)
array = array[:,:,:2].max(2).astype(np.float)
arr_box = cut_bounding_box(array)
s = model_config['filter']['kshape']
X = np.zeros(s)
(hx,wx) = X.shape
(ha,wa) = arr_box.shape
hx0 = max((hx - ha) / 2, 0)
hx1 = min(ha + hx0,hx)
ha0 = max((ha - hx) / 2, 0)
ha1 = min(hx + ha0, ha)
wx0 = max((wx - wa) / 2, 0)
wx1 = min(wa + wx0,wx)
wa0 = max((wa - wx) / 2, 0)
wa1 = min(wx + wa0, wa)
X[hx0:hx1, wx0:wx1] = arr_box[ha0:ha1, wa0:wa1]
X = normalize(X)
L[2*ind,:,:] = X
L[2*ind + 1,:,:] = X.T
return L
# factors = [0.5, 0.75, 1.0, 1.25, 1.5, 2, 5, 10]
nsegs = [5, 10, 20]
# nsegs = [30, 35, 40]
miters = [10]
for nseg in nsegs:
for miter in miters:
for factor in factors:
print('tryng ', nseg, ' segments with compactness ', factor,
' and ', miter, ' iterations')
for sample in samples:
if (sample == 'paper_grid'):
current = 0.0
else:
setD = re.search('D', sample)
if setD:
current = re.search('\d+(_\d+)?', sample).group()
cube, wn = get_cube(sample)
nonzero_cube = remove_zero(cube)
img_data = preprocess(nonzero_cube)
segment_data = get_segments(
img_data, factor, nseg, miter)
if current in names:
spectra_data, sid = get_spectra(
segment_data, nonzero_cube, wn, current,
True)
create_plots(img_data, segment_data,
spectra_data, sid, True)
outdir = 'output/segment_cube/slic_nocon_' + str(
nseg) + 'x' + str(miter)
save_plt(outdir, sample, factor)
import glob
import time
import json
from utils import calc_from_cam_to_map_matrix
from processing import grey_world, Sobel_3_colors, read_calib_params, correct_distortion, preprocess, sift
import utils
#img = cv.imread("../test_images/test_img.png")
img = cv.imread("../test_images/2019-03-02-151138.jpg")
#img = cv.imread("../test_images/Image.png")
MAX_DESC_MSE = 0.007
MIN_QUALITY = 0.0
img = preprocess(img)
kp, des = sift.detectAndCompute(img, None)
kpdes = list(zip(kp, des))
kpdes = list(filter(lambda k: k[0].response >= MIN_QUALITY, kpdes))
kp = list(map(lambda k: k[0], kpdes))
img0 = img.copy()
img_kp0 = img.copy()
RESET_POINTS = True
points_good = []
des_good = []
if RESET_POINTS:
print("Running in 'image' mode")
if not FLAGS.input:
print("FAILED: no input image")
sys.exit(1)
inputs = []
outputs = []
inputs.append(grpcclient.InferInput('data', [1, 3, 608, 608], "FP32"))
outputs.append(grpcclient.InferRequestedOutput('prob'))
print("Creating buffer from image file...")
input_image = cv2.imread(str(FLAGS.input))
if input_image is None:
print(f"FAILED: could not load input image {str(FLAGS.input)}")
sys.exit(1)
input_image_buffer = preprocess(input_image)
input_image_buffer = np.expand_dims(input_image_buffer, axis=0)
inputs[0].set_data_from_numpy(input_image_buffer)
print("Invoking inference...")
results = triton_client.infer(model_name=FLAGS.model,
inputs=inputs,
outputs=outputs,
client_timeout=FLAGS.client_timeout)
if FLAGS.model_info:
statistics = triton_client.get_inference_statistics(
model_name=FLAGS.model)
if len(statistics.model_stats) != 1:
print("FAILED: get_inference_statistics")
sys.exit(1)
print(statistics)
plt.savefig(outdir + '/' + sample[:-4] + "_c" + str(factor) + "_part" +
part + ".png")
plt.close()
if __name__ == "__main__":
os.chdir('/home/block-am/Documents/substrates/on_move_testing')
samples = [line.rstrip() for line in open('full_samples.txt', 'r')]
# factors = [0.60, 0.70, 1.3, 1.4]
factors = [0.80, 0.90, 1.0, 1.1, 1.2]
# factors = [0.75, 1.0]
# factors = [.50, 0.75, 1.0, 1.25]
# factors = [.60, .70, .75, .80, .90, 1.0, 1.1, 1.2]
nsegs = [5]
for nseg in nsegs:
for factor in factors:
print('trying ', nseg, ' segments with compactness ', factor)
for sample in samples:
cube, wn = get_cube(sample)
cube[np.isnan(cube)] = 0
minis = get_mini_cubes(cube, sample[7:12])
for m, mini in enumerate(minis):
img_data = preprocess(mini)
segment_data = get_segments(img_data, factor, nseg)
spectra_data, sid = get_spectra(segment_data, mini, wn)
create_plots(img_data, segment_data, spectra_data, sid)
outdir = 'output/hand_isolated_400_5clusters/' + sample[:-4]
save_plt(outdir, sample, factor, str(m))
def infer(self,input_img,triton_client,confidence = 0.5):
confidence = confidence
out = "output/"+input_img.split("/")[1]
# IMAGE MODE
# print("Running in 'image' mode")
if not input_img:
print("FAILED: no input image")
sys.exit(1)
inputs = []
outputs = []
inputs.append(grpcclient.InferInput('data', [1, 3, 640, 640], "FP32"))
outputs.append(grpcclient.InferRequestedOutput('prob'))
# print("Creating buffer from image file...")
input_image = cv2.imread(input_img)
if input_image is None:
print(f"FAILED: could not load input image {str(input_img)}")
sys.exit(1)
input_image_buffer,dw,dh,padding_w,padding_h= preprocess(input_image)
input_image_buffer = np.expand_dims(input_image_buffer, axis=0)
inputs[0].set_data_from_numpy(input_image_buffer)
# print("Invoking inference...")
results = self.triton_client.infer(model_name=self.model,
inputs=inputs,
outputs=outputs,
client_timeout=self.client_timeout)
if self.model_info:
statistics = self.triton_client.get_inference_statistics(model_name=self.model)
if len(statistics.model_stats) != 1:
print("FAILED: get_inference_statistics")
sys.exit(1)
# print(statistics)
# print("load model done")
result = results.as_numpy('prob')
# print(f"Received result buffer of size {result.shape}")
# print(f"Naive buffer sum: {np.sum(result)}")
detected_objects = postprocess(result, input_image.shape[1], input_image.shape[0],dw,dh,padding_w,padding_h,confidence, self.nms)
print(f"Raw boxes: {int(result[0, 0, 0, 0])}")
print(f"Detected objects: {len(detected_objects)}")
return_info = []
for box in detected_objects:
return_info.append([box.classID,box.u1,box.u2,box.v1,box.v2])
print(f"{COCOLabels(box.classID).name}: {box.confidence}")
input_image = render_box(input_image, box.box(), color=tuple(RAND_COLORS[box.classID % 64].tolist()))
size = get_text_size(input_image, f"{COCOLabels(box.classID).name}: {box.confidence:.2f}", normalised_scaling=0.6)
input_image = render_filled_box(input_image, (box.x1 - 3, box.y1 - 3, box.x1 + size[0], box.y1 + size[1]), color=(220, 220, 220))
input_image = render_text(input_image, f"{COCOLabels(box.classID).name}: {box.confidence:.2f}", (box.x1, box.y1), color=(30, 30, 30), normalised_scaling=0.5)
if out:
cv2.imwrite(out, input_image)
print(f"Saved result to {out}")
else:
cv2.imshow('image', input_image)
cv2.waitKey(0)
cv2.destroyAllWindows()
return return_info
import pathlib
from util import get_datasets, calculate_accuracy
from processing import preprocess, postprocess
from tflite_helper import TFLiteConvertor
##################
# Prepare datasets
##################
cv_dataset, test_dataset = get_datasets()
num_images_in_cv_dataset = 100
# For 224 x 224
preprocess_cv_dataset_224 = list()
for i in range(0, num_images_in_cv_dataset):
preprocess_cv_dataset_224.append(preprocess(cv_dataset[i], 224, 224))
# For 299 x 299
preprocess_cv_dataset_299 = list()
for i in range(0, num_images_in_cv_dataset):
preprocess_cv_dataset_299.append(preprocess(cv_dataset[i], 299, 299))
def representative_data_gen_224():
for input_image in preprocess_cv_dataset_224:
yield [input_image]
def representative_data_gen_299():
for input_image in preprocess_cv_dataset_299:
yield [input_image]
def main(rootpath,
data_dir,
device,
training=True,
model_type="cnn",
sample_name=None,
display=False):
"""
The primary method for controlling the training and testing of data. rootpath is the parent directory of main.py, data_dir
is the subdirectory containing all training data, device is either CPU or GPU. Training is true if we are training a model
and false otherwise. If training is false, sample name is the path to the image that you are converting to LaTeX. If display
is true, print the output of the model for each character it is passed. The model type can be a convolutional neural network (cnn),
k-nearest neighbors (knn), or a decision tree (tree). The knn and tree models have already been trained and will not be trained if
training is true. This will only train the cnn.
"""
dataset = processing.get_dataset(data_dir,
img_size=28,
filename="symbols_rectified")
mean, std = torch.Tensor([.0942]), torch.Tensor(
[.2202]
) # The values that were computed using the processing.normalize() function
dataset.transforms = torchvision.transforms.Compose([
torchvision.transforms.Grayscale(num_output_channels=1),
torchvision.transforms.Resize([224, 224]),
torchvision.transforms.RandomRotation(degrees=5),
torchvision.transforms.ToTensor(),
torchvision.transforms.RandomErasing(p=.2),
torchvision.transforms.Normalize((mean), (std))
])
if training:
train_index, test_index = processing.test_train_split(dataset)
dataloader = processing.get_dataloaders(dataset,
train_index,
test_index,
batch_size=128)
model, val_acc_history = train_cnn(rootpath, dataset, dataloader,
device)
else:
output = []
characters, corners, areas = processing.find_symbols(sample_name,
display=True)
if model_type == "cnn":
num_classes = len(dataset.classes)
model, input_size = cnn.initialize_model(
"simple",
num_classes,
resume_from=os.path.join(rootpath, "weights",
"cnn_weights_epoch_4"))
model.eval()
for character in characters:
img = processing.preprocess(character,
img_size=224,
mean=mean,
std=std,
to_numpy=False,
display=True)
out = model(img.unsqueeze(0))
_, prediction = torch.topk(out, k=2, dim=1)
symbols = []
for pred in prediction[0]:
symbols.append(cnn.label_number_to_name(dataset, pred))
if display:
print(symbols[0])
output.append(symbols)
elif model_type == "knn":
for character in characters:
img = processing.preprocess(character,
img_size=224,
mean=None,
std=None,
to_numpy=True,
display=True)
out = knn.get_nearest_neighbors(
img, k=51) # The two most likely outputs
symbols = [out[0], out[1]]
if display:
print(symbols[0])
output.append(symbols)
elif model_type == "tree":
for character in characters:
img = processing.preprocess(character,
img_size=224,
mean=None,
std=None,
to_numpy=True,
display=True)
out = tree.get_label(img)
symbols = [out, out]
if display:
print(symbols[0])
output.append(symbols)
else:
print("That is not a valid model name")
equation = convert.to_latex(output, corners)
print(equation)
import processing
import os
import json
root=os.getenv("root_folder")
os.system(f"mkdir -p result")
cases = ["web-service", "load-balancer"]
for case in cases:
processing.preprocess(root_dir=f"{root}/{case}")
root_dir = f"json/{case}"
latencies = processing.total_mean_latencies_obj_in_second(root_dir)
processing.write_result(f"result/latencies_{case}_.json", latencies)
success_rate = processing.calculate_avg_success_rate(root_dir)
processing.write_result(f"result/success_rate_{case}_.json", success_rate)
