def load_tile(tuple,
crop=None,
aspect_ratio=None,
tile_width=None,
tile_height=None):
"""
Load an image into memory and crop/resize it to fit into a tile.
This is the time-consuming part of the tiling process, so I'm breaking it into a function to multiprocess it.
:param tuple:
:param crop:
:param aspect_ratio:
:param tile_width:
:param tile_height:
:return:
"""
if crop:
return resize_image(crop_image_to_aspect_ratio(
Image.open(tuple.get('filename')), aspect_ratio),
width=tile_width), (tuple.get('c') * tile_width,
tuple.get('r') * tile_height)
else:
return resize_image(Image.open(tuple.get('filename')),
width=tile_width,
height=tile_height), (tuple.get('c') * tile_width,
tuple.get('r') * tile_height)
def scatterplot_images(embeddings, images, width=1200, height=900, max_dim=40):
'''
Plots images in a scatterplot where coordinates are from
embeddings.
'''
tx, ty = embeddings[:, 0], embeddings[:, 1]
tx = (tx - np.min(tx)) / (np.max(tx) - np.min(tx))
ty = (ty - np.min(ty)) / (np.max(ty) - np.min(ty))
full_image = Image.new('RGB', size=(width, height), color=(55, 61, 71))
for f_img, x, y in tqdm(zip(images, tx, ty)):
# read and resize image
tile = Image.open(f_img)
rs = max(1, tile.width / max_dim, tile.height / max_dim)
tile_width = int(tile.width / rs)
tile_height = int(tile.height / rs)
aspect_ratio = float(tile_width) / tile_height
tile = resize_image(tile, tile_width, tile_height, aspect_ratio)
# add the image to the graph
x_coord = int((width - max_dim) * x)
y_coord = int((height - max_dim) * y)
img_coords = (x_coord, y_coord)
full_image.paste(tile, box=img_coords, mask=tile.convert('RGBA'))
return full_image
def process_sample(img_path, xml_path, index, classes, type='default'):
img = cv2.imread(img_path)
if xml_path is not None:
xml = ET.parse(xml_path).getroot()
xml_labels = label_utils.parse_xml(xml)
else:
xml_labels = []
if opt.resolution != [0, 0]:
(h, w) = img.shape[:2]
old_dims = [w, h]
scale, offset = image_utils.get_scale_val(old_dims, opt.resolution,
opt.keep_ar)
img = image_utils.resize_image(img, opt.resolution, scale, offset)
xml_labels = image_utils.scale_xml_labels(xml_labels, scale, offset)
(h, w) = img.shape[:2]
new_dims = [w, h]
txt_labels = image_utils.xyxy_to_xywh(xml_labels, new_dims)
path_txt = "{}labels/{}_{}.txt".format(opt.root, index, type)
path_xml = "{}labels_xml/{}_{}.xml".format(opt.root, index, type)
path_img = "{}images/{}_{}.jpg".format(opt.root, index, type)
label_utils.save_as_txt(txt_labels, path_txt, classes, opt.single_cls)
label_utils.save_as_xml(xml_labels, path_xml, old_dims, opt.single_cls)
cv2.imwrite(path_img, img)
def calibration_fn():
"""function to pass as argument to the builder function
it will be used to optimize the network based on given examples """
print("[CALIBRATION] Starting calibration process...")
images_found = sorted(os.listdir(IMAGES_PATH))
print("[CALIBRATION] Obtaining calibration images from {}"
.format(images_found))
print("[CALIBRATION] Done! Found {} images for calibration"
.format(len(images_found)))
print("[CALIBRATION] Starting image yielding...")
start_yielding = time.time()
for image_path in images_found:
input_image = imgutils.read_image_from_cv2(IMAGES_PATH + image_path)
resized_image = imgutils.resize_image(input_image, (INPUT_SIZE, INPUT_SIZE))
final_image = resized_image[np.newaxis, ...].astype("uint8")
print("[CALIBRATION] Yielding image from {}".format(image_path))
start_yielding = time.time()
yield (final_image,)
print("[CALIBRATION] Image yielding Done, it took {} ms"
.format((time.time()-start_yielding)*1000))
CALIBRATION_TIME = (time.time()-start_yielding)*1000
print("[CALIBRATION] Calibration procces finished, it took {} ms"
.format(CALIBRATION_TIME))
def tile_old(tile_filename: str,
output_filename: str,
output_width: int,
output_height: int,
choice: int = 0,
crop: bool = True):
"""
Deprecated. It will populate the tiles but you have to specify the same choice number for every tile and it doesn't have any smarts about consecutive duplicates or multithreading.
:param tile_filename:
:param output_filename:
:param output_width:
:param output_height:
:param choice:
:param crop:
:return:
"""
df = pd.read_csv(tile_filename)
df = df[df['choice'] == choice]
column_count = df['col'].max()
row_count = df['row'].max()
tile_width = int(output_width / column_count)
tile_height = int(output_height / row_count)
result = Image.new('RGB',
(column_count * tile_width, row_count * tile_height))
pbar = tqdm.tqdm(total=row_count * column_count)
for r in range(row_count):
for c in range(column_count):
choice_row = df[(df['row'] == r) & (df['col'] == c)]
filename = choice_row['filename'].iloc[0]
if crop:
resized_child = crop_image(resize_image(Image.open(filename),
width=tile_width),
width=tile_width,
height=tile_height)
else:
resized_child = resize_image(Image.open(filename),
width=tile_width,
height=tile_height)
result.paste(im=resized_child,
box=(c * tile_width, r * tile_height))
pbar.update(1)
result.save(output_filename)
def dream(image_filename, layer):
"""
image_filename is the path to the image to process
layer is a dictionnary, containing the layer name and the feature channels to maximise.
It has the following format:
{
name: 'conv2d0:0',
fromChannel: 0,
toChannel: 10
}
"""
global model
start_loading = time.time()
model = inception5h.Inception5h()
session = tf.compat.v1.Session(graph=model.graph)
image = image_utils.load_image(filename=image_filename)
start_processing = time.time()
# The image we're going to be working on should be within 400x600 pixels for performance reasons.
DESIRED_WIDTH = 400
DESIRED_HEIGHT = 600
rescale_factor = min(DESIRED_WIDTH / image.shape[1],
DESIRED_HEIGHT / image.shape[0])
image_downscaled = image_utils.resize_image(image=image,
factor=rescale_factor)
layer_tensor = model.get_layer_tensor_by_name(
layer['name'])[:, :, :, layer['fromChannel']:layer['toChannel'] + 1]
print('Layer tensor: ' + str(layer_tensor))
img_result = recursive_optimize(layer_tensor=layer_tensor,
image=image_downscaled,
num_iterations=10,
step_size=3.0,
num_repeats=4,
blend=0.2,
session=session)
upscaled_result = image_utils.resize_image(image=img_result,
size=image.shape)
image_utils.save_image(upscaled_result, filename='img/deapdream_image.jpg')
session.close()
end_processing = time.time()
print(
"Loading time: %f sec, processing time: %f sec" %
(start_processing - start_loading, end_processing - start_processing))
def generate_mosaic(embeddings,
images,
mosaic_width,
mosaic_height,
tile_width=72,
tile_height=56,
title="Doppler Mosaic",
title_rbg=(229, 229, 229),
save_as_file=False,
return_image=True,
verbose=False):
'''
Transforms 2-dimensional embeddings to a grid.
Plots the images for each embedding in the corresponding grid (mosaic).
Includes arguments for the dimensions of each tile and the the mosaic.
'''
# assign to grid
grid_assignment = transformPointCloud2D(embeddings,
target=(mosaic_width,
mosaic_height))
full_width = tile_width * mosaic_width
full_height = tile_height * (mosaic_height + 2)
aspect_ratio = float(tile_width) / tile_height
# create an empty image for the mosaic
mosaic = Image.new('RGB', (full_width, full_height))
# iterate through each image and where it is possed to live.
for f_img, (idx_x, idx_y) in tqdm(zip(images, grid_assignment[0]),
disable=not verbose):
# Find exactly where the image will be
x, y = tile_width * idx_x, tile_height * idx_y
# read the image, center crop the image and add it to the mosaic
try:
img = Image.open(f_img).convert('RGBA')
tile = resize_image(img, tile_width, tile_height, aspect_ratio)
mosaic.paste(tile, (int(x), int(y)))
except Exception as e:
print(f"Failed to add image {f_img} see error:\n{e}")
# write an annotation
#fnt = ImageFont.truetype('Pillow/Tests/fonts/FreeMono.ttf', int(tile_height * 1.2) )
draw = ImageDraw.Draw(mosaic)
draw.text((4, (tile_height * (mosaic_height)) + 10), title, title_rbg)
if save_as_file and not os.path.exists(save_as_file):
try:
mosaic.save(save_as_file)
except Exception as e:
print(
f'Saving the mosaic to {save_as_file} failed, see error:\n{e}')
if return_image:
return mosaic
def find_dominant_color_and_generate_thumb(image, pic, thumbs_folder, num_clusters, thumb_size, save_thumb):
"""
Find dominant color of a picture and returns color
If save_thumb is True than create and save thumbnail with the num_clusters common colors
"""
image = resize_image(image, (thumb_size, thumb_size))
ar, shape, codes, vecs = calculate_dominant_colors(image, num_clusters)
dominant_color = calculate_dominant_color_from_clusters(codes, vecs)
if save_thumb:
im = convert_scipy_image_to_n_colors(ar, shape, codes, vecs)
if save_image(im, pic, thumbs_folder):
return dominant_color, im
else:
return None, None
else:
return dominant_color, None
def find_dominant_color_and_generate_thumb(image, pic, thumbs_folder,
num_clusters, thumb_size,
save_thumb):
"""
Find dominant color of a picture and returns color
If save_thumb is True than create and save thumbnail with the num_clusters common colors
"""
image = resize_image(image, (thumb_size, thumb_size))
ar, shape, codes, vecs = calculate_dominant_colors(image, num_clusters)
dominant_color = calculate_dominant_color_from_clusters(codes, vecs)
if save_thumb:
im = convert_scipy_image_to_n_colors(ar, shape, codes, vecs)
if save_image(im, pic, thumbs_folder):
return dominant_color, im
else:
return None, None
else:
return dominant_color, None
def run(content_images_paths,style_images_paths,style_square_crop=False,style_image_size=400,content_square_crop=False,
image_size=400,checkpoint='arbitrary_style_transfer/model.ckpt',maximum_styles_to_evaluate=1024,
interpolation_weights='[1.0]'):
f = open('docs/accessKeys.csv', 'r')
reader = csv.reader(f,delimiter=',')
i = 1
keys = []
for row in reader:
if i == 2:
keys = row
i = i + 1
f.close()
s3 = boto3.client('s3',
aws_access_key_id=keys[0],
aws_secret_access_key=keys[1])
start = time.time()
front = content_images_paths.split('.')[0]
# bucket name, dir of file in s3 bucket, local name
s3.download_file(config.bucketname, 'target1/'+content_images_paths, content_images_paths)
convertimage(content_images_paths, False)
content = 'c_'+front+'.jpg'
s3.download_file(config.bucketname, 'style1/'+style_images_paths, style_images_paths)
convertimage(style_images_paths, True)
style = 's_'+front+'.jpg'
# start evaluate
tf.logging.set_verbosity(tf.logging.INFO)
# if not tf.gfile.Exists(output_dir):
# tf.gfile.MkDir(output_dir)
with tf.Graph().as_default(), tf.Session() as sess:
# Defines place holder for the style image.
style_img_ph = tf.placeholder(tf.float32, shape=[None, None, 3])
if style_square_crop:
style_img_preprocessed = image_utils.center_crop_resize_image(
style_img_ph, style_image_size)
else:
style_img_preprocessed = image_utils.resize_image(style_img_ph,
style_image_size)
# Defines place holder for the content image.
content_img_ph = tf.placeholder(tf.float32, shape=[None, None, 3])
if content_square_crop:
content_img_preprocessed = image_utils.center_crop_resize_image(
content_img_ph, image_size)
else:
content_img_preprocessed = image_utils.resize_image(
content_img_ph, image_size)
# Defines the model.
stylized_images, _, _, bottleneck_feat = build_model.build_model(
content_img_preprocessed,
style_img_preprocessed,
trainable=False,
is_training=False,
inception_end_point='Mixed_6e',
style_prediction_bottleneck=100,
adds_losses=False)
if tf.gfile.IsDirectory(checkpoint):
checkpoint = tf.train.latest_checkpoint(checkpoint)
else:
checkpoint = checkpoint
tf.logging.info('loading latest checkpoint file: {}'.format(checkpoint))
init_fn = slim.assign_from_checkpoint_fn(checkpoint,
slim.get_variables_to_restore())
sess.run([tf.local_variables_initializer()])
init_fn(sess)
# Gets the list of the input style images.
style_img_list = tf.gfile.Glob(style)
if len(style_img_list) > maximum_styles_to_evaluate:
np.random.seed(1234)
style_img_list = np.random.permutation(style_img_list)
style_img_list = style_img_list[:maximum_styles_to_evaluate]
# Gets list of input content images.
content_img_list = tf.gfile.Glob(content)
for content_i, content_img_path in enumerate(content_img_list):
content_img_np = image_utils.load_np_image_uint8(content_img_path)[:, :, :
3]
content_img_name = os.path.basename(content_img_path)[:-4]
# Saves preprocessed content image.
inp_img_croped_resized_np = sess.run(
content_img_preprocessed, feed_dict={
content_img_ph: content_img_np
})
# image_utils.save_np_image(inp_img_croped_resized_np,
# os.path.join(output_dir,
# '%s.jpg' % (content_img_name)))
# Computes bottleneck features of the style prediction network for the
# identity transform.
identity_params = sess.run(
bottleneck_feat, feed_dict={style_img_ph: content_img_np})
for style_i, style_img_path in enumerate(style_img_list):
if style_i > maximum_styles_to_evaluate:
break
style_img_name = os.path.basename(style_img_path)[:-4]
style_image_np = image_utils.load_np_image_uint8(style_img_path)[:, :, :
3]
if style_i % 10 == 0:
tf.logging.info('Stylizing (%d) %s with (%d) %s' %
(content_i, content_img_name, style_i,
style_img_name))
# Saves preprocessed style image.
style_img_croped_resized_np = sess.run(
style_img_preprocessed, feed_dict={
style_img_ph: style_image_np
})
# image_utils.save_np_image(style_img_croped_resized_np,
# os.path.join(output_dir,
# '%s.jpg' % (style_img_name)))
# Computes bottleneck features of the style prediction network for the
# given style image.
style_params = sess.run(
bottleneck_feat, feed_dict={style_img_ph: style_image_np})
interpolation_weights = ast.literal_eval(interpolation_weights)
# Interpolates between the parameters of the identity transform and
# style parameters of the given style image.
for interp_i, wi in enumerate(interpolation_weights):
stylized_image_res = sess.run(
stylized_images,
feed_dict={
bottleneck_feat:
identity_params * (1 - wi) + style_params * wi,
content_img_ph:
content_img_np
})
# Saves stylized image.
image_utils.save_np_image(
stylized_image_res,
front+'.jpg')
upload_file(front+'.jpg','styletransferimage','output1/'+front+'.jpg')
print("finished upload")
os.remove('c_'+front+'.jpg')
os.remove('s_'+front+'.jpg')
os.remove(front+'.jpg')
status = False
retryCounter = 2
while retryCounter > 0:
try:
channel = grpc.insecure_channel('54.164.44.43:50051')
stub = uid_management_pb2_grpc.UidManagementStub(channel)
status = stub.TransferCompleted(uid_management_pb2.Id(id=front))
retryCounter = 0
except:
retryCounter -= 1
def run_inference(self,
image_path=None,
labels=None,
bbox_results=None,
performance_results=None,
warmup_iters=0,
threshold=0.5,
image=None,
resize=False,
opencv=False):
input_size = int(self.attributes["INPUT_SIZE"])
# Load image with opencv backend
if image_path is not None:
if opencv:
print("[INFERENCE] Loading image with opencv backend...")
# opencv option
image, total_image_loading = imgutils.load_image_as_np(
image_path, "CV2")
# preprocess image to work on tf
print("[INFERENCE] Preprocessing image...")
start_preprocessing = time.time()
# resize image to netwrk input dimensions
if resize:
resized_image = imgutils.resize_image(
image, (input_size, input_size))
else:
resized_image = image
# conventional conversion (use with opencv option)
# The input needs to be a tensor, convert it using `tf.convert_to_tensor`.
input_tensor = tf.convert_to_tensor(resized_image)
#input_tensor = tf.convert_to_tensor(image)
# The model expects a batch of images, so add an axis with `tf.newaxis`.
input_tensor = input_tensor[tf.newaxis, ...]
total_preprocessing = (time.time() -
start_preprocessing) * 1000
print("[INFERENCE] Preprocessing done!, it took {}ms".format(
total_preprocessing))
# tf backend
else:
print("[INFERENCE] Loading image with tf backend...")
if resize:
# dataset option
dataset, total_image_loading, total_preprocessing = imgutils.get_dataset_tf(
image_path=image_path, input_size=input_size)
else:
# dataset option
dataset, total_image_loading, total_preprocessing = imgutils.get_dataset_tf(
image_path=image_path)
# take the batched image
dataset_enum = enumerate(dataset)
input_tensor = list(dataset_enum)[0][1]
# take image as np and convert to rgb
image_bgr = input_tensor.numpy()[0]
resized_image = image_bgr[..., ::-1].copy()
# get a copy of the graph func
#graph_func = self.attributes["graph_func"]
saved_model_loaded = self.attributes["saved_model_loaded"]
if warmup_iters == 0:
print("[INFERENCE] Now performing inference...")
inference_start_time = time.time()
# get the detections
detections = saved_model_loaded(input_tensor)
total_inference = (time.time() - inference_start_time) * 1000
print("[INFERENCE] Inference took {} ms".format(total_inference))
# TODO: ADD tracker updating
# draw results on image
if (bbox_results is not None) and (performance_results
is not None):
drawing_time = imgutils.draw_bounding_boxes(
resized_image,
detections,
labels,
threshold,
bbox_results=bbox_results)
height, width, _ = resized_image.shape
# add new performance results to object
performance_results.add_new_result(width, height,
total_image_loading,
total_preprocessing,
total_inference,
drawing_time)
else:
# draw case of just one image
imgutils.draw_bounding_boxes(image,
detections,
labels,
threshold,
save_results=True)
else:
warmup_start = time.time()
if opencv:
print(
"[WARMUP] Starting warmup with opencv backend on {} iters".
format(warmup_iters))
for i in range(warmup_iters):
print(
"[WARMUP] Generating image {} with dims {}x{}".format(
i + 1, input_size, input_size))
image = np.random.normal(size=(input_size, input_size,
3)).astype(np.uint8)
# conventional conversion (use with opencv option)
# The input needs to be a tensor, convert it using `tf.convert_to_tensor`.
input_tensor = tf.convert_to_tensor(image)
# The model expects a batch of images, so add an axis with `tf.newaxis`.
input_tensor = input_tensor[tf.newaxis, ...]
# get the detections
print("[WARMUP] Now performing warmup inference...")
inference_start_time = time.time()
detections = saved_model_loaded(input_tensor)
print("[WARMUP] Warmup inference took {} ms".format(
(time.time() - inference_start_time) * 1000))
_ = imgutils.draw_bounding_boxes(image, detections, labels,
0.05)
else:
print(
"[WARMUP] Starting warmup with tf backend on {} iterations..."
.format(warmup_iters))
for i in range(warmup_iters):
print(
"[WARMUP] Generating image {} with dims {}x{}".format(
i + 1, input_size, input_size))
features = np.random.normal(loc=112,
scale=70,
size=(1, input_size,
input_size,
3)).astype(np.float32)
print("[WARMUP] Creating features...")
features = np.clip(features, 0.0, 255.0).astype(np.uint8)
features = tf.convert_to_tensor(
value=tf.compat.v1.get_variable(
"features", initializer=tf.constant(features)))
print("[WARMUP] Creating dataset from features...")
dataset = tf.data.Dataset.from_tensor_slices([features])
dataset = dataset.repeat(count=1)
dataset_enum = enumerate(dataset)
print("[WARMUP] Retrieving image and input tensor...")
# get input tensor and cast to image (np)
input_tensor = list(dataset_enum)[0][1]
image = input_tensor.numpy()[0]
# get the detections
print("[WARMUP] Now performing warmup inference...")
inference_start_time = time.time()
detections = saved_model_loaded(input_tensor)
print("[WARMUP] Warmup inference took {} ms".format(
(time.time() - inference_start_time) * 1000))
#perform drawing warmup with very low threshold
_ = imgutils.draw_bounding_boxes(image, detections, labels,
0.05)
# display results in ms
print("[WARMUP] Warmup finished, it took {} ms".format(
(time.time() - warmup_start) * 1000))
def minimal_example(input_image):
input_text = ["a diagram", "a dog", "a cat"]
st.write("{}".format(input_text))
probs = compute_probs(input_image, input_text)
st.write("Label probs: {}".format(probs))
return
def imagenet_example(input_image):
predicted_class = predict_class(input_image)
formatted_predictions = "\n".join(
f"{i}) {label}" for i, label in enumerate(predicted_class, start=1))
st.text("ImageNet labels:\n{}\n".format(formatted_predictions))
return
if __name__ == "__main__":
st.title("Predict top-5 ImageNet labels with OpenAI's CLIP.")
uploaded_file = st.file_uploader("Choose an image...", type="jpg")
if uploaded_file is not None:
input_image = pil_image.open(uploaded_file)
input_image = resize_image(input_image, target_size=[224, 224])
st.image(input_image, caption="Query image")
imagenet_example(input_image)
if not roll(prob):
inds_to_keep.append(j)
intervals[i] = list(np.array(row)[inds_to_keep])
return intervals
if __name__ == "__main__":
from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
import pdb
import timeit
im = Image.open('source/temple.jpg')
im = util.resize_image(im, 4)
original_size = im.size
a = np.asarray(im)
intervals = threshold_interval(a,
np.shape(a),
0.6,
0.75,
width=1,
func=util.lightness,
inverted=True)
intervals = randomly_filter_interval(intervals, 0.3)
colors = [np.array([0, 255, 0])]
ac = a.copy()
def track_objects(detections_path, frames_path, output_path, display_results=False, save_results=False):
# perform warmup to visualize results in "real time"
warmup(20, display_results=display_results)
# instantiate a tracker results object
sort_results = SortResults()
sort_performance = SortPerformanceResults()
# get the detections file
detections = pd.read_csv(args.detections_path).iloc[:, 1:]
# get image dims to set boundaries
width = detections["width"].unique()[0]
height = detections["height"].unique()[0]
# instantiate the tracker with given boundaries
tracker = Sort(dims=(width, height))
# read frames sorted by numerical order
frame_paths = sorted(glob.glob(frames_path + "*.png"), key=numericalSort)
frame_count = 1
for frame_path in frame_paths:
frame_filename = frame_path.split('/')[-1]
# init sort results metadata (imaage and frame count)
sort_results.init_frame_metadata(frame_path, frame_count)
sort_performance.init_frame_metadata(frame_path, frame_count)
# init a bbox array to store bboxes
current_bboxes = np.array([[]]).astype(int)
# get detections of current frame
pig_detections_data = detections.loc[(detections["frame"] == frame_count) & (detections["detection_class"] == 'pig')]
if pig_detections_data.shape[0] == 0:
print("[SORT] No detections for pigs here, skipping frame {}"
.format(frame_count))
frame_count += 1
continue
print(pig_detections_data.head())
#sys.exit()
# get frame and its dimensions
print("[SORT] Loading image with opencv...")
start_loading = time.time()
frame = imgutils.read_image_from_cv2(frame_path)
total_loading = (time.time()-start_loading)*1000
print("[SORT] Done!, it took {}ms".format(total_loading))
frame_height, frame_width, _ = frame.shape
# get detections results image dims
width = pig_detections_data["width"].unique()[0]
height = pig_detections_data["height"].unique()[0]
print("[SORT] Starting image preprocessing...")
start_preprocessing = time.time()
# resize image to match detections results
if (frame_height != height) or (frame_width != width):
print("[SORT] Frame dims are not the same from the detections")
print("[SORT] Resizing...")
resized_frame = imgutils.resize_image(frame, (width, height))
print("[SORT] Done!")
else:
print("[SORT] Frame dims are the same from the detections")
resized_frame = frame
total_preprocessing = (time.time()-start_preprocessing)*1000
print("[SORT] Done!, it took {}ms".format(total_preprocessing))
print("[SORT] Preparing detections for tracker...")
start_preparing = time.time()
for _, row in pig_detections_data.iterrows():
# prepare bbox detections to format required by sort tracker
x1 = row.xmin
x2 = row.xmax
y1 = row.ymin
y2 = row.ymax
# check if there is a valid detection
if valid_detection([x1, x2, y1, y2]):
print("[SORT] Detection is valid!")
# fomat detection for sort input
bbox = np.array([[x1, y1, x2, y2]]).astype(int)
print("[SORT] Bbox: ", bbox)
if current_bboxes.size == 0:
current_bboxes = bbox
else:
np.append(current_bboxes, bbox, axis=0).astype(int)
else:
print("[SORT] Detection is invalid!, skipping...")
total_preparing = (time.time()-start_preparing)*1000
print("[SORT] Done!, it took {}ms".format(total_preparing))
# send bbox to tracker
# if detection is unmatched, it will initialize a new tracker
# if its matched it should should, predict and update
print("[SORT] Updating trackers...")
start_update = time.time()
objects_tracked = tracker.update(current_bboxes)
total_update = (time.time()-start_update)*1000
print("[SORT] Done!, it took {}ms".format(total_update))
print("[SORT] Actual trackers: ", tracker.total_trackers)
print("[SORT] Objects tracked: ", objects_tracked)
#sys.exit()
# get tracking results
print("[SORT] Getting tracker results...")
start_results = time.time()
# set a copy of the current frame to display results
drawed_frame = np.copy(resized_frame)
for object_tracked in objects_tracked:
# get trackers info like state and time since update
tracker_id = object_tracked.id
tracker_state = "active" if object_tracked.active else "inactive"
time_since_update = object_tracked.time_since_update
initialized_in_roi = object_tracked.initialized_in_ROI
# get the bbox returned as [x1, y1, x2, y2]
bbox = object_tracked.get_state().astype(int)
first_centroid = (object_tracked.first_centroid[0], object_tracked.first_centroid[1])
last_centroid = (object_tracked.last_centroid[0], object_tracked.last_centroid[1])
# get x-y coords
xmin = bbox[0]
ymin = bbox[1]
xmax = bbox[2]
ymax = bbox[3]
print("[SORT] Bbox from tracker: ", bbox)
print("[SORT] Active?: ", tracker_state)
print("[SORT] Time since update: ", time_since_update)
print("[SORT] Initialized in roi?: " , initialized_in_roi)
# draw bboxes and label on frame
drawed_frame = imgutils.draw_trackers_bounding_box(drawed_frame, "PIG", object_tracked)
# add a new result to trackers results
sort_results.add_new_result(width, height,
tracker_id, tracker_state,
time_since_update,
initialized_in_roi,
first_centroid, last_centroid,
[xmin, xmax, ymin, ymax])
total_results = (time.time()-start_results)*1000
print("[SORT] Tracker results done!, it took {}ms".format(total_results))
# update trackers state to active/inactive depending on position
print("[SORT] Updating state to trackers")
start_update_state = time.time()
tracker.update_trackers_state()
total_update_state = (time.time()-start_update_state)*1000
print("[SORT] States update!, it took {}ms".format(total_update_state))
# draw trackers info on the image
drawed_frame = imgutils.draw_tracker_info(drawed_frame, "PIG", tracker)
# display results on screen, if scaled will adapt frame to screen dimensions
if display_results:
imgutils.display_frame(drawed_frame, scaled=True, message="Tracker")
# save results to a given path
if save_results:
imgutils.save_frame(drawed_frame, output_path, detections_path, frame_count, "tracker_frames")
total_trackers = tracker.total_trackers
sort_performance.add_new_result(width, height, total_loading, total_preprocessing,
total_preparing, total_update, total_results,
total_update_state, total_trackers)
frame_count+=1
# save tracker results on given path
sort_results.save_results(output_path, detections_path)
# save performance results on given path
sort_performance.save_results(output_path, detections_path)
return tracker.total_trackers
def run_inference_on_images(self,
images_path,
warmup_iters=0,
model_dir=None,
labels=None,
threshold=0.3,
iou=0.45):
input_size = int(self.attributes["INPUT_SIZE"])
labels = self.attributes["LABELS"]
# get a copy of the graph func
#graph_func = self.attributes["graph_func"]
if model_dir is not None:
saved_model_loaded = self.get_func_from_saved_model(model_dir)
else:
saved_model_loaded = self.attributes["saved_model_loaded"]
#warmup
if warmup_iters > 0:
print("[INFERENCE] Starting warmup on {} iterations...".format(
warmup_iters))
warmup_start = time.time()
# get input size from model attributes
input_size = int(self.attributes["INPUT_SIZE"])
# create a set of random images and perform inference
for i in range(warmup_iters):
print("[INFERENCE] Generating image {} with dims {}x{}".format(
i + 1, input_size, input_size))
# create random image
image = np.random.normal(size=(input_size, input_size,
3)).astype(np.float32) / 255.
# conventional conversion (use with opencv option)
# The input needs to be a tensor, convert it using `tf.convert_to_tensor`.
# resize image to netwrk input dimensions
resized_image = imgutils.resize_image(image,
(input_size, input_size))
images_data = []
for i in range(1):
images_data.append(resized_image)
images_data = np.asarray(images_data).astype(np.float32)
input_tensor = tf.constant(images_data)
# get the detections
print("[WARMUP] Now performing warmup inference...")
inference_start_time = time.time()
detections = saved_model_loaded(input_tensor)
print("[WARMUP] Warmup inference took {} ms".format(
(time.time() - inference_start_time) * 1000))
print("[INFERENCE] Preprocessing network outputs...")
start_output = time.time()
# extract output tensors metadata: boxes, confidence scores
print("[INFERENCE] Extracting output tensors metadata...")
keyval_start = time.time()
for key, value in detections.items():
#print("[INFERENCE-DEBUG] key {}: value {}".format(key, value))
boxes = value[:, :, 0:4]
pred_conf = value[:, :, 4:]
#print("[INFERENCE-DEBUG] boxes ", boxes, type(boxes))
#print("[INFERENCE-DEBUG] confidence ", pred_conf, type(pred_conf))
print(
"[INFERENCE] Done extracting metadata, it took {}".format(
(time.time() - keyval_start) * 1000))
print("[INFERENCE] Performing NMS to output...")
nms_start = time.time()
# perform non-max supression to retrieve valid detections only
boxes, scores, classes, valid_detections = tf.image.combined_non_max_suppression(
boxes=tf.reshape(boxes, (tf.shape(boxes)[0], -1, 1, 4)),
scores=tf.reshape(
pred_conf,
(tf.shape(pred_conf)[0], -1, tf.shape(pred_conf)[-1])),
max_output_size_per_class=50,
max_total_size=50,
iou_threshold=iou,
score_threshold=threshold)
results = [
boxes.numpy(),
scores.numpy(),
classes.numpy(),
valid_detections.numpy()
]
print("[INFERENCE] NMS done, it took {} ms".format(
(time.time() - nms_start) * 1000))
total_output = (time.time() - start_output) * 1000
print(
"[INFERENCE] Done procesing output!, it took {}ms".format(
total_output))
_ = imgutils.draw_yolo_bounding_boxes(resized_image, results,
labels)
# display results in ms
print("[WARMUP] Warmup finished, it took {} ms".format(
(time.time() - warmup_start) * 1000))
# define frame count and a helper function to read
# the images sorted by numerical index
frame_count = 1
numbers = re.compile(r'(\d+)')
def numericalSort(value):
parts = numbers.split(value)
parts[1::2] = map(int, parts[1::2])
return parts
# read image sorted by numerical order
image_paths = sorted(glob.glob(images_path + "*.png"),
key=numericalSort)
print("[INFERENCE] Found {} images in {} ...".format(
len(image_paths), images_path))
#create a class to store results
bbox_results = BboxResults()
performance_results = YoloPerformanceResults()
# Iterate over all images, perform inference and update
# results dataframe
for image_path in image_paths:
print("[INFERENCE] Processing frame/image {} from {}".format(
frame_count, image_path))
image_filename = image_path.split('/')[-1]
# init metadata
bbox_results.init_frame_metadata(image_filename, frame_count)
performance_results.init_frame_metadata(image_filename,
frame_count)
# case inference
print("[INFERENCE] Loading image with opencv backend...")
# opencv option
image_loading_start = time.time()
image = imgutils.read_image_from_cv2(image_path)
total_image_loading = (time.time() - image_loading_start) * 1000
image = image.astype(np.float32)
# preprocess image to work on tf
print("[INFERENCE] Preprocessing image...")
start_preprocessing = time.time()
# resize image to netwrk input dimensions
resized_image = imgutils.resize_image(image, (input_size, input_size))
resized_image = resized_image / 255.
images_data = []
for i in range(1):
images_data.append(resized_image)
images_data = np.asarray(images_data).astype(np.float32)
input_tensor = tf.constant(images_data)
total_preprocessing = (time.time() - start_preprocessing) * 1000
print("[INFERENCE] Preprocessing done!, it took {}ms".format(
total_preprocessing))
print("[INFERENCE] Images data: {} - shape: {} - dtype {}".format(
images_data, images_data.shape, images_data.dtype))
print("[INFERENCE] Input tensor: {} - shape: {} - dtype {}".format(
input_tensor, input_tensor.shape, input_tensor.dtype))
print("[INFERENCE] Now performing inference...")
inference_start_time = time.time()
# get the detections
detections = saved_model_loaded(input_tensor)
total_inference = (time.time() - inference_start_time) * 1000
print("[INFERENCE] Inference took {} ms".format(total_inference))
print("[INFERENCE] Preprocessing network outputs...")
start_output = time.time()
# extract output tensors metadata: boxes, confidence scores
print("[INFERENCE] Extracting output tensors metadata...")
keyval_start = time.time()
for key, value in detections.items():
#print("[INFERENCE-DEBUG] key {}: value {}".format(key, value))
boxes = value[:, :, 0:4]
pred_conf = value[:, :, 4:]
#print("[INFERENCE-DEBUG] boxes ", boxes, type(boxes))
#print("[INFERENCE-DEBUG] confidence ", pred_conf, type(pred_conf))
print("[INFERENCE] Done extracting metadata, it took {}".format(
(time.time() - keyval_start) * 1000))
print("[INFERENCE] Performing NMS to output...")
nms_start = time.time()
# perform non-max supression to retrieve valid detections only
boxes, scores, classes, valid_detections = tf.image.combined_non_max_suppression(
boxes=tf.reshape(boxes, (tf.shape(boxes)[0], -1, 1, 4)),
scores=tf.reshape(
pred_conf,
(tf.shape(pred_conf)[0], -1, tf.shape(pred_conf)[-1])),
max_output_size_per_class=50,
max_total_size=50,
iou_threshold=iou,
score_threshold=threshold)
results = [
boxes.numpy(),
scores.numpy(),
classes.numpy(),
valid_detections.numpy()
]
print("[INFERENCE] NMS done, it took {} ms".format(
(time.time() - nms_start) * 1000))
total_output = (time.time() - start_output) * 1000
print("[INFERENCE] Done procesing output!, it took {}ms".format(
total_output))
# draw results on image
if (bbox_results is not None) and (performance_results is not None):
drawing_time = imgutils.draw_yolo_bounding_boxes(
resized_image, results, labels, bbox_results=bbox_results)
height, width, _ = image.shape
# add new performance results to object
performance_results.add_new_result(width, height,
total_image_loading,
total_preprocessing,
total_inference, total_output,
drawing_time)
else:
_ = imgutils.draw_yolo_bounding_boxes(resized_image,
results,
labels,
save=True)
print("[INFERENCE] Image/frame {} processed".format(frame_count))
frame_count += 1
print("[INFERENCE] All frames procesed!")
output_path = "{}results/{}".format(images_path,
self.attributes["MODEL_NAME"])
output_path += "/opencv-backend"
model_name = self.attributes["MODEL_NAME"]
precision = self.attributes["precision"]
# save results obtained from performance and detections to output
bbox_results.save_results(output_path,
model_name,
precision,
threshold,
resize=True,
opencv=True)
performance_results.save_results(output_path,
model_name,
precision,
threshold,
resize=True,
opencv=True)
def main(unused_argv=None):
tf.logging.set_verbosity(tf.logging.INFO)
if not tf.gfile.Exists(FLAGS.output_dir):
tf.gfile.MkDir(FLAGS.output_dir)
with tf.Graph().as_default(), tf.Session() as sess:
# Defines place holder for the style image.
style_img_ph = tf.placeholder(tf.float32, shape=[None, None, 3])
if FLAGS.style_square_crop:
style_img_preprocessed = image_utils.center_crop_resize_image(
style_img_ph, FLAGS.style_image_size)
else:
style_img_preprocessed = image_utils.resize_image(
style_img_ph, FLAGS.style_image_size)
# Defines place holder for the content image.
content_img_ph = tf.placeholder(tf.float32, shape=[None, None, 3])
if FLAGS.content_square_crop:
content_img_preprocessed = image_utils.center_crop_resize_image(
content_img_ph, FLAGS.image_size)
else:
content_img_preprocessed = image_utils.resize_image(
content_img_ph, FLAGS.image_size)
# Defines the model.
stylized_images, _, _, bottleneck_feat = build_model.build_model(
content_img_preprocessed,
style_img_preprocessed,
trainable=False,
is_training=False,
inception_end_point='Mixed_6e',
style_prediction_bottleneck=100,
adds_losses=False)
if tf.gfile.IsDirectory(FLAGS.checkpoint):
checkpoint = tf.train.latest_checkpoint(FLAGS.checkpoint)
else:
checkpoint = FLAGS.checkpoint
tf.logging.info(
'loading latest checkpoint file: {}'.format(checkpoint))
init_fn = slim.assign_from_checkpoint_fn(
checkpoint, slim.get_variables_to_restore())
sess.run([tf.local_variables_initializer()])
init_fn(sess)
# Gets the list of the input style images.
style_img_list = tf.gfile.Glob(FLAGS.style_images_paths)
if len(style_img_list) > FLAGS.maximum_styles_to_evaluate:
np.random.seed(1234)
style_img_list = np.random.permutation(style_img_list)
style_img_list = style_img_list[:FLAGS.maximum_styles_to_evaluate]
# Gets list of input content images.
content_img_list = tf.gfile.Glob(FLAGS.content_images_paths)
for content_i, content_img_path in enumerate(content_img_list):
content_img_np = image_utils.load_np_image_uint8(
content_img_path)[:, :, :3]
content_img_name = os.path.basename(content_img_path)[:-4]
# Saves preprocessed content image.
inp_img_croped_resized_np = sess.run(
content_img_preprocessed,
feed_dict={content_img_ph: content_img_np})
image_utils.save_np_image(
inp_img_croped_resized_np,
os.path.join(FLAGS.output_dir, '%s.jpg' % (content_img_name)))
# Computes bottleneck features of the style prediction network for the
# identity transform.
identity_params = sess.run(
bottleneck_feat, feed_dict={style_img_ph: content_img_np})
for style_i, style_img_path in enumerate(style_img_list):
if style_i > FLAGS.maximum_styles_to_evaluate:
break
style_img_name = os.path.basename(style_img_path)[:-4]
style_image_np = image_utils.load_np_image_uint8(
style_img_path)[:, :, :3]
if style_i % 10 == 0:
tf.logging.info(
'Stylizing (%d) %s with (%d) %s' %
(content_i, content_img_name, style_i, style_img_name))
# Saves preprocessed style image.
style_img_croped_resized_np = sess.run(
style_img_preprocessed,
feed_dict={style_img_ph: style_image_np})
image_utils.save_np_image(
style_img_croped_resized_np,
os.path.join(FLAGS.output_dir,
'%s.jpg' % (style_img_name)))
# Computes bottleneck features of the style prediction network for the
# given style image.
style_params = sess.run(
bottleneck_feat, feed_dict={style_img_ph: style_image_np})
interpolation_weights = ast.literal_eval(
FLAGS.interpolation_weights)
# Interpolates between the parameters of the identity transform and
# style parameters of the given style image.
for interp_i, wi in enumerate(interpolation_weights):
stylized_image_res = sess.run(
stylized_images,
feed_dict={
bottleneck_feat:
identity_params * (1 - wi) + style_params * wi,
content_img_ph:
content_img_np
})
# Saves stylized image.
image_utils.save_np_image(
stylized_image_res,
os.path.join(
FLAGS.output_dir, '%s_stylized_%s_%d.jpg' %
(content_img_name, style_img_name, interp_i)))
detection_graph = tf_detections.load_frozen_graph(FROZEN_GRAPH)
with detection_graph.as_default():
with tf.Session() as session:
cache_image_size = (0, 0)
tensor_dict_cache = {}
for image_dict in image_dicts:
if image_dict['image_complete']:
continue
image_start = time.time()
image = Image.open(image_dict['image_path'])
if IMAGE_RESIZE:
image = image_utils.resize_image(image, "image", RESIZE_SCALE,
RESIZE_IMAGE_X,
RESIZE_IMAGE_Y,
RESIZE_PADDING, RESIZE_BORDER)
if SAVE_IMAGE_RESIZE:
resized_filename = os.path.basename(
image_dict['image_path'])
image.save(os.path.join(OUTPUT_DIR, resized_filename))
if image.size != cache_image_size:
# Reset tensor dict cache
cache_image_size = image.size
tensor_dict_cache = {}
image_np = image_utils.numpy_from_image(image)
if image_dict['mask_exists']:
mask = Image.open(image_dict['mask_path'])
mask_np = image_utils.numpy_from_image(mask)
built_dict = png_masks.rebuild_from_mask(
mask_np, MASK_DECODE, CODEC_OFFSET, category_index)
def resize_and_save_pic(uri, width, height, elaborated_folder):
image = image_utils.open_image(uri, "RGB")
image = image_utils.resize_image(image, (width,height))
image_utils.save_image(image, basename(uri), elaborated_folder)
return True
def run_inference(self,
image_path=None,
warmup_iters=0,
threshold=0.5,
iou=0.45,
model_dir=None,
bbox_results=None,
performance_results=None,
opencv=False,
resize=False):
input_size = int(self.attributes["INPUT_SIZE"])
labels = self.attributes["LABELS"]
# get a copy of the graph func
#graph_func = self.attributes["graph_func"]
# Load the saved model
model_loaded = tf.saved_model.load(model_dir,
tags=[tag_constants.SERVING])
saved_model_loaded = model_loaded.signatures['serving_default']
#warmup
if warmup_iters > 0:
print("[INFERENCE] Starting warmup on {} iterations...".format(
warmup_iters))
warmup_start = time.time()
# get input size from model attributes
input_size = int(self.attributes["INPUT_SIZE"])
# create a set of random images and perform inference
for i in range(warmup_iters):
print("[INFERENCE] Generating image {} with dims {}x{}".format(
i + 1, input_size, input_size))
# working with numpy/cv backend
if opencv:
# create random image
resized_image = np.random.normal(size=(
input_size, input_size, 3)).astype(np.float32) / 255.
# conventional conversion (use with opencv option)
# The input needs to be a tensor, convert it using `tf.convert_to_tensor`.
#input_tensor = tf.convert_to_tensor(resized_image)
#input_tensor = tf.convert_to_tensor(image)
# The model expects a batch of images, so add an axis with `tf.newaxis`.
#input_tensor = input_tensor[tf.newaxis, ...]
images_data = []
for i in range(1):
images_data.append(resized_image)
images_data = np.asarray(images_data).astype(np.float32)
input_tensor = tf.constant(images_data)
# working with tf backend for image
else:
dataset, _, _ = imgutils.get_dataset_tf(
tensor_type='float', input_size=input_size)
"""print("[WARMUP] Creating features...")
features = np.random.normal(loc=112, scale=70,
size=(1, input_size, input_size, 3)).astype(np.float32)
features = np.clip(features, 0.0, 255.0).astype(np.float32)
features = tf.convert_to_tensor(value=tf.compat.v1.get_variable(
"features", initializer=tf.constant(features)))
print("[WARMUP] Creating dataset from features...")
dataset = tf.data.Dataset.from_tensor_slices([features])
dataset = dataset.repeat(count=1)"""
print("[WARMUP] Retrieving image and input tensor...")
dataset_enum = enumerate(dataset)
# get input tensor and cast to image (np)
input_tensor = list(dataset_enum)[0][1]
resized_image = input_tensor.numpy()[0]
images_data = []
for i in range(1):
images_data.append(resized_image)
images_data = np.asarray(images_data).astype(np.float32)
input_tensor = tf.constant(images_data)
# get the detections
print("[WARMUP] Now performing warmup inference...")
inference_start_time = time.time()
detections = saved_model_loaded(input_tensor)
print("[WARMUP] Warmup inference took {} ms".format(
(time.time() - inference_start_time) * 1000))
print("[INFERENCE] Preprocessing network outputs...")
start_output = time.time()
# extract output tensors metadata: boxes, confidence scores
print("[INFERENCE] Extracting output tensors metadata...")
keyval_start = time.time()
for key, value in detections.items():
#print("[INFERENCE-DEBUG] key {}: value {}".format(key, value))
boxes = value[:, :, 0:4]
pred_conf = value[:, :, 4:]
#print("[INFERENCE-DEBUG] boxes ", boxes, type(boxes))
#print("[INFERENCE-DEBUG] confidence ", pred_conf, type(pred_conf))
print(
"[INFERENCE] Done extracting metadata, it took {}".format(
(time.time() - keyval_start) * 1000))
print("[INFERENCE] Performing NMS to output...")
nms_start = time.time()
# perform non-max supression to retrieve valid detections only
boxes, scores, classes, valid_detections = tf.image.combined_non_max_suppression(
boxes=tf.reshape(boxes, (tf.shape(boxes)[0], -1, 1, 4)),
scores=tf.reshape(
pred_conf,
(tf.shape(pred_conf)[0], -1, tf.shape(pred_conf)[-1])),
max_output_size_per_class=50,
max_total_size=50,
iou_threshold=iou,
score_threshold=threshold)
results = [
boxes.numpy(),
scores.numpy(),
classes.numpy(),
valid_detections.numpy()
]
print("[INFERENCE] NMS done, it took {} ms".format(
(time.time() - nms_start) * 1000))
total_output = (time.time() - start_output) * 1000
print(
"[INFERENCE] Done procesing output!, it took {}ms".format(
total_output))
_ = imgutils.draw_yolo_bounding_boxes(resized_image, results,
labels)
# display results in ms
print("[WARMUP] Warmup finished, it took {} ms".format(
(time.time() - warmup_start) * 1000))
# case inference
if opencv:
print("[INFERENCE] Loading image with opencv backend...")
# opencv option
image_loading_start = time.time()
image = imgutils.read_image_from_cv2(image_path)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
total_image_loading = (time.time() - image_loading_start) * 1000
# preprocess image to work on tf
print("[INFERENCE] Preprocessing image...")
start_preprocessing = time.time()
# resize image to netwrk input dimensions
resized_image = imgutils.resize_image(image,
(input_size, input_size))
resized_image = resized_image / 255.
# conventional conversion (use with opencv option)
# The input needs to be a tensor, convert it using `tf.convert_to_tensor`.
#input_tensor = tf.convert_to_tensor(resized_image)
#input_tensor = tf.convert_to_tensor(image)
# The model expects a batch of images, so add an axis with `tf.newaxis`.
#input_tensor = input_tensor[tf.newaxis, ...]
images_data = []
for i in range(1):
images_data.append(resized_image)
images_data = np.asarray(images_data).astype(np.float32)
input_tensor = tf.constant(images_data)
total_preprocessing = (time.time() - start_preprocessing) * 1000
print("[INFERENCE] Preprocessing done!, it took {}ms".format(
total_preprocessing))
# case tf backend to manipulate images
else:
print("[INFERENCE] Loading image with tf backend...")
# dataset option, yolo models require resizing to input size
dataset, total_image_loading, total_preprocessing = imgutils.get_dataset_tf(
image_path=image_path,
input_size=input_size,
tensor_type='float')
#print("[INFERENCE] dataset {}".format(dataset))
# take the batched image
dataset_enum = enumerate(dataset)
input_tensor = list(dataset_enum)[0][1]
# take image as np and convert to rgb
image_bgr = input_tensor.numpy()[0]
resized_image = image_bgr[..., ::-1].copy() / 255.
images_data = []
for i in range(1):
images_data.append(resized_image)
images_data = np.asarray(images_data).astype(np.float32)
input_tensor = tf.constant(images_data)
print(
"[INFERENCE] Images data: {} - shape: {} - dtype {} - type {}".
format(images_data, images_data.shape, images_data.dtype,
type(images_data)))
print(
"[INFERENCE] Input tensor: {} - shape: {} - dtype {} - type {}"
.format(input_tensor, input_tensor.shape, input_tensor.dtype,
type(input_tensor)))
print("[INFERENCE] Now performing inference...")
inference_start_time = time.time()
#print("[INFERENCE] Input tensor: {} - dtype {}"
# .format(input_tensor, input_tensor.dtype))
# get the detections
detections = saved_model_loaded(input_tensor)
total_inference = (time.time() - inference_start_time) * 1000
print("[INFERENCE] Inference took {} ms".format(total_inference))
print("[INFERENCE] Preprocessing network outputs...")
start_output = time.time()
# extract output tensors metadata: boxes, confidence scores
print("[INFERENCE] Extracting output tensors metadata...")
keyval_start = time.time()
for key, value in detections.items():
#print("[INFERENCE-DEBUG] key {}: value {}".format(key, value))
boxes = value[:, :, 0:4]
pred_conf = value[:, :, 4:]
#print("[INFERENCE-DEBUG] boxes ", boxes, type(boxes))
#print("[INFERENCE-DEBUG] confidence ", pred_conf, type(pred_conf))
print("[INFERENCE] Done extracting metadata, it took {}".format(
(time.time() - keyval_start) * 1000))
print("[INFERENCE] Performing NMS to output...")
nms_start = time.time()
# perform non-max supression to retrieve valid detections only
boxes, scores, classes, valid_detections = tf.image.combined_non_max_suppression(
boxes=tf.reshape(boxes, (tf.shape(boxes)[0], -1, 1, 4)),
scores=tf.reshape(
pred_conf,
(tf.shape(pred_conf)[0], -1, tf.shape(pred_conf)[-1])),
max_output_size_per_class=50,
max_total_size=50,
iou_threshold=iou,
score_threshold=threshold)
results = [
boxes.numpy(),
scores.numpy(),
classes.numpy(),
valid_detections.numpy()
]
print("[INFERENCE] NMS done, it took {} ms".format(
(time.time() - nms_start) * 1000))
total_output = (time.time() - start_output) * 1000
print("[INFERENCE] Done procesing output!, it took {}ms".format(
total_output))
# draw results on image
if (bbox_results is not None) and (performance_results is not None):
drawing_time = imgutils.draw_yolo_bounding_boxes(
resized_image, results, labels, bbox_results=bbox_results)
height, width, _ = image.shape
# add new performance results to object
performance_results.add_new_result(width, height,
total_image_loading,
total_preprocessing,
total_inference, total_output,
drawing_time)
else:
_ = imgutils.draw_yolo_bounding_boxes(resized_image,
results,
labels,
save=True)
# debugging info
"""print("[INFERENCE-DEBUG] tf-boxes ", boxes, type(boxes))
import pdb import timeit from sort import sort_pixels import interval import image_utils as util # Contains image source, rotation information, threshold settings, sorting settings, etc from parameters import * # Open image im = Image.open(source) # Rotate the image if needed im = im.rotate(base_rotation, expand=True) # Resize the image (useful for testing, keep at 1 for final images) im = util.resize_image(im, resize_factor) original_size = im.size im = im.rotate(image_rotation, expand=True) a = np.asarray(im) # get intervals intervals = (interval.threshold_interval(a, np.shape(a), 0.7, 0.91, width=width, func=util.lightness, inverted=True)) intervals = interval.randomly_filter_interval(intervals, 0) ac = a.copy() # Sort the pixels new = sort.sort_pixels(ac, intervals, width=width, sorting_func=util.lightness, reverse=False) im2 = Image.fromarray(new) # Unrotate the image
def run_inference(self,
image_path=None,
warmup_iters=0,
threshold=0.5,
iou=0.45,
model_dir=None):
input_size = int(self.attributes["INPUT_SIZE"])
labels = self.attributes["LABELS"]
# get a copy of the graph func
#graph_func = self.attributes["graph_func"]
if model_dir is not None:
saved_model_loaded = self.get_func_from_saved_model(model_dir)
else:
saved_model_loaded = self.attributes["saved_model_loaded"]
#warmup
if warmup_iters > 0:
print("[INFERENCE] Starting warmup on {} iterations...".format(
warmup_iters))
warmup_start = time.time()
# get input size from model attributes
input_size = int(self.attributes["INPUT_SIZE"])
# create a set of random images and perform inference
for i in range(warmup_iters):
print("[INFERENCE] Generating image {} with dims {}x{}".format(
i + 1, input_size, input_size))
# working with numpy/cv backend
# create random image
resized_image = np.random.normal(
size=(input_size, input_size, 3)).astype(np.float32) / 255.
images_data = np.asarray([resized_image]).astype(np.float32)
input_tensor = tf.constant(images_data)
images_data = []
for i in range(1):
images_data.append(resized_image)
images_data = np.asarray(images_data).astype(np.float32)
input_tensor = tf.constant(images_data)
# get the detections
print("[WARMUP] Now performing warmup inference...")
inference_start_time = time.time()
detections = saved_model_loaded(input_tensor)
print("[WARMUP] Warmup inference took {} ms".format(
(time.time() - inference_start_time) * 1000))
print("[INFERENCE] Preprocessing network outputs...")
start_output = time.time()
# extract output tensors metadata: boxes, confidence scores
print("[INFERENCE] Extracting output tensors metadata...")
keyval_start = time.time()
for key, value in detections.items():
#print("[INFERENCE-DEBUG] key {}: value {}".format(key, value))
boxes = value[:, :, 0:4]
pred_conf = value[:, :, 4:]
#print("[INFERENCE-DEBUG] boxes ", boxes, type(boxes))
#print("[INFERENCE-DEBUG] confidence ", pred_conf, type(pred_conf))
print(
"[INFERENCE] Done extracting metadata, it took {}".format(
(time.time() - keyval_start) * 1000))
print("[INFERENCE] Performing NMS to output...")
nms_start = time.time()
# perform non-max supression to retrieve valid detections only
boxes, scores, classes, valid_detections = tf.image.combined_non_max_suppression(
boxes=tf.reshape(boxes, (tf.shape(boxes)[0], -1, 1, 4)),
scores=tf.reshape(
pred_conf,
(tf.shape(pred_conf)[0], -1, tf.shape(pred_conf)[-1])),
max_output_size_per_class=50,
max_total_size=50,
iou_threshold=iou,
score_threshold=threshold)
results = [
boxes.numpy(),
scores.numpy(),
classes.numpy(),
valid_detections.numpy()
]
print("[INFERENCE] NMS done, it took {} ms".format(
(time.time() - nms_start) * 1000))
total_output = (time.time() - start_output) * 1000
print(
"[INFERENCE] Done procesing output!, it took {}ms".format(
total_output))
# draw dummy results
_ = imgutils.draw_yolo_bounding_boxes(resized_image, results,
labels)
# display results in ms
print("[WARMUP] Warmup finished, it took {} ms".format(
(time.time() - warmup_start) * 1000))
# case inference
print("[INFERENCE] Loading image with opencv backend...")
# opencv option
image_loading_start = time.time()
image = imgutils.read_image_from_cv2(image_path)
total_image_loading = (time.time() - image_loading_start) * 1000
image = image.astype(np.float32)
# preprocess image to work on tf
print("[INFERENCE] Preprocessing image...")
start_preprocessing = time.time()
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
# resize image to netwrk input dimensions
resized_image = imgutils.resize_image(image, (input_size, input_size))
resized_image = resized_image / 255.
images_data = []
for i in range(1):
images_data.append(resized_image)
images_data = np.asarray(images_data).astype(np.float32)
input_tensor = tf.constant(images_data)
total_preprocessing = (time.time() - start_preprocessing) * 1000
print("[INFERENCE] Preprocessing done!, it took {}ms".format(
total_preprocessing))
print("[INFERENCE] Images data: {} - shape: {} - dtype {}".format(
images_data, images_data.shape, images_data.dtype))
print("[INFERENCE] Input tensor: {} - shape: {} - dtype {}".format(
input_tensor, input_tensor.shape, input_tensor.dtype))
print("[INFERENCE] Now performing inference...")
inference_start_time = time.time()
# get the detections
detections = saved_model_loaded(input_tensor)
total_inference = (time.time() - inference_start_time) * 1000
print("[INFERENCE] Inference took {} ms".format(total_inference))
print("[INFERENCE] Preprocessing network outputs...")
start_output = time.time()
# extract output tensors metadata: boxes, confidence scores
print("[INFERENCE] Extracting output tensors metadata...")
keyval_start = time.time()
for key, value in detections.items():
#print("[INFERENCE-DEBUG] key {}: value {}".format(key, value))
boxes = value[:, :, 0:4]
pred_conf = value[:, :, 4:]
#print("[INFERENCE-DEBUG] boxes ", boxes, type(boxes))
#print("[INFERENCE-DEBUG] confidence ", pred_conf, type(pred_conf))
print("[INFERENCE] Done extracting metadata, it took {}".format(
(time.time() - keyval_start) * 1000))
print("[INFERENCE] Performing NMS to output...")
nms_start = time.time()
# perform non-max supression to retrieve valid detections only
boxes, scores, classes, valid_detections = tf.image.combined_non_max_suppression(
boxes=tf.reshape(boxes, (tf.shape(boxes)[0], -1, 1, 4)),
scores=tf.reshape(
pred_conf,
(tf.shape(pred_conf)[0], -1, tf.shape(pred_conf)[-1])),
max_output_size_per_class=50,
max_total_size=50,
iou_threshold=iou,
score_threshold=threshold)
results = [
boxes.numpy(),
scores.numpy(),
classes.numpy(),
valid_detections.numpy()
]
print("[INFERENCE] NMS done, it took {} ms".format(
(time.time() - nms_start) * 1000))
total_output = (time.time() - start_output) * 1000
print("[INFERENCE] Done procesing output!, it took {}ms".format(
total_output))
# draw results and save
_ = imgutils.draw_yolo_bounding_boxes(resized_image,
results,
labels,
save=True)
# debugging info
"""print("[INFERENCE-DEBUG] tf-boxes ", boxes, type(boxes))
urls.append(base_url + image_name + ".png")
urls.append(base_url_alt + image_name_alt + ".png")
urls.append(base_url + image_name_alt2 + ".png")
urls.append(base_url + image_name + '2.png')
urls.append(alt_base_url + image_name + ".png")
downloaded = False
for url in urls:
response = requests.get(url)
if response.ok and not downloaded:
downloaded = True
# print('Downloading {}'.format(url))
download_directory = 'images/monsters'
output_directory = 'images/monsters/rescaled'
if not os.path.exists(output_directory):
os.makedirs(output_directory)
output_file_name = '{}.png'.format(
str(monster_name.lower().replace(' ', '-')))
download_file(url, 'images/monsters', output_file_name)
resize_image(
'{}/{}'.format(download_directory, output_file_name),
'{}/{}'.format(output_directory, output_file_name),
output_bb_size, output_image_size)
break
# else:
# print('Url failed {}'.format(url))
if not downloaded:
print("{} no download".format(monster_name))
def tile_no_neighbors(tile_filename: str,
output_filename: str,
output_width: int,
output_height: int,
aspect_ratio: float,
crop: bool = True):
"""
Also deprecated-ish. Similar to tile_old, but at least this one makes sure that neighboring tiles aren't the same image.
This has cropping of candidate images, which is nice. This does not leverage any multiprocessing.
It is therefore slower, but should use less RAM and is less likely to result in filling up memory if you're creating a massive image.
Once the concurrent version is fixed, then they'd both be unlikely to do that.
:param tile_filename:
:param output_filename:
:param output_width:
:param output_height:
:param aspect_ratio:
:param choice:
:param crop:
:return:
"""
df = pd.read_csv(tile_filename)
column_count = df['col'].max() + 1
row_count = df['row'].max() + 1
tile_width = int(output_width / column_count)
tile_height = int(output_height / row_count)
result = Image.new('RGB',
(column_count * tile_width, row_count * tile_height))
pbar = tqdm.tqdm(total=row_count * column_count)
selected_filenames = []
for r in range(row_count):
row_filenames = []
for c in range(column_count):
# Find neighbors
neighbors = []
if r > 0:
if c > 0:
neighbors.append(selected_filenames[r - 1][c - 1])
neighbors.append(selected_filenames[r - 1][c])
if c < column_count - 1:
neighbors.append(selected_filenames[r - 1][c + 1])
if c > 0:
neighbors.append(row_filenames[c - 1])
choice_rows = df[(df['row'] == r) & (df['col'] == c) & (
~df['filename'].isin(neighbors))].sort_values('choice')
filename = choice_rows['filename'].iloc[0]
if crop:
resized_child = resize_image(crop_image_to_aspect_ratio(
Image.open(filename), aspect_ratio),
width=tile_width)
else:
resized_child = resize_image(Image.open(filename),
width=tile_width,
height=tile_height)
result.paste(im=resized_child,
box=(c * tile_width, r * tile_height))
row_filenames.append(filename)
pbar.update(1)
selected_filenames.append(row_filenames)
save_final_choices(selected_filenames)
result.save(output_filename)
pygame.init()
WIDTH = 1000
HEIGHT = 1000
FRAME_RATE = 2
# images = ['hard_left_small.png', 'left_small.png', 'small.png', 'right_small.png', 'hard_right_small.png']
images = [
'hardest_left.png', 'hardest_left_fin.png', 'harder_left.png',
'harder_left_fin.png', 'left.png', 'left_fin.png', 'center_left.png',
'center.png', 'center_right.png', 'right.png', 'right_fin.png',
'hard_right.png', 'hard_right_fin.png', 'hardest_right.png',
'hardest_right_fin.png'
]
images = ['movement_images/v2/' + im for im in images]
images = [resize_image(im, (250, 250)) for im in images]
images = [pygame.image.load(im) for im in images]
my_thing = Thing(images, (250 + 250 / 2, 250 + 250 / 2))
bg_color = (0, 0, 0)
size = (WIDTH, HEIGHT)
display = pygame.display.set_mode(size)
pygame.display.set_caption('Thing class test')
clock = pygame.time.Clock()
go = True
while go:
display.fill(bg_color)
mouse_cords = pygame.mouse.get_pos()
my_thing.point_at_cords(mouse_cords)
def augmentation_function(images, labels, **kwargs):
'''
Function for augmentation of minibatches. It will transform a set of images and corresponding labels
by a number of optional transformations. Each image/mask pair in the minibatch will be seperately transformed
with random parameters.
:param images: A numpy array of shape [minibatch, X, Y, (Z), nchannels]
:param labels: A numpy array containing a corresponding label mask
:param do_rotations: Rotate the input images by a random angle between -15 and 15 degrees.
:param do_scaleaug: Do scale augmentation by sampling one length of a square, then cropping and upsampling the image
back to the original size.
:param do_fliplr: Perform random flips with a 50% chance in the left right direction.
:return: A mini batch of the same size but with transformed images and masks.
'''
if images.ndim > 4:
raise AssertionError('Augmentation will only work with 2D images')
do_rotations = kwargs.get('do_rotations', False)
do_scaleaug = kwargs.get('do_scaleaug', False)
do_fliplr = kwargs.get('do_fliplr', False)
new_images = []
new_labels = []
num_images = images.shape[0]
for ii in range(num_images):
img = np.squeeze(images[ii, ...])
lbl = np.squeeze(labels[ii, ...])
# ROTATE
if do_rotations:
angles = kwargs.get('angles', (-15, 15))
random_angle = np.random.uniform(angles[0], angles[1])
img = image_utils.rotate_image(img, random_angle)
lbl = image_utils.rotate_image(lbl,
random_angle,
interp=cv2.INTER_NEAREST)
# RANDOM CROP SCALE
if do_scaleaug:
offset = kwargs.get('offset', 30)
n_x, n_y = img.shape
r_y = np.random.random_integers(n_y - offset, n_y)
p_x = np.random.random_integers(0, n_x - r_y)
p_y = np.random.random_integers(0, n_y - r_y)
img = image_utils.resize_image(
img[p_y:(p_y + r_y), p_x:(p_x + r_y)], (n_x, n_y))
lbl = image_utils.resize_image(lbl[p_y:(p_y + r_y),
p_x:(p_x + r_y)], (n_x, n_y),
interp=cv2.INTER_NEAREST)
# RANDOM FLIP
if do_fliplr:
coin_flip = np.random.randint(2)
if coin_flip == 0:
img = np.fliplr(img)
lbl = np.fliplr(lbl)
new_images.append(img[..., np.newaxis])
new_labels.append(lbl[...])
sampled_image_batch = np.asarray(new_images)
sampled_label_batch = np.asarray(new_labels)
return sampled_image_batch, sampled_label_batch
def verify_image_size_and_resize(image, tile_size):
if image.size > (tile_size, tile_size):
return image_utils.resize_image(image,(tile_size, tile_size))
return image
