def ck_custom_preprocess(image_files, iter_num,processed_image_ids,params): if params["RESIZE_WIDTH_SIZE"] != params["RESIZE_HEIGHT_SIZE"]: raise ValueError ("this model works with square images, and you provided different height and width") image_file = image_files[iter_num] image_id = ck_utils.filename_to_id(image_file, params["DATASET_TYPE"]) processed_image_ids.append(image_id) image_path = os.path.join(params["IMAGES_DIR"], image_file) orig_image = cv2.imread(image_path) orig_image = cv2.cvtColor(orig_image, cv2.COLOR_BGR2RGB) original_image_size = orig_image.shape[:2] image = cv2.cvtColor(orig_image, cv2.COLOR_BGR2RGB).astype(np.float32) ih, iw = params["RESIZE_HEIGHT_SIZE"],params["RESIZE_WIDTH_SIZE"] ##They are equal. h, w, _ = image.shape scale = min(iw/w, ih/h) nw, nh = int(scale * w), int(scale * h) image_resized = cv2.resize(image, (nw, nh)) image_paded = np.full(shape=[ih, iw, 3], fill_value=128.0) dw, dh = (iw - nw) // 2, (ih-nh) // 2 image_paded[dh:nh+dh, dw:nw+dw, :] = image_resized image_paded = image_paded / 255. image_paded = image_paded[np.newaxis, ...] return image_paded,processed_image_ids,original_image_size,orig_image
def ck_custom_preprocess_batch(image_files, iter_num, processed_image_ids, params): batch_data = [] batch_sizes = [] orig_images = [] for img in range(params["BATCH_SIZE"]): image_file = image_files[iter_num * params["BATCH_SIZE"] + img] image_id = ck_utils.filename_to_id(image_file, params["DATASET_TYPE"]) processed_image_ids.append(image_id) image_path = os.path.join(params["IMAGES_DIR"], image_file) orig_image = cv2.imread(image_path) orig_image = cv2.cvtColor(orig_image, cv2.COLOR_BGR2RGB) orig_images.append(orig_image) batch_sizes.append(orig_image.shape[:2]) image = cv2.cvtColor(orig_image, cv2.COLOR_BGR2RGB).astype(np.float32) ih, iw = 416, 416 ##They are equal. h, w, _ = image.shape scale = min(iw / w, ih / h) nw, nh = int(scale * w), int(scale * h) image_resized = cv2.resize(image, (nw, nh)) image_paded = np.full(shape=[ih, iw, 3], fill_value=128.0) dw, dh = (iw - nw) // 2, (ih - nh) // 2 image_paded[dh:nh + dh, dw:nw + dw, :] = image_resized image_paded = image_paded / 255. print(np.shape(image_paded)) #image_paded = image_paded[np.newaxis, ...] #print(np.shape(image_paded)) batch_data.append(image_paded) return batch_data, processed_image_ids, batch_sizes, orig_images
def load_images_batch(image_list, iter_num, processed_image_ids, params): batch_data = [] batch_sizes = [] for img in range(params["BATCH_SIZE"]): image = PIL.Image.open( os.path.join(params["IMAGES_DIR"], image_list[iter_num * params["BATCH_SIZE"] + img])) batch_sizes.append(image.size) print("resizing image to: ", params["RESIZE_WIDTH_SIZE"], params["RESIZE_HEIGHT_SIZE"]) image = image.resize( (params["RESIZE_WIDTH_SIZE"], params["RESIZE_HEIGHT_SIZE"]), PIL.Image.BILINEAR) image_id = ck_utils.filename_to_id( image_list[iter_num * params["BATCH_SIZE"] + img], params["DATASET_TYPE"]) processed_image_ids.append(image_id) # Check if not RGB and convert to RGB if image.mode != 'RGB': image = image.convert('RGB') # Convert to NumPy array img_data = np.array(image.getdata()) img_data = img_data.astype(np.uint8) img_data = img_data.reshape( (params["RESIZE_HEIGHT_SIZE"], params["RESIZE_WIDTH_SIZE"], 3)) batch_data.append(img_data) return batch_data, processed_image_ids, batch_sizes, image_id #last value is not needed actually.
def load_image(image_files, iter_num, processed_image_ids, params): image_file = image_files[iter_num] image_id = ck_utils.filename_to_id(image_file, params["DATASET_TYPE"]) processed_image_ids.append(image_id) image_path = os.path.join(params["IMAGES_DIR"], image_file) img = cv2.imread(image_path) orig_image = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # Load to numpy array, separately from original image image = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.uint8) # Make batch from single image im_height, im_width, _ = image.shape batch_shape = (1, im_height, im_width, 3) batch_data = image.reshape(batch_shape) return batch_data, processed_image_ids, (im_width, im_height), orig_image
def load_images_batch(image_list,iter_num,processed_image_ids,params): batch_data = [] batch_sizes = [] for img in range(params["BATCH_SIZE"]): img_rd = cv2.imread(os.path.join(params["IMAGES_DIR"], image_list[iter_num*params["BATCH_SIZE"]+img])) image = cv2.cvtColor(img_rd, cv2.COLOR_BGR2RGB).astype(np.uint8) im_height, im_width, _ = image.shape batch_sizes.append((im_width, im_height)) image = cv2.resize(image,(params["RESIZE_WIDTH_SIZE"],params["RESIZE_HEIGHT_SIZE"])) image_id = ck_utils.filename_to_id(image_list[iter_num*params["BATCH_SIZE"]+img], params["DATASET_TYPE"]) processed_image_ids.append(image_id) img_data = image.reshape((params["RESIZE_HEIGHT_SIZE"],params["RESIZE_WIDTH_SIZE"],3)) batch_data.append(img_data) return batch_data,processed_image_ids,batch_sizes,image_id #last value is not needed actually.
def convert_to_coco(detection_files, detections_dir, target_dir, dataset_type, model_dataset_type): res_array = [] for file_name in detection_files: read_file = os.path.join(detections_dir, file_name) file_id = helper.filename_to_id(file_name, dataset_type) with open(read_file, 'r') as rf: rf.readline() # first line is image size for line in rf: det = helper.Detection(line) res = detection_to_coco_object(det, model_dataset_type, file_id) if (res): res_array.append(res) results_file = os.path.join(target_dir, 'coco_results.json') with open(results_file, 'w') as f: f.write(json.dumps(res_array, indent=2, sort_keys=False)) return results_file
def load_image(image_files, iter_num, processed_image_ids, params): image_file = image_files[iter_num] image_id = ck_utils.filename_to_id(image_file, params["DATASET_TYPE"]) processed_image_ids.append(image_id) image_path = os.path.join(params["IMAGES_DIR"], image_file) image = PIL.Image.open(image_path) # Check if not RGB and convert to RGB if image.mode != 'RGB': image = image.convert('RGB') # Convert to NumPy array img_data = np.array(image.getdata()) img_data = img_data.astype(np.uint8) # Make batch from single image (im_width, im_height) = image.size batch_shape = (1, im_height, im_width, 3) batch_data = img_data.reshape(batch_shape) return batch_data, processed_image_ids, image.size, image
def load_pil_images_into_numpy_batch_array(image_list, batch_id, images_ids): batch_data = [] batch_sizes = [] for img in range(BATCH_SIZE): image = PIL.Image.open( os.path.join(IMAGES_DIR, image_list[batch_id * BATCH_SIZE + img])) batch_sizes.append(image.size) image = image.resize((RESIZE_WIDTH_SIZE, RESIZE_HEIGHT_SIZE), PIL.Image.BILINEAR) image_id = ck_utils.filename_to_id( image_list[batch_id * BATCH_SIZE + img], DATASET_TYPE) images_ids.append(image_id) # Check if not RGB and convert to RGB if image.mode != 'RGB': image = image.convert('RGB') # Convert to NumPy array img_data = np.array(image.getdata()) img_data = img_data.astype(np.uint8) img_data = img_data.reshape((RESIZE_HEIGHT_SIZE, RESIZE_WIDTH_SIZE, 3)) batch_data.append(img_data) return batch_data, images_ids, batch_sizes
def ck_custom_preprocess(image_files, iter_num, processed_image_ids, params): image_file = image_files[iter_num] image_id = ck_utils.filename_to_id(image_file, params["DATASET_TYPE"]) processed_image_ids.append(image_id) image_path = os.path.join(params["IMAGES_DIR"], image_file) orig_image = cv2.imread(image_path) orig_image = cv2.cvtColor(orig_image, cv2.COLOR_BGR2RGB) original_image_size = orig_image.shape[:2] image = cv2.cvtColor(orig_image, cv2.COLOR_BGR2RGB).astype(np.float32) ih, iw = 416, 416 ##They are equal, and the model requires that size so must be hardcoded. h, w, _ = image.shape scale = min(iw / w, ih / h) nw, nh = int(scale * w), int(scale * h) image_resized = cv2.resize(image, (nw, nh)) image_paded = np.full(shape=[ih, iw, 3], fill_value=128.0) dw, dh = (iw - nw) // 2, (ih - nh) // 2 image_paded[dh:nh + dh, dw:nw + dw, :] = image_resized image_paded = image_paded / 255. image_paded = image_paded[np.newaxis, ...] return image_paded, processed_image_ids, original_image_size, orig_image
def detect(category_index): # Prepare TF config options tf_config = make_tf_config() # Prepare directories ck_utils.prepare_dir(RESULTS_OUT_DIR) ck_utils.prepare_dir(ANNOTATIONS_OUT_DIR) ck_utils.prepare_dir(IMAGES_OUT_DIR) ck_utils.prepare_dir(DETECTIONS_OUT_DIR) # Load processing image filenames image_files = ck_utils.load_image_list(IMAGES_DIR, IMAGE_COUNT, SKIP_IMAGES) with tf.Graph().as_default(), tf.Session(config=tf_config) as sess: setup_time_begin = time.time() # Make TF graph def from frozen graph file begin_time = time.time() graph_def = tf.GraphDef() with tf.gfile.GFile(FROZEN_GRAPH, 'rb') as f: graph_def.ParseFromString(f.read()) tf.import_graph_def(graph_def, name='') graph_load_time = time.time() - begin_time print('Graph loaded in {:.4f}s'.format(graph_load_time)) # NOTE: Load checkpoint here when they are needed # Get handles to input and output tensors tensor_dict, input_tensor = get_handles_to_tensors() setup_time = time.time() - setup_time_begin # Process images # TODO: implement batched mode test_time_begin = time.time() image_index = 0 load_time_total = 0 detect_time_total = 0 images_processed = 0 processed_image_ids = [] for file_counter, image_file in enumerate(image_files): if FULL_REPORT or (file_counter + 1) % 10 == 0: print('\nDetect image: {} ({} of {})'.format( image_file, file_counter + 1, len(image_files))) # Load image load_time_begin = time.time() image = PIL.Image.open(os.path.join(IMAGES_DIR, image_file)) image_id = ck_utils.filename_to_id(image_file, DATASET_TYPE) processed_image_ids.append(image_id) # The array based representation of the image will be used later # in order to prepare the result image with boxes and labels on it. image_data = load_pil_image_into_numpy_array(image) load_time = time.time() - load_time_begin load_time_total += load_time # Detect image detect_time_begin = time.time() feed_dict = {input_tensor: image_data} output_dict = sess.run(tensor_dict, feed_dict) detect_time = time.time() - detect_time_begin # Exclude first image from averaging if file_counter > 0 or IMAGE_COUNT == 1: detect_time_total += detect_time images_processed += 1 # Process results # All outputs are float32 numpy arrays, so convert types as appropriate # TODO: implement batched mode (0 here is the image index in the batch) output_dict['num_detections'] = int( output_dict['num_detections'][0]) output_dict['detection_classes'] = output_dict[ 'detection_classes'][0].astype(np.uint8) output_dict['detection_boxes'] = output_dict['detection_boxes'][0] output_dict['detection_scores'] = output_dict['detection_scores'][ 0] save_detection_txt(image_file, image.size, output_dict, category_index) save_detection_img(image_file, image_data[0], output_dict, category_index) if FULL_REPORT: print('Detected in {:.4f}s'.format(detect_time)) # Save processed images ids list to be able to run # evaluation without repeating detections (CK_SKIP_DETECTION=YES) with open(IMAGE_LIST_FILE, 'w') as f: f.write(json.dumps(processed_image_ids)) test_time = time.time() - test_time_begin detect_avg_time = detect_time_total / images_processed load_avg_time = load_time_total / len(processed_image_ids) OPENME['setup_time_s'] = setup_time OPENME['test_time_s'] = test_time OPENME['graph_load_time_s'] = graph_load_time OPENME['images_load_time_s'] = load_time_total OPENME['images_load_time_avg_s'] = load_avg_time OPENME['detection_time_total_s'] = detect_time_total OPENME['detection_time_avg_s'] = detect_avg_time OPENME['avg_time_ms'] = detect_avg_time * 1000 OPENME['avg_fps'] = 1.0 / detect_avg_time if detect_avg_time > 0 else 0 with open(TIMER_JSON, 'w') as o: json.dump(OPENME, o, indent=2, sort_keys=True) return processed_image_ids
def ck_postprocess(i): def my_env(var): return i['env'].get(var) def dep_env(dep, var): return i['deps'][dep]['dict']['env'].get(var) def set_in_my_env(var): return my_env(var) and my_env(var).lower() in [ 'yes', 'true', 'on', '1' ] def set_in_dep_env(dep, var): return dep_env(dep, var) and dep_env( dep, var).lower() in ['yes', 'true', 'on', '1'] def has_dep_env(dep, var): return var in i['deps'][dep]['dict']['env'] def has_dep(dep): return dep in i['deps'] import os import json import sys import inspect # gain access to other scripts in the same directory as the postprocess.py : # SCRIPT_DIR = os.path.dirname( os.path.abspath(inspect.getfile(inspect.currentframe()))) sys.path.append(SCRIPT_DIR) # gain access to the some python dependencies : PYTHONPATH = '' for dep_name in ['tool-coco', 'lib-python-matplotlib']: PYTHONPATH = dep_env(dep_name, 'PYTHONPATH') + ':' + PYTHONPATH split_path = set() for p in PYTHONPATH.split(":"): if p in ["${PYTHONPATH}", "$PYTHONPATH", ""]: continue split_path.add(p) sys.path.extend(list( split_path)) # allow THIS SCRIPT to be able to use numpy, pillow, etc. # get some parameters directly from the deps' environment: # MODEL_ROOT = dep_env('weights', "CK_ENV_TENSORFLOW_MODEL_ROOT") if MODEL_ROOT: LABELMAP_FILE = os.path.join( MODEL_ROOT, dep_env('weights', 'CK_ENV_TENSORFLOW_MODEL_LABELMAP_FILE') or "") MODEL_DATASET_TYPE = dep_env('weights', "CK_ENV_TENSORFLOW_MODEL_DATASET_TYPE") else: MODEL_ROOT = dep_env('weights', "CK_ENV_ONNX_MODEL_ROOT") LABELMAP_FILE = os.path.join( MODEL_ROOT, dep_env('weights', 'CK_ENV_ONNX_MODEL_CLASSES_LABELS') or "") MODEL_DATASET_TYPE = dep_env('weights', "CK_ENV_ONNX_MODEL_DATASET_TYPE") # Annotations can be a directory or a single file, depending on dataset type: ANNOTATIONS_PATH = dep_env('dataset', "CK_ENV_DATASET_ANNOTATIONS") TIMER_JSON = my_env('CK_TIMER_FILE') PREPROCESSED_FILES = dep_env( 'dataset', 'CK_ENV_DATASET_OBJ_DETECTION_PREPROCESSED_SUBSET_FOF' ) or my_env('CK_PREPROCESSED_FOF_WITH_ORIGINAL_DIMENSIONS') DETECTIONS_OUT_DIR = my_env('CK_DETECTIONS_OUT_DIR') RESULTS_OUT_DIR = my_env('CK_RESULTS_OUT_DIR') ANNOTATIONS_OUT_DIR = my_env('CK_ANNOTATIONS_OUT_DIR') DATASET_TYPE = dep_env('dataset', "CK_ENV_DATASET_TYPE") METRIC_TYPE = (my_env("CK_METRIC_TYPE") or DATASET_TYPE).lower() FULL_REPORT = not set_in_my_env("CK_SILENT_MODE") import ck_utils import converter_annotations import converter_results ck_utils.prepare_dir(RESULTS_OUT_DIR) ck_utils.prepare_dir(ANNOTATIONS_OUT_DIR) if METRIC_TYPE != ck_utils.COCO: import calc_metrics_coco_tf import calc_metrics_kitti import calc_metrics_oid from object_detection.utils import label_map_util else: import calc_metrics_coco_pycocotools def evaluate(processed_image_ids, categories_list): # Convert annotations from original format of the dataset # to a format specific for a tool that will calculate metrics if DATASET_TYPE != METRIC_TYPE: print('\nConvert annotations from {} to {} ...'.format( DATASET_TYPE, METRIC_TYPE)) annotations = converter_annotations.convert( ANNOTATIONS_PATH, ANNOTATIONS_OUT_DIR, DATASET_TYPE, METRIC_TYPE) else: annotations = ANNOTATIONS_PATH # Convert detection results from our universal text format # to a format specific for a tool that will calculate metrics print('\nConvert results to {} ...'.format(METRIC_TYPE)) results = converter_results.convert(DETECTIONS_OUT_DIR, RESULTS_OUT_DIR, DATASET_TYPE, MODEL_DATASET_TYPE, METRIC_TYPE) # Run evaluation tool print('\nEvaluate metrics as {} ...'.format(METRIC_TYPE)) if METRIC_TYPE == ck_utils.COCO: mAP, recall, all_metrics = calc_metrics_coco_pycocotools.evaluate( processed_image_ids, results, annotations) elif METRIC_TYPE == ck_utils.COCO_TF: mAP, recall, all_metrics = calc_metrics_coco_tf.evaluate( categories_list, results, annotations, FULL_REPORT) elif METRIC_TYPE == ck_utils.OID: mAP, _, all_metrics = calc_metrics_oid.evaluate( results, annotations, LABELMAP_FILE, FULL_REPORT) recall = 'N/A' else: raise ValueError( 'Metrics type is not supported: {}'.format(METRIC_TYPE)) OPENME['mAP'] = mAP OPENME['recall'] = recall OPENME['metrics'] = all_metrics return OPENME = {} with open(PREPROCESSED_FILES, 'r') as f: processed_image_filenames = [x.split(';')[0] for x in f.readlines()] processed_image_ids = [ ck_utils.filename_to_id(image_filename, DATASET_TYPE) for image_filename in processed_image_filenames ] if os.path.isfile(TIMER_JSON): with open(TIMER_JSON, 'r') as f: OPENME = json.load(f) # Run evaluation ck_utils.print_header('Process results') if METRIC_TYPE != ck_utils.COCO: category_index = label_map_util.create_category_index_from_labelmap( LABELMAP_FILE, use_display_name=True) categories_list = category_index.values() else: categories_list = [] evaluate(processed_image_ids, categories_list) OPENME[ 'frame_predictions'] = converter_results.convert_to_frame_predictions( DETECTIONS_OUT_DIR) OPENME['execution_time'] = OPENME['run_time_state'].get( 'test_time_s', 0) + OPENME['run_time_state'].get('setup_time_s', 0) # Store benchmark results with open(TIMER_JSON, 'w') as o: json.dump(OPENME, o, indent=2, sort_keys=True) # Print metrics print('\nSummary:') print('-------------------------------') print('All images loaded in {:.6f}s'.format(OPENME['run_time_state'].get( 'load_images_time_total_s', 0))) print('Average image load time: {:.6f}s'.format( OPENME['run_time_state'].get('load_images_time_avg_s', 0))) print('All images detected in {:.6f}s'.format(OPENME['run_time_state'].get( 'prediction_time_total_s', 0))) print('Average detection time: {:.6f}s'.format( OPENME['run_time_state'].get('prediction_time_avg_s', 0))) print('mAP: {}'.format(OPENME['mAP'])) print('Recall: {}'.format(OPENME['recall'])) print('--------------------------------\n') return {'return': 0}