def process_example(input_dir, output_dir): """Process a single example, save its outputs, and return the latency. In particular, this function requires that the submitted model (the run_model function in the wod_latency_submission module) takes in a bunch of numpy arrays as keyword arguments, whose names are defined in wod_latency_submission.DATA_FIELDS. It also assumes that input_directory contains a bunch of .npy data files with basenames corresponding to the valid values of DATA_FIELDS. Thus, this function loads the required fields from the input_directory, runs and times the run_model function, saves the model's outputs (a 'boxes' N x 7 ndarray, a 'scores' N ndarray, and a 'classes' N ndarray) to the output directory, and returns the model's runtime in seconds. Args: input_dir: string directory name to find the input .npy data files output_dir: string directory name to save the model results to. Returns: float latency value of the run_model call, in seconds. """ # Load all the data fields that the user requested. data = { field: np.load(os.path.join(input_dir, f'{field}.npy')) for field in wod_latency_submission.DATA_FIELDS } # Time the run_model function of the user's submitted module, with the data # fields passed in as keyword arguments. tic = time.perf_counter() output = wod_latency_submission.run_model(**data) toc = time.perf_counter() # Sanity check the output before saving. assert len(output) == 3 assert set(output.keys()) == set(('boxes', 'scores', 'classes')) num_objs = output['boxes'].shape[0] assert output['scores'].shape[0] == num_objs assert output['classes'].shape[0] == num_objs # Save the outputs as numpy files. for k, v in output.items(): np.save(os.path.join(output_dir, k), v) # Save the list of input fields in a text file. with open(os.path.join(output_dir, 'input_fields.txt'), 'w') as f: f.write('\n'.join(wod_latency_submission.DATA_FIELDS)) # Return the elapsed time of the run_model call. return toc - tic
def process_allimages_example(input_dir, output_dir): # Load all the data fields that the user requested. latency=[] result_dict={} for imagename in allcameras:#go through all cameras data = { allcameras[0]: np.load(os.path.join(input_dir, f'{imagename}.npy')) } # Time the run_model function of the user's submitted module, with the data # fields passed in as keyword arguments. tic = time.perf_counter() output = wod_latency_submission.run_model(**data) toc = time.perf_counter() latency.append(toc - tic) # Sanity check the output before saving. assert len(output) == 3 assert set(output.keys()) == set(('boxes', 'scores', 'classes')) num_objs = output['boxes'].shape[0] print(f'num_objs:{num_objs}') assert output['scores'].shape[0] == num_objs assert output['classes'].shape[0] == num_objs # Save the outputs as numpy files. # for k, v in output.items(): # npfilename=imagename+'_'+k #k add image name before the key # np.save(os.path.join(output_dir, npfilename), v) result_dict[imagename]=output #print(f'Num objs: {num_objs}, output: {output}') del data # # Save the list of input fields in a text file. # with open(os.path.join(output_dir, 'input_fields.txt'), 'w') as f: # f.write('\n'.join(wod_latency_submission.DATA_FIELDS)) npfilename=os.path.join(output_dir, 'allcameraresult.npy') np.save(npfilename, result_dict) del result_dict # Return the elapsed time of the run_model call. latency_np=np.array(latency) return np.mean(latency_np)
def evaluateWaymoValidationFramesAllcameraSubmission(PATH, model_path, config_path, validation_folders, outputsubmissionfilepath, VisEnable, imagesavepath, nameprefix="output"): data_files = [ path for x in validation_folders for path in glob(os.path.join(PATH, x, "*.tfrecord")) ] print(data_files) # all TFRecord file list print(len(data_files)) # create a list of dataset for each TFRecord file dataset = [ tf.data.TFRecordDataset(FILENAME, compression_type='') for FILENAME in data_files ] # total number of TFrecord files * 40 frame(each TFrecord) wod_latency_submission.setupmodeldir(model_path, config_path) wod_latency_submission.initialize_model() objects = metrics_pb2.Objects() # submission objects required_field = wod_latency_submission.DATA_FIELDS print(required_field) fps = FPS().start() #allcameras=["FRONT_IMAGE", "FRONT_LEFT_IMAGE", "FRONT_RIGHT_IMAGE", "SIDE_LEFT_IMAGE", "SIDE_RIGHT_IMAGE"] frameid = 0 testImagedict = {} for i, data_file in enumerate(dataset): print("Datafile: ", i) # Each TFrecord file # Create frame based on Waymo API, 199 frames per TFrecord (20s, 10Hz) for idx, data in enumerate(data_file): # if idx % 5 != 0: #Downsample every 5 images, reduce to 2Hz, total around 40 frames # continue frame = open_dataset.Frame() frame.ParseFromString(bytearray(data.numpy())) #frame=frames[frameid] convertedframesdict = convert_frame_to_dict_cameras( frame) #data_array[frameid] #Allconvertedframesdict.append(convertedframesdict) frame_timestamp_micros = convertedframesdict['TIMESTAMP'] #['key'] context_name = frame.context.name print( f'Current frame id: {frameid}, idx: {idx}, context_name: {context_name}, frame_timestamp_micros: {frame_timestamp_micros}' ) for imagename in allcameras: #go through all cameras testImagedict[ required_field[0]] = convertedframesdict[imagename].copy() start_time = time.perf_counter() #.time() #result = wod_latency_submission.run_model(Front_image) result = wod_latency_submission.run_model( **testImagedict) #All images end_time = time.perf_counter() #.time() elapsed_time = end_time - start_time print('Inference time: ' + str(elapsed_time * 1000) + f'ms, camera name: {imagename}') # createsubmisionobject(objects, result['boxes'], result['classes'], # result['scores'], context_name, frame_timestamp_micros) createsubmisioncameraobject(objects, result['boxes'], result['classes'], result['scores'], context_name, frame_timestamp_micros, cameraname_map[imagename]) if VisEnable == True: display_str = f'Inference time: {str(elapsed_time*1000)}ms, camera name: {imagename}, context_name: {context_name}, timestamp_micros: {frame_timestamp_micros}' savedetectedimagetofile(convertedframesdict[imagename], frameid, result, imagename, display_str, nameprefix, imagesavepath) fps.update() frameid = frameid + 1 if frameid % 5000 == 0: with open( os.path.join( imagesavepath, nameprefix + '_' + str(frameid) + '.pickle'), 'wb') as f: pickle.dump(objects, f) # stop the timer and display FPS information fps.stop() print("[INFO] elasped time: {:.2f}".format(fps.elapsed())) print("[INFO] approx. FPS: {:.2f}".format(fps.fps())) with open(os.path.join(imagesavepath, nameprefix + '.pickle'), 'wb') as f: pickle.dump(objects, f) # with open('allframedics.pickle', 'wb') as f: # pickle.dump(Allconvertedframesdict, f) submission = submission_pb2.Submission() submission.task = submission_pb2.Submission.DETECTION_2D submission.account_name = '*****@*****.**' submission.authors.append('Kaikai Liu') submission.affiliation = 'San Jose State University' submission.unique_method_name = nameprefix #'fake' submission.description = 'none' submission.method_link = "empty method" submission.sensor_type = submission_pb2.Submission.CAMERA_ALL submission.number_past_frames_exclude_current = 0 submission.number_future_frames_exclude_current = 0 submission.inference_results.CopyFrom(objects) f = open(outputsubmissionfilepath, 'wb') # output submission file f.write(submission.SerializeToString()) f.close() now = datetime.datetime.now() print("Finished validation, current date and time : ") print(now.strftime("%Y-%m-%d %H:%M:%S"))
def evaluateWaymoValidationFramesSubmission(PATH, validation_folders, outputsubmissionfilepath, VisEnable, outputfile="./output_video1.mp4"): data_files = [ path for x in validation_folders for path in glob(os.path.join(PATH, x, "*.tfrecord")) ] print(data_files) # all TFRecord file list print(len(data_files)) # create a list of dataset for each TFRecord file dataset = [ tf.data.TFRecordDataset(FILENAME, compression_type='') for FILENAME in data_files ] # total number of TFrecord files * 40 frame(each TFrecord) objects = metrics_pb2.Objects() # submission objects wod_latency_submission.initialize_model() required_field = wod_latency_submission.DATA_FIELDS print(required_field) if VisEnable == True: frame_width = 1920 frame_height = 1280 out = cv2.VideoWriter(outputfile, cv2.VideoWriter_fourcc('M', 'P', '4', 'V'), 5, (frame_width, frame_height)) fps = FPS().start() frameid = 0 for i, data_file in enumerate(dataset): print("Datafile: ", i) # Each TFrecord file # Create frame based on Waymo API, 199 frames per TFrecord (20s, 10Hz) for idx, data in enumerate(data_file): # if idx % 5 != 0: #Downsample every 5 images, reduce to 2Hz, total around 40 frames # continue frame = open_dataset.Frame() frame.ParseFromString(bytearray(data.numpy())) #frame=frames[frameid] convertedframesdict = convert_frame_to_dict_cameras( frame) #data_array[frameid] #Allconvertedframesdict.append(convertedframesdict) frame_timestamp_micros = convertedframesdict['TIMESTAMP'] #['key'] context_name = frame.context.name print( f'Current frame id: {frameid}, context_name: {context_name}, frame_timestamp_micros: {frame_timestamp_micros}' ) start_time = time.perf_counter() #.time() #result = wod_latency_submission.run_model(Front_image) result = wod_latency_submission.run_model( **convertedframesdict) #All images end_time = time.perf_counter() #.time() elapsed_time = end_time - start_time print('Inference time: ' + str(elapsed_time) + 's') # print(result) createsubmisionobject(objects, result['boxes'], result['classes'], result['scores'], context_name, frame_timestamp_micros) if VisEnable == True: Front_image = convertedframesdict[required_field[0]] #Front_image = convertedframesdict['FRONT_IMAGE'] image_np_with_detections = Front_image.copy() visualization_util.visualize_boxes_and_labels_on_image_array( image_np_with_detections, result['boxes'], result['classes'], result['scores'], category_index, use_normalized_coordinates=False, max_boxes_to_draw=200, min_score_thresh=Threshold, agnostic_mode=False) display_str = f'Inference time: {str(elapsed_time*1000)}ms, context_name: {context_name}, timestamp_micros: {frame_timestamp_micros}' visualization_util.draw_text_on_image(image_np_with_detections, 0, 0, display_str, color='black') #visualization_util.save_image_array_as_png(image_np_with_detections, outputfile) name = './Test_data/frame' + str(frameid) + '.jpg' #print ('Creating\...' + name) # cv2.imwrite(name, image_np_with_detections) #write to image folder #out.write(image_np_with_detections) out.write( cv2.cvtColor(image_np_with_detections, cv2.COLOR_RGB2BGR)) #cv2.cvtColor(img, cv2.COLOR_RGB2BGR) frameid = frameid + 1 fps.update() # stop the timer and display FPS information fps.stop() print("[INFO] elasped time: {:.2f}".format(fps.elapsed())) print("[INFO] approx. FPS: {:.2f}".format(fps.fps())) if VisEnable == True: out.release() with open('objectsresult0525_detectron282k_valall_front.pickle', 'wb') as f: pickle.dump(objects, f) # with open('allframedics.pickle', 'wb') as f: # pickle.dump(Allconvertedframesdict, f) submission = submission_pb2.Submission() submission.task = submission_pb2.Submission.DETECTION_2D submission.account_name = '*****@*****.**' submission.authors.append('Kaikai Liu') submission.affiliation = 'None' submission.unique_method_name = 'fake' submission.description = 'none' submission.method_link = "empty method" submission.sensor_type = submission_pb2.Submission.CAMERA_ALL submission.number_past_frames_exclude_current = 0 submission.number_future_frames_exclude_current = 0 submission.inference_results.CopyFrom(objects) f = open(outputsubmissionfilepath, 'wb') # output submission file f.write(submission.SerializeToString()) f.close() now = datetime.datetime.now() print("Finished validation, current date and time : ") print(now.strftime("%Y-%m-%d %H:%M:%S"))
def evaluateallframescreatesubmission(frames, outputsubmissionfilepath, outputfile="./output_video1.mp4"): array_len = len(frames) #4931 frames for validation_0000 # 20, 200 frames in one file, downsample by 10 print("Frames lenth:", array_len) print("Final_array type:", type(frames)) # class 'list' objects = metrics_pb2.Objects() # submission objects frame_width = 1920 frame_height = 1280 out = cv2.VideoWriter(outputfile, cv2.VideoWriter_fourcc( 'M', 'P', '4', 'V'), 5, (frame_width, frame_height)) fps = FPS().start() wod_latency_submission.initialize_model() required_field = wod_latency_submission.DATA_FIELDS print(required_field) for frameid in range(array_len): #frameid = 5 print("frameid:", frameid) # {'key':key, 'context_name':context_name, 'framedict':framedict} currentframe=frames[frameid] convertedframesdict = convert_frame_to_dict_cameras(currentframe) #data_array[frameid] frame_timestamp_micros = convertedframesdict['TIMESTAMP']#['key'] context_name = currentframe.context.name #convertedframesdict['context_name'] #framedict = convertedframesdict['framedict'] # 10017090168044687777_6380_000_6400_000 #print('context_name:', context_name) # print('frame_timestamp_micros:', frame_timestamp_micros) # 1550083467346370 #result = wod_latency_submission.run_model(framedict[required_field[0]], framedict[required_field[1]]) #result = wod_latency_submission.run_model(**framedict) #Front_image = framedict[required_field[0]] start_time = time.time() #result = wod_latency_submission.run_model(Front_image) result = wod_latency_submission.run_model(**convertedframesdict)#All images end_time = time.time() elapsed_time = end_time - start_time print('Inference time: ' + str(elapsed_time) + 's') # print(result) createsubmisionobject(objects, result['boxes'], result['classes'], result['scores'], context_name, frame_timestamp_micros) # Save the original image #output_path = "./test.png" #visualization_util.save_image_array_as_png(Front_image, output_path) Front_image = convertedframesdict[required_field[0]] image_np_with_detections = Front_image.copy() visualization_util.visualize_boxes_and_labels_on_image_array(image_np_with_detections, result['boxes'], result['classes'], result['scores'], category_index, use_normalized_coordinates=False, max_boxes_to_draw=200, min_score_thresh=Threshold, agnostic_mode=False) display_str=f'Inference time: {str(elapsed_time*1000)}ms, context_name: {context_name}, timestamp_micros: {frame_timestamp_micros}' visualization_util.draw_text_on_image(image_np_with_detections, 0, 0, display_str, color='black') #visualization_util.save_image_array_as_png(image_np_with_detections, outputfile) name = './Test_data/frame' + str(frameid) + '.jpg' #print ('Creating\...' + name) # cv2.imwrite(name, image_np_with_detections) #write to image folder fps.update() #out.write(image_np_with_detections) out.write(cv2.cvtColor(image_np_with_detections, cv2.COLOR_RGB2BGR)) #cv2.cvtColor(img, cv2.COLOR_RGB2BGR) # stop the timer and display FPS information fps.stop() print("[INFO] elasped time: {:.2f}".format(fps.elapsed())) print("[INFO] approx. FPS: {:.2f}".format(fps.fps())) out.release() with open('objectsresult.pickle', 'wb') as f: pickle.dump(objects, f) submission = submission_pb2.Submission() submission.task = submission_pb2.Submission.DETECTION_2D submission.account_name = '*****@*****.**' submission.authors.append('Kaikai Liu') submission.affiliation = 'None' submission.unique_method_name = 'torchvisionfaster' submission.description = 'none' submission.method_link = "empty method" submission.sensor_type = submission_pb2.Submission.CAMERA_ALL submission.number_past_frames_exclude_current = 0 submission.number_future_frames_exclude_current = 0 submission.inference_results.CopyFrom(objects) f = open(outputsubmissionfilepath, 'wb') # output submission file f.write(submission.SerializeToString()) f.close() now = datetime.datetime.now() print("Finished validation, current date and time : ") print(now.strftime("%Y-%m-%d %H:%M:%S"))
def evaluatesingleframe(base_dir, filename, frameid, outputfile="./testresult.png"): Final_array = np.load(base_dir / filename, allow_pickle=True, mmap_mode='r') data_array = Final_array['arr_0'] array_len = len(data_array) # 20, 200 frames in one file, downsample by 10 print("Final_array lenth:", array_len) print("Final_array type:", type(data_array)) # numpy.ndarray # for frameid in range(array_len): #frameid = 5 print("frameid:", frameid) # {'key':key, 'context_name':context_name, 'framedict':framedict} convertedframesdict = data_array[frameid] frame_timestamp_micros = convertedframesdict['key'] context_name = convertedframesdict['context_name'] framedict = convertedframesdict['framedict'] # 10017090168044687777_6380_000_6400_000 print('context_name:', context_name) print('frame_timestamp_micros:', frame_timestamp_micros) # 1550083467346370 start_time = time.time() wod_latency_submission.initialize_model() end_time = time.time() elapsed_time = end_time - start_time print('Model Inalization elapsed time: ' + str(elapsed_time) + 's') required_field = wod_latency_submission.DATA_FIELDS print(required_field) #result = wod_latency_submission.run_model(framedict[required_field[0]], framedict[required_field[1]]) #result = wod_latency_submission.run_model(**framedict) Front_image = framedict[required_field[0]] start_time = time.time() result = wod_latency_submission.run_model(Front_image) end_time = time.time() elapsed_time = end_time - start_time print('Inference time: ' + str(elapsed_time) + 's') #print(result) #Save the original image #output_path = "./test.png" #visualization_util.save_image_array_as_png(Front_image, output_path) image_np_with_detections = Front_image.copy() # category_index = { 1: {'id': 1, 'name': 'VEHICLE'}, \ # 2: {'id': 2, 'name': 'PEDESTRIAN'}, \ # 3: {'id': 3, 'name': 'SIGN'}, \ # 4: {'id': 4, 'name': 'CYCLIST'}} # label_map_path = '2DObject/tfobjectdetection/waymo_labelmap.txt' # label_map = label_map_util.load_labelmap(label_map_path) # categories = label_map_util.convert_label_map_to_categories( # label_map, # max_num_classes=label_map_util.get_max_label_map_index(label_map), # use_display_name=True) # category_index = label_map_util.create_category_index(categories) visualization_util.visualize_boxes_and_labels_on_image_array( image_np_with_detections, result['boxes'], result['classes'], result['scores'], category_index, use_normalized_coordinates=False, max_boxes_to_draw=200, min_score_thresh=Threshold, agnostic_mode=False) visualization_util.save_image_array_as_png(image_np_with_detections, outputfile)
def evaluateallframes(base_dir, filename, outputfile="./output_video1.mp4"): Final_array = np.load(base_dir / filename, allow_pickle=True, mmap_mode='r') data_array = Final_array['arr_0'] array_len = len(data_array) # 20, 200 frames in one file, downsample by 10 print("Final_array lenth:", array_len) print("Final_array type:", type(data_array)) # numpy.ndarray frame_width = 1920 frame_height = 1280 out = cv2.VideoWriter(outputfile, cv2.VideoWriter_fourcc('M', 'P', '4', 'V'), 2, (frame_width, frame_height)) fps = FPS().start() wod_latency_submission.initialize_model() required_field = wod_latency_submission.DATA_FIELDS print(required_field) for frameid in range(array_len): #frameid = 5 print("frameid:", frameid) # {'key':key, 'context_name':context_name, 'framedict':framedict} convertedframesdict = data_array[frameid] frame_timestamp_micros = convertedframesdict['key'] context_name = convertedframesdict['context_name'] framedict = convertedframesdict['framedict'] # 10017090168044687777_6380_000_6400_000 #print('context_name:', context_name) #print('frame_timestamp_micros:', frame_timestamp_micros) # 1550083467346370 #result = wod_latency_submission.run_model(framedict[required_field[0]], framedict[required_field[1]]) #result = wod_latency_submission.run_model(**framedict) Front_image = framedict[required_field[0]] start_time = time.time() result = wod_latency_submission.run_model(Front_image) end_time = time.time() elapsed_time = end_time - start_time print('Inference time: ' + str(elapsed_time) + 's') #print(result) #Save the original image #output_path = "./test.png" #visualization_util.save_image_array_as_png(Front_image, output_path) image_np_with_detections = Front_image.copy() visualization_util.visualize_boxes_and_labels_on_image_array( image_np_with_detections, result['boxes'], result['classes'], result['scores'], category_index, use_normalized_coordinates=False, max_boxes_to_draw=200, min_score_thresh=Threshold, agnostic_mode=False) #visualization_util.save_image_array_as_png(image_np_with_detections, outputfile) name = './Test_data/frame' + str(frameid) + '.jpg' #print ('Creating\...' + name) #cv2.imwrite(name, image_np_with_detections) #write to image folder fps.update() out.write(image_np_with_detections) # stop the timer and display FPS information fps.stop() print("[INFO] elasped time: {:.2f}".format(fps.elapsed())) print("[INFO] approx. FPS: {:.2f}".format(fps.fps())) out.release()