def process_example(input_dir, output_dir):
"""Process a single example, save its outputs, and return the latency.
In particular, this function requires that the submitted model (the run_model
function in the wod_latency_submission module) takes in a bunch of numpy
arrays as keyword arguments, whose names are defined in
wod_latency_submission.DATA_FIELDS. It also assumes that input_directory
contains a bunch of .npy data files with basenames corresponding to the valid
values of DATA_FIELDS. Thus, this function loads the required fields from the
input_directory, runs and times the run_model function, saves the model's
outputs (a 'boxes' N x 7 ndarray, a 'scores' N ndarray, and a 'classes' N
ndarray) to the output directory, and returns the model's runtime in seconds.
Args:
input_dir: string directory name to find the input .npy data files
output_dir: string directory name to save the model results to.
Returns:
float latency value of the run_model call, in seconds.
"""
# Load all the data fields that the user requested.
data = {
field: np.load(os.path.join(input_dir, f'{field}.npy'))
for field in wod_latency_submission.DATA_FIELDS
}
# Time the run_model function of the user's submitted module, with the data
# fields passed in as keyword arguments.
tic = time.perf_counter()
output = wod_latency_submission.run_model(**data)
toc = time.perf_counter()
# Sanity check the output before saving.
assert len(output) == 3
assert set(output.keys()) == set(('boxes', 'scores', 'classes'))
num_objs = output['boxes'].shape[0]
assert output['scores'].shape[0] == num_objs
assert output['classes'].shape[0] == num_objs
# Save the outputs as numpy files.
for k, v in output.items():
np.save(os.path.join(output_dir, k), v)
# Save the list of input fields in a text file.
with open(os.path.join(output_dir, 'input_fields.txt'), 'w') as f:
f.write('\n'.join(wod_latency_submission.DATA_FIELDS))
# Return the elapsed time of the run_model call.
return toc - tic
def process_allimages_example(input_dir, output_dir):
# Load all the data fields that the user requested.
latency=[]
result_dict={}
for imagename in allcameras:#go through all cameras
data = {
allcameras[0]: np.load(os.path.join(input_dir, f'{imagename}.npy'))
}
# Time the run_model function of the user's submitted module, with the data
# fields passed in as keyword arguments.
tic = time.perf_counter()
output = wod_latency_submission.run_model(**data)
toc = time.perf_counter()
latency.append(toc - tic)
# Sanity check the output before saving.
assert len(output) == 3
assert set(output.keys()) == set(('boxes', 'scores', 'classes'))
num_objs = output['boxes'].shape[0]
print(f'num_objs:{num_objs}')
assert output['scores'].shape[0] == num_objs
assert output['classes'].shape[0] == num_objs
# Save the outputs as numpy files.
# for k, v in output.items():
# npfilename=imagename+'_'+k #k add image name before the key
# np.save(os.path.join(output_dir, npfilename), v)
result_dict[imagename]=output
#print(f'Num objs: {num_objs}, output: {output}')
del data
# # Save the list of input fields in a text file.
# with open(os.path.join(output_dir, 'input_fields.txt'), 'w') as f:
# f.write('\n'.join(wod_latency_submission.DATA_FIELDS))
npfilename=os.path.join(output_dir, 'allcameraresult.npy')
np.save(npfilename, result_dict)
del result_dict
# Return the elapsed time of the run_model call.
latency_np=np.array(latency)
return np.mean(latency_np)
def evaluateWaymoValidationFramesAllcameraSubmission(PATH,
model_path,
config_path,
validation_folders,
outputsubmissionfilepath,
VisEnable,
imagesavepath,
nameprefix="output"):
data_files = [
path for x in validation_folders
for path in glob(os.path.join(PATH, x, "*.tfrecord"))
]
print(data_files) # all TFRecord file list
print(len(data_files))
# create a list of dataset for each TFRecord file
dataset = [
tf.data.TFRecordDataset(FILENAME, compression_type='')
for FILENAME in data_files
]
# total number of TFrecord files * 40 frame(each TFrecord)
wod_latency_submission.setupmodeldir(model_path, config_path)
wod_latency_submission.initialize_model()
objects = metrics_pb2.Objects() # submission objects
required_field = wod_latency_submission.DATA_FIELDS
print(required_field)
fps = FPS().start()
#allcameras=["FRONT_IMAGE", "FRONT_LEFT_IMAGE", "FRONT_RIGHT_IMAGE", "SIDE_LEFT_IMAGE", "SIDE_RIGHT_IMAGE"]
frameid = 0
testImagedict = {}
for i, data_file in enumerate(dataset):
print("Datafile: ", i) # Each TFrecord file
# Create frame based on Waymo API, 199 frames per TFrecord (20s, 10Hz)
for idx, data in enumerate(data_file):
# if idx % 5 != 0: #Downsample every 5 images, reduce to 2Hz, total around 40 frames
# continue
frame = open_dataset.Frame()
frame.ParseFromString(bytearray(data.numpy()))
#frame=frames[frameid]
convertedframesdict = convert_frame_to_dict_cameras(
frame) #data_array[frameid]
#Allconvertedframesdict.append(convertedframesdict)
frame_timestamp_micros = convertedframesdict['TIMESTAMP'] #['key']
context_name = frame.context.name
print(
f'Current frame id: {frameid}, idx: {idx}, context_name: {context_name}, frame_timestamp_micros: {frame_timestamp_micros}'
)
for imagename in allcameras: #go through all cameras
testImagedict[
required_field[0]] = convertedframesdict[imagename].copy()
start_time = time.perf_counter() #.time()
#result = wod_latency_submission.run_model(Front_image)
result = wod_latency_submission.run_model(
**testImagedict) #All images
end_time = time.perf_counter() #.time()
elapsed_time = end_time - start_time
print('Inference time: ' + str(elapsed_time * 1000) +
f'ms, camera name: {imagename}')
# createsubmisionobject(objects, result['boxes'], result['classes'],
# result['scores'], context_name, frame_timestamp_micros)
createsubmisioncameraobject(objects, result['boxes'],
result['classes'],
result['scores'], context_name,
frame_timestamp_micros,
cameraname_map[imagename])
if VisEnable == True:
display_str = f'Inference time: {str(elapsed_time*1000)}ms, camera name: {imagename}, context_name: {context_name}, timestamp_micros: {frame_timestamp_micros}'
savedetectedimagetofile(convertedframesdict[imagename],
frameid, result, imagename,
display_str, nameprefix,
imagesavepath)
fps.update()
frameid = frameid + 1
if frameid % 5000 == 0:
with open(
os.path.join(
imagesavepath,
nameprefix + '_' + str(frameid) + '.pickle'),
'wb') as f:
pickle.dump(objects, f)
# stop the timer and display FPS information
fps.stop()
print("[INFO] elasped time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
with open(os.path.join(imagesavepath, nameprefix + '.pickle'), 'wb') as f:
pickle.dump(objects, f)
# with open('allframedics.pickle', 'wb') as f:
# pickle.dump(Allconvertedframesdict, f)
submission = submission_pb2.Submission()
submission.task = submission_pb2.Submission.DETECTION_2D
submission.account_name = '*****@*****.**'
submission.authors.append('Kaikai Liu')
submission.affiliation = 'San Jose State University'
submission.unique_method_name = nameprefix #'fake'
submission.description = 'none'
submission.method_link = "empty method"
submission.sensor_type = submission_pb2.Submission.CAMERA_ALL
submission.number_past_frames_exclude_current = 0
submission.number_future_frames_exclude_current = 0
submission.inference_results.CopyFrom(objects)
f = open(outputsubmissionfilepath, 'wb') # output submission file
f.write(submission.SerializeToString())
f.close()
now = datetime.datetime.now()
print("Finished validation, current date and time : ")
print(now.strftime("%Y-%m-%d %H:%M:%S"))
def evaluateWaymoValidationFramesSubmission(PATH,
validation_folders,
outputsubmissionfilepath,
VisEnable,
outputfile="./output_video1.mp4"):
data_files = [
path for x in validation_folders
for path in glob(os.path.join(PATH, x, "*.tfrecord"))
]
print(data_files) # all TFRecord file list
print(len(data_files))
# create a list of dataset for each TFRecord file
dataset = [
tf.data.TFRecordDataset(FILENAME, compression_type='')
for FILENAME in data_files
]
# total number of TFrecord files * 40 frame(each TFrecord)
objects = metrics_pb2.Objects() # submission objects
wod_latency_submission.initialize_model()
required_field = wod_latency_submission.DATA_FIELDS
print(required_field)
if VisEnable == True:
frame_width = 1920
frame_height = 1280
out = cv2.VideoWriter(outputfile,
cv2.VideoWriter_fourcc('M', 'P', '4', 'V'), 5,
(frame_width, frame_height))
fps = FPS().start()
frameid = 0
for i, data_file in enumerate(dataset):
print("Datafile: ", i) # Each TFrecord file
# Create frame based on Waymo API, 199 frames per TFrecord (20s, 10Hz)
for idx, data in enumerate(data_file):
# if idx % 5 != 0: #Downsample every 5 images, reduce to 2Hz, total around 40 frames
# continue
frame = open_dataset.Frame()
frame.ParseFromString(bytearray(data.numpy()))
#frame=frames[frameid]
convertedframesdict = convert_frame_to_dict_cameras(
frame) #data_array[frameid]
#Allconvertedframesdict.append(convertedframesdict)
frame_timestamp_micros = convertedframesdict['TIMESTAMP'] #['key']
context_name = frame.context.name
print(
f'Current frame id: {frameid}, context_name: {context_name}, frame_timestamp_micros: {frame_timestamp_micros}'
)
start_time = time.perf_counter() #.time()
#result = wod_latency_submission.run_model(Front_image)
result = wod_latency_submission.run_model(
**convertedframesdict) #All images
end_time = time.perf_counter() #.time()
elapsed_time = end_time - start_time
print('Inference time: ' + str(elapsed_time) + 's')
# print(result)
createsubmisionobject(objects, result['boxes'], result['classes'],
result['scores'], context_name,
frame_timestamp_micros)
if VisEnable == True:
Front_image = convertedframesdict[required_field[0]]
#Front_image = convertedframesdict['FRONT_IMAGE']
image_np_with_detections = Front_image.copy()
visualization_util.visualize_boxes_and_labels_on_image_array(
image_np_with_detections,
result['boxes'],
result['classes'],
result['scores'],
category_index,
use_normalized_coordinates=False,
max_boxes_to_draw=200,
min_score_thresh=Threshold,
agnostic_mode=False)
display_str = f'Inference time: {str(elapsed_time*1000)}ms, context_name: {context_name}, timestamp_micros: {frame_timestamp_micros}'
visualization_util.draw_text_on_image(image_np_with_detections,
0,
0,
display_str,
color='black')
#visualization_util.save_image_array_as_png(image_np_with_detections, outputfile)
name = './Test_data/frame' + str(frameid) + '.jpg'
#print ('Creating\...' + name)
# cv2.imwrite(name, image_np_with_detections) #write to image folder
#out.write(image_np_with_detections)
out.write(
cv2.cvtColor(image_np_with_detections, cv2.COLOR_RGB2BGR))
#cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
frameid = frameid + 1
fps.update()
# stop the timer and display FPS information
fps.stop()
print("[INFO] elasped time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
if VisEnable == True:
out.release()
with open('objectsresult0525_detectron282k_valall_front.pickle',
'wb') as f:
pickle.dump(objects, f)
# with open('allframedics.pickle', 'wb') as f:
# pickle.dump(Allconvertedframesdict, f)
submission = submission_pb2.Submission()
submission.task = submission_pb2.Submission.DETECTION_2D
submission.account_name = '*****@*****.**'
submission.authors.append('Kaikai Liu')
submission.affiliation = 'None'
submission.unique_method_name = 'fake'
submission.description = 'none'
submission.method_link = "empty method"
submission.sensor_type = submission_pb2.Submission.CAMERA_ALL
submission.number_past_frames_exclude_current = 0
submission.number_future_frames_exclude_current = 0
submission.inference_results.CopyFrom(objects)
f = open(outputsubmissionfilepath, 'wb') # output submission file
f.write(submission.SerializeToString())
f.close()
now = datetime.datetime.now()
print("Finished validation, current date and time : ")
print(now.strftime("%Y-%m-%d %H:%M:%S"))
def evaluateallframescreatesubmission(frames, outputsubmissionfilepath, outputfile="./output_video1.mp4"):
array_len = len(frames) #4931 frames for validation_0000
# 20, 200 frames in one file, downsample by 10
print("Frames lenth:", array_len)
print("Final_array type:", type(frames)) # class 'list'
objects = metrics_pb2.Objects() # submission objects
frame_width = 1920
frame_height = 1280
out = cv2.VideoWriter(outputfile, cv2.VideoWriter_fourcc(
'M', 'P', '4', 'V'), 5, (frame_width, frame_height))
fps = FPS().start()
wod_latency_submission.initialize_model()
required_field = wod_latency_submission.DATA_FIELDS
print(required_field)
for frameid in range(array_len):
#frameid = 5
print("frameid:", frameid)
# {'key':key, 'context_name':context_name, 'framedict':framedict}
currentframe=frames[frameid]
convertedframesdict = convert_frame_to_dict_cameras(currentframe) #data_array[frameid]
frame_timestamp_micros = convertedframesdict['TIMESTAMP']#['key']
context_name = currentframe.context.name #convertedframesdict['context_name']
#framedict = convertedframesdict['framedict']
# 10017090168044687777_6380_000_6400_000
#print('context_name:', context_name)
# print('frame_timestamp_micros:', frame_timestamp_micros) # 1550083467346370
#result = wod_latency_submission.run_model(framedict[required_field[0]], framedict[required_field[1]])
#result = wod_latency_submission.run_model(**framedict)
#Front_image = framedict[required_field[0]]
start_time = time.time()
#result = wod_latency_submission.run_model(Front_image)
result = wod_latency_submission.run_model(**convertedframesdict)#All images
end_time = time.time()
elapsed_time = end_time - start_time
print('Inference time: ' + str(elapsed_time) + 's')
# print(result)
createsubmisionobject(objects, result['boxes'], result['classes'],
result['scores'], context_name, frame_timestamp_micros)
# Save the original image
#output_path = "./test.png"
#visualization_util.save_image_array_as_png(Front_image, output_path)
Front_image = convertedframesdict[required_field[0]]
image_np_with_detections = Front_image.copy()
visualization_util.visualize_boxes_and_labels_on_image_array(image_np_with_detections, result['boxes'], result['classes'], result['scores'], category_index, use_normalized_coordinates=False,
max_boxes_to_draw=200,
min_score_thresh=Threshold,
agnostic_mode=False)
display_str=f'Inference time: {str(elapsed_time*1000)}ms, context_name: {context_name}, timestamp_micros: {frame_timestamp_micros}'
visualization_util.draw_text_on_image(image_np_with_detections, 0, 0, display_str, color='black')
#visualization_util.save_image_array_as_png(image_np_with_detections, outputfile)
name = './Test_data/frame' + str(frameid) + '.jpg'
#print ('Creating\...' + name)
# cv2.imwrite(name, image_np_with_detections) #write to image folder
fps.update()
#out.write(image_np_with_detections)
out.write(cv2.cvtColor(image_np_with_detections, cv2.COLOR_RGB2BGR))
#cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
# stop the timer and display FPS information
fps.stop()
print("[INFO] elasped time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
out.release()
with open('objectsresult.pickle', 'wb') as f:
pickle.dump(objects, f)
submission = submission_pb2.Submission()
submission.task = submission_pb2.Submission.DETECTION_2D
submission.account_name = '*****@*****.**'
submission.authors.append('Kaikai Liu')
submission.affiliation = 'None'
submission.unique_method_name = 'torchvisionfaster'
submission.description = 'none'
submission.method_link = "empty method"
submission.sensor_type = submission_pb2.Submission.CAMERA_ALL
submission.number_past_frames_exclude_current = 0
submission.number_future_frames_exclude_current = 0
submission.inference_results.CopyFrom(objects)
f = open(outputsubmissionfilepath, 'wb') # output submission file
f.write(submission.SerializeToString())
f.close()
now = datetime.datetime.now()
print("Finished validation, current date and time : ")
print(now.strftime("%Y-%m-%d %H:%M:%S"))
def evaluatesingleframe(base_dir,
filename,
frameid,
outputfile="./testresult.png"):
Final_array = np.load(base_dir / filename,
allow_pickle=True,
mmap_mode='r')
data_array = Final_array['arr_0']
array_len = len(data_array)
# 20, 200 frames in one file, downsample by 10
print("Final_array lenth:", array_len)
print("Final_array type:", type(data_array)) # numpy.ndarray
# for frameid in range(array_len):
#frameid = 5
print("frameid:", frameid)
# {'key':key, 'context_name':context_name, 'framedict':framedict}
convertedframesdict = data_array[frameid]
frame_timestamp_micros = convertedframesdict['key']
context_name = convertedframesdict['context_name']
framedict = convertedframesdict['framedict']
# 10017090168044687777_6380_000_6400_000
print('context_name:', context_name)
print('frame_timestamp_micros:',
frame_timestamp_micros) # 1550083467346370
start_time = time.time()
wod_latency_submission.initialize_model()
end_time = time.time()
elapsed_time = end_time - start_time
print('Model Inalization elapsed time: ' + str(elapsed_time) + 's')
required_field = wod_latency_submission.DATA_FIELDS
print(required_field)
#result = wod_latency_submission.run_model(framedict[required_field[0]], framedict[required_field[1]])
#result = wod_latency_submission.run_model(**framedict)
Front_image = framedict[required_field[0]]
start_time = time.time()
result = wod_latency_submission.run_model(Front_image)
end_time = time.time()
elapsed_time = end_time - start_time
print('Inference time: ' + str(elapsed_time) + 's')
#print(result)
#Save the original image
#output_path = "./test.png"
#visualization_util.save_image_array_as_png(Front_image, output_path)
image_np_with_detections = Front_image.copy()
# category_index = { 1: {'id': 1, 'name': 'VEHICLE'}, \
# 2: {'id': 2, 'name': 'PEDESTRIAN'}, \
# 3: {'id': 3, 'name': 'SIGN'}, \
# 4: {'id': 4, 'name': 'CYCLIST'}}
# label_map_path = '2DObject/tfobjectdetection/waymo_labelmap.txt'
# label_map = label_map_util.load_labelmap(label_map_path)
# categories = label_map_util.convert_label_map_to_categories(
# label_map,
# max_num_classes=label_map_util.get_max_label_map_index(label_map),
# use_display_name=True)
# category_index = label_map_util.create_category_index(categories)
visualization_util.visualize_boxes_and_labels_on_image_array(
image_np_with_detections,
result['boxes'],
result['classes'],
result['scores'],
category_index,
use_normalized_coordinates=False,
max_boxes_to_draw=200,
min_score_thresh=Threshold,
agnostic_mode=False)
visualization_util.save_image_array_as_png(image_np_with_detections,
outputfile)
def evaluateallframes(base_dir, filename, outputfile="./output_video1.mp4"):
Final_array = np.load(base_dir / filename,
allow_pickle=True,
mmap_mode='r')
data_array = Final_array['arr_0']
array_len = len(data_array)
# 20, 200 frames in one file, downsample by 10
print("Final_array lenth:", array_len)
print("Final_array type:", type(data_array)) # numpy.ndarray
frame_width = 1920
frame_height = 1280
out = cv2.VideoWriter(outputfile,
cv2.VideoWriter_fourcc('M', 'P', '4', 'V'), 2,
(frame_width, frame_height))
fps = FPS().start()
wod_latency_submission.initialize_model()
required_field = wod_latency_submission.DATA_FIELDS
print(required_field)
for frameid in range(array_len):
#frameid = 5
print("frameid:", frameid)
# {'key':key, 'context_name':context_name, 'framedict':framedict}
convertedframesdict = data_array[frameid]
frame_timestamp_micros = convertedframesdict['key']
context_name = convertedframesdict['context_name']
framedict = convertedframesdict['framedict']
# 10017090168044687777_6380_000_6400_000
#print('context_name:', context_name)
#print('frame_timestamp_micros:', frame_timestamp_micros) # 1550083467346370
#result = wod_latency_submission.run_model(framedict[required_field[0]], framedict[required_field[1]])
#result = wod_latency_submission.run_model(**framedict)
Front_image = framedict[required_field[0]]
start_time = time.time()
result = wod_latency_submission.run_model(Front_image)
end_time = time.time()
elapsed_time = end_time - start_time
print('Inference time: ' + str(elapsed_time) + 's')
#print(result)
#Save the original image
#output_path = "./test.png"
#visualization_util.save_image_array_as_png(Front_image, output_path)
image_np_with_detections = Front_image.copy()
visualization_util.visualize_boxes_and_labels_on_image_array(
image_np_with_detections,
result['boxes'],
result['classes'],
result['scores'],
category_index,
use_normalized_coordinates=False,
max_boxes_to_draw=200,
min_score_thresh=Threshold,
agnostic_mode=False)
#visualization_util.save_image_array_as_png(image_np_with_detections, outputfile)
name = './Test_data/frame' + str(frameid) + '.jpg'
#print ('Creating\...' + name)
#cv2.imwrite(name, image_np_with_detections) #write to image folder
fps.update()
out.write(image_np_with_detections)
# stop the timer and display FPS information
fps.stop()
print("[INFO] elasped time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
out.release()
