def createsubmissionfromobject(objects, outputsubmissionfilepath, prefix):
submission = submission_pb2.Submission()
submission.task = submission_pb2.Submission.DETECTION_2D
submission.account_name = '*****@*****.**'
submission.authors.append('Kaikai Liu')
submission.affiliation = 'San Jose State University'
# 'fake' unique_method_name should not be too long
submission.unique_method_name = prefix
submission.description = 'none'
submission.method_link = "empty method"
submission.sensor_type = submission_pb2.Submission.CAMERA_ALL
submission.number_past_frames_exclude_current = 0
submission.number_future_frames_exclude_current = 0
submission.inference_results.CopyFrom(objects)
f = open(outputsubmissionfilepath, 'wb') # output submission file
f.write(submission.SerializeToString())
f.close()
def createsubmissionfromobject(objects, outputsubmissionfilepath, prefix):
submission = submission_pb2.Submission()
submission.task = submission_pb2.Submission.DETECTION_3D #DETECTION_2D
submission.account_name = '*****@*****.**'
submission.authors.append('Kaikai Liu')
submission.affiliation = 'San Jose State University'
# 'fake' unique_method_name should not be too long
submission.unique_method_name = prefix
submission.description = 'mm3d'
submission.docker_image_source = "us-west1-docker.pkg.dev/cmpelkk/mycontainers/mymm3d@sha256:d2853bdc620a4e3aa434b1d22bbec8dfcedb09e6378978872d64dd9d1ab20d1b"
submission.method_link = "https://drive.google.com/drive/folders/1qy8FmmrQfCKjg0G6nKVjME6tmZnlLBTz?usp=sharing"
submission.sensor_type = submission_pb2.Submission.LIDAR_ALL #CAMERA_ALL
submission.number_past_frames_exclude_current = 0
submission.number_future_frames_exclude_current = 0
submission.inference_results.CopyFrom(objects)
f = open(outputsubmissionfilepath, 'wb') # output submission file
f.write(submission.SerializeToString())
f.close()
def create_pb_submission(detections, unique_method_name, description,
account_name, tracking):
submission = submission_pb2.Submission()
submission.task = submission_pb2.Submission.TRACKING_2D if tracking else submission_pb2.Submission.DETECTION_2D
submission.account_name = account_name
submission.authors.append('Yuan Xu')
submission.authors.append('Erdene-Ochir Tuguldur')
submission.affiliation = 'DAInamite'
submission.unique_method_name = unique_method_name
submission.description = description
submission.method_link = ""
submission.sensor_type = submission_pb2.Submission.CAMERA_ALL
submission.number_past_frames_exclude_current = 0
submission.number_future_frames_exclude_current = 0
objects = create_pd_objects(detections)
submission.inference_results.CopyFrom(objects)
# submission.object_types = [LABEL_TYPE.items()] # all types by default
# submission.latency_second = ?
return submission
def createsubmissionfromobject(objects, outputsubmissionfilepath, prefix):
submission = submission_pb2.Submission()
submission.task = submission_pb2.Submission.DETECTION_2D
submission.account_name = '*****@*****.**'
submission.authors.append('Kaikai Liu')
submission.affiliation = 'San Jose State University'
# 'fake' unique_method_name should not be too long
submission.unique_method_name = prefix
submission.docker_image_source = "us-west1-docker.pkg.dev/cmpelkk/mycontainers/mytftorchvision@sha256:354ae11bb8947bfc2295aabb789b54b98ea92100425a061084723a9f00b6a35b"
#Detectron2: "gcr.io/cmpelkk/mytftorch@sha256:629f60dd1b93c1750f909dd3064292f51d04db3b5cc041cf6fdfb26a49028c61"
submission.description = 'Torchvision'
submission.method_link = "https://drive.google.com/drive/folders/1qy8FmmrQfCKjg0G6nKVjME6tmZnlLBTz?usp=sharing"
submission.sensor_type = submission_pb2.Submission.CAMERA_ALL
submission.number_past_frames_exclude_current = 0
submission.number_future_frames_exclude_current = 0
submission.inference_results.CopyFrom(objects)
f = open(outputsubmissionfilepath, 'wb') # output submission file
f.write(submission.SerializeToString())
f.close()
def generatevalidationsubmission(PATH, outputfilepath, MODEL_DIR):
now = datetime.datetime.now()
print("In generatevalidationsubmission, current date and time : ")
print(now.strftime("%Y-%m-%d %H:%M:%S"))
tf.enable_eager_execution()
print(tf.__version__)
print(torch.cuda.is_available())
print(torch.cuda.device_count())
print(torch.cuda.get_device_name())
device = torch.device("cuda")
print("Loading Waymo validation frames...")
waymovalidationframes = loadWaymoValidationFrames(PATH)
#mywaymovaldataset = myNewWaymoDataset(PATH, waymovalidationframes, get_transform(train=False))
print("Total validation frames: ", len(waymovalidationframes))
num_classes = 4 #Unknown:0, Vehicles: 1, Pedestrians: 2, Cyclists: 3, Signs (removed)
# get the model using our helper function
print("Loading previous model: " + MODEL_DIR)
#model = get_previous_object_detection_model(num_classes, previous_model_path)
model = load_previous_object_detection_model(num_classes, MODEL_DIR)
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
# move model to the right device
model.to(device)
model.eval()
objects = metrics_pb2.Objects()
f = open(outputfilepath, 'wb') #waymovalidationframes, waymotestframes
outputallframes = waymovalidationframes
print("Total frames: ", len(outputallframes))
#step=5
for i in range(len(outputallframes)): #len(outputallframes)
if i % 10 == 0:
print("current frame: ", i)
frame = outputallframes[i]
image = tf.image.decode_jpeg(
frame.images[0].image).numpy() #front camera image
img = Image.fromarray(image)
boxes, pred_cls, scores = get_prediction(model, img, device,
score_threshold)
total_boxes = len(boxes)
if len(boxes) == 0:
continue
for i in range(total_boxes): #patch in pred_bbox:
label = pred_cls[i]
bbox = boxes[i]
score = scores[i]
o = metrics_pb2.Object()
o.context_name = frame.context.name
o.frame_timestamp_micros = int(frame.timestamp_micros)
o.camera_name = dataset_pb2.CameraName.FRONT
o.score = score
# Populating box and score.
box = label_pb2.Label.Box()
box.length = bbox[1][0] - bbox[0][0]
box.width = bbox[1][1] - bbox[0][1]
box.center_x = bbox[0][0] + box.length * 0.5
box.center_y = bbox[0][1] + box.width * 0.5
o.object.box.CopyFrom(box)
o.object.detection_difficulty_level = label_pb2.Label.LEVEL_1
o.object.num_lidar_points_in_box = 100
o.object.type = INSTANCE_pb2[
label] # INSTANCE_CATEGORY_NAMES.index(label) #INSTANCE_pb2[label]
print(
f'Object type label: {label}, {INSTANCE_pb2[label]}, {INSTANCE_CATEGORY_NAMES.index(label)}'
)
assert o.object.type != label_pb2.Label.TYPE_UNKNOWN
objects.objects.append(o)
submission = submission_pb2.Submission()
submission.task = submission_pb2.Submission.DETECTION_2D
submission.account_name = '*****@*****.**'
submission.authors.append('Kaikai Liu')
submission.affiliation = 'None'
submission.unique_method_name = 'torchvisionfaster'
submission.description = 'none'
submission.method_link = "empty method"
submission.sensor_type = submission_pb2.Submission.CAMERA_ALL
submission.number_past_frames_exclude_current = 0
submission.number_future_frames_exclude_current = 0
submission.inference_results.CopyFrom(objects)
f = open(outputfilepath, 'wb')
#f = open("./drive/My Drive/waymo_submission/waymo35.bin", 'wb')
f.write(submission.SerializeToString())
f.close()
now = datetime.datetime.now()
print("Finished validation, current date and time : ")
print(now.strftime("%Y-%m-%d %H:%M:%S"))
def evaluateWaymoValidationFramesAllcameraSubmission(PATH,
model_path,
config_path,
validation_folders,
outputsubmissionfilepath,
VisEnable,
imagesavepath,
nameprefix="output"):
data_files = [
path for x in validation_folders
for path in glob(os.path.join(PATH, x, "*.tfrecord"))
]
print(data_files) # all TFRecord file list
print(len(data_files))
# create a list of dataset for each TFRecord file
dataset = [
tf.data.TFRecordDataset(FILENAME, compression_type='')
for FILENAME in data_files
]
# total number of TFrecord files * 40 frame(each TFrecord)
wod_latency_submission.setupmodeldir(model_path, config_path)
wod_latency_submission.initialize_model()
objects = metrics_pb2.Objects() # submission objects
required_field = wod_latency_submission.DATA_FIELDS
print(required_field)
fps = FPS().start()
#allcameras=["FRONT_IMAGE", "FRONT_LEFT_IMAGE", "FRONT_RIGHT_IMAGE", "SIDE_LEFT_IMAGE", "SIDE_RIGHT_IMAGE"]
frameid = 0
testImagedict = {}
for i, data_file in enumerate(dataset):
print("Datafile: ", i) # Each TFrecord file
# Create frame based on Waymo API, 199 frames per TFrecord (20s, 10Hz)
for idx, data in enumerate(data_file):
# if idx % 5 != 0: #Downsample every 5 images, reduce to 2Hz, total around 40 frames
# continue
frame = open_dataset.Frame()
frame.ParseFromString(bytearray(data.numpy()))
#frame=frames[frameid]
convertedframesdict = convert_frame_to_dict_cameras(
frame) #data_array[frameid]
#Allconvertedframesdict.append(convertedframesdict)
frame_timestamp_micros = convertedframesdict['TIMESTAMP'] #['key']
context_name = frame.context.name
print(
f'Current frame id: {frameid}, idx: {idx}, context_name: {context_name}, frame_timestamp_micros: {frame_timestamp_micros}'
)
for imagename in allcameras: #go through all cameras
testImagedict[
required_field[0]] = convertedframesdict[imagename].copy()
start_time = time.perf_counter() #.time()
#result = wod_latency_submission.run_model(Front_image)
result = wod_latency_submission.run_model(
**testImagedict) #All images
end_time = time.perf_counter() #.time()
elapsed_time = end_time - start_time
print('Inference time: ' + str(elapsed_time * 1000) +
f'ms, camera name: {imagename}')
# createsubmisionobject(objects, result['boxes'], result['classes'],
# result['scores'], context_name, frame_timestamp_micros)
createsubmisioncameraobject(objects, result['boxes'],
result['classes'],
result['scores'], context_name,
frame_timestamp_micros,
cameraname_map[imagename])
if VisEnable == True:
display_str = f'Inference time: {str(elapsed_time*1000)}ms, camera name: {imagename}, context_name: {context_name}, timestamp_micros: {frame_timestamp_micros}'
savedetectedimagetofile(convertedframesdict[imagename],
frameid, result, imagename,
display_str, nameprefix,
imagesavepath)
fps.update()
frameid = frameid + 1
if frameid % 5000 == 0:
with open(
os.path.join(
imagesavepath,
nameprefix + '_' + str(frameid) + '.pickle'),
'wb') as f:
pickle.dump(objects, f)
# stop the timer and display FPS information
fps.stop()
print("[INFO] elasped time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
with open(os.path.join(imagesavepath, nameprefix + '.pickle'), 'wb') as f:
pickle.dump(objects, f)
# with open('allframedics.pickle', 'wb') as f:
# pickle.dump(Allconvertedframesdict, f)
submission = submission_pb2.Submission()
submission.task = submission_pb2.Submission.DETECTION_2D
submission.account_name = '*****@*****.**'
submission.authors.append('Kaikai Liu')
submission.affiliation = 'San Jose State University'
submission.unique_method_name = nameprefix #'fake'
submission.description = 'none'
submission.method_link = "empty method"
submission.sensor_type = submission_pb2.Submission.CAMERA_ALL
submission.number_past_frames_exclude_current = 0
submission.number_future_frames_exclude_current = 0
submission.inference_results.CopyFrom(objects)
f = open(outputsubmissionfilepath, 'wb') # output submission file
f.write(submission.SerializeToString())
f.close()
now = datetime.datetime.now()
print("Finished validation, current date and time : ")
print(now.strftime("%Y-%m-%d %H:%M:%S"))
def evaluateWaymoValidationFramesSubmission(PATH,
validation_folders,
outputsubmissionfilepath,
VisEnable,
outputfile="./output_video1.mp4"):
data_files = [
path for x in validation_folders
for path in glob(os.path.join(PATH, x, "*.tfrecord"))
]
print(data_files) # all TFRecord file list
print(len(data_files))
# create a list of dataset for each TFRecord file
dataset = [
tf.data.TFRecordDataset(FILENAME, compression_type='')
for FILENAME in data_files
]
# total number of TFrecord files * 40 frame(each TFrecord)
objects = metrics_pb2.Objects() # submission objects
wod_latency_submission.initialize_model()
required_field = wod_latency_submission.DATA_FIELDS
print(required_field)
if VisEnable == True:
frame_width = 1920
frame_height = 1280
out = cv2.VideoWriter(outputfile,
cv2.VideoWriter_fourcc('M', 'P', '4', 'V'), 5,
(frame_width, frame_height))
fps = FPS().start()
frameid = 0
for i, data_file in enumerate(dataset):
print("Datafile: ", i) # Each TFrecord file
# Create frame based on Waymo API, 199 frames per TFrecord (20s, 10Hz)
for idx, data in enumerate(data_file):
# if idx % 5 != 0: #Downsample every 5 images, reduce to 2Hz, total around 40 frames
# continue
frame = open_dataset.Frame()
frame.ParseFromString(bytearray(data.numpy()))
#frame=frames[frameid]
convertedframesdict = convert_frame_to_dict_cameras(
frame) #data_array[frameid]
#Allconvertedframesdict.append(convertedframesdict)
frame_timestamp_micros = convertedframesdict['TIMESTAMP'] #['key']
context_name = frame.context.name
print(
f'Current frame id: {frameid}, context_name: {context_name}, frame_timestamp_micros: {frame_timestamp_micros}'
)
start_time = time.perf_counter() #.time()
#result = wod_latency_submission.run_model(Front_image)
result = wod_latency_submission.run_model(
**convertedframesdict) #All images
end_time = time.perf_counter() #.time()
elapsed_time = end_time - start_time
print('Inference time: ' + str(elapsed_time) + 's')
# print(result)
createsubmisionobject(objects, result['boxes'], result['classes'],
result['scores'], context_name,
frame_timestamp_micros)
if VisEnable == True:
Front_image = convertedframesdict[required_field[0]]
#Front_image = convertedframesdict['FRONT_IMAGE']
image_np_with_detections = Front_image.copy()
visualization_util.visualize_boxes_and_labels_on_image_array(
image_np_with_detections,
result['boxes'],
result['classes'],
result['scores'],
category_index,
use_normalized_coordinates=False,
max_boxes_to_draw=200,
min_score_thresh=Threshold,
agnostic_mode=False)
display_str = f'Inference time: {str(elapsed_time*1000)}ms, context_name: {context_name}, timestamp_micros: {frame_timestamp_micros}'
visualization_util.draw_text_on_image(image_np_with_detections,
0,
0,
display_str,
color='black')
#visualization_util.save_image_array_as_png(image_np_with_detections, outputfile)
name = './Test_data/frame' + str(frameid) + '.jpg'
#print ('Creating\...' + name)
# cv2.imwrite(name, image_np_with_detections) #write to image folder
#out.write(image_np_with_detections)
out.write(
cv2.cvtColor(image_np_with_detections, cv2.COLOR_RGB2BGR))
#cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
frameid = frameid + 1
fps.update()
# stop the timer and display FPS information
fps.stop()
print("[INFO] elasped time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
if VisEnable == True:
out.release()
with open('objectsresult0525_detectron282k_valall_front.pickle',
'wb') as f:
pickle.dump(objects, f)
# with open('allframedics.pickle', 'wb') as f:
# pickle.dump(Allconvertedframesdict, f)
submission = submission_pb2.Submission()
submission.task = submission_pb2.Submission.DETECTION_2D
submission.account_name = '*****@*****.**'
submission.authors.append('Kaikai Liu')
submission.affiliation = 'None'
submission.unique_method_name = 'fake'
submission.description = 'none'
submission.method_link = "empty method"
submission.sensor_type = submission_pb2.Submission.CAMERA_ALL
submission.number_past_frames_exclude_current = 0
submission.number_future_frames_exclude_current = 0
submission.inference_results.CopyFrom(objects)
f = open(outputsubmissionfilepath, 'wb') # output submission file
f.write(submission.SerializeToString())
f.close()
now = datetime.datetime.now()
print("Finished validation, current date and time : ")
print(now.strftime("%Y-%m-%d %H:%M:%S"))
def evaluateWaymoValidationFramesFakeSubmission(
PATH,
validation_folders,
outputsubmissionfilepath,
outputfile="./output_video1.mp4"):
data_files = [
path for x in validation_folders
for path in glob(os.path.join(PATH, x, "*.tfrecord"))
]
print(data_files) # all TFRecord file list
print(len(data_files))
# create a list of dataset for each TFRecord file
dataset = [
tf.data.TFRecordDataset(FILENAME, compression_type='')
for FILENAME in data_files
]
# total number of TFrecord files * 40 frame(each TFrecord)
objects = metrics_pb2.Objects() # submission objects
frame_width = 1920
frame_height = 1280
out = cv2.VideoWriter(outputfile,
cv2.VideoWriter_fourcc('M', 'P', '4', 'V'), 5,
(frame_width, frame_height))
fps = FPS().start()
frameid = 0
for i, data_file in enumerate(dataset):
print("Datafile: ", i) # Each TFrecord file
# Create frame based on Waymo API, 199 frames per TFrecord (20s, 10Hz)
for idx, data in enumerate(data_file):
# if idx % 5 != 0: #Downsample every 5 images, reduce to 2Hz, total around 40 frames
# continue
frame = open_dataset.Frame()
frame.ParseFromString(bytearray(data.numpy()))
#frame=frames[frameid]
convertedframesdict = convert_frame_to_dict_cameras(
frame) #data_array[frameid]
#Allconvertedframesdict.append(convertedframesdict)
frame_timestamp_micros = convertedframesdict['TIMESTAMP'] #['key']
context_name = frame.context.name
print(
f'Current frame id: {frameid}, context_name: {context_name}, frame_timestamp_micros: {frame_timestamp_micros}'
)
o_list = []
boundingbox = []
boxscore = []
boxtype = []
for camera_labels in frame.camera_labels:
if camera_labels.name != 1: #Only use front camera
continue
for gt_label in camera_labels.labels:
o = metrics_pb2.Object()
# The following 3 fields are used to uniquely identify a frame a prediction
# is predicted at.
o.context_name = frame.context.name
# The frame timestamp for the prediction. See Frame::timestamp_micros in
# dataset.proto.
o.frame_timestamp_micros = frame.timestamp_micros
# This is only needed for 2D detection or tracking tasks.
# Set it to the camera name the prediction is for.
o.camera_name = camera_labels.name
# Populating box and score.
box = label_pb2.Label.Box()
box.center_x = gt_label.box.center_x
box.center_y = gt_label.box.center_y
box.length = gt_label.box.length
box.width = gt_label.box.width
boundingbox.append(
[box.center_x, box.center_y, box.length,
box.width]) #width height
o.object.box.CopyFrom(box)
# This must be within [0.0, 1.0]. It is better to filter those boxes with
# small scores to speed up metrics computation.
o.score = 0.9
boxscore.append(o.score)
# Use correct type.
o.object.type = gt_label.type
boxtype.append(o.object.type)
o_list.append(o)
print(
f'Camera labelname: {camera_labels.name}, object type: { gt_label.type}, box:{box}'
)
for o in o_list:
objects.objects.append(o)
# Save the original image
#output_path = "./test.png"
#visualization_util.save_image_array_as_png(Front_image, output_path)
boundingbox = np.array(boundingbox)
boxscore = np.array(boxscore)
boxtype = np.array(boxtype).astype(np.uint8)
Front_image = convertedframesdict['FRONT_IMAGE']
image_np_with_detections = Front_image.copy()
visualization_util.visualize_boxes_and_labels_on_image_array(
image_np_with_detections,
boundingbox,
boxtype,
boxscore,
category_index,
use_normalized_coordinates=False,
max_boxes_to_draw=200,
min_score_thresh=Threshold,
agnostic_mode=False)
display_str = f'context_name: {context_name}, timestamp_micros: {frame_timestamp_micros}'
visualization_util.draw_text_on_image(image_np_with_detections,
0,
0,
display_str,
color='black')
#visualization_util.save_image_array_as_png(image_np_with_detections, outputfile)
name = './Test_data/frame' + str(frameid) + '.jpg'
#print ('Creating\...' + name)
# cv2.imwrite(name, image_np_with_detections) #write to image folder
fps.update()
#out.write(image_np_with_detections)
out.write(cv2.cvtColor(image_np_with_detections,
cv2.COLOR_RGB2BGR))
#cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
frameid = frameid + 1
# stop the timer and display FPS information
fps.stop()
print("[INFO] elasped time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
out.release()
with open('objectsresult_gtvalall.pickle', 'wb') as f:
pickle.dump(objects, f)
# with open('allframedics.pickle', 'wb') as f:
# pickle.dump(Allconvertedframesdict, f)
submission = submission_pb2.Submission()
submission.task = submission_pb2.Submission.DETECTION_2D
submission.account_name = '*****@*****.**'
submission.authors.append('Kaikai Liu')
submission.affiliation = 'None'
submission.unique_method_name = 'fake'
submission.description = 'none'
submission.method_link = "empty method"
submission.sensor_type = submission_pb2.Submission.CAMERA_ALL
submission.number_past_frames_exclude_current = 0
submission.number_future_frames_exclude_current = 0
submission.inference_results.CopyFrom(objects)
f = open(outputsubmissionfilepath, 'wb') # output submission file
f.write(submission.SerializeToString())
f.close()
now = datetime.datetime.now()
print("Finished validation, current date and time : ")
print(now.strftime("%Y-%m-%d %H:%M:%S"))
def evaluateallframesgtfakesubmission(frames,
outputsubmissionfilepath,
outputfile="./output_video1.mp4"):
array_len = len(frames) #4931 frames for validation_0000
# 20, 200 frames in one file, downsample by 10
print("Frames lenth:", array_len)
print("Final_array type:", type(frames)) # class 'list'
objects = metrics_pb2.Objects() # submission objects
frame_width = 1920
frame_height = 1280
out = cv2.VideoWriter(outputfile,
cv2.VideoWriter_fourcc('M', 'P', '4', 'V'), 5,
(frame_width, frame_height))
fps = FPS().start()
#wod_latency_submission.initialize_model()
#required_field = wod_latency_submission.DATA_FIELDS
#print(required_field)
#Allconvertedframesdict=[]
for frameid in range(array_len):
#frameid = 5
print("frameid:", frameid)
# {'key':key, 'context_name':context_name, 'framedict':framedict}
frame = frames[frameid]
convertedframesdict = convert_frame_to_dict_cameras(
frame) #data_array[frameid]
#Allconvertedframesdict.append(convertedframesdict)
frame_timestamp_micros = convertedframesdict['TIMESTAMP'] #['key']
context_name = frame.context.name
o_list = []
boundingbox = []
boxscore = []
boxtype = []
for camera_labels in frame.camera_labels:
if camera_labels.name != 1: #Only use front camera
continue
for gt_label in camera_labels.labels:
o = metrics_pb2.Object()
# The following 3 fields are used to uniquely identify a frame a prediction
# is predicted at.
o.context_name = frame.context.name
# The frame timestamp for the prediction. See Frame::timestamp_micros in
# dataset.proto.
o.frame_timestamp_micros = frame.timestamp_micros
# This is only needed for 2D detection or tracking tasks.
# Set it to the camera name the prediction is for.
o.camera_name = camera_labels.name
# Populating box and score.
box = label_pb2.Label.Box()
box.center_x = gt_label.box.center_x
box.center_y = gt_label.box.center_y
box.length = gt_label.box.length
box.width = gt_label.box.width
boundingbox.append(
[box.center_x, box.center_y, box.length,
box.width]) #width height
o.object.box.CopyFrom(box)
# This must be within [0.0, 1.0]. It is better to filter those boxes with
# small scores to speed up metrics computation.
o.score = 0.9
boxscore.append(o.score)
# Use correct type.
o.object.type = gt_label.type
boxtype.append(o.object.type)
o_list.append(o)
print(
f'Camera labelname: {camera_labels.name}, object type: { gt_label.type}, box:{box}'
)
# Save the original image
#output_path = "./test.png"
#visualization_util.save_image_array_as_png(Front_image, output_path)
boundingbox = np.array(boundingbox)
boxscore = np.array(boxscore)
boxtype = np.array(boxtype).astype(np.uint8)
Front_image = convertedframesdict['FRONT_IMAGE']
image_np_with_detections = Front_image.copy()
visualization_util.visualize_boxes_and_labels_on_image_array(
image_np_with_detections,
boundingbox,
boxtype,
boxscore,
category_index,
use_normalized_coordinates=False,
max_boxes_to_draw=200,
min_score_thresh=Threshold,
agnostic_mode=False)
display_str = f'context_name: {context_name}, timestamp_micros: {frame_timestamp_micros}'
visualization_util.draw_text_on_image(image_np_with_detections,
0,
0,
display_str,
color='black')
#visualization_util.save_image_array_as_png(image_np_with_detections, outputfile)
name = './Test_data/frame' + str(frameid) + '.jpg'
#print ('Creating\...' + name)
# cv2.imwrite(name, image_np_with_detections) #write to image folder
fps.update()
#out.write(image_np_with_detections)
out.write(cv2.cvtColor(image_np_with_detections, cv2.COLOR_RGB2BGR))
#cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
# stop the timer and display FPS information
fps.stop()
print("[INFO] elasped time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
out.release()
with open('objectsresult_gtvalall.pickle', 'wb') as f:
pickle.dump(objects, f)
# with open('allframedics.pickle', 'wb') as f:
# pickle.dump(Allconvertedframesdict, f)
submission = submission_pb2.Submission()
submission.task = submission_pb2.Submission.DETECTION_2D
submission.account_name = '*****@*****.**'
submission.authors.append('Kaikai Liu')
submission.affiliation = 'None'
submission.unique_method_name = 'fake'
submission.description = 'none'
submission.method_link = "empty method"
submission.sensor_type = submission_pb2.Submission.CAMERA_ALL
submission.number_past_frames_exclude_current = 0
submission.number_future_frames_exclude_current = 0
submission.inference_results.CopyFrom(objects)
f = open(outputsubmissionfilepath, 'wb') # output submission file
f.write(submission.SerializeToString())
f.close()
now = datetime.datetime.now()
print("Finished validation, current date and time : ")
print(now.strftime("%Y-%m-%d %H:%M:%S"))
for i, data_file in enumerate(dataset):
print("Datafile: ", i) #Each TFrecord file
for idx, data in enumerate(
data_file
): #Create frame based on Waymo API, 199 frames per TFrecord (20s, 10Hz)
# if idx % 5 != 0: #Downsample every 5 images, reduce to 2Hz, total around 40 frames
# continue
frame = dataset_pb2.Frame()
frame.ParseFromString(bytearray(data.numpy()))
o_list = _fancy_deep_learning(frame)
frames.append(frame)
for o in o_list:
objects.objects.append(o)
#https://github.com/waymo-research/waymo-open-dataset/blob/master/waymo_open_dataset/protos/submission.proto
submission = submission_pb2.Submission()
submission.task = submission_pb2.Submission.DETECTION_2D
submission.account_name = '*****@*****.**'
submission.authors.append('Kaikai Liu')
submission.affiliation = 'None'
submission.unique_method_name = 'fake'
submission.description = 'none'
submission.method_link = "empty method"
submission.sensor_type = submission_pb2.Submission.CAMERA_ALL
submission.number_past_frames_exclude_current = 0
submission.number_future_frames_exclude_current = 0
submission.inference_results.CopyFrom(objects)
submission.docker_image_source = '' #// Link to the latency submission Docker image stored in Google Storage bucket
#object_types // Object types this submission contains. By default, we assume all types.
#latency_second Self-reported end to end inference latency in seconds
def evaluateallframescreatesubmission(frames, outputsubmissionfilepath, outputfile="./output_video1.mp4"):
array_len = len(frames) #4931 frames for validation_0000
# 20, 200 frames in one file, downsample by 10
print("Frames lenth:", array_len)
print("Final_array type:", type(frames)) # class 'list'
objects = metrics_pb2.Objects() # submission objects
frame_width = 1920
frame_height = 1280
out = cv2.VideoWriter(outputfile, cv2.VideoWriter_fourcc(
'M', 'P', '4', 'V'), 5, (frame_width, frame_height))
fps = FPS().start()
wod_latency_submission.initialize_model()
required_field = wod_latency_submission.DATA_FIELDS
print(required_field)
for frameid in range(array_len):
#frameid = 5
print("frameid:", frameid)
# {'key':key, 'context_name':context_name, 'framedict':framedict}
currentframe=frames[frameid]
convertedframesdict = convert_frame_to_dict_cameras(currentframe) #data_array[frameid]
frame_timestamp_micros = convertedframesdict['TIMESTAMP']#['key']
context_name = currentframe.context.name #convertedframesdict['context_name']
#framedict = convertedframesdict['framedict']
# 10017090168044687777_6380_000_6400_000
#print('context_name:', context_name)
# print('frame_timestamp_micros:', frame_timestamp_micros) # 1550083467346370
#result = wod_latency_submission.run_model(framedict[required_field[0]], framedict[required_field[1]])
#result = wod_latency_submission.run_model(**framedict)
#Front_image = framedict[required_field[0]]
start_time = time.time()
#result = wod_latency_submission.run_model(Front_image)
result = wod_latency_submission.run_model(**convertedframesdict)#All images
end_time = time.time()
elapsed_time = end_time - start_time
print('Inference time: ' + str(elapsed_time) + 's')
# print(result)
createsubmisionobject(objects, result['boxes'], result['classes'],
result['scores'], context_name, frame_timestamp_micros)
# Save the original image
#output_path = "./test.png"
#visualization_util.save_image_array_as_png(Front_image, output_path)
Front_image = convertedframesdict[required_field[0]]
image_np_with_detections = Front_image.copy()
visualization_util.visualize_boxes_and_labels_on_image_array(image_np_with_detections, result['boxes'], result['classes'], result['scores'], category_index, use_normalized_coordinates=False,
max_boxes_to_draw=200,
min_score_thresh=Threshold,
agnostic_mode=False)
display_str=f'Inference time: {str(elapsed_time*1000)}ms, context_name: {context_name}, timestamp_micros: {frame_timestamp_micros}'
visualization_util.draw_text_on_image(image_np_with_detections, 0, 0, display_str, color='black')
#visualization_util.save_image_array_as_png(image_np_with_detections, outputfile)
name = './Test_data/frame' + str(frameid) + '.jpg'
#print ('Creating\...' + name)
# cv2.imwrite(name, image_np_with_detections) #write to image folder
fps.update()
#out.write(image_np_with_detections)
out.write(cv2.cvtColor(image_np_with_detections, cv2.COLOR_RGB2BGR))
#cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
# stop the timer and display FPS information
fps.stop()
print("[INFO] elasped time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
out.release()
with open('objectsresult.pickle', 'wb') as f:
pickle.dump(objects, f)
submission = submission_pb2.Submission()
submission.task = submission_pb2.Submission.DETECTION_2D
submission.account_name = '*****@*****.**'
submission.authors.append('Kaikai Liu')
submission.affiliation = 'None'
submission.unique_method_name = 'torchvisionfaster'
submission.description = 'none'
submission.method_link = "empty method"
submission.sensor_type = submission_pb2.Submission.CAMERA_ALL
submission.number_past_frames_exclude_current = 0
submission.number_future_frames_exclude_current = 0
submission.inference_results.CopyFrom(objects)
f = open(outputsubmissionfilepath, 'wb') # output submission file
f.write(submission.SerializeToString())
f.close()
now = datetime.datetime.now()
print("Finished validation, current date and time : ")
print(now.strftime("%Y-%m-%d %H:%M:%S"))