def preprocess_data(video_input_path, flow_video_output_path, image_folder_path, flow_image_folder_path, type):
if os.path.exists(image_folder_path):
shutil.rmtree(image_folder_path)
os.makedirs(image_folder_path)
if os.path.exists(flow_image_folder_path):
shutil.rmtree(flow_image_folder_path)
os.makedirs(flow_image_folder_path)
print("Converting video to optical flow for: ", video_input_path)
video_reader = cv2.VideoCapture(video_input_path)
num_frames = video_reader.get(cv2.CAP_PROP_FRAME_COUNT)
frame_size = (int(video_reader.get(cv2.CAP_PROP_FRAME_WIDTH)), int(video_reader.get(cv2.CAP_PROP_FRAME_HEIGHT)))
fps = int(video_reader.get(cv2.CAP_PROP_FPS))
# fourcc = cv2.VideoWriter_fourcc(*'XVID')
fourcc = 0x00000021
video_writer = cv2.VideoWriter(flow_video_output_path, fourcc, fps, frame_size)
t1 = time.time()
ret, prev_frame = video_reader.read()
hsv = np.zeros_like(prev_frame)
image_path_out = os.path.join(image_folder_path, str(0) + '.jpg')
cv2.imwrite(image_path_out, prev_frame)
count = 1
while True:
ret, next_frame = video_reader.read()
if next_frame is None:
break
bgr_flow = convertToOptical(prev_frame, next_frame)
image_path_out = os.path.join(image_folder_path, str(count) + '.jpg')
flow_image_path_out = os.path.join(flow_image_folder_path, str(count) + '.jpg')
cv2.imwrite(image_path_out, next_frame)
cv2.imwrite(flow_image_path_out, bgr_flow)
video_writer.write(bgr_flow)
prev_frame = next_frame
count += 1
t2 = time.time()
video_reader.release()
video_writer.release()
print(' Conversion completed !')
print(' Time Taken:', (t2 - t1), 'seconds')
return
def generatorData(samples, batch_size=32, type=TYPE_FLOW_PRECOMPUTED):
num_samples = len(samples)
while 1: # Loop forever so the generator never terminates
samples = sklearn.utils.shuffle(samples)
for offset in range(0, num_samples, batch_size):
batch_samples = samples[offset:offset + batch_size]
images = []
angles = []
for imagePath, measurement in batch_samples:
combined_image = None
flow_image_bgr = None
if type == TYPE_FLOW_PRECOMPUTED:
# curr_image_path, flow_image_path = imagePath
# flow_image_bgr = cv2.imread(flow_image_path)
curr_image_path, flow_image_path1, flow_image_path2, flow_image_path3, flow_image_path4 = imagePath
flow_image_bgr = (cv2.imread(flow_image_path1) +
cv2.imread(flow_image_path2) +
cv2.imread(flow_image_path3) +
cv2.imread(flow_image_path4)) / 4
curr_image = cv2.imread(curr_image_path)
curr_image = cv2.cvtColor(curr_image, cv2.COLOR_BGR2RGB)
else:
prev_image_path, curr_image_path = imagePath
prev_image = cv2.imread(prev_image_path)
curr_image = cv2.imread(curr_image_path)
flow_image_bgr = convertToOptical(prev_image, curr_image)
curr_image = cv2.cvtColor(curr_image, cv2.COLOR_BGR2RGB)
combined_image = 0.1 * curr_image + flow_image_bgr
#CHOOSE IF WE WANT TO TEST WITH ONLY OPTICAL FLOW OR A COMBINATION OF VIDEO AND OPTICAL FLOW
combined_image = flow_image_bgr
combined_image = cv2.normalize(combined_image,
None,
alpha=-1,
beta=1,
norm_type=cv2.NORM_MINMAX,
dtype=cv2.CV_32F)
combined_image = cv2.resize(combined_image, (0, 0),
fx=0.5,
fy=0.5)
# im = Image.fromarray(combined_image)
# plt.imshow(im)
# plt.show()
images.append(combined_image)
angles.append(measurement)
# AUGMENTING DATA
# Flipping image, correcting measurement and measuerement
images.append(cv2.flip(combined_image, 1))
angles.append(measurement)
inputs = np.array(images)
outputs = np.array(angles)
yield sklearn.utils.shuffle(inputs, outputs)
def preprocess_data(video_input_path, flow_video_output_path,
image_folder_path, flow_image_folder_path, type):
if os.path.exists(image_folder_path):
shutil.rmtree(image_folder_path)
os.makedirs(image_folder_path)
if os.path.exists(flow_image_folder_path):
shutil.rmtree(flow_image_folder_path)
os.makedirs(flow_image_folder_path)
print("Converting video to optical flow for: ", video_input_path)
video_reader = cv2.VideoCapture(video_input_path)
num_frames = video_reader.get(cv2.CAP_PROP_FRAME_COUNT)
frame_size = (int(video_reader.get(cv2.CAP_PROP_FRAME_WIDTH)),
int(video_reader.get(cv2.CAP_PROP_FRAME_HEIGHT)))
fps = int(video_reader.get(cv2.CAP_PROP_FPS))
# fourcc = cv2.VideoWriter_fourcc(*'XVID')
fourcc = 0x00000021
video_writer = cv2.VideoWriter(flow_video_output_path, fourcc, fps,
frame_size)
t1 = time.time()
ret, prev_frame = video_reader.read()
hsv = np.zeros_like(prev_frame)
image_path_out = os.path.join(image_folder_path, str(0) + '.jpg')
cv2.imwrite(image_path_out, prev_frame)
count = 1
while True:
ret, next_frame = video_reader.read()
if next_frame is None:
break
bgr_flow = convertToOptical(prev_frame, next_frame)
image_path_out = os.path.join(image_folder_path, str(count) + '.jpg')
flow_image_path_out = os.path.join(flow_image_folder_path,
str(count) + '.jpg')
cv2.imwrite(image_path_out, next_frame)
cv2.imwrite(flow_image_path_out, bgr_flow)
video_writer.write(bgr_flow)
prev_frame = next_frame
count += 1
'''FOR FLIP PREPROCESSING, CURRENTLY HANDLED IN GENERATE DATA IN TRAIN'''
# if type == 'train':
# image_flip = cv2.flip( next_frame, 1 )
# bgr_flow_flip = cv2.flip( bgr_flow, 1 )
#
# image_path_out_flip = os.path.join(image_folder_path, str(count) + '.jpg')
# flow_image_path_out_flip = os.path.join(flow_image_folder_path, str(count) + '.jpg')
#
# cv2.imwrite(image_path_out_flip, image_flip)
# cv2.imwrite(flow_image_path_out_flip, bgr_flow_flip)
#
# sys.stdout.write('\rprocessed frames: %d of %d' % (count//2, num_frames))
# count += 1
# else:
# sys.stdout.write('\rprocessed frames: %d of %d' % (count, num_frames))
t2 = time.time()
video_reader.release()
video_writer.release()
print(' Conversion completed !')
print(' Time Taken:', (t2 - t1), 'seconds')
'''FOR FLIP PREPROCESSING, CURRENTLY HANDLED IN GENERATE DATA IN TRAIN'''
# if type == 'train':
# file_r = open(PATH_TRAIN_LABEL, 'r')
#
# if os.path.exists(PATH_TRAIN_LABEL_PREPROCESSED):
# os.remove(PATH_TRAIN_LABEL_PREPROCESSED)
# file = open(PATH_TRAIN_LABEL_PREPROCESSED, 'w')
#
# speed_list = file_r.read().split()
# for i in range(len(speed_list)-1):
# speed= speed_list[i] + speed_list[i+1]/2
# file.write(speed+ '\n')
# file.write(speed + '\n')
#
# file_r.close()
# file.close()
#
# print(' New labels written !')
return
def generatorData(samples, batch_size=32, type=TYPE_FLOW_PRECOMPUTED):
num_samples = len(samples)
while 1: # Loop forever so the generator never terminates
samples = sklearn.utils.shuffle(samples)
for offset in range(0, num_samples, batch_size):
batch_samples = samples[offset:offset+batch_size]
images = []
angles = []
for imagePath, measurement in batch_samples:
combined_image = None
flow_image_bgr = None
if type == 'augmentation':
# curr_image_path, flow_image_path = imagePath
# flow_image_bgr = cv2.imread(flow_image_path)
curr_image_path, flow_image_path1, flow_image_path2,flow_image_path3, flow_image_path4 = imagePath
try:
im1=cv2.imread(flow_image_path1)
except:
print("An exception occurred :" ,flow_image_path1)
# im2=cv2.imread(flow_image_path2)
# im3=cv2.imread(flow_image_path3)
#im4=cv2.imread(flow_image_path4)
#flow_image_bgr = random_augment((cv2.imread(flow_image_path1) )+random_augment(cv2.imread(flow_image_path2)) +random_augment(cv2.imread(flow_image_path3)) +random_augment(cv2.imread(flow_image_path4) ))/4
try:
flow_image_bgr=random_augment(im1)
except:
print("An exception occurred :", flow_image_path1)
curr_image = cv2.imread(curr_image_path)
curr_image = cv2.cvtColor(curr_image, cv2.COLOR_BGR2RGB)
elif type == 'no-augmentation':
# curr_image_path, flow_image_path = imagePath
# flow_image_bgr = cv2.imread(flow_image_path)
curr_image_path, flow_image_path1, flow_image_path2,flow_image_path3, flow_image_path4 = imagePath
flow_image_bgr = (cv2.imread(flow_image_path1) +cv2.imread(flow_image_path2) +cv2.imread(flow_image_path3) +cv2.imread(flow_image_path4) )/4
curr_image = cv2.imread(curr_image_path)
curr_image = cv2.cvtColor(curr_image, cv2.COLOR_BGR2RGB)
else:
prev_image_path, curr_image_path = imagePath
prev_image = cv2.imread(prev_image_path)
curr_image = cv2.imread(curr_image_path)
flow_image_bgr = convertToOptical(prev_image, curr_image)
curr_image = cv2.cvtColor(curr_image, cv2.COLOR_BGR2RGB)
#combined_image = 0.1*curr_image + flow_image_bgr
#CHOOSE IF WE WANT TO TEST WITH ONLY OPTICAL FLOW OR A COMBINATION OF VIDEO AND OPTICAL FLOW
#combined_image = flow_image_bgr[500:1050, 0:850]
combined_image=flow_image_bgr
combined_image=cv2.resize(combined_image, (640,480))
combined_image = cv2.normalize(combined_image, None, alpha=-1, beta=1, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F)
combined_image = cv2.resize(combined_image, (0,0), fx=0.5, fy=0.5)
images.append(combined_image)
angles.append(measurement)
images.append(combined_image)
angles.append(measurement)
inputs = np.array(images)
outputs = np.array(angles)
yield sklearn.utils.shuffle(inputs, outputs)
def predict_from_video(video_input_path, original_video_output_path,
combined_video_output_path):
predicted_labels = []
video_reader = cv2.VideoCapture(video_input_path)
num_frames = video_reader.get(cv2.CAP_PROP_FRAME_COUNT)
frame_size = (int(video_reader.get(cv2.CAP_PROP_FRAME_WIDTH)),
int(video_reader.get(cv2.CAP_PROP_FRAME_HEIGHT)))
fps = int(video_reader.get(cv2.CAP_PROP_FPS))
# fourcc = cv2.VideoWriter_fourcc(*'XVID')
fourcc = 0x00000021
video_writer = cv2.VideoWriter(original_video_output_path, fourcc, fps,
frame_size)
video_writer_combined = cv2.VideoWriter(combined_video_output_path, fourcc,
fps, frame_size)
t1 = time.time()
ret, prev_frame = video_reader.read()
hsv = np.zeros_like(prev_frame)
video_writer.write(prev_frame)
predicted_labels.append(0.0)
flow_image_bgr_prev1 = np.zeros_like(prev_frame)
flow_image_bgr_prev2 = np.zeros_like(prev_frame)
flow_image_bgr_prev3 = np.zeros_like(prev_frame)
flow_image_bgr_prev4 = np.zeros_like(prev_frame)
font = cv2.FONT_HERSHEY_SIMPLEX
place = (50, 50)
fontScale = 1
fontColor = (255, 255, 255)
lineType = 2
count = 0
while True:
ret, next_frame = video_reader.read()
if ret is False:
break
flow_image_bgr_next = convertToOptical(prev_frame, next_frame)
flow_image_bgr = (flow_image_bgr_prev1 + flow_image_bgr_prev2 +
flow_image_bgr_prev3 + flow_image_bgr_prev4 +
flow_image_bgr_next) / 4
curr_image = cv2.cvtColor(next_frame, cv2.COLOR_BGR2RGB)
combined_image_save = 0.1 * curr_image + flow_image_bgr
#CHOOSE IF WE WANT TO TEST WITH ONLY OPTICAL FLOW OR A COMBINATION OF VIDEO AND OPTICAL FLOW
combined_image = flow_image_bgr
# combined_image = combined_image_save
combined_image_test = cv2.normalize(combined_image,
None,
alpha=-1,
beta=1,
norm_type=cv2.NORM_MINMAX,
dtype=cv2.CV_32F)
# plt.imshow(combined_image)
# plt.show()
#CHOOSE IF WE WANT TO TEST WITH ONLY OPTICAL FLOW OR A COMBINATION OF VIDEO AND OPTICAL FLOW
# combined_image_test = cv2.resize(combined_image, (0,0), fx=0.5, fy=0.5)
combined_image_test = cv2.resize(combined_image_test, (0, 0),
fx=0.5,
fy=0.5)
combined_image_test = combined_image_test.reshape(
1, combined_image_test.shape[0], combined_image_test.shape[1],
combined_image_test.shape[2])
prediction = model.predict(combined_image_test)
predicted_labels.append(prediction[0][0])
# print(combined_image.shape, np.mean(flow_image_bgr), prediction[0][0])
cv2.putText(next_frame, str(prediction[0][0]), place, font, fontScale,
fontColor, lineType)
cv2.putText(combined_image_save, str(prediction[0][0]), place, font,
fontScale, fontColor, lineType)
video_writer.write(next_frame)
video_writer_combined.write(combined_image_save.astype('uint8'))
prev_frame = next_frame
flow_image_bgr_prev4 = flow_image_bgr_prev3
flow_image_bgr_prev3 = flow_image_bgr_prev2
flow_image_bgr_prev2 = flow_image_bgr_prev1
flow_image_bgr_prev1 = flow_image_bgr_next
count += 1
sys.stdout.write('\rprocessed frames: %d of %d' % (count, num_frames))
t2 = time.time()
video_reader.release()
video_writer.release()
video_writer_combined.release()
print(' Prediction completed !')
print(' Time Taken:', (t2 - t1), 'seconds')
predicted_labels[0] = predicted_labels[1]
return predicted_labels
def predict_from_video(video_input_path, Ground_truth,
original_video_output_path, combined_video_output_path):
predicted_labels = []
video_reader = cv2.VideoCapture(video_input_path)
num_frames = video_reader.get(cv2.CAP_PROP_FRAME_COUNT)
frame_size = (int(video_reader.get(cv2.CAP_PROP_FRAME_WIDTH)),
int(video_reader.get(cv2.CAP_PROP_FRAME_HEIGHT)))
fps = int(video_reader.get(cv2.CAP_PROP_FPS))
# fourcc = cv2.VideoWriter_fourcc(*'XVID')
fourcc = 0x00000021
video_writer = cv2.VideoWriter(original_video_output_path, fourcc, fps,
frame_size)
video_writer_combined = cv2.VideoWriter(combined_video_output_path, fourcc,
fps, frame_size)
t1 = time.time()
ret, prev_frame = video_reader.read()
prev_frame = cv2.resize(prev_frame, (640, 480))
# prev_frame = cv2.resize(prev_frame, (640, 480))
hsv = np.zeros_like(prev_frame)
video_writer.write(prev_frame)
predicted_labels.append(0.0)
flow_image_bgr_prev1 = np.zeros_like(prev_frame)
#flow_image_bgr_prev2 = np.zeros_like(prev_frame)
# flow_image_bgr_prev3 = np.zeros_like(prev_frame)
#flow_image_bgr_prev4 = np.zeros_like(prev_frame)
prediction_output_file = file1 = open(PATH_TEST_LABEL, "a+")
font = cv2.FONT_HERSHEY_SIMPLEX
place = (300, 50)
place2 = (300, 75)
place3 = (50, 50)
place4 = (50, 75)
fontScale = 1
fontColor = (255, 255, 255)
lineType = 2
count = 0
while True:
ret, next_frame = video_reader.read()
if ret is False:
break
t_start = time.time()
next_frame = cv2.resize(next_frame, (640, 480))
#next_frame = cv2.resize(next_frame, (640, 480))
flow_image_bgr_next = convertToOptical(prev_frame, next_frame)
t_after_optixcal_conversion = time.time()
#flow_image_bgr = (flow_image_bgr_prev1 + flow_image_bgr_prev2 +flow_image_bgr_prev3 +flow_image_bgr_prev4 + flow_image_bgr_next)/4
flow_image_bgr = flow_image_bgr_prev1
curr_image = cv2.cvtColor(next_frame, cv2.COLOR_BGR2RGB)
t_after_color_conversion = time.time()
#combined_image_save = 0.1*curr_image + flow_image_bgr
#CHOOSE IF WE WANT TO TEST WITH ONLY OPTICAL FLOW OR A COMBINATION OF VIDEO AND OPTICAL FLOW
combined_image = flow_image_bgr
#combined_image = flow_image_bgr[500:1050, 0:850]
t_recize = time.time()
# combined_image = combined_image_save
combined_image_test = cv2.normalize(combined_image,
None,
alpha=-1,
beta=1,
norm_type=cv2.NORM_MINMAX,
dtype=cv2.CV_32F)
t_normalise = time.time()
# plt.imshow(combined_image)
# plt.show()
#CHOOSE IF WE WANT TO TEST WITH ONLY OPTICAL FLOW OR A COMBINATION OF VIDEO AND OPTICAL FLOW
# combined_image_test = cv2.resize(combined_image, (0,0), fx=0.5, fy=0.5)
combined_image_test = cv2.resize(combined_image_test, (0, 0),
fx=0.5,
fy=0.5)
combined_image_test = combined_image_test.reshape(
1, combined_image_test.shape[0], combined_image_test.shape[1],
combined_image_test.shape[2])
t_reshapeandrecize = time.time()
prediction = model.predict(combined_image_test)
t_predict = time.time()
predicted_labels.append(prediction[0][0])
truth = float(Ground_truth[count])
file1.writelines(str(prediction[0][0]) + '\n')
# print(combined_image.shape, np.mean(flow_image_bgr), prediction[0][0])
cv2.putText(next_frame, str(int(prediction[0][0] - 2)), place, font,
fontScale, (0, 0, 255), lineType)
cv2.putText(next_frame, str(int(truth)), place2, font, fontScale,
(0, 255, 0), lineType)
cv2.putText(next_frame, 'prediction : ', place3, font, fontScale,
(0, 0, 255), lineType)
cv2.putText(next_frame, 'Ground truth : ', place4, font, fontScale,
(0, 255, 0), lineType)
#cv2.putText(combined_image_save, str(prediction[0][0]), place, font, fontScale,fontColor,lineType)
cv2.imshow('frame', next_frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
video_writer.write(next_frame)
# video_writer_combined.write(combined_image_save.astype('uint8'))
prev_frame = next_frame
#flow_image_bgr_prev4 = flow_image_bgr_prev3
#flow_image_bgr_prev3 = flow_image_bgr_prev2
#flow_image_bgr_prev2 = flow_image_bgr_prev1
flow_image_bgr_prev1 = flow_image_bgr_next
count += 1
sys.stdout.write('\rprocessed frames: %d of %d' % (count, num_frames))
t_end = time.time()
print('total time : ', t_end - t_start)
print('optical flow conversion time: ',
t_after_optixcal_conversion - t_start)
print('color conversion time : ',
t_after_color_conversion - t_after_optixcal_conversion)
print('recize time : ', t_recize - t_after_color_conversion)
print(' normalize time : ', t_normalise - t_recize)
print(' reshape time : ', t_recize - t_reshapeandrecize)
print(' predict time : ', t_predict - t_reshapeandrecize)
t2 = time.time()
video_reader.release()
video_writer.release()
video_writer_combined.release()
file1.close()
print(' Prediction completed !')
print(' Time Taken:', (t2 - t1), 'seconds')
predicted_labels[0] = predicted_labels[1]
return predicted_labels