def run_yolo(split_command, path, idx, num_images, json_struct):
pipe = Popen(split_command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
output, err = pipe.communicate(path)
# print output
object_label_probs = output.split('\n')
predict_message = object_label_probs[0]
time_for_prediction = predict_message.split(' ')[-2]
object_label_probs = object_label_probs[1:-1]
for i, s in enumerate(object_label_probs):
split = s.split(':')
object_label_probs[i] = {'class': split[0], 'score': split[1][1:]}
log("Processed image {}/{} object detection in {}s".format(idx, num_images, time_for_prediction))
# print ("Processed image {}/{} object detection in {}s".format(idx, num_images, time_for_prediction))
if object_label_probs:
log('Objects detected: ', object_label_probs)
# TODO open predictions.png save in section other than full images!!!!
mutex.acquire()
try:
json_struct['images'][idx]['object_lists']['yolo_20'] = object_label_probs
finally:
mutex.release()
def main_separate_scenes_and_extract_frames(json_struct, video_path, video_url, verbose=True, multiplier=1.0):
start = timer()
directory = os.path.dirname(video_path)
name = video_path.split('/')[-1][:-4] # todo watch out for .4 letters at end
log('Main function. Separating video into scenes:', name, header=HEADER_SIZE - 1, color='darkturquoise')
json_struct['info'] = getInfo(video_path)
json_struct['info']['name'] = name
json_struct['info']['INITIAL_NUM_FRAMES_IN_SCENE'] = 5
json_struct['info']['multiplier'] = multiplier
json_struct['info']['url'] = video_url
makeOutputDirs(directory)
# todo calculateFrameStats could still be useful
process_video(video_path, directory, name, json_struct, verbose)
json_path = os.path.join(directory, 'metadata', 'result_struct.json')
json.dump(json_struct, open(json_path, 'w'), indent=4)
#todo log info one at a time!!!!!!!!
log("Video Info: ", json_struct['info'], color='green', header=HEADER_SIZE)
log("Video scene and frame extraction complete.", color='green', header=HEADER_SIZE)
log("Video separated into ", json_struct['info']['num_scenes'], " scenes.", color='green', header=HEADER_SIZE)
end = timer()
log('Time taken for scene separation and extracting relevant frames:', round((end - start), 5), 'seconds.', color='chartreuse')
def run_yolo(split_command, path, idx, num_images, json_struct):
pipe = Popen(split_command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
output, err = pipe.communicate(path)
# print output
object_label_probs = output.split('\n')
predict_message = object_label_probs[0]
time_for_prediction = predict_message.split(' ')[-2]
object_label_probs = object_label_probs[1:-1]
for i, s in enumerate(object_label_probs):
split = s.split(':')
object_label_probs[i] = {'class': split[0], 'score': split[1][1:]}
log("Processed image {}/{} object detection in {}s".format(
idx, num_images, time_for_prediction))
# print ("Processed image {}/{} object detection in {}s".format(idx, num_images, time_for_prediction))
if object_label_probs:
log('Objects detected: ', object_label_probs)
# TODO open predictions.png save in section other than full images!!!!
mutex.acquire()
try:
json_struct['images'][idx]['object_lists'][
'yolo_20'] = object_label_probs
finally:
mutex.release()
def batch_scene_classification(video_path, models):
plt.rcParams['figure.figsize'] = (10, 10)
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'
caffe.set_mode_gpu()
net0 = caffe.Net(models[0]['prototxt'], models[0]['caffemodel'], caffe.TEST)
net1 = caffe.Net(models[1]['prototxt'], models[1]['caffemodel'], caffe.TEST)
directory = os.path.dirname(video_path)
image_directory_path = os.path.join(directory, 'images', 'full')
json_struct_path = os.path.join(directory, 'metadata', 'result_struct.json')
json_struct = {}
if os.path.isfile(json_struct_path):
with open(json_struct_path) as data_file:
json_struct = json.load(data_file)
num_images = len(json_struct['images'])
for idx, image in enumerate(json_struct['images']):
image_path = os.path.join(image_directory_path, image['image_name'])
results1 = classify_scene(net0, image_path)
results2 = classify_scene(net1, image_path)
json_struct['images'][idx]['scene_results'] = [] #todo ????
json_struct['images'][idx]['scene_results'] = {'scene_results1' : results1, 'scene_results2' : results2}
log('Image {}/{} scene classified'.format(idx, num_images))
json.dump(json_struct, open(json_struct_path, 'w'), indent=4)
def download_video():
data = dict((key, request.form.getlist(key) if len(request.form.getlist(key)) > 1 else request.form.getlist(key)[0]) for key in request.form.keys())
url = data.get('url')
log('Downloading youtube URL: ', url, header=HEADER_SIZE)
# todo stop js from clicking twice.
name = main.download_video(url)
return name
def download_video():
data = dict(
(key, request.form.getlist(key) if len(request.form.getlist(key)) > 1
else request.form.getlist(key)[0]) for key in request.form.keys())
url = data.get('url')
log('Downloading youtube URL: ', url, header=HEADER_SIZE)
# todo stop js from clicking twice.
name = main.download_video(url)
return name
def main_object_detect(json_struct, video_path):
# log('Main function. Starting YOLO Object Detection', header=HEADER_SIZE - 1, color='darkturquoise')
log('Main function. Starting Faster R-CNN Object Detection', header=HEADER_SIZE - 1, color='darkturquoise')
start = timer()
directory = os.path.dirname(video_path)
image_directory_path = os.path.join(directory, 'images', 'full')
json_struct_path = os.path.join(directory, 'metadata', 'result_struct.json')
with cd('/home/ben/VideoUnderstanding/python_features/faster_rcnn_VOC_object_detection'):
command = 'python faster_rcnn_VOC_object_detection.py ' + json_struct_path + ' ' + video_path
split_command = command.split(' ')
if os.path.isfile(json_struct_path):
with open(json_struct_path) as data_file:
json_struct = json.load(data_file)
pipe = Popen(split_command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
object_list_next = False
object_cls_score_list = []
while True:
line = pipe.stdout.readline()
if not line: break
else:
line = line[:-1]
print line
if line[:3] == '~~~':
object_list_next = False
# print 'OBJECT LIST: ', object_cls_score_list
log('Object list: ', object_cls_score_list)
object_cls_score_list = []
if object_list_next:
if line[:7] == 'NOTHING':
continue
split_line = line.split()
object_cls_score_list.append({'class': split_line[0], 'score': split_line[1]})
elif line[:9] == 'Detection':
log(line)
object_list_next = True
# json.dump(json_struct, open(json_struct_path, 'w'), indent=4)
end = timer()
log('Faster R-CNN Object Detection complete', header=HEADER_SIZE, color='green')
log('Time taken for Faster R-CNN object detection:', round((end - start), 5), 'seconds.', color='chartreuse')
# main_object_detect('/home/ben/VideoUnderstanding/example_images/Funny_Videos_Of_Funny_Animals_NEW_2015/metadata/result_struct.json', '/home/ben/VideoUnderstanding/example_images/Funny_Videos_Of_Funny_Animals_NEW_2015/Funny_Videos_Of_Funny_Animals_NEW_2015.mp4')
def compute_avg_col_dist_and_chi_diff(hist_features, json_struct):
log('')
log('Computing average colour, and euclidean distances of colours between consecutive images.', header=HEADER_SIZE)
# compute euclidian distance from avg colours. Then computer histogram chi distance between every consecutive frame.
# Function computes chi-distance to next frame while compute_chi_diff_on_all_interval computes chi distance to last frame
num_images = len(json_struct['images'])
for idx, image in enumerate(json_struct['images']):
if idx + 1 == num_images:
break
next_avg_colour = json_struct['images'][idx + 1]['dominant_colours']['avg_colour']['col']
cur_avg_colour = image['dominant_colours']['avg_colour']['col']
next_avg_colour = rgb2lab(next_avg_colour)
cur_avg_colour = rgb2lab(cur_avg_colour)
dist = distance.euclidean(next_avg_colour, cur_avg_colour)
image['dominant_colours']['l2distnext'] = round(dist, 3)
# Calculate chi distance between next image
cur_hist = hist_features[image['image_name']]
next_hist = hist_features[json_struct['images'][idx + 1]['image_name']]
chi_dist_next = histogram.chi2_distance(cur_hist, next_hist)
image['dominant_colours']['chi_dist_next'] = round(chi_dist_next, 3)
def main_object_detect(json_struct, video_path):
log('Main function. Starting YOLO Object Detection',
header=HEADER_SIZE - 1,
color='darkturquoise')
start = timer()
directory = os.path.dirname(video_path)
image_directory_path = os.path.join(directory, 'images', 'full')
json_struct_path = os.path.join(directory, 'metadata',
'result_struct.json')
with cd('/home/ben/Documents/darknet'):
darknet_command = "./darknet yolo test cfg/yolo-tiny.cfg yolo-tiny.weights"
# darknet_command = "./darknet yolo test cfg/yolo-small.cfg yolo-small.weights"
split_command = darknet_command.split(' ')
if os.path.isfile(json_struct_path):
with open(json_struct_path) as data_file:
json_struct = json.load(data_file)
num_images = len(json_struct['images'])
# pipe = Popen(split_command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
threads = []
for idx, image in enumerate(json_struct['images']):
path = os.path.join(image_directory_path,
image['image_name']) + '\n'
t = Thread(target=run_yolo,
args=(split_command, path, idx, num_images,
json_struct))
threads.append(t)
t.start()
time.sleep(1)
# if idx > 10:
# break
time.sleep(4)
print threads
for idx, t in enumerate(threads):
t.join()
# print 'joined ', idx, ' thread'
json.dump(json_struct, open(json_struct_path, 'w'), indent=4)
end = timer()
log('YOLO Object Detection complete',
header=HEADER_SIZE,
color='green')
log('Time taken for YOLO object detection:',
round((end - start), 5),
'seconds',
color='chartreuse')
def main_scene_classification(json_struct, video_path):
log('Main function. Starting Scene Classification', header=HEADER_SIZE - 1, color='darkturquoise')
start = timer()
places205model = {'prototxt' : '/home/ben/VideoUnderstanding/models_prototxts/places205CNN_deploy.prototxt', 'caffemodel' : '/home/ben/VideoUnderstanding/models_prototxts/places205CNN_iter_300000.caffemodel'}
googlenet205model = {'prototxt' : '/home/ben/VideoUnderstanding/models_prototxts/deploy_places205.protxt', 'caffemodel' : '/home/ben/VideoUnderstanding/models_prototxts/googlelet_places205_train_iter_2400000.caffemodel'}
all_scene_models = []
all_scene_models.append(places205model)
all_scene_models.append(googlenet205model)
batch_scene_classification(video_path, all_scene_models)
end = timer()
log('Scene Classification complete', header=HEADER_SIZE, color='green')
log('Time taken for scene classification:', round((end - start), 5), 'seconds', color='chartreuse')
def main_object_detect(json_struct, video_path):
log('Main function. Starting YOLO Object Detection', header=HEADER_SIZE - 1, color='darkturquoise')
start = timer()
directory = os.path.dirname(video_path)
image_directory_path = os.path.join(directory, 'images', 'full')
json_struct_path = os.path.join(directory, 'metadata', 'result_struct.json')
with cd('/home/ben/Documents/darknet'):
darknet_command = "./darknet yolo test cfg/yolo-tiny.cfg yolo-tiny.weights"
# darknet_command = "./darknet yolo test cfg/yolo-small.cfg yolo-small.weights"
split_command = darknet_command.split(' ')
if os.path.isfile(json_struct_path):
with open(json_struct_path) as data_file:
json_struct = json.load(data_file)
num_images = len(json_struct['images'])
# pipe = Popen(split_command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
threads = []
for idx, image in enumerate(json_struct['images']):
path = os.path.join(image_directory_path, image['image_name']) + '\n'
t = Thread(target=run_yolo, args=(split_command, path, idx, num_images, json_struct))
threads.append(t)
t.start()
time.sleep(1)
# if idx > 10:
# break
time.sleep(4)
print threads
for idx, t in enumerate(threads):
t.join()
# print 'joined ', idx, ' thread'
json.dump(json_struct, open(json_struct_path, 'w'), indent=4)
end = timer()
log('YOLO Object Detection complete', header=HEADER_SIZE, color='green')
log('Time taken for YOLO object detection:', round((end - start), 5), 'seconds', color='chartreuse')
def test_connect():
print('Client connected')
# emit('print_event', {'s': 'Connected to server', 'header' : 6})
log('Connected to server', header=4)
def download_video(url,changed_name=None):
yt = api.YouTube(url)
pprint(yt.get_videos())
# [<Video: MPEG-4 Visual (.3gp) - 144p>,
# <Video: MPEG-4 Visual (.3gp) - 240p>,
# <Video: Sorenson H.263 (.flv) - 240p>,
# <Video: H.264 (.flv) - 360p>,
# <Video: H.264 (.flv) - 480p>,
# <Video: H.264 (.mp4) - 360p>,
# <Video: H.264 (.mp4) - 720p>,
# <Video: VP8 (.webm) - 360p>,
# <Video: VP8 (.webm) - 480p>]
# set the filename:
if changed_name:
yt.set_filename(changed_name)
yt.set_filename(yt.filename.replace(" ", "_"))
yt.set_filename(yt.filename.replace("&", "and"))
yt.set_filename((yt.filename).encode('ascii', 'ignore').decode('ascii'))
log('Video Name: ', yt.filename)
# Notice that the list is ordered by lowest resolution to highest. If you
# wanted the highest resolution available for a specific file type, you
# can simply do:
log('Highest mp4 resolution video: ', yt.filter('mp4')[-1])
if not yt.filter(extension='mp4'):
log('No mp4 vidoes found!', color='red')
return 'No Video Found'
video = yt.filter('mp4')[-1]
current_dir = os.path.dirname(os.path.realpath(__file__))
video_folder_path = os.path.join(current_dir, 'example_images', yt.filename)
if not os.path.isdir(video_folder_path):
os.makedirs(video_folder_path)
try:
log('Downloading!', color='green')
video.download(video_folder_path)
log('Finished downloading!', color='green')
except:
#TODO TODO TODO TEST TEST TEST TEST BELOW CAREFUL in case recursive
os.rmdir(video_folder_path)
else:
log('Folder and file already there:', video_folder_path, color='red')
return yt.filename
def process_video(video_path, video_url, multiplier=1.0):
start = timer()
#todo pass all below in one dictionary to save calculating every time?
video_name = video_path.split('/')[-1][:-4]
directory = os.path.dirname(video_path)
image_directory_path = os.path.join(directory, 'images', 'full')
json_struct_path = os.path.join(directory, 'metadata', 'result_struct.json')
json_struct = {'images': []}
if os.path.isfile(json_struct_path):
with open(json_struct_path) as data_file:
json_struct = json.load(data_file)
log('Processing video:', video_name, header=1)
# main functions to process video
# ---------------------------------
# RENAME ALL FUNCTIONS AND TAKE SCREENSHOT TODO
filmstrip.main_separate_scenes_and_extract_frames(json_struct, video_path, video_url, False, multiplier)
scene_classification.main_scene_classification(json_struct, video_path)
faster_rcnn_20_detection.main_object_detect(json_struct, video_path)
yolo_object_detection.main_object_detect(json_struct, video_path)
scene_results.main_average_all_scene_results(json_struct, json_struct_path)
# ---------------------------------
# Open URL in a new tab, if a browser window is already open.
log('Opening browser tab with results')
url = 'http://localhost:5000/video_results.html?video='
#todo make open browser another time. For now it opens file. uncomment.
webbrowser.open_new_tab('file:///' + json_struct_path)
webbrowser.open_new_tab(url + json_struct['info']['name'])
# Save video in processed videos json.
all_videos_json = {'videos': []}
all_videos_json_path = '/home/ben/VideoUnderstanding/example_images/all_videos.json'
if os.path.isfile(all_videos_json_path):
with open(all_videos_json_path) as data_file:
all_videos_json = json.load(data_file)
for idx, video in enumerate(all_videos_json['videos']):
if video['video_name'] == video_name:
all_videos_json['videos'][idx]['last_processed_datetime'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
break
else: # if no breaks
all_videos_json['videos'].append({'video_name': video_name,
'video_url': video_url,
'last_processed_datetime': datetime.datetime.now().strftime("%Y-%m-%d %H:%M")})
json.dump(all_videos_json, open(all_videos_json_path, 'w'), indent=4)
end = timer()
seconds = round(end - start, 0);
m, s = divmod(seconds, 60)
log('All processing completed', color='green', header=1)
log('TOTAL PROCESSING TIME TAKEN: ', "%02d:%02d" % (m, s), color='chartreuse', header=1)
log('')
log('')
def save_all_relevant_frames(cap, sourcePath, destPath, name, json_struct, verbose):
log('Saving relevant frames between keyframe ranges.', header=HEADER_SIZE)
dest_dir = os.path.join(destPath, "images")
json_struct['images'] = []
scene_num = 0
hist_features = {}
# todo below into function ASAP FUNCTION CALLED create frames
left_scene_first_frame = 0
for scene_change in json_struct['scene_changes']:
num_range = scene_change['keyframe_range'].split('-')
# log(num_range)
right_scene_first_frame = int(num_range[0])
num_frames_in_scene = json_struct['info']['INITIAL_NUM_FRAMES_IN_SCENE']
range = right_scene_first_frame - left_scene_first_frame
jump_rate = range / num_frames_in_scene
current_frame_num = left_scene_first_frame
log('Scene number: ', scene_num, 'Saving scene frames between: ', left_scene_first_frame, '-', right_scene_first_frame, color='lime')
frames_taken = 0
while frames_taken < num_frames_in_scene:
cap.set(cv.CV_CAP_PROP_POS_FRAMES, current_frame_num)
ret, frame = cap.read()
# extract dominant color
small = resize(frame, 100, 100)
# Todo make 5 a global?
dom_colours = extract_cols(small, 3)
if frame != None:
#todo png always?
image_name = name + "-" + str(current_frame_num) + ".png"
fullPath = os.path.join(dest_dir, "full", image_name)
cv2.imwrite(fullPath, frame)
log(image_name)
avg_colour = [0.0, 0.0, 0.0]
total = 10000.0
for colour in dom_colours:
weight = colour['count'] / total
for idx, num in enumerate(colour['col']):
avg_colour[idx] += weight * num
hist_features[image_name] = histogram.calculate_histograms(frame)
json_struct['images'].append({'image_name': image_name, 'frame_number': current_frame_num, 'scene_num': scene_num,
'dominant_colours': {'kmeans' : dom_colours, 'avg_colour': {'col': avg_colour}}, })
if verbose:
cv2.imshow('extract', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
current_frame_num += jump_rate
frames_taken += 1
else:
log('breaking')
break
scene_num += 1
left_scene_first_frame = int(num_range[1])
json_struct['info']['num_scenes'] = scene_num - 1 # TODO double check if right?
return hist_features
def detect_scenes(cap, json_struct, data, verbose):
log('Using statistics to compute keyframe ranges.', header=HEADER_SIZE)
multiplier = json_struct['info']['multiplier']
multplier_times_sd = (json_struct["stats"]["sd"] * multiplier)
mean_plus_multiplier_times_sd = json_struct["stats"]["mean"] + multplier_times_sd
json_struct['info']['mean_plus_multiplier_times_sd'] = mean_plus_multiplier_times_sd
log('Standard Deviation Multiplier:', multiplier)
log('Mean + (multiplier * standard deviation) chi-difference: ', round(mean_plus_multiplier_times_sd, 3))
log('Any chi-differences over ', round(mean_plus_multiplier_times_sd, 3), 'count as a scene change/keyframe range')
all_chi_diffs = []
count = 0
for idx, fi in enumerate(data['frame_info']):
if idx > 0 and fi['chi_diff'] > mean_plus_multiplier_times_sd:
right_frame_no = fi['frame_number']
left_frame_no = data['frame_info'][idx - 1]['frame_number']
# isolate down to range within 50 for the moment
left_frame_no, right_frame_no, chi_diff_between_50 = isolate_from_100_to_10_range(cap, left_frame_no, right_frame_no)
left_frame_no, right_frame_no, chi_diff_between_50 = isolate_from_100_to_10_range(cap, left_frame_no, right_frame_no)
keyframe_range = ('{0}-{1}').format(left_frame_no, right_frame_no)
# log(keyframe_range)
all_chi_diffs.append({'chi_diff': round(fi['chi_diff'], 4), 'keyframe_range': keyframe_range,
'sds_over_mean': round((fi['chi_diff'] - (json_struct["stats"]["mean"])) / json_struct["stats"]["sd"], 4)})
count += 1
all_scene_changes_by_frame_no = sorted(all_chi_diffs, key=lambda k: int(k['keyframe_range'].split('-')[0]))
all_scene_changes_by_chi_diff = sorted(all_chi_diffs, key=lambda k: k['chi_diff'])
json_struct['scene_changes'] = all_scene_changes_by_frame_no
# if verbose:
log('')
log('Keyframe ranges in order of frame number', color='lightblue')
for i in all_scene_changes_by_frame_no:
log('keyframe_range', i['keyframe_range'], 'chi_diff: ', i['chi_diff'], 'sds over mean:', i['sds_over_mean'])
log('')
log('Keyframe ranges in order of chi-difference', color='lightblue')
for i in all_scene_changes_by_chi_diff:
log('sds over meani', i['sds_over_mean'], 'chi_diff: ', i['chi_diff'], 'keyframe_range', i['keyframe_range'])
# TODO keep expanding this whole section so that when I change to other videos (videos with only 1-2 scenes) I can keep debugging
log('Keyframe ranges found.', header=HEADER_SIZE)
def compute_chi_diff_on_all_interval(sourcePath, json_struct, verbose, interval):
log('Computing chi-differences (between current frame and frame one interval before) on all frames at an interval of ', interval, header=HEADER_SIZE)
# stores all chi differences between every 100 images.
data = {
"frame_info": []
}
cap = cv2.VideoCapture(sourcePath)
ret, last_frame = cap.read()
frame_number = 0
frame_number += interval
while (cap.isOpened()):
cap.set(cv.CV_CAP_PROP_POS_FRAMES, frame_number)
ret, frame = cap.read()
if frame != None:
chi_diff = histogram.chi2_distance_two_images(frame, last_frame)
frame_info = {
"frame_number": int(frame_number),
"chi_diff": chi_diff
}
log('Frame number: ', frame_number, ' chi-difference: ', round(chi_diff, 3))
data["frame_info"].append(frame_info)
# cv2.putText(frame, "chidiff = " + str(chi_diff), (600, 45), cv2.FONT_HERSHEY_SIMPLEX, 2.0, (255, 255, 255))
# cv2.imshow('frame', frame)
k = cv2.waitKey(1)
if k == ord('q'):
break
last_frame = frame
else:
break
frame_number += interval
log('Computing statistics of all chi-differences.', header=HEADER_SIZE)
# Compute some stats
chi_diff_counts = [fi["chi_diff"] for fi in data["frame_info"]]
json_struct['stats'] = {
"num": len(chi_diff_counts),
"min": np.min(chi_diff_counts),
"max": np.max(chi_diff_counts),
"mean": np.mean(chi_diff_counts),
"median": np.median(chi_diff_counts),
"sd": np.std(chi_diff_counts)
}
greater_than_mean = [fi for fi in data["frame_info"] if fi["chi_diff"] > json_struct["stats"]["mean"]]
greater_than_median = [fi for fi in data["frame_info"] if fi["chi_diff"] > json_struct["stats"]["median"]]
greater_than_one_sd = [fi for fi in data["frame_info"] if fi["chi_diff"] > json_struct["stats"]["sd"] + json_struct["stats"]["mean"]]
greater_than_two_sd = [fi for fi in data["frame_info"] if fi["chi_diff"] > (json_struct["stats"]["sd"] * 2) + json_struct["stats"]["mean"]]
greater_than_three_sd = [fi for fi in data["frame_info"] if fi["chi_diff"] > (json_struct["stats"]["sd"] * 3) + json_struct["stats"]["mean"]]
json_struct["stats"]["greater_than_mean"] = len(greater_than_mean)
json_struct["stats"]["greater_than_median"] = len(greater_than_median)
json_struct["stats"]["greater_than_one_sd"] = len(greater_than_one_sd)
json_struct["stats"]["greater_than_two_sd"] = len(greater_than_two_sd)
json_struct["stats"]["greater_than_three_sd"] = len(greater_than_three_sd)
json_struct["stats"]['mean_plus_one_sd'] = (json_struct["stats"]["sd"]) + json_struct["stats"]["mean"]
json_struct["stats"]['mean_plus_two_sd'] = (json_struct["stats"]["sd"] * 2) + json_struct["stats"]["mean"]
# if verbose:
# log(json.dumps(json_struct, indent=4))
return cap, data
def main_object_detect(json_struct, video_path):
# log('Main function. Starting YOLO Object Detection', header=HEADER_SIZE - 1, color='darkturquoise')
log('Main function. Starting Faster R-CNN Object Detection',
header=HEADER_SIZE - 1,
color='darkturquoise')
start = timer()
directory = os.path.dirname(video_path)
image_directory_path = os.path.join(directory, 'images', 'full')
json_struct_path = os.path.join(directory, 'metadata',
'result_struct.json')
with cd('/home/ben/VideoUnderstanding/python_features/faster_rcnn_VOC_object_detection'
):
command = 'python faster_rcnn_VOC_object_detection.py ' + json_struct_path + ' ' + video_path
split_command = command.split(' ')
if os.path.isfile(json_struct_path):
with open(json_struct_path) as data_file:
json_struct = json.load(data_file)
pipe = Popen(split_command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
object_list_next = False
object_cls_score_list = []
while True:
line = pipe.stdout.readline()
if not line: break
else:
line = line[:-1]
print line
if line[:3] == '~~~':
object_list_next = False
# print 'OBJECT LIST: ', object_cls_score_list
log('Object list: ', object_cls_score_list)
object_cls_score_list = []
if object_list_next:
if line[:7] == 'NOTHING':
continue
split_line = line.split()
object_cls_score_list.append({
'class': split_line[0],
'score': split_line[1]
})
elif line[:9] == 'Detection':
log(line)
object_list_next = True
# json.dump(json_struct, open(json_struct_path, 'w'), indent=4)
end = timer()
log('Faster R-CNN Object Detection complete',
header=HEADER_SIZE,
color='green')
log('Time taken for Faster R-CNN object detection:',
round((end - start), 5),
'seconds.',
color='chartreuse')
# main_object_detect('/home/ben/VideoUnderstanding/example_images/Funny_Videos_Of_Funny_Animals_NEW_2015/metadata/result_struct.json', '/home/ben/VideoUnderstanding/example_images/Funny_Videos_Of_Funny_Animals_NEW_2015/Funny_Videos_Of_Funny_Animals_NEW_2015.mp4')
def main_average_all_scene_results(json_struct, json_struct_path):
start = timer()
log('Main function. Averaging all results for a scene',
header=HEADER_SIZE - 1,
color='darkturquoise')
#TODO so much horrible naming
current_scene_num = 0
all_averaged_results_for_scene = {
'current_scene_scene_results': [[], []],
'current_scene_object_lists': {
'yolo_20': [],
'faster_rcnn_20': []
},
'average_colour': []
}
json_struct = {}
if os.path.isfile(json_struct_path):
with open(json_struct_path) as data_file:
json_struct = json.load(data_file)
json_struct['scenes'] = []
num_images = json_struct['info']['num_images']
# json.dumps(parsed, indent=4, sort_keys=True)
for idx, image in enumerate(json_struct['images']):
#todo replace all with .gets
print image
scene_results1 = image['scene_results']['scene_results1']
scene_results2 = image['scene_results']['scene_results2']
object_list_YOLO = image['object_lists']['yolo_20']
object_list_RCNN = image['object_lists']['faster_rcnn_20']
average_colour = image['dominant_colours']['avg_colour']['col']
print 'scene_results1: ', scene_results1, '\n'
print 'scene_results2: ', scene_results2, '\n'
print 'object list YOLO: ', object_list_YOLO, '\n'
print 'object list Faster R-CNN: ', object_list_RCNN, '\n'
print 'average colour: ', average_colour, '\n'
all_averaged_results_for_scene['current_scene_scene_results'][
0].append(scene_results1)
all_averaged_results_for_scene['current_scene_scene_results'][
1].append(scene_results2)
all_averaged_results_for_scene['current_scene_object_lists'][
'yolo_20'].append(object_list_YOLO)
all_averaged_results_for_scene['current_scene_object_lists'][
'faster_rcnn_20'].append(object_list_RCNN)
all_averaged_results_for_scene['average_colour'].append(average_colour)
#
# If next image doesn't exist or next image is another scene, average results
if idx + 1 >= num_images or json_struct['images'][
idx + 1]['scene_num'] != current_scene_num:
# print '\n-------------------------------------\n-------------------------------------\n-------------------------------------\n'
average_scene_results(json_struct, all_averaged_results_for_scene,
current_scene_num)
# all_averaged_results_for_scene['current_scene_scene_results'][0][:] = [] ##todo this or below?
all_averaged_results_for_scene = {
'current_scene_scene_results': [[], []],
'current_scene_object_lists': {
'yolo_20': [],
'faster_rcnn_20': []
},
'average_colour': []
}
current_scene_num += 1
# print '\n-------------------------------------\n-------------------------------------\nSCENE HAS CHANGED TO: ', current_scene_num, '\n-------------------------------------\n'
json.dump(json_struct, open(json_struct_path, 'w'), indent=4)
end = timer()
log('Scene result averaging complete', header=HEADER_SIZE, color='green')
log('Time taken for scene result averaging:',
round((end - start), 5),
'seconds',
color='chartreuse')
def test_connect():
print('Client connected')
# emit('print_event', {'s': 'Connected to server', 'header' : 6})
log('Connected to server', header=4)
