def recursive_merge_data(derived_class, original, updated):
"""Recursive class function to move deep into dictionary and merge contents."""
try:
if (len(updated.keys()) == 0):
return {}
for key in updated.keys():
if (original.has_key(key)) and (type(original[key]) is dict) and (type(original[key]) is dict):
original[key] = MongoDocument.recursive_merge_data(original[key],updated[key])
else:
original[key] = updated[key]
except:
Tools.print_errors ("Error recursively updating object")
return None
return original
def create(self):
self.doc_id = ""
"""Insert the object into the mongodb. It should not exist in mongo before this call."""
if (not self.attributes.has_key('created_at')):
self.attributes['created_at'] = datetime.datetime.now()
self.attributes['status'] = {'imagetagger_status':'created'}
self.attributes['modified_at'] = datetime.datetime.now()
cherrylog ("Creating item in database with values " + str(self.attributes))
#insert a doc
try:
insert = self.my_collection.insert(self.attributes, safe=True)
except Exception as e:
Tools.print_errors ("Error inserting object into database: " + str(self.__class__.__name__))
return False
cherrylog ("Insert of item successul: " + (str(insert)))
self.doc_id = str(insert)
return True
def sanitize_items_results(self, input_array):
try:
if (input_array == None):
return None
clean_array = []
for item in input_array:
if (item == None):
continue
if (len(item) > 80):
continue
item_clean = str(re.sub(r'[^\w\s&]+','',item))
item_clean.strip()
if (len(item_clean) >= 3):
clean_array.append(item_clean)
return clean_array
except Exception as e:
Tools.print_errors ("Error sanitizing result from Crowd Source!")
return ""
def get_scene_stats(video_id, motionIndexVector,fps):
cherrylog ("Running get_scene_stats")
all_scenes = []
try:
for i in range(len(motionIndexVector)-1):
num_images = motionIndexVector[i+1] - motionIndexVector[i]
start_time = time_of_frame_raw(motionIndexVector[i], fps)
stop_time = time_of_frame_raw(motionIndexVector[i+1], fps)
length = time_of_frame(start_time - stop_time)
scene = {'video': int(video_id),'scene_id': int(i),
'start_time':float(start_time),'stop_time':float(stop_time),
'num_images':int(num_images),'length':str(length),
'status':{'processed':True}}
all_scenes.append(scene)
except Exception as e:
Tools.print_errors ("Error running get_scene_stats on video_id: " + str(video_id))
return all_scenes
def update(self, merge_data={}, commit=True):
""" Update object with current data into mongo.
Arguments:
merge_data -- dictionary used to represent the entire object. This will overwrite the object's current data.
commit -- Should this update of the data in memory be commited to mongo? Default: True
"""
if (not merge_data=={}):
self.merge_data(merge_data) #Intelligently merge the argument into the self.attributes variable
cherrylog ("Data merged into object: "+ str(merge_data))
if (commit):
self.attributes['modified_at'] = datetime.datetime.now()
cherrylog ("updating item in database")
try:
update = self.my_collection.update({'_id':oid(self.doc_id)}, self.attributes, safe=True)
except Exception as e:
Tools.print_errors ("Error updating object in database: " + str(self.__class__.__name__))
return False
cherrylog ("Update of item successul: " + (str(update)))
return True
def load(self, lookup_doc_id=None, alt_matching={}):
"""Load data from database into object in memory
Arguments:
lookup_doc_id -- the doc_id of the object that will be used for matching.
alt_matching -- An alternate dictionary for doing the lookup.
One of these two parameters is required.
"""
if (lookup_doc_id != None):
cherrylog ("Loading " + str(self.__class__.__name__) + " with params: id=" + str(lookup_doc_id))
else:
cherrylog ("Loading " + str(self.__class__.__name__) + " with params: " + str(alt_matching))
try:
match_param = {}
if (alt_matching != {}):
match_param = alt_matching
else:
match_param = {'_id':oid(lookup_doc_id)}
docs = self.my_collection.find(match_param, safe=True)
cherrylog ("got number of results as: " + str(docs.count()))
if (docs.count() == 0):
cherrylog ("Couldn't load object, 0 matching results in database!!")
return False
if (docs.count() == 1):
self.doc_id = str(docs[0]['_id'])
if docs[0].has_key('_id'): del docs[0]['_id']
self.attributes = docs[0]
return True
else:
raise Exception("Retrieved multiple videos when only 1 was expected")
except Exception as e:
Tools.print_errors ("Unable to load " + str(self.__class__.__name__) + " from database with params: id=" + str(self.doc_id))
return False
cherrylog ("Successfully loaded object: " + str(self.attributes))
return True
def delete_data(self):
cherrylog("Removing Video files, including hidden files (.*)")
video_root_dir = c.configs["imagetagger"]["processed_videos_dir"] + "/" + self.attributes["hashstring"]
# image_files = glob.glob(video_root_dir + '/' + c.configs['imagetagger']['image_dir_name'] + '/*') + glob.glob(video_root_dir + '/' + c.configs['imagetagger']['image_dir_name'] + '/.*')
# video_files = glob.glob(video_root_dir + '/' + c.configs['imagetagger']['video_dir_name'] + '/*') + glob.glob(video_root_dir + '/' + c.configs['imagetagger']['video_dir_name'] + '/.*')
# report_files = glob.glob(video_root_dir + '/' + c.configs['imagetagger']['report_dir_name'] + '/*') + glob.glob(video_root_dir + '/' + c.configs['imagetagger']['report_dir_name'] + '/.*')
try:
# Have to first delete all files, when using the OS lib
# for image_count in range(len(image_files)):
# os.remove(image_files[image_count])
# for video_count in range(len(video_files)):
# os.remove(video_files[video_count])
# for report_count in range(len(report_files)):
# os.remove(report_files[report_count])
# os.removedirs(video_root_dir)
shutil.rmtree(video_root_dir)
except OSError:
Tools.print_errors(
"Unable to delete directory or files in "
+ c.configs["imagetagger"]["processed_videos_dir"]
+ "/"
+ self.attributes["hashstring"]
)
def get_video_info(videoInputPath, videoName, processedVideoPath, video_dir, image_dir, report_dir, thumb_dir):
video_name_without_extension = videoName[0:videoName.rfind(".")]
full_path = videoInputPath + "/" + videoName
running_os = ""
if (sys.platform.find("linux") >= 0):
running_os = "linux"
else:
running_os = "osx"
md5 = get_md5(full_path, running_os)
#Create destination directories
try:
os.makedirs(processedVideoPath + "/" + md5)
os.makedirs(processedVideoPath + "/" + md5 + "/" + video_dir)
os.makedirs(processedVideoPath + "/" + md5 + "/" + image_dir)
os.makedirs(processedVideoPath + "/" + md5 + "/" + report_dir)
os.makedirs(processedVideoPath + "/" + md5 + "/" + thumb_dir)
except OSError as err:
Tools.print_errors("Error creating directories for video proprocess: " + str(videoName))
cherrylog ("file exists, so skipping this video " + full_path)
return None
#Gathering specs for the video
ffmpeg_info_command = "ffmpeg -i \"%s\"" % ( full_path )
args = shlex.split (ffmpeg_info_command)
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout_value, stderr_value = p.communicate()
video_info = repr(stderr_value)
vid_res = re.search ('\d\d\d[\d]?x\d\d\d[\d]?' , video_info)
if (vid_res != None):
video_res = vid_res.group(0)
if (video_res == None or video_res == ""):
video_res = ""
else:
resolution = getResolution(video_res)
duration_string = re.search('\d\d:\d\d:\d\d\.\d\d', video_info).group(0)
duration = get_video_duration(duration_string)
fps_query = re.search('\d\d\.\d\d fps', video_info)
if (fps_query == None):
fps_query = re.search('\d\d\.\d fps', video_info)
if (fps_query == None):
fps_query = re.search('\d\d\ fps', video_info)
if (fps_query == None):
fps = 0
else:
fps_string = fps_query.group(0)
fps = get_FPS(fps_string)
return ({
'hashstring':str(md5),
'width':str(resolution["width"]),
'height':str(resolution["height"]),
'length':duration,
'friendly_id': int(video_name_without_extension),
'fps': fps
})
def index(self, *args, **kw):
cherrylog("Args: " + str(kw))
# submit a new video
# http://localhost:8080/video?submit={%22display_name%22:%22testvid%22,%22video_description%22:%22Some%20Sample%20Video%22,%22video_meta_tags%22:%22apple,banana%22,%22account_id%22:1,%22friendly_id%22:12893712}
try:
# Check that the submitted request is in a valid form
if not mv.MVideo.validate_cmd(kw["action"]):
return "Unknown command for video " + str(i)
if "friendly_id" in kw:
kw["friendly_id"] = int(kw["friendly_id"])
video_action = ""
if "action" in kw:
video_action = kw["action"]
cherrylog("Getting video operation of " + video_action)
if video_action == "submit":
submit_args = kw
del submit_args["action"]
cherrylog("Received video submit with parameters: " + str(submit_args))
if not mv.MVideo.validate_submit_params(submit_args):
return "Invalid or missing parameter for Video"
cherrylog("Got a submit for a valid video! Checking to make sure this video doesn't already exist")
# Create new MVideo object, which will make an entry in mongo
new_video = mv.MVideo()
if new_video.load(alt_matching={"friendly_id": submit_args["friendly_id"]}):
return "Video already exists"
new_video.attributes = submit_args
new_video.create()
return_attributes = new_video.attributes # Why does this object contain the mongo _id attribute??
return self.clean_attributes(return_attributes)
elif video_action == "retrieve":
retrieve_args = kw
del retrieve_args["action"]
cherrylog("Received request to retrieve Video with values: " + str(retrieve_args))
if not mv.MVideo.validate_retrieve_params(retrieve_args):
return "Invalid or missing parameter for Video"
retrieve_video = mv.MVideo()
if not retrieve_video.load(alt_matching={"friendly_id": retrieve_args["friendly_id"]}):
return "Could not find video"
return_attributes = retrieve_video.attributes
return self.clean_attributes(return_attributes)
elif video_action == "update":
update_args = kw
del update_args["action"]
cherrylog("Received request to update Video with values: " + str(update_args))
if not mv.MVideo.validate_update_params(update_args):
return "Invalid or missing parameter for Video"
update_video = mv.MVideo()
if not update_video.load(alt_matching={"friendly_id": update_args["friendly_id"]}):
return "Could not find video"
update_video.update(merge_data=update_args, commit=True)
return self.clean_attributes(update_video.attributes)
elif video_action == "analyze":
analyze_args = kw
del analyze_args["action"]
cherrylog("Received command to analyze this video in the crowd: " + str(analyze_args))
if not mv.MVideo.validate_analyze_params(analyze_args):
return "Invalid or missing parameter for Video"
analyze_video = mv.MVideo()
if not analyze_video.load(alt_matching={"friendly_id": analyze_args["friendly_id"]}):
return "Could not find video"
analyze_quality = analyze_args["quality"]
if analyze_quality != "premium":
analyze_quality = "basic"
# can include the crowds to use during analysis depending on needs and send that as a parameter
# default in CF
result = analyze_video.send_to_providers(quality=analyze_quality)
if result:
cherrylog("Video has been submitted to sources to process")
return "Analyzing!"
else:
cherrylog("Something went wrong when submitting process command")
return "Unable to process video at this time"
elif video_action == "reprocess_metadata":
reprocess_args = kw
del reprocess_args["action"]
cherrylog("Received command to reprocess the metadata of video with data: " + str(reprocess_args))
if not mv.MVideo.validate_reprocess_metadata_params(reprocess_args):
return "Invalid or missing parameter for Video"
reprocess_video = mv.MVideo()
if not reprocess_video.load(alt_matching={"friendly_id": reprocess_args["friendly_id"]}):
return "Could not find video."
reprocess_video.process_imagetagger_metadata()
return "Done reprocessing video!"
elif video_action == "rebuild_report":
rebuild_args = kw
del rebuild_args["action"]
cherrylog("Received command to rebuild the report for video with data: " + str(rebuild_args))
if not mv.MVideo.validate_rebuild_report_params(rebuild_args):
return "Invalid or missing parameter for Video"
rebuild_video = mv.MVideo()
if not rebuild_video.load(alt_matching={"friendly_id": rebuild_args["friendly_id"]}):
return "Could not find video."
rebuild_video.build_report(report_quality=rebuild_args["quality"])
return "Reports have been rebuilt."
elif video_action == "destroy":
destroy_args = kw
del destroy_args["action"]
cherrylog("Received command to destroy the video from all systems: " + str(destroy_args))
destroy_video = mv.MVideo()
if not destroy_video.load(alt_matching={"friendly_id": destroy_args["friendly_id"]}):
return "Could not find video."
destroy_video.destroy()
return "Video has been cleared from database and deleted."
else:
return "Invalid Command"
except Exception as e:
Tools.print_errors("Receievd Invalid Video Command in Video Route")
return "Invalid Command"
