def ffprobe_metadata(input_file): """This function uses pretty and parsable ffmpeg output to access all the metadata of a videofile correctly @param input_file: fullpath to the media file @type input_file: string @return: a dictionary containing the metadata @rtype: dictionary """ global _FFPROBE_METADATA_CACHE if input_file in _FFPROBE_METADATA_CACHE: return _FFPROBE_METADATA_CACHE[input_file] ffprobe_output = probe(input_file, True) if ffprobe_output is None: return None meta_dict = { 'format': {}, 'streams': [] } format_start = re.compile("^\[FORMAT\]$") format_end = re.compile("^\[\/FORMAT\]$") stream_start = re.compile("^\[STREAM\]$") stream_end = re.compile("^\[\/STREAM\]$") lines = ffprobe_output.splitlines() format_section = False stream_section = False for line in lines: if format_start.match(line): format_section = True continue if format_end.match(line): format_section = False continue if stream_start.match(line): meta_dict['streams'].append(dict()) stream_section = True continue if stream_end.match(line): stream_section = False continue if format_section: key, value = line.split("=", 1) meta_dict['format'][key] = value if stream_section: key, value = line.split("=", 1) meta_dict['streams'][-1][key] = value _FFPROBE_METADATA_CACHE[input_file] = meta_dict return meta_dict
## Remove all previous uploads filelist = os.listdir(os.path.split(formfields.file.name)[0]) for afile in filelist: if argd['access'] in afile: os.remove(os.path.join(os.path.split(formfields.file.name)[0], afile)) ## Check if the file is a readable video ## We must exclude all image and audio formats that are readable by ffprobe if (os.path.splitext(filename)[1] in ['jpg', 'jpeg', 'gif', 'tiff', 'bmp', 'png', 'tga', 'jp2', 'j2k', 'jpf', 'jpm', 'mj2', 'biff', 'cgm', 'exif', 'img', 'mng', 'pic', 'pict', 'raw', 'wmf', 'jpe', 'jif', 'jfif', 'jfi', 'tif', 'webp', 'svg', 'ai', 'ps', 'psd', 'wav', 'mp3', 'pcm', 'aiff', 'au', 'flac', 'wma', 'm4a', 'wv', 'oga', 'm4a', 'm4b', 'm4p', 'm4r', 'aac', 'mp4', 'vox', 'amr', 'snd'] or not probe(formfields.file.name)): formfields.file.close() raise apache.SERVER_RETURN(apache.HTTP_FORBIDDEN) ## We have no "delete" attribute in Python 2.4 if sys.hexversion < 0x2050000: ## We need to rename first and create a dummy file ## Rename the temporary file for the garbage collector new_tmp_fullpath = os.path.split(formfields.file.name)[0] + "/" + CFG_WEBSUBMIT_TMP_VIDEO_PREFIX + argd['access'] + "_" + os.path.split(formfields.file.name)[1] os.rename(formfields.file.name, new_tmp_fullpath) dummy = open(formfields.file.name, "w") dummy.close() formfields.file.close() else: # Mark the NamedTemporatyFile as not to be deleted formfields.file.delete = False
os.path.join( os.path.split(formfields.file.name)[0], afile)) ## Check if the file is a readable video ## We must exclude all image and audio formats that are readable by ffprobe if (os.path.splitext(filename)[1] in [ 'jpg', 'jpeg', 'gif', 'tiff', 'bmp', 'png', 'tga', 'jp2', 'j2k', 'jpf', 'jpm', 'mj2', 'biff', 'cgm', 'exif', 'img', 'mng', 'pic', 'pict', 'raw', 'wmf', 'jpe', 'jif', 'jfif', 'jfi', 'tif', 'webp', 'svg', 'ai', 'ps', 'psd', 'wav', 'mp3', 'pcm', 'aiff', 'au', 'flac', 'wma', 'm4a', 'wv', 'oga', 'm4a', 'm4b', 'm4p', 'm4r', 'aac', 'mp4', 'vox', 'amr', 'snd' ] or not probe(formfields.file.name)): formfields.file.close() raise apache.SERVER_RETURN(apache.HTTP_FORBIDDEN) ## We have no "delete" attribute in Python 2.4 if sys.hexversion < 0x2050000: ## We need to rename first and create a dummy file ## Rename the temporary file for the garbage collector new_tmp_fullpath = os.path.split( formfields.file.name )[0] + "/" + CFG_WEBSUBMIT_TMP_VIDEO_PREFIX + argd[ 'access'] + "_" + os.path.split( formfields.file.name)[1] os.rename(formfields.file.name, new_tmp_fullpath) dummy = open(formfields.file.name, "w") dummy.close()
def upload_video(self, req, form): """ A clone of uploadfile but for (large) videos. Does not copy the uploaded file to the websubmit directory. Instead, the path to the file is stored inside the submission directory. """ def gcd(a, b): """ the euclidean algorithm """ while a: a, b = b % a, a return b from invenio.bibencode_extract import extract_frames from invenio.bibencode_config import CFG_BIBENCODE_WEBSUBMIT_ASPECT_SAMPLE_DIR, CFG_BIBENCODE_WEBSUBMIT_ASPECT_SAMPLE_FNAME from invenio.bibencode_encode import determine_aspect from invenio.bibencode_utils import probe from invenio.bibencode_metadata import ffprobe_metadata from invenio.websubmit_config import CFG_WEBSUBMIT_TMP_VIDEO_PREFIX argd = wash_urlargd( form, { 'doctype': (str, ''), 'access': (str, ''), 'indir': (str, ''), 'session_id': (str, ''), 'rename': (str, ''), }) curdir = None if not form.has_key("indir") or \ not form.has_key("doctype") or \ not form.has_key("access"): raise apache.SERVER_RETURN(apache.HTTP_BAD_REQUEST) else: curdir = os.path.join(CFG_WEBSUBMIT_STORAGEDIR, argd['indir'], argd['doctype'], argd['access']) user_info = collect_user_info(req) if form.has_key("session_id"): # Are we uploading using Flash, which does not transmit # cookie? The expect to receive session_id as a form # parameter. First check that IP addresses do not # mismatch. uid = session.uid user_info = collect_user_info(uid) try: act_fd = file(os.path.join(curdir, 'act')) action = act_fd.read() act_fd.close() except: act = "" # Is user authorized to perform this action? (auth_code, auth_message) = acc_authorize_action( uid, "submit", authorized_if_no_roles=not isGuestUser(uid), verbose=0, doctype=argd['doctype'], act=action) if acc_is_role("submit", doctype=argd['doctype'], act=action) and auth_code != 0: # User cannot submit raise apache.SERVER_RETURN(apache.HTTP_UNAUTHORIZED) else: # Process the upload and get the response json_response = {} for key, formfields in form.items(): filename = key.replace("[]", "") if hasattr(formfields, "filename") and formfields.filename: dir_to_open = os.path.abspath( os.path.join(curdir, 'files', str(user_info['uid']), key)) try: assert ( dir_to_open.startswith(CFG_WEBSUBMIT_STORAGEDIR)) except AssertionError: register_exception(req=req, prefix='curdir="%s", key="%s"' % (curdir, key)) raise apache.SERVER_RETURN(apache.HTTP_FORBIDDEN) if not os.path.exists(dir_to_open): try: os.makedirs(dir_to_open) except OSError, e: if e.errno != errno.EEXIST: # If the issue is only that directory # already exists, then continue, else # report register_exception(req=req, alert_admin=True) raise apache.SERVER_RETURN( apache.HTTP_FORBIDDEN) filename = formfields.filename ## Before saving the file to disc, wash the filename (in particular ## washing away UNIX and Windows (e.g. DFS) paths): filename = os.path.basename(filename.split('\\')[-1]) filename = filename.strip() if filename != "": # Check that file does not already exist while os.path.exists( os.path.join(dir_to_open, filename)): #dirname, basename, extension = decompose_file(new_destination_path) basedir, name, extension = decompose_file(filename) new_name = propose_next_docname(name) filename = new_name + extension #-------------# # VIDEO STUFF # #-------------# ## Remove all previous uploads filelist = os.listdir( os.path.split(formfields.file.name)[0]) for afile in filelist: if argd['access'] in afile: os.remove( os.path.join( os.path.split(formfields.file.name)[0], afile)) ## Check if the file is a readable video ## We must exclude all image and audio formats that are readable by ffprobe if (os.path.splitext(filename)[1] in [ 'jpg', 'jpeg', 'gif', 'tiff', 'bmp', 'png', 'tga', 'jp2', 'j2k', 'jpf', 'jpm', 'mj2', 'biff', 'cgm', 'exif', 'img', 'mng', 'pic', 'pict', 'raw', 'wmf', 'jpe', 'jif', 'jfif', 'jfi', 'tif', 'webp', 'svg', 'ai', 'ps', 'psd', 'wav', 'mp3', 'pcm', 'aiff', 'au', 'flac', 'wma', 'm4a', 'wv', 'oga', 'm4a', 'm4b', 'm4p', 'm4r', 'aac', 'mp4', 'vox', 'amr', 'snd' ] or not probe(formfields.file.name)): formfields.file.close() raise apache.SERVER_RETURN(apache.HTTP_FORBIDDEN) ## We have no "delete" attribute in Python 2.4 if sys.hexversion < 0x2050000: ## We need to rename first and create a dummy file ## Rename the temporary file for the garbage collector new_tmp_fullpath = os.path.split( formfields.file.name )[0] + "/" + CFG_WEBSUBMIT_TMP_VIDEO_PREFIX + argd[ 'access'] + "_" + os.path.split( formfields.file.name)[1] os.rename(formfields.file.name, new_tmp_fullpath) dummy = open(formfields.file.name, "w") dummy.close() formfields.file.close() else: # Mark the NamedTemporatyFile as not to be deleted formfields.file.delete = False formfields.file.close() ## Rename the temporary file for the garbage collector new_tmp_fullpath = os.path.split( formfields.file.name )[0] + "/" + CFG_WEBSUBMIT_TMP_VIDEO_PREFIX + argd[ 'access'] + "_" + os.path.split( formfields.file.name)[1] os.rename(formfields.file.name, new_tmp_fullpath) # Write the path to the temp file to a file in STORAGEDIR fp = open(os.path.join(dir_to_open, "filepath"), "w") fp.write(new_tmp_fullpath) fp.close() fp = open(os.path.join(dir_to_open, "filename"), "w") fp.write(filename) fp.close() ## We are going to extract some thumbnails for websubmit ## sample_dir = os.path.join( curdir, 'files', str(user_info['uid']), CFG_BIBENCODE_WEBSUBMIT_ASPECT_SAMPLE_DIR) try: ## Remove old thumbnails shutil.rmtree(sample_dir) except OSError: register_exception(req=req, alert_admin=False) try: os.makedirs( os.path.join(curdir, 'files', str(user_info['uid']), sample_dir)) except OSError: register_exception(req=req, alert_admin=False) try: extract_frames( input_file=new_tmp_fullpath, output_file=os.path.join( sample_dir, CFG_BIBENCODE_WEBSUBMIT_ASPECT_SAMPLE_FNAME ), size="600x600", numberof=5) json_response['frames'] = [] for extracted_frame in os.listdir(sample_dir): json_response['frames'].append(extracted_frame) except: ## If the frame extraction fails, something was bad with the video os.remove(new_tmp_fullpath) register_exception(req=req, alert_admin=False) raise apache.SERVER_RETURN(apache.HTTP_FORBIDDEN) ## Try to detect the aspect. if this fails, the video is not readable ## or a wrong file might have been uploaded try: (aspect, width, height) = determine_aspect(new_tmp_fullpath) if aspect: aspx, aspy = aspect.split(':') else: the_gcd = gcd(width, height) aspx = str(width / the_gcd) aspy = str(height / the_gcd) json_response['aspx'] = aspx json_response['aspy'] = aspy except TypeError: ## If the aspect detection completely fails os.remove(new_tmp_fullpath) register_exception(req=req, alert_admin=False) raise apache.SERVER_RETURN(apache.HTTP_FORBIDDEN) ## Try to extract some metadata from the video container metadata = ffprobe_metadata(new_tmp_fullpath) json_response['meta_title'] = metadata['format'].get( 'TAG:title') json_response['meta_description'] = metadata[ 'format'].get('TAG:description') json_response['meta_year'] = metadata['format'].get( 'TAG:year') json_response['meta_author'] = metadata['format'].get( 'TAG:author') ## Empty file name else: raise apache.SERVER_RETURN(apache.HTTP_BAD_REQUEST) ## We found our file, we can break the loop break # Send our response if CFG_JSON_AVAILABLE: dumped_response = json.dumps(json_response) # store the response in the websubmit directory # this is needed if the submission is not finished and continued later response_dir = os.path.join(curdir, 'files', str(user_info['uid']), "response") try: os.makedirs(response_dir) except OSError: # register_exception(req=req, alert_admin=False) pass fp = open(os.path.join(response_dir, "response"), "w") fp.write(dumped_response) fp.close() return dumped_response
def DEMOVID_Validation(parameters, curdir, form, user_info=None): """ """ messages = [] malformed = False file_storing_path = os.path.join(curdir, "files", str(user_info['uid']), "NewFile", 'filepath') file_storing_name = os.path.join(curdir, "files", str(user_info['uid']), "NewFile", 'filename') file_storing_aspect = os.path.join(curdir, "DEMOVID_ASPECT") file_storing_title = os.path.join(curdir, "DEMOVID_TITLE") file_storing_description = os.path.join(curdir, "DEMOVID_DESCR") file_storing_author = os.path.join(curdir, "DEMOVID_AU") file_storing_year = os.path.join(curdir, "DEMOVID_YEAR") ## Validate the uploaded video try: fp = open(file_storing_path) fullpath = fp.read() fp.close() fp = open(file_storing_name) filename = fp.read() fp.close() if not probe(fullpath) or os.path.splitext(filename)[1] in ['jpg', 'jpeg', 'gif', 'tiff', 'bmp', 'png', 'tga']: malformed = True messages.append("The uploaded file is not a supported video format.") except: malformed = True messages.append("Please upload a video.") ## Validate the title try: fp = open(file_storing_title) title = fp.read() fp.close() if len(title) < 2 or len(title) > 50: malformed = True messages.append("The title is too short, please enter at least 2 characters.") except: malformed = True messages.append("Please enter a title.") ## Validate the description try: fp = open(file_storing_description) description = fp.read() fp.close() if len(description) < 10: malformed = True messages.append("The description must be at least 10 characters long.") except: malformed = True messages.append("Please enter a description.") ## Validate author try: fp = open(file_storing_author) author = fp.read() fp.close() except: malformed = True messages.append("Please enter at least one author.") ## Validate year try: fp = open(file_storing_year) year = fp.read() fp.close() except: malformed = True messages.append("Please enter a year.") try: if int(year) < 1000 or int(year) > 9999: malformed = True messages.append("Please enter a valid year. It must consist of 4 digits.") except: malformed = True messages.append("Please enter a valid year. It must consist of 4 digits.") ## Validate the aspect ratio try: fp = open(file_storing_aspect) aspect = fp.read() fp.close() try: aspectx, aspecty = aspect.split(':') aspectx = int(aspectx) aspecty = int(aspecty) except: malformed = True messages.append("Aspect ratio was not provided as 'Number:Number' format") except: malformed = True messages.append("Please enter an aspect ratio.") if malformed: raise InvenioWebSubmitFunctionStop(""" <SCRIPT> document.forms[0].action="/submit"; document.forms[0].curpage.value = 1; document.forms[0].step.value = 0; user_must_confirm_before_leaving_page = false; alert('%s'); document.forms[0].submit(); </SCRIPT>""" % "\\n".join(messages) ) else: return
def DEMOVID_Validation(parameters, curdir, form, user_info=None): """ """ messages = [] malformed = False file_storing_path = os.path.join(curdir, "files", str(user_info['uid']), "NewFile", 'filepath') file_storing_name = os.path.join(curdir, "files", str(user_info['uid']), "NewFile", 'filename') file_storing_aspect = os.path.join(curdir, "DEMOVID_ASPECT") file_storing_title = os.path.join(curdir, "DEMOVID_TITLE") file_storing_description = os.path.join(curdir, "DEMOVID_DESCR") file_storing_author = os.path.join(curdir, "DEMOVID_AU") file_storing_year = os.path.join(curdir, "DEMOVID_YEAR") ## Validate the uploaded video try: fp = open(file_storing_path) fullpath = fp.read() fp.close() fp = open(file_storing_name) filename = fp.read() fp.close() if not probe(fullpath) or os.path.splitext(filename)[1] in [ 'jpg', 'jpeg', 'gif', 'tiff', 'bmp', 'png', 'tga' ]: malformed = True messages.append( "The uploaded file is not a supported video format.") except: malformed = True messages.append("Please upload a video.") ## Validate the title try: fp = open(file_storing_title) title = fp.read() fp.close() if len(title) < 2 or len(title) > 50: malformed = True messages.append( "The title is too short, please enter at least 2 characters.") except: malformed = True messages.append("Please enter a title.") ## Validate the description try: fp = open(file_storing_description) description = fp.read() fp.close() if len(description) < 10: malformed = True messages.append( "The description must be at least 10 characters long.") except: malformed = True messages.append("Please enter a description.") ## Validate author try: fp = open(file_storing_author) author = fp.read() fp.close() except: malformed = True messages.append("Please enter at least one author.") ## Validate year try: fp = open(file_storing_year) year = fp.read() fp.close() except: malformed = True messages.append("Please enter a year.") try: if int(year) < 1000 or int(year) > 9999: malformed = True messages.append( "Please enter a valid year. It must consist of 4 digits.") except: malformed = True messages.append( "Please enter a valid year. It must consist of 4 digits.") ## Validate the aspect ratio try: fp = open(file_storing_aspect) aspect = fp.read() fp.close() try: aspectx, aspecty = aspect.split(':') aspectx = int(aspectx) aspecty = int(aspecty) except: malformed = True messages.append( "Aspect ratio was not provided as 'Number:Number' format") except: malformed = True messages.append("Please enter an aspect ratio.") if malformed: raise InvenioWebSubmitFunctionStop(""" <SCRIPT> document.forms[0].action="/submit"; document.forms[0].curpage.value = 1; document.forms[0].step.value = 0; user_must_confirm_before_leaving_page = false; alert('%s'); document.forms[0].submit(); </SCRIPT>""" % "\\n".join(messages)) else: return