def test_ffrobe_metadata(self):
"""Test if ffprobe metadata outputs correctly"""
metadata_check = {
"format": {
"TAG:album": '"Invenio Album"',
"TAG:album_artist": '"Invenio Album Artist"',
"TAG:comment": '"Invenio Comment"',
"TAG:compatible_brands": "isomiso2avc1mp41",
"TAG:copyright": '"Invenio Copyright"',
"TAG:creation_time": "1970-01-01 00:00:00",
"TAG:description": '"Invenio Description"',
"TAG:encoder": "Lavf53.1.0",
"TAG:episode_id": '"S04x42"',
"TAG:genre": '"Invenio Genre"',
"TAG:grouping": '"Invenio Grouping"',
"TAG:lyrics": '"Invenio Lyrics"',
"TAG:major_brand": "isom",
"TAG:minor_version": "512",
"TAG:network": '"Invenio Network"',
"TAG:show": '"Invenio Show"',
"TAG:synopsis": '"Invenio Synopsis"',
"TAG:title": '"Super Duper Difficult Test Metadata Video File"',
"bit_rate": "7606651.000000 ",
"duration": "10.000000 ",
"filename": "/home/oldi/videos/park_joy_1080p.mp4",
"format_long_name": "QuickTime/MPEG-4/Motion JPEG 2000 format",
"format_name": "mov,mp4,m4a,3gp,3g2,mj2",
"nb_streams": "1",
"size": "9508314.000000 ",
"start_time": "0.000000 ",
},
"streams": [
{
"TAG:creation_time": "1970-01-01 00:00:00",
"TAG:language": "und",
"avg_frame_rate": "50/1",
"codec_long_name": "H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10",
"codec_name": "h264",
"codec_tag": "0x31637661",
"codec_tag_string": "avc1",
"codec_time_base": "1/100",
"codec_type": "video",
"display_aspect_ratio": "30:17",
"duration": "10.000000 ",
"has_b_frames": "2",
"height": "1088",
"index": "0",
"nb_frames": "500",
"pix_fmt": "yuv420p",
"r_frame_rate": "50/1",
"sample_aspect_ratio": "1:1",
"start_time": "0.000000 ",
"time_base": "1/50",
"width": "1920",
}
],
}
self.assertEqual(bibencode_metadata.ffprobe_metadata(video01_out01), metadata_check)
def test_ffrobe_metadata(self):
"""Test if ffprobe metadata outputs correctly"""
metadata_check = {
'format': {
'TAG:album': '"Invenio Album"',
'TAG:album_artist': '"Invenio Album Artist"',
'TAG:comment': '"Invenio Comment"',
'TAG:compatible_brands': 'isomiso2avc1mp41',
'TAG:copyright': '"Invenio Copyright"',
'TAG:creation_time': '1970-01-01 00:00:00',
'TAG:description': '"Invenio Description"',
'TAG:encoder': 'Lavf53.1.0',
'TAG:episode_id': '"S04x42"',
'TAG:genre': '"Invenio Genre"',
'TAG:grouping': '"Invenio Grouping"',
'TAG:lyrics': '"Invenio Lyrics"',
'TAG:major_brand': 'isom',
'TAG:minor_version': '512',
'TAG:network': '"Invenio Network"',
'TAG:show': '"Invenio Show"',
'TAG:synopsis': '"Invenio Synopsis"',
'TAG:title':
'"Super Duper Difficult Test Metadata Video File"',
'bit_rate': '7606651.000000 ',
'duration': '10.000000 ',
'filename': '/home/oldi/videos/park_joy_1080p.mp4',
'format_long_name': 'QuickTime/MPEG-4/Motion JPEG 2000 format',
'format_name': 'mov,mp4,m4a,3gp,3g2,mj2',
'nb_streams': '1',
'size': '9508314.000000 ',
'start_time': '0.000000 '
},
'streams': [{
'TAG:creation_time': '1970-01-01 00:00:00',
'TAG:language': 'und',
'avg_frame_rate': '50/1',
'codec_long_name': 'H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10',
'codec_name': 'h264',
'codec_tag': '0x31637661',
'codec_tag_string': 'avc1',
'codec_time_base': '1/100',
'codec_type': 'video',
'display_aspect_ratio': '30:17',
'duration': '10.000000 ',
'has_b_frames': '2',
'height': '1088',
'index': '0',
'nb_frames': '500',
'pix_fmt': 'yuv420p',
'r_frame_rate': '50/1',
'sample_aspect_ratio': '1:1',
'start_time': '0.000000 ',
'time_base': '1/50',
'width': '1920'
}]
}
self.assertEqual(bibencode_metadata.ffprobe_metadata(video01_out01),
metadata_check)
def test_ffrobe_metadata(self):
"""Test if ffprobe metadata outputs correctly"""
metadata_check = {
'format': {'TAG:album': '"Invenio Album"',
'TAG:album_artist': '"Invenio Album Artist"',
'TAG:comment': '"Invenio Comment"',
'TAG:compatible_brands': 'isomiso2avc1mp41',
'TAG:copyright': '"Invenio Copyright"',
'TAG:creation_time': '1970-01-01 00:00:00',
'TAG:description': '"Invenio Description"',
'TAG:encoder': 'Lavf53.1.0',
'TAG:episode_id': '"S04x42"',
'TAG:genre': '"Invenio Genre"',
'TAG:grouping': '"Invenio Grouping"',
'TAG:lyrics': '"Invenio Lyrics"',
'TAG:major_brand': 'isom',
'TAG:minor_version': '512',
'TAG:network': '"Invenio Network"',
'TAG:show': '"Invenio Show"',
'TAG:synopsis': '"Invenio Synopsis"',
'TAG:title': '"Super Duper Difficult Test Metadata Video File"',
'bit_rate': '7606651.000000 ',
'duration': '10.000000 ',
'filename': '/home/oldi/videos/park_joy_1080p.mp4',
'format_long_name': 'QuickTime/MPEG-4/Motion JPEG 2000 format',
'format_name': 'mov,mp4,m4a,3gp,3g2,mj2',
'nb_streams': '1',
'size': '9508314.000000 ',
'start_time': '0.000000 '},
'streams': [{'TAG:creation_time': '1970-01-01 00:00:00',
'TAG:language': 'und',
'avg_frame_rate': '50/1',
'codec_long_name': 'H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10',
'codec_name': 'h264',
'codec_tag': '0x31637661',
'codec_tag_string': 'avc1',
'codec_time_base': '1/100',
'codec_type': 'video',
'display_aspect_ratio': '30:17',
'duration': '10.000000 ',
'has_b_frames': '2',
'height': '1088',
'index': '0',
'nb_frames': '500',
'pix_fmt': 'yuv420p',
'r_frame_rate': '50/1',
'sample_aspect_ratio': '1:1',
'start_time': '0.000000 ',
'time_base': '1/50',
'width': '1920'}]}
self.assertEqual(bibencode_metadata.ffprobe_metadata(video01_out01), metadata_check)
def determine_aspect(input_file):
""" Checks video metadata to find the display aspect ratio.
Returns None if the DAR is not stored in the video container.
@param input_file: full path of the video
@type input_file: string
"""
videoinfo = ffprobe_metadata(input_file)
if not videoinfo:
return None
for stream in videoinfo['streams']:
if stream['codec_type'] == 'video':
fwidth = int(stream['width'])
fheight = int(stream['height'])
if 'display_aspect_ratio' in stream:
return (stream['display_aspect_ratio'], fwidth, fheight)
return (None, fwidth, fheight)
if aspect:
aspx, aspy = aspect.split(':')
else:
the_gcd = gcd(width, height)
aspx = str(width / the_gcd)
aspy = str(height / the_gcd)
json_response['aspx'] = aspx
json_response['aspy'] = aspy
except TypeError:
## If the aspect detection completely fails
os.remove(new_tmp_fullpath)
register_exception(req=req, alert_admin=False)
raise apache.SERVER_RETURN(apache.HTTP_FORBIDDEN)
## Try to extract some metadata from the video container
metadata = ffprobe_metadata(new_tmp_fullpath)
json_response['meta_title'] = metadata['format'].get('TAG:title')
json_response['meta_description'] = metadata['format'].get('TAG:description')
json_response['meta_year'] = metadata['format'].get('TAG:year')
json_response['meta_author'] = metadata['format'].get('TAG:author')
## Empty file name
else:
raise apache.SERVER_RETURN(apache.HTTP_BAD_REQUEST)
## We found our file, we can break the loop
break;
# Send our response
if CFG_JSON_AVAILABLE:
dumped_response = json.dumps(json_response)
def propose_resolutions(video_file, display_aspect=None, res_16_9=['1920x1080', '1280x720', '854x480', '640x360'], res_4_3=['640x480'], lq_fallback=True):
""" Returns a list of possible resolutions that would work with the given
video, based on its own resultion ans aspect ratio
@ param display_aspect: Sets the display aspect ratio for videos where
this might not be detectable
@param res_16_9: Possible resolutions to select from for 16:9 videos
@param res_4_3: Possible resolutions to select from for 4:3 videos
@param lq_fallback: Return the videos own resultion if none of the given fits
"""
def eq(a,b):
if abs(a-b) < 0.01:
return 1
else:
return 0
def get_smaler_or_equal_res(height, avail_res):
smaler_res = []
for res in avail_res:
vres = int(res.split('x')[1])
if vres <= height:
smaler_res.append(res)
return smaler_res
def get_res_for_weird_aspect(width, aspect, avail_res):
smaler_res = []
for res in avail_res:
hres, vres = res.split('x')
hres = int(hres)
vres = int(vres)
if hres <= width:
height = hres * (1.0 / aspect)
if height % 2 != 0:
height = height-1
smaler_res.append(str(hres) + 'x' + str(int(height)))
return smaler_res
meta_dict = ffprobe_metadata(video_file)
for stream in meta_dict['streams']:
if stream['codec_type'] == 'video':
width = int(stream['width'])
height = int(stream['height'])
# If the display aspect ratio is in the meta, we can even override
# the ratio that was given to the function as a fallback
# But the information in the file could be wrong ...
# Which information is trustful?
if 'display_aspect_ratio' in stream:
display_aspect = stream['display_aspect_ratio']
break
# Calculate the aspect factors
if display_aspect == None:
# Assume square pixels
display_aspect = float(width) / float(height)
else:
asp_w, asp_h = display_aspect.split(':')
display_aspect = float(asp_w) / float(asp_h)
# Check if 16:9
if eq(display_aspect, 1.77):
possible_res = get_smaler_or_equal_res(height, res_16_9)
# Check if 4:3
elif eq(display_aspect, 1.33):
possible_res = get_smaler_or_equal_res(height, res_4_3)
# Weird aspect
else:
possible_res = get_res_for_weird_aspect(width, display_aspect, res_16_9)
# If the video is crap
if not possible_res and lq_fallback:
return [str(width) + 'x' + str(height)]
else:
return possible_res
def encode_video(input_file, output_file,
acodec=None, vcodec=None,
abitrate=None, vbitrate=None,
resolution=None,
passes=1,
special=None, specialfirst=None, specialsecond=None,
metadata=None,
width=None, height=None, aspect=None,
profile=None,
update_fnc=task_update_progress,
message_fnc=write_message
):
""" Starts an ffmpeg encoding process based on the given parameters.
The encoding is run as a subprocess. The progress of the subprocess is
continiously written to the given messaging functions. In a normale case,
these should be the ones of BibTask.
@param input_file: Path to the input video.
@type input_file: string
@param output_file: Path to the output file. If no other parameters are giv
than input and output files, FFmpeg tries to auto-discover the right codecs
for the given file extension. In this case, every other aspect like
resolution and bitrates will be the same as in the input video.
@type output_file: string
@param acodec: The audio codec to use. This must be an available codec of
libavcodec within FFmpeg.
@type acodec: string
@param vcodec: The video codec to use. This must be an available codec of
libavcodec within FFmpeg.
@type vcodec: string
@param abitrate: Bitrate of the audio stream. In bit/s.
@type abitrate: int
@param vbitrate: Bitrate of the video stream. In bit/s.
@type vbitrate: int
@param resolution: Fixed size of the frames in the transcoded video.
FFmpeg notation: 'WxH' or preset like 'vga'. See also 'width'
@param passes: Number of encoding passes. Either 1 or 2.
@type passes: int
@param special: Additional FFmpeg parameters.
@type special: string
@param specialfirst: Additional FFmpeg parameters for the first pass.
The 'special' parameter is ignored if this ist not 'None'
@type specialfirst: string
@param specialsecond: Additional FFmpeg parameters for the second pass.
The 'special' parameter is ignored if this is not 'None'
@type specialsecond: string
@param metadata: Metadata that should be added to the transcoded video.
This must be a dictionary. As with as metadata in FFmpeg, there is no
guarantee that the metadata specified in the dictionary will really be added
to the file, because it will largelydepend on the container format and its
supported fields.
@type metadata: dict
@param width: Instead of giving a fixed resolution, you can use width and
height as dimensional constrains. The algorithm will try to preserve the
original aspect and fit the new frame size into the given dimensions.
@type width: int
@param height: see 'width'
@type height: int
@param aspect: A float representing the aspect ratio of the video:
4:3 equals 1.33 and 16:9 equals 1.77.
This is a fallback in case the algorithm fails to determine the real aspect
ratio from the video. See also 'width'
@type aspect: float or "4:3" like string
@param profile: A profile to use. The priority is on the parameters
directly given to the function.
@type profile: string
@param update_fnc: A function called to display or log an the encoding
status. This function must accept a string.
@type update_fnc: function
@param message_fnc: A function to log important messages or errors.
This function must accept a string.
@type message_fnc: function
@return: True if the encoding was successful, False if not
@rtype: boolean
"""
def encode():
""" Subfunction to run the acutal encoding
"""
## Start process
process = subprocess.Popen(command,
stderr=log_file_handle,
close_fds=True)
## While the process is running
time.sleep(1)
while process.poll() is None:
# Update the status in bibsched
update_status()
time.sleep(4)
## If the process was terminated
if process.poll() == -15:
# Encoding was terminated by system
message_fnc("FFMPEG was terminated")
update_fnc(" FFMPEG was terminated")
return 0
## If there was an error during encoding
if process.poll() == 1:
update_fnc(" An FFMPEG error has appeared, see log")
message_fnc("An FFMPEG error has appeared encoding %s" % output_file)
message_fnc("Command was: %s" % ' '.join(command))
message_fnc("Last lines of the FFmpeg log:")
## open the logfile again an retrieve the size
log_file_handle2 = open(log_file_name, 'rb')
size = os.fstat(log_file_handle2.fileno())[6]
## Read the last lines
log_file_handle2.seek(-min(size, 10000), 2)
lastlines = log_file_handle2.read().splitlines()[-5:]
for line in lastlines:
message_fnc(line)
return 0
## If everything went fine
if process.poll() == 0:
message_fnc("Encoding of %s done" % output_file)
update_fnc("Encoding of %s done" % output_file)
return 1
def build_command(nofpass=1):
""" Builds the ffmpeg command according to the function params
"""
def insert(key, value):
""" Shortcut for inserting parameters into the arg list
"""
base_args.insert(-1, key)
base_args.insert(-1, value)
## Determine base command arguments from the pass to run
base_args = None
if passes == 1:
base_args = [CFG_PATH_FFMPEG, '-y', '-i', input_file, output_file]
elif passes == 2:
if nofpass == 1:
base_args = [CFG_PATH_FFMPEG, '-y', '-i', input_file,
'-pass', '1', '-passlogfile', pass_log_file,
'-an', '-f', 'rawvideo', '/dev/null']
elif nofpass == 2:
base_args = [CFG_PATH_FFMPEG, '-y', '-i', input_file,
'-pass', '2', '-passlogfile',
pass_log_file, output_file]
## Insert additional arguments
if acodec is not None:
insert('-acodec', acodec)
if vcodec is not None:
insert('-vcodec', vcodec)
if abitrate is not None:
insert('-b:a', str(abitrate))
if vbitrate is not None:
insert('-b:v', str(vbitrate))
## If a resolution is given
if resolution:
insert('-s', resolution)
## If not, you can give width and height and generate the resolution
else:
## Use our new function to get the size of the input
nresolution = determine_resolution_preserving_aspect(input_file,
width,
height,
aspect)
insert('-s', nresolution)
## Metadata additions
if type(metadata) is type(dict()):
## build metadata arguments for ffmpeg
for key, value in metadata.iteritems():
if value is not None:
meta_arg = (
CFG_BIBENCODE_FFMPEG_METADATA_ARGUMENT % (key, value)
)
insert("-metadata", meta_arg)
## Special argument additions
if passes == 1:
if passes == 1 and special is not None:
for val in special.split():
base_args.insert(-1, val)
elif passes == 2:
if nofpass == 1:
if specialfirst is not None:
for val in specialfirst.split():
base_args.insert(-1, val)
if nofpass == 2:
if specialsecond is not None:
for val in specialsecond.split():
base_args.insert(-1, val)
return base_args
def update_status():
""" Parses the encoding status and updates the task in bibsched
"""
def graphical(value):
""" Converts a percentage value to a nice graphical representation
"""
## If the given value is a valid precentage
if value >= 0 and value <= 100:
## This is to get nice, aligned output in bibsched
oval = str(value).zfill(3)
return (
"[" + "#"*(value/10) + " "*(10-(value/10)) +
"][%d/%d] %s%%" % (nofpass, passes, oval)
)
else:
## Sometimes the parsed values from FFMPEG are totaly off.
## Or maybe nneeded values are not avail. for the given video.
## In this case there is no estimate.
return "[ no est. ][%d/%d] " % (nofpass, passes)
## init variables
time_string = '0.0'
percentage_done = -1
## try to read the encoding log
try:
filehandle = open(log_file_name, 'rb')
except IOError:
message_fnc("Error opening %s" % log_file_name)
update_fnc("Could not open encoding log")
return
## Check the size of the file before reading from the end
size = os.path.getsize(log_file_name)
if not size:
return
## Go to the end of the log
filehandle.seek(-min(10000, size), 2)
chunk = filehandle.read()
lines = chunk.splitlines()
## try to parse the status
for line in reversed(lines):
if CFG_BIBENCODE_FFMPEG_ENCODE_TIME.match(line):
time_string = (
CFG_BIBENCODE_FFMPEG_ENCODE_TIME.match(line).groups()
)[0]
break
filehandle.close()
try:
percentage_done = int(timecode_to_seconds(time_string) / total_seconds * 100)
except:
precentage_done = -1
## Now update the bibsched progress
opath, ofile = os.path.split(output_file)
if len(opath) > 8:
opath = "..." + opath[-8:]
ohint = opath + '/' + ofile
update_fnc(graphical(percentage_done) + " > " + ohint)
#------------------#
# PROFILE HANDLING #
#------------------#
if profile:
profile = get_encoding_profile(profile)
acodec = chose(acodec, 'audiocodec', profile)
vcodec = chose(vcodec, 'videocodec', profile)
abitrate = chose(abitrate, 'audiobitrate', profile)
vbitrate = chose(vbitrate, 'videobitrate', profile)
resolution = chose(resolution, 'resolution', profile)
passes = getval(profile, 'passes', 1)
special = chose(special, 'special', profile)
specialfirst = chose(specialfirst, 'special_firstpass', profile)
specialsecond = chose(specialsecond, 'special_secondpass', profile)
metadata = chose(metadata, 'metadata', profile)
width = chose(width, 'width', profile)
height = chose(height, 'height', profile)
aspect = chose(aspect, 'aspect', profile)
#----------#
# ENCODING #
#----------#
## Mark Task as stoppable
# task_sleep_now_if_required()
tech_metadata = ffprobe_metadata(input_file)
try:
total_seconds = float(tech_metadata['format']['duration'])
except:
total_seconds = 0.0
## Run the encoding
pass_log_file = CFG_BIBENCODE_FFMPEG_PASSLOGFILE_PREFIX % (
os.path.splitext(os.path.split(input_file)[1])[0],
str(uuid.uuid4()))
no_error = True
## For every encoding pass to do
for apass in range(0, passes):
nofpass = apass + 1
if no_error:
## Create Logfiles
log_file_name = _filename_log(output_file, nofpass)
try:
log_file_handle = open(log_file_name, 'w')
except IOError:
message_fnc("Error creating %s" % log_file_name)
update_fnc("Error creating logfile")
return 0
## Build command for FFMPEG
command = build_command(nofpass)
## Start encoding, result will define to continue or not to
no_error = encode()
## !!! Status Update
return no_error
def extract_frames(input_file,
output_file=None,
size=None,
positions=None,
numberof=None,
extension='jpg',
width=None,
height=None,
aspect=None,
profile=None,
update_fnc=task_update_progress,
message_fnc=write_message):
""" Extracts frames from a given video using ffmpeg based on the given
parameters. Starts a subprocess. The status of the process is continously
written to the given messaging functions.
@param input_file: Full path to the input video.
@type input_file: String
@param output_file: Full path to the output file, in case of multiple outs,
there will be squential numbers appended to the file's name
automatically. If this parameter is not given, the
output filename will be generated from the input file
The output can be substituted with information.
Valid substrings for substitution are:
%(input)s for the input filename
%(timecode)s for the timecode
%(size)s for the frame size
%(number)d for sequential numbers
Everything else that could be a python substitution substring
should be escaped accordingly.
!!! Warnning !!! FFmpeg will also try to substitude if there
are any '%' left. This will likely screw up the extraction.
@type output_file: string
@param size: The size of the frames. Format is WxH
@type size: string
@param positions: A list of positions within the video where the frames
should be shot. Percentual values between 0 and 100 or
HH:MM:SS.ss are accepted.
@type positions: string
@param numberof: In case you don't want to give positions but just a fixed
number of frames to extract.
@type numberof: nt
@param extension: If no output filename is given, construct the name with
this extension
@type extension: string
@param width: The width of the extracted frame.
@type width: int
@param height: The height of the extracted frame
@type height: int
@param aspect: A float representing the aspect ratio of the video.
4:3 equals 1.33 and 16:9 equals 1.77. See also 'width'
@type aspect: float or "4:3" like string
@param profile: A profile to use. The priority is on the parameters directly
given to the function.
@type profile: string
@param update_fnc: A function called to display or log an the encoding
status. This function must accept a string.
@type update_fnc: function
@param message_fnc: A function to log important messages or errors.
This function must accept a string.
@type message_fnc: function
@return: 1 if the extraction was successful, 0 if not
@rtype: bool
"""
#---------#
# PROFILE #
#---------#
## Takes parameters from the profile if they are not directly given
if profile:
profile = get_extract_profile(profile)
size = chose(size, 'size', profile)
positions = chose(positions, 'positions', profile)
numberof = chose(numberof, 'numberof', profile)
extension = chose(extension, 'extension', profile)
width = chose(width, 'width', profile)
height = chose(height, 'height', profile)
#---------------#
# Check and fix #
#---------------#
## If neither positions nor a number of shots are given
if not positions and not numberof:
raise ValueError(
"Either argument \'positions\' xor argument \'numberof\' must be given"
)
## If both are given
if positions and numberof:
raise ValueError(
"Arguments \'positions\' and \'numberof\' exclude each other")
## If just a number of shots to take is given by 'numberof'
if numberof and not positions:
## Parse the duration from the input
info = ffprobe_metadata(input_file)
if info is None:
message_fnc("An error occured while receiving the video log")
return 0
duration = float(info['format']['duration'])
if duration is None:
message_fnc(
"Could not extract by \'numberof\' because video duration is unknown."
)
return 0
positions = []
for pos in range(numberof):
## Calculate the position for every shot and append it to the list
position = pos * (duration / numberof)
positions.append(position)
## If specific positions are given
elif positions and not numberof:
## Check if correct timecodes or seconds are given
i = 0
for pos in positions:
if not (is_seconds(pos) or is_timecode(pos)):
raise ValueError(
"The given position \'%s\' is neither a value in seconds nor a timecode!"
% str(pos))
## if a timecode is given, convert it to seconds
if is_timecode(pos):
positions[i] = timecode_to_seconds(pos)
i += 1
## If no output filename is given, use input filename and append jpg
if output_file is None:
ipath = os.path.splitext(input_file)[0]
if not extension.startswith("."):
extension = "." + extension
output_file = ipath + extension
## If no explizit size for the frames is given
if not size:
size = determine_resolution_preserving_aspect(input_file, width,
height, aspect)
#------------#
# Extraction #
#------------#
counter = 1
for position in positions:
#---------------------------#
# Generate output file name #
#---------------------------#
number_substituted = False
if '%(number)' in output_file:
number_substituted = True
## If the output filename should be stubstituted
try:
output_filename = output_file % {
'input': os.path.splitext(os.path.split(input_file)[1])[0],
'timecode': seconds_to_timecode(position),
'size': size,
'number': counter
}
except KeyError:
raise
## In case that more than one shot is taken and you don't want to substitute
if not number_substituted:
if len(positions) > 1:
path, ext = os.path.splitext(output_file)
output_filename = path + str(counter).zfill(
len(str(len(positions)))) + ext
## If you dont want to substitute and only one file is selected,
## it will just take the output or input name without altering it
else:
output_filename = output_file
#-------------#
# Run process #
#-------------#
## Build the command for ffmpeg
command = (CFG_BIBENCODE_FFMPEG_EXTRACT_COMMAND %
(position, input_file, size, output_filename)).split()
## Start subprocess and poll the output until it finishes
process = subprocess.Popen(command, stderr=subprocess.PIPE)
stderr = []
while process.poll() is None:
## We want to keep the last lines of output in case of an error
stderr += process.communicate()[1].splitlines()
stderr = stderr[-5:]
## If something went wrong, print the last lines of the log
if process.poll() != 0:
msg = ("Error while extracting frame %d of %d" %
(counter, len(positions)))
message_fnc(msg)
update_fnc(msg)
## Print the end of the log
message_fnc("Last lines of the FFmpeg log:")
for line in stderr:
message_fnc(line)
return 0
else:
update_fnc("Frame %d of %d extracted" % (counter, len(positions)))
counter += 1
## Everything should be fine if this position is reached
message_fnc("Extraction of frames was successful")
return 1
if aspect:
aspx, aspy = aspect.split(':')
else:
the_gcd = gcd(width, height)
aspx = str(width / the_gcd)
aspy = str(height / the_gcd)
json_response['aspx'] = aspx
json_response['aspy'] = aspy
except TypeError:
## If the aspect detection completely fails
os.remove(new_tmp_fullpath)
register_exception(req=req, alert_admin=False)
raise apache.SERVER_RETURN(apache.HTTP_FORBIDDEN)
## Try to extract some metadata from the video container
metadata = ffprobe_metadata(new_tmp_fullpath)
json_response['meta_title'] = metadata['format'].get(
'TAG:title')
json_response['meta_description'] = metadata[
'format'].get('TAG:description')
json_response['meta_year'] = metadata['format'].get(
'TAG:year')
json_response['meta_author'] = metadata['format'].get(
'TAG:author')
## Empty file name
else:
raise apache.SERVER_RETURN(apache.HTTP_BAD_REQUEST)
## We found our file, we can break the loop
break
# Send our response
def upload_video(self, req, form):
"""
A clone of uploadfile but for (large) videos.
Does not copy the uploaded file to the websubmit directory.
Instead, the path to the file is stored inside the submission directory.
"""
def gcd(a, b):
""" the euclidean algorithm """
while a:
a, b = b % a, a
return b
from invenio.bibencode_extract import extract_frames
from invenio.bibencode_config import CFG_BIBENCODE_WEBSUBMIT_ASPECT_SAMPLE_DIR, CFG_BIBENCODE_WEBSUBMIT_ASPECT_SAMPLE_FNAME
from invenio.bibencode_encode import determine_aspect
from invenio.bibencode_utils import probe
from invenio.bibencode_metadata import ffprobe_metadata
from invenio.websubmit_config import CFG_WEBSUBMIT_TMP_VIDEO_PREFIX
argd = wash_urlargd(
form, {
'doctype': (str, ''),
'access': (str, ''),
'indir': (str, ''),
'session_id': (str, ''),
'rename': (str, ''),
})
curdir = None
if not form.has_key("indir") or \
not form.has_key("doctype") or \
not form.has_key("access"):
raise apache.SERVER_RETURN(apache.HTTP_BAD_REQUEST)
else:
curdir = os.path.join(CFG_WEBSUBMIT_STORAGEDIR, argd['indir'],
argd['doctype'], argd['access'])
user_info = collect_user_info(req)
if form.has_key("session_id"):
# Are we uploading using Flash, which does not transmit
# cookie? The expect to receive session_id as a form
# parameter. First check that IP addresses do not
# mismatch.
uid = session.uid
user_info = collect_user_info(uid)
try:
act_fd = file(os.path.join(curdir, 'act'))
action = act_fd.read()
act_fd.close()
except:
act = ""
# Is user authorized to perform this action?
(auth_code, auth_message) = acc_authorize_action(
uid,
"submit",
authorized_if_no_roles=not isGuestUser(uid),
verbose=0,
doctype=argd['doctype'],
act=action)
if acc_is_role("submit", doctype=argd['doctype'],
act=action) and auth_code != 0:
# User cannot submit
raise apache.SERVER_RETURN(apache.HTTP_UNAUTHORIZED)
else:
# Process the upload and get the response
json_response = {}
for key, formfields in form.items():
filename = key.replace("[]", "")
if hasattr(formfields, "filename") and formfields.filename:
dir_to_open = os.path.abspath(
os.path.join(curdir, 'files', str(user_info['uid']),
key))
try:
assert (
dir_to_open.startswith(CFG_WEBSUBMIT_STORAGEDIR))
except AssertionError:
register_exception(req=req,
prefix='curdir="%s", key="%s"' %
(curdir, key))
raise apache.SERVER_RETURN(apache.HTTP_FORBIDDEN)
if not os.path.exists(dir_to_open):
try:
os.makedirs(dir_to_open)
except OSError, e:
if e.errno != errno.EEXIST:
# If the issue is only that directory
# already exists, then continue, else
# report
register_exception(req=req, alert_admin=True)
raise apache.SERVER_RETURN(
apache.HTTP_FORBIDDEN)
filename = formfields.filename
## Before saving the file to disc, wash the filename (in particular
## washing away UNIX and Windows (e.g. DFS) paths):
filename = os.path.basename(filename.split('\\')[-1])
filename = filename.strip()
if filename != "":
# Check that file does not already exist
while os.path.exists(
os.path.join(dir_to_open, filename)):
#dirname, basename, extension = decompose_file(new_destination_path)
basedir, name, extension = decompose_file(filename)
new_name = propose_next_docname(name)
filename = new_name + extension
#-------------#
# VIDEO STUFF #
#-------------#
## Remove all previous uploads
filelist = os.listdir(
os.path.split(formfields.file.name)[0])
for afile in filelist:
if argd['access'] in afile:
os.remove(
os.path.join(
os.path.split(formfields.file.name)[0],
afile))
## Check if the file is a readable video
## We must exclude all image and audio formats that are readable by ffprobe
if (os.path.splitext(filename)[1] in [
'jpg', 'jpeg', 'gif', 'tiff', 'bmp', 'png',
'tga', 'jp2', 'j2k', 'jpf', 'jpm', 'mj2',
'biff', 'cgm', 'exif', 'img', 'mng', 'pic',
'pict', 'raw', 'wmf', 'jpe', 'jif', 'jfif',
'jfi', 'tif', 'webp', 'svg', 'ai', 'ps', 'psd',
'wav', 'mp3', 'pcm', 'aiff', 'au', 'flac',
'wma', 'm4a', 'wv', 'oga', 'm4a', 'm4b', 'm4p',
'm4r', 'aac', 'mp4', 'vox', 'amr', 'snd'
] or not probe(formfields.file.name)):
formfields.file.close()
raise apache.SERVER_RETURN(apache.HTTP_FORBIDDEN)
## We have no "delete" attribute in Python 2.4
if sys.hexversion < 0x2050000:
## We need to rename first and create a dummy file
## Rename the temporary file for the garbage collector
new_tmp_fullpath = os.path.split(
formfields.file.name
)[0] + "/" + CFG_WEBSUBMIT_TMP_VIDEO_PREFIX + argd[
'access'] + "_" + os.path.split(
formfields.file.name)[1]
os.rename(formfields.file.name, new_tmp_fullpath)
dummy = open(formfields.file.name, "w")
dummy.close()
formfields.file.close()
else:
# Mark the NamedTemporatyFile as not to be deleted
formfields.file.delete = False
formfields.file.close()
## Rename the temporary file for the garbage collector
new_tmp_fullpath = os.path.split(
formfields.file.name
)[0] + "/" + CFG_WEBSUBMIT_TMP_VIDEO_PREFIX + argd[
'access'] + "_" + os.path.split(
formfields.file.name)[1]
os.rename(formfields.file.name, new_tmp_fullpath)
# Write the path to the temp file to a file in STORAGEDIR
fp = open(os.path.join(dir_to_open, "filepath"), "w")
fp.write(new_tmp_fullpath)
fp.close()
fp = open(os.path.join(dir_to_open, "filename"), "w")
fp.write(filename)
fp.close()
## We are going to extract some thumbnails for websubmit ##
sample_dir = os.path.join(
curdir, 'files', str(user_info['uid']),
CFG_BIBENCODE_WEBSUBMIT_ASPECT_SAMPLE_DIR)
try:
## Remove old thumbnails
shutil.rmtree(sample_dir)
except OSError:
register_exception(req=req, alert_admin=False)
try:
os.makedirs(
os.path.join(curdir, 'files',
str(user_info['uid']),
sample_dir))
except OSError:
register_exception(req=req, alert_admin=False)
try:
extract_frames(
input_file=new_tmp_fullpath,
output_file=os.path.join(
sample_dir,
CFG_BIBENCODE_WEBSUBMIT_ASPECT_SAMPLE_FNAME
),
size="600x600",
numberof=5)
json_response['frames'] = []
for extracted_frame in os.listdir(sample_dir):
json_response['frames'].append(extracted_frame)
except:
## If the frame extraction fails, something was bad with the video
os.remove(new_tmp_fullpath)
register_exception(req=req, alert_admin=False)
raise apache.SERVER_RETURN(apache.HTTP_FORBIDDEN)
## Try to detect the aspect. if this fails, the video is not readable
## or a wrong file might have been uploaded
try:
(aspect, width,
height) = determine_aspect(new_tmp_fullpath)
if aspect:
aspx, aspy = aspect.split(':')
else:
the_gcd = gcd(width, height)
aspx = str(width / the_gcd)
aspy = str(height / the_gcd)
json_response['aspx'] = aspx
json_response['aspy'] = aspy
except TypeError:
## If the aspect detection completely fails
os.remove(new_tmp_fullpath)
register_exception(req=req, alert_admin=False)
raise apache.SERVER_RETURN(apache.HTTP_FORBIDDEN)
## Try to extract some metadata from the video container
metadata = ffprobe_metadata(new_tmp_fullpath)
json_response['meta_title'] = metadata['format'].get(
'TAG:title')
json_response['meta_description'] = metadata[
'format'].get('TAG:description')
json_response['meta_year'] = metadata['format'].get(
'TAG:year')
json_response['meta_author'] = metadata['format'].get(
'TAG:author')
## Empty file name
else:
raise apache.SERVER_RETURN(apache.HTTP_BAD_REQUEST)
## We found our file, we can break the loop
break
# Send our response
if CFG_JSON_AVAILABLE:
dumped_response = json.dumps(json_response)
# store the response in the websubmit directory
# this is needed if the submission is not finished and continued later
response_dir = os.path.join(curdir, 'files',
str(user_info['uid']), "response")
try:
os.makedirs(response_dir)
except OSError:
# register_exception(req=req, alert_admin=False)
pass
fp = open(os.path.join(response_dir, "response"), "w")
fp.write(dumped_response)
fp.close()
return dumped_response
def extract_frames(input_file, output_file=None, size=None, positions=None,
numberof=None, extension='jpg',
width=None, height=None, aspect=None, profile=None,
update_fnc=task_update_progress,
message_fnc=write_message):
""" Extracts frames from a given video using ffmpeg based on the given
parameters. Starts a subprocess. The status of the process is continously
written to the given messaging functions.
@param input_file: Full path to the input video.
@type input_file: String
@param output_file: Full path to the output file, in case of multiple outs,
there will be squential numbers appended to the file's name
automatically. If this parameter is not given, the
output filename will be generated from the input file
The output can be substituted with information.
Valid substrings for substitution are:
%(input)s for the input filename
%(timecode)s for the timecode
%(size)s for the frame size
%(number)d for sequential numbers
Everything else that could be a python substitution substring
should be escaped accordingly.
!!! Warnning !!! FFmpeg will also try to substitude if there
are any '%' left. This will likely screw up the extraction.
@type output_file: string
@param size: The size of the frames. Format is WxH
@type size: string
@param positions: A list of positions within the video where the frames
should be shot. Percentual values between 0 and 100 or
HH:MM:SS.ss are accepted.
@type positions: string
@param numberof: In case you don't want to give positions but just a fixed
number of frames to extract.
@type numberof: nt
@param extension: If no output filename is given, construct the name with
this extension
@type extension: string
@param width: The width of the extracted frame.
@type width: int
@param height: The height of the extracted frame
@type height: int
@param aspect: A float representing the aspect ratio of the video.
4:3 equals 1.33 and 16:9 equals 1.77. See also 'width'
@type aspect: float or "4:3" like string
@param profile: A profile to use. The priority is on the parameters directly
given to the function.
@type profile: string
@param update_fnc: A function called to display or log an the encoding
status. This function must accept a string.
@type update_fnc: function
@param message_fnc: A function to log important messages or errors.
This function must accept a string.
@type message_fnc: function
@return: 1 if the extraction was successful, 0 if not
@rtype: bool
"""
#---------#
# PROFILE #
#---------#
## Takes parameters from the profile if they are not directly given
if profile:
profile = get_extract_profile(profile)
size = chose(size, 'size', profile)
positions = chose(positions, 'positions', profile)
numberof = chose(numberof, 'numberof', profile)
extension = chose(extension, 'extension', profile)
width = chose(width, 'width', profile)
height = chose(height, 'height', profile)
#---------------#
# Check and fix #
#---------------#
## If neither positions nor a number of shots are given
if not positions and not numberof:
raise ValueError("Either argument \'positions\' xor argument \'numberof\' must be given")
## If both are given
if positions and numberof:
raise ValueError("Arguments \'positions\' and \'numberof\' exclude each other")
## If just a number of shots to take is given by 'numberof'
if numberof and not positions:
## Parse the duration from the input
info = ffprobe_metadata(input_file)
if info is None:
message_fnc("An error occured while receiving the video log")
return 0
duration = float(info['format']['duration'])
if duration is None:
message_fnc("Could not extract by \'numberof\' because video duration is unknown.")
return 0
positions = []
for pos in range(numberof):
## Calculate the position for every shot and append it to the list
position = pos * (duration / numberof)
positions.append(position)
## If specific positions are given
elif positions and not numberof:
## Check if correct timecodes or seconds are given
i = 0
for pos in positions:
if not (is_seconds(pos) or is_timecode(pos)):
raise ValueError("The given position \'%s\' is neither a value in seconds nor a timecode!" % str(pos))
## if a timecode is given, convert it to seconds
if is_timecode(pos):
positions[i] = timecode_to_seconds(pos)
i += 1
## If no output filename is given, use input filename and append jpg
if output_file is None:
ipath = os.path.splitext(input_file)[0]
if not extension.startswith("."):
extension = "." + extension
output_file = ipath + extension
## If no explizit size for the frames is given
if not size:
size = determine_resolution_preserving_aspect(input_file, width, height, aspect)
#------------#
# Extraction #
#------------#
counter = 1
for position in positions:
#---------------------------#
# Generate output file name #
#---------------------------#
number_substituted = False
if '%(number)' in output_file:
number_substituted = True
## If the output filename should be stubstituted
try:
output_filename = output_file % {
'input': os.path.splitext(os.path.split(input_file)[1])[0],
'timecode': seconds_to_timecode(position),
'size': size,
'number': counter
}
except KeyError:
raise
## In case that more than one shot is taken and you don't want to substitute
if not number_substituted:
if len(positions) > 1:
path, ext = os.path.splitext(output_file)
output_filename = path + str(counter).zfill(len(str(len(positions)))) + ext
## If you dont want to substitute and only one file is selected,
## it will just take the output or input name without altering it
else:
output_filename = output_file
#-------------#
# Run process #
#-------------#
## Build the command for ffmpeg
command = (CFG_BIBENCODE_FFMPEG_EXTRACT_COMMAND % (
position, input_file, size, output_filename
)).split()
## Start subprocess and poll the output until it finishes
process = subprocess.Popen(command, stderr=subprocess.PIPE)
stderr = []
while process.poll() is None:
## We want to keep the last lines of output in case of an error
stderr += process.communicate()[1].splitlines()
stderr = stderr[-5:]
## If something went wrong, print the last lines of the log
if process.poll() != 0:
msg = ("Error while extracting frame %d of %d" % (counter, len(positions)))
message_fnc(msg)
update_fnc(msg)
## Print the end of the log
message_fnc("Last lines of the FFmpeg log:")
for line in stderr:
message_fnc(line)
return 0
else:
update_fnc("Frame %d of %d extracted" % (counter, len(positions)))
counter += 1
## Everything should be fine if this position is reached
message_fnc("Extraction of frames was successful")
return 1