def transcode(self, video_id):
video = db_session.query(models.Video).filter_by(id=video_id).one_or_none()
video.status = 'encoding'
db_session.commit()
outdir = f"{celery.conf.get('MOVIE_PATH')}/{video.id}"
try:
os.mkdir(outdir)
except FileExistsError:
pass
try:
ffmpeg = FfmpegTranscode(video, self, outdir)
ffmpeg.run()
transcode_video(video, self)
except FfmpegException as e:
video.status = 'error'
video.status_message = str(e)
db_session.commit()
self.update_state(
state = states.FAILURE,
meta = str(e)
)
raise Ignore()
def delete(self, video_id):
owner_id = request.cookies.get(app.config['COOKIE_OWNER_ID'])
video = db_session.query(models.Video).filter_by(
owner=owner_id, id=video_id).one_or_none()
if not video:
abort(404, 'Video not found')
if video.status in ['encoding', 'start-encoding']:
abort(409, 'Cannot delete while encoding')
try:
if (video.playlist):
rm_f(video.playlist)
if (video.orig_file):
rm_f(os.path.join(app.config['MOVIE_PATH'], video.orig_file))
if app.config['STORAGE_BACKEND'] == 'S3':
tasks.s3_delete.delay(video.id)
else:
viddir = f"{app.config['MOVIE_PATH']}/{video.id}"
shutil.rmtree(viddir, ignore_errors=True)
except:
abort(500)
db_session.delete(video)
db_session.commit()
return {'message': 'Video deleted'}, 204
def create_stream(self, command, filename, stream_type):
outfile = f'{self.outdir}/{filename}'
command_hash = hashlib.sha256(str(command).encode('utf-8')).hexdigest()
encoded_files = db_session.query(models.EncodedFile).filter_by(
video_id = self.video.id,
encoded_file_name = filename,
).all()
encoded_file = None
for file in encoded_files:
if file.encoding_hash != command_hash:
db_session.delete(file)
db_session.commit()
encoded_file = None
else:
encoded_file = file
if not encoded_file:
encoded_file = models.EncodedFile(
video_id = self.video.id,
encoded_file_name = filename,
encoding_hash = command_hash,
track_type = stream_type
)
self.encoded_files.append(encoded_file)
self.ffmpeg_command.extend(command)
self.ffmpeg_command.append(outfile)
self.has_work = True
def delete(self, id):
owner_id = request.cookies.get(app.config['COOKIE_OWNER_ID'])
video = db_session.query(models.Video).filter_by(owner=owner_id,
id=id).one_or_none()
if not video:
abort(404)
try:
if (video.playlist):
rm_f(video.playlist)
if (video.orig_file):
rm_f(os.path.join(app.config['MOVIE_PATH'], video.orig_file))
if app.config['STORAGE_BACKEND'] == 'S3':
tasks.s3_delete.delay(video.id)
else:
viddir = f"{app.config['MOVIE_PATH']}/{video.id}"
shutil.rmtree(viddir, ignore_errors=True)
except:
abort(500)
db_session.delete(video)
db_session.commit()
return "Video deleted", 204
def ffmpeg_progress(self, logfile):
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.settimeout(15)
sock.bind(self.socketfile)
sock.listen(1)
ffmpeg_retval = multiprocessing.Value('i', -1)
ffmpeg = multiprocessing.Process(target=self.run_ffmpeg, args=(self.ffmpeg_command, logfile, ffmpeg_retval))
ffmpeg.start()
percentage = 0
speed = 0
ffmpeg_clean_end = False
try:
connection, client_address = sock.accept()
while True:
data = connection.recv(1024)
if data:
string = data.decode('utf-8')
for line in string.splitlines():
if line.startswith('out_time_ms'):
progress = int(line.split('=')[1].strip()) / 1000000
percentage = (progress / self.duration) * 100
percentage = min(percentage, 100)
if line.startswith('speed'):
speed = float(line.split('=')[1].strip().split('x')[0])
if line.startswith('progress'):
if line.split('=')[1].strip() == 'end':
ffmpeg_clean_end = True
self.video.encoding_progress = percentage
self.video.encoding_speed = speed
db_session.commit()
else:
break
except socket.timeout:
pass
finally:
# Give ffmpeg time to shut down
time.sleep(2)
ffmpeg.terminate()
connection.close()
print(f"ffmpeg_clean: {ffmpeg_clean_end}, ffmpeg_retval: {ffmpeg_retval.value}")
if ffmpeg_clean_end and ffmpeg_retval.value == 0:
return
if ffmpeg_clean_end and ffmpeg_retval.value == 255:
raise Ignore
status = ""
if ffmpeg_clean_end and ffmpeg_retval.value != 0:
status = f"Encoding failed at {percentage}%"
else:
status = f"Encoder crash at {percentage}%"
raise FfmpegException(status)
def put(self):
owner_id = request.cookies.get(app.config['COOKIE_OWNER_ID'])
if not owner_id:
abort(403)
args = video_parser.parse_args()
video = models.Video(title=args['title'], owner=owner_id)
db_session.add(video)
db_session.commit()
return video
def run(self):
self.video.status = 'encoding'
self.video.width = self.vwidth
self.video.height = self.vheight
self.video.duration = self.duration
db_session.commit()
if self.has_work:
self.ffmpeg_progress(f'{self.orig_file}.log')
for encoded_file in self.encoded_files:
db_session.add(encoded_file)
db_session.commit()
def post(self, video_id):
args = video_parser.parse_args()
owner_id = request.cookies.get(app.config['COOKIE_OWNER_ID'])
video = db_session.query(models.Video).filter_by(
owner=owner_id, id=video_id).one_or_none()
if not video:
abort(404, 'Video not found')
if args.get('title'):
if not video.status in ['start-encoding', 'encoding']:
video.title = args.get('title')
else:
abort(409, 'Cannot change video title while encoding')
if args.get('tune'):
if not video.status in ['start-encoding', 'encoding']:
video.tune = args.get('tune')
else:
abort(409, 'Cannot change video tuning while encoding')
if args.get('status'):
status = args.get('status')
if status == 'file-waiting':
if not video.status in ['encoding', 'start-encoding']:
video.status = status
video.upload_identifier = None
else:
abort(409, 'Cannot upload new file while encoding')
if status == 'start-encoding':
if video.status in ['file-uploaded', 'ready', 'error']:
video.status = status
video.encoding_progress = 0
video.status_error = ""
video.celery_taskid = tasks.transcode.delay(video.id)
else:
abort(
409,
'Cannot start encoding while video is in this state')
db_session.commit()
return video
def post(self, video_id, subtitle_id):
owner_id = request.cookies.get(app.config['COOKIE_OWNER_ID'])
video = db_session.query(models.Video).filter_by(
owner=owner_id, id=video_id).one_or_none()
subtitle = db_session.query(models.Subtitle).filter_by(
id=subtitle_id, video_id=video_id).one_or_none()
if not video:
abort(404, 'Video not found')
if not subtitle:
abort(404, 'Subtitle not found')
file = request.files['file']
filename = f'{video.id}_sub_{subtitle.id}_orig'
file.save(os.path.join(app.config['MOVIE_PATH'], filename))
subtitle.orig_file_name = filename
db_session.commit()
def post(self, video_id, subtitle_id):
args = new_subtitle_parser.parse_args()
owner_id = request.cookies.get(app.config['COOKIE_OWNER_ID'])
video = db_session.query(models.Video).filter_by(
owner=owner_id, id=video_id).one_or_none()
subtitle = db_session.query(models.Subtitle).filter_by(
id=subtitle_id, video_id=video_id).one_or_none()
if not video:
abort(404, 'Video not found')
if not subtitle:
abort(404, 'Subtitle not found')
subtitle.title = args['title']
subtitle.language = args['language']
db_session.commit()
return subtitle
def put(self, video_id):
args = new_subtitle_parser.parse_args()
owner_id = request.cookies.get(app.config['COOKIE_OWNER_ID'])
if not owner_id:
abort(403)
video = db_session.query(models.Video).filter_by(
owner=owner_id, id=video_id).one_or_none()
if not video:
abort(404, 'Video not found')
subtitle = models.Subtitle(
video_id=video.id,
language=args['language'],
title=args['title'],
)
db_session.add(subtitle)
db_session.commit()
return subtitle
def post(self, video_id):
owner_id = request.cookies.get(app.config['COOKIE_OWNER_ID'])
video = db_session.query(models.Video).filter_by(
owner=owner_id, id=video_id).one_or_none()
if not video:
return {'message': 'Video not found'}, 403
resumableTotalChunks = request.form.get('resumableTotalChunks',
type=int)
resumableChunkNumber = request.form.get('resumableChunkNumber',
default=1,
type=int)
resumableIdentifier = request.form.get('resumableIdentifier',
default='error',
type=str)
resumableFilename = request.form.get('resumableFilename',
default='error',
type=str)
resumableTotalSize = request.form.get('resumableTotalSize',
default=0,
type=int)
resumableChunkSize = request.form.get('resumableChunkSize',
default=0,
type=int)
if not resumableIdentifier or not resumableChunkNumber or not resumableTotalSize or not resumableChunkSize:
return {'message': 'Parameter error'}, 500
target_file_name = target_name(video.id)
chunk_data = request.files['file']
if video.status in ['file-waiting', 'file-uploaded', 'ready', 'error']:
video.status = 'file-uploading'
video.upload_identifier = resumableIdentifier
video.orig_file_name = resumableFilename
video.orig_file = target_file_name.name
db_session.commit()
if video.upload_identifier != resumableIdentifier:
return {'message': 'Different upload already in progress'}, 409
try:
if target_file_name.stat(
).st_size != resumableTotalSize or video.orig_file_name != resumableFilename:
rm_f(target_file_name)
except FileNotFoundError:
pass
if not target_file_name.exists():
target_file = open(target_file_name, "wb")
target_file.truncate(resumableTotalSize)
target_file.close()
upload_complete = False
with open(target_file_name, "r+b") as target_file:
offset = (resumableChunkNumber - 1) * resumableChunkSize
target_file.seek(offset, os.SEEK_SET)
target_file.write(chunk_data.read())
fh = target_file.fileno()
if os.lseek(fh, 0, os.SEEK_HOLE) == resumableTotalSize:
upload_complete = True
os.fsync(fh)
last_chunk_offset = resumableTotalChunks * resumableChunkSize
if resumableTotalSize >= resumableChunkSize:
if (not is_hole(fh, resumableChunkSize - 1)
and resumableChunkNumber == resumableTotalChunks) or (
not is_hole(fh, resumableTotalSize - 1)
and resumableChunkNumber == 1):
update_video_metadata(video, target_file_name)
if upload_complete:
if not is_video_file(target_file_name):
video.status = 'error'
video.status_message = 'File uploaded was not a video file. Please use a different file.'
db_session.commit()
return {'message': video.status_message}, 501
update_video_metadata(video, target_file_name)
video.status = 'file-uploaded'
video.encoding_progress = 0
for encoded_file in video.encoded_files:
rm_f(
os.path.join(app.config['MOVIE_PATH'], video.id,
encoded_file.encoded_file_name))
db_session.delete(encoded_file)
db_session.commit()
def update_video_metadata(video, filename):
title = get_video_title(filename)
if title:
video.title = title
db_session.commit()
def transcode_video(video, task):
status = 'error'
output = ""
outdir = f"{celery.conf.get('MOVIE_PATH')}/{video.id}"
try:
os.mkdir(outdir)
except FileExistsError:
pass
master_playlist = f"{outdir}/playlist.mpd"
rm_f(master_playlist)
dash_size = 4
dash_command = ['MP4Box', '-dash', f'{dash_size * 1000}', '-rap', '-frag-rap', '-min-buffer', '16000', '-profile', 'dashavc264:onDemand', '-mpd-title', video.title ,'-out', master_playlist]
try:
print("Reencoded file")
def sort_video(video):
return int(video.split("_")[1])
def sort_audio(audio):
return int(audio.split("_")[1].split("k")[0])
video_files = []
encoded_files = db_session.query(models.EncodedFile).filter_by(video_id = video.id, track_type='video').all()
for encoded_file in encoded_files:
video_files.append(f'{outdir}/{encoded_file.encoded_file_name}')
video_files.sort(key=sort_video)
audio_files = []
encoded_files = db_session.query(models.EncodedFile).filter_by(video_id = video.id, track_type='audio').all()
for encoded_file in encoded_files:
audio_files.append(f'{outdir}/{encoded_file.encoded_file_name}')
audio_files.sort(key=sort_audio)
dash_command.extend(video_files)
dash_command.extend(audio_files)
print(f'Executing: {" ".join(dash_command)}')
output = subprocess.check_call(dash_command, stderr=subprocess.STDOUT)
print("DASHed file")
status = 'ready'
except Exception as e:
print(output)
print(e)
video.status = 'error'
video.status_message = 'MP4Box failed'
db_session.commit()
if celery.conf.get('STORAGE_BACKEND') == "S3":
print("Uploading to S3")
nthreads = celery.conf.get('S3_UPLOAD_THREADS')
g = glob.glob(f"{outdir}/*")
splits = numpy.array_split(g, nthreads)
threads = list()
for index in range(nthreads):
x = threading.Thread(target=s3_upload, args=(splits[index].copy(),))
threads.append(x)
x.start()
for index, thread in enumerate(threads):
thread.join()
print("Done uploading")
video.playlist = f'{video.id}/playlist.mpd'
video.encoding_progress = 100
video.status = status
db_session.commit()
def transcode(tmpfile, streaminfo, video_id):
status = 'error'
output = ""
outdir = f"{celery.conf.get('MOVIE_PATH')}/{video_id}"
shutil.rmtree(outdir, ignore_errors=True)
os.mkdir(outdir)
master_playlist = f"{outdir}/playlist.mpd"
rm_f(master_playlist)
vwidth = 0
vheight = 0
duration = 0
acodec = ""
vcodec = ""
framerate = 24
chunk_size = 4
video_streamidx = -1
audio_streamidx = -1
has_audio = False
video = db_session.query(models.Video).filter_by(id=video_id).one_or_none()
duration = float(streaminfo['format']['duration'])
for stream in streaminfo['streams']:
if stream['codec_type'] == 'video':
vcodec = stream['codec_name']
vwidth = stream['width']
vheight = stream['height']
framerate = stream['r_frame_rate']
video_streamidx = stream['index']
if stream['codec_type'] == 'audio':
has_audio = True
if audio_streamidx == -1 and stream['tags']['language'] == 'und':
audio_streamidx = stream['index']
audio_codec = stream['codec_name']
if stream['tags']['language'] == 'eng':
audio_streamidx = stream['index']
audio_codec = stream['codec_name']
if video_streamidx == -1:
video_streamidx = 0
if audio_streamidx == -1 and has_audio:
audio_streamidx = 1
try:
framerate = round(float(framerate))
except ValueError:
x, y = framerate.split("/")
framerate = round(int(x) / int(y))
dash_size = 4
keyint = framerate
if vwidth > 1920:
vheight = int(vheight / (vwidth / 1920))
vwidth = 1920
audio_formats = []
if has_audio:
audio_formats = [{
'rate': '64k',
'channels': '1'
}, {
'rate': '128k',
'channels': '2'
}, {
'rate': '196k',
'channels': '2'
}]
video_profiles = [
{
'profile': 'main',
'preset': 'veryslow',
'crf': '22',
'maxrate': '600k',
'bufsize': '800k',
'width': 480
},
{
'profile': 'main',
'preset': 'slow',
'crf': '22',
'maxrate': '900k',
'bufsize': '1200k',
'width': 640
},
{
'profile': 'high',
'preset': 'slow',
'crf': '22',
'maxrate': '1200k',
'bufsize': '1500k',
'width': 960
},
{
'profile': 'high',
'preset': 'slow',
'crf': '21',
'maxrate': '2000k',
'bufsize': '4000k',
'width': 1280
},
{
'profile': 'high',
'preset': 'slow',
'crf': '21',
'maxrate': '4500k',
'bufsize': '8000k',
'width': 1920
},
]
video_formats = [{
'profile': 'baseline',
'preset': 'veryslow',
'crf': '22',
'maxrate': '200k',
'bufsize': '300k',
'width': 320
}, {
'profile': 'baseline',
'preset': 'veryslow',
'crf': '22',
'maxrate': '400k',
'bufsize': '500k',
'width': 320
}]
sizes = [1, 1.5, 2, 3]
for size in sizes:
this_width = int(vwidth / size) + (int(vwidth / size) % 2)
if this_width < video_profiles[0]['width']:
next
this_profile = None
for idx in range(len(video_profiles)):
if this_width == video_profiles[idx]['width']:
this_profile = video_profiles[idx].copy()
break
if this_width > video_profiles[idx][
'width'] and this_width < video_profiles[idx + 1]['width']:
this_profile = video_profiles[idx + 1].copy()
this_profile['width'] = this_width
break
if this_profile:
video_formats.append(this_profile)
print(video_formats)
tmpdir = tempfile.mkdtemp()
socketfile = os.path.join(tmpdir, 'progress')
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.bind(socketfile)
sock.listen(1)
transcode_command = [
'ffmpeg', '-y', '-nostdin', '-i', f'{tmpfile}', '-progress',
f'unix://{socketfile}', '-loglevel', '24'
]
dash_command = [
'MP4Box', '-dash', f'{dash_size * 1000}', '-rap', '-frag-rap',
'-min-buffer', '16000', '-profile', 'dashavc264:onDemand',
'-mpd-title', video.title, '-out', master_playlist
]
tmpfiles = []
for num, f in enumerate(video_formats):
stream = num
filename = f'{outdir}/video_{f["width"]}_{f["maxrate"]}.mp4'
transcode_command.extend([
'-map', f'0:{video_streamidx}', f'-c:v', 'libx264', '-x264-params',
f'no-scenecut', f'-profile:v', f['profile'], '-preset:v',
f["preset"], '-tune:v', video.tune, '-keyint_min', f'{keyint}',
'-g', f'{keyint}', '-sc_threshold', '0', '-bf', '1', '-b_strategy',
'0', f'-crf', f['crf'], f'-maxrate', f'{f["maxrate"]}',
f'-bufsize', f'{f["bufsize"]}', f'-filter',
f'scale={f["width"]}:-2', '-map_chapters', '-1', filename
])
dash_command.append(filename)
tmpfiles.append(filename)
for num, f in enumerate(audio_formats):
stream = num
filename = f'{outdir}/audio_{f["rate"]}.mp4'
transcode_command.extend([
'-map', f'0:{audio_streamidx}', f'-c:a', 'aac', f'-b:a', f['rate'],
f'-ac', f['channels'], '-map_chapters', '-1', filename
])
dash_command.append(filename)
tmpfiles.append(filename)
video.encoding_status = 'encoding'
db_session.commit()
ffmpeg = multiprocessing.Process(target=run_ffmpeg,
args=(transcode_command,
f'{tmpfile}.log'))
ffmpeg.start()
connection, client_address = sock.accept()
percentage = 0
speed = 0
try:
while True:
data = connection.recv(1024)
if data:
string = data.decode('utf-8')
for line in string.splitlines():
if line.startswith('out_time_ms'):
progress = int(line.split('=')[1]) / 1000000
percentage = (progress / duration) * 100
percentage = min(percentage, 100)
if line.startswith('speed'):
speed = float(line.split('=')[1].strip().split('x')[0])
video.encoding_progress = percentage
video.encoding_speed = speed
db_session.commit()
else:
break
finally:
ffmpeg.terminate()
connection.close()
shutil.rmtree(tmpdir, ignore_errors=True)
if percentage < 100:
video.status = 'error'
db_session.commit()
try:
print("Reencoded file")
print(f'Executing: {" ".join(dash_command)}')
output = subprocess.check_call(dash_command, stderr=subprocess.STDOUT)
print("DASHed file")
status = 'ready'
except Exception as e:
print(output)
print(e)
for f in tmpfiles:
os.unlink(f)
if celery.conf.get('STORAGE_BACKEND') == "S3":
print("Uploading to S3")
nthreads = celery.conf.get('S3_UPLOAD_THREADS')
g = glob.glob(f"{outdir}/*")
splits = numpy.array_split(g, nthreads)
threads = list()
for index in range(nthreads):
x = threading.Thread(target=s3_upload,
args=(splits[index].copy(), ))
threads.append(x)
x.start()
for index, thread in enumerate(threads):
thread.join()
shutil.rmtree(outdir, ignore_errors=True)
print("Done uploading")
video.playlist = f'{video_id}/playlist.mpd'
video.width = vwidth
video.height = vheight
video.duration = duration
video.encoding_status = status
db_session.commit()