def hook_install(self):
cfg = self.config
self.hook_uninstall()
self.generate_locales((u'fr_CH.UTF-8',))
try_makedirs(u'/etc/mysql')
debconf, mysql = u'debconf-set-selections', u'mysql-server mysql-server'
# Tip : http://ubuntuforums.org/showthread.php?t=981801
self.cmd(debconf, input=u'{0}/root_password select {1}'.format(mysql, cfg.mysql_root_password))
self.cmd(debconf, input=u'{0}/root_password_again select {1}'.format(mysql, cfg.mysql_root_password))
self.install_packages(WebuiHooks.PACKAGES)
self.restart_ntp()
self.info(u'Import Web UI database and create user')
hostname = socket.gethostname()
self.cmd(u'service mysql start', fail=False)
self.mysql_do(u"DROP USER ''@'localhost'; DROP USER ''@'{0}';".format(hostname), fail=False)
self.mysql_do(u"GRANT ALL PRIVILEGES ON *.* TO 'root'@'%%' WITH GRANT OPTION;")
self.mysql_do(u'DROP DATABASE IF EXISTS webui')
self.mysql_do(cli_input=open(self.local_config.site_database_file, u'r', u'utf-8').read())
self.mysql_do(u"GRANT ALL ON webui.* TO 'webui'@'%%' IDENTIFIED BY '{0}';".format(cfg.mysql_user_password))
self.info(u'Configure Apache 2')
self.cmd(u'a2enmod rewrite')
self.info(u'Copy and pre-configure Web UI')
rsync(u'www/', self.local_config.site_directory, archive=True, delete=True, exclude_vcs=True, recursive=True)
chown(self.local_config.site_directory, DAEMON_USER, DAEMON_GROUP, recursive=True)
self.local_config.encryption_key = WebuiHooks.randpass(32)
self.info(u'Expose Apache 2 service')
self.open_port(80, u'TCP')
def test_transcode_the_media_assets(self):
with mock.patch('encodebox.celeryconfig.CELERY_ALWAYS_EAGER',
True,
create=True):
from encodebox import tasks
media_filenames = sorted(
f for f in os.listdir(MEDIA_INPUTS_DIRECTORY)
if not f.startswith('.git'))
for index, filename in enumerate(media_filenames, 1):
index, name = unicode(index), basename(filename)
in_relpath = join('2', index, 'uploaded', name)
in_abspath = join(LOCAL_DIRECTORY, in_relpath)
unguessable = generate_unguessable_filename(
SETTINGS['filenames_seed'], name)
try_makedirs(dirname(in_abspath))
shutil.copy(join(MEDIA_INPUTS_DIRECTORY, filename), in_abspath)
tasks.transcode(json.dumps(in_relpath))
ok_(
exists(join(LOCAL_DIRECTORY, '2', index, 'completed',
name)))
ok_(self.is_empty(join(LOCAL_DIRECTORY, '2', index, 'failed')))
ok_(
self.is_empty(join(LOCAL_DIRECTORY, '2', index,
'uploaded')))
ok_(
exists(
join(REMOTE_DIRECTORY, '2', index,
unguessable + '.smil')))
rsync(source=join(REMOTE_DIRECTORY, '2', index),
destination=join(MEDIA_REMOTE_DIRECTORY, filename),
destination_is_dir=True,
archive=True,
delete=True,
makedest=True,
recursive=True)
def setUp(self):
set_test_settings()
for directory in (LOCAL_DIRECTORY, REMOTE_DIRECTORY):
try_makedirs(directory)
logging.config.dictConfig({
'version': 1,
'disable_existing_loggers': True,
'formatters': {
'simple': {
'format': '%(levelname)s %(message)s'
}
},
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'formatter': 'simple'
}
},
'loggers': {
'encodebox.tasks.transcode': {
'handlers': ['console'],
'propagate': True,
'level': 'DEBUG'
}
}
})
def add_media(config, media):
if media.status != Media.PENDING:
media_src_path = config.storage_medias_path(media, generate=False)
if media_src_path:
media_dst_path = config.storage_medias_path(media, generate=True)
if media_dst_path != media_src_path:
# Generate media storage uri and move it to media storage path + set permissions
media.uri = config.storage_medias_uri(media)
try_makedirs(os.path.dirname(media_dst_path))
the_error = None
for i in xrange(5):
try:
os.rename(media_src_path, media_dst_path)
# FIXME chown chmod
the_error = None
break
except OSError as error:
the_error = error
time.sleep(1)
if the_error:
raise IndexError(to_bytes(u'An error occured : {0} ({1} -> {2}).'.format(
the_error, media_src_path, media_dst_path)))
try:
size = get_size(os.path.dirname(media_dst_path))
except OSError:
raise ValueError(to_bytes(u'Unable to detect size of media asset {0}.'.format(media_dst_path)))
duration = get_media_duration(media_dst_path)
if duration is None:
raise ValueError(to_bytes(u'Unable to detect duration of media asset {0}.'.format(media_dst_path)))
return (size, duration)
else:
raise NotImplementedError(to_bytes(u'FIXME Add of external URI not implemented.'))
return (0, None)
def post_install():
from encodebox import lib
from pytoolbox.console import confirm
from pytoolbox.encoding import to_bytes
from pytoolbox.filesystem import chown, from_template, try_makedirs, try_remove
from pytoolbox.network.http import download
if not exists(u'/usr/local/bin/neroAacEnc'):
try:
print(u'Download and install Nero AAC encoder')
download(u'ftp://ftp6.nero.com/tools/NeroDigitalAudio.zip',
u'/tmp/nero.zip')
zipfile.ZipFile(u'/tmp/nero.zip').extract(u'linux/neroAacEnc',
u'/usr/local/bin')
os.chmod(
u'/usr/local/bin/neroAacEnc',
os.stat(u'/usr/local/bin/neroAacEnc').st_mode | stat.S_IEXEC)
finally:
try_remove(u'/tmp/nero.zip')
filename = lib.SETTINGS_FILENAME
settings = lib.load_settings(u'etc/config.yaml')
if not exists(filename) or confirm(
u'Overwrite existing configuration file "{0}"'.format(filename)):
print(u'Generate configuration file "{0}"'.format(filename))
password = lib.generate_password()
settings[u'rabbit_password'] = password
lib.save_settings(filename, settings)
print(u'Configure RabbitMQ Message Broker')
check_call([u'service', u'rabbitmq-server', u'start'])
call([u'rabbitmqctl', u'add_vhost', u'/'])
call([u'rabbitmqctl', u'delete_user', u'guest'])
call([u'rabbitmqctl', u'delete_user', u'encodebox'])
call([
u'rabbitmqctl', u'add_user', u'encodebox', settings[u'rabbit_password']
])
check_call([
u'rabbitmqctl', u'set_permissions', u'-p', u'/', u'encodebox', u'.*',
u'.*', u'.*'
])
users, vhosts = lib.rabbit_users(), lib.rabbit_vhosts()
print(u'RabbitMQ users: {0} vhosts: {1}'.format(users, vhosts))
if u'guest' in users or u'encodebox' not in users:
raise RuntimeError(to_bytes(u'Unable to configure RabbitMQ'))
print(u'Create directory for storing persistent data')
try_makedirs(lib.LIB_DIRECTORY)
chown(lib.LIB_DIRECTORY,
lib.USERNAME,
pwd.getpwnam(lib.USERNAME).pw_gid,
recursive=True)
print(u'Register and start our services as user ' + lib.USERNAME)
from_template(u'etc/encodebox.conf.template',
u'/etc/supervisor/conf.d/encodebox.conf', {
u'lib_directory': lib.LIB_DIRECTORY,
u'user': lib.USERNAME
})
call([u'service', u'supervisor', u'force-reload'])
def load_settings(filename=None, create_directories=False):
default = os.environ.get(u'ENCODEBOX_SETTINGS_FILENAME', SETTINGS_FILENAME)
filename = filename or default
if not exists(filename):
raise IOError(to_bytes(u'Unable to find settings file "{0}".'.format(filename)))
with open(filename, u'r', u'utf-8') as f:
settings = yaml.load(f)
for key, value in settings.iteritems():
if u'directory' in key and not u'remote' in key:
settings[key] = abspath(expanduser(value))
if create_directories:
try_makedirs(settings[key])
return settings
def move(source, destination):
u"""
Create the destination directory if missing and recursively move a file/directory from source to destination.
**Example usage**
>>> import os
>>> open(u'/tmp/move_test_file', u'w', u'utf-8').close()
>>> move(u'/tmp/move_test_file', u'/tmp/move_demo/another/file')
>>> os.remove(u'/tmp/move_demo/another/file')
>>> shutil.rmtree(u'/tmp/move_demo')
"""
try_makedirs(dirname(destination))
shutil.move(source, destination)
def test_transcode_mp4(self):
with mock.patch('encodebox.celeryconfig.CELERY_ALWAYS_EAGER',
True,
create=True):
from encodebox import tasks
in_relpath = '1/2/uploaded/test.mp4'
in_abspath = join(LOCAL_DIRECTORY, in_relpath)
unguessable = generate_unguessable_filename(
SETTINGS['filenames_seed'], 'test.mp4')
try_makedirs(dirname(in_abspath))
download('http://techslides.com/demos/sample-videos/small.mp4',
in_abspath)
tasks.transcode(json.dumps(in_relpath))
ok_(exists(join(LOCAL_DIRECTORY, '1/2/completed/test.mp4')))
ok_(self.is_empty(join(LOCAL_DIRECTORY, '1/2/failed')))
ok_(self.is_empty(join(LOCAL_DIRECTORY, '1/2/uploaded')))
ok_(exists(join(REMOTE_DIRECTORY, '1/2', unguessable + '.smil')))
def hook_uninstall(self):
self.info(u'Uninstall prerequisities, unregister service and load default configuration')
self.hook_stop()
try_makedirs(u'/var/log/rabbitmq') # Fix rabbitmq-server package uninstall error
#self.cmd('juju destroy-environment')
#self.cmd('... --purge apt-cacher-ng charm-tools juju libzookeeper-java lxc zookeeper')
self.storage_unregister()
if self.config.cleanup:
self.cmd(u'apt-get -y remove --purge {0}'.format(u' '.join(OrchestraHooks.PACKAGES)))
self.cmd(u'apt-get -y remove --purge {0}'.format(u' '.join(OrchestraHooks.FIX_PACKAGES)), fail=False)
self.cmd(u'apt-get -y autoremove')
#shutil.rmtree('$HOME/.juju $HOME/.ssh/id_rsa*
shutil.rmtree(u'/etc/apache2/', ignore_errors=True)
shutil.rmtree(u'/var/log/apache2/', ignore_errors=True)
shutil.rmtree(u'/etc/rabbitmq/', ignore_errors=True)
shutil.rmtree(u'/var/log/rabbitmq/', ignore_errors=True)
shutil.rmtree(self.local_config.site_directory, ignore_errors=True)
self.local_config.reset()
def setUp(self):
OrchestraLocalConfig().write(u'test.pkl')
self.hooks = OrchestraHooks_tmp(None, CONFIG, u'test.pkl', OS_ENV)
shutil.copy(self.hooks.local_config.hosts_file, 'hosts')
shutil.copy(u'mongodb.conf', u'mongodb_test.conf')
self.hooks.local_config.hosts_file = u'hosts' # Avoid writing to system hosts file !
self.hooks.local_config.celery_config_file = u'celeryconfig.py'
self.hooks.local_config.celery_template_file = os.path.join(
u'../../charms/oscied-orchestra', self.hooks.local_config.celery_template_file)
self.hooks.local_config.site_path = u'.'
self.hooks.local_config.site_template_file = os.path.join(
u'../../charms/oscied-orchestra', self.hooks.local_config.site_template_file)
self.hooks.local_config.ssh_template_path = os.path.join(
u'../../charms/oscied-orchestra', self.hooks.local_config.ssh_template_path)
self.hooks.local_config.mongo_config_file = u'mongodb_test.conf'
try_makedirs(os.path.join(self.hooks.local_config.charms_repository, u'default'))
try_makedirs(u'bibi')
self.hooks.directory = u'bibi'
def hook_install(self):
cfg = self.config
self.hook_uninstall()
self.generate_locales((u'fr_CH.UTF-8',))
self.install_packages(StorageHooks.PACKAGES)
self.restart_ntp()
self.info(u'Configure storage bricks root')
if cfg.bricks_root_device:
self.cmd(u'umount {0}'.format(cfg.bricks_root_path), fail=False)
if cfg.format_bricks_root:
self.cmd(u'mkfs.xfs {0} -f'.format(cfg.bricks_root_device)) # FIXME detect based on the mount point
self.cmd(u'mount {0} {1}'.format(cfg.bricks_root_device, cfg.bricks_root_path)) # FIXME add mdadm support?
try_makedirs(self.bricks_path)
self.info(u'Expose GlusterFS Server service')
self.open_port(111, u'TCP') # For portmapper, and should have both TCP and UDP open
self.open_port(111, u'UDP')
self.open_port(24007, u'TCP') # For the Gluster Daemon
#self.open_port(24008, u'TCP') # Infiniband management (optional unless you are using IB)
self.open_port(24009, u'TCP') # We have only 1 storage brick (24009-24009)
def test_transcode_text(self):
with mock.patch('encodebox.celeryconfig.CELERY_ALWAYS_EAGER',
True,
create=True):
from encodebox import tasks
in_relpath = '1/1/uploaded/test.txt'
in_abspath = join(LOCAL_DIRECTORY, in_relpath)
try_makedirs(dirname(in_abspath))
open(in_abspath, 'w', 'utf-8').write('salut')
try:
tasks.transcode(json.dumps(in_relpath))
raise ValueError(
u'Transcoding task does not raised an exception.')
except RuntimeError:
pass
ok_(exists(join(LOCAL_DIRECTORY, '1/1/failed/test.txt')))
ok_(self.is_empty(join(LOCAL_DIRECTORY, '1/1/completed')))
ok_(self.is_empty(join(LOCAL_DIRECTORY, '1/1/uploaded')))
ok_(self.is_empty(join(REMOTE_DIRECTORY, '1/1')))
def storage_remount(self, address=None, fstype=None, mountpoint=None, options=u''):
if self.storage_config_is_enabled:
self.info(u'Override storage parameters with charm configuration')
address = self.config.storage_address
nat_address = self.config.storage_nat_address
fstype = self.config.storage_fstype
mountpoint = self.config.storage_mountpoint
options = self.config.storage_options
elif address and fstype and mountpoint:
self.info(u'Use storage parameters from charm storage relation')
nat_address = u''
else:
return
if nat_address:
self.info(u'Update hosts file to map storage internal address {0} to {1}'.format(address, nat_address))
lines = filter(lambda l: nat_address not in l, open(self.local_config.hosts_file, u'r', u'utf-8'))
lines += u'{0} {1}\n'.format(nat_address, address)
open(self.local_config.hosts_file, u'w', u'utf-8').write(u''.join(lines))
# Avoid unregistering and registering storage if it does not change ...
if (address == self.local_config.storage_address and nat_address == self.local_config.storage_nat_address and
fstype == self.local_config.storage_fstype and mountpoint == self.local_config.storage_mountpoint and
options == self.local_config.storage_options):
self.remark(u'Skip remount already mounted shared storage')
else:
self.storage_unregister()
self.debug(u"Mount shared storage [{0}] {1}:{2} type {3} options '{4}' -> {5}".format(nat_address, address,
mountpoint, fstype, options, self.local_config.storage_path))
try_makedirs(self.local_config.storage_path)
# FIXME try X times, a better way to handle failure
for i in xrange(self.local_config.storage_mount_max_retry):
if self.storage_is_mounted:
break
mount_address = u'{0}:/{1}'.format(nat_address or address, mountpoint)
mount_path = self.local_config.storage_path
if options:
self.cmd([u'mount', u'-t', fstype, u'-o', options, mount_address, mount_path])
else:
self.cmd([u'mount', u'-t', fstype, mount_address, mount_path])
time.sleep(self.local_config.storage_mount_sleep_delay)
if self.storage_is_mounted:
# FIXME update /etc/fstab (?)
self.local_config.storage_address = address
self.local_config.storage_nat_address = nat_address
self.local_config.storage_fstype = fstype
self.local_config.storage_mountpoint = mountpoint
self.local_config.storage_options = options
self.remark(u'Shared storage successfully registered')
self.debug(u'Create directories in the shared storage and ensure it is owned by the right user')
try_makedirs(self.local_config.storage_medias_path())
try_makedirs(self.local_config.storage_uploads_path)
chown(self.local_config.storage_path, DAEMON_USER, DAEMON_GROUP, recursive=True)
else:
raise IOError(to_bytes(u'Unable to mount shared storage'))
def transform_task(media_in_json, media_out_json, profile_json, callback_json):
def copy_callback(start_date, elapsed_time, eta_time, src_size, dst_size, ratio):
transform_task.update_state(state=TransformTask.PROGRESS, meta={
u'hostname': request.hostname, 'start_date': start_date, u'elapsed_time': elapsed_time,
u'eta_time': eta_time, u'media_in_size': src_size, u'media_out_size': dst_size,
u'percent': int(100 * ratio)})
def transform_callback(status, measures):
data_json = object2json({u'task_id': request.id, u'status': status, u'measures': measures}, include_properties=False)
if callback is None:
print(u'{0} [ERROR] Unable to callback orchestrator: {1}'.format(request.id, data_json))
else:
r = callback.post(data_json)
print(u'{0} Code {1} {2} : {3}'.format(request.id, r.status_code, r.reason, r._content))
# ------------------------------------------------------------------------------------------------------------------
RATIO_DELTA, TIME_DELTA = 0.01, 1 # Update status if at least 1% of progress and 1 second elapsed.
MAX_TIME_DELTA = 5 # Also ensure status update every 5 seconds.
DASHCAST_TIMEOUT_TIME = 10
try:
# Avoid 'referenced before assignment'
callback = dashcast_conf = None
encoder_out, request = u'', current_task.request
# Let's the task begin !
print(u'{0} Transformation task started'.format(request.id))
# Read current configuration to translate files uri to local paths
local_config = TransformLocalConfig.read(LOCAL_CONFIG_FILENAME, inspect_constructor=False)
print(object2json(local_config, include_properties=True))
# Load and check task parameters
callback = Callback.from_json(callback_json, inspect_constructor=True)
callback.is_valid(True)
# Update callback socket according to configuration
if local_config.api_nat_socket and len(local_config.api_nat_socket) > 0:
callback.replace_netloc(local_config.api_nat_socket)
media_in = Media.from_json(media_in_json, inspect_constructor=True)
media_out = Media.from_json(media_out_json, inspect_constructor=True)
profile = TransformProfile.from_json(profile_json, inspect_constructor=True)
media_in.is_valid(True)
media_out.is_valid(True)
profile.is_valid(True)
# Verify that media file can be accessed and create output path
media_in_path = local_config.storage_medias_path(media_in, generate=False)
if not media_in_path:
raise NotImplementedError(to_bytes(u'Input media asset will not be readed from shared storage : {0}'.format(
media_in.uri)))
media_out_path = local_config.storage_medias_path(media_out, generate=True)
if not media_out_path:
raise NotImplementedError(to_bytes(u'Output media asset will not be written to shared storage : {0}'.format(
media_out.uri)))
media_in_root = dirname(media_in_path)
media_out_root = dirname(media_out_path)
try_makedirs(media_out_root)
# Get input media duration and frames to be able to estimate ETA
media_in_duration = get_media_duration(media_in_path)
# Keep potential PSNR status
measures = {}
# NOT A REAL TRANSFORM : FILE COPY -----------------------------------------------------------------------------
if profile.encoder_name == u'copy':
infos = recursive_copy(media_in_root, media_out_root, copy_callback, RATIO_DELTA, TIME_DELTA)
media_out_tmp = media_in_path.replace(media_in_root, media_out_root)
os.rename(media_out_tmp, media_out_path)
start_date = infos[u'start_date']
elapsed_time = infos[u'elapsed_time']
media_in_size = infos[u'src_size']
# A REAL TRANSFORM : TRANSCODE WITH FFMPEG ---------------------------------------------------------------------
elif profile.encoder_name == u'ffmpeg':
start_date, start_time = datetime_now(), time.time()
prev_ratio = prev_time = 0
# Get input media size to be able to estimate ETA
media_in_size = get_size(media_in_root)
# Create FFmpeg subprocess
cmd = u'ffmpeg -y -i "{0}" {1} "{2}"'.format(media_in_path, profile.encoder_string, media_out_path)
print(cmd)
ffmpeg = Popen(shlex.split(cmd), stderr=PIPE, close_fds=True)
make_async(ffmpeg.stderr)
while True:
# Wait for data to become available
select.select([ffmpeg.stderr], [], [])
chunk = ffmpeg.stderr.read()
encoder_out += chunk
elapsed_time = time.time() - start_time
match = FFMPEG_REGEX.match(chunk)
if match:
stats = match.groupdict()
media_out_duration = stats[u'time']
try:
ratio = total_seconds(media_out_duration) / total_seconds(media_in_duration)
ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio
except ZeroDivisionError:
ratio = 1.0
delta_time = elapsed_time - prev_time
if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA:
prev_ratio, prev_time = ratio, elapsed_time
eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0
transform_task.update_state(
state=TransformTask.PROGRESS,
meta={u'hostname': request.hostname,
u'start_date': start_date,
u'elapsed_time': elapsed_time,
u'eta_time': eta_time,
u'media_in_size': media_in_size,
u'media_in_duration': media_in_duration,
u'media_out_size': get_size(media_out_root),
u'media_out_duration': media_out_duration,
u'percent': int(100 * ratio),
u'encoding_frame': stats[u'frame'],
u'encoding_fps': stats[u'fps'],
u'encoding_bitrate': stats[u'bitrate'],
u'encoding_quality': stats[u'q']})
returncode = ffmpeg.poll()
if returncode is not None:
break
# FFmpeg output sanity check
if returncode != 0:
raise OSError(to_bytes(u'FFmpeg return code is {0}, encoding probably failed.'.format(returncode)))
# compute stats about the video
measures['psnr'] = get_media_psnr(media_in_path, media_out_path)
measures['ssim'] = get_media_ssim(media_in_path, media_out_path)
# measures of the data and its metadata
measures['bitrate'] = get_media_bitrate(media_out_path)
# FIXME: fake git url, commit
measures['git_url'] = 'https://github.com/videolan/x265'
measures['git_commit'] = 'd2051f9544434612a105d2f5267db23018cb3454'
# Output media file sanity check
# media_out_duration = get_media_duration(media_out_path)
# if total_seconds(media_out_duration) / total_seconds(media_in_duration) > 1.5 or < 0.8:
# salut
elif profile.encoder_name == u'from_git':
start_date, start_time = datetime_now(), time.time()
prev_ratio = prev_time = 0
# Get input media size to be able to estimate ETA
media_in_size = get_size(media_in_root)
metadata = media_out.metadata
dirpath = tempfile.mkdtemp()
prepare_cmd = u'git clone --depth=1 "{0}" "{1}" && cd "{1}" && git checkout "{2}" && {3}'.format(metadata['git_url'], dirpath, metadata['git_commit'], metadata['build_cmds'])
check_call(prepare_cmd, shell=True)
# Templated parameter
encoder_string = profile.encoder_string.replace(u"BITRATE", str(metadata['input_bitrate']))
cmd = u'cd "{0}" && ffmpeg -y -i "{1}" -f yuv4mpegpipe - | {2} "{3}"'.format(dirpath, media_in_path, encoder_string, media_out_path)
returncode = call(cmd, shell=True)
if returncode != 0:
raise OSError(to_bytes(u'Encoding return code is {0}, encoding probably failed.'.format(returncode)))
# compute stats about the video
measures['psnr'] = get_media_psnr(media_in_path, media_out_path)
measures['ssim'] = get_media_ssim(media_in_path, media_out_path)
# measures of the data and its metadata
measures['bitrate'] = get_media_bitrate(media_out_path)
# FIXME: don't put this in measures
measures['git_url'] = metadata['git_url']
measures['git_commit'] = metadata['git_commit']
# A REAL TRANSFORM : TRANSCODE WITH DASHCAST -------------------------------------------------------------------
elif profile.encoder_name == u'dashcast':
start_date, start_time = datetime_now(), time.time()
prev_ratio = prev_time = 0
# Get input media size and frames to be able to estimate ETA
media_in_size = get_size(media_in_root)
try:
media_in_frames = int(get_media_tracks(media_in_path)[u'video'][u'0:0'][u'estimated_frames'])
media_out_frames = 0
except:
raise ValueError(to_bytes(u'Unable to estimate # frames of input media asset'))
# Create DashCast configuration file and subprocess
dashcast_conf = u'dashcast_{0}.conf'.format(uuid.uuid4())
with open(dashcast_conf, u'w', u'utf-8') as f:
f.write(profile.dash_config)
cmd = u'DashCast -conf {0} -av "{1}" {2} -out "{3}" -mpd "{4}"'.format(
dashcast_conf, media_in_path, profile.dash_options, media_out_root, media_out.filename)
print(cmd)
dashcast = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE, close_fds=True)
make_async(dashcast.stdout.fileno())
make_async(dashcast.stderr.fileno())
while True:
# Wait for data to become available
select.select([dashcast.stdout.fileno()], [], [])
stdout, stderr = read_async(dashcast.stdout), read_async(dashcast.stderr)
elapsed_time = time.time() - start_time
match = DASHCAST_REGEX.match(stdout)
if match:
stats = match.groupdict()
media_out_frames = int(stats[u'frame'])
try:
ratio = float(media_out_frames) / media_in_frames
ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio
except ZeroDivisionError:
ratio = 1.0
delta_time = elapsed_time - prev_time
if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA:
prev_ratio, prev_time = ratio, elapsed_time
eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0
transform_task.update_state(
state=TransformTask.PROGRESS,
meta={u'hostname': request.hostname,
u'start_date': start_date,
u'elapsed_time': elapsed_time,
u'eta_time': eta_time,
u'media_in_size': media_in_size,
u'media_in_duration': media_in_duration,
u'media_out_size': get_size(media_out_root),
u'percent': int(100 * ratio),
u'encoding_frame': media_out_frames})
match = DASHCAST_SUCCESS_REGEX.match(stdout)
returncode = dashcast.poll()
if returncode is not None or match:
encoder_out = u'stdout: {0}\nstderr: {1}'.format(stdout, stderr)
break
if media_out_frames == 0 and elapsed_time > DASHCAST_TIMEOUT_TIME:
encoder_out = u'stdout: {0}\nstderr: {1}'.format(stdout, stderr)
raise OSError(to_bytes(u'DashCast does not output frame number, encoding probably failed.'))
# DashCast output sanity check
if not exists(media_out_path):
raise OSError(to_bytes(u'Output media asset not found, DashCast encoding probably failed.'))
if returncode != 0:
raise OSError(to_bytes(u'DashCast return code is {0}, encoding probably failed.'.format(returncode)))
# FIXME check duration too !
# Here all seem okay -------------------------------------------------------------------------------------------
elapsed_time = time.time() - start_time
media_out_size = get_size(media_out_root)
media_out_duration = get_media_duration(media_out_path)
print(u'{0} Transformation task successful, output media asset {1}'.format(request.id, media_out.filename))
transform_callback(TransformTask.SUCCESS, measures)
return {u'hostname': request.hostname, u'start_date': start_date, u'elapsed_time': elapsed_time,
u'eta_time': 0, u'media_in_size': media_in_size, u'media_in_duration': media_in_duration,
u'media_out_size': media_out_size, u'media_out_duration': media_out_duration, u'percent': 100 }
except Exception as error:
# Here something went wrong
print(u'{0} Transformation task failed '.format(request.id))
transform_callback(u'ERROR\n{0}\n\nOUTPUT\n{1}'.format(unicode(error), encoder_out), {})
raise
finally:
if dashcast_conf:
try_remove(dashcast_conf)
common_file.write(common_data)
with open(DAVID_REPORT_REFERENCES_FILE, "w", "utf-8") as references_file:
references_file.write(references_data)
print("Append header files into common file")
for header_filename in glob.glob(join(DAVID_REPORT_SOURCE_PATH, "*.rst.header")):
rst_filename = join(dirname(header_filename), splitext(header_filename)[0])
with open(rst_filename, "rw+", "utf-8") as rst_file:
data = rst_file.read()
with open(header_filename, "r", "utf-8") as header_file:
rst_file.seek(0)
rst_file.write(header_file.read() + data)
# FIXME echo about.rst -> index.rst for html version at least
shutil.rmtree(DAVID_REPORT_BUILD_PATH, ignore_errors=True)
try_makedirs(DAVID_REPORT_BUILD_PATH)
os.chdir(DAVID_REPORT_PATH)
if args.html:
print(HELP_HTML)
result = cmd("sudo make html", fail=False)
with open("build_html.log", "w", "utf-8") as log_file:
log_file.write("Output:\n{0}\nError:\n{1}".format(result["stdout"], result["stderr"]))
if result["returncode"] != 0:
print_error("Unable to generate HTML version of the report, see build_html.log")
if args.pdf:
print(HELP_PDF)
result = cmd("sudo make latexpdf", fail=False)
with open("build_pdf.log", "w", "utf-8") as log_file:
log_file.write("Output:\n{0}\nError:\n{1}".format(result["stdout"], result["stderr"]))
def transform_task(media_in_json, media_out_json, profile_json, callback_json):
def copy_callback(start_date, elapsed_time, eta_time, src_size, dst_size, ratio):
transform_task.update_state(
state=TransformTask.PROGRESS,
meta={
"hostname": request.hostname,
"start_date": start_date,
"elapsed_time": elapsed_time,
"eta_time": eta_time,
"media_in_size": src_size,
"media_out_size": dst_size,
"percent": int(100 * ratio),
},
)
def transform_callback(status):
data_json = object2json({"task_id": request.id, "status": status}, include_properties=False)
if callback is None:
print("{0} [ERROR] Unable to callback orchestrator: {1}".format(request.id, data_json))
else:
r = callback.post(data_json)
print("{0} Code {1} {2} : {3}".format(request.id, r.status_code, r.reason, r._content))
# ------------------------------------------------------------------------------------------------------------------
RATIO_DELTA, TIME_DELTA = 0.01, 1 # Update status if at least 1% of progress and 1 second elapsed.
MAX_TIME_DELTA = 5 # Also ensure status update every 5 seconds.
DASHCAST_TIMEOUT_TIME = 10
try:
# Avoid 'referenced before assignment'
callback = dashcast_conf = None
encoder_out, request = "", current_task.request
# Let's the task begin !
print("{0} Transformation task started".format(request.id))
# Read current configuration to translate files uri to local paths
local_config = TransformLocalConfig.read(LOCAL_CONFIG_FILENAME, inspect_constructor=False)
print(object2json(local_config, include_properties=True))
# Load and check task parameters
callback = Callback.from_json(callback_json, inspect_constructor=True)
callback.is_valid(True)
# Update callback socket according to configuration
if local_config.api_nat_socket and len(local_config.api_nat_socket) > 0:
callback.replace_netloc(local_config.api_nat_socket)
media_in = Media.from_json(media_in_json, inspect_constructor=True)
media_out = Media.from_json(media_out_json, inspect_constructor=True)
profile = TransformProfile.from_json(profile_json, inspect_constructor=True)
media_in.is_valid(True)
media_out.is_valid(True)
profile.is_valid(True)
# Verify that media file can be accessed and create output path
media_in_path = local_config.storage_medias_path(media_in, generate=False)
if not media_in_path:
raise NotImplementedError(
to_bytes("Input media asset will not be readed from shared storage : {0}".format(media_in.uri))
)
media_out_path = local_config.storage_medias_path(media_out, generate=True)
if not media_out_path:
raise NotImplementedError(
to_bytes("Output media asset will not be written to shared storage : {0}".format(media_out.uri))
)
media_in_root = dirname(media_in_path)
media_out_root = dirname(media_out_path)
try_makedirs(media_out_root)
# Get input media duration and frames to be able to estimate ETA
media_in_duration = get_media_duration(media_in_path)
# NOT A REAL TRANSFORM : FILE COPY -----------------------------------------------------------------------------
if profile.encoder_name == "copy":
infos = recursive_copy(media_in_root, media_out_root, copy_callback, RATIO_DELTA, TIME_DELTA)
media_out_tmp = media_in_path.replace(media_in_root, media_out_root)
os.rename(media_out_tmp, media_out_path)
start_date = infos["start_date"]
elapsed_time = infos["elapsed_time"]
media_in_size = infos["src_size"]
# A REAL TRANSFORM : TRANSCODE WITH FFMPEG ---------------------------------------------------------------------
elif profile.encoder_name == "ffmpeg":
start_date, start_time = datetime_now(), time.time()
prev_ratio = prev_time = 0
# Get input media size to be able to estimate ETA
media_in_size = get_size(media_in_root)
# Create FFmpeg subprocess
cmd = 'ffmpeg -y -i "{0}" {1} "{2}"'.format(media_in_path, profile.encoder_string, media_out_path)
print(cmd)
ffmpeg = Popen(shlex.split(cmd), stderr=PIPE, close_fds=True)
make_async(ffmpeg.stderr)
while True:
# Wait for data to become available
select.select([ffmpeg.stderr], [], [])
chunk = ffmpeg.stderr.read()
encoder_out += chunk
elapsed_time = time.time() - start_time
match = FFMPEG_REGEX.match(chunk)
if match:
stats = match.groupdict()
media_out_duration = stats["time"]
try:
ratio = total_seconds(media_out_duration) / total_seconds(media_in_duration)
ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio
except ZeroDivisionError:
ratio = 1.0
delta_time = elapsed_time - prev_time
if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA:
prev_ratio, prev_time = ratio, elapsed_time
eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0
transform_task.update_state(
state=TransformTask.PROGRESS,
meta={
"hostname": request.hostname,
"start_date": start_date,
"elapsed_time": elapsed_time,
"eta_time": eta_time,
"media_in_size": media_in_size,
"media_in_duration": media_in_duration,
"media_out_size": get_size(media_out_root),
"media_out_duration": media_out_duration,
"percent": int(100 * ratio),
"encoding_frame": stats["frame"],
"encoding_fps": stats["fps"],
"encoding_bitrate": stats["bitrate"],
"encoding_quality": stats["q"],
},
)
returncode = ffmpeg.poll()
if returncode is not None:
break
# FFmpeg output sanity check
if returncode != 0:
raise OSError(to_bytes("FFmpeg return code is {0}, encoding probably failed.".format(returncode)))
# Output media file sanity check
# media_out_duration = get_media_duration(media_out_path)
# if total_seconds(media_out_duration) / total_seconds(media_in_duration) > 1.5 or < 0.8:
# salut
# A REAL TRANSFORM : TRANSCODE WITH DASHCAST -------------------------------------------------------------------
elif profile.encoder_name == "dashcast":
start_date, start_time = datetime_now(), time.time()
prev_ratio = prev_time = 0
# Get input media size and frames to be able to estimate ETA
media_in_size = get_size(media_in_root)
try:
media_in_frames = int(get_media_tracks(media_in_path)["video"]["0:0"]["estimated_frames"])
media_out_frames = 0
except:
raise ValueError(to_bytes("Unable to estimate # frames of input media asset"))
# Create DashCast configuration file and subprocess
dashcast_conf = "dashcast_{0}.conf".format(uuid.uuid4())
with open(dashcast_conf, "w", "utf-8") as f:
f.write(profile.dash_config)
cmd = 'DashCast -conf {0} -av "{1}" {2} -out "{3}" -mpd "{4}"'.format(
dashcast_conf, media_in_path, profile.dash_options, media_out_root, media_out.filename
)
print(cmd)
dashcast = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE, close_fds=True)
make_async(dashcast.stdout.fileno())
make_async(dashcast.stderr.fileno())
while True:
# Wait for data to become available
select.select([dashcast.stdout.fileno()], [], [])
stdout, stderr = read_async(dashcast.stdout), read_async(dashcast.stderr)
elapsed_time = time.time() - start_time
match = DASHCAST_REGEX.match(stdout)
if match:
stats = match.groupdict()
media_out_frames = int(stats["frame"])
try:
ratio = float(media_out_frames) / media_in_frames
ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio
except ZeroDivisionError:
ratio = 1.0
delta_time = elapsed_time - prev_time
if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA:
prev_ratio, prev_time = ratio, elapsed_time
eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0
transform_task.update_state(
state=TransformTask.PROGRESS,
meta={
"hostname": request.hostname,
"start_date": start_date,
"elapsed_time": elapsed_time,
"eta_time": eta_time,
"media_in_size": media_in_size,
"media_in_duration": media_in_duration,
"media_out_size": get_size(media_out_root),
"percent": int(100 * ratio),
"encoding_frame": media_out_frames,
},
)
match = DASHCAST_SUCCESS_REGEX.match(stdout)
returncode = dashcast.poll()
if returncode is not None or match:
encoder_out = "stdout: {0}\nstderr: {1}".format(stdout, stderr)
break
if media_out_frames == 0 and elapsed_time > DASHCAST_TIMEOUT_TIME:
encoder_out = "stdout: {0}\nstderr: {1}".format(stdout, stderr)
raise OSError(to_bytes("DashCast does not output frame number, encoding probably failed."))
# DashCast output sanity check
if not exists(media_out_path):
raise OSError(to_bytes("Output media asset not found, DashCast encoding probably failed."))
if returncode != 0:
raise OSError(to_bytes("DashCast return code is {0}, encoding probably failed.".format(returncode)))
# FIXME check duration too !
# Here all seem okay -------------------------------------------------------------------------------------------
media_out_size = get_size(media_out_root)
media_out_duration = get_media_duration(media_out_path)
print("{0} Transformation task successful, output media asset {1}".format(request.id, media_out.filename))
transform_callback(TransformTask.SUCCESS)
return {
"hostname": request.hostname,
"start_date": start_date,
"elapsed_time": elapsed_time,
"eta_time": 0,
"media_in_size": media_in_size,
"media_in_duration": media_in_duration,
"media_out_size": media_out_size,
"media_out_duration": media_out_duration,
"percent": 100,
}
except Exception as error:
# Here something went wrong
print("{0} Transformation task failed ".format(request.id))
transform_callback("ERROR\n{0}\n\nOUTPUT\n{1}".format(unicode(error), encoder_out))
raise
finally:
if dashcast_conf:
try_remove(dashcast_conf)
def save_settings(filename, settings):
try_makedirs(dirname(filename))
with open(filename, u'w', u'utf-8') as f:
f.write(yaml.safe_dump(settings, default_flow_style=False))
def transcode(in_relpath_json):
u"""Convert an input media file to 3 (SD) or 5 (HD) output files."""
logger = get_task_logger(u'encodebox.tasks.transcode')
report = None
in_abspath = None
failed_abspath = None
temporary_directory = None
outputs_directory = None
final_state = states.FAILURE
final_url = None
try:
settings = load_settings()
in_relpath = json.loads(in_relpath_json)
in_abspath = join(settings[u'local_directory'], in_relpath)
try:
in_directories = in_relpath.split(os.sep)
assert (len(in_directories) == 4)
publisher_id = in_directories[0]
product_id = in_directories[1]
assert (in_directories[2] == u'uploaded')
filename = in_directories[3]
name, extension = splitext(filename)
except:
raise ValueError(
to_bytes(
u'Input file path does not respect template publisher_id/product_id/filename'
))
# Generate a unguessable filename using a seed and the original filename
name = generate_unguessable_filename(settings[u'filenames_seed'],
filename)
completed_abspath = join(settings[u'local_directory'], publisher_id,
product_id, u'completed', filename)
failed_abspath = join(settings[u'local_directory'], publisher_id,
product_id, u'failed', filename)
temporary_directory = join(settings[u'local_directory'], publisher_id,
product_id, u'temporary', filename)
outputs_directory = join(settings[u'local_directory'], publisher_id,
product_id, u'outputs', filename)
remote_directory = join(settings[u'remote_directory'], publisher_id,
product_id)
remote_url = settings[u'remote_url'].format(publisher_id=publisher_id,
product_id=product_id,
name=name)
report = TranscodeProgressReport(settings[u'api_servers'],
publisher_id, product_id, filename,
getsize(in_abspath), logger)
report.send_report(states.STARTED, counter=0)
logger.info(u'Create outputs directories')
for path in (completed_abspath, failed_abspath, temporary_directory,
outputs_directory):
shutil.rmtree(path, ignore_errors=True)
try_makedirs(temporary_directory)
try_makedirs(outputs_directory)
resolution = get_media_resolution(in_abspath)
if not resolution:
raise IOError(
to_bytes(u'Unable to detect resolution of video "{0}"'.format(
in_relpath)))
quality = u'hd' if resolution[HEIGHT] >= HD_HEIGHT else u'sd'
template_transcode_passes = settings[quality + u'_transcode_passes']
template_smil_filename = settings[quality + u'_smil_template']
logger.info(u'Media {0} {1}p {2}'.format(quality.upper(),
resolution[HEIGHT],
in_relpath))
logger.info(u'Generate SMIL file from template SMIL file')
from_template(template_smil_filename,
join(outputs_directory, name + u'.smil'),
{u'name': name})
logger.info(
u'Generate transcoding passes from templated transcoding passes')
transcode_passes = passes_from_template(template_transcode_passes,
input=in_abspath,
name=name,
out=outputs_directory,
tmp=temporary_directory)
report.transcode_passes = transcode_passes
logger.info(u'Execute transcoding passes')
for counter, transcode_pass in enumerate(transcode_passes, 1):
if transcode_pass[0] in (u'ffmpeg', u'x264'):
encoder_module = globals()[transcode_pass[0]]
for statistics in encoder_module.encode(
transcode_pass[1], transcode_pass[2],
transcode_pass[3]):
status = statistics.pop(u'status').upper()
if status == u'PROGRESS':
for info in (u'output', u'returncode', u'sanity'):
statistics.pop(info, None)
report.send_report(states.ENCODING,
counter=counter,
statistics=statistics)
elif status == u'ERROR':
raise RuntimeError(statistics)
else:
try:
check_call(transcode_pass)
except OSError:
raise OSError(
to_bytes(u'Missing encoder ' + transcode_pass[0]))
logger.info(
u'Move the input file to the completed directory and send outputs to the remote host'
)
move(in_abspath, completed_abspath)
try:
report.send_report(states.TRANSFERRING)
is_remote = u':' in remote_directory
if is_remote:
# Create directory in remote host
username_host, directory = remote_directory.split(u':')
username, host = username_host.split(u'@')
ssh_client = paramiko.SSHClient()
ssh_client.load_system_host_keys()
ssh_client.set_missing_host_key_policy(
paramiko.AutoAddPolicy()) # FIXME man-in-the-middle attack
ssh_client.connect(host, username=username)
ssh_client.exec_command(u'mkdir -p "{0}"'.format(directory))
else:
# Create directory in local host
try_makedirs(remote_directory)
rsync(source=outputs_directory,
destination=remote_directory,
source_is_dir=True,
destination_is_dir=True,
archive=True,
progress=True,
recursive=True,
extra=u'ssh' if is_remote else None)
final_state, final_url = states.SUCCESS, remote_url
except Exception as e:
logger.exception(u'Transfer of outputs to remote host failed')
final_state = states.TRANSFER_ERROR
with open(join(outputs_directory, u'transfer-error.log'), u'w',
u'utf-8') as log:
log.write(repr(e))
except Exception as e:
logger.exception(u'Transcoding task failed')
try:
logger.info(u'Report the error by e-mail')
send_error_email(exception=e,
filename=in_abspath,
settings=settings)
except:
logger.exception(u'Unable to report the error by e-mail')
logger.info(
u'Move the input file to the failed directory and remove the outputs'
)
if in_abspath and failed_abspath:
move(in_abspath, failed_abspath)
if outputs_directory and exists(outputs_directory):
shutil.rmtree(outputs_directory)
raise
finally:
if report:
report.send_report(final_state, url=final_url)
logger.info(u'Remove the temporary files')
if temporary_directory and exists(temporary_directory):
shutil.rmtree(temporary_directory)
def setUp(self):
set_test_settings()
for filename in COMPLETED_FILES + OTHER_FILES:
try_makedirs(dirname(filename))