def hook_install(self): cfg = self.config self.hook_uninstall() self.generate_locales((u'fr_CH.UTF-8',)) try_makedirs(u'/etc/mysql') debconf, mysql = u'debconf-set-selections', u'mysql-server mysql-server' # Tip : http://ubuntuforums.org/showthread.php?t=981801 self.cmd(debconf, input=u'{0}/root_password select {1}'.format(mysql, cfg.mysql_root_password)) self.cmd(debconf, input=u'{0}/root_password_again select {1}'.format(mysql, cfg.mysql_root_password)) self.install_packages(WebuiHooks.PACKAGES) self.restart_ntp() self.info(u'Import Web UI database and create user') hostname = socket.gethostname() self.cmd(u'service mysql start', fail=False) self.mysql_do(u"DROP USER ''@'localhost'; DROP USER ''@'{0}';".format(hostname), fail=False) self.mysql_do(u"GRANT ALL PRIVILEGES ON *.* TO 'root'@'%%' WITH GRANT OPTION;") self.mysql_do(u'DROP DATABASE IF EXISTS webui') self.mysql_do(cli_input=open(self.local_config.site_database_file, u'r', u'utf-8').read()) self.mysql_do(u"GRANT ALL ON webui.* TO 'webui'@'%%' IDENTIFIED BY '{0}';".format(cfg.mysql_user_password)) self.info(u'Configure Apache 2') self.cmd(u'a2enmod rewrite') self.info(u'Copy and pre-configure Web UI') rsync(u'www/', self.local_config.site_directory, archive=True, delete=True, exclude_vcs=True, recursive=True) chown(self.local_config.site_directory, DAEMON_USER, DAEMON_GROUP, recursive=True) self.local_config.encryption_key = WebuiHooks.randpass(32) self.info(u'Expose Apache 2 service') self.open_port(80, u'TCP')
def test_transcode_the_media_assets(self): with mock.patch('encodebox.celeryconfig.CELERY_ALWAYS_EAGER', True, create=True): from encodebox import tasks media_filenames = sorted( f for f in os.listdir(MEDIA_INPUTS_DIRECTORY) if not f.startswith('.git')) for index, filename in enumerate(media_filenames, 1): index, name = unicode(index), basename(filename) in_relpath = join('2', index, 'uploaded', name) in_abspath = join(LOCAL_DIRECTORY, in_relpath) unguessable = generate_unguessable_filename( SETTINGS['filenames_seed'], name) try_makedirs(dirname(in_abspath)) shutil.copy(join(MEDIA_INPUTS_DIRECTORY, filename), in_abspath) tasks.transcode(json.dumps(in_relpath)) ok_( exists(join(LOCAL_DIRECTORY, '2', index, 'completed', name))) ok_(self.is_empty(join(LOCAL_DIRECTORY, '2', index, 'failed'))) ok_( self.is_empty(join(LOCAL_DIRECTORY, '2', index, 'uploaded'))) ok_( exists( join(REMOTE_DIRECTORY, '2', index, unguessable + '.smil'))) rsync(source=join(REMOTE_DIRECTORY, '2', index), destination=join(MEDIA_REMOTE_DIRECTORY, filename), destination_is_dir=True, archive=True, delete=True, makedest=True, recursive=True)
def setUp(self): set_test_settings() for directory in (LOCAL_DIRECTORY, REMOTE_DIRECTORY): try_makedirs(directory) logging.config.dictConfig({ 'version': 1, 'disable_existing_loggers': True, 'formatters': { 'simple': { 'format': '%(levelname)s %(message)s' } }, 'handlers': { 'console': { 'level': 'DEBUG', 'class': 'logging.StreamHandler', 'formatter': 'simple' } }, 'loggers': { 'encodebox.tasks.transcode': { 'handlers': ['console'], 'propagate': True, 'level': 'DEBUG' } } })
def add_media(config, media): if media.status != Media.PENDING: media_src_path = config.storage_medias_path(media, generate=False) if media_src_path: media_dst_path = config.storage_medias_path(media, generate=True) if media_dst_path != media_src_path: # Generate media storage uri and move it to media storage path + set permissions media.uri = config.storage_medias_uri(media) try_makedirs(os.path.dirname(media_dst_path)) the_error = None for i in xrange(5): try: os.rename(media_src_path, media_dst_path) # FIXME chown chmod the_error = None break except OSError as error: the_error = error time.sleep(1) if the_error: raise IndexError(to_bytes(u'An error occured : {0} ({1} -> {2}).'.format( the_error, media_src_path, media_dst_path))) try: size = get_size(os.path.dirname(media_dst_path)) except OSError: raise ValueError(to_bytes(u'Unable to detect size of media asset {0}.'.format(media_dst_path))) duration = get_media_duration(media_dst_path) if duration is None: raise ValueError(to_bytes(u'Unable to detect duration of media asset {0}.'.format(media_dst_path))) return (size, duration) else: raise NotImplementedError(to_bytes(u'FIXME Add of external URI not implemented.')) return (0, None)
def post_install(): from encodebox import lib from pytoolbox.console import confirm from pytoolbox.encoding import to_bytes from pytoolbox.filesystem import chown, from_template, try_makedirs, try_remove from pytoolbox.network.http import download if not exists(u'/usr/local/bin/neroAacEnc'): try: print(u'Download and install Nero AAC encoder') download(u'ftp://ftp6.nero.com/tools/NeroDigitalAudio.zip', u'/tmp/nero.zip') zipfile.ZipFile(u'/tmp/nero.zip').extract(u'linux/neroAacEnc', u'/usr/local/bin') os.chmod( u'/usr/local/bin/neroAacEnc', os.stat(u'/usr/local/bin/neroAacEnc').st_mode | stat.S_IEXEC) finally: try_remove(u'/tmp/nero.zip') filename = lib.SETTINGS_FILENAME settings = lib.load_settings(u'etc/config.yaml') if not exists(filename) or confirm( u'Overwrite existing configuration file "{0}"'.format(filename)): print(u'Generate configuration file "{0}"'.format(filename)) password = lib.generate_password() settings[u'rabbit_password'] = password lib.save_settings(filename, settings) print(u'Configure RabbitMQ Message Broker') check_call([u'service', u'rabbitmq-server', u'start']) call([u'rabbitmqctl', u'add_vhost', u'/']) call([u'rabbitmqctl', u'delete_user', u'guest']) call([u'rabbitmqctl', u'delete_user', u'encodebox']) call([ u'rabbitmqctl', u'add_user', u'encodebox', settings[u'rabbit_password'] ]) check_call([ u'rabbitmqctl', u'set_permissions', u'-p', u'/', u'encodebox', u'.*', u'.*', u'.*' ]) users, vhosts = lib.rabbit_users(), lib.rabbit_vhosts() print(u'RabbitMQ users: {0} vhosts: {1}'.format(users, vhosts)) if u'guest' in users or u'encodebox' not in users: raise RuntimeError(to_bytes(u'Unable to configure RabbitMQ')) print(u'Create directory for storing persistent data') try_makedirs(lib.LIB_DIRECTORY) chown(lib.LIB_DIRECTORY, lib.USERNAME, pwd.getpwnam(lib.USERNAME).pw_gid, recursive=True) print(u'Register and start our services as user ' + lib.USERNAME) from_template(u'etc/encodebox.conf.template', u'/etc/supervisor/conf.d/encodebox.conf', { u'lib_directory': lib.LIB_DIRECTORY, u'user': lib.USERNAME }) call([u'service', u'supervisor', u'force-reload'])
def load_settings(filename=None, create_directories=False): default = os.environ.get(u'ENCODEBOX_SETTINGS_FILENAME', SETTINGS_FILENAME) filename = filename or default if not exists(filename): raise IOError(to_bytes(u'Unable to find settings file "{0}".'.format(filename))) with open(filename, u'r', u'utf-8') as f: settings = yaml.load(f) for key, value in settings.iteritems(): if u'directory' in key and not u'remote' in key: settings[key] = abspath(expanduser(value)) if create_directories: try_makedirs(settings[key]) return settings
def move(source, destination): u""" Create the destination directory if missing and recursively move a file/directory from source to destination. **Example usage** >>> import os >>> open(u'/tmp/move_test_file', u'w', u'utf-8').close() >>> move(u'/tmp/move_test_file', u'/tmp/move_demo/another/file') >>> os.remove(u'/tmp/move_demo/another/file') >>> shutil.rmtree(u'/tmp/move_demo') """ try_makedirs(dirname(destination)) shutil.move(source, destination)
def test_transcode_mp4(self): with mock.patch('encodebox.celeryconfig.CELERY_ALWAYS_EAGER', True, create=True): from encodebox import tasks in_relpath = '1/2/uploaded/test.mp4' in_abspath = join(LOCAL_DIRECTORY, in_relpath) unguessable = generate_unguessable_filename( SETTINGS['filenames_seed'], 'test.mp4') try_makedirs(dirname(in_abspath)) download('http://techslides.com/demos/sample-videos/small.mp4', in_abspath) tasks.transcode(json.dumps(in_relpath)) ok_(exists(join(LOCAL_DIRECTORY, '1/2/completed/test.mp4'))) ok_(self.is_empty(join(LOCAL_DIRECTORY, '1/2/failed'))) ok_(self.is_empty(join(LOCAL_DIRECTORY, '1/2/uploaded'))) ok_(exists(join(REMOTE_DIRECTORY, '1/2', unguessable + '.smil')))
def hook_uninstall(self): self.info(u'Uninstall prerequisities, unregister service and load default configuration') self.hook_stop() try_makedirs(u'/var/log/rabbitmq') # Fix rabbitmq-server package uninstall error #self.cmd('juju destroy-environment') #self.cmd('... --purge apt-cacher-ng charm-tools juju libzookeeper-java lxc zookeeper') self.storage_unregister() if self.config.cleanup: self.cmd(u'apt-get -y remove --purge {0}'.format(u' '.join(OrchestraHooks.PACKAGES))) self.cmd(u'apt-get -y remove --purge {0}'.format(u' '.join(OrchestraHooks.FIX_PACKAGES)), fail=False) self.cmd(u'apt-get -y autoremove') #shutil.rmtree('$HOME/.juju $HOME/.ssh/id_rsa* shutil.rmtree(u'/etc/apache2/', ignore_errors=True) shutil.rmtree(u'/var/log/apache2/', ignore_errors=True) shutil.rmtree(u'/etc/rabbitmq/', ignore_errors=True) shutil.rmtree(u'/var/log/rabbitmq/', ignore_errors=True) shutil.rmtree(self.local_config.site_directory, ignore_errors=True) self.local_config.reset()
def setUp(self): OrchestraLocalConfig().write(u'test.pkl') self.hooks = OrchestraHooks_tmp(None, CONFIG, u'test.pkl', OS_ENV) shutil.copy(self.hooks.local_config.hosts_file, 'hosts') shutil.copy(u'mongodb.conf', u'mongodb_test.conf') self.hooks.local_config.hosts_file = u'hosts' # Avoid writing to system hosts file ! self.hooks.local_config.celery_config_file = u'celeryconfig.py' self.hooks.local_config.celery_template_file = os.path.join( u'../../charms/oscied-orchestra', self.hooks.local_config.celery_template_file) self.hooks.local_config.site_path = u'.' self.hooks.local_config.site_template_file = os.path.join( u'../../charms/oscied-orchestra', self.hooks.local_config.site_template_file) self.hooks.local_config.ssh_template_path = os.path.join( u'../../charms/oscied-orchestra', self.hooks.local_config.ssh_template_path) self.hooks.local_config.mongo_config_file = u'mongodb_test.conf' try_makedirs(os.path.join(self.hooks.local_config.charms_repository, u'default')) try_makedirs(u'bibi') self.hooks.directory = u'bibi'
def hook_install(self): cfg = self.config self.hook_uninstall() self.generate_locales((u'fr_CH.UTF-8',)) self.install_packages(StorageHooks.PACKAGES) self.restart_ntp() self.info(u'Configure storage bricks root') if cfg.bricks_root_device: self.cmd(u'umount {0}'.format(cfg.bricks_root_path), fail=False) if cfg.format_bricks_root: self.cmd(u'mkfs.xfs {0} -f'.format(cfg.bricks_root_device)) # FIXME detect based on the mount point self.cmd(u'mount {0} {1}'.format(cfg.bricks_root_device, cfg.bricks_root_path)) # FIXME add mdadm support? try_makedirs(self.bricks_path) self.info(u'Expose GlusterFS Server service') self.open_port(111, u'TCP') # For portmapper, and should have both TCP and UDP open self.open_port(111, u'UDP') self.open_port(24007, u'TCP') # For the Gluster Daemon #self.open_port(24008, u'TCP') # Infiniband management (optional unless you are using IB) self.open_port(24009, u'TCP') # We have only 1 storage brick (24009-24009)
def test_transcode_text(self): with mock.patch('encodebox.celeryconfig.CELERY_ALWAYS_EAGER', True, create=True): from encodebox import tasks in_relpath = '1/1/uploaded/test.txt' in_abspath = join(LOCAL_DIRECTORY, in_relpath) try_makedirs(dirname(in_abspath)) open(in_abspath, 'w', 'utf-8').write('salut') try: tasks.transcode(json.dumps(in_relpath)) raise ValueError( u'Transcoding task does not raised an exception.') except RuntimeError: pass ok_(exists(join(LOCAL_DIRECTORY, '1/1/failed/test.txt'))) ok_(self.is_empty(join(LOCAL_DIRECTORY, '1/1/completed'))) ok_(self.is_empty(join(LOCAL_DIRECTORY, '1/1/uploaded'))) ok_(self.is_empty(join(REMOTE_DIRECTORY, '1/1')))
def storage_remount(self, address=None, fstype=None, mountpoint=None, options=u''): if self.storage_config_is_enabled: self.info(u'Override storage parameters with charm configuration') address = self.config.storage_address nat_address = self.config.storage_nat_address fstype = self.config.storage_fstype mountpoint = self.config.storage_mountpoint options = self.config.storage_options elif address and fstype and mountpoint: self.info(u'Use storage parameters from charm storage relation') nat_address = u'' else: return if nat_address: self.info(u'Update hosts file to map storage internal address {0} to {1}'.format(address, nat_address)) lines = filter(lambda l: nat_address not in l, open(self.local_config.hosts_file, u'r', u'utf-8')) lines += u'{0} {1}\n'.format(nat_address, address) open(self.local_config.hosts_file, u'w', u'utf-8').write(u''.join(lines)) # Avoid unregistering and registering storage if it does not change ... if (address == self.local_config.storage_address and nat_address == self.local_config.storage_nat_address and fstype == self.local_config.storage_fstype and mountpoint == self.local_config.storage_mountpoint and options == self.local_config.storage_options): self.remark(u'Skip remount already mounted shared storage') else: self.storage_unregister() self.debug(u"Mount shared storage [{0}] {1}:{2} type {3} options '{4}' -> {5}".format(nat_address, address, mountpoint, fstype, options, self.local_config.storage_path)) try_makedirs(self.local_config.storage_path) # FIXME try X times, a better way to handle failure for i in xrange(self.local_config.storage_mount_max_retry): if self.storage_is_mounted: break mount_address = u'{0}:/{1}'.format(nat_address or address, mountpoint) mount_path = self.local_config.storage_path if options: self.cmd([u'mount', u'-t', fstype, u'-o', options, mount_address, mount_path]) else: self.cmd([u'mount', u'-t', fstype, mount_address, mount_path]) time.sleep(self.local_config.storage_mount_sleep_delay) if self.storage_is_mounted: # FIXME update /etc/fstab (?) self.local_config.storage_address = address self.local_config.storage_nat_address = nat_address self.local_config.storage_fstype = fstype self.local_config.storage_mountpoint = mountpoint self.local_config.storage_options = options self.remark(u'Shared storage successfully registered') self.debug(u'Create directories in the shared storage and ensure it is owned by the right user') try_makedirs(self.local_config.storage_medias_path()) try_makedirs(self.local_config.storage_uploads_path) chown(self.local_config.storage_path, DAEMON_USER, DAEMON_GROUP, recursive=True) else: raise IOError(to_bytes(u'Unable to mount shared storage'))
def transform_task(media_in_json, media_out_json, profile_json, callback_json): def copy_callback(start_date, elapsed_time, eta_time, src_size, dst_size, ratio): transform_task.update_state(state=TransformTask.PROGRESS, meta={ u'hostname': request.hostname, 'start_date': start_date, u'elapsed_time': elapsed_time, u'eta_time': eta_time, u'media_in_size': src_size, u'media_out_size': dst_size, u'percent': int(100 * ratio)}) def transform_callback(status, measures): data_json = object2json({u'task_id': request.id, u'status': status, u'measures': measures}, include_properties=False) if callback is None: print(u'{0} [ERROR] Unable to callback orchestrator: {1}'.format(request.id, data_json)) else: r = callback.post(data_json) print(u'{0} Code {1} {2} : {3}'.format(request.id, r.status_code, r.reason, r._content)) # ------------------------------------------------------------------------------------------------------------------ RATIO_DELTA, TIME_DELTA = 0.01, 1 # Update status if at least 1% of progress and 1 second elapsed. MAX_TIME_DELTA = 5 # Also ensure status update every 5 seconds. DASHCAST_TIMEOUT_TIME = 10 try: # Avoid 'referenced before assignment' callback = dashcast_conf = None encoder_out, request = u'', current_task.request # Let's the task begin ! print(u'{0} Transformation task started'.format(request.id)) # Read current configuration to translate files uri to local paths local_config = TransformLocalConfig.read(LOCAL_CONFIG_FILENAME, inspect_constructor=False) print(object2json(local_config, include_properties=True)) # Load and check task parameters callback = Callback.from_json(callback_json, inspect_constructor=True) callback.is_valid(True) # Update callback socket according to configuration if local_config.api_nat_socket and len(local_config.api_nat_socket) > 0: callback.replace_netloc(local_config.api_nat_socket) media_in = Media.from_json(media_in_json, inspect_constructor=True) media_out = Media.from_json(media_out_json, inspect_constructor=True) profile = TransformProfile.from_json(profile_json, inspect_constructor=True) media_in.is_valid(True) media_out.is_valid(True) profile.is_valid(True) # Verify that media file can be accessed and create output path media_in_path = local_config.storage_medias_path(media_in, generate=False) if not media_in_path: raise NotImplementedError(to_bytes(u'Input media asset will not be readed from shared storage : {0}'.format( media_in.uri))) media_out_path = local_config.storage_medias_path(media_out, generate=True) if not media_out_path: raise NotImplementedError(to_bytes(u'Output media asset will not be written to shared storage : {0}'.format( media_out.uri))) media_in_root = dirname(media_in_path) media_out_root = dirname(media_out_path) try_makedirs(media_out_root) # Get input media duration and frames to be able to estimate ETA media_in_duration = get_media_duration(media_in_path) # Keep potential PSNR status measures = {} # NOT A REAL TRANSFORM : FILE COPY ----------------------------------------------------------------------------- if profile.encoder_name == u'copy': infos = recursive_copy(media_in_root, media_out_root, copy_callback, RATIO_DELTA, TIME_DELTA) media_out_tmp = media_in_path.replace(media_in_root, media_out_root) os.rename(media_out_tmp, media_out_path) start_date = infos[u'start_date'] elapsed_time = infos[u'elapsed_time'] media_in_size = infos[u'src_size'] # A REAL TRANSFORM : TRANSCODE WITH FFMPEG --------------------------------------------------------------------- elif profile.encoder_name == u'ffmpeg': start_date, start_time = datetime_now(), time.time() prev_ratio = prev_time = 0 # Get input media size to be able to estimate ETA media_in_size = get_size(media_in_root) # Create FFmpeg subprocess cmd = u'ffmpeg -y -i "{0}" {1} "{2}"'.format(media_in_path, profile.encoder_string, media_out_path) print(cmd) ffmpeg = Popen(shlex.split(cmd), stderr=PIPE, close_fds=True) make_async(ffmpeg.stderr) while True: # Wait for data to become available select.select([ffmpeg.stderr], [], []) chunk = ffmpeg.stderr.read() encoder_out += chunk elapsed_time = time.time() - start_time match = FFMPEG_REGEX.match(chunk) if match: stats = match.groupdict() media_out_duration = stats[u'time'] try: ratio = total_seconds(media_out_duration) / total_seconds(media_in_duration) ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio except ZeroDivisionError: ratio = 1.0 delta_time = elapsed_time - prev_time if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA: prev_ratio, prev_time = ratio, elapsed_time eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0 transform_task.update_state( state=TransformTask.PROGRESS, meta={u'hostname': request.hostname, u'start_date': start_date, u'elapsed_time': elapsed_time, u'eta_time': eta_time, u'media_in_size': media_in_size, u'media_in_duration': media_in_duration, u'media_out_size': get_size(media_out_root), u'media_out_duration': media_out_duration, u'percent': int(100 * ratio), u'encoding_frame': stats[u'frame'], u'encoding_fps': stats[u'fps'], u'encoding_bitrate': stats[u'bitrate'], u'encoding_quality': stats[u'q']}) returncode = ffmpeg.poll() if returncode is not None: break # FFmpeg output sanity check if returncode != 0: raise OSError(to_bytes(u'FFmpeg return code is {0}, encoding probably failed.'.format(returncode))) # compute stats about the video measures['psnr'] = get_media_psnr(media_in_path, media_out_path) measures['ssim'] = get_media_ssim(media_in_path, media_out_path) # measures of the data and its metadata measures['bitrate'] = get_media_bitrate(media_out_path) # FIXME: fake git url, commit measures['git_url'] = 'https://github.com/videolan/x265' measures['git_commit'] = 'd2051f9544434612a105d2f5267db23018cb3454' # Output media file sanity check # media_out_duration = get_media_duration(media_out_path) # if total_seconds(media_out_duration) / total_seconds(media_in_duration) > 1.5 or < 0.8: # salut elif profile.encoder_name == u'from_git': start_date, start_time = datetime_now(), time.time() prev_ratio = prev_time = 0 # Get input media size to be able to estimate ETA media_in_size = get_size(media_in_root) metadata = media_out.metadata dirpath = tempfile.mkdtemp() prepare_cmd = u'git clone --depth=1 "{0}" "{1}" && cd "{1}" && git checkout "{2}" && {3}'.format(metadata['git_url'], dirpath, metadata['git_commit'], metadata['build_cmds']) check_call(prepare_cmd, shell=True) # Templated parameter encoder_string = profile.encoder_string.replace(u"BITRATE", str(metadata['input_bitrate'])) cmd = u'cd "{0}" && ffmpeg -y -i "{1}" -f yuv4mpegpipe - | {2} "{3}"'.format(dirpath, media_in_path, encoder_string, media_out_path) returncode = call(cmd, shell=True) if returncode != 0: raise OSError(to_bytes(u'Encoding return code is {0}, encoding probably failed.'.format(returncode))) # compute stats about the video measures['psnr'] = get_media_psnr(media_in_path, media_out_path) measures['ssim'] = get_media_ssim(media_in_path, media_out_path) # measures of the data and its metadata measures['bitrate'] = get_media_bitrate(media_out_path) # FIXME: don't put this in measures measures['git_url'] = metadata['git_url'] measures['git_commit'] = metadata['git_commit'] # A REAL TRANSFORM : TRANSCODE WITH DASHCAST ------------------------------------------------------------------- elif profile.encoder_name == u'dashcast': start_date, start_time = datetime_now(), time.time() prev_ratio = prev_time = 0 # Get input media size and frames to be able to estimate ETA media_in_size = get_size(media_in_root) try: media_in_frames = int(get_media_tracks(media_in_path)[u'video'][u'0:0'][u'estimated_frames']) media_out_frames = 0 except: raise ValueError(to_bytes(u'Unable to estimate # frames of input media asset')) # Create DashCast configuration file and subprocess dashcast_conf = u'dashcast_{0}.conf'.format(uuid.uuid4()) with open(dashcast_conf, u'w', u'utf-8') as f: f.write(profile.dash_config) cmd = u'DashCast -conf {0} -av "{1}" {2} -out "{3}" -mpd "{4}"'.format( dashcast_conf, media_in_path, profile.dash_options, media_out_root, media_out.filename) print(cmd) dashcast = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE, close_fds=True) make_async(dashcast.stdout.fileno()) make_async(dashcast.stderr.fileno()) while True: # Wait for data to become available select.select([dashcast.stdout.fileno()], [], []) stdout, stderr = read_async(dashcast.stdout), read_async(dashcast.stderr) elapsed_time = time.time() - start_time match = DASHCAST_REGEX.match(stdout) if match: stats = match.groupdict() media_out_frames = int(stats[u'frame']) try: ratio = float(media_out_frames) / media_in_frames ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio except ZeroDivisionError: ratio = 1.0 delta_time = elapsed_time - prev_time if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA: prev_ratio, prev_time = ratio, elapsed_time eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0 transform_task.update_state( state=TransformTask.PROGRESS, meta={u'hostname': request.hostname, u'start_date': start_date, u'elapsed_time': elapsed_time, u'eta_time': eta_time, u'media_in_size': media_in_size, u'media_in_duration': media_in_duration, u'media_out_size': get_size(media_out_root), u'percent': int(100 * ratio), u'encoding_frame': media_out_frames}) match = DASHCAST_SUCCESS_REGEX.match(stdout) returncode = dashcast.poll() if returncode is not None or match: encoder_out = u'stdout: {0}\nstderr: {1}'.format(stdout, stderr) break if media_out_frames == 0 and elapsed_time > DASHCAST_TIMEOUT_TIME: encoder_out = u'stdout: {0}\nstderr: {1}'.format(stdout, stderr) raise OSError(to_bytes(u'DashCast does not output frame number, encoding probably failed.')) # DashCast output sanity check if not exists(media_out_path): raise OSError(to_bytes(u'Output media asset not found, DashCast encoding probably failed.')) if returncode != 0: raise OSError(to_bytes(u'DashCast return code is {0}, encoding probably failed.'.format(returncode))) # FIXME check duration too ! # Here all seem okay ------------------------------------------------------------------------------------------- elapsed_time = time.time() - start_time media_out_size = get_size(media_out_root) media_out_duration = get_media_duration(media_out_path) print(u'{0} Transformation task successful, output media asset {1}'.format(request.id, media_out.filename)) transform_callback(TransformTask.SUCCESS, measures) return {u'hostname': request.hostname, u'start_date': start_date, u'elapsed_time': elapsed_time, u'eta_time': 0, u'media_in_size': media_in_size, u'media_in_duration': media_in_duration, u'media_out_size': media_out_size, u'media_out_duration': media_out_duration, u'percent': 100 } except Exception as error: # Here something went wrong print(u'{0} Transformation task failed '.format(request.id)) transform_callback(u'ERROR\n{0}\n\nOUTPUT\n{1}'.format(unicode(error), encoder_out), {}) raise finally: if dashcast_conf: try_remove(dashcast_conf)
common_file.write(common_data) with open(DAVID_REPORT_REFERENCES_FILE, "w", "utf-8") as references_file: references_file.write(references_data) print("Append header files into common file") for header_filename in glob.glob(join(DAVID_REPORT_SOURCE_PATH, "*.rst.header")): rst_filename = join(dirname(header_filename), splitext(header_filename)[0]) with open(rst_filename, "rw+", "utf-8") as rst_file: data = rst_file.read() with open(header_filename, "r", "utf-8") as header_file: rst_file.seek(0) rst_file.write(header_file.read() + data) # FIXME echo about.rst -> index.rst for html version at least shutil.rmtree(DAVID_REPORT_BUILD_PATH, ignore_errors=True) try_makedirs(DAVID_REPORT_BUILD_PATH) os.chdir(DAVID_REPORT_PATH) if args.html: print(HELP_HTML) result = cmd("sudo make html", fail=False) with open("build_html.log", "w", "utf-8") as log_file: log_file.write("Output:\n{0}\nError:\n{1}".format(result["stdout"], result["stderr"])) if result["returncode"] != 0: print_error("Unable to generate HTML version of the report, see build_html.log") if args.pdf: print(HELP_PDF) result = cmd("sudo make latexpdf", fail=False) with open("build_pdf.log", "w", "utf-8") as log_file: log_file.write("Output:\n{0}\nError:\n{1}".format(result["stdout"], result["stderr"]))
def transform_task(media_in_json, media_out_json, profile_json, callback_json): def copy_callback(start_date, elapsed_time, eta_time, src_size, dst_size, ratio): transform_task.update_state( state=TransformTask.PROGRESS, meta={ "hostname": request.hostname, "start_date": start_date, "elapsed_time": elapsed_time, "eta_time": eta_time, "media_in_size": src_size, "media_out_size": dst_size, "percent": int(100 * ratio), }, ) def transform_callback(status): data_json = object2json({"task_id": request.id, "status": status}, include_properties=False) if callback is None: print("{0} [ERROR] Unable to callback orchestrator: {1}".format(request.id, data_json)) else: r = callback.post(data_json) print("{0} Code {1} {2} : {3}".format(request.id, r.status_code, r.reason, r._content)) # ------------------------------------------------------------------------------------------------------------------ RATIO_DELTA, TIME_DELTA = 0.01, 1 # Update status if at least 1% of progress and 1 second elapsed. MAX_TIME_DELTA = 5 # Also ensure status update every 5 seconds. DASHCAST_TIMEOUT_TIME = 10 try: # Avoid 'referenced before assignment' callback = dashcast_conf = None encoder_out, request = "", current_task.request # Let's the task begin ! print("{0} Transformation task started".format(request.id)) # Read current configuration to translate files uri to local paths local_config = TransformLocalConfig.read(LOCAL_CONFIG_FILENAME, inspect_constructor=False) print(object2json(local_config, include_properties=True)) # Load and check task parameters callback = Callback.from_json(callback_json, inspect_constructor=True) callback.is_valid(True) # Update callback socket according to configuration if local_config.api_nat_socket and len(local_config.api_nat_socket) > 0: callback.replace_netloc(local_config.api_nat_socket) media_in = Media.from_json(media_in_json, inspect_constructor=True) media_out = Media.from_json(media_out_json, inspect_constructor=True) profile = TransformProfile.from_json(profile_json, inspect_constructor=True) media_in.is_valid(True) media_out.is_valid(True) profile.is_valid(True) # Verify that media file can be accessed and create output path media_in_path = local_config.storage_medias_path(media_in, generate=False) if not media_in_path: raise NotImplementedError( to_bytes("Input media asset will not be readed from shared storage : {0}".format(media_in.uri)) ) media_out_path = local_config.storage_medias_path(media_out, generate=True) if not media_out_path: raise NotImplementedError( to_bytes("Output media asset will not be written to shared storage : {0}".format(media_out.uri)) ) media_in_root = dirname(media_in_path) media_out_root = dirname(media_out_path) try_makedirs(media_out_root) # Get input media duration and frames to be able to estimate ETA media_in_duration = get_media_duration(media_in_path) # NOT A REAL TRANSFORM : FILE COPY ----------------------------------------------------------------------------- if profile.encoder_name == "copy": infos = recursive_copy(media_in_root, media_out_root, copy_callback, RATIO_DELTA, TIME_DELTA) media_out_tmp = media_in_path.replace(media_in_root, media_out_root) os.rename(media_out_tmp, media_out_path) start_date = infos["start_date"] elapsed_time = infos["elapsed_time"] media_in_size = infos["src_size"] # A REAL TRANSFORM : TRANSCODE WITH FFMPEG --------------------------------------------------------------------- elif profile.encoder_name == "ffmpeg": start_date, start_time = datetime_now(), time.time() prev_ratio = prev_time = 0 # Get input media size to be able to estimate ETA media_in_size = get_size(media_in_root) # Create FFmpeg subprocess cmd = 'ffmpeg -y -i "{0}" {1} "{2}"'.format(media_in_path, profile.encoder_string, media_out_path) print(cmd) ffmpeg = Popen(shlex.split(cmd), stderr=PIPE, close_fds=True) make_async(ffmpeg.stderr) while True: # Wait for data to become available select.select([ffmpeg.stderr], [], []) chunk = ffmpeg.stderr.read() encoder_out += chunk elapsed_time = time.time() - start_time match = FFMPEG_REGEX.match(chunk) if match: stats = match.groupdict() media_out_duration = stats["time"] try: ratio = total_seconds(media_out_duration) / total_seconds(media_in_duration) ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio except ZeroDivisionError: ratio = 1.0 delta_time = elapsed_time - prev_time if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA: prev_ratio, prev_time = ratio, elapsed_time eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0 transform_task.update_state( state=TransformTask.PROGRESS, meta={ "hostname": request.hostname, "start_date": start_date, "elapsed_time": elapsed_time, "eta_time": eta_time, "media_in_size": media_in_size, "media_in_duration": media_in_duration, "media_out_size": get_size(media_out_root), "media_out_duration": media_out_duration, "percent": int(100 * ratio), "encoding_frame": stats["frame"], "encoding_fps": stats["fps"], "encoding_bitrate": stats["bitrate"], "encoding_quality": stats["q"], }, ) returncode = ffmpeg.poll() if returncode is not None: break # FFmpeg output sanity check if returncode != 0: raise OSError(to_bytes("FFmpeg return code is {0}, encoding probably failed.".format(returncode))) # Output media file sanity check # media_out_duration = get_media_duration(media_out_path) # if total_seconds(media_out_duration) / total_seconds(media_in_duration) > 1.5 or < 0.8: # salut # A REAL TRANSFORM : TRANSCODE WITH DASHCAST ------------------------------------------------------------------- elif profile.encoder_name == "dashcast": start_date, start_time = datetime_now(), time.time() prev_ratio = prev_time = 0 # Get input media size and frames to be able to estimate ETA media_in_size = get_size(media_in_root) try: media_in_frames = int(get_media_tracks(media_in_path)["video"]["0:0"]["estimated_frames"]) media_out_frames = 0 except: raise ValueError(to_bytes("Unable to estimate # frames of input media asset")) # Create DashCast configuration file and subprocess dashcast_conf = "dashcast_{0}.conf".format(uuid.uuid4()) with open(dashcast_conf, "w", "utf-8") as f: f.write(profile.dash_config) cmd = 'DashCast -conf {0} -av "{1}" {2} -out "{3}" -mpd "{4}"'.format( dashcast_conf, media_in_path, profile.dash_options, media_out_root, media_out.filename ) print(cmd) dashcast = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE, close_fds=True) make_async(dashcast.stdout.fileno()) make_async(dashcast.stderr.fileno()) while True: # Wait for data to become available select.select([dashcast.stdout.fileno()], [], []) stdout, stderr = read_async(dashcast.stdout), read_async(dashcast.stderr) elapsed_time = time.time() - start_time match = DASHCAST_REGEX.match(stdout) if match: stats = match.groupdict() media_out_frames = int(stats["frame"]) try: ratio = float(media_out_frames) / media_in_frames ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio except ZeroDivisionError: ratio = 1.0 delta_time = elapsed_time - prev_time if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA: prev_ratio, prev_time = ratio, elapsed_time eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0 transform_task.update_state( state=TransformTask.PROGRESS, meta={ "hostname": request.hostname, "start_date": start_date, "elapsed_time": elapsed_time, "eta_time": eta_time, "media_in_size": media_in_size, "media_in_duration": media_in_duration, "media_out_size": get_size(media_out_root), "percent": int(100 * ratio), "encoding_frame": media_out_frames, }, ) match = DASHCAST_SUCCESS_REGEX.match(stdout) returncode = dashcast.poll() if returncode is not None or match: encoder_out = "stdout: {0}\nstderr: {1}".format(stdout, stderr) break if media_out_frames == 0 and elapsed_time > DASHCAST_TIMEOUT_TIME: encoder_out = "stdout: {0}\nstderr: {1}".format(stdout, stderr) raise OSError(to_bytes("DashCast does not output frame number, encoding probably failed.")) # DashCast output sanity check if not exists(media_out_path): raise OSError(to_bytes("Output media asset not found, DashCast encoding probably failed.")) if returncode != 0: raise OSError(to_bytes("DashCast return code is {0}, encoding probably failed.".format(returncode))) # FIXME check duration too ! # Here all seem okay ------------------------------------------------------------------------------------------- media_out_size = get_size(media_out_root) media_out_duration = get_media_duration(media_out_path) print("{0} Transformation task successful, output media asset {1}".format(request.id, media_out.filename)) transform_callback(TransformTask.SUCCESS) return { "hostname": request.hostname, "start_date": start_date, "elapsed_time": elapsed_time, "eta_time": 0, "media_in_size": media_in_size, "media_in_duration": media_in_duration, "media_out_size": media_out_size, "media_out_duration": media_out_duration, "percent": 100, } except Exception as error: # Here something went wrong print("{0} Transformation task failed ".format(request.id)) transform_callback("ERROR\n{0}\n\nOUTPUT\n{1}".format(unicode(error), encoder_out)) raise finally: if dashcast_conf: try_remove(dashcast_conf)
def save_settings(filename, settings): try_makedirs(dirname(filename)) with open(filename, u'w', u'utf-8') as f: f.write(yaml.safe_dump(settings, default_flow_style=False))
def transcode(in_relpath_json): u"""Convert an input media file to 3 (SD) or 5 (HD) output files.""" logger = get_task_logger(u'encodebox.tasks.transcode') report = None in_abspath = None failed_abspath = None temporary_directory = None outputs_directory = None final_state = states.FAILURE final_url = None try: settings = load_settings() in_relpath = json.loads(in_relpath_json) in_abspath = join(settings[u'local_directory'], in_relpath) try: in_directories = in_relpath.split(os.sep) assert (len(in_directories) == 4) publisher_id = in_directories[0] product_id = in_directories[1] assert (in_directories[2] == u'uploaded') filename = in_directories[3] name, extension = splitext(filename) except: raise ValueError( to_bytes( u'Input file path does not respect template publisher_id/product_id/filename' )) # Generate a unguessable filename using a seed and the original filename name = generate_unguessable_filename(settings[u'filenames_seed'], filename) completed_abspath = join(settings[u'local_directory'], publisher_id, product_id, u'completed', filename) failed_abspath = join(settings[u'local_directory'], publisher_id, product_id, u'failed', filename) temporary_directory = join(settings[u'local_directory'], publisher_id, product_id, u'temporary', filename) outputs_directory = join(settings[u'local_directory'], publisher_id, product_id, u'outputs', filename) remote_directory = join(settings[u'remote_directory'], publisher_id, product_id) remote_url = settings[u'remote_url'].format(publisher_id=publisher_id, product_id=product_id, name=name) report = TranscodeProgressReport(settings[u'api_servers'], publisher_id, product_id, filename, getsize(in_abspath), logger) report.send_report(states.STARTED, counter=0) logger.info(u'Create outputs directories') for path in (completed_abspath, failed_abspath, temporary_directory, outputs_directory): shutil.rmtree(path, ignore_errors=True) try_makedirs(temporary_directory) try_makedirs(outputs_directory) resolution = get_media_resolution(in_abspath) if not resolution: raise IOError( to_bytes(u'Unable to detect resolution of video "{0}"'.format( in_relpath))) quality = u'hd' if resolution[HEIGHT] >= HD_HEIGHT else u'sd' template_transcode_passes = settings[quality + u'_transcode_passes'] template_smil_filename = settings[quality + u'_smil_template'] logger.info(u'Media {0} {1}p {2}'.format(quality.upper(), resolution[HEIGHT], in_relpath)) logger.info(u'Generate SMIL file from template SMIL file') from_template(template_smil_filename, join(outputs_directory, name + u'.smil'), {u'name': name}) logger.info( u'Generate transcoding passes from templated transcoding passes') transcode_passes = passes_from_template(template_transcode_passes, input=in_abspath, name=name, out=outputs_directory, tmp=temporary_directory) report.transcode_passes = transcode_passes logger.info(u'Execute transcoding passes') for counter, transcode_pass in enumerate(transcode_passes, 1): if transcode_pass[0] in (u'ffmpeg', u'x264'): encoder_module = globals()[transcode_pass[0]] for statistics in encoder_module.encode( transcode_pass[1], transcode_pass[2], transcode_pass[3]): status = statistics.pop(u'status').upper() if status == u'PROGRESS': for info in (u'output', u'returncode', u'sanity'): statistics.pop(info, None) report.send_report(states.ENCODING, counter=counter, statistics=statistics) elif status == u'ERROR': raise RuntimeError(statistics) else: try: check_call(transcode_pass) except OSError: raise OSError( to_bytes(u'Missing encoder ' + transcode_pass[0])) logger.info( u'Move the input file to the completed directory and send outputs to the remote host' ) move(in_abspath, completed_abspath) try: report.send_report(states.TRANSFERRING) is_remote = u':' in remote_directory if is_remote: # Create directory in remote host username_host, directory = remote_directory.split(u':') username, host = username_host.split(u'@') ssh_client = paramiko.SSHClient() ssh_client.load_system_host_keys() ssh_client.set_missing_host_key_policy( paramiko.AutoAddPolicy()) # FIXME man-in-the-middle attack ssh_client.connect(host, username=username) ssh_client.exec_command(u'mkdir -p "{0}"'.format(directory)) else: # Create directory in local host try_makedirs(remote_directory) rsync(source=outputs_directory, destination=remote_directory, source_is_dir=True, destination_is_dir=True, archive=True, progress=True, recursive=True, extra=u'ssh' if is_remote else None) final_state, final_url = states.SUCCESS, remote_url except Exception as e: logger.exception(u'Transfer of outputs to remote host failed') final_state = states.TRANSFER_ERROR with open(join(outputs_directory, u'transfer-error.log'), u'w', u'utf-8') as log: log.write(repr(e)) except Exception as e: logger.exception(u'Transcoding task failed') try: logger.info(u'Report the error by e-mail') send_error_email(exception=e, filename=in_abspath, settings=settings) except: logger.exception(u'Unable to report the error by e-mail') logger.info( u'Move the input file to the failed directory and remove the outputs' ) if in_abspath and failed_abspath: move(in_abspath, failed_abspath) if outputs_directory and exists(outputs_directory): shutil.rmtree(outputs_directory) raise finally: if report: report.send_report(final_state, url=final_url) logger.info(u'Remove the temporary files') if temporary_directory and exists(temporary_directory): shutil.rmtree(temporary_directory)
def setUp(self): set_test_settings() for filename in COMPLETED_FILES + OTHER_FILES: try_makedirs(dirname(filename))