def generate_config_from_template(self, overwrite=False, **kwargs):
if not os.path.exists(self.config) or overwrite:
chars, size = string.ascii_letters + string.digits, 16
params = {p: u''.join(random.choice(chars) for i in xrange(size)) for p in self.config_passwords}
params.update({u'charms_release': self.release})
params.update(kwargs)
from_template(self.config_template, self.config, params)
def post_install():
from encodebox import lib
from pytoolbox.console import confirm
from pytoolbox.encoding import to_bytes
from pytoolbox.filesystem import chown, from_template, try_makedirs, try_remove
from pytoolbox.network.http import download
if not exists(u'/usr/local/bin/neroAacEnc'):
try:
print(u'Download and install Nero AAC encoder')
download(u'ftp://ftp6.nero.com/tools/NeroDigitalAudio.zip',
u'/tmp/nero.zip')
zipfile.ZipFile(u'/tmp/nero.zip').extract(u'linux/neroAacEnc',
u'/usr/local/bin')
os.chmod(
u'/usr/local/bin/neroAacEnc',
os.stat(u'/usr/local/bin/neroAacEnc').st_mode | stat.S_IEXEC)
finally:
try_remove(u'/tmp/nero.zip')
filename = lib.SETTINGS_FILENAME
settings = lib.load_settings(u'etc/config.yaml')
if not exists(filename) or confirm(
u'Overwrite existing configuration file "{0}"'.format(filename)):
print(u'Generate configuration file "{0}"'.format(filename))
password = lib.generate_password()
settings[u'rabbit_password'] = password
lib.save_settings(filename, settings)
print(u'Configure RabbitMQ Message Broker')
check_call([u'service', u'rabbitmq-server', u'start'])
call([u'rabbitmqctl', u'add_vhost', u'/'])
call([u'rabbitmqctl', u'delete_user', u'guest'])
call([u'rabbitmqctl', u'delete_user', u'encodebox'])
call([
u'rabbitmqctl', u'add_user', u'encodebox', settings[u'rabbit_password']
])
check_call([
u'rabbitmqctl', u'set_permissions', u'-p', u'/', u'encodebox', u'.*',
u'.*', u'.*'
])
users, vhosts = lib.rabbit_users(), lib.rabbit_vhosts()
print(u'RabbitMQ users: {0} vhosts: {1}'.format(users, vhosts))
if u'guest' in users or u'encodebox' not in users:
raise RuntimeError(to_bytes(u'Unable to configure RabbitMQ'))
print(u'Create directory for storing persistent data')
try_makedirs(lib.LIB_DIRECTORY)
chown(lib.LIB_DIRECTORY,
lib.USERNAME,
pwd.getpwnam(lib.USERNAME).pw_gid,
recursive=True)
print(u'Register and start our services as user ' + lib.USERNAME)
from_template(u'etc/encodebox.conf.template',
u'/etc/supervisor/conf.d/encodebox.conf', {
u'lib_directory': lib.LIB_DIRECTORY,
u'user': lib.USERNAME
})
call([u'service', u'supervisor', u'force-reload'])
def main():
roles = {Path(p).name for p in ROLES_SOURCE_DIRECTORY.glob('*')}
filesystem.from_template(REQUIREMENTS_TEMPLATE,
REQUIREMENTS_FILENAME,
values={
'GITHUB_USER': GITHUB_USER,
'roles': roles
},
jinja2=True)
print('OK')
def process_role(role, roles):
directory = ROLES_TARGET_DIRECTORY / f'ansible-role-{role}'
if not directory.exists():
print('Create role', role)
try:
rsync(LIBRARY_DIRECTORY, directory, destination_is_dir=True)
os.chdir(directory)
paths = PATHS + [f'roles/{r}' for r in roles - {role}]
print('\tFiltering')
subprocess.check_output([
'git', 'filter-branch', '--force', '--index-filter',
f"git rm --cached --ignore-unmatch -r {' '.join(paths)}",
'--prune-empty', '--tag-name-filter', 'cat', '--', '--all'
])
for path in (Path('roles') / role).glob('*'):
print('\tMove directory', path.name)
subprocess.check_output(['git', 'mv', path, path.name])
subprocess.check_output(['git', 'clean', '-f', '-d'])
print('\tGenerate README')
filesystem.from_template(README_TEMPLATE,
'README.md',
values={
'has_meta': Path('meta').exists(),
'role': role
},
jinja2=True)
subprocess.check_output(['git', 'add', 'README.md'])
subprocess.check_output(['git', 'commit', '-m', MESSAGE])
print('\tJob done!')
except Exception:
filesystem.remove(directory, recursive=True)
raise
print('Push role', role)
os.chdir(directory)
url = create_github_repo(role)
subprocess.check_call(['git', 'remote', 'remove', 'origin'])
subprocess.check_call(['git', 'remote', 'add', 'origin', url])
subprocess.check_call(['git', 'push', '--all'])
m.replace('.py', '').replace('./', '').replace('/', '.')
for m in cmd('find . -type f -name "*.py"', cwd='../pytoolbox', shell=True)
['stdout'].split() if m.endswith('.py') and not '__init__' in m)
print('Detected modules are: {0}'.format(modules))
api_toc = ''
for module in modules:
if 'django' in module or 'crypto' in module:
continue # FIXME temporary hack, see issue #6
module = 'pytoolbox.{0}'.format(module)
title = module.replace('.', ' > ')
api_toc += ' {0}\n'.format(module)
from_template('templates/module.rst.template',
'source/{0}.rst'.format(module), {
'module': module,
'title': title,
'equals': '=' * len(title)
})
from_template('templates/api.rst.template', 'source/api.rst',
{'api_toc': api_toc})
shutil.rmtree('build/html', ignore_errors=True)
result = cmd('make html', fail=False)
print('\nOutputs\n=======\n')
print(result['stdout'])
print('\nErrors\n======\n')
print(result['stderr'])
sys.exit(0 if not result['stderr'] else 1)
def transcode(in_relpath_json):
u"""Convert an input media file to 3 (SD) or 5 (HD) output files."""
logger = get_task_logger(u'encodebox.tasks.transcode')
report = None
in_abspath = None
failed_abspath = None
temporary_directory = None
outputs_directory = None
final_state = states.FAILURE
final_url = None
try:
settings = load_settings()
in_relpath = json.loads(in_relpath_json)
in_abspath = join(settings[u'local_directory'], in_relpath)
try:
in_directories = in_relpath.split(os.sep)
assert (len(in_directories) == 4)
publisher_id = in_directories[0]
product_id = in_directories[1]
assert (in_directories[2] == u'uploaded')
filename = in_directories[3]
name, extension = splitext(filename)
except:
raise ValueError(
to_bytes(
u'Input file path does not respect template publisher_id/product_id/filename'
))
# Generate a unguessable filename using a seed and the original filename
name = generate_unguessable_filename(settings[u'filenames_seed'],
filename)
completed_abspath = join(settings[u'local_directory'], publisher_id,
product_id, u'completed', filename)
failed_abspath = join(settings[u'local_directory'], publisher_id,
product_id, u'failed', filename)
temporary_directory = join(settings[u'local_directory'], publisher_id,
product_id, u'temporary', filename)
outputs_directory = join(settings[u'local_directory'], publisher_id,
product_id, u'outputs', filename)
remote_directory = join(settings[u'remote_directory'], publisher_id,
product_id)
remote_url = settings[u'remote_url'].format(publisher_id=publisher_id,
product_id=product_id,
name=name)
report = TranscodeProgressReport(settings[u'api_servers'],
publisher_id, product_id, filename,
getsize(in_abspath), logger)
report.send_report(states.STARTED, counter=0)
logger.info(u'Create outputs directories')
for path in (completed_abspath, failed_abspath, temporary_directory,
outputs_directory):
shutil.rmtree(path, ignore_errors=True)
try_makedirs(temporary_directory)
try_makedirs(outputs_directory)
resolution = get_media_resolution(in_abspath)
if not resolution:
raise IOError(
to_bytes(u'Unable to detect resolution of video "{0}"'.format(
in_relpath)))
quality = u'hd' if resolution[HEIGHT] >= HD_HEIGHT else u'sd'
template_transcode_passes = settings[quality + u'_transcode_passes']
template_smil_filename = settings[quality + u'_smil_template']
logger.info(u'Media {0} {1}p {2}'.format(quality.upper(),
resolution[HEIGHT],
in_relpath))
logger.info(u'Generate SMIL file from template SMIL file')
from_template(template_smil_filename,
join(outputs_directory, name + u'.smil'),
{u'name': name})
logger.info(
u'Generate transcoding passes from templated transcoding passes')
transcode_passes = passes_from_template(template_transcode_passes,
input=in_abspath,
name=name,
out=outputs_directory,
tmp=temporary_directory)
report.transcode_passes = transcode_passes
logger.info(u'Execute transcoding passes')
for counter, transcode_pass in enumerate(transcode_passes, 1):
if transcode_pass[0] in (u'ffmpeg', u'x264'):
encoder_module = globals()[transcode_pass[0]]
for statistics in encoder_module.encode(
transcode_pass[1], transcode_pass[2],
transcode_pass[3]):
status = statistics.pop(u'status').upper()
if status == u'PROGRESS':
for info in (u'output', u'returncode', u'sanity'):
statistics.pop(info, None)
report.send_report(states.ENCODING,
counter=counter,
statistics=statistics)
elif status == u'ERROR':
raise RuntimeError(statistics)
else:
try:
check_call(transcode_pass)
except OSError:
raise OSError(
to_bytes(u'Missing encoder ' + transcode_pass[0]))
logger.info(
u'Move the input file to the completed directory and send outputs to the remote host'
)
move(in_abspath, completed_abspath)
try:
report.send_report(states.TRANSFERRING)
is_remote = u':' in remote_directory
if is_remote:
# Create directory in remote host
username_host, directory = remote_directory.split(u':')
username, host = username_host.split(u'@')
ssh_client = paramiko.SSHClient()
ssh_client.load_system_host_keys()
ssh_client.set_missing_host_key_policy(
paramiko.AutoAddPolicy()) # FIXME man-in-the-middle attack
ssh_client.connect(host, username=username)
ssh_client.exec_command(u'mkdir -p "{0}"'.format(directory))
else:
# Create directory in local host
try_makedirs(remote_directory)
rsync(source=outputs_directory,
destination=remote_directory,
source_is_dir=True,
destination_is_dir=True,
archive=True,
progress=True,
recursive=True,
extra=u'ssh' if is_remote else None)
final_state, final_url = states.SUCCESS, remote_url
except Exception as e:
logger.exception(u'Transfer of outputs to remote host failed')
final_state = states.TRANSFER_ERROR
with open(join(outputs_directory, u'transfer-error.log'), u'w',
u'utf-8') as log:
log.write(repr(e))
except Exception as e:
logger.exception(u'Transcoding task failed')
try:
logger.info(u'Report the error by e-mail')
send_error_email(exception=e,
filename=in_abspath,
settings=settings)
except:
logger.exception(u'Unable to report the error by e-mail')
logger.info(
u'Move the input file to the failed directory and remove the outputs'
)
if in_abspath and failed_abspath:
move(in_abspath, failed_abspath)
if outputs_directory and exists(outputs_directory):
shutil.rmtree(outputs_directory)
raise
finally:
if report:
report.send_report(final_state, url=final_url)
logger.info(u'Remove the temporary files')
if temporary_directory and exists(temporary_directory):
shutil.rmtree(temporary_directory)
configure_unicode()
# Detect modules, thanks to find !
modules = sorted(m.replace('.py', '').replace('./', '').replace('/', '.')
for m in cmd('find . -type f -name "*.py"', cwd='../pytoolbox', shell=True)['stdout'].split()
if m.endswith('.py') and not '__init__' in m)
print('Detected modules are: {0}'.format(modules))
api_toc = ''
for module in modules:
if 'django' in module or 'crypto' in module:
continue # FIXME temporary hack, see issue #6
module = 'pytoolbox.{0}'.format(module)
title = module.replace('.', ' > ')
api_toc += ' {0}\n'.format(module)
from_template('templates/module.rst.template', 'source/{0}.rst'.format(module),
{'module': module, 'title': title, 'equals': '='*len(title)})
from_template('templates/api.rst.template', 'source/api.rst', {'api_toc': api_toc})
shutil.rmtree('build/html', ignore_errors=True)
result = cmd('make html', fail=False)
print('\nOutputs\n=======\n')
print(result['stdout'])
print('\nErrors\n======\n')
print(result['stderr'])
sys.exit(0 if not result['stderr'] else 1)
configure_unicode()
# Detect modules, thanks to find !
modules = sorted(m.replace(u'.py', u'').replace(u'./', u'').replace(u'/', u'.')
for m in cmd(u'find . -type f -name "*.py"', cwd=u'../pytoolbox', shell=True)[u'stdout'].split()
if m.endswith(u'.py') and not u'__init__' in m)
print(u'Detected modules are: {0}'.format(modules))
api_toc = u''
for module in modules:
if u'django' in module or u'crypto' in module:
continue # FIXME temporary hack, see issue #6
module = u'pytoolbox.{0}'.format(module)
title = module.replace(u'.', u' > ')
api_toc += u' {0}\n'.format(module)
from_template(u'templates/module.rst.template', u'source/{0}.rst'.format(module),
{u'module': module, u'title': title, u'equals': u'='*len(title)})
from_template(u'templates/api.rst.template', u'source/api.rst', {u'api_toc': api_toc})
shutil.rmtree(u'build/html', ignore_errors=True)
result = cmd(u'make html', fail=False)
print(u'\nOutputs\n=======\n')
print(result[u'stdout'])
print(u'\nErrors\n======\n')
print(result[u'stderr'])
sys.exit(0 if not result[u'stderr'] else 1)
