def run(self):
print(description)
env = self.dev
env.symlink_local_charms()
env.generate_config_from_template()
if confirm(u'Deploy the platform now', default=True):
do_merge = confirm(u'Merge services (takes more time to setup, cost less if running for hours)')
print(u'')
env.bootstrap(wait_started=True)
env.auto = True
ensure = partial(env.ensure_num_units, constraints=C1_MEDIUM, local=True)
ensure(u'oscied-orchestra', u'oscied-orchestra', expose=True)
ensure(u'oscied-storage', u'oscied-storage')
ensure(u'oscied-transform', u'oscied-transform')
ensure(u'oscied-webui', u'oscied-webui', to=1 if do_merge else None, expose=True)
ensure(u'oscied-publisher', u'oscied-publisher', to=2 if do_merge else None, expose=True)
for peer in (u'orchestra', u'webui', u'transform', u'publisher'):
env.add_relation(u'oscied-storage', u'oscied-{0}'.format(peer))
env.add_relation(u'oscied-orchestra:transform', u'oscied-transform:transform')
env.add_relation(u'oscied-orchestra:publisher', u'oscied-publisher:publisher')
env.add_relation(u'oscied-orchestra:api', u'oscied-webui:api')
env.auto = False
#self.dev.check_status(raise_if_errors=True, wait_all_started=True)
if confirm(u'Initialize orchestra (will wait until orchestra is ready)', default=True):
env.init_api(SCENARIO_PATH, flush=True)#, wait_started=True)
def run(self):
print (description)
if confirm(u"Deploy on MAAS", default=False):
self.deploy_maas()
if confirm(u"Deploy on Amazon", default=False):
self.deploy_amazon()
if confirm(u"Initialize orchestra on MAAS", default=False):
self.maas.init_api(SCENARIO_PATH, add_users=False, flush=True, backup_medias_in_remote=False)
if confirm(u"Start events loop", default=True):
self.events_loop()
def run(self):
print(description)
if confirm(u'Deploy on MAAS'):
self.deploy_maas()
if confirm(u'Initialize orchestra on MAAS'):
self.maas.init_api(SCENARIO_PATH, flush=True)
if confirm(u'Deploy on Amazon'):
self.deploy_amazon()
if confirm(u'Initialize orchestra on Amazon'):
self.amazon.init_api(SCENARIO_PATH, flush=True)
if confirm(u'Start events loop', default=True):
self.events_loop()
def deploy_maas(self):
u"""Deploy a full OSCIED setup in the EBU's private cluster (4 machines) provisioned by the MAAS controller."""
self.maas.symlink_local_charms()
self.maas.generate_config_from_template()
self.maas.bootstrap(wait_started=True, timeout=1200, polling_delay=30, synchronize_tools=True)
ensure = partial(self.maas.ensure_num_units, local=True)
ensure(u'oscied-storage', u'oscied-storage', num_units=3, expose=True) # 1,2,3
# WAIT
ensure(u'oscied-orchestra', u'oscied-orchestra', to=1, expose=True)
# WAIT
ensure(u'oscied-webui', u'oscied-webui', to=1, expose=True)
ensure(u'oscied-publisher', u'oscied-publisher', to=2, expose=True)
ensure(u'oscied-publisher', u'oscied-publisher', to=3, expose=True) #3=5
# WAIT
ensure(u'oscied-transform', u'oscied-transform', to=1)
ensure(u'oscied-transform', u'oscied-transform', to=2)
ensure(u'oscied-transform', u'oscied-transform', to=3) #3=5
# WAIT -> Makes juju crazy (/var/log/juju/all-machines.log -> growing to GB)
ensure(u'oscied-storage', u'oscied-storage', to=0, expose=True)
# WAIT -> Makes juju crazy (/var/log/juju/all-machines.log -> growing to GB)
ensure(u'oscied-transform', u'oscied-transform', to=0, expose=True)
if confirm(u'Disconnect all services [DEBUG PURPOSE ONLY] (with juju remove-relation)'):
for peer in (u'orchestra', u'webui', u'transform', u'publisher'):
self.maas.remove_relation(u'oscied-storage', u'oscied-{0}'.format(peer))
self.maas.remove_relation(u'oscied-orchestra:api', u'oscied-webui:api')
self.maas.remove_relation(u'oscied-orchestra:transform', u'oscied-transform:transform')
self.maas.remove_relation(u'oscied-orchestra:publisher', u'oscied-publisher:publisher')
for peer in (u'orchestra', u'webui', u'transform', u'publisher'):
self.maas.add_relation(u'oscied-storage', u'oscied-{0}'.format(peer))
self.maas.add_relation(u'oscied-orchestra:api', u'oscied-webui:api')
self.maas.add_relation(u'oscied-orchestra:transform', u'oscied-transform:transform')
self.maas.add_relation(u'oscied-orchestra:publisher', u'oscied-publisher:publisher')
def post_install():
from encodebox import lib
from pytoolbox.console import confirm
from pytoolbox.encoding import to_bytes
from pytoolbox.filesystem import chown, from_template, try_makedirs, try_remove
from pytoolbox.network.http import download
if not exists(u'/usr/local/bin/neroAacEnc'):
try:
print(u'Download and install Nero AAC encoder')
download(u'ftp://ftp6.nero.com/tools/NeroDigitalAudio.zip',
u'/tmp/nero.zip')
zipfile.ZipFile(u'/tmp/nero.zip').extract(u'linux/neroAacEnc',
u'/usr/local/bin')
os.chmod(
u'/usr/local/bin/neroAacEnc',
os.stat(u'/usr/local/bin/neroAacEnc').st_mode | stat.S_IEXEC)
finally:
try_remove(u'/tmp/nero.zip')
filename = lib.SETTINGS_FILENAME
settings = lib.load_settings(u'etc/config.yaml')
if not exists(filename) or confirm(
u'Overwrite existing configuration file "{0}"'.format(filename)):
print(u'Generate configuration file "{0}"'.format(filename))
password = lib.generate_password()
settings[u'rabbit_password'] = password
lib.save_settings(filename, settings)
print(u'Configure RabbitMQ Message Broker')
check_call([u'service', u'rabbitmq-server', u'start'])
call([u'rabbitmqctl', u'add_vhost', u'/'])
call([u'rabbitmqctl', u'delete_user', u'guest'])
call([u'rabbitmqctl', u'delete_user', u'encodebox'])
call([
u'rabbitmqctl', u'add_user', u'encodebox', settings[u'rabbit_password']
])
check_call([
u'rabbitmqctl', u'set_permissions', u'-p', u'/', u'encodebox', u'.*',
u'.*', u'.*'
])
users, vhosts = lib.rabbit_users(), lib.rabbit_vhosts()
print(u'RabbitMQ users: {0} vhosts: {1}'.format(users, vhosts))
if u'guest' in users or u'encodebox' not in users:
raise RuntimeError(to_bytes(u'Unable to configure RabbitMQ'))
print(u'Create directory for storing persistent data')
try_makedirs(lib.LIB_DIRECTORY)
chown(lib.LIB_DIRECTORY,
lib.USERNAME,
pwd.getpwnam(lib.USERNAME).pw_gid,
recursive=True)
print(u'Register and start our services as user ' + lib.USERNAME)
from_template(u'etc/encodebox.conf.template',
u'/etc/supervisor/conf.d/encodebox.conf', {
u'lib_directory': lib.LIB_DIRECTORY,
u'user': lib.USERNAME
})
call([u'service', u'supervisor', u'force-reload'])
def cleanup_transform_tasks(self, api_client, auto=False, cleanup_progress_time=20):
u"""Cleanup transformation tasks that stuck in progress status without updating the eta_time."""
tasks, new_time = api_client.transform_tasks.list(head=True), time.time()
progress_tasks = [t for t in tasks if t.status == TaskModel.PROGRESS]
delta_time = new_time - self.progress_tasks[0]
if cleanup_progress_time and delta_time > cleanup_progress_time:
for task in progress_tasks:
try:
prev_task = next(t for t in self.progress_tasks[1] if t._id == task._id)
prev_eta_time = prev_task.statistic.get(u'eta_time')
eta_time = task.statistic.get(u'eta_time')
logging.debug(u'PROGRESS task {0} previous eta_time {1}, current {2}'.format(
task._id, prev_eta_time, eta_time))
if eta_time == prev_eta_time:
logging.warning(u"PROGRESS task {0} hasn't updated is eta_time for at least {1} seconds.".
format(task._id, delta_time))
logging.info(task.__dict__)
if auto or confirm(u'Revoke the task now ?'):
del api_client.transform_tasks[task._id]
except StopIteration:
pass
self.progress_tasks = (new_time, progress_tasks)
def run(self):
print(description)
self.amazon.symlink_local_charms()
self.amazon.generate_config_from_template()
self.amazon.bootstrap(wait_started=True)
ensure = self.amazon.ensure_num_units
ensure(u'oscied-orchestra', u'oscied-orchestra', local=True, expose=True)
ensure(u'oscied-storage', u'oscied-storage', local=True)
ensure(u'oscied-transform', u'oscied-transform', local=True, to=2)
ensure(u'oscied-webui', u'oscied-webui', local=True, to=1, expose=True)
ensure(u'oscied-publisher', u'oscied-publisher', local=True, to=2, expose=True)
ensure(u'haproxy', u'haproxy', expose=True, local=False, release=u'trusty', required=False)
for peer in (u'orchestra', u'webui', u'transform', u'publisher'):
self.amazon.add_relation(u'oscied-storage', u'oscied-{0}'.format(peer))
self.amazon.add_relation(u'oscied-orchestra:transform', u'oscied-transform:transform')
self.amazon.add_relation(u'oscied-orchestra:publisher', u'oscied-publisher:publisher')
self.amazon.add_relation(u'oscied-orchestra:api', u'oscied-webui:api')
if self.amazon.get_service(u'haproxy', fail=False):
if self.amazon.add_relation(u'haproxy', u'oscied-webui'):
self.amazon.unexpose_service(u'oscied-webui')
if confirm(u'Initialize orchestra'):
self.amazon.init_api(SCENARIO_PATH, flush=True)
def run(self, **kwargs):
u"""
Run the Benchmark One scenario.
Keyword arguments:
concurrency -- the number of concurrent worker per transformation unit (default 1)
overwrite_config -- overwrite previously generated configuration file (default False)
"""
# get configuration parameters
overwrite = kwargs.get('overwrite_config', False)
concurrency = kwargs.get('concurrency', 8)
benchmark = self.dev
# initialize environment configuration and bootstrap it
benchmark.symlink_local_charms()
benchmark.generate_config_from_template(overwrite=overwrite, concurrency=concurrency)
benchmark.bootstrap(wait_started=True)
# deploy juju units
if confirm(u'Deploy OSCIED units'):
benchmark.auto = False
ensure_num_units = partial(benchmark.ensure_num_units, constraints=C1_XLARGE, local=True)
ensure_num_units(u'oscied-orchestra', u'oscied-orchestra', expose=True, constraints=C1_MEDIUM)
ensure_num_units(u'oscied-storage', u'oscied-storage', num_units=STORAGE_TRANSFORM_UNITS)
transform_units = tuple(u'transform-{0}'.format(i) for i in xrange(STORAGE_TRANSFORM_UNITS))
for i,_ in enumerate(transform_units):
ensure_num_units(u'oscied-transform', u'oscied-transform', num_units=i+1, to=i+2)
# setup units relations (except orchestra-transform)
for peer in (u'oscied-orchestra', u'oscied-transform'):
benchmark.add_relation(u'oscied-storage', peer)
# benchmark.add_relation(u'oscied-orchestra:transform', u'oscied-transform:transform')
benchmark.auto = False
print(u'start units status monitoring')
history = paya.history.FileHistory(u'{0}/units-status.paya'.format(SCENARIO_PATH))
start_monitor(target=monitor_unit_status, args=[benchmark, history])
# wait for orchestra to be STARTED
time_start = datetime.now()
while True:
print(u'wait for orchestra to start, elapsed: {0}'.format((datetime.now() - time_start)))
time_zero = time.time()
units = benchmark.get_units(u'oscied-orchestra')
state = units['0'].get(u'agent-state', u'unknown')
if state in py_juju.STARTED_STATES: break
elif state in py_juju.ERROR_STATES: raise Exception(u'oscied-orchestra failed while starting')
else: time.sleep(max(0, 15 - (time.time() - time_zero)))
# initialize client API (add users and transform profiles)
if confirm(u'Initialize OSCIED API'):
benchmark.init_api(SCENARIO_PATH, flush=True, add_tasks=False, wait_started=True,
backup_medias_in_remote=False)
# setup missing units relations (orchestra-transform)
# we put the relation between orchestra and transform after we could successfully
# communicated with orchestra, in order to avoid any misfunction of the unit relation
benchmark.add_relation(u'oscied-orchestra:transform', u'oscied-transform:transform')
# TODO: read tasks config file
config = {
u'task_sets': [{
u'input': u'chsrf.mxf',
u'output': u'chsrf.mp4',
u'profile': u'Tablet 480p/25',
u'metadata': {u'title': u'task-mxf-mp4'},
u'count': 50
}]
}
# get client API object
api_client = benchmark.api_client
api_client.login('*****@*****.**', 'passw0rd')
if confirm(u'revoke previous tasks'):
for task in api_client.transform_tasks.list():
try:
del api_client.transform_tasks[task._id]
except Exception as e:
print(repr(e))
if confirm(u'send task sets to the API'):
scheduled_tasks = []
for ts in config['task_sets']:
scheduled_tasks += send_task_set(api_client, ts)
else: exit(0)
print(u'start tasks status monitoring')
history = paya.history.FileHistory(u'{0}/task-status.paya'.format(SCENARIO_PATH))
start_monitor(target=monitor_task_status,
args=[api_client, [t._id for t in scheduled_tasks], history])
loop = len(scheduled_tasks) > 0
while loop:
print(u'wait for tasks completion')
states = {}
percent = 0.0
try:
for st in scheduled_tasks:
task = api_client.transform_tasks[st._id]
states[task.status] = states.get(task.status, 0) + 1
percent += task.statistic.get('percent', 0)
undef = task.status in TransformTask.UNDEF_STATUS
running = task.status in TransformTask.RUNNING_STATUS
pending = task.status in TransformTask.PENDING_STATUS
loop = running or pending or undef
print(u'\tstates: ' + u', '.join(['{0}: {1}'.format(k, v) for k,v in states.iteritems()]))
print(u'\tprogress: ' + str(percent / len(scheduled_tasks)) + '%')
time.sleep(10)
except Exception as e: # except (ConnectionError, Timeout) as e:
print(u'WARNING! Communication error, details: {1}.'.format(e))
print(u'retrieve paya histories')
for unit_type in ['orchestra', 'storage', 'transform']:
units = benchmark.get_units(u'oscied-{0}'.format(unit_type))
for unit_no in units:
try:
src = u'oscied-{0}/{1}:/tmp/{0}.paya'.format(unit_type, unit_no)
dst = u'{0}/{1}-{2}.paya'.format(SCENARIO_PATH, unit_type, unit_no)
cmd(u'juju scp {0} {1}'.format(src, dst))
if not os.path.exists(dst):
print(u'failed to download {0}'.format(src))
except Exception as e:
print(u'failed to download history from oscied-{0}/{1}'.format(unit_type, unit_no))
#print(client.transform_units.add(num_units=2))
#david = User(first_name='David', last_name='Fischer', mail='*****@*****.**', secret='oscied3D1', admin_platform=True)
#print(client.users.add(david))
#client.transform_units.add(1)
#client.transform_units.remove(1)
#print(client.users.list()[0].first_name)
#print(david._id in client.users)
#print(str(uuid.uuid4()) in client.users)
#print(client.user[str(uuid.uuid4())])
#del client.user[str(uuid.uuid4())]
#import copy
#david2 = copy.copy(david)
#client.users[client.auth._id] = david
#david2.mail = '*****@*****.**'
#david2.first_name = 'David 2nd'
#client.users.add(david2)
#client.medias.add(Media(user_id=david._id, uri=None, public_uris=None, filename='test.mp4', metadata={'title': 'test import'}))
#del client.environments['amazon']
#media = client.medias.list()[0]
#del client.medias[media._id]
#del client.transform_profiles[client.transform_profiles.list()[0]._id]
#client.transform_profiles.add(TransformProfile(title='salut', description='yo', encoder_name='copy'))
if confirm(u'Live-test the orchestrator'):
test_api(client)
