def run(self):
from pytoolbox import encoding, filesystem
from pytoolbox.subprocess import cmd
encoding.configure_unicode()
docs_directory = os.path.join(os.path.dirname(__file__), 'docs')
source_directory = os.path.join(docs_directory, 'source')
package_directory = os.path.join(os.path.dirname(__file__),
'pytoolbox')
# Cleanup previously generated restructured files
for path in filesystem.find_recursive(source_directory,
r'^pytoolbox.*\.rst$',
unix_wildcards=False):
os.remove(path)
cmd([
'sphinx-apidoc', '--force', '--module-first', '--separate', '-o',
source_directory, package_directory
])
shutil.rmtree(os.path.join(docs_directory, 'build', 'html'),
ignore_errors=True)
result = cmd('make html', cwd=docs_directory, fail=False)
print(
'{0}Outputs{0}======={0}{1}{0}{0}Errors{0}======{0}{2}{0}'.format(
os.linesep, result['stdout'].decode('utf-8'),
result['stderr'].decode('utf-8')))
sys.exit(1 if result['stderr'] else 0)
def test_retry_missing_binary_no_retry(self):
log = Mock()
with self.assertRaises(OSError):
cmd('hfuejnvwqkdivengz', log=log, tries=5)
validate_list(log.call_args_list, [
r"call\(u*'Execute hfuejnvwqkdivengz'\)",
r"call\(FileNotFoundError.*\)" if six.PY3 else r"call\(OSError.*\)"
])
def test_retry_missing_binary_no_retry():
log = mock.Mock()
with pytest.raises(OSError):
subprocess.cmd('hfuejnvwqkdivengz', log=log, tries=5)
validate_list(log.call_args_list, [
r"call\(u*'Execute hfuejnvwqkdivengz'\)",
r'call\(FileNotFoundError.*\)'
])
def test_retry_missing_binary_no_retry(self):
log = mock.Mock()
with self.raises(OSError):
cmd('hfuejnvwqkdivengz', log=log, tries=5)
validate_list(log.call_args_list, [
r"call\(u*'Execute hfuejnvwqkdivengz'\)",
r"call\(FileNotFoundError.*\)" if six.PY3 else r"call\(OSError.*\)"
])
def test_cmd(self):
cmd_log = mock_cmd()
cmd([u'echo', u'it seem to work'], log=cmd_log)
assert_equal(cmd(u'cat missing_file', fail=False, log=cmd_log)[u'returncode'], 1)
validate_list(cmd_log.call_args_list, [
r"call\(u*\"Execute \[u*'echo', u*'it seem to work'\]\"\)",
r"call\(u*'Execute cat missing_file'\)"])
assert(cmd(u'my.funny.missing.script.sh', fail=False)[u'stderr'] != u'')
result = cmd(u'cat {0}'.format(__file__))
# There are at least 30 lines in this source file !
assert(len(result[u'stdout'].splitlines()) > 30)
def test_retry_no_success(self):
log = Mock()
cmd('ls hfuejnvwqkdivengz', log=log, fail=False, tries=5, delay_min=0.0, delay_max=0.95)
validate_list(log.call_args_list, [
r"call\(u*'Execute ls hfuejnvwqkdivengz'\)",
r"call\(u*'Attempt 1 out of 5: Will retry in 0\.[0-9]+ seconds'\)",
r"call\(u*'Attempt 2 out of 5: Will retry in 0\.[0-9]+ seconds'\)",
r"call\(u*'Attempt 3 out of 5: Will retry in 0\.[0-9]+ seconds'\)",
r"call\(u*'Attempt 4 out of 5: Will retry in 0\.[0-9]+ seconds'\)",
r"call\(u*'Attempt 5 out of 5: Failed'\)"
])
def test_retry_no_success(self):
log = mock.Mock()
cmd('ls hfuejnvwqkdivengz', log=log, fail=False, tries=5, delay_min=0.0, delay_max=0.95)
validate_list(log.call_args_list, [
r"call\(u*'Execute ls hfuejnvwqkdivengz'\)",
r"call\(u*'Attempt 1 out of 5: Will retry in 0\.[0-9]+ seconds'\)",
r"call\(u*'Attempt 2 out of 5: Will retry in 0\.[0-9]+ seconds'\)",
r"call\(u*'Attempt 3 out of 5: Will retry in 0\.[0-9]+ seconds'\)",
r"call\(u*'Attempt 4 out of 5: Will retry in 0\.[0-9]+ seconds'\)",
r"call\(u*'Attempt 5 out of 5: Failed'\)"
])
def test_cmd(self):
log = mock.Mock()
cmd(['echo', 'it seem to work'], log=log)
self.equal(cmd('cat missing_file', fail=False, log=log)['returncode'], 1)
validate_list(log.call_args_list, [
r"call\(u*'Execute echo it seem to work'\)",
r"call\(u*'Execute cat missing_file'\)",
r"call\(u*'Attempt 1 out of 1: Failed'\)"
])
self.not_equal(cmd('my.funny.missing.script.sh', fail=False)['stderr'], '')
result = cmd('cat {0}'.format(__file__))
# There are at least 30 lines in this source file !
self.greater(len(result['stdout'].splitlines()), 30)
def test_cmd(self):
log = Mock()
cmd(['echo', 'it seem to work'], log=log)
self.assertEqual(cmd('cat missing_file', fail=False, log=log)['returncode'], 1)
validate_list(log.call_args_list, [
r"call\(u*'Execute echo it seem to work'\)",
r"call\(u*'Execute cat missing_file'\)",
r"call\(u*'Attempt 1 out of 1: Failed'\)"
])
self.assertNotEqual(cmd('my.funny.missing.script.sh', fail=False)['stderr'], '')
result = cmd('cat {0}'.format(__file__))
# There are at least 30 lines in this source file !
self.assertGreater(len(result['stdout'].splitlines()), 30)
def test_cmd():
log = mock.Mock()
subprocess.cmd(['echo', 'it seem to work'], log=log)
assert subprocess.cmd('cat missing_file', fail=False,
log=log)['returncode'] == 1
validate_list(log.call_args_list, [
r"call\(u*'Execute echo it seem to work'\)",
r"call\(u*'Execute cat missing_file'\)",
r"call\(u*'Attempt 1 out of 1: Failed'\)"
])
assert subprocess.cmd('my.funny.missing.script.sh',
fail=False)['stderr'] != ''
result = subprocess.cmd(f'cat {__file__}')
# There are at least 30 lines in this source file !
assert len(result['stdout'].splitlines()) > 30
def run(self):
from pytoolbox import encoding, filesystem
from pytoolbox.subprocess import cmd
encoding.configure_unicode()
docs_directory = os.path.join(os.path.dirname(__file__), 'docs')
source_directory = os.path.join(docs_directory, 'source')
package_directory = os.path.join(os.path.dirname(__file__), 'pytoolbox')
# Cleanup previously generated restructured files
for path in filesystem.find_recursive(
source_directory, r'^pytoolbox.*\.rst$', unix_wildcards=False
):
os.remove(path)
cmd([
'sphinx-apidoc', '--force', '--module-first', '--separate', '-o',
source_directory, package_directory
])
shutil.rmtree(os.path.join(docs_directory, 'build', 'html'), ignore_errors=True)
subprocess.check_call(['make', 'html'], cwd=docs_directory)
def run(self):
from pytoolbox import encoding, filesystem
from pytoolbox.subprocess import cmd
encoding.configure_unicode()
docs_directory = os.path.join(os.path.dirname(__file__), 'docs')
source_directory = os.path.join(docs_directory, 'source')
package_directory = os.path.join(os.path.dirname(__file__), 'pytoolbox')
# Cleanup previously generated restructured files
for path in filesystem.find_recursive(source_directory, r'^pytoolbox.*\.rst$', unix_wildcards=False):
os.remove(path)
cmd(['sphinx-apidoc', '--force', '--module-first', '--separate', '-o', source_directory, package_directory])
shutil.rmtree(os.path.join(docs_directory, 'build', 'html'), ignore_errors=True)
result = cmd('make html', cwd=docs_directory, fail=False)
print('{0}Outputs{0}======={0}{1}{0}{0}Errors{0}======{0}{2}{0}'.format(
os.linesep, result['stdout'].decode('utf-8'), result['stderr'].decode('utf-8'))
)
sys.exit(1 if result['stderr'] else 0)
def run(self):
from pytoolbox import encoding, filesystem
from pytoolbox.subprocess import cmd
encoding.configure_unicode()
docs_directory = os.path.join(os.path.dirname(__file__), 'docs')
source_directory = os.path.join(docs_directory, 'source')
package_directory = os.path.join(os.path.dirname(__file__),
'pytoolbox')
# Cleanup previously generated restructured files
for path in filesystem.find_recursive(source_directory,
r'^pytoolbox.*\.rst$',
unix_wildcards=False):
os.remove(path)
cmd([
'sphinx-apidoc', '--force', '--module-first', '--separate', '-o',
source_directory, package_directory
])
shutil.rmtree(os.path.join(docs_directory, 'build', 'html'),
ignore_errors=True)
subprocess.check_call(['make', 'html'], cwd=docs_directory)
def test_retry_first_try(self):
log = mock.Mock()
cmd('ls', log=log, tries=5, delay_min=1, delay_max=1)
validate_list(log.call_args_list, [r"call\(u*'Execute ls'\)"])
def test_cmd_missing_binary(self):
self.equal(cmd('hfuejnvwqkdivengz', fail=False)['returncode'], 2)
def test_cmd_missing_binary():
assert subprocess.cmd('hfuejnvwqkdivengz', fail=False)['returncode'] == 2
# 22-07-2013, <https://joinup.ec.europa.eu/software/page/eupl/licence-eupl>
#
# Retrieved from https://github.com/davidfischer-ch/pytoolbox.git
from __future__ import absolute_import, division, print_function, unicode_literals
import shutil, sys
from pytoolbox.encoding import configure_unicode
from pytoolbox.subprocess import cmd
from pytoolbox.filesystem import from_template
configure_unicode()
# Detect modules, thanks to find !
modules = sorted(m.replace('.py', '').replace('./', '').replace('/', '.')
for m in cmd('find . -type f -name "*.py"', cwd='../pytoolbox', shell=True)['stdout'].split()
if m.endswith('.py') and not '__init__' in m)
print('Detected modules are: {0}'.format(modules))
api_toc = ''
for module in modules:
if 'django' in module or 'crypto' in module:
continue # FIXME temporary hack, see issue #6
module = 'pytoolbox.{0}'.format(module)
title = module.replace('.', ' > ')
api_toc += ' {0}\n'.format(module)
from_template('templates/module.rst.template', 'source/{0}.rst'.format(module),
{'module': module, 'title': title, 'equals': '='*len(title)})
from_template('templates/api.rst.template', 'source/api.rst', {'api_toc': api_toc})
parser = ArgumentParser(
formatter_class=ArgumentDefaultsHelpFormatter,
epilog="""Generate OSCIED project documentation from source reStructuredText files.""",
)
parser.add_argument("--html", help=HELP_HTML, action="store_true")
parser.add_argument("--pdf", help=HELP_PDF, action="store_true")
# parser.add_argument(u'--wiki', help=HELP_WIKI, action=u'store_true')
args = parser.parse_args()
if not args.html and not args.pdf and not args.wiki:
parser.print_help()
print("")
print_error("At least one target must be enabled")
print("Install prerequisites")
cmd("sudo apt-get -y install " + " ".join(PACKAGES))
cmd("sudo pip install " + " ".join(PYTHON_PACKAGES))
cmd("sudo easy_install rednose") # NEVER install it with pip ;-)
os.chdir(TOOLS_PATH)
plantuml = "http://downloads.sourceforge.net/project/plantuml"
cmd("wget -N {0}/plantuml.jar".format(plantuml))
cmd("wget -N {0}/PlantUML%20Language%20Reference%20Guide.pdf".format(plantuml))
print("Fixes bitbucket.org/birkenfeld/sphinx/pull-request/98/fixes-typeerror-raised-from/diff")
for path in site.getsitepackages():
file_to_fix = join(path, "sphinx/writers/latex.py")
if exists(file_to_fix):
cmd(
'sudo sed -i "s:letter.translate(tex_escape_map)):unicode(letter).translate(tex_escape_map)):g" {0}'.format(
file_to_fix
)
def run(self, **kwargs):
u"""
Run the Benchmark One scenario.
Keyword arguments:
concurrency -- the number of concurrent worker per transformation unit (default 1)
overwrite_config -- overwrite previously generated configuration file (default False)
"""
# get configuration parameters
overwrite = kwargs.get('overwrite_config', False)
concurrency = kwargs.get('concurrency', 8)
benchmark = self.dev
# initialize environment configuration and bootstrap it
benchmark.symlink_local_charms()
benchmark.generate_config_from_template(overwrite=overwrite, concurrency=concurrency)
benchmark.bootstrap(wait_started=True)
# deploy juju units
if confirm(u'Deploy OSCIED units'):
benchmark.auto = False
ensure_num_units = partial(benchmark.ensure_num_units, constraints=C1_XLARGE, local=True)
ensure_num_units(u'oscied-orchestra', u'oscied-orchestra', expose=True, constraints=C1_MEDIUM)
ensure_num_units(u'oscied-storage', u'oscied-storage', num_units=STORAGE_TRANSFORM_UNITS)
transform_units = tuple(u'transform-{0}'.format(i) for i in xrange(STORAGE_TRANSFORM_UNITS))
for i,_ in enumerate(transform_units):
ensure_num_units(u'oscied-transform', u'oscied-transform', num_units=i+1, to=i+2)
# setup units relations (except orchestra-transform)
for peer in (u'oscied-orchestra', u'oscied-transform'):
benchmark.add_relation(u'oscied-storage', peer)
# benchmark.add_relation(u'oscied-orchestra:transform', u'oscied-transform:transform')
benchmark.auto = False
print(u'start units status monitoring')
history = paya.history.FileHistory(u'{0}/units-status.paya'.format(SCENARIO_PATH))
start_monitor(target=monitor_unit_status, args=[benchmark, history])
# wait for orchestra to be STARTED
time_start = datetime.now()
while True:
print(u'wait for orchestra to start, elapsed: {0}'.format((datetime.now() - time_start)))
time_zero = time.time()
units = benchmark.get_units(u'oscied-orchestra')
state = units['0'].get(u'agent-state', u'unknown')
if state in py_juju.STARTED_STATES: break
elif state in py_juju.ERROR_STATES: raise Exception(u'oscied-orchestra failed while starting')
else: time.sleep(max(0, 15 - (time.time() - time_zero)))
# initialize client API (add users and transform profiles)
if confirm(u'Initialize OSCIED API'):
benchmark.init_api(SCENARIO_PATH, flush=True, add_tasks=False, wait_started=True,
backup_medias_in_remote=False)
# setup missing units relations (orchestra-transform)
# we put the relation between orchestra and transform after we could successfully
# communicated with orchestra, in order to avoid any misfunction of the unit relation
benchmark.add_relation(u'oscied-orchestra:transform', u'oscied-transform:transform')
# TODO: read tasks config file
config = {
u'task_sets': [{
u'input': u'chsrf.mxf',
u'output': u'chsrf.mp4',
u'profile': u'Tablet 480p/25',
u'metadata': {u'title': u'task-mxf-mp4'},
u'count': 50
}]
}
# get client API object
api_client = benchmark.api_client
api_client.login('*****@*****.**', 'passw0rd')
if confirm(u'revoke previous tasks'):
for task in api_client.transform_tasks.list():
try:
del api_client.transform_tasks[task._id]
except Exception as e:
print(repr(e))
if confirm(u'send task sets to the API'):
scheduled_tasks = []
for ts in config['task_sets']:
scheduled_tasks += send_task_set(api_client, ts)
else: exit(0)
print(u'start tasks status monitoring')
history = paya.history.FileHistory(u'{0}/task-status.paya'.format(SCENARIO_PATH))
start_monitor(target=monitor_task_status,
args=[api_client, [t._id for t in scheduled_tasks], history])
loop = len(scheduled_tasks) > 0
while loop:
print(u'wait for tasks completion')
states = {}
percent = 0.0
try:
for st in scheduled_tasks:
task = api_client.transform_tasks[st._id]
states[task.status] = states.get(task.status, 0) + 1
percent += task.statistic.get('percent', 0)
undef = task.status in TransformTask.UNDEF_STATUS
running = task.status in TransformTask.RUNNING_STATUS
pending = task.status in TransformTask.PENDING_STATUS
loop = running or pending or undef
print(u'\tstates: ' + u', '.join(['{0}: {1}'.format(k, v) for k,v in states.iteritems()]))
print(u'\tprogress: ' + str(percent / len(scheduled_tasks)) + '%')
time.sleep(10)
except Exception as e: # except (ConnectionError, Timeout) as e:
print(u'WARNING! Communication error, details: {1}.'.format(e))
print(u'retrieve paya histories')
for unit_type in ['orchestra', 'storage', 'transform']:
units = benchmark.get_units(u'oscied-{0}'.format(unit_type))
for unit_no in units:
try:
src = u'oscied-{0}/{1}:/tmp/{0}.paya'.format(unit_type, unit_no)
dst = u'{0}/{1}-{2}.paya'.format(SCENARIO_PATH, unit_type, unit_no)
cmd(u'juju scp {0} {1}'.format(src, dst))
if not os.path.exists(dst):
print(u'failed to download {0}'.format(src))
except Exception as e:
print(u'failed to download history from oscied-{0}/{1}'.format(unit_type, unit_no))
def test_retry_first_try(self):
log = Mock()
cmd('ls', log=log, tries=5, delay_min=1, delay_max=1)
validate_list(log.call_args_list, [
r"call\(u*'Execute ls'\)"
])
def test_cmd_missing_binary(self):
self.assertEqual(cmd('hfuejnvwqkdivengz', fail=False)['returncode'], 2)
parser = ArgumentParser(
formatter_class=ArgumentDefaultsHelpFormatter,
epilog=u'''Generate OSCIED project documentation from source reStructuredText files.''')
parser.add_argument(u'--html', help=HELP_HTML, action=u'store_true')
parser.add_argument(u'--pdf', help=HELP_PDF, action=u'store_true')
#parser.add_argument(u'--wiki', help=HELP_WIKI, action=u'store_true')
args = parser.parse_args()
if not args.html and not args.pdf and not args.wiki:
parser.print_help()
print(u'')
print_error(u'At least one target must be enabled')
print(u'Install prerequisites')
cmd(u'sudo apt-get -y install ' + u' '.join(PACKAGES))
cmd(u'sudo pip install ' + u' '.join(PYTHON_PACKAGES))
cmd(u'sudo easy_install rednose') # NEVER install it with pip ;-)
os.chdir(TOOLS_PATH)
plantuml = u'http://downloads.sourceforge.net/project/plantuml'
cmd(u'wget -N {0}/plantuml.jar'.format(plantuml))
cmd(u'wget -N {0}/PlantUML%20Language%20Reference%20Guide.pdf'.format(plantuml))
print(u'Fixes bitbucket.org/birkenfeld/sphinx/pull-request/98/fixes-typeerror-raised-from/diff')
for path in site.getsitepackages():
file_to_fix = join(path, 'sphinx/writers/latex.py')
if exists(file_to_fix):
cmd('sudo sed -i "s:letter.translate(tex_escape_map)):unicode(letter).translate(tex_escape_map)):g" {0}'.
format(file_to_fix))
print(u'Fixed source file {0}'.format(file_to_fix))
#
# Retrieved from https://github.com/davidfischer-ch/pytoolbox.git
from __future__ import absolute_import, division, print_function, unicode_literals
import shutil, sys
from pytoolbox.encoding import configure_unicode
from pytoolbox.subprocess import cmd
from pytoolbox.filesystem import from_template
configure_unicode()
# Detect modules, thanks to find !
modules = sorted(
m.replace('.py', '').replace('./', '').replace('/', '.')
for m in cmd('find . -type f -name "*.py"', cwd='../pytoolbox', shell=True)
['stdout'].split() if m.endswith('.py') and not '__init__' in m)
print('Detected modules are: {0}'.format(modules))
api_toc = ''
for module in modules:
if 'django' in module or 'crypto' in module:
continue # FIXME temporary hack, see issue #6
module = 'pytoolbox.{0}'.format(module)
title = module.replace('.', ' > ')
api_toc += ' {0}\n'.format(module)
from_template('templates/module.rst.template',
'source/{0}.rst'.format(module), {
'module': module,
'title': title,
'equals': '=' * len(title)
import os, threading, time
from functools import partial
from library.oscied_lib.models import Media, TransformTask
from pytoolbox import juju as py_juju
from pytoolbox.console import confirm
from pytoolbox.juju import DeploymentScenario, C1_MEDIUM
from pytoolbox.subprocess import cmd
from datetime import datetime
try:
import paya.history
except:
cmd(u'pip install git+git://github.com/kyouko-taiga/paya.git#egg=paya')
import paya.history
SCENARIO_PATH = os.path.dirname(__file__)
CONFIG = os.path.join(SCENARIO_PATH, u'config.yaml')
STORAGE_TRANSFORM_UNITS = 2
C1_XLARGE = u'arch=amd64 cpu-cores=8 cpu-power=2000 mem=6G'
def start_monitor(target, args=[], daemon=True):
t = threading.Thread(target=target, args=args)
t.daemon = daemon
t.start()
def monitor_unit_status(environment, history, interval=15):
while True:
