def test_parallel_execute_with_upstream_errors():
log = []
def process(x):
if x is data_volume:
raise APIError(None, None, "Something went wrong")
log.append(x)
parallel_execute(
objects=objects,
func=process,
get_name=lambda obj: obj,
msg="Processing",
get_deps=get_deps,
)
assert log == [cache]
events = [(obj, result, type(exception))
for obj, result, exception in parallel_execute_iter(
objects, process, get_deps, None)]
assert (cache, None, type(None)) in events
assert (data_volume, None, APIError) in events
assert (db, None, UpstreamError) in events
assert (web, None, UpstreamError) in events
def test_parallel_execute_with_upstream_errors():
log = []
def process(x):
if x is data_volume:
raise APIError(None, None, "Something went wrong")
log.append(x)
parallel_execute(
objects=objects,
func=process,
get_name=lambda obj: obj,
msg="Processing",
get_deps=get_deps,
)
assert log == [cache]
events = [
(obj, result, type(exception))
for obj, result, exception
in parallel_execute_stream(objects, process, get_deps)
]
assert (cache, None, type(None)) in events
assert (data_volume, None, APIError) in events
assert (db, None, UpstreamError) in events
assert (web, None, UpstreamError) in events
def test_parallel_execute_with_deps():
log = []
def process(x):
log.append(x)
parallel_execute(objects=objects, func=process, get_name=lambda obj: obj, msg="Processing", get_deps=get_deps)
assert sorted(log) == sorted(objects)
assert log.index(data_volume) < log.index(db)
assert log.index(db) < log.index(web)
assert log.index(cache) < log.index(web)
def test_parallel_execute():
results, errors = parallel_execute(
objects=[1, 2, 3, 4, 5], func=lambda x: x * 2, get_name=six.text_type, msg="Doubling"
)
assert sorted(results) == [2, 4, 6, 8, 10]
assert errors == {}
def run(self):
if not self.options['--repo']:
log.error(
'--repo is required and must be a docker repository base url')
self.docker_registry_base = self.options['--repo']
services = []
for service in self.project.get_services_without_duplicate(
service_names=self.options['SERVICES']):
if service.can_be_built():
services.append(service)
else:
log.info('%s uses an image, skipping' % service.name)
def process_service(service):
self.process_service(service)
if not self.options['--disable-parallel']:
_, errors = parallel.parallel_execute(
services,
process_service,
operator.attrgetter('name'),
'Syncing',
limit=5,
)
if len(errors):
combined_errors = '\n'.join([
e.decode('utf-8') if isinstance(e, six.binary_type) else e
for e in errors.values()
])
raise ProjectError(combined_errors)
else:
for service in services:
process_service(service)
def test_parallel_execute():
results = parallel_execute(
objects=[1, 2, 3, 4, 5],
func=lambda x: x * 2,
get_name=six.text_type,
msg="Doubling",
)
assert sorted(results) == [2, 4, 6, 8, 10]
def test_parallel_execute_with_deps():
log = []
def process(x):
log.append(x)
parallel_execute(
objects=objects,
func=process,
get_name=lambda obj: obj,
msg="Processing",
get_deps=get_deps,
)
assert sorted(log) == sorted(objects)
assert log.index(data_volume) < log.index(db)
assert log.index(db) < log.index(web)
assert log.index(cache) < log.index(web)
def test_parallel_execute(self):
results, errors = parallel_execute(
objects=[1, 2, 3, 4, 5],
func=lambda x: x * 2,
get_name=str,
msg="Doubling",
)
assert sorted(results) == [2, 4, 6, 8, 10]
assert errors == {}
def test_parallel_execute_alignment(capsys):
results, errors = parallel_execute(
objects=["short", "a very long name"],
func=lambda x: x,
get_name=six.text_type,
msg="Aligning",
)
assert errors == {}
_, err = capsys.readouterr()
a, b = err.split('\n')[:2]
assert a.index('...') == b.index('...')
def test_parallel_execute_alignment(capsys):
results, errors = parallel_execute(
objects=["short", "a very long name"],
func=lambda x: x,
get_name=six.text_type,
msg="Aligning",
)
assert errors == {}
_, err = capsys.readouterr()
a, b = err.split('\n')[:2]
assert a.index('...') == b.index('...')
def test_parallel_execute_noansi(capsys):
ParallelStreamWriter.set_noansi()
results, errors = parallel_execute(
objects=["something", "something more"],
func=lambda x: x,
get_name=six.text_type,
msg="Control characters",
)
assert errors == {}
_, err = capsys.readouterr()
assert "\x1b" not in err
def up(self,
service_names=None,
start_deps=True,
strategy=ConvergenceStrategy.changed,
do_build=BuildAction.none,
timeout=DEFAULT_TIMEOUT,
detached=False,
remove_orphans=False):
warn_for_swarm_mode(self.project.client)
self.project.initialize()
self.project.find_orphan_containers(remove_orphans)
services = self.project.get_services_without_duplicate(
service_names,
include_deps=start_deps)
for svc in services:
svc.ensure_image_exists(do_build=do_build)
plans = self.project._get_convergence_plans(services, strategy)
def do(service):
return service.execute_convergence_plan(
plans[service.name],
timeout=timeout,
detached=detached
)
def get_deps(service):
return {self.project.get_service(dep) for dep in service.get_dependency_names()}
results, errors = parallel.parallel_execute(
services,
do,
operator.attrgetter('name'),
None,
get_deps
)
if errors:
raise ProjectError(
'Encountered errors while bringing up the project.', errors
)
return [
container
for svc_containers in results
if svc_containers is not None
for container in svc_containers
]
def test_parallel_execute_ansi(capsys):
ParallelStreamWriter.instance = None
ParallelStreamWriter.set_noansi(value=False)
results, errors = parallel_execute(
objects=["something", "something more"],
func=lambda x: x,
get_name=str,
msg="Control characters",
)
assert errors == {}
_, err = capsys.readouterr()
assert "\x1b" in err
def test_parallel_execute_with_limit():
limit = 1
tasks = 20
lock = Lock()
def f(obj):
locked = lock.acquire(False)
# we should always get the lock because we're the only thread running
assert locked
lock.release()
return None
results, errors = parallel_execute(
objects=list(range(tasks)),
func=f,
get_name=six.text_type,
msg="Testing",
limit=limit,
)
assert results == tasks * [None]
assert errors == {}
def test_parallel_execute_with_limit():
limit = 1
tasks = 20
lock = Lock()
def f(obj):
locked = lock.acquire(False)
# we should always get the lock because we're the only thread running
assert locked
lock.release()
return None
results, errors = parallel_execute(
objects=list(range(tasks)),
func=f,
get_name=six.text_type,
msg="Testing",
limit=limit,
)
assert results == tasks * [None]
assert errors == {}
def test_parallel_execute_with_global_limit(self):
GlobalLimit.set_global_limit(1)
self.addCleanup(GlobalLimit.set_global_limit, None)
tasks = 20
lock = Lock()
def f(obj):
locked = lock.acquire(False)
# we should always get the lock because we're the only thread running
assert locked
lock.release()
return None
results, errors = parallel_execute(
objects=list(range(tasks)),
func=f,
get_name=str,
msg="Testing",
)
assert results == tasks * [None]
assert errors == {}
