def ClusterConstructor(**kwargs):
log = logging.getLogger(__file__)
log.setLevel(logging.DEBUG)
log.handlers = [logging.StreamHandler(sys.stdout)]
kwargs['log'] = log
kwargs.setdefault("controller", controller_launcher_class)
kwargs.setdefault("engines", engine_launcher_class)
cfg = kwargs.setdefault("config", cluster_config)
cfg.EngineLauncher.engine_args = ['--log-level=10']
cfg.ControllerLauncher.controller_args = ['--log-level=10']
kwargs.setdefault("controller_args", ['--ping=250'])
kwargs.setdefault("load_profile", False)
c = ipp.Cluster(**kwargs)
if not kwargs['load_profile']:
assert c.config == cfg
request.addfinalizer(c.stop_cluster_sync)
return c
def test_ipcluster_clean(ipython_dir):
default_profile = ProfileDir.find_profile_dir_by_name(ipython_dir)
default_profile_dir = default_profile.location
log_file = os.path.join(default_profile.log_dir, "test.log")
with open(log_file, "w") as f:
f.write("logsssss")
cluster_file = os.path.join(default_profile.security_dir,
"cluster-abc.json")
c = ipp.Cluster()
with open(cluster_file, 'w') as f:
json.dump(c.to_dict(), f)
connection_file = os.path.join(default_profile.security_dir,
"ipcontroller-client.json")
with open(connection_file, 'w') as f:
f.write("{}")
check_call([sys.executable, "-m", "ipyparallel.cluster", "clean"])
assert not os.path.exists(log_file)
assert not os.path.exists(cluster_file)
assert not os.path.exists(connection_file)
def __init__(self, view=None, **kwargs):
super().__init__(**kwargs)
self._cluster_owner = False
self._client_owner = False
if view is None:
self._client_owner = True
try:
# load the default cluster
cluster = ipp.Cluster.from_file()
except FileNotFoundError:
# other load errors?
cluster = self._cluster = ipp.Cluster()
self._cluster_owner = True
cluster.start_cluster_sync()
else:
# cluster running, ensure some engines are, too
if not cluster.engines:
cluster.start_engines_sync()
rc = cluster.connect_client_sync()
rc.wait_for_engines(cluster.n or 1)
view = rc.load_balanced_view()
# use cloudpickle or dill for closures, if available.
# joblib tends to create closures default pickle can't handle.
try:
import cloudpickle # noqa
except ImportError:
try:
import dill # noqa
except ImportError:
pass
else:
view.client[:].use_dill()
else:
view.client[:].use_cloudpickle()
self._view = view
#! /usr/bin/env python3
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2021 <@localhost>
#
# Distributed under terms of the MIT license.
import time
import ipyparallel as ipp
task_durations = [1] * 25
# request a cluster
with ipp.Cluster() as rc:
# get a view on the cluster
view = rc.load_balanced_view()
# submit the tasks
asyncresult = view.map_async(time.sleep, task_durations)
# wait interactively for results
asyncresult.wait_interactive()
# retrieve actual results
result = asyncresult.get()
# at this point, the cluster processes have been shutdown
def test_ipcluster_start_stop(request, ipython_dir, daemonize):
default_profile = ProfileDir.find_profile_dir_by_name(ipython_dir)
default_profile_dir = default_profile.location
# cleanup the security directory to avoid leaking files from one test to the next
def cleanup_security():
for f in glob.glob(os.path.join(default_profile.security_dir,
"*.json")):
print(f"Cleaning up {f}")
try:
os.remove(f)
except Exception as e:
print(f"Error removing {f}: {e}")
request.addfinalizer(cleanup_security)
n = 2
start_args = ["-n", str(n)]
if daemonize:
start_args.append("--daemonize")
start = Popen(
[sys.executable, "-m", "ipyparallel.cluster", "start", "--debug"] +
start_args)
request.addfinalizer(start.terminate)
if daemonize:
# if daemonize, should exit after starting
start.wait(30)
else:
# wait for file to appear
# only need this if not daemonize
cluster_file = ipp.Cluster(profile_dir=default_profile_dir,
cluster_id="").cluster_file
for i in range(100):
if os.path.isfile(cluster_file) or start.poll() is not None:
break
else:
time.sleep(0.1)
assert os.path.isfile(cluster_file)
# list should show a file
out = ipcluster_list()
assert len(out.splitlines()) == 2
# cluster running, try to connect with default args
cluster = ipp.Cluster.from_file(log_level=10)
try:
with cluster.connect_client_sync() as rc:
rc.wait_for_engines(n=2, timeout=60)
rc[:].apply_async(os.getpid).get(timeout=10)
except Exception:
print("controller output")
print(cluster.controller.get_output())
print("engine output")
for engine_set in cluster.engines.values():
print(engine_set.get_output())
raise
# stop with ipcluster stop
check_call([sys.executable, "-m", "ipyparallel.cluster", "stop"])
# start should exit when this happens
start.wait(30)
# and ipcluster list should return empty
out = ipcluster_list()
assert len(out.splitlines()) == 1
# stop all succeeds even if there's nothing to stop
check_call([sys.executable, "-m", "ipyparallel.cluster", "stop", "--all"])
#! /usr/bin/env python3
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2021 <@localhost>
#
# Distributed under terms of the MIT license.
import ipyparallel as ipp
# start cluster, connect client
with ipp.Cluster(n=4) as rc:
print(rc)
e_all = rc[:]
print(e_all)
ar = e_all.apply(lambda x, y: x**2 + y**2, 5, 12)
print(ar)
rc.wait_interactive()
results = ar.get()
print(results)
# have results, cluster is shutdown
#!/usr/bin/env python import ipyparallel as ipp rc = ipp.Cluster(n=4).start_and_connect_sync() rc.wait_for_engines(n=4) print(rc.ids) print(rc[:].apply_sync(lambda: "Hello, World"))
comm = MPI.COMM_WORLD
size = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
steps = 100000
re = 1000
base_lenght = 100
uw = 0.1
relaxation = (2 * re) / (6 * base_lenght * uw + re)
process_info = slidingLidMPI.fill_mpi_struct_fields(
rank, size, 2, 2, base_lenght, relaxation, steps, uw)
# process_info.boundaries_info.apply_left = True
# process_info.boundaries_info.apply_right = True
# process_info.boundaries_info.apply_top = True
# process_info.boundaries_info.apply_bottom = True
slidingLidMPI.sliding_lid_mpi(process_info, comm)
return f"{process_info}"
# Main caller
# request an MPI cluster with 2 engines
with ipp.Cluster(engines='mpi', n=cores) as rc:
# get a broadcast_view on the cluster which is best
# print(rc.ids)
# suited for MPI style computation
view = rc.broadcast_view()
# run the mpi_example function on all engines in parallel
r = view.apply_sync(test_4_simulation)
# Retrieve and print the result from the engines
print("\n".join(r))
# at this point, the cluster processes have been shutdow