def connect_client(self):
"""connect a client with my Context, and track its sockets for cleanup"""
c = Client(profile='iptest', context=self.context)
c.wait = lambda *a, **kw: self.client_wait(c, *a, **kw)
for name in filter(lambda n:n.endswith('socket'), dir(c)):
s = getattr(c, name)
s.setsockopt(zmq.LINGER, 0)
self.sockets.append(s)
return c
def connect_client(self):
"""connect a client with my Context, and track its sockets for cleanup"""
c = Client(profile='iptest', context=self.context)
c.wait = lambda *a, **kw: self.client_wait(c, *a, **kw)
for name in filter(lambda n: n.endswith('socket'), dir(c)):
s = getattr(c, name)
s.setsockopt(zmq.LINGER, 0)
self.sockets.append(s)
return c
for strike in strike_vals:
for sigma in sigma_vals:
ar = view.apply_async(price_options, price, strike, sigma, rate, days, paths)
async_results.append(ar)
# <codecell>
print "Submitted tasks: ", len(async_results)
# <markdowncell>
# Block until all tasks are completed.
# <codecell>
c.wait(async_results)
t2 = time.time()
t = t2-t1
print "Parallel calculation completed, time = %s s" % t
# <markdowncell>
# ## Process and visualize results
# <markdowncell>
# Get the results using the `get` method:
# <codecell>
class Bakapara:
"""A "bakapara" client for IPython cluster.
Args: identical to IPython.parallel.Client
"""
def __init__(self, **args):
self.rc = Client(**args)
self.lview = None
self.ar = None
self.jobs = None
self.indices = None
self.finished = set()
# Obtain the host names and PIDs of engines.
self.pids = self.rc[:].apply(os.getpid).get_dict()
self.hosts = self.rc[:].apply(socket.getfqdn).get_dict()
# Change the working directory of each engine.
self.rc[:].apply(os.chdir, os.getcwd())
def run(self, jobs, targets=None):
"""Runs the jobs on the cluster.
Args:
jobs (list): list of dictionary objects describing the jobs.
targets (int, list of ints, 'all', or None): the engine(s) on which the jobs will run.
Returns:
bool: True if successful, False otherwise (e.g., jobs are running).
"""
if self.ar is not None and not self.ar.ready():
return False
self.lview = self.rc.load_balanced_view(targets)
self.ar = self.lview.map_async(runjob, jobs)
self.jobs = jobs
self.indices = dict([(k, v) for v, k in enumerate(self.ar.msg_ids)])
self.finished = set()
return True
def wait(self, timeout=1e-3):
"""Waits for the jobs to complete and writes job results.
Args:
timeout (float): a time in seconds, after which to give up.
"""
if self.ar is None:
return
# Find finished msg_ids.
pending = set(self.ar.msg_ids)
try:
self.rc.wait(pending, timeout)
except TimeoutError:
pass
finished = pending.difference(self.rc.outstanding)
# Overwrite the results in the job array.
for msg_id in finished:
i = self.indices[msg_id]
if i in self.finished:
continue
job = self.jobs[i]
meta = self.rc.metadata[msg_id]
result = self.rc.results[msg_id][0]
for key in ('submitted', 'started', 'completed', 'received'):
result[key] = meta[key].isoformat()
for key in ('engine_id', 'pyerr', 'pyout', 'status', 'msg_id'):
result[key] = meta[key]
result['elapsed'] = str(meta['completed'] - meta['started'])
result['host'] = self.hosts[meta['engine_id']]
job['result'] = result
self.finished.add(i)
def ready():
"""Returns whether the jobs have completed."""
return self.ar is not None or self.ar.ready()
def successful():
"""Returns whether the jobs completed without raising an exception.
Raises:
AssertionError: the result is not ready.
"""
return self.ar is not None and self.ar.successful()
def abort(self, **args):
"""Aborts jobs.
Args: identical to IPython.parallel.client.view.LoadBalancedView.abort()
"""
if self.lview is not None:
self.lview.abort()
def interrupt(self):
"""Sends SIGINT signal to engines (experimental).
http://mail.scipy.org/pipermail/ipython-dev/2014-March/013426.html
"""
self.abort()
for i in self.rc.ids:
host = self.hosts[i]
pid = self.pids[i]
if host == socket.getfqdn():
os.kill(pid, signal.SIGINT)
else:
os.system('ssh {} kill -INT {}'.format(host, pid))
def shutdown(self, **args):
"""Terminates one or more engine processes, optionally including the hub.
Args: identical to IPython.parallel.Client.shutdown
"""
if self.lview is None:
return False
self.lview.shutdown(**args)
def status(self, interval=1., timeout=-1, fo=sys.stdout):
"""Waits for the jobs, printing progress at regular intervals
Args:
interval (float): a time in seconds, after which to print the progress.
timeout (float): a time in seconds, after which to give up waiting.
fo (file): a file object to which the progress is printed.
"""
if self.ar is None:
return
if timeout is None:
timeout = -1
# Make sure to write the job results into the job objects.
self.wait(1e-3)
tic = time.time()
while not self.ar.ready() and (timeout < 0
or time.time() - tic <= timeout):
self.wait(interval)
clear_output(wait=True)
dt = datetime.timedelta(seconds=self.ar.elapsed)
fo.write('{}/{} tasks finished after {}'.format(
self.ar.progress, len(self.ar), str(dt)))
fo.flush()
else:
fo.write('\n')
dt = datetime.timedelta(seconds=self.ar.elapsed)
clear_output(wait=True)
fo.write('{} tasks completed in {}\n'.format(len(self.ar), str(dt)))
fo.flush()
def __len__(self):
"""Returns the number of engines."""
return len(self.rc)
for strike in strike_vals:
for sigma in sigma_vals:
ar = view.apply_async(price_options, price, strike, sigma, rate, days, paths)
async_results.append(ar)
# <codecell>
print("Submitted tasks: ", len(async_results))
# <markdowncell>
# Block until all tasks are completed.
# <codecell>
c.wait(async_results)
t2 = time.time()
t = t2-t1
print("Parallel calculation completed, time = %s s" % t)
# <markdowncell>
# ## Process and visualize results
# <markdowncell>
# Get the results using the `get` method:
# <codecell>
def client_wait(self, client, jobs=None, timeout=-1):
"""my wait wrapper, sets a default finite timeout to avoid hangs"""
if timeout < 0:
timeout = self.timeout
return Client.wait(client, jobs, timeout)
except (NoOptionError, exceptions.IOError) as e:
log.error("SKIP TEST: " + e.message)
if config.getboolean("run", "run_chdiff"):
print "CHDIFF"
for freq in freqs:
for surv in survs:
if paral:
tasks.append(lview.apply_async(chdiff,freq, ["LFI%d" % h for
h in
utils.HORNS[freq]],
surv, pol='I',
smooth_combine_config=smooth_combine_config,
root_folder=root_folder,
log_to_file=True,
mapreader=mapreader))
else:
try:
chdiff(freq, ["LFI%d" % h for h in utils.HORNS[freq]], surv,
pol='I', smooth_combine_config=smooth_combine_config,
root_folder=root_folder,
log_to_file=False,
mapreader=mapreader)
except (NoOptionError, exceptions.IOError) as e:
log.error("SKIP TEST: " + e.message)
if paral:
print("Wait for %d tasks to complete" % len(tasks))
tc.wait(tasks)
class Bakapara:
"""A "bakapara" client for IPython cluster.
Args: identical to IPython.parallel.Client
"""
def __init__(self, **args):
self.rc = Client(**args)
self.lview = None
self.ar = None
self.jobs = None
self.indices = None
self.finished = set()
# Obtain the host names and PIDs of engines.
self.pids = self.rc[:].apply(os.getpid).get_dict()
self.hosts = self.rc[:].apply(socket.getfqdn).get_dict()
# Change the working directory of each engine.
self.rc[:].apply(os.chdir, os.getcwd())
def run(self, jobs, targets=None):
"""Runs the jobs on the cluster.
Args:
jobs (list): list of dictionary objects describing the jobs.
targets (int, list of ints, 'all', or None): the engine(s) on which the jobs will run.
Returns:
bool: True if successful, False otherwise (e.g., jobs are running).
"""
if self.ar is not None and not self.ar.ready():
return False
self.lview = self.rc.load_balanced_view(targets)
self.ar = self.lview.map_async(runjob, jobs)
self.jobs = jobs
self.indices = dict([(k, v) for v, k in enumerate(self.ar.msg_ids)])
self.finished = set()
return True
def wait(self, timeout=1e-3):
"""Waits for the jobs to complete and writes job results.
Args:
timeout (float): a time in seconds, after which to give up.
"""
if self.ar is None:
return
# Find finished msg_ids.
pending = set(self.ar.msg_ids)
try:
self.rc.wait(pending, timeout)
except TimeoutError:
pass
finished = pending.difference(self.rc.outstanding)
# Overwrite the results in the job array.
for msg_id in finished:
i = self.indices[msg_id]
if i in self.finished:
continue
job = self.jobs[i]
meta = self.rc.metadata[msg_id]
result = self.rc.results[msg_id][0]
for key in ('submitted', 'started', 'completed', 'received'):
result[key] = meta[key].isoformat()
for key in ('engine_id', 'pyerr', 'pyout', 'status', 'msg_id'):
result[key] = meta[key]
result['elapsed'] = str(meta['completed'] - meta['started'])
result['host'] = self.hosts[meta['engine_id']]
job['result'] = result
self.finished.add(i)
def ready():
"""Returns whether the jobs have completed."""
return self.ar is not None or self.ar.ready()
def successful():
"""Returns whether the jobs completed without raising an exception.
Raises:
AssertionError: the result is not ready.
"""
return self.ar is not None and self.ar.successful()
def abort(self, **args):
"""Aborts jobs.
Args: identical to IPython.parallel.client.view.LoadBalancedView.abort()
"""
if self.lview is not None:
self.lview.abort()
def interrupt(self):
"""Sends SIGINT signal to engines (experimental).
http://mail.scipy.org/pipermail/ipython-dev/2014-March/013426.html
"""
self.abort()
for i in self.rc.ids:
host = self.hosts[i]
pid = self.pids[i]
if host == socket.getfqdn():
os.kill(pid, signal.SIGINT)
else:
os.system('ssh {} kill -INT {}'.format(host, pid))
def shutdown(self, **args):
"""Terminates one or more engine processes, optionally including the hub.
Args: identical to IPython.parallel.Client.shutdown
"""
if self.lview is None:
return False
self.lview.shutdown(**args)
def status(self, interval=1., timeout=-1, fo=sys.stdout):
"""Waits for the jobs, printing progress at regular intervals
Args:
interval (float): a time in seconds, after which to print the progress.
timeout (float): a time in seconds, after which to give up waiting.
fo (file): a file object to which the progress is printed.
"""
if self.ar is None:
return
if timeout is None:
timeout = -1
# Make sure to write the job results into the job objects.
self.wait(1e-3)
tic = time.time()
while not self.ar.ready() and (timeout < 0 or time.time() - tic <= timeout):
self.wait(interval)
clear_output(wait=True)
dt = datetime.timedelta(seconds=self.ar.elapsed)
fo.write('{}/{} tasks finished after {}'.format(self.ar.progress, len(self.ar), str(dt)))
fo.flush()
else:
fo.write('\n')
dt = datetime.timedelta(seconds=self.ar.elapsed)
clear_output(wait=True)
fo.write('{} tasks completed in {}\n'.format(len(self.ar), str(dt)))
fo.flush()
def __len__(self):
"""Returns the number of engines."""
return len(self.rc)
#-------------------------------------------------------------------------
# Imports
#-------------------------------------------------------------------------
from __future__ import print_function
import time
from IPython.parallel import Client
#-------------------------------------------------------------------------
# Setup
#-------------------------------------------------------------------------
mux = Client()[:]
mux.clear()
mux.block = False
ar1 = mux.apply(time.sleep, 5)
ar2 = mux.push(dict(a=10, b=30, c=range(20000), d='The dog went swimming.'))
ar3 = mux.pull(('a', 'b', 'd'), block=False)
print("Try a non-blocking get_result")
ar4 = mux.get_result()
print("Now wait for all the results")
mux.wait([ar1, ar2, ar3, ar4])
print("The last pull got:", ar4.r)
def client_wait(self, client, jobs=None, timeout=-1):
"""my wait wrapper, sets a default finite timeout to avoid hangs"""
if timeout < 0:
timeout = self.timeout
return Client.wait(client, jobs, timeout)
#-------------------------------------------------------------------------------
# Imports
#-------------------------------------------------------------------------------
from __future__ import print_function
import time
from IPython.parallel import Client
#-------------------------------------------------------------------------------
# Setup
#-------------------------------------------------------------------------------
mux = Client()[:]
mux.clear()
mux.block = False
ar1 = mux.apply(time.sleep, 5)
ar2 = mux.push(dict(a=10, b=30, c=range(20000), d='The dog went swimming.'))
ar3 = mux.pull(('a', 'b', 'd'), block=False)
print("Try a non-blocking get_result")
ar4 = mux.get_result()
print("Now wait for all the results")
mux.wait([ar1, ar2, ar3, ar4])
print("The last pull got:", ar4.r)
import time
import os
client = Client()
lbview = client.load_balanced_view()
def do_nothing(render_script, env_dict):
os.listdir(".")
num_tasks = 100
task_frames = range(1, num_tasks + 1, 1)
for x in task_frames:
env_dict = {}
render_script = "foo"
ar = lbview.apply(do_nothing, render_script, env_dict)
# avoid race condition
#time.sleep(0.2)
client.wait()
time.sleep(5)
records = client.db_query({'msg_id' : {'$in' : client.history}}, keys=['header', 'completed', 'engine_uuid'])
print "records: "+str(len(records))
print "history: "+str(len(client.history))
arrived = filter(lambda rec: rec['engine_uuid'] is not None, records)
print "arrived tasks: "+str(len(arrived))
finished = filter(lambda rec: rec['completed'] is not None, records)
print "finished tasks: "+str(len(finished))
