def parallel_run():
"""
Start parallel engines to run
"""
from IPython.parallel import Client
c = Client() # here is where the client establishes the connection
lv = c.load_balanced_view() # this object represents the engines (workers)
rays = []
maxs=25
bounding = AABA(xmin=0, ymin=0, zmin=0, xmax=maxs, ymax=maxs, zmax=maxs,)
gridd = np.zeros((maxs,maxs,maxs))
# spectrum for red to nir leaves
red_nir_leaves = spectrum(np.array([0.5, 0.85]), np.array([0.1, 0.6]), np.array([0.5, 0.1]))
# spectrum for soil
red_nir_soil = spectrum(np.array([0.5, 0.85]), np.array([0.3, 0.4]), np.array([0.0, 0.0]))
# scattering setup
scatt = BRDSF(red_nir_leaves, 0.0)
lf = leaf(55.0, 0.8) # leaf angle distribution and leaf area density
tasks = []
for x in xrange(maxs):
for y in xrange(maxs):
tasks.append(lv.apply(prun, x,y, maxs, gridd, scatt, red_nir_soil, bounding, lf))
result = [task.get() for task in tasks] # blocks until all results are back
return results
def main():
parser = argparse.ArgumentParser()
parser.add_argument('db_fname',
help="Provide the filename of the HDF database "
"file here.")
args = parser.parse_args()
image_names = get_image_names_from_db(args.db_fname)
logging.info('Found {} image_names'.format(len(image_names)))
c = Client()
dview = c.direct_view()
lbview = c.load_balanced_view()
dview.push({'do_clustering': do_clustering,
'dbfile': args.db_fname})
results = lbview.map_async(process_image_name, image_names)
import time
import sys
import os
dirname = os.path.join(os.environ['HOME'], 'data/planet4/catalog_2_and_3')
while not results.ready():
print("{:.1f} %".format(100 * results.progress / len(image_names)))
sys.stdout.flush()
time.sleep(10)
for res in results.result:
print(res)
logging.info('Catalog production done. Results in {}.'.format(dirname))
def add_engines(n=1, profile='iptest', total=False):
"""add a number of engines to a given profile.
If total is True, then already running engines are counted, and only
the additional engines necessary (if any) are started.
"""
rc = Client(profile=profile)
base = len(rc)
if total:
n = max(n - base, 0)
eps = []
for i in range(n):
ep = TestProcessLauncher()
ep.cmd_and_args = ipengine_cmd_argv + ['--profile=%s'%profile, '--log-level=50']
ep.start()
launchers.append(ep)
eps.append(ep)
tic = time.time()
while len(rc) < base+n:
if any([ ep.poll() is not None for ep in eps ]):
raise RuntimeError("A test engine failed to start.")
elif time.time()-tic > 15:
raise RuntimeError("Timeout waiting for engines to connect.")
time.sleep(.1)
rc.spin()
rc.close()
return eps
def run_jobs_on_ipythoncluster(worker, task_list, shutdown_ipengines_after_done=False):
t0 = time.time()
rc = Client(CLUSTER_CLIENT_JSON)
lview = rc.load_balanced_view()
print "\t# nodes in use: {}".format(len(lview.targets or rc.ids))
lview.block = False
print "\t# of tasks: {}".format(len(task_list))
print "\tsubmitting...",
job = lview.map_async(worker, task_list)
print "done."
try:
job.wait_interactive()
except KeyboardInterrupt:
#handle "Ctrl-C"
if ask("\nAbort all submitted jobs?") == 'Y':
lview.abort()
print "Aborted, all submitted jobs are cancelled."
else:
print "Aborted, but your jobs are still running on the cluster."
return
if len(job.result) != len(task_list):
print "WARNING:\t# of results returned ({}) != # of tasks ({}).".format(len(job.result), len(task_list))
print "\ttotal time: {}".format(timesofar(t0))
if shutdown_ipengines_after_done:
print "\tshuting down all ipengine nodes...",
lview.shutdown()
print 'Done.'
return job.result
def analyze_log_file_in_phases(file_id, nstates, trials, iter):
print_n_flush("Starting phase by phase analysis...")
# id_to_log = lambda x: "logs/%s.exp.log" % x
filename_log = id_to_log(file_id)
responses, tests, responses_t, tests_t, images = toCSV(filename_log)
from IPython.parallel import Client
# from functools import partial
from rpy2.rinterface import initr
rinterface.set_initoptions(("--max-ppsize=100000"))
initr()
client = Client(profile="default")
# client[:].push(dict(initr=initr))
# client[:].apply_sync(lambda: initr())
lview = client.load_balanced_view() # default load-balanced view
lview.block = True
# func = lambda args: train_hmm_n_times(file_id=args[0], nstates=args[1], trials=args[2], iter=args[3])
# trials = 4
client[:].push(dict(train_hmm_once=train_hmm_once))
# args = [(file_id, nstates, trials, 1000) for nstates in range(5,10)]
# results = lview.map(func, args)# hmm, d, results = train_hmm_n_times(file_id, nstates, trials=20, iter=1000)
# pool.join()
results = {}
for i in range(3):
results[i] = train_hmm_n_times(file_id, nstates=nstates, trials=trials, iter=iter, phase=i)
return results
def __init__(self,mdl):
"""The constructor.
mdl : model (of class Red3 or Red6)
"""
IntGen.__init__(self,mdl)
#find the engine processes
rc = Client(profile='mpi')
rc.clear()
#Create a view of the processes
self.view = rc[:]
#number of clients
nCl = len(rc.ids)
if mdl.Y.ndim >2:
#divisors of nCl
div = [i for i in range(1,nCl+1) if nCl%i==0]
ldiv = len(div)
#the surface will be divided into nbx rows and nby columns
if ldiv %2 == 0:
self.nbx = div[ldiv/2]
self.nby = div[ldiv/2-1]
else:
self.nbx = self.nby = div[ldiv/2]
else:
self.nbx = nCl
self.nby = 0
def map(r,func, args=None, modules=None):
"""
Before you run parallel.map, start your cluster (e.g. ipcluster start -n 4)
map(r,func, args=None, modules=None):
args=dict(arg0=arg0,...)
modules='numpy, scipy'
examples:
func= lambda x: numpy.random.rand()**2.
z=parallel.map(r_[0:1000], func, modules='numpy, numpy.random')
plot(z)
A=ones((1000,1000));
l=range(0,1000)
func=lambda x : A[x,l]**2.
z=parallel.map(r_[0:1000], func, dict(A=A, l=l))
z=array(z)
"""
mec = Client()
mec.clear()
lview=mec.load_balanced_view()
for k in mec.ids:
mec[k].activate()
if args is not None:
mec[k].push(args)
if modules is not None:
mec[k].execute('import '+modules)
z=lview.map(func, r)
out=z.get()
return out
def remove_duplicates(df):
logging.info('Removing duplicates.')
image_names = df.image_name.unique()
def process_image_name(image_name):
data = df[df.image_name == image_name]
data = remove_duplicates_from_image_name_data(data)
data.to_hdf(get_temp_fname(image_name), 'df')
# parallel approach, u need to launch an ipcluster/controller for this work!
c = Client()
dview = c.direct_view()
dview.push({'remove_duplicates_from_image_name_data':
remove_duplicates_from_image_name_data,
'data_root': data_root})
lbview = c.load_balanced_view()
lbview.map_sync(process_image_name, image_names)
df = []
for image_name in image_names:
try:
df.append(pd.read_hdf(get_temp_fname(image_name), 'df'))
except OSError:
continue
else:
os.remove(get_temp_fname(image_name))
df = pd.concat(df, ignore_index=True)
logging.info('Duplicates removal complete.')
return df
def ipythonMP(m):
from IPython.parallel import Client
cli = Client()
dview = cli[:]
lbview = cli.load_balanced_view()
return dview.map_sync(factorize,range(m))
def subsample(cache_dir, image_sets, ipython_profile):
parameters = [(cache_dir, images) for images in image_sets]
if ipython_profile:
from IPython.parallel import Client, LoadBalancedView
client = Client(profile='lsf')
lview = client.load_balanced_view()
generator = lview.imap(_compute_group_subsample, parameters)
elif ipython_profile == False:
generator = (_compute_group_subsample(p) for p in parameters)
else:
from multiprocessing import Pool
lview = Pool()
generator = lview.imap(_compute_group_subsample, parameters)
progress = progressbar.ProgressBar(widgets=['Subsampling:',
progressbar.Percentage(), ' ',
progressbar.Bar(), ' ',
progressbar.Counter(), '/',
str(len(parameters)), ' ',
progressbar.ETA()],
maxval=len(parameters))
results = list(generator)
subsample = []
for i, (p, r) in enumerate(zip(parameters, results)):
if r is None:
print >>sys.stderr, '#### There was an error, recomputing locally: %s' % parameters[i][1]
results[i] = _compute_group_subsample(p) # just to see throw the exception
subsample.extend(r)
print "the subsampling set contains %d items" % len(subsample)
return subsample
def test_run_from_multiple_files_without_cache_on_ipy_cluster(self):
try:
from IPython.parallel import Client
client = Client()
pool = client.load_balanced_view()
except:
raise unittest.SkipTest("Cluster connection failed")
models = [self.transport]
p = Point(self.start_lon, self.start_lat)
model = IPythonClusterModelController(geometry=p,
depth=self.start_depth,
start=self.start_time,
step=self.time_step,
nstep=self.num_steps,
npart=self.num_particles,
models=models,
use_bathymetry=False,
use_shoreline=False,
pool=pool)
model.setup_run("/data/lm/tests/pws_das_2014*.nc")
model.run(output_formats=self.output_formats, output_path=self.output_path)
self.assertTrue(os.path.exists(os.path.join(self.output_path, "simple_trackline.geojson")))
self.draw_trackline(os.path.join(self.output_path, "simple_trackline.geojson"))
# Not a caching controller, no cache path should exist
self.assertFalse(os.path.exists(self.cache_path))
def __init__(self, config_filename=None, profile=None, seed=None, sshkey=None, packer='json'):
"""Initialize a IPClusterEngine
Do IPython.parallel operations to set up cluster and generate mapper.
"""
super(IPClusterEngine, self).__init__(seed=seed)
rc = Client(config_filename, profile=profile, sshkey=sshkey, packer=packer)
# FIXME: add a warning if environment in direct view is not 'empty'?
# else, might become dependent on an object created in
# environemnt in a prior run
dview = rc.direct_view()
lview = rc.load_balanced_view()
with dview.sync_imports(local=True):
import crosscat
mapper = lambda f, tuples: self.lview.map(f, *tuples)
# if you're trying to debug issues, consider clearning to start fresh
# rc.clear(block=True)
#
self.rc = rc
self.dview = dview
self.lview = lview
self.mapper = mapper
self.do_initialize = None
self.do_analyze = None
return
def _is_up(profile, cluster_id, n):
try:
#client = Client(profile=profile, cluster_id=cluster_id)
client = Client(profile=profile)
up = len(client.ids)
client.close()
except IOError, msg:
return False
def _test_wrapper_remote(func):
"""Execute a function on a remote ipengine"""
from IPython.parallel import Client
from qiita_core.configuration_manager import ConfigurationManager
config = ConfigurationManager()
c = Client(profile=config.ipython_default)
bv = c.load_balanced_view()
return _ipy_wait(bv.apply_async(func))
def cluster_view(scheduler, queue, num_jobs, cores_per_job=1, profile=None,
start_wait=16, extra_params=None, retries=None):
"""Provide a view on an ipython cluster for processing.
- scheduler: The type of cluster to start (lsf, sge, pbs, torque).
- num_jobs: Number of jobs to start.
- cores_per_job: The number of cores to use for each job.
- start_wait: How long to wait for the cluster to startup, in minutes.
Defaults to 16 minutes. Set to longer for slow starting clusters.
- retries: Number of retries to allow for failed tasks.
"""
if extra_params is None:
extra_params = {}
delay = 10
max_delay = start_wait * 60
# Increase default delay without changing max_delay for back compatibility
delay = delay * 3
max_tries = 10
if profile is None:
has_throwaway = True
profile = create_throwaway_profile()
else:
# ensure we have an .ipython directory to prevent issues
# creating it during parallel startup
cmd = [sys.executable, "-c", "from IPython import start_ipython; start_ipython()",
"profile", "create"]
subprocess.check_call(cmd)
has_throwaway = False
num_tries = 0
cluster_id = str(uuid.uuid4())
url_file = get_url_file(profile, cluster_id)
while 1:
try:
_start(scheduler, profile, queue, num_jobs, cores_per_job, cluster_id, extra_params)
break
except subprocess.CalledProcessError:
if num_tries > max_tries:
raise
num_tries += 1
time.sleep(delay)
try:
client = None
slept = 0
while not _is_up(url_file, num_jobs):
time.sleep(delay)
slept += delay
if slept > max_delay:
raise IOError("Cluster startup timed out.")
client = Client(url_file, timeout=60)
yield _get_balanced_blocked_view(client, retries)
finally:
if client:
client.close()
_stop(profile, cluster_id)
if has_throwaway:
delete_profile(profile)
def calibrate_multiple():
num_runs = 12
rc = Client()
lview = rc.load_balanced_view()
lview.block = True
print(datetime.datetime.now())
res = lview.map(lambda q: calibrate_mh(), range(num_runs))
print(datetime.datetime.now())
1/0
def _is_up(url_file, n):
try:
client = Client(url_file)
up = len(client.ids)
client.close()
except IOError:
return False
else:
return up >= n
def main_loop(self, time_budget=None, parallel=False, client_kwargs=None,
view_flags=None):
"""
Run main_loop of each trainer.
Note: if you get PickleErrors when running in parallel, make sure
you have `dill` installed.
Parameters
----------
time_budget : int, optional
The maximum number of seconds before interrupting
training. Default is `None`, no time limit.
parallel : bool, optional
Whether to train subtrainers in parallel using
IPython.parallel (default False).
client_kwargs : dict, optional
Keyword arguments for IPython.parallel Client.
view_flags : dict, optional
Flags for IPython.parallel LoadBalancedView.
"""
self.setup()
if parallel:
from IPython.parallel import Client
def _train(trainer, time_budget=None):
"""
Run main_loop of this trainer.
Parameters
----------
trainer : Train object
Train object.
time_budget : int, optional
The maximum number of seconds before interrupting
training. Default is `None`, no time limit.
"""
trainer.main_loop(time_budget)
return trainer
if client_kwargs is None:
client_kwargs = {}
if view_flags is None:
view_flags = {}
client = Client(**client_kwargs)
view = client.load_balanced_view()
view.set_flags(**view_flags)
call = view.map(_train,
self.trainers[self.skip_folds:],
[time_budget] * len(self.trainers[self.skip_folds:]),
block=False)
self.trainers = call.get()
else:
for trainer in self.trainers[self.skip_folds:]:
trainer.main_loop(time_budget)
self.save()
def main():
parser = OptionParser()
parser.add_option('-d','--dataset',dest='dataset',help='path to dataset')
parser.add_option('--input_format',dest='input_format',help='format of training dataset(s) tsv | csv | mm (matrixmarket) | fsm (fast_sparse_matrix)')
parser.add_option('--l1_min',dest='l1_min',type='float',help='min l1 constant to try (expected to be a power of 10)')
parser.add_option('--l1_max',dest='l1_max',type='float',help='max l1 constant to try (expected to be a power of 10)')
parser.add_option('--l2_min',dest='l2_min',type='float',help='min l2 constant to try (expected to be a power of 10)')
parser.add_option('--l2_max',dest='l2_max',type='float',help='max l2 constant to try (expected to be a power of 10)')
parser.add_option('--max_sims',dest='max_sims',type='int',default=2000,help='max desired number of positive item similarity weights (default: %default)')
parser.add_option('--min_sims',dest='min_sims',type='int',default=15,help='min desired number of positive item similarity weights (default: %default)')
parser.add_option('--max_sparse',dest='max_sparse',type='float',default=0.01,help='max allowable proportion of items with less than min_sims positive similarity weights (default: %default)')
parser.add_option('--num_samples',dest='num_samples',type='int',default=100,help='number of sample items to evaluate for each regularization setting')
parser.add_option('--packer',dest='packer',default='json',help='packer for IPython.parallel (default: %default)')
parser.add_option('--add_module_paths',dest='add_module_paths',help='comma-separated list of paths to append to pythonpath to enable import of uninstalled modules')
(opts,args) = parser.parse_args()
if not opts.dataset or not opts.input_format or not opts.l1_min or not opts.l1_max or not opts.l2_min or not opts.l2_max:
parser.print_help()
raise SystemExit
logging.basicConfig(level=logging.INFO,format='[%(asctime)s] %(levelname)s: %(message)s')
dataset = load_fast_sparse_matrix(opts.input_format,opts.dataset)
params = {'l1_reg':pow_range(opts.l1_min,opts.l1_max),
'l2_reg':pow_range(opts.l2_min,opts.l2_max)}
num_items = dataset.shape[1]
sample_items = random.sample(xrange(num_items),opts.num_samples)
logging.info('preparing tasks for a grid search of these values:')
logging.info(params)
tasks = [(args,dataset,opts.min_sims,sample_items) for args in ParameterGrid(params)]
c = Client(packer=opts.packer)
view = c.load_balanced_view()
if opts.add_module_paths:
c[:].execute('import sys')
for path in opts.add_module_paths.split(','):
logging.info('adding {0} to pythonpath on all engines'.format(path))
c[:].execute("sys.path.append('{0}')".format(path))
logging.info('running {0} tasks in parallel...'.format(len(tasks)))
results = view.map(estimate_sparsity,tasks,ordered=False)
candidates = [(args,nsims,nsparse,nneg) for args,nsims,nsparse,nneg in results if nsims <= opts.max_sims and nsparse <= opts.max_sparse]
if candidates:
best = min(candidates,key=itemgetter(1))
print 'best parameter setting: {0}'.format(best[0])
print 'mean # positive similarity weights per item = {0:.3}'.format(best[1])
print 'proportion of items with fewer than {0} positive similarity weights = {1:.3}'.format(opts.min_sims,best[2])
print 'mean # negative similarity weights per item = {0:.3}'.format(best[3])
else:
print 'no parameter settings satisfied the conditions, try increasing --min_sims, --max_sims or --max_sparse'
def test_run_from_multiple_files_without_cache_on_ipy_cluster(self):
try:
from IPython.parallel import Client
client = Client()
pool = client.load_balanced_view()
except:
raise unittest.SkipTest("Cluster connection failed")
self.test_run_from_multiple_files_without_cache(pool=pool)
def connect_client(self):
"""connect a client with my Context, and track its sockets for cleanup"""
c = Client(profile='iptest', context=self.context)
c.wait = lambda *a, **kw: self.client_wait(c, *a, **kw)
for name in filter(lambda n:n.endswith('socket'), dir(c)):
s = getattr(c, name)
s.setsockopt(zmq.LINGER, 0)
self.sockets.append(s)
return c
def init(profile="mpi"):
"""Initialize pyDive.
:param str profile: The name of the cluster profile of *IPython.parallel*. Has to be an MPI-profile.\
Defaults to 'mpi'.
"""
# init direct view
global view
client = Client(profile=profile)
client.clear()
view = client[:]
view.block = True
view.execute(
"""\
import numpy as np
from mpi4py import MPI
import h5py as h5
import os, sys
import psutil
import math
os.environ["onTarget"] = 'True'
from pyDive import structured
from pyDive import algorithm
from pyDive.distribution import interengine
try:
import pyDive.arrays.local.h5_ndarray
except ImportError:
pass
try:
import pyDive.arrays.local.ad_ndarray
except ImportError:
pass
try:
import pyDive.arrays.local.gpu_ndarray
import pycuda.autoinit
except ImportError:
pass
"""
)
# get number of processes per node (ppn)
def hostname():
import socket
return socket.gethostname()
hostnames = view.apply(interactive(hostname))
global ppn
ppn = max(Counter(hostnames).values())
# mpi ranks
get_rank = interactive(lambda: MPI.COMM_WORLD.Get_rank())
all_ranks = view.apply(get_rank)
view["target2rank"] = all_ranks
def _is_up(url_file, n):
try:
client = Client(url_file, timeout=60)
up = len(client.ids)
client.close()
except iperror.TimeoutError:
return False
except IOError:
return False
else:
return up >= n
def main():
partial_results=[]
c=Client(profile='default')
print c.ids
view=c.load_balanced_view()
ar = view.map_async(func, range(10))
#print ar.get_dict(timeout=0)
print ar.msg_ids
for i, r in enumerate(ar):
print r[1]
print ar.get()
def importanceSampling_parallel4(generateSample,args_generateSample,likelihoodRatio,args_likelihoodRatio, N):
# not working
temp = 3
from IPython.parallel import Client
p = Client()[:]
p.use_dill()
print('This is the parallel output')
p.map_sync(generateSample,np.arange(6))
#sample = p.map_sync(generateSample,np.array((10,)))
#print(sample)
return(1)
def run_parallel_jobs(jobs, job_fn, ipython_profile=None):
# IPython will error out if jobs is empty.
if jobs:
if ipython_profile is None:
c = Client()
else:
c = Client(profile=ipython_profile)
lview = c.load_balanced_view()
lview.block = True
lview.map(job_fn, jobs)
def run_nb(beliefs, meta, params, num_samples):
if params['do_parallel']:
rc = Client()
lview = rc.load_balanced_view()
lview.block = True
else:
lview = None
print(datetime.datetime.now().time())
results = aggregation.importance_multiple(beliefs, meta, params,
num_samples, lview)
print(datetime.datetime.now().time())
return results
def featurize(self, mols, parallel=False, client_kwargs=None,
view_flags=None, verbosity=None, log_every_n=1000):
"""
Calculate features for molecules.
Parameters
----------
mols : iterable
RDKit Mol objects.
parallel : bool, optional
Whether to train subtrainers in parallel using
IPython.parallel (default False).
client_kwargs : dict, optional
Keyword arguments for IPython.parallel Client.
view_flags : dict, optional
Flags for IPython.parallel LoadBalancedView.
"""
if self.conformers and isinstance(mols, types.GeneratorType):
mols = list(mols)
assert verbosity in [None, "low", "high"]
if parallel:
from IPython.parallel import Client
if client_kwargs is None:
client_kwargs = {}
if view_flags is None:
view_flags = {}
client = Client(**client_kwargs)
client.direct_view().use_dill() # use dill
view = client.load_balanced_view()
view.set_flags(**view_flags)
call = view.map(self._featurize, mols, block=False)
features = call.get()
# get output from engines
call.display_outputs()
else:
features = []
for i, mol in enumerate(mols):
if verbosity is not None and i % log_every_n == 0:
log("Featurizing %d / %d" % (i, len(mols)))
if mol is not None:
features.append(self._featurize(mol))
else:
features.append(np.array([]))
if self.conformers:
features = self.conformer_container(mols, features)
else:
features = np.asarray(features)
return features
def calibrate():
num_runs = 12
rc = Client()
lview = rc.load_balanced_view()
lview.block = True
print(datetime.datetime.now())
res = lview.map(lambda q: all_together_q(), range(num_runs))
print(datetime.datetime.now())
act = np.array([r[0] for r in res])
mh = np.array([r[1] for r in res])
nb = np.array([r[2] for r in res])
gp = np.array([r[3] for r in res])
1/0
def cluster_view(scheduler, queue, num_jobs, cores_per_job=1, profile=None,
start_wait=16, extra_params=None, retries=None):
"""Provide a view on an ipython cluster for processing.
- scheduler: The type of cluster to start (lsf, sge, pbs, torque).
- num_jobs: Number of jobs to start.
- cores_per_job: The number of cores to use for each job.
- start_wait: How long to wait for the cluster to startup, in minutes.
Defaults to 16 minutes. Set to longer for slow starting clusters.
- retries: Number of retries to allow for failed tasks.
"""
if extra_params is None:
extra_params = {}
delay = 10
max_delay = start_wait * 60
max_tries = 10
if profile is None:
has_throwaway = True
profile = create_throwaway_profile()
else:
has_throwaway = False
num_tries = 0
cluster_id = str(uuid.uuid4())
url_file = get_url_file(profile, cluster_id)
#cluster_id = ""
while 1:
try:
_start(scheduler, profile, queue, num_jobs, cores_per_job, cluster_id, extra_params)
break
except subprocess.CalledProcessError:
if num_tries > max_tries:
raise
num_tries += 1
time.sleep(delay)
try:
client = None
slept = 0
while not _is_up(url_file, num_jobs):
time.sleep(delay)
slept += delay
if slept > max_delay:
raise IOError("Cluster startup timed out.")
client = Client(url_file)
yield _get_balanced_blocked_view(client, retries)
finally:
if client:
client.close()
_stop(profile, cluster_id)
if has_throwaway:
delete_profile(profile)
def main():
N = 500
base = ['a', 'c', 'g', 't']
seqPairs = []
np.random.seed(1234)
for i in xrange(8):
seqA = ''.join([base[np.random.randint(0, 4)] for i in xrange(N)])
seqB = ''.join([base[np.random.randint(0, 4)] for i in xrange(N)])
seqPairs.append([seqA, seqB])
client = Client()
# Note: for ipclustergenomic to be loaded on the cluster side, engines
# must have been started in the same directory as ipclustergenomic.py
with client[:].sync_imports():
from ipclustergenomic import editDistance
printRes(client[:].map_sync(edProxy, seqPairs))
class ClusterPool(object):
def __init__(self, *args, **kwargs):
self.client = Client(*args, **kwargs)
self.lbview = self.client.load_balanced_view()
self.chunksize = 1
def map_with_shared_data(self, func, shared_data, args, chunksize=None):
"""Map a function over each in a set of arguments, also passing a constant shared variable to each invocation."""
# no imap with shared data, since we couldn't guarantee the integrity of FUNC and SHARED_DATA
self.dview.push(dict(FUNC=func, SHARED_DATA=shared_data), block=True)
return self.lbview.map_sync(func_int,
args,
chunksize=chunksize or self.chunksize,
ordered=True)
def map(self, func, args, chunksize=None):
map = self.lbview.map
return iter(
map(func,
args,
chunksize=chunksize or self.chunksize,
block=True,
ordered=True))
def imap(self, func, args, chunksize=None):
map = self.lbview.map
return iter(
map(func,
args,
chunksize=chunksize or self.chunksize,
block=False,
ordered=True))
def imap_unordered(self, func, args, chunksize=None):
map = self.lbview.map
return iter(
map(func,
args,
chunksize=chunksize or self.chunksize,
block=False,
ordered=False))
class MulticoreJob(object):
def __init__(self):
self.tasks = {}
self.client = Client()
self.lb_view = self.client.load_balanced_view()
def apply(self, f, named_tasks):
"""named_tasks: dict of {nametask: taskparams}
"""
self.tasks = {
tname: self.lb_view.apply(f, **param)
for (tname, param) in named_tasks.items()
}
return self
def isready(self):
return all([t.ready() for t in self.tasks.values()])
def progress(self):
return np.mean([t.ready() for t in self.tasks.values()])
def partial_result(self):
return {
tname: tresult.get()
for (tname, tresult) in self.tasks.items() if tresult.ready()
}
def wait(self):
for (tname, tresult) in self.tasks.items():
tresult.wait()
return self
def abort(self):
for (tname, tresult) in self.tasks.items():
if not tresult.ready():
try:
tresult.abort()
except:
pass
return self
def run_parallel_map():
""" Run a test with synthetic data and MCMC inference
"""
# Parse command line args
(options, args) = parse_cmd_line_args()
# Load the data
data = load_data(options)
# Get a model for the data
model_type = 'standard_glm'
model = make_model(model_type, N=data['N'])
# Get parallel clients
rc = Client(profile="sge")
dview = rc[:]
# dview = get_engines(n_workers=8)
# Load imports on the client
load_imports_on_client(dview)
# Initialize population objects on the clients
dview.apply_sync(initialize_client, (model_type,N,data))
def create_from_options(cls, parser, options):
noptions = ((options.ipython_profile and 1 or 0) +
(options.lsf_directory and 1 or 0) +
(options.multiprocessing and 1 or 0))
if noptions > 1:
parser.error(
'You can only specify one of --ipython-profile, --lsf-directory, and --multiprocessing.'
)
if options.lsf_directory:
from . import lsf
return lsf.LSF(options.njobs,
options.lsf_directory,
memory=options.memory,
job_array_name=options.jobname)
elif options.ipython_profile:
from IPython.parallel import Client, LoadBalancedView
client = Client(profile=options.ipython_profile)
return IPython(client)
elif options.multiprocessing:
return Multiprocessing()
else:
return Uniprocessing()
def main():
parser = optparse.OptionParser(
usage="""Usage: %prog [options] infolder outfolder
infolder: input folder containing .jpg files
outfolder: output folder to contain collection
""")
# implementing this means dependence on IPython
parser.add_option("-m", "--mec", dest="mec", default=False,
help="Use IPython parallel, say 'default'")
parser.add_option("-n", "--name", dest="name", default='dzc',
help="collection name, default 'dzc'")
(options, args) = parser.parse_args()
if len(args)!=2:
parser.print_help()
sys.exit(1)
infolder,outfolder = map(
lambda d:os.path.abspath(os.path.expanduser(os.path.expandvars(d))),args
)
for folder in (infolder, outfolder):
if not os.path.exists(folder):
print "Folder %s not found" % folder
sys.exit(1)
mec = None
if options.mec:
from IPython.parallel import Client
mec = Client(profile=mec)
assert len(mec)>0, 'No engines found'
print infolder, '-->', outfolder
mc = MakeCollection(infolder, outfolder, collection_name=options.name)
mc.convert(mec)
mc.collect()
def map_on_cluster(parallel_profile, *args, **kw_args):
"""
The same as map, but the first argument is ipc_profile. Distributes the task over IPython cluster.
:param parallel_profile: the IPython cluster profile to use.
:type parallel_profile: None or str
:param list args: function, arguments
:param dict kw_args: kwargs for LoadBalacedView.map_sync
:return: the result of mapping
"""
if parallel_profile is None:
return map(*args)
elif str.startswith(parallel_profile, 'threads-'):
n_threads = int(parallel_profile[len('threads-'):])
pool = ThreadPool(processes=n_threads)
func, params = args[0], args[1:]
return pool.map(_threads_wrapper, zip(itertools.cycle([func]),
*params))
else:
from IPython.parallel import Client
return Client(profile=parallel_profile).load_balanced_view().map_sync(
*args, **kw_args)
def test_dask_client_from_ipclient():
c = Client()
dask_client = dask_client_from_ipclient(c)
# data
a = np.arange(100).reshape(10, 10)
d = da.from_array(a, ((5, 5), (5, 5)))
try:
# test array.mean
expected = a.mean(axis=0)
d1 = d.mean(axis=0)
result = d1.compute(get=dask_client.get)
assert_array_almost_equal(result, expected)
# test ghosting
d2 = da.ghost.ghost(d, depth=1, boundary='reflect')
d3 = da.ghost.trim_internal(d2, {0: 1, 1: 1})
result1 = d3.compute(get=dask_client.get)
assert_array_almost_equal(result1, a)
finally:
# close the workers
dask_client.close(close_scheduler=True)
def parallel_directview_generic(PROFILE_NAME, module_names, FILE_REPOSITORY,
LOG_LEVEL, FILES_TO_PROCESS, input_dir,
output_dir, reports_dir, COMMENT,
MAKE_KML_FILES):
'''sets up worker namespaces for ipython parallel runs'''
print "Run 'ipcluster start -n 10' from the command line first!"
#from IPython import parallel
from IPython.parallel import Client
c = Client() #c=Client(debug=True)
print c.ids
#engine_count = len(c.ids)
dview = c[:] #DirectView list of engines
dview.clear() #clean up the namespaces on the eng
dview.block = True
#build parallel namespace
dview['module_names'] = module_names
dview['PROFILE_NAME'] = PROFILE_NAME
dview['COMMENT'] = COMMENT
dview['LOG_LEVEL'] = LOG_LEVEL
dview.scatter('files_to_process', FILES_TO_PROCESS)
dview['file_repository'] = FILE_REPOSITORY
dview['MAKE_KML_FILES'] = MAKE_KML_FILES
reports_dir = settings.PROFILE_REPORTS_PATH
output_dir = settings.PROFILE_DATA_PATH + PROFILE_NAME + '/'
if not os.path.exists(output_dir):
os.makedirs(output_dir)
dview['input_dir '] = input_dir
dview['output_dir '] = output_dir
dview['reports_dir '] = reports_dir
#logger = logging.Logger('ipcluster')
#logger.setLevel(LOG_LEVEL)
#dview['logger'] = logger
return dview
def fit(df, alpha=0.1, eta0=0.01, power_t=0.25, epochs=3,
latent_dimensions=10):
rc = Client()
dview = rc[:]
k = float(len(rc))
with dview.sync_imports():
import random
from numpy.linalg import norm
from numpy import dot, isnan
from numpy.random import normal
from pandas import DataFrame
dview.scatter('df', df)
res = dview.apply_sync(sgd, alpha=0.1, eta0=0.01, power_t=0.25, epochs=3,
latent_dimensions=latent_dimensions)
add = lambda a,b: a.add(b, fill_value=0)
U = reduce(add, (r[0] for r in res))/k
V = reduce(add, (r[1] for r in res))/k
return U, V
def create_runners(self):
# Check the status of runners and the cluster process
# If cluster process dead (non-None return to p.poll)
# kill and re-start (add config for this?) and then
# create an in-process kernel and pop it on the queue
# to keep things ticking
if self.p is None:
# Use the in-process runner for now
# self.runners = [self.in_process_runner]
self.start_cluster()
elif self.p.poll() is None:
# Create matching runners for the client
# note that these may already exist; we need to check
if self.client is None:
self.client = Client(timeout=5)
# FIXME: Inline plots are fine as long as we don't do it on the cluster+the interactive kernel; this results
# in an image cache being generated that breaks the pickle
for e in self.client:
found = False
for r in self.runners:
if e.targets == r.e.targets:
found = True
if not found:
runner = ClusterRunner(e)
runner.e.execute('%reset -f')
runner.e.execute('%matplotlib inline')
self.runners.append(runner)
else:
# We've got a -value for poll; it's terminated this will trigger restart on next poll
self.stop_cluster()
def featurize(self,
mols,
parallel=False,
client_kwargs=None,
view_flags=None):
"""
Calculate features for molecules.
Parameters
----------
mols : iterable
RDKit Mol objects.
parallel : bool, optional (default False)
Train subtrainers in parallel using IPython.parallel.
client_kwargs : dict, optional
Keyword arguments for IPython.parallel Client.
view_flags : dict, optional
Flags for IPython.parallel LoadBalancedView.
"""
if parallel:
from IPython.parallel import Client
if client_kwargs is None:
client_kwargs = {}
if view_flags is None:
view_flags = {}
client = Client(**client_kwargs)
client.direct_view().use_dill() # use dill
view = client.load_balanced_view()
view.set_flags(**view_flags)
call = view.map(self._featurize,
np.array_split(mols, len(client.direct_view())),
block=False)
features = call.get()
features = np.concatenate(features)
# get output from engines
call.display_outputs()
else:
features = self._featurize(mols)
return np.asarray(features)
def add_engines(n=1, profile='iptest'):
rc = Client(profile=profile)
base = len(rc)
eps = []
for i in range(n):
ep = TestProcessLauncher()
ep.cmd_and_args = ipengine_cmd_argv + ['--profile=%s'%profile, '--log-level=50']
ep.start()
launchers.append(ep)
eps.append(ep)
tic = time.time()
while len(rc) < base+n:
if any([ ep.poll() is not None for ep in eps ]):
raise RuntimeError("A test engine failed to start.")
elif time.time()-tic > 10:
raise RuntimeError("Timeout waiting for engines to connect.")
time.sleep(.1)
rc.spin()
rc.close()
return eps
def add_engines(n=1, profile='iptest'):
rc = Client(profile=profile)
base = len(rc)
eps = []
for i in range(n):
ep = Popen(
['ipengine'] +
['--profile', profile, '--log-level', '10', '--log-to-file'],
stdout=blackhole,
stderr=STDOUT)
# ep.start()
processes.append(ep)
eps.append(ep)
tic = time.time()
while len(rc) < base + n:
if any([ep.poll() is not None for ep in eps]):
raise RuntimeError("A test engine failed to start.")
elif time.time() - tic > 10:
raise RuntimeError("Timeout waiting for engines to connect.")
time.sleep(.1)
rc.spin()
rc.close()
return eps
def starmap(func, iterable, **kwargs):
"""
A dynamic load balancing parallel implementation of itertools.starmap for IPython.parallel.
The reason for it's existence was twofold.
First, the desire to easily submit a 'map' onto inputs
already grouped in tuples in IPython.parallel.
Second was the ability to submit a 'map' onto very large
sequences. Potentially infinite sequences.
This function allows one to do that. It is a generator function, so it is iterable.
It maintains an internal list of returned results that are removed once yielded.
The iterable passed as an argument need only have a next() method and raise StopIteration
when it is finished iterating.
Arguments
---------
*func* - The function to be called (remotely) on each iterable.next()
*iterable* - An iterable, generator, generator function...etc. Something with a .next() that
will raise StopIteration when finished
*profile* - (optional keyword argument. Default = None) The ipython parallel cluster profile.
This function expects the cluster to already be 'up'. Under the default of None,
this will start a client and load balanced view under the default profile, if
possible. If the profile specified is not running, an IO error will be raised.
(Ignored if client keyword argument is specified)
*client* - (optional keyword argument. Default = None) An instance of
IPython.parallel.Client
*max_fill* - (optional keyword argument. Default = 500000)The maximum number of
'jobs' to submit to the cluster before waiting for earlier jobs to finish.
*wait* - (optional keyword argument. Default = 1) Number of seconds to wait when
submission queue is full, and no further output may be yielded.
*kwargs* - Additional keyword arguments are treated as keyword arguments to func.
A note on the profile and client keyword arguments: If client is specified, the profile
kwarg will be ignored.
"""
profile = kwargs.pop('profile', None)
rc = kwargs.pop('client', None)
max_fill = kwargs.pop('max_fill', 50000)
wait = kwargs.pop('wait', 1)
if rc is None:
rc = Client(profile=profile)
elif not isinstance(rc, Client):
raise ValueError(
'client keyword value expected an instance of IPython.parallel.Client'
)
lbv = rc.load_balanced_view()
async_results_list = [] #This will serve as our output queue
while True: #GO until StopIteration is raised
if n_queued_jobs(lbv) < max_fill:
#If there are less than the maximum number of jobs waiting to run,
#submit the next job, unless we cannot.
try:
async_results_list.append(
lbv.apply(func, *iterable.next(), **kwargs))
except StopIteration:
if len(async_results_list) == 0:
raise
while len(async_results_list) > 0 and async_results_list[0].ready():
#If there are results ready to be read, pop them off
yield async_results_list.pop(0).get()
if n_queued_jobs(lbv) >= max_fill:
time.sleep(wait)
import os
import pickle
from datetime import datetime
from ConfigParser import ConfigParser
from IPython.parallel import Client
config = ConfigParser()
config.read('{0}/config.ini'.format(os.path.dirname(
os.path.realpath(__file__))))
rc = Client(packer="pickle")
dview = rc[:]
print rc.ids
@dview.remote(block=True)
def fetch():
import os
os.chdir(node)
for i, f in enumerate(files):
fname = f.split("/")[-1].split(".")[0]
if not os.path.exists("{0}.xml".format(fname)):
os.system("wget {0}".format(f))
os.system("unzip {0}.zip".format(fname))
fname = open("urls.pickle", "rb")
urls = pickle.load(fname)
master = config.get('directory', 'home')
node = config.get('directory', 'local')
Reference('it' + name))
# This causes the StopIteration exception to be raised.
except RemoteError, e:
if e.ename == 'StopIteration':
raise StopIteration
else:
raise e
else:
yield result
# Main, interactive testing
if __name__ == '__main__':
from IPython.parallel import Client, Reference
rc = Client()
view = rc[:]
print 'Engine IDs:', rc.ids
# Make a set of 'sorted datasets'
a0 = range(5, 20)
a1 = range(10)
a2 = range(15, 25)
# Now, imagine these had been created in the remote engines by some long
# computation. In this simple example, we just send them over into the
# remote engines. They will all be called 'a' in each engine.
rc[0]['a'] = a0
rc[1]['a'] = a1
rc[2]['a'] = a2
# encoding: utf-8
# ---------------------------------------------------------------------------
# Copyright (C) 2008-2014, IPython Development Team and Enthought, Inc.
# Distributed under the terms of the BSD License. See COPYING.rst.
# ---------------------------------------------------------------------------
"""
Calculate pi using a Monte Carlo method using IPython Parallel.
"""
from IPython.parallel import Client, interactive
from util import timer
client = Client()
view = client[:]
view.execute('import numpy')
@interactive # this runs on the engins
def calc_pi_on_engines(n):
x = numpy.random.rand(n)
y = numpy.random.rand(n)
r = numpy.hypot(x, y)
return 4. * (r < 1.).sum() / n
@timer
def calc_pi(n):
"""Estimate pi using IPython.parallel."""
n_engines = n / len(view)
results = view.apply_sync(calc_pi_on_engines, n_engines)
def fit(self, X, y, sample_weight=None):
"""
Run fit with all sets of parameters.
:param X: array-like, shape = [n_samples, n_features]
Training vector, where n_samples is the number of samples and n_features is the number of features.
:param y: array-like, shape = [n_samples] or [n_samples, n_output], optional
:param sample_weight: array-like, shape = [n_samples], weight
"""
X, y, sample_weight = check_inputs(X,
y,
sample_weight=sample_weight,
allow_none_weights=True)
if self.parallel_profile is None:
while self.evaluations_done < self.params_generator.n_evaluations:
state_indices, state_dict = self.params_generator.generate_next_point(
)
status, value = apply_scorer(self.scorer, state_dict,
self.base_estimator, X, y,
sample_weight)
assert status == 'success', 'Error during grid search ' + str(
value)
self.params_generator.add_result(state_indices, value)
self.evaluations_done += 1
state_string = ", ".join(
[k + '=' + str(v) for k, v in state_dict.items()])
self._log('{}: {}'.format(value, state_string))
else:
if str.startswith(self.parallel_profile, 'threads'):
_, n_threads = str.split(self.parallel_profile, '-')
portion = int(n_threads)
print("Performing grid search in {} threads".format(portion))
else:
from IPython.parallel import Client
direct_view = Client(
profile=self.parallel_profile).direct_view()
portion = len(direct_view)
print(
"There are {0} cores in cluster, the portion is equal {1}".
format(len(direct_view), portion))
while self.evaluations_done < self.params_generator.n_evaluations:
state_indices_array, state_dict_array = self.params_generator.generate_batch_points(
size=portion)
current_portion = len(state_indices_array)
result = map_on_cluster(
self.parallel_profile, apply_scorer,
[self.scorer] * current_portion, state_dict_array,
[self.base_estimator] * current_portion,
[X] * current_portion, [y] * current_portion,
[sample_weight] * current_portion)
assert len(
result
) == current_portion, "The length of result is very strange"
for state_indices, state_dict, (status, score) in zip(
state_indices_array, state_dict_array, result):
params = ", ".join(
[k + '=' + str(v) for k, v in state_dict.items()])
if status != 'success':
message = 'Fail during training on the node \nException {exc}\n Parameters {params}'
self._log(message.format(exc=score, params=params),
level=40)
else:
self.params_generator.add_result(state_indices, score)
self._log("{}: {}".format(score, params))
self.evaluations_done += current_portion
print("%i evaluations done" % self.evaluations_done)
return self
def _ipython_map(func, iterable, cfg):
import IPython
if IPython.version_info[0] < 4:
from IPython.parallel import Client
else:
from ipyparallel import Client
rc = Client()
rc[:].clear()
### Make modules for all dependencies on the engines
for dep in cfg['dependencies']:
mod_name = os.path.splitext(
os.path.basename(dep)
)[0]
with open(dep) as f:
code = f.read()
code = code.encode('string_escape')
rc[:].execute(
"""
import imp
import sys
_mod = imp.new_module('{mod_name}')
sys.modules['{mod_name}'] = _mod
exec '''{code}''' in _mod.__dict__
del _mod
""".format(code=code, mod_name=mod_name),
block=True
)
### Make sure all definitions surrounding the func are present on
### the engines (evaluate the code from the file of the func)
fname = inspect.getfile(func)
with open(fname) as f:
code = f.read()
logger.info("IPython engine IDs: {}".format(rc.ids))
## Need to escape all ' and " in order to embed the code into
## execute string
# code = code.replace("\'", "\\\'")
# code = code.replace("\"", "\\\"")
code = code.encode('string_escape')
## The trick with `exec in {}' is done because we want to avoid
## executing `__main__'
rc[:].execute(
"""
_tmp_dict = dict()
exec '''{code}''' in _tmp_dict
globals().update(_tmp_dict)
del _tmp_dict
""".format(code=code),
block=True
)
# status.wait()
# res = rc[:].apply(dir)
# print(res.get())
wrap = _FuncWrap(func)
pool = rc.load_balanced_view()
results = []
for args in iterable:
results.append( pool.apply_async(wrap, args) )
for result in results:
yield result.get()
def load_client():
global client, view
client = Client()
view = client.load_balanced_view()
client.block = False
client[:].use_dill()
def runScan(options):
import os, sys, pickle
import numpy as np
scan_settings = pickle.load(
open(
os.path.join(user.scans_dir, options.NAME,
'OscFit_ScanSettings.pckl')))
print '\nLLH scan: Running with the following fit settings'
for one_key in scan_settings['fit_settings']:
print '\t', one_key, '\t', scan_settings['fit_settings'][one_key]
total_jobs = scan_settings['dm31_map'].size
print '\nLLH scan: Total jobs ', total_jobs
job_script = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'oscFit_oneJob_mctest.py')
if len(options.JOBS) == 0:
job_range = range(0, total_jobs)
else:
job_range = np.array(options.JOBS, dtype=int) - 1
if options.TEST:
print 'LLH scan: Running first point as a test!'
print '\tYou will be asked if you wish to continue before moving on'
os.system(' '.join(['python', job_script, '1', '1', options.NAME]))
print 'LLH scan: Continue execution? ... '
raw_input()
if options.MODE == 'uge_farm':
#print job_script
farm_multiplicity = 1
if len(options.JOBS) == 0:
total_jobs = np.ceil(total_jobs * 1. / farm_multiplicity)
job_array = '1-' + "%i" % total_jobs
qsub_line = ' '.join([
'qsub -t', job_array, './oscFit_farmScript.sh', job_script,
"%i" % farm_multiplicity, options.NAME
])
print qsub_line
os.system(qsub_line)
else:
for job_array in options.JOBS:
qsub_line = ' '.join([
'qsub -t', job_array, './oscFit_farmScript.sh', job_script,
"1", options.NAME
])
print qsub_line
os.system(qsub_line)
elif options.MODE == 'iparallel':
from IPython.parallel import Client, interactive
import iparallel
rc = Client(profile='sge')
lview = rc.load_balanced_view()
result = lview.map_async(doOscFit, [options.NAME] * len(job_range),
job_range)
iparallel.waitOn(result)
elif options.MODE == 'local':
import os
for i in job_range:
os.system(' '.join(
['python', job_script,
"%i" % i, '1', options.NAME]))
print 'Finished!'
grid = ns.grid
partition = ns.partition
Lx = ns.Lx
Ly = ns.Ly
c = ns.c
tstop = ns.tstop
if ns.save:
user_action = wave_saver
else:
user_action = None
num_cells = 1.0*(grid[0]-1)*(grid[1]-1)
final_test = True
# create the Client
rc = Client(profile=ns.profile)
num_procs = len(rc.ids)
if partition is None:
partition = [1,num_procs]
assert partition[0]*partition[1] == num_procs, "can't map partition %s to %i engines"%(partition, num_procs)
view = rc[:]
print("Running %s system on %s processes until %f" % (grid, partition, tstop))
# functions defining initial/boundary/source conditions
def I(x,y):
from numpy import exp
return 1.5*exp(-100*((x-0.5)**2+(y-0.5)**2))
def f(x,y,t):
del (grid)
except:
pass
return 0
md = '/lcrc/group/earthscience/radar/nexrad/chicago_floods/'
idir = md
filelist = os.listdir(md)
good_files = []
for fl in filelist:
if 'KLOT' in fl:
good_files.append(idir + fl)
good_files.sort()
t1 = time()
My_Cluster = Client()
My_View = My_Cluster[:]
print My_View
print len(My_View)
#Turn off blocking so all engines can work async
My_View.block = False
#on all engines do an import of Py-ART
My_View.execute('import matplotlib')
My_View.execute('matplotlib.use("agg")')
#Map the code and input to all workers
result = My_View.map_async(do_grid_map_gates_to_grid, good_files)
#Reduce the result to get a list of output
def client_wait(self, client, jobs=None, timeout=-1):
"""my wait wrapper, sets a default finite timeout to avoid hangs"""
if timeout < 0:
timeout = self.timeout
return Client.wait(client, jobs, timeout)
from IPython.parallel import Client
c = Client(profile='mpi')
view = c[:]
view.activate() # enable magics
view.run('psum.py')
view.scatter('a',np.arange(16,dtype='float'))
view['a']
%px totalsum = psum(a)
view['totalsum']
# <codecell>
strike_vals = np.linspace(min_strike, max_strike, n_strikes)
sigma_vals = np.linspace(min_sigma, max_sigma, n_sigmas)
# <markdowncell>
# ## Parallel computation across strike prices and volatilities
# <markdowncell>
# The Client is used to setup the calculation and works with all engines.
# <codecell>
c = Client(profile=cluster_profile)
# <markdowncell>
# A LoadBalancedView is an interface to the engines that provides dynamic load
# balancing at the expense of not knowing which engine will execute the code.
# <codecell>
view = c.load_balanced_view()
# <codecell>
print("Strike prices: ", strike_vals)
print("Volatilities: ", sigma_vals)
def proc_data(data_folder,
h5name,
multiproc=False,
chunk_size=4,
filetype='aia',
reffile=None,
fittype=None,
calfile=None,
picts=False,
**kwargs):
if filetype == 'aia':
GcmsObj = gcf.AiaFile
ends = ('CDF', 'AIA', 'cdf', 'aia')
files = os.listdir(data_folder)
files = [f for f in files if f.endswith(ends)]
files = [os.path.join(data_folder, f) for f in files]
ref = None
if reffile:
if reffile.endswith(('txt', 'TXT')):
ref = gcr.TxtReference(reffile, **kwargs)
fit = None
if fittype:
if fittype.lower() == 'nnls':
fit = gcfit.Nnls(**kwargs)
h5 = gcd.GcmsStore(h5name, **kwargs)
if multiproc:
try:
client = Client()
except:
error = "ERROR! You do not have an IPython Cluster running.\n\n"
error += "Start cluster with: ipcluster start -n # &\n"
error += "Where # == the number of processors.\n\n"
error += "Stop cluster with: ipcluster stop"
print(error)
h5.close()
return
dview = client[:]
dview.block = True
dview['ref'] = ref
dview['fit'] = fit
dview['GcmsObj'] = GcmsObj
chunk_size = len(dview)
# Chunk the data so lots of data files aren't opened in memory.
for chunk in _chunker(files, chunk_size):
if multiproc:
datafiles = dview.map_sync(_proc_file,
[(i, kwargs) for i in chunk])
else:
datafiles = [GcmsObj(f, **kwargs) for f in chunk]
if ref:
ref(datafiles)
if fit:
fit(datafiles)
h5.append_gcms(datafiles)
if calfile:
cal = gcc.Calibrate(h5, **kwargs)
cal.curvegen(calfile, picts=picts, **kwargs)
cal.datagen(picts=picts, **kwargs)
h5.compress()
class Bakapara:
"""A "bakapara" client for IPython cluster.
Args: identical to IPython.parallel.Client
"""
def __init__(self, **args):
self.rc = Client(**args)
self.lview = None
self.ar = None
self.jobs = None
self.indices = None
self.finished = set()
# Obtain the host names and PIDs of engines.
self.pids = self.rc[:].apply(os.getpid).get_dict()
self.hosts = self.rc[:].apply(socket.getfqdn).get_dict()
# Change the working directory of each engine.
self.rc[:].apply(os.chdir, os.getcwd())
def run(self, jobs, targets=None):
"""Runs the jobs on the cluster.
Args:
jobs (list): list of dictionary objects describing the jobs.
targets (int, list of ints, 'all', or None): the engine(s) on which the jobs will run.
Returns:
bool: True if successful, False otherwise (e.g., jobs are running).
"""
if self.ar is not None and not self.ar.ready():
return False
self.lview = self.rc.load_balanced_view(targets)
self.ar = self.lview.map_async(runjob, jobs)
self.jobs = jobs
self.indices = dict([(k, v) for v, k in enumerate(self.ar.msg_ids)])
self.finished = set()
return True
def wait(self, timeout=1e-3):
"""Waits for the jobs to complete and writes job results.
Args:
timeout (float): a time in seconds, after which to give up.
"""
if self.ar is None:
return
# Find finished msg_ids.
pending = set(self.ar.msg_ids)
try:
self.rc.wait(pending, timeout)
except TimeoutError:
pass
finished = pending.difference(self.rc.outstanding)
# Overwrite the results in the job array.
for msg_id in finished:
i = self.indices[msg_id]
if i in self.finished:
continue
job = self.jobs[i]
meta = self.rc.metadata[msg_id]
result = self.rc.results[msg_id][0]
for key in ('submitted', 'started', 'completed', 'received'):
result[key] = meta[key].isoformat()
for key in ('engine_id', 'pyerr', 'pyout', 'status', 'msg_id'):
result[key] = meta[key]
result['elapsed'] = str(meta['completed'] - meta['started'])
result['host'] = self.hosts[meta['engine_id']]
job['result'] = result
self.finished.add(i)
def ready():
"""Returns whether the jobs have completed."""
return self.ar is not None or self.ar.ready()
def successful():
"""Returns whether the jobs completed without raising an exception.
Raises:
AssertionError: the result is not ready.
"""
return self.ar is not None and self.ar.successful()
def abort(self, **args):
"""Aborts jobs.
Args: identical to IPython.parallel.client.view.LoadBalancedView.abort()
"""
if self.lview is not None:
self.lview.abort()
def interrupt(self):
"""Sends SIGINT signal to engines (experimental).
http://mail.scipy.org/pipermail/ipython-dev/2014-March/013426.html
"""
self.abort()
for i in self.rc.ids:
host = self.hosts[i]
pid = self.pids[i]
if host == socket.getfqdn():
os.kill(pid, signal.SIGINT)
else:
os.system('ssh {} kill -INT {}'.format(host, pid))
def shutdown(self, **args):
"""Terminates one or more engine processes, optionally including the hub.
Args: identical to IPython.parallel.Client.shutdown
"""
if self.lview is None:
return False
self.lview.shutdown(**args)
def status(self, interval=1., timeout=-1, fo=sys.stdout):
"""Waits for the jobs, printing progress at regular intervals
Args:
interval (float): a time in seconds, after which to print the progress.
timeout (float): a time in seconds, after which to give up waiting.
fo (file): a file object to which the progress is printed.
"""
if self.ar is None:
return
if timeout is None:
timeout = -1
# Make sure to write the job results into the job objects.
self.wait(1e-3)
tic = time.time()
while not self.ar.ready() and (timeout < 0
or time.time() - tic <= timeout):
self.wait(interval)
clear_output(wait=True)
dt = datetime.timedelta(seconds=self.ar.elapsed)
fo.write('{}/{} tasks finished after {}'.format(
self.ar.progress, len(self.ar), str(dt)))
fo.flush()
else:
fo.write('\n')
dt = datetime.timedelta(seconds=self.ar.elapsed)
clear_output(wait=True)
fo.write('{} tasks completed in {}\n'.format(len(self.ar), str(dt)))
fo.flush()
def __len__(self):
"""Returns the number of engines."""
return len(self.rc)
from IPython.parallel import Client
from numpy import array, savez, percentile, nan
from arch.compat.python import range, lmap
# Time in seconds to sleep before checking if ready
SLEEP = 10
# Number of repetitions
EX_NUM = 500
# Number of simulations per exercise
EX_SIZE = 200000
# Approximately controls memory use, in MiB
MAX_MEMORY_SIZE = 100
rc = Client()
dview = rc.direct_view()
with dview.sync_imports():
from numpy import ones, vstack, arange, cumsum, sum, dot, zeros
from numpy.random import RandomState, seed, random_integers
from numpy.linalg import pinv
def clear_cache(client, view):
"""Cache-clearing function from mailing list"""
assert not rc.outstanding, "don't clear history when tasks are outstanding"
client.purge_results('all') # clears controller
client.results.clear()
client.metadata.clear()
view.results.clear()
client.history = []