def launch_ipcluster_dv(profile="default",
targets="all",
block=True,
max_engines=None):
# initiate ipcluster
rc = Client(profile=profile)
# print ipcluster information
n_proc = len(rc.ids)
if targets == "all":
targets = rc.ids
dv = rc.direct_view(targets=targets)
# check number of engines
# print(rc.ids, dv.targets, targets, max_engines)
if max_engines is not None:
if len(dv.targets) > max_engines:
targets = deepcopy(dv.targets)
np.random.shuffle(targets)
targets = targets[:max_engines]
targets.sort()
dv = rc.direct_view(targets=targets)
print("===================================================")
print("@Slam: ipcluster[{}, n_engines={}/{}]".format(
profile, len(dv.targets), n_proc))
print("---------------------------------------------------")
dv.block = block
# import basic modules in ipcluster
dv.execute("import os")
dv.execute("import numpy as np")
dv.execute("from joblib import Parallel, delayed, dump, load")
# print host information
dv.execute("host_names = os.uname()[1]").get()
u_host_names, u_counts = np.unique(dv["host_names"], return_counts=True)
for i in range(len(u_counts)):
print("host: {} x {}".format(u_host_names[i], u_counts[i]))
print("===================================================")
return dv
def parallel_combinatorialSelection(self,
target,
N=2,
n_jobs=None,
c=None):
'''
score_log:
key - name of feature removed
val - expected value of class1 withou the key feature
This works with N=1, where expected vals with each feature
is calculated.
'''
if N > self.num_features:
print('Error: N has to be smaller than num_features')
return
def getExp2(combo):
test_feature = self.data[list(combo)]
jp = JointProbability()
jp.fit(test_feature, target)
contingency = jp.contingency
E = np.sum(contingency[1] * contingency['cond_proba'])
return E
# Set up parallel calculation
if c is None:
c = Client(profile="default")
else:
c = c
feature_combo = list(combinations(self.feature_list_, N))
if n_jobs is None:
vals = c[:].map(getExp2, feature_combo)
else:
vals = c[:n_jobs].map(getExp2, feature_combo)
c.wait(vals)
expected_vals = dict(zip(feature_combo, vals.get()))
self.best_expected_value_ = np.max(vals.get())
best_ind = np.argmax(vals.get())
self.best_combination_ = feature_combo[best_ind]
return expected_vals
def __init__(self, ensemble_controller=None, ensemble_mode=None, **specs):
SurveyEnsemble.__init__(self, **specs)
# allow bail-out
if ensemble_mode and 'init-only' in ensemble_mode:
self.vprint("SurveyEnsemble: initialize-only mode")
return
self.verb = specs.get('verbose', True)
# specify the cluster
if ensemble_controller:
if '.json' in ensemble_controller:
arglist = dict(url_file=ensemble_controller)
else:
arglist = dict(profile=ensemble_controller)
else:
arglist = dict()
# access the cluster
self.rc = Client(**arglist)
self.dview = self.rc[:]
self.dview.block = True
# these are the imports needed by the run_one()
with self.dview.sync_imports():
import EXOSIMS, EXOSIMS.util.get_module, \
time, os, os.path, random, cPickle, gzip, traceback
if specs.has_key('logger'):
specs.pop('logger')
if specs.has_key('seed'):
specs.pop('seed')
self.dview.push(dict(specs=specs))
res = self.dview.execute(
"SS = EXOSIMS.util.get_module.get_module_from_specs" +
"(specs, 'SurveySimulation')(**specs)")
self.vprint("Created SurveySimulation objects on %d engines." %
len(self.rc.ids))
# optionally print stdout of each engine's activity
if False:
for row, id in zip(res.stdout, res.engine_id):
print(''.join(
['[#%d] %s\n' % (id, line) for line in row.split('\n')]))
self.lview = self.rc.load_balanced_view()
def run_ipyparallel(func, tasks, *args, **kwargs):
from ipyparallel import Client
profile = kwargs.get("profile", get_conda_env())
url_file = kwargs.get("url_file", None)
if not profile and url_file:
raise ValueError(
"you must create a ipython profile first, try using 'fxdayu create_profile' in commands"
)
client = Client(url_file=url_file, profile=profile)
_dview = client[:]
_lview = client.load_balanced_view()
delayed = [
_lview.apply_async(func, task, *args, **kwargs) for task in tasks
]
_lview.wait(delayed)
results = [delayed.get() for delayed in delayed]
return results
def variables(dx):
"""
Write a function that accepts a dictionary of variables. Create a Client
object, a Direct View, and distribute the variables. Then, pull the
variables in both a blocking and non-blocking format.
:param dx: Dictionary of variables
:return: (list of results, list of ASyncResult objects)
"""
client = Client() #initialize the client
dview = client[:]
dview.push(dx) #push the dictionary
results = []
results_asyncObj = []
for x in dx: #pull all of the info in non-blocking format
results.append(dview[x])
results_asyncObj.append(dview.pull(x))
return (results, results_asyncObj)
def test_ipyparallel_executor():
from ipyparallel import Client
if OPERATING_SYSTEM == "Windows":
import wexpect as expect
else:
import pexpect as expect
child = expect.spawn("ipcluster start -n 1")
child.expect("Engines appear to have started successfully", timeout=35)
ipyparallel_executor = Client()
learner = Learner1D(linear, (-1, 1))
BlockingRunner(learner, trivial_goal, executor=ipyparallel_executor)
assert learner.npoints > 0
if not child.terminate(force=True):
raise RuntimeError("Could not stop ipcluster")
def start_server(slurm_script: str = None, ipcluster: str = "ipcluster", ncpus: int = None) -> None:
"""
programmatically start the ipyparallel server
Args:
ncpus: int
number of processors
ipcluster : str
ipcluster binary file name; requires 4 path separators on Windows. ipcluster="C:\\\\Anaconda3\\\\Scripts\\\\ipcluster.exe"
Default: "ipcluster"
"""
logger.info("Starting cluster...")
if ncpus is None:
ncpus = psutil.cpu_count()
if slurm_script is None:
if ipcluster == "ipcluster":
subprocess.Popen("ipcluster start -n {0}".format(ncpus), shell=True, close_fds=(os.name != 'nt'))
else:
subprocess.Popen(shlex.split("{0} start -n {1}".format(ipcluster, ncpus)),
shell=True,
close_fds=(os.name != 'nt'))
time.sleep(1.5)
# Check that all processes have started
client = ipyparallel.Client()
time.sleep(1.5)
while len(client) < ncpus:
sys.stdout.write(".") # Give some visual feedback of things starting
sys.stdout.flush() # (de-buffered)
time.sleep(0.5)
logger.debug('Making sure everything is up and running')
client.direct_view().execute('__a=1', block=True) # when done on all, we're set to go
else:
shell_source(slurm_script)
pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE']
logger.debug([pdir, profile])
c = Client(ipython_dir=pdir, profile=profile)
ee = c[:]
ne = len(ee)
logger.info(('Running on %d engines.' % (ne)))
c.close()
sys.stdout.write("start_server: done\n")
def main():
parser = OptionParser()
parser.set_defaults(n=100)
parser.set_defaults(tmin=1e-3)
parser.set_defaults(tmax=1)
parser.set_defaults(profile='default')
parser.add_option("-n", type='int', dest='n',
help='the number of tasks to run')
parser.add_option("-t", type='float', dest='tmin',
help='the minimum task length in seconds')
parser.add_option("-T", type='float', dest='tmax',
help='the maximum task length in seconds')
parser.add_option("-p", '--profile', type='str', dest='profile',
help="the cluster profile [default: 'default']")
(opts, args) = parser.parse_args()
assert opts.tmax >= opts.tmin, "tmax must not be smaller than tmin"
rc = Client()
view = rc.load_balanced_view()
print(view)
rc.block=True
nengines = len(rc.ids)
with rc[:].sync_imports():
from IPython.utils.timing import time
# the jobs should take a random time within a range
times = [random.random()*(opts.tmax-opts.tmin)+opts.tmin for i in range(opts.n)]
stime = sum(times)
print("executing %i tasks, totalling %.1f secs on %i engines"%(opts.n, stime, nengines))
time.sleep(1)
start = time.time()
amr = view.map(time.sleep, times)
amr.get()
stop = time.time()
ptime = stop-start
scale = stime/ptime
print("executed %.1f secs in %.1f secs"%(stime, ptime))
print("%.3fx parallel performance on %i engines"%(scale, nengines))
print("%.1f%% of theoretical max"%(100*scale/nengines))
def start_server(slurm_script=None, ipcluster="ipcluster", ncpus = None):
'''
programmatically start the ipyparallel server
Parameters
----------
ncpus: int
number of processors
ipcluster : str
ipcluster binary file name; requires 4 path separators on Windows. ipcluster="C:\\\\Anaconda2\\\\Scripts\\\\ipcluster.exe"
Default: "ipcluster"
'''
sys.stdout.write("Starting cluster...")
sys.stdout.flush()
if ncpus is None:
ncpus=psutil.cpu_count()
if slurm_script is None:
if ipcluster == "ipcluster":
p1 = subprocess.Popen("ipcluster start -n {0}".format(ncpus), shell=True, close_fds=(os.name != 'nt'))
else:
p1 = subprocess.Popen(shlex.split("{0} start -n {1}".format(ipcluster, ncpus)), shell=True, close_fds=(os.name != 'nt'))
#
# Check that all processes have started
time.sleep(10)
client = ipyparallel.Client()
while len(client) < ncpus:
sys.stdout.write(".")
sys.stdout.flush()
client.close()
time.sleep(1)
client = ipyparallel.Client()
client.close()
else:
shell_source(slurm_script)
pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE']
c = Client(ipython_dir=pdir, profile=profile)
ee = c[:]
ne = len(ee)
print(('Running on %d engines.' % (ne)))
c.close()
sys.stdout.write(" done\n")
def analyzer_map(parallel=True):
'''returns configured bluepyopt.optimisations.DEAPOptimisation'''
if parallel:
from ipyparallel import Client
rc = Client(profile=os.getenv('IPYTHON_PROFILE'))
logger.debug('Using ipyparallel with %d engines', len(rc))
lview = rc.load_balanced_view()
def mapper(func, it):
ret = lview.map_sync(func, it)
return ret
map_function = mapper
else:
map_function = None
return map_function
def run(dest, results_path, parallel, seed):
np.random.seed(seed)
# load the responses
old_store_pth = os.path.abspath(os.path.join(
results_path, 'model_fall_responses_raw.h5'))
old_store = pd.HDFStore(old_store_pth, mode='r')
# open up the store for saving
store = pd.HDFStore(dest, mode='w')
# create the ipython parallel client
if parallel:
rc = Client()
lview = rc.load_balanced_view()
# start the tasks
results = []
for key in old_store.keys():
if key.split('/')[-1] == 'param_ref':
store.append(key, old_store[key])
continue
args = [key, old_store[key]]
if parallel:
result = lview.apply(model_fall_responses, *args)
else:
result = model_fall_responses(*args)
results.append(result)
# collect and save results
while len(results) > 0:
result = results.pop(0)
if parallel:
key, responses = result.get()
result.display_outputs()
else:
key, responses = result
store.append(key, responses)
store.close()
old_store.close()
def get_instance() -> Client:
"""
Get the singleton instance
Arguments
---------
None
Returns
-------
Client
The singleton client
"""
SingletonClient.__instance.close()
SingletonClient.__instance = Client()
return SingletonClient.__instance
def __init__(self, mod, engine='multiproc'):
"""
Instantiate the parallel scanner class with a PySCeS model instance
and an optional 'engine' argument specifying the parallel engine:
'multiproc' -- multiprocessing (default)
'ipcluster' -- IPython cluster
"""
self.engine = engine
if engine == 'multiproc':
print('parallel engine: multiproc')
elif engine == 'ipcluster':
print('parallel engine: ipcluster')
try:
from ipyparallel import Client
except ImportError as ex:
print('\n', ex)
raise ImportError(
'PARSCANNER: Requires IPython and ipyparallel version >=4.0 (http://ipython.org) and 0MQ (http://zero.mq).'
)
try:
rc = Client()
self.rc = rc
except OSError as ex:
raise OSError(
str(ex) +
'\nPARSCANNER: Requires a running IPython cluster. See "ipcluster --help".\n'
)
dv = rc[:] # direct view
lv = rc.load_balanced_view()
self.dv = dv
self.lv = lv
dv.execute('from pysces.PyscesParScan import Analyze, setModValue')
else:
raise UserWarning(engine + " is not a valid parallel engine!")
self.GenDict = {}
self.GenOrder = []
self.ScanSpace = []
self.mod = mod
self.SteadyStateResults = []
self.UserOutputList = []
self.UserOutputResults = []
self.scanT = TimerBox()
def extract_rois_patch(file_name, d1, d2, rf=5, stride=5):
idx_flat, idx_2d = extract_patch_coordinates(d1, d2, rf=rf, stride=stride)
perctl = 95
n_components = 2
tol = 1e-6
max_iter = 5000
args_in = []
for id_f, id_2d in zip(idx_flat, idx_2d):
args_in.append((file_name, id_f, id_2d[0].shape, perctl, n_components,
tol, max_iter))
st = time.time()
print len(idx_flat)
try:
if 1:
c = Client()
dview = c[:]
file_res = dview.map_sync(nmf_patches, args_in)
else:
file_res = map(nmf_patches, args_in)
finally:
dview.results.clear()
c.purge_results('all')
c.purge_everything()
c.close()
print time.time() - st
A1 = lil_matrix((d1 * d2, len(file_res)))
C1 = []
A2 = lil_matrix((d1 * d2, len(file_res)))
C2 = []
for count, f in enumerate(file_res):
idx_, flt, ca, d = f
#flt,ca,_=cse.order_components(coo_matrix(flt),ca)
A1[idx_, count] = flt[:, 0][:, np.newaxis]
A2[idx_, count] = flt[:, 1][:, np.newaxis]
C1.append(ca[0, :])
C2.append(ca[1, :])
# pl.imshow(np.reshape(flt[:,0],d,order='F'),vmax=10)
# pl.pause(.1)
return A1, A2, C1, C2
def __init__(self, filename='./data/2_sslp/sslp_5_25_3_mymod'):
super(Elimination_TwoStage_SMPS_InnerProblem, self).__init__(filename)
# TODO add a check that this is not run unless the problem is purely integer in the first stage.
# introduce a list to record the points we want to cut
self.points_to_eliminate = [] # [[0,1,1],[0,0,0]]
self.best_current_primal_value = []
self.with_elimination = 0
self.stop_dual_iterations = 0
self.seen_x_ks = [
] # memory of inner solutions seen, used in the 'frequent' variant of elimination procedure
self.time_generating = 0
self.time_writing = 0
# write local optimization problems for each scenario; from iteration to iteration, the only thing that will
# change is the objective (and the cutting planes added to eliminate solutions).
for scenario, scenario_label in enumerate(self.stoch_model.scenarios):
scenario_program, x, y = self._construct_initial_local_problem(
self.stoch_model.scenarios[scenario_label])
scenario_program.write(HOME_PATH +
'/sc_model_{}.mps'.format(str(scenario)))
# For the distributed implementation, direct views of the engines; this requires ipcluster to be started
try:
self.rc = Client()
self.dview = self.rc[:]
self.dview.execute('import gurobipy as gb')
@self.dview.parallel
def temp(scenario_n, scenario_lambda_k, points_to_eliminate):
return _external_optimization_of_single_scenario_distributed(
scenario_n, scenario_lambda_k, points_to_eliminate)
self._optimization_of_single_scenario_distributed = temp
except:
print(
'WARNING. Could not connect to the engines. This is likely due to ipcluster not being started.'
)
print('Distributed computations will not be available.')
self._optimization_of_single_scenario_distributed = 0
def test_hubresult_timestamps(self):
self.minimum_engines(4)
v = self.client[:]
ar = v.apply_async(time.sleep, 0.25)
ar.get(2)
rc2 = Client(profile='iptest')
# must have try/finally to close second Client, otherwise
# will have dangling sockets causing problems
try:
time.sleep(0.25)
hr = rc2.get_result(ar.msg_ids)
self.assertTrue(hr.elapsed > 0.,
"got bad elapsed: %s" % hr.elapsed)
hr.get(1)
self.assertTrue(
hr.wall_time < ar.wall_time + 0.2,
"got bad wall_time: %s > %s" % (hr.wall_time, ar.wall_time))
self.assertEqual(hr.serial_time, ar.serial_time)
finally:
rc2.close()
def start(self):
self.controller.profile = self.label
self.controller.ipython_dir = self.run_dir
if self.engine_dir is None:
parent, child = pathlib.Path(self.run_dir).parts[-2:]
self.engine_dir = os.path.join(parent, child)
self.controller.start()
self.engine_file = self.controller.engine_file
with wait_for_file(self.controller.client_file, seconds=120):
logger.debug("Waiting for {0}".format(self.controller.client_file))
if not os.path.exists(self.controller.client_file):
raise Exception("Controller client file is missing at {0}".format(
self.controller.client_file))
command_composer = self.compose_launch_cmd
self.executor = Client(url_file=self.controller.client_file)
if self.container_image:
command_composer = self.compose_containerized_launch_cmd
logger.info("Launching IPP with Docker image: {0}".format(
self.container_image))
self.launch_cmd = command_composer(self.engine_file, self.engine_dir,
self.container_image)
self.engines = []
self._scaling_enabled = self.provider.scaling_enabled
logger.debug("Starting IPyParallelExecutor with provider:\n%s",
self.provider)
if hasattr(self.provider, 'init_blocks'):
try:
self.scale_out(blocks=self.provider.init_blocks)
except Exception as e:
logger.error("Scaling out failed: %s" % e)
raise e
self.lb_view = self.executor.load_balanced_view()
logger.debug("Starting executor")
def setupParallel(self):
self.SlurmID = os.getenv('SLURM_JOBID')
if (self.SlurmID is None):
self.SlurmID = self.nextRunID()
else:
self.SlurmID = int(self.SlurmID)
self.logFile.write("Using SlurmID: " + str(self.SlurmID))
if (os.getenv('IPYTHON_PROFILE') is not None):
self.logFile.write('Creating ipyparallel client\n')
from ipyparallel import Client
#self.rc = Client(profile=os.getenv('IPYTHON_PROFILE'),
# # sshserver='127.0.0.1',
# debug=False)
ufile = os.getenv('IPYTHONDIR') + "/profile_" \
+ os.getenv('IPYTHON_PROFILE') \
+ "/security/ipcontroller-client.json"
self.rc = Client(url_file=ufile, timeout=120, debug=False)
self.logFile.write('Client IDs: ' + str(self.rc.ids))
# http://davidmasad.com/blog/simulation-with-ipyparallel/
# http://people.duke.edu/~ccc14/sta-663-2016/19C_IPyParallel.html
self.dView = self.rc.direct_view(
targets='all') # rc[:] # Direct view into clients
self.lbView = self.rc.load_balanced_view(targets='all')
# Define nc globally
# self.dView.execute("nc = None",block=True)
else:
self.logFile.write(
"No IPYTHON_PROFILE enviroment variable set, running in serial"
)
self.dView = None
self.lbView = None
self.rc = None
def _perform_evolution(self, algo, pop):
try:
from ipyparallel import Client
# Create client
rc = Client()
# Create Load-balanced view
lbview = rc.load_balanced_view()
# Run the task
lbview.block = True
ar = lbview.apply(_maptask_target, args=(algo, pop))
# Get retval
retval = ar.get()
if isinstance(retval, BaseException):
raise retval
return retval
except BaseException as e:
print('Exception caught during evolution:')
print(e)
raise RuntimeError()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('directory',
help="Provide the directory of the HDF files "
"that shall be converted to csv here.")
args = parser.parse_args()
root = os.path.abspath(args.directory)
fnames = glob.glob(os.path.join(root, '*.hdf'))
logging.info('Found %i files to convert.', len(fnames))
c = Client()
lbview = c.load_balanced_view()
results = lbview.map_async(process_fname, fnames)
# progress display
while not results.ready():
print("{:.1f} %".format(100 * results.progress / len(fnames)))
sys.stdout.flush()
time.sleep(10)
logging.info('Conversion done.')
def test_ipy_island(self):
from PyGMO import ipy_island, algorithm, problem
try:
from ipyparallel import Client
rc = Client()
if len(rc.ids) == 0:
raise RuntimeError()
return
except BaseException as e:
print(
'\nThere is a problem with parallel IPython setup. The error message is:')
print(e)
print('Tests for ipy_island will not be run.')
return
isl_type = ipy_island
algo_list = [algorithm.py_example(1), algorithm.de(5)]
prob_list = [problem.py_example(), problem.dejong(1)]
for algo in algo_list:
for prob in prob_list:
self.__test_impl(isl_type, algo, prob)
def bench_ipp_seq(tasks, workers, task_duration, warmup=True):
from ipyparallel import Client
rc = Client()
#dview = rc[:]
dview = rc.load_balanced_view()
if warmup:
dview.map_sync(sleep_worker, [task_duration for i in range(tasks)])
dview.block = True
def run(tasks):
objs = [
dview.apply_async(sleep_worker, task_duration)
for i in range(tasks)
]
for task in objs:
task.get()
res, elapsed = timeit(run, tasks)
return elapsed
def test_convergence(run_model, chains, saveplots=False):
"""
Uses ipyparallel client to run <chains> model fits, then runs r-hat (gelman-rubin) statistic
on the resulting traces. Finally, the models are concatenated and their posteriors are plotted for assessing
convergence.
"""
from ipyparallel import Client
c = Client()[:] # create the ipython client
jobs = c.map(run_model, range(chains)) # c.map(function, number of CPUs)
models = jobs.get() # run the jobs
# Calculate convergence statistics
from kabuki.analyze import gelman_rubin
rhat = gelman_rubin(models)
# Create a new model that contains all traces concatenated from the individual models
from kabuki.utils import concat_models
combined_model = concat_models(models)
# Plot posteriors
combined_model.plot_posteriors(save=saveplots)
return rhat
def main():
"""Start IO loop that checks every 60 seconds whether the engines are
running, if inactive for two hours they are culled."""
options.define('timeout',
default=900,
help="""Time (in seconds) after which to consider an engine
idle that should be shutdown.""")
options.define(
'interval',
default=60,
help="""Interval (in seconds) at which state should be checked
and culling performed.""")
options.define('profile', default='pbs', help="""Profile name.""")
options.parse_command_line()
kill_running_cullers(profile=options.options.profile)
loop = ioloop.IOLoop.current()
culler = EngineCuller(Client(profile=options.options.profile),
options.options.timeout, options.options.interval)
ioloop.PeriodicCallback(culler.update_state,
options.options.interval * 1000).start()
loop.start()
def run_evolve(self, algo, pop):
"""Evolve population.
This method will evolve the input :class:`~pygmo.population` *pop* using the input
:class:`~pygmo.algorithm` *algo*, and return *algo* and the evolved population. The evolution
task is submitted to the ipyparallel cluster via a global :class:`ipyparallel.LoadBalancedView`
instance initialised either implicitly by the first invocation of this method,
or by an explicit call to the :func:`~pygmo.ipyparallel_island.init_view()` method.
Args:
pop(:class:`~pygmo.population`): the input population
algo(:class:`~pygmo.algorithm`): the input algorithm
Returns:
:class:`tuple`: a tuple of 2 elements containing *algo* (i.e., the :class:`~pygmo.algorithm` object that was used for the evolution) and the evolved :class:`~pygmo.population`
Raises:
unspecified: any exception thrown by the evolution, by the creation of a
:class:`ipyparallel.LoadBalancedView`, or by the sumission of the evolution task
to the ipyparallel cluster
"""
# NOTE: as in the mp_island, we pre-serialize
# the algo and pop, so that we can catch
# serialization errors early.
import pickle
ser_algo_pop = pickle.dumps((algo, pop))
with ipyparallel_island._view_lock:
if ipyparallel_island._view is None:
from ipyparallel import Client
ipyparallel_island._view = Client().load_balanced_view()
ret = ipyparallel_island._view.apply_async(_evolve_func_ipy,
ser_algo_pop)
return pickle.loads(ret.get())
def start_server(ncpus, slurm_script=None):
'''
programmatically start the ipyparallel server
Parameters
----------
ncpus: int
number of processors
'''
sys.stdout.write("Starting cluster...")
sys.stdout.flush()
if slurm_script is None:
subprocess.Popen(["ipcluster start -n {0}".format(ncpus)], shell=True)
while True:
try:
c = ipyparallel.Client()
if len(c) < ncpus:
sys.stdout.write(".")
sys.stdout.flush()
raise ipyparallel.error.TimeoutError
c.close()
break
except (IOError, ipyparallel.error.TimeoutError):
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(1)
else:
shell_source(slurm_script)
from ipyparallel import Client
pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE']
c = Client(ipython_dir=pdir, profile=profile)
ee = c[:]
ne = len(ee)
print 'Running on %d engines.' % (ne)
c.close()
sys.stdout.write(" done\n")
def run_jobs_on_ipythoncluster(worker,
task_list,
shutdown_ipengines_after_done=False):
t0 = time.time()
rc = Client(config.CLUSTER_CLIENT_JSON)
lview = rc.load_balanced_view()
cnt_nodes = len(lview.targets or rc.ids)
print("\t# nodes in use: {}".format(cnt_nodes))
lview.block = False
# move to app path
lview.map(os.chdir, [config.APP_PATH] * cnt_nodes)
print("\t# of tasks: {}".format(len(task_list)))
print("\tsubmitting...", end='')
job = lview.map_async(worker, task_list)
print("done.")
try:
job.wait_interactive()
except KeyboardInterrupt:
#handle "Ctrl-C"
if ask("\nAbort all submitted jobs?") == 'Y':
lview.abort()
print("Aborted, all submitted jobs are cancelled.")
else:
print("Aborted, but your jobs are still running on the cluster.")
return
if len(job.result()) != len(task_list):
print(
"WARNING:\t# of results returned ({}) != # of tasks ({}).".format(
len(job.result()), len(task_list)))
print("\ttotal time: {}".format(timesofar(t0)))
if shutdown_ipengines_after_done:
print("\tshuting down all ipengine nodes...", end='')
lview.shutdown()
print('Done.')
return job.result()
def profile_running(profile):
""" Checks if there is an ipyparallel cluster running with profile
Args:
profile: name of profile
Returns:
is_running: true if found
"""
# TODO: more sophisticated checks
# maybe we can just check to see if the connection files exist?
try:
client = Client(profile=profile)
return True
except OSError as os_err:
print("Caught OSError while attempting to connect to {}: {}.".format(
profile, os_err))
return False
except TimeoutError as timeout_err:
print(
"Caught TimeoutError while attempting to connect to {}: {}".format(
profile, timeout_err))
return False
def Main():
args = parser.parse_args()
cty_type = args.cty
cty_list = args.cty_list
data_path = os.path.join(os.path.dirname(man_opt.__file__), os.pardir,
'assets', 'aggregated_data')
mouse_data_filename = os.path.join(data_path, 'Mouse_class_data.csv')
mouse_datatype_filename = os.path.join(data_path,
'Mouse_class_datatype.csv')
me_ttype_map_path = os.path.join(data_path, 'me_ttype.pkl')
sdk_data_filename = os.path.join(data_path, 'sdk.csv')
sdk_datatype_filename = os.path.join(data_path, 'sdk_datatype.csv')
sdk_data = man_utils.read_csv_with_dtype(sdk_data_filename,
sdk_datatype_filename)
if cty_type == 'ttype':
mouse_data = man_utils.read_csv_with_dtype(mouse_data_filename,
mouse_datatype_filename)
me_ttype_map = utility.load_pickle(me_ttype_map_path)
metype_cluster = mouse_data.loc[
mouse_data.hof_index == 0, ['Cell_id', 'Dendrite_type', 'me_type']]
sdk_me = pd.merge(sdk_data, metype_cluster, how='left',
on='Cell_id').dropna(how='any', subset=['me_type'])
sdk_me['ttype'] = sdk_me['me_type'].apply(lambda x: me_ttype_map[x])
cell_df = sdk_me.loc[sdk_me.ttype.isin(cty_list), ]
elif cty_type == 'Cre_line':
cell_df = sdk_data.loc[sdk_data.line_name.isin(cty_list), ]
cell_ids = cell_df.Cell_id.unique().tolist()
rc = Client(profile=os.getenv('IPYTHON_PROFILE'))
logger.debug('Using ipyparallel with %d engines', len(rc))
lview = rc.load_balanced_view()
lview.map_sync(get_efeatures, cell_ids)
def execute_in_parallel(func, iterable):
"""Use IPyparallel's load_balanced_view to execute in parallel.
Function will create a Client() object, a load_balanced_view and
a notebook widget based progressbar automatically. The processing
will be performed using the `map_async` method of the
load_balanced_view object.
Parameters
----------
func : function
Function to call in parallel.
iterable : iterable
Iterable container that will be used as input for `func`.
Returns
-------
The results object from executing `lbview.map_async`.
"""
c = Client()
lbview = c.load_balanced_view()
results = lbview.map_async(func, iterable)
display_multi_progress(results, iterable)
return results
