def main(): parser = OptionParser() parser.set_defaults(n=100) parser.set_defaults(tmin=1e-3) parser.set_defaults(tmax=1) parser.set_defaults(profile='default') parser.add_option("-n", type='int', dest='n', help='the number of tasks to run') parser.add_option("-t", type='float', dest='tmin', help='the minimum task length in seconds') parser.add_option("-T", type='float', dest='tmax', help='the maximum task length in seconds') parser.add_option("-p", '--profile', type='str', dest='profile', help="the cluster profile [default: 'default']") (opts, args) = parser.parse_args() assert opts.tmax >= opts.tmin, "tmax must not be smaller than tmin" rc = Client() view = rc.load_balanced_view() print view rc.block = True nengines = len(rc.ids) with rc[:].sync_imports(): from IPython.utils.timing import time # the jobs should take a random time within a range times = [ random.random() * (opts.tmax - opts.tmin) + opts.tmin for i in range(opts.n) ] stime = sum(times) print "executing %i tasks, totalling %.1f secs on %i engines" % ( opts.n, stime, nengines) time.sleep(1) start = time.time() amr = view.map(time.sleep, times) amr.get() stop = time.time() ptime = stop - start scale = stime / ptime print "executed %.1f secs in %.1f secs" % (stime, ptime) print "%.3fx parallel performance on %i engines" % (scale, nengines) print "%.1f%% of theoretical max" % (100 * scale / nengines)
def main(): parser = OptionParser() parser.set_defaults(n=100) parser.set_defaults(tmin=1) parser.set_defaults(tmax=60) parser.set_defaults(controller='localhost') parser.set_defaults(meport=10105) parser.set_defaults(tport=10113) parser.add_option("-n", type='int', dest='n', help='the number of tasks to run') parser.add_option("-t", type='float', dest='tmin', help='the minimum task length in seconds') parser.add_option("-T", type='float', dest='tmax', help='the maximum task length in seconds') parser.add_option("-c", type='string', dest='controller', help='the address of the controller') parser.add_option("-p", type='int', dest='meport', help="the port on which the controller listens for the MultiEngine/RemoteController client") parser.add_option("-P", type='int', dest='tport', help="the port on which the controller listens for the TaskClient client") (opts, args) = parser.parse_args() assert opts.tmax >= opts.tmin, "tmax must not be smaller than tmin" rc = client.MultiEngineClient() tc = client.TaskClient() print(tc.task_controller) rc.block=True nengines = len(rc.get_ids()) rc.execute('from IPython.utils.timing import time') # the jobs should take a random time within a range times = [random.random()*(opts.tmax-opts.tmin)+opts.tmin for i in range(opts.n)] tasks = [client.StringTask("time.sleep(%f)"%t) for t in times] stime = sum(times) print("executing %i tasks, totalling %.1f secs on %i engines"%(opts.n, stime, nengines)) time.sleep(1) start = time.time() taskids = [tc.run(t) for t in tasks] tc.barrier(taskids) stop = time.time() ptime = stop-start scale = stime/ptime print("executed %.1f secs in %.1f secs"%(stime, ptime)) print("%.3fx parallel performance on %i engines"%(scale, nengines)) print("%.1f%% of theoretical max"%(100*scale/nengines))
def _add_tasks(self, new_tasks): """ Accounting routine for the parallel tasks, only used by :meth:`.run`. """ if new_tasks is not None: for mid in new_tasks.msg_ids: self.pending.add(mid) self.new_finished = self.pending.difference(self.evaluators.outstanding) self.pending = self.pending.difference(self.new_finished) for tid in self.new_finished: self.finished.append(tid) self.tasks_walltimes[tid] = self.evaluators.get_result(tid).elapsed if time.time() - self.show_last > self.config.show_interval: self.info() self.show_last = time.time()
def main(): parser = OptionParser() parser.set_defaults(n=100) parser.set_defaults(tmin=1e-3) parser.set_defaults(tmax=1) parser.set_defaults(profile='default') parser.add_option("-n", type='int', dest='n', help='the number of tasks to run') parser.add_option("-t", type='float', dest='tmin', help='the minimum task length in seconds') parser.add_option("-T", type='float', dest='tmax', help='the maximum task length in seconds') parser.add_option("-p", '--profile', type='str', dest='profile', help="the cluster profile [default: 'default']") (opts, args) = parser.parse_args() assert opts.tmax >= opts.tmin, "tmax must not be smaller than tmin" rc = Client() view = rc.load_balanced_view() print(view) rc.block = True nengines = len(rc.ids) with rc[:].sync_imports(): from IPython.utils.timing import time # the jobs should take a random time within a range times = [ random.random() * (opts.tmax - opts.tmin) + opts.tmin for i in range(opts.n)] stime = sum(times) print("executing %i tasks, totalling %.1f secs on %i engines" % (opts.n, stime, nengines)) time.sleep(1) start = time.time() amr = view.map(time.sleep, times) amr.get() stop = time.time() ptime = stop - start scale = stime / ptime print("executed %.1f secs in %.1f secs" % (stime, ptime)) print("%.3fx parallel performance on %i engines" % (scale, nengines)) print("%.1f%% of theoretical max" % (100 * scale / nengines))
def __init__(self, problem, parse_args=False): """ @type problem: panobbgo_lib.lib.Problem @param problem: @param parse_args: """ self._name = name = self.__class__.__name__ self.config = config = Config(parse_args) self.logger = logger = config.get_logger('STRAT') self.slogger = config.get_logger('STATS') logger.info("Init of '%s'" % (name)) logger.info("%s" % problem) # aux configuration import pandas as pd # determine width based on console info pd.set_option('display.width', None) pd.set_option('display.precision', 2) # default 7 # statistics self.show_last = 0 # for printing the info line in _add_tasks() self.time_start = time.time() self.tasks_walltimes = {} # task accounting (tasks != points !!!) self.jobs_per_client = 1 # number of tasks per client in 'chunksize' self.pending = set([]) self.new_finished = [] self.finished = [] # init & start everything self._setup_cluster(0, problem) self._threads = [] self._hs = [] import collections self._heuristics = collections.OrderedDict() self._analyzers = collections.OrderedDict() self.problem = problem self.eventbus = EventBus(config) self.results = Results(self) # UI if config.ui_show: from .ui import UI self.ui = UI() self.ui._init_module(self) self.ui.show()
def __init__(self, point, fx, cv_vec=None, cv_norm=None, error=0.0): """ Args: - ``cv``: the constraint violation vector - ``cv_norm``: the norm used to calculate :attr:`.cv`. (see :func:`numpy.linalg.norm`, default ``None`` means 2-norm) """ if point and not isinstance(point, Point): raise ValueError("point must be an instance of lib.Point") self._point = point self._fx = fx self._error = error self._cv_vec = cv_vec self._cv_norm = cv_norm self._time = time.time()
def _run(self): self.eventbus.publish('start', terminate=True) from IPython.parallel import Reference prob_ref = Reference(StrategyBase.PROBLEM_KEY) # see _setup_cluster self._start = time.time() self.eventbus.register(self) self.logger.info("Strategy '%s' started" % self._name) self.loops = 0 while True: self.loops += 1 # execute the actual strategy points = self.execute() # distribute work new_tasks = self.evaluators.map_async(prob_ref, points, chunksize=self.jobs_per_client, ordered=False) # and don't forget, this updates the statistics self._add_tasks(new_tasks) # collect new results for each finished task, hand them over to result DB new_results = [] for msg_id in self.new_finished: list(map(new_results.append, self.evaluators.get_result(msg_id).result)) self.results += new_results self.jobs_per_client = max(1, int(min(self.config.max_eval / 50., 1. / self.avg_time_per_task))) # show heuristic performances after each round # logger.info(' '.join(('%s:%.3f' % (h, h.performance) for h in # heurs))) # stopping criteria if len(self.results) > self.config.max_eval: break # limit loop speed self.evaluators.wait(None, 1e-3) self._cleanup()
def _cleanup(self): """ cleanup + shutdown """ self.eventbus.publish('finished') self._end = time.time() for msg_id in self.evaluators.outstanding: try: self.evaluators.get_result(msg_id).abort() except: pass self.logger.info("Strategy '%s' finished after %.3f [s] and %d loops." % (self._name, self._end - self._start, self.loops)) self.info() self.results.info() [m.__stop__() for m in self.analyzers + self.heuristics] if self.config.ui_show: self.ui.finish() # blocks figure window
def filter(self, record): from IPython.utils.timing import time record.runtime = time.time() - self._start record.where = "%s:%s" % (record.filename[:-3], record.lineno) return True
def __init__(self): logging.Filter.__init__(self) from IPython.utils.timing import time self._start = time.time()
def time_wall(self): """ wall time in seconds """ return time.time() - self.time_start
def __init__(self, **kwargs): self._when = time.time() self._kwargs = kwargs for k, v in list(kwargs.items()): setattr(self, k, v)
def main(): parser = OptionParser() parser.set_defaults(n=100) parser.set_defaults(tmin=1) parser.set_defaults(tmax=60) parser.set_defaults(controller='localhost') parser.set_defaults(meport=10105) parser.set_defaults(tport=10113) parser.add_option("-n", type='int', dest='n', help='the number of tasks to run') parser.add_option("-t", type='float', dest='tmin', help='the minimum task length in seconds') parser.add_option("-T", type='float', dest='tmax', help='the maximum task length in seconds') parser.add_option("-c", type='string', dest='controller', help='the address of the controller') parser.add_option( "-p", type='int', dest='meport', help= "the port on which the controller listens for the MultiEngine/RemoteController client" ) parser.add_option( "-P", type='int', dest='tport', help= "the port on which the controller listens for the TaskClient client") (opts, args) = parser.parse_args() assert opts.tmax >= opts.tmin, "tmax must not be smaller than tmin" rc = client.MultiEngineClient() tc = client.TaskClient() print tc.task_controller rc.block = True nengines = len(rc.get_ids()) rc.execute('from IPython.utils.timing import time') # the jobs should take a random time within a range times = [ random.random() * (opts.tmax - opts.tmin) + opts.tmin for i in range(opts.n) ] tasks = [client.StringTask("time.sleep(%f)" % t) for t in times] stime = sum(times) print "executing %i tasks, totalling %.1f secs on %i engines" % ( opts.n, stime, nengines) time.sleep(1) start = time.time() taskids = [tc.run(t) for t in tasks] tc.barrier(taskids) stop = time.time() ptime = stop - start scale = stime / ptime print "executed %.1f secs in %.1f secs" % (stime, ptime) print "%.3fx parallel performance on %i engines" % (scale, nengines) print "%.1f%% of theoretical max" % (100 * scale / nengines)
best_obj = numpy.infty last_best = best_obj def status(): global last_best s = "*" if last_best != best_obj else " " logger.info( "pend %4d | + %2d | tot: %4d | finished: %4d | gen: %3d | best_obj: %.10f %s" % (queue_size, new, added, nb_finished, nb_generated, best_obj, s) ) last_best = best_obj logger.info("start") start_time = time.time() # pending is the set of jobs we are expecting in each loop pending = set([]) pending_generators = set([]) new_points = [] # collects all returns results = [] allx = dict() # store all x vectors def gen_points(new, DIMSIZE, cur_best_res=None, cur_best_x=None): """ generates @new new points, depends on results and allx """ np = numpy