def start(self): if self.started: return self.init() env.start(True, isLocal=self.isLocal) self.scheduler.start() self.started = True atexit.register(self.stop) def handler(signm, frame): logger.error("got signal %d, exit now", signm) self.scheduler.shutdown() signal.signal(signal.SIGTERM, handler) signal.signal(signal.SIGHUP, handler) signal.signal(signal.SIGABRT, handler) signal.signal(signal.SIGQUIT, handler) try: from rfoo.utils import rconsole rconsole.spawn_server(locals(), 0) except ImportError: pass
def test(): l = [] for i in range(10): d = zip(range(10000), range(10000)) l.append(sorted_items(d)) hl = heap_merged(l, lambda x,y:x+y) for i in range(10): print i, hl.next() import logging logging.basicConfig(level=logging.INFO) from env import env import cPickle env.start(True) path = LocalFileShuffle.getOutputFile(1, 0, 0) f = open(path, 'w') f.write(cPickle.dumps([('key','value')], -1)) f.close() uri = LocalFileShuffle.getServerUri() env.mapOutputTracker.registerMapOutput(1, 1, 0, uri) fetcher = SimpleShuffleFetcher() def func(k,v): assert k=='key' assert v=='value' fetcher.fetch(1, 0, func) tracker = MapOutputTracker(True) tracker.registerMapOutput(2, 5, 1, uri) assert tracker.getServerUris(2) == [None, uri, None, None, None] ntracker = MapOutputTracker(False) assert ntracker.getServerUris(2) == [None, uri, None, None, None] ntracker.stop() tracker.stop()
def test(): import logging logging.basicConfig(level=logging.INFO) from env import env import cPickle env.start(True) path = LocalFileShuffle.getOutputFile(1, 0, 0) f = open(path, 'w') f.write(cPickle.dumps([('key','value')], -1)) f.close() uri = LocalFileShuffle.getServerUri() env.mapOutputTracker.registerMapOutput(1, 1, 0, uri) fetcher = SimpleShuffleFetcher() def func(k,v): assert k=='key' assert v=='value' fetcher.fetch(1, 0, func) tracker = MapOutputTracker(True) tracker.registerMapOutput(2, 5, 1, uri) assert tracker.getServerUris(2) == [None, uri, None, None, None] ntracker = MapOutputTracker(False) assert ntracker.getServerUris(2) == [None, uri, None, None, None] ntracker.stop() tracker.stop()
def run_task_in_process(task, tid, environ): from env import env env.start(False, environ) logger.debug("run task in process %s %s", task, tid) try: return run_task(task, tid) except KeyboardInterrupt: sys.exit(0)
def start(self): if self.started: return env.start(True, isLocal=self.isLocal) self.scheduler.start() self.started = True atexit.register(self.stop) def handler(signm, frame): logger.error("got signal %d, exit now", signm) self.scheduler.shutdown() signal.signal(signal.SIGTERM, handler) signal.signal(signal.SIGHUP, handler) signal.signal(signal.SIGABRT, handler) signal.signal(signal.SIGQUIT, handler)