def start(self, isMaster, environ={}): if self.started: return logger.debug("start env in %s: %s %s", os.getpid(), isMaster, environ) self.isMaster = isMaster if isMaster: roots = conf.DPARK_WORK_DIR if isinstance(roots, str): roots = roots.split(',') name = '%s-%s-%d' % (time.strftime("%Y%m%d-%H%M%S"), socket.gethostname(), os.getpid()) self.workdir = [os.path.join(root, name) for root in roots] for d in self.workdir: if not os.path.exists(d): try: os.makedirs(d) except OSError: pass self.environ['SERVER_URI'] = 'file://' + self.workdir[0] self.environ['WORKDIR'] = self.workdir self.environ['COMPRESS'] = util.COMPRESS else: self.environ.update(environ) if self.environ['COMPRESS'] != util.COMPRESS: raise Exception("no %s available" % self.environ['COMPRESS']) self.ctx = zmq.Context() from dpark.tracker import TrackerServer, TrackerClient if isMaster: self.trackerServer = TrackerServer() self.trackerServer.start() addr = self.trackerServer.addr env.register('TrackerAddr', addr) else: addr = env.get('TrackerAddr') self.trackerClient = TrackerClient(addr) from dpark.cache import CacheTracker self.cacheTracker = CacheTracker() from dpark.shuffle import LocalFileShuffle, MapOutputTracker LocalFileShuffle.initialize(isMaster) self.mapOutputTracker = MapOutputTracker() from dpark.shuffle import SimpleShuffleFetcher, ParallelShuffleFetcher #self.shuffleFetcher = SimpleShuffleFetcher() self.shuffleFetcher = ParallelShuffleFetcher(2) from dpark.broadcast import start_manager start_manager(isMaster) self.started = True logger.debug("env started")
def start(self, isMaster, environ={}): if self.started: return logger.debug("start env in %s: %s %s", os.getpid(), isMaster, environ) self.isMaster = isMaster if isMaster: roots = conf.DPARK_WORK_DIR if isinstance(roots, str): roots = roots.split(',') name = '%s-%s-%d' % (time.strftime("%Y%m%d-%H%M%S"), socket.gethostname(), os.getpid()) self.workdir = [os.path.join(root, name) for root in roots] try: for d in self.workdir: util.mkdir_p(d) except OSError as e: if environ.get('is_local', False): raise e self.environ['SERVER_URI'] = 'file://' + self.workdir[0] self.environ['WORKDIR'] = self.workdir self.environ['COMPRESS'] = util.COMPRESS else: self.environ.update(environ) if self.environ['COMPRESS'] != util.COMPRESS: raise Exception("no %s available" % self.environ['COMPRESS']) self.ctx = zmq.Context() from dpark.tracker import TrackerServer, TrackerClient if isMaster: self.trackerServer = TrackerServer() self.trackerServer.start() addr = self.trackerServer.addr env.register('TrackerAddr', addr) else: addr = env.get('TrackerAddr') self.trackerClient = TrackerClient(addr) from dpark.cache import CacheTracker self.cacheTracker = CacheTracker() from dpark.shuffle import LocalFileShuffle, MapOutputTracker LocalFileShuffle.initialize(isMaster) self.mapOutputTracker = MapOutputTracker() from dpark.shuffle import ParallelShuffleFetcher self.shuffleFetcher = ParallelShuffleFetcher(2) from dpark.broadcast import start_manager start_manager(isMaster) self.started = True logger.debug("env started")
def start(self, isMaster, environ={}, isLocal=False): if self.started: return logger.debug("start env in %s: %s %s", os.getpid(), isMaster, environ) self.isMaster = isMaster self.isLocal = isLocal if isMaster: roots = conf.DPARK_WORK_DIR if isinstance(roots, str): roots = roots.split(',') if isLocal: root = roots[0] # for local mode if not os.path.exists(root): os.mkdir(root, 0777) os.chmod(root, 0777) # because of umask name = '%s-%s-%d' % (time.strftime("%Y%m%d-%H%M%S"), socket.gethostname(), os.getpid()) self.workdir = [os.path.join(root, name) for root in roots] for d in self.workdir: if not os.path.exists(d): try: os.makedirs(d) except OSError: pass self.environ['WORKDIR'] = self.workdir self.environ['COMPRESS'] = util.COMPRESS else: self.environ.update(environ) if self.environ['COMPRESS'] != util.COMPRESS: raise Exception("no %s available" % self.environ['COMPRESS']) self.ctx = zmq.Context() from dpark.tracker import TrackerServer, TrackerClient if isMaster: self.trackerServer = TrackerServer() self.trackerServer.start() addr = self.trackerServer.addr env.register('TrackerAddr', addr) else: addr = env.get('TrackerAddr') self.trackerClient = TrackerClient(addr) from dpark.cache import CacheTracker, LocalCacheTracker if isLocal: self.cacheTracker = LocalCacheTracker() else: self.cacheTracker = CacheTracker() from dpark.shuffle import LocalFileShuffle, MapOutputTracker, LocalMapOutputTracker LocalFileShuffle.initialize(isMaster) if isLocal: self.mapOutputTracker = LocalMapOutputTracker() else: self.mapOutputTracker = MapOutputTracker() from dpark.shuffle import SimpleShuffleFetcher, ParallelShuffleFetcher #self.shuffleFetcher = SimpleShuffleFetcher() self.shuffleFetcher = ParallelShuffleFetcher(2) from dpark.broadcast import start_manager start_manager(isMaster) self.started = True logger.debug("env started")