def start(self): if self.started: return self.started = True logger.debug("start env in %s", os.getpid()) for d in self.workdir: util.mkdir_p(d) if 'TRACKER_ADDR' not in self.environ: from dpark.tracker import TrackerServer trackerServer = self.trackerServer = TrackerServer() self.trackerServer.start() self.register('TRACKER_ADDR', trackerServer.addr) from dpark.tracker import TrackerClient addr = self.get('TRACKER_ADDR') self.trackerClient = TrackerClient(addr) from dpark.cache import CacheTracker self.cacheTracker = CacheTracker() from dpark.shuffle import MapOutputTracker self.mapOutputTracker = MapOutputTracker() from dpark.shuffle import ParallelShuffleFetcher self.shuffleFetcher = ParallelShuffleFetcher(2) from dpark.broadcast import start_guide_manager, GUIDE_ADDR if GUIDE_ADDR not in self.environ: start_guide_manager() logger.debug("env started")
def start_master(self): if self.master_started: return self.master_started = True logger.debug("start master env in pid %s", os.getpid()) self._start_common() self.register(self.COMPRESS, utils.COMPRESS) dpark_id = '{}-{}'.format(socket.gethostname(), uuid_pkg.uuid4()) logger.info("dpark_id = %s", dpark_id) self.register(self.DPARK_ID, dpark_id) self.workdir.init(dpark_id) self.register(self.SERVER_URI, 'file://' + self.workdir.main) # overwrited when executor starts # start centra servers in scheduler from dpark.tracker import TrackerServer self.trackerServer = TrackerServer() self.trackerServer.start() self.register(self.TRACKER_ADDR, self.trackerServer.addr) from dpark.cache import CacheTrackerServer self.cacheTrackerServer = CacheTrackerServer() from dpark.broadcast import start_guide_manager start_guide_manager() self._start_common() logger.debug("master env started")
def start(self, isMaster, environ={}): if self.started: return logger.debug("start env in %s: %s %s", os.getpid(), isMaster, environ) self.isMaster = isMaster if isMaster: roots = conf.DPARK_WORK_DIR if isinstance(roots, str): roots = roots.split(',') name = '%s-%s-%d' % (time.strftime("%Y%m%d-%H%M%S"), socket.gethostname(), os.getpid()) self.workdir = [os.path.join(root, name) for root in roots] for d in self.workdir: if not os.path.exists(d): try: os.makedirs(d) except OSError: pass self.environ['SERVER_URI'] = 'file://' + self.workdir[0] self.environ['WORKDIR'] = self.workdir self.environ['COMPRESS'] = util.COMPRESS else: self.environ.update(environ) if self.environ['COMPRESS'] != util.COMPRESS: raise Exception("no %s available" % self.environ['COMPRESS']) self.ctx = zmq.Context() from dpark.tracker import TrackerServer, TrackerClient if isMaster: self.trackerServer = TrackerServer() self.trackerServer.start() addr = self.trackerServer.addr env.register('TrackerAddr', addr) else: addr = env.get('TrackerAddr') self.trackerClient = TrackerClient(addr) from dpark.cache import CacheTracker self.cacheTracker = CacheTracker() from dpark.shuffle import LocalFileShuffle, MapOutputTracker LocalFileShuffle.initialize(isMaster) self.mapOutputTracker = MapOutputTracker() from dpark.shuffle import SimpleShuffleFetcher, ParallelShuffleFetcher #self.shuffleFetcher = SimpleShuffleFetcher() self.shuffleFetcher = ParallelShuffleFetcher(2) from dpark.broadcast import start_manager start_manager(isMaster) self.started = True logger.debug("env started")