def start(self, isMaster, environ={}, isLocal=False): if self.started: return logger.debug("start env in %s: %s %s", os.getpid(), isMaster, environ) self.isMaster = isMaster self.isLocal = isLocal if isMaster: if os.environ.has_key('DPARK_WORK_DIR'): root = os.environ['DPARK_WORK_DIR'] else: root = '/tmp/dpark' if not os.path.exists(root): os.mkdir(root, 0777) os.chmod(root, 0777) # because of umask name = '%s-%s-%d' % (time.strftime("%Y%m%d-%H%M%S"), socket.gethostname(), os.getpid()) self.workdir = os.path.join(root, name) os.makedirs(self.workdir) self.environ['WORKDIR'] = self.workdir self.environ['COMPRESS'] = util.COMPRESS else: self.environ.update(environ) if self.environ['COMPRESS'] != util.COMPRESS: raise Exception("no %s available" % self.environ['COMPRESS']) self.ctx = zmq.Context() from cache import CacheTracker, LocalCacheTracker if isLocal: self.cacheTracker = LocalCacheTracker(isMaster) else: self.cacheTracker = CacheTracker(isMaster) from shuffle import LocalFileShuffle, MapOutputTracker, LocalMapOutputTracker LocalFileShuffle.initialize(isMaster) if isLocal: self.mapOutputTracker = LocalMapOutputTracker(isMaster) else: self.mapOutputTracker = MapOutputTracker(isMaster) from shuffle import SimpleShuffleFetcher, ParallelShuffleFetcher #self.shuffleFetcher = SimpleShuffleFetcher() self.shuffleFetcher = ParallelShuffleFetcher(2) from broadcast import Broadcast Broadcast.initialize(isMaster) self.started = True logger.debug("env started")