def start(self, isMaster, environ={}, isLocal=False):
if getattr(self, 'started', False):
return
logger.debug("start env in %s: %s %s", os.getpid(), isMaster, environ)
if isMaster:
root = '/tmp/dpark'
if not isLocal:
root = os.environ.get("DPARK_SHARE_DIR")
if not root:
raise Exception("no shuffle dir exists")
if not os.path.exists(root):
os.mkdir(root, 0777)
os.chmod(root, 0777) # because of umask
name = '%s-%s-%d' % (time.strftime("%Y%m%d-%H%M%S"),
socket.gethostname(), os.getpid())
self.workdir = os.path.join(root, name)
os.makedirs(self.workdir)
self.environ['WORKDIR'] = self.workdir
else:
self.environ.update(environ)
from cache import CacheTracker
self.cacheTracker = CacheTracker(isMaster)
from shuffle import LocalFileShuffle, MapOutputTracker, SimpleShuffleFetcher
LocalFileShuffle.initialize(isMaster)
self.mapOutputTracker = MapOutputTracker(isMaster)
self.shuffleFetcher = SimpleShuffleFetcher()
from broadcast import Broadcast
Broadcast.initialize(isMaster)
self.started = True
logger.debug("env started")
