def init(self):
if self.initialized:
return
register_sighandlers()
cls = self.__class__
options = cls.options
if options is None:
options = cls.options = parse_options()
try:
import dpark.web
from dpark.web.ui import create_app
app = create_app(self)
self.web_port = dpark.web.start(app)
self.webui_url = 'http://%s:%s' % (socket.gethostname(),
self.web_port)
logger.info('start listening on Web UI with port: %d' %
self.web_port)
except ImportError as e:
self.webui_url = None
logger.info('no web server created as %s', e)
master = self.master or options.master
if master == 'local':
self.scheduler = LocalScheduler()
self.isLocal = True
elif master == 'process':
self.scheduler = MultiProcessScheduler(options.parallel)
self.isLocal = False
else:
if master == 'mesos':
master = conf.MESOS_MASTER
if master.startswith('mesos://'):
if '@' in master:
master = master[master.rfind('@') + 1:]
else:
master = master[master.rfind('//') + 2:]
elif master.startswith('zoo://'):
master = 'zk' + master[3:]
if ':' not in master:
master += ':5050'
self.scheduler = MesosScheduler(master,
options,
webui_url=self.webui_url)
self.data_limit = 1024 * 1024 # 1MB
self.isLocal = False
self.master = master
if options.parallel:
self.defaultParallelism = options.parallel
else:
self.defaultParallelism = self.scheduler.defaultParallelism()
self.defaultMinSplits = max(self.defaultParallelism, 2)
self.initialized = True
def init(self):
if self.initialized:
return
options = parse_options()
self.options = options
setup_conf(options)
master = self.master or options.master
if master == 'local':
self.scheduler = LocalScheduler()
self.isLocal = True
elif master == 'process':
self.scheduler = MultiProcessScheduler(options.parallel)
self.isLocal = False
else:
if master == 'mesos':
master = conf.MESOS_MASTER
if master.startswith('mesos://'):
if '@' in master:
master = master[master.rfind('@') + 1:]
else:
master = master[master.rfind('//') + 2:]
elif master.startswith('zoo://'):
master = 'zk' + master[3:]
if ':' not in master:
master += ':5050'
self.scheduler = MesosScheduler(master, options)
self.isLocal = False
self.master = master
if options.parallel:
self.defaultParallelism = options.parallel
else:
self.defaultParallelism = self.scheduler.defaultParallelism()
self.defaultMinSplits = max(self.defaultParallelism, 2)
self.initialized = True