def main():
parser = ArgumentParser()
parser.add_argument("-p", "--port", type=int, default=9000)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("-l",
"--log",
help="from loghub dir. e.g. examples/ui/loghub")
group.add_argument(
"-d",
"--dag",
help="from intermediate dag file. e.g. examples/ui/dag.json")
parser.add_argument(
"-g",
"--gen",
help="gen intermediate dag file from loghub dir, used with -l")
args = parser.parse_args()
if args.log:
context = dag.from_loghub(args.log)
context = json.dumps(context, indent=4)
if args.gen:
with open(args.gen, "w") as f:
f.write(context)
return
else:
with open(args.dag) as f:
context = f.read()
app = create_app(context)
run(app, args.port, sys.stderr)
def main():
parser = ArgumentParser()
parser.add_argument("-p", "--port",
type=int,
default=9000)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("-l", "--log",
help="from loghub dir. e.g. examples/ui/loghub")
group.add_argument("-d", "--dag",
help="from intermediate dag file. e.g. examples/ui/dag.json")
parser.add_argument("-g", "--gen",
help="gen intermediate dag file from loghub dir, used with -l")
args = parser.parse_args()
if args.log:
context = dag.from_loghub(args.log)
context = json.dumps(context, indent=4)
if args.gen:
with open(args.gen, "w") as f:
f.write(context)
return
else:
with open(args.dag) as f:
context = f.read()
app = create_app(context)
run(app, args.port, sys.stderr)
def init(self):
if self.initialized:
return
register_sighandlers()
cls = self.__class__
options = cls.options
if options is None:
options = cls.options = parse_options()
try:
import dpark.web
from dpark.web.ui import create_app
app = create_app(self)
self.web_port = dpark.web.start(app)
self.webui_url = 'http://%s:%s' % (socket.gethostname(),
self.web_port)
logger.info('start listening on Web UI with port: %d' %
self.web_port)
except ImportError as e:
self.webui_url = None
logger.info('no web server created as %s', e)
master = self.master or options.master
if master == 'local':
self.scheduler = LocalScheduler()
self.isLocal = True
elif master == 'process':
self.scheduler = MultiProcessScheduler(options.parallel)
self.isLocal = False
else:
if master == 'mesos':
master = conf.MESOS_MASTER
if master.startswith('mesos://'):
if '@' in master:
master = master[master.rfind('@') + 1:]
else:
master = master[master.rfind('//') + 2:]
elif master.startswith('zoo://'):
master = 'zk' + master[3:]
if ':' not in master:
master += ':5050'
self.scheduler = MesosScheduler(master,
options,
webui_url=self.webui_url)
self.data_limit = 1024 * 1024 # 1MB
self.isLocal = False
self.master = master
if options.parallel:
self.defaultParallelism = options.parallel
else:
self.defaultParallelism = self.scheduler.defaultParallelism()
self.defaultMinSplits = max(self.defaultParallelism, 2)
self.initialized = True
def init(self):
if self.initialized:
return
options = parse_options()
self.options = options
try:
import dpark.web
from dpark.web.ui import create_app
app = create_app(self)
self.web_port = dpark.web.start(app)
self.options.webui_url = 'http://%s:%s' % (
socket.gethostname(),
self.web_port
)
logger.info('start listening on Web UI with port: %d' % self.web_port)
except ImportError as e:
self.options.webui_url = ''
logger.info('no web server created as %s', e)
master = self.master or options.master
if master == 'local':
self.scheduler = LocalScheduler()
self.isLocal = True
elif master == 'process':
self.scheduler = MultiProcessScheduler(options.parallel)
self.isLocal = False
else:
if master == 'mesos':
master = conf.MESOS_MASTER
if master.startswith('mesos://'):
if '@' in master:
master = master[master.rfind('@')+1:]
else:
master = master[master.rfind('//')+2:]
elif master.startswith('zoo://'):
master = 'zk' + master[3:]
if ':' not in master:
master += ':5050'
self.scheduler = MesosScheduler(master, options)
self.isLocal = False
self.master = master
if options.parallel:
self.defaultParallelism = options.parallel
else:
self.defaultParallelism = self.scheduler.defaultParallelism()
self.defaultMinSplits = max(self.defaultParallelism, 2)
self.initialized = True
def init(self):
if self.initialized:
return
register_sighandlers()
cls = self.__class__
options = cls.options
if options is None:
options = cls.options = parse_options()
try:
import dpark.web
from dpark.web.ui import create_app
app = create_app(self)
self.web_port = dpark.web.start(app)
self.webui_url = 'http://%s:%s' % (
socket.gethostname(),
self.web_port
)
logger.info('start listening on Web UI: %s', self.webui_url)
except ImportError as e:
self.webui_url = None
logger.info('no web server created as %s', e)
origin_master = master = self.master or options.master
if master == 'local':
logger.info("use local scheduler: %s", master)
self.scheduler = LocalScheduler()
self.is_local = True
elif master == 'process':
logger.info("use process scheduler: %s", master)
self.scheduler = MultiProcessScheduler(options.parallel)
self.is_local = False
else:
if master == 'mesos':
master = conf.MESOS_MASTER
else:
master = conf.MESOS_MASTERS.get(master, master)
if master.startswith('mesos://'):
if '@' in master:
master = master[master.rfind('@') + 1:]
else:
master = master[master.rfind('//') + 2:]
elif master.startswith('zoo://'):
master = 'zk' + master[3:]
if ':' not in master:
master += ':5050'
self.scheduler = MesosScheduler(
master, options, webui_url=self.webui_url
)
self.data_limit = 1024 * 1024 # 1MB
self.is_local = False
logger.info("use mesos scheduler: %s", master)
self.master = master
if options.parallel:
self.defaultParallelism = options.parallel
else:
self.defaultParallelism = self.scheduler.defaultParallelism()
self.defaultMinSplits = max(self.defaultParallelism, 2)
self.initialized = True
self.scheduler.is_dstream = self.is_dstream
logger.info("DparkContext initialized, use master %s -> %s, default_rddconf = %s",
origin_master, master,
conf.default_rddconf)