def start_driver(self):
name = '[dpark] ' + \
os.path.abspath(sys.argv[0]) + ' ' + ' '.join(sys.argv[1:])
if len(name) > 256:
name = name[:256] + '...'
framework = Dict()
framework.user = getuser()
if framework.user == 'root':
raise Exception('dpark is not allowed to run as \'root\'')
framework.name = name
framework.hostname = socket.gethostname()
framework.webui_url = self.options.webui_url
self.driver = MesosSchedulerDriver(
self, framework, self.master, use_addict=True
)
self.driver.start()
logger.debug('Mesos Scheudler driver started')
self.started = True
self.last_finish_time = time.time()
def check():
while self.started:
now = time.time()
if (not self.activeJobs and
now - self.last_finish_time > MAX_IDLE_TIME):
logger.info('stop mesos scheduler after %d seconds idle',
now - self.last_finish_time)
self.stop()
break
time.sleep(1)
spawn(check)
def start_driver(self):
name = '[dpark] ' + os.path.abspath(sys.argv[0]) + ' ' + ' '.join(
sys.argv[1:])
if len(name) > 256:
name = name[:256] + '...'
framework = mesos_pb2.FrameworkInfo()
framework.user = getuser()
if framework.user == 'root':
raise Exception("dpark is not allowed to run as 'root'")
framework.name = name
framework.hostname = socket.gethostname()
self.driver = mesos.MesosSchedulerDriver(self, framework, self.master)
self.driver.start()
logger.debug("Mesos Scheudler driver started")
self.started = True
self.last_finish_time = time.time()
def check():
while self.started:
now = time.time()
if not self.activeJobs and now - self.last_finish_time > MAX_IDLE_TIME:
logger.info("stop mesos scheduler after %d seconds idle",
now - self.last_finish_time)
self.stop()
break
time.sleep(1)
spawn(check)
def start_driver(self):
name = '[dpark] ' + \
os.path.abspath(sys.argv[0]) + ' ' + ' '.join(sys.argv[1:])
if len(name) > 256:
name = name[:256] + '...'
framework = Dict()
framework.user = getuser()
if framework.user == 'root':
raise Exception('dpark is not allowed to run as \'root\'')
framework.name = name
framework.hostname = socket.gethostname()
framework.webui_url = self.options.webui_url
self.driver = MesosSchedulerDriver(self,
framework,
self.master,
use_addict=True)
self.driver.start()
logger.debug('Mesos Scheudler driver started')
self.started = True
self.last_finish_time = time.time()
def check():
while self.started:
now = time.time()
if (not self.activeJobs
and now - self.last_finish_time > MAX_IDLE_TIME):
logger.info('stop mesos scheduler after %d seconds idle',
now - self.last_finish_time)
self.stop()
break
time.sleep(1)
spawn(check)
def start_driver(self):
name = '[dpark@%s] ' % socket.gethostname()
name += os.path.abspath(sys.argv[0]) + ' ' + ' '.join(sys.argv[1:])
if len(name) > 512:
name = name[:512] + '...'
framework = mesos_pb2.FrameworkInfo()
framework.user = getpass.getuser()
if framework.user == 'root':
raise Exception("dpark is not allowed to run as 'root'")
framework.name = name
self.driver = mesos.MesosSchedulerDriver(self, framework,
self.master)
self.driver.start()
logger.debug("Mesos Scheudler driver started")
self.started = True
self.last_finish_time = time.time()
def check():
while self.started:
now = time.time()
if not self.activeJobs and now - self.last_finish_time > MAX_IDLE_TIME:
logger.info("stop mesos scheduler after %d seconds idle",
now - self.last_finish_time)
self.stop()
break
time.sleep(1)
spawn(check)
def registered(self, driver, executorInfo, frameworkInfo, slaveInfo):
try:
global Script
Script, cwd, python_path, osenv, self.parallel, out_logger, err_logger, logLevel, args = marshal.loads(executorInfo.data)
self.init_args = args
try:
os.chdir(cwd)
except OSError:
driver.sendFrameworkMessage("switch cwd failed: %s not exists!" % cwd)
sys.path = python_path
os.environ.update(osenv)
prefix = '[%s] ' % socket.gethostname()
self.outt, sys.stdout = start_forword(out_logger, prefix, sys.stdout)
self.errt, sys.stderr = start_forword(err_logger, prefix, sys.stderr)
logging.basicConfig(format='%(asctime)-15s [%(levelname)s] [%(name)-9s] %(message)s', level=logLevel)
self.workdir = args['WORKDIR']
root = os.path.dirname(self.workdir[0])
if not os.path.exists(root):
os.mkdir(root)
os.chmod(root, 0777) # because umask
args['SERVER_URI'] = startWebServer(self.workdir[0])
if 'MESOS_SLAVE_PID' in os.environ: # make unit test happy
setup_cleaner_process(self.workdir)
spawn(self.check_memory, driver)
logger.debug("executor started at %s", slaveInfo.hostname)
except Exception, e:
import traceback
msg = traceback.format_exc()
logger.error("init executor failed: %s", msg)
raise
def registered(self, driver, executorInfo, frameworkInfo, agent_info):
try:
global Script
(
Script, cwd, python_path, osenv, self.parallel,
out_logger, err_logger, logLevel, args
) = marshal.loads(decode_data(executorInfo.data))
self.init_args = args
sys.path = python_path
os.environ.update(osenv)
setproctitle('[Executor]' + Script)
prefix = '[%s] ' % socket.gethostname()
fmt = '%(asctime)-15s [%(levelname)s] [%(name)-9s] %(message)s'
logging.basicConfig(format=fmt, level=logLevel)
r1 = self.stdout_redirect = Redirect(1, out_logger, prefix)
sys.stdout = r1.pipe_wfile
r2 = self.stderr_redirect = Redirect(2, err_logger, prefix)
sys.stderr = r2.pipe_wfile
if os.path.exists(cwd):
try:
os.chdir(cwd)
except Exception as e:
logger.warning('change cwd to %s failed: %s', cwd, e)
else:
logger.warning('cwd (%s) not exists', cwd)
self.workdir = args['WORKDIR']
main_workdir = self.workdir[0]
root = os.path.dirname(main_workdir)
if not os.path.exists(root):
os.mkdir(root)
os.chmod(root, 0o777) # because umask
mkdir_p(main_workdir)
self._try_flock(main_workdir)
args['SERVER_URI'] = startWebServer(main_workdir)
if 'MESOS_SLAVE_PID' in os.environ: # make unit test happy
setup_cleaner_process(self.workdir)
spawn(self.check_memory, driver)
spawn(self.replier, driver)
logger.debug('executor started at %s', agent_info.hostname)
except Exception as e:
import traceback
msg = traceback.format_exc()
logger.error('init executor failed: %s', msg)
raise
def startReceiver(self):
def _run():
while True:
generator = self.func()
try:
for message in generator:
if not self.ssc.sc.started:
return
with self._lock:
self._messages.append(message)
except:
logger.exception('fail to receive')
spawn(_run)
def startReceiver(self):
def _run():
while True:
generator = self.func()
try:
for message in generator:
if not self.ssc.sc.started:
return
with self._lock:
self._messages.append(message)
except:
logger.exception('fail to receive')
spawn(_run)
def start_logger(self, output):
sock = env.ctx.socket(zmq.PULL)
port = sock.bind_to_random_port("tcp://0.0.0.0")
def collect_log():
while True:
line = sock.recv()
output.write(line)
spawn(collect_log)
host = socket.gethostname()
addr = "tcp://%s:%d" % (host, port)
logger.debug("log collecter start at %s", addr)
return addr
def start_logger(self, output):
sock = env.ctx.socket(zmq.PULL)
port = sock.bind_to_random_port("tcp://0.0.0.0")
def collect_log():
while True:
line = sock.recv()
output.write(line)
spawn(collect_log)
host = socket.gethostname()
addr = "tcp://%s:%d" % (host, port)
logger.debug("log collecter start at %s", addr)
return addr
def start_logger(self, output):
sock = env.ctx.socket(zmq.PULL)
port = sock.bind_to_random_port('tcp://0.0.0.0')
def collect_log():
while not self._shutdown:
if sock.poll(1000, zmq.POLLIN):
line = sock.recv()
output.write(line)
spawn(collect_log)
host = socket.gethostname()
addr = 'tcp://%s:%d' % (host, port)
logger.debug('log collecter start at %s', addr)
return addr
def start_logger(self, output):
sock = env.ctx.socket(zmq.PULL)
port = sock.bind_to_random_port('tcp://0.0.0.0')
def collect_log():
while not self._shutdown:
if sock.poll(1000, zmq.POLLIN):
line = sock.recv()
output.write(line)
spawn(collect_log)
host = socket.gethostname()
addr = 'tcp://%s:%d' % (host, port)
logger.debug('log collecter start at %s', addr)
return addr
def startGuide(self):
sock = env.ctx.socket(zmq.REP)
port = sock.bind_to_random_port("tcp://0.0.0.0")
self.guide_addr = "tcp://%s:%d" % (self.host, port)
self.guide_thread = spawn(self.guide, sock)
self.guides[self.uuid] = self
def start_server(self):
sock = env.ctx.socket(zmq.REP)
sock.setsockopt(zmq.LINGER, 0)
port = sock.bind_to_random_port("tcp://0.0.0.0")
server_addr = 'tcp://%s:%d' % (self.host, port)
def run():
logger.debug("server started at %s", server_addr)
while True:
type, msg = sock.recv_pyobj()
logger.debug('server recv: %s %s', type, msg)
if type == SERVER_STOP:
sock.send_pyobj(None)
break
elif type == SERVER_FETCH:
uuid, id = msg
if uuid not in self.published:
sock.send_pyobj((SERVER_FETCH_FAIL, None))
else:
blocks = self.published[uuid]
if id >= len(blocks):
sock.send_pyobj((SERVER_FETCH_FAIL, None))
else:
sock.send_pyobj(
(SERVER_FETCH_OK, (id, blocks[id])))
else:
logger.error('Unknown server message: %s %s', type, msg)
sock.close()
logger.debug("stop Broadcast server %s", server_addr)
for uuid in list(self.published.keys()):
self.clear(uuid)
return server_addr, spawn(run)
def startGuide(self):
sock = env.ctx.socket(zmq.REP)
port = sock.bind_to_random_port("tcp://0.0.0.0")
self.guide_addr = "tcp://%s:%d" % (self.host, port)
self.guide_thread = spawn(self.guide, sock)
self.guides[self.uuid] = self
def initialize(cls, is_master):
Broadcast.initialize(is_master)
sock = env.ctx.socket(zmq.REP)
sock.setsockopt(zmq.LINGER, 0)
port = sock.bind_to_random_port("tcp://0.0.0.0")
cls.tracker_addr = 'tcp://%s:%d' % (cls.host, port)
def run():
logger.debug("TreeBroadcast tracker started at %s",
cls.tracker_addr)
while True:
uuid = sock.recv_pyobj()
obj = cls.guides.get(uuid, '')
sock.send_pyobj(obj and (obj.guide_addr, len(obj.blocks)))
if not uuid:
break
sock.close()
logger.debug("TreeBroadcast tracker stopped")
if is_master:
cls.tracker_thread = spawn(run)
env.register('TreeBroadcastTrackerAddr', cls.tracker_addr)
else:
cls.tracker_addr = env.get('TreeBroadcastTrackerAddr')
logger.debug("TreeBroadcast initialized")
def start_guide(self):
sock = env.ctx.socket(zmq.REP)
port = sock.bind_to_random_port("tcp://0.0.0.0")
guide_addr = "tcp://%s:%d" % (self.host, port)
def run():
logger.debug("guide start at %s", guide_addr)
while True:
type, msg = sock.recv_pyobj()
if type == GUIDE_STOP:
sock.send_pyobj(0)
break
elif type == GUIDE_SOURCES:
uuid, addr, bitmap = msg
sources = self.guides[uuid]
sock.send_pyobj(sources)
if any(bitmap):
sources[addr] = bitmap
elif type == GUIDE_REPORT_BAD:
uuid, addr = msg
sock.send_pyobj(0)
sources = self.guides[uuid]
if addr in sources:
del sources[addr]
else:
logger.error('Unknown guide message: %s %s', type, msg)
sock.close()
logger.debug("Sending stop notification to all servers ...")
for uuid, sources in self.guides.iteritems():
for addr in sources:
self.stop_server(addr)
return guide_addr, spawn(run)
def start_server(self):
sock = env.ctx.socket(zmq.REP)
sock.setsockopt(zmq.LINGER, 0)
port = sock.bind_to_random_port("tcp://0.0.0.0")
server_addr = 'tcp://%s:%d' % (self.host, port)
def run():
logger.debug("server started at %s", server_addr)
while True:
type, msg = sock.recv_pyobj()
logger.debug('server recv: %s %s', type, msg)
if type == SERVER_STOP:
sock.send_pyobj(None)
break
elif type == SERVER_FETCH:
uuid, id = msg
if uuid not in self.published:
sock.send_pyobj((SERVER_FETCH_FAIL, None))
else:
blocks = self.published[uuid]
if id >= len(blocks):
sock.send_pyobj((SERVER_FETCH_FAIL, None))
else:
sock.send_pyobj((SERVER_FETCH_OK, (id,
blocks[id])))
else:
logger.error('Unknown server message: %s %s', type, msg)
sock.close()
logger.debug("stop Broadcast server %s", server_addr)
for uuid in self.published.keys():
self.clear(uuid)
return server_addr, spawn(run)
def start_guide(self):
sock = env.ctx.socket(zmq.REP)
port = sock.bind_to_random_port("tcp://0.0.0.0")
guide_addr = "tcp://%s:%d" % (self.host, port)
def run():
logger.debug("guide start at %s", guide_addr)
while True:
type, msg = sock.recv_pyobj()
if type == GUIDE_STOP:
sock.send_pyobj(0)
break
elif type == GUIDE_SOURCES:
uuid, addr, bitmap = msg
sources = self.guides[uuid]
sock.send_pyobj(sources)
if any(bitmap):
sources[addr] = bitmap
elif type == GUIDE_REPORT_BAD:
uuid, addr = msg
sock.send_pyobj(0)
sources = self.guides[uuid]
if addr in sources:
del sources[addr]
else:
logger.error('Unknown guide message: %s %s', type, msg)
sock.close()
logger.debug("Sending stop notification to all servers ...")
for uuid, sources in six.iteritems(self.guides):
for addr in sources:
self.stop_server(addr)
return guide_addr, spawn(run)
def initialize(cls, is_master):
Broadcast.initialize(is_master)
sock = env.ctx.socket(zmq.REP)
sock.setsockopt(zmq.LINGER, 0)
port = sock.bind_to_random_port("tcp://0.0.0.0")
cls.tracker_addr = 'tcp://%s:%d' % (cls.host, port)
def run():
logger.debug("TreeBroadcast tracker started at %s",
cls.tracker_addr)
while True:
uuid = sock.recv_pyobj()
obj = cls.guides.get(uuid, '')
sock.send_pyobj(obj and (obj.guide_addr, len(obj.blocks)))
if not uuid:
break
sock.close()
logger.debug("TreeBroadcast tracker stopped")
if is_master:
cls.tracker_thread = spawn(run)
env.register('TreeBroadcastTrackerAddr', cls.tracker_addr)
else:
cls.tracker_addr = env.get('TreeBroadcastTrackerAddr')
logger.debug("TreeBroadcast initialized")
def registered(self, driver, executorInfo, frameworkInfo, slaveInfo):
try:
global Script
Script, cwd, python_path, osenv, self.parallel, out_logger, err_logger, logLevel, args = marshal.loads(executorInfo.data)
self.init_args = args
try:
os.chdir(cwd)
except OSError:
driver.sendFrameworkMessage("switch cwd failed: %s not exists!" % cwd)
sys.path = python_path
os.environ.update(osenv)
prefix = '[%s] ' % socket.gethostname()
if out_logger:
self.outt, sys.stdout = start_forword(out_logger, prefix)
if err_logger:
self.errt, sys.stderr = start_forword(err_logger, prefix)
logging.basicConfig(format='%(asctime)-15s [%(levelname)s] [%(name)-9s] %(message)s', level=logLevel)
self.workdir = args['WORKDIR']
root = os.path.dirname(self.workdir)
if not os.path.exists(root):
os.mkdir(root)
os.chmod(root, 0777) # because umask
if not os.path.exists(self.workdir):
os.mkdir(self.workdir)
args['SERVER_URI'] = startWebServer(args['WORKDIR'])
spawn(self.check_memory, driver)
# wait background threads to been initialized
time.sleep(0.1)
if out_logger:
os.close(1)
assert os.dup(sys.stdout.fileno()) == 1, 'redirect stdout failed'
if err_logger:
os.close(2)
assert os.dup(sys.stderr.fileno()) == 2, 'redirect stderr failed'
multiprocessing.Pool(1, clean_work_dir, [self.workdir])
logger.debug("executor started at %s", slaveInfo.hostname)
except Exception, e:
import traceback
msg = traceback.format_exc()
logger.error("init executor failed: %s", msg)
raise
def registered(self, driver, executorInfo, frameworkInfo, slaveInfo):
try:
global Script
Script, cwd, python_path, osenv, self.parallel, out_logger, err_logger, logLevel, args = marshal.loads(executorInfo.data)
self.init_args = args
sys.path = python_path
os.environ.update(osenv)
prefix = '[%s] ' % socket.gethostname()
self.outt = spawn(forward, self.stdout, out_logger, prefix)
self.errt = spawn(forward, self.stderr, err_logger, prefix)
logging.basicConfig(format='%(asctime)-15s [%(levelname)s] [%(name)-9s] %(message)s', level=logLevel)
if os.path.exists(cwd):
try:
os.chdir(cwd)
except Exception, e:
logger.warning("change cwd to %s failed: %s", cwd, e)
else:
def start(self):
if self._started:
return
self._started = True
self.requests = six.moves.queue.Queue()
self.results = six.moves.queue.Queue(self.nthreads)
self.threads = [
spawn(self._fetch_thread) for i in range(self.nthreads)
]
def registered(self, driver, executorInfo, frameworkInfo, slaveInfo):
try:
global Script
Script, cwd, python_path, osenv, self.parallel, out_logger, err_logger, logLevel, args = marshal.loads(executorInfo.data)
self.init_args = args
sys.path = python_path
os.environ.update(osenv)
prefix = '[%s] ' % socket.gethostname()
self.outt = spawn(forward, self.stdout, out_logger, prefix)
self.errt = spawn(forward, self.stderr, err_logger, prefix)
logging.basicConfig(format='%(asctime)-15s [%(levelname)s] [%(name)-9s] %(message)s', level=logLevel)
if os.path.exists(cwd):
try:
os.chdir(cwd)
except Exception, e:
logger.warning("change cwd to %s failed: %s", cwd, e)
else:
def _createThriftServer(self):
buf_que = deque()
handler = ScribeHandler(buf_que)
protocol_factory = TBinaryProtocol.TBinaryProtocolFactory(
False, False)
transport = TSocket.TServerSocket(host='0.0.0.0')
processor = Processor(handler)
server = TNonblockingServer.TNonblockingServer(
processor, transport, protocol_factory)
server._stop = False
while True:
try:
server.prepare()
port = transport.handle.getsockname()[1]
logger.info('get scribe port succeed: %d', port)
break
except socket.error:
pass
spawn(server.serve)
return server, port, buf_que
def start(self):
ctx = zmq.Context()
sock = ctx.socket(zmq.PULL)
port = sock.bind_to_random_port('tcp://0.0.0.0')
self._started = True
def collect_log():
while self._started:
if sock.poll(1000, zmq.POLLIN):
line = sock.recv()
self.output.write(line)
sock.close()
ctx.destroy()
spawn(collect_log)
host = socket.gethostname()
self.addr = 'tcp://%s:%d' % (host, port)
logger.debug('log collecter start at %s', self.addr)
def start_guide(self):
sock = self.ctx.socket(zmq.REP)
port = sock.bind_to_random_port('tcp://0.0.0.0')
self.guide_addr = 'tcp://%s:%d' % (self.host, port)
def run():
logger.debug("guide start at %s", self.guide_addr)
while self._started:
if not sock.poll(1000, zmq.POLLIN):
continue
type, msg = sock.recv_pyobj()
if type == GUIDE_STOP:
sock.send_pyobj(0)
break
elif type == GUIDE_GET_SOURCES:
uuid = msg
sources = None
if uuid in self.guides:
sources = self.guides[uuid]
else:
logger.warning(
'uuid %s NOT REGISTERED in guide server', uuid)
sock.send_pyobj(sources)
elif type == GUIDE_SET_SOURCES:
uuid, addr, bitmap = msg
if any(bitmap):
sources = None
if uuid in self.guides:
sources = self.guides[uuid]
if sources:
sources[addr] = bitmap
else:
self.guides[uuid] = {addr: bitmap}
self.register_addr[uuid] = addr
sock.send_pyobj(None)
elif type == GUIDE_REPORT_BAD:
uuid, addr = msg
sources = self.guides[uuid]
if addr in sources:
if addr != self.register_addr[uuid]:
del sources[addr]
else:
logger.warning(
'The addr %s to delete is the register Quit!!!',
addr)
sock.send_pyobj(None)
else:
logger.error('Unknown guide message: %s %s', type, msg)
sock.send_pyobj(None)
return spawn(run)
def startWebServer(path):
# check the default web server
if not os.path.exists(path):
os.makedirs(path)
testpath = os.path.join(path, 'test')
with open(testpath, 'w') as f:
f.write(path)
default_uri = 'http://%s:%d/%s' % (socket.gethostname(), DEFAULT_WEB_PORT,
os.path.basename(path))
try:
data = urllib.request.urlopen(default_uri + '/' + 'test').read()
if data == path.encode('utf-8'):
return default_uri
except IOError:
pass
logger.warning('default webserver at %s not available', DEFAULT_WEB_PORT)
LocalizedHTTP.basedir = os.path.dirname(path)
ss = six.moves.socketserver.TCPServer(('0.0.0.0', 0), LocalizedHTTP)
spawn(ss.serve_forever)
uri = 'http://%s:%d/%s' % (socket.gethostname(), ss.server_address[1],
os.path.basename(path))
return uri
def startWebServer(path):
# check the default web server
if not os.path.exists(path):
os.makedirs(path)
testpath = os.path.join(path, 'test')
with open(testpath, 'w') as f:
f.write(path)
default_uri = 'http://%s:%d/%s' % (socket.gethostname(), DEFAULT_WEB_PORT,
os.path.basename(path))
try:
data = urllib2.urlopen(default_uri + '/' + 'test').read()
if data == path:
return default_uri
except IOError:
pass
logger.warning('default webserver at %s not available', DEFAULT_WEB_PORT)
LocalizedHTTP.basedir = os.path.dirname(path)
ss = SocketServer.TCPServer(('0.0.0.0', 0), LocalizedHTTP)
spawn(ss.serve_forever)
uri = 'http://%s:%d/%s' % (socket.gethostname(), ss.server_address[1],
os.path.basename(path))
return uri
def __init__(self, fd, addr, prefix):
self.fd = fd
self.addr = addr
self.prefix = prefix
self.fd_dup = os.dup(self.fd)
self.origin_wfile = None
self.pipe_rfd, self.pipe_wfd = os.pipe()
self.pipe_rfile = os.fdopen(self.pipe_rfd, 'rb')
self.pipe_wfile = os.fdopen(self.pipe_wfd, 'wb', 0)
os.close(self.fd)
os.dup2(self.pipe_wfd, self.fd)
# assert os.dup(self.pipe_wfd) == self.fd, 'redirect io failed'
self.ctx = zmq.Context()
self._shutdown = False
self.thread = None
self.sock = None
self.thread = spawn(self._forward)
def __init__(self, fd, addr, prefix):
self.fd = fd
self.addr = addr
self.prefix = prefix
self.fd_dup = os.dup(self.fd)
self.origin_wfile = None
self.pipe_rfd, self.pipe_wfd = os.pipe()
self.pipe_rfile = os.fdopen(self.pipe_rfd, 'r')
self.pipe_wfile = os.fdopen(self.pipe_wfd, 'w', 0)
os.close(self.fd)
os.dup2(self.pipe_wfd, self.fd)
# assert os.dup(self.pipe_wfd) == self.fd, 'redirect io failed'
self.ctx = zmq.Context()
self._shutdown = False
self.thread = None
self.sock = None
self.thread = spawn(self._forward)
def startServer(self):
sock = env.ctx.socket(zmq.REP)
sock.setsockopt(zmq.LINGER, 0)
port = sock.bind_to_random_port("tcp://0.0.0.0")
self.server_addr = 'tcp://%s:%d' % (self.host,port)
def run():
logger.debug("server started at %s", self.server_addr)
while True:
id = sock.recv_pyobj()
if id == SourceInfo.Stop:
sock.send_pyobj(0)
break
sock.send_pyobj(id < len(self.blocks) and self.blocks[id] or None)
sock.close()
logger.debug("stop TreeBroadcast server %s", self.server_addr)
Broadcast.clear(self) # release obj
self.server_thread = spawn(run)
def startServer(self):
sock = env.ctx.socket(zmq.REP)
sock.setsockopt(zmq.LINGER, 0)
port = sock.bind_to_random_port("tcp://0.0.0.0")
self.server_addr = 'tcp://%s:%d' % (self.host, port)
def run():
logger.debug("server started at %s", self.server_addr)
while True:
id = sock.recv_pyobj()
if id == SourceInfo.Stop:
sock.send_pyobj(0)
break
sock.send_pyobj(id < len(self.blocks) and self.blocks[id]
or None)
sock.close()
logger.debug("stop TreeBroadcast server %s", self.server_addr)
Broadcast.clear(self) # release obj
self.server_thread = spawn(run)
def registered(self, driver, executorInfo, frameworkInfo, agent_info):
try:
global Script
(Script, cwd, python_path, osenv, self.parallel, out_logger,
err_logger, logLevel, use_color,
args) = marshal.loads(decode_data(executorInfo.data))
sys.path = python_path
os.environ.update(osenv)
setproctitle('[Executor]' + Script)
prefix = formatter_message(
'{MAGENTA}[%s]{RESET} ' % socket.gethostname().ljust(10),
use_color)
init_dpark_logger(logLevel, use_color=use_color)
logging.root.setLevel(logLevel)
r1 = self.stdout_redirect = Redirect(1, out_logger, prefix)
sys.stdout = r1.pipe_wfile
r2 = self.stderr_redirect = Redirect(2, err_logger, prefix)
sys.stderr = r2.pipe_wfile
if os.path.exists(cwd):
try:
os.chdir(cwd)
except Exception as e:
logger.warning('change cwd to %s failed: %s', cwd, e)
else:
logger.warning('cwd (%s) not exists', cwd)
self.workdir = args['WORKDIR']
main_workdir = self.workdir[0]
root = os.path.dirname(main_workdir)
if not os.path.exists(root):
os.mkdir(root)
os.chmod(root, 0o777) # because umask
mkdir_p(main_workdir)
self._try_flock(main_workdir)
args['SERVER_URI'] = startWebServer(main_workdir)
if 'MESOS_SLAVE_PID' in os.environ: # make unit test happy
setup_cleaner_process(self.workdir)
spawn(self.check_memory, driver)
spawn(self.replier, driver)
env.environ.update(args)
from dpark.broadcast import start_download_manager
start_download_manager()
logger.debug('executor started at %s', agent_info.hostname)
except Exception as e:
import traceback
msg = traceback.format_exc()
logger.error('init executor failed: %s', msg)
raise
def start(self):
self.requests = Queue.Queue()
self.results = Queue.Queue(self.nthreads)
self.threads = [
spawn(self._worker_thread) for i in range(self.nthreads)
]
def start(self):
self.requests = Queue.Queue()
self.results = Queue.Queue(self.nthreads)
self.threads = [spawn(self._worker_thread) for i in range(self.nthreads)]
testpath = os.path.join(path, 'test')
with open(testpath, 'w') as f:
f.write(path)
default_uri = 'http://%s:%d/%s' % (socket.gethostname(), DEFAULT_WEB_PORT,
os.path.basename(path))
try:
data = urllib2.urlopen(default_uri + '/' + 'test').read()
if data == path:
return default_uri
except IOError, e:
pass
logger.warning("default webserver at %s not available", DEFAULT_WEB_PORT)
LocalizedHTTP.basedir = os.path.dirname(path)
ss = SocketServer.TCPServer(('0.0.0.0', 0), LocalizedHTTP)
spawn(ss.serve_forever)
uri = "http://%s:%d/%s" % (socket.gethostname(), ss.server_address[1],
os.path.basename(path))
return uri
def forward(fd, addr, prefix=''):
f = os.fdopen(fd, 'r')
ctx = zmq.Context()
out = [None]
buf = []
def send(buf):
if not out[0]:
out[0] = ctx.socket(zmq.PUSH)
out[0].connect(addr)
def start(self, start):
self.nextTime = (int(start / self.period) + 1) * self.period
self.stopped = False
self.thread = spawn(self.run)
logger.debug("RecurringTimer started, nextTime is %d", self.nextTime)
testpath = os.path.join(path, 'test')
with open(testpath, 'w') as f:
f.write(path)
default_uri = 'http://%s:%d/%s' % (socket.gethostname(), DEFAULT_WEB_PORT,
os.path.basename(path))
try:
data = urllib2.urlopen(default_uri + '/' + 'test').read()
if data == path:
return default_uri
except IOError, e:
pass
logger.warning("default webserver at %s not available", DEFAULT_WEB_PORT)
LocalizedHTTP.basedir = os.path.dirname(path)
ss = SocketServer.TCPServer(('0.0.0.0', 0), LocalizedHTTP)
spawn(ss.serve_forever)
uri = "http://%s:%d/%s" % (socket.gethostname(), ss.server_address[1],
os.path.basename(path))
return uri
def forward(fd, addr, prefix=''):
f = os.fdopen(fd, 'r')
ctx = zmq.Context()
out = [None]
buf = []
def send(buf):
if not out[0]:
out[0] = ctx.socket(zmq.PUSH)
out[0].connect(addr)
out[0].send(prefix+''.join(buf))
def start_forword(addr, prefix, oldfile):
rfd, wfd = os.pipe()
t = spawn(forword, rfd, addr, prefix)
newfile = os.fdopen(wfd, 'w', 0)
return t, newfile
def run():
logger.debug("server started at %s", server_addr)
while True:
type, msg = sock.recv_pyobj()
logger.debug('server recv: %s %s', type, msg)
if type == SERVER_STOP:
sock.send_pyobj(None)
break
elif type == SERVER_FETCH:
uuid, indices, client_addr = msg
if uuid in self.master_broadcast_blocks:
block_num = len(self.master_broadcast_blocks[uuid])
bls = []
for index in indices:
if index >= block_num:
logger.warning('input index too big %s for '
'len of blocks %d from host %s',
str(indices), block_num, client_addr)
sock.send_pyobj((SERVER_FETCH_FAIL, None))
else:
bls.append(self.master_broadcast_blocks[uuid][index])
sock.send_pyobj((SERVER_FETCH_OK, (indices, bls)))
elif uuid in self.uuid_state_dict:
fd = os.open(self.uuid_state_dict[uuid][0], os.O_RDONLY)
mmfp = mmap.mmap(fd, 0, access=ACCESS_READ)
os.close(fd)
bitmap = self.uuid_map_dict[uuid]
block_num = len(bitmap)
bls = []
for index in indices:
if index >= block_num:
logger.warning('input index too big %s for '
'len of blocks %d from host %s',
str(indices), block_num, client_addr)
sock.send_pyobj((SERVER_FETCH_FAIL, None))
else:
mmfp.seek(bitmap[index][0])
block = mmfp.read(bitmap[index][1])
bls.append(block)
mmfp.close()
sock.send_pyobj((SERVER_FETCH_OK, (indices, bls)))
else:
logger.warning('server fetch failed for uuid %s '
'not exists in server %s from host %s',
uuid, socket.gethostname(), client_addr)
sock.send_pyobj((SERVER_FETCH_FAIL, None))
elif type == DATA_GET:
uuid, compressed_size = msg
if uuid not in self.uuid_state_dict or not self.uuid_state_dict[uuid][1]:
if uuid not in self.download_threads:
sources = self._get_sources(uuid, guide_sock)
if not sources:
logger.warning('get sources from guide server failed in host %s',
socket.gethostname())
sock.send_pyobj(DATA_GET_FAIL)
continue
self.download_threads[uuid] = spawn(self._download_blocks,
*[sources, uuid, compressed_size])
sock.send_pyobj(DATA_DOWNLOADING)
else:
sock.send_pyobj(DATA_DOWNLOADING)
else:
sock.send_pyobj(DATA_GET_OK)
elif type == SERVER_CLEAR_ITEM:
uuid = msg
self.clear(uuid)
sock.send_pyobj(None)
else:
logger.error('Unknown server message: %s %s', type, msg)
sock.send_pyobj(None)
sock.close()
logger.debug("stop Broadcast server %s", server_addr)
for uuid in list(self.uuid_state_dict.keys()):
self.clear(uuid)
testpath = os.path.join(path, 'test')
with open(testpath, 'w') as f:
f.write(path)
default_uri = 'http://%s:%d/%s' % (socket.gethostname(), DEFAULT_WEB_PORT,
os.path.basename(path))
try:
data = urllib2.urlopen(default_uri + '/' + 'test').read()
if data == path:
return default_uri
except IOError, e:
pass
logger.warning("default webserver at %s not available", DEFAULT_WEB_PORT)
LocalizedHTTP.basedir = os.path.dirname(path)
ss = SocketServer.TCPServer(('0.0.0.0', 0), LocalizedHTTP)
spawn(ss.serve_forever)
uri = "http://%s:%d/%s" % (socket.gethostname(), ss.server_address[1],
os.path.basename(path))
return uri
def forword(fd, addr, prefix=''):
f = os.fdopen(fd, 'r')
ctx = zmq.Context()
out = [None]
buf = []
def send(buf):
if not out[0]:
out[0] = ctx.socket(zmq.PUSH)
out[0].connect(addr)
out[0].send(prefix+''.join(buf))
def start_forword(addr, prefix=''):
rfd, wfd = os.pipe()
t = spawn(forword, rfd, addr, prefix)
return t, os.fdopen(wfd, 'w', 0)
testpath = os.path.join(path, 'test')
with open(testpath, 'w') as f:
f.write(path)
default_uri = 'http://%s:%d/%s' % (socket.gethostname(), DEFAULT_WEB_PORT,
os.path.basename(path))
try:
data = urllib2.urlopen(default_uri + '/' + 'test').read()
if data == path:
return default_uri
except IOError, e:
pass
logger.warning("default webserver at %s not available", DEFAULT_WEB_PORT)
LocalizedHTTP.basedir = os.path.dirname(path)
ss = SocketServer.TCPServer(('0.0.0.0', 0), LocalizedHTTP)
spawn(ss.serve_forever)
uri = "http://%s:%d/%s" % (socket.gethostname(), ss.server_address[1],
os.path.basename(path))
return uri
def forword(fd, addr, prefix=''):
f = os.fdopen(fd, 'r')
ctx = zmq.Context()
out = [None]
buf = []
def send(buf):
if not out[0]:
out[0] = ctx.socket(zmq.PUSH)
out[0].connect(addr)
out[0].send(prefix+''.join(buf))
def start(self):
self.thread = spawn(self.run)
while self.addr is None:
time.sleep(0.01)
def start(self):
self.thread = spawn(self.run)
while self.addr is None:
time.sleep(0.01)
def __init__(self, nthreads):
self.nthreads = nthreads
self.requests = Queue.Queue()
self.results = Queue.Queue(1)
self.threads = [spawn(self._worker_thread) for i in range(nthreads)]
def start(self):
if self.ctx is None:
self.ctx = zmq.Context()
self.thread = spawn(self.run)
while self.addr is None:
time.sleep(0.01)
def start(self, start):
self.nextTime = (int(start / self.period) + 1) * self.period
self.stopped = False
self.thread = spawn(self.run)
logger.debug("RecurringTimer started, nextTime is %d", self.nextTime)