def start(self):
if self._started:
return
self._started = True
self.guide_thread = self.start_guide()
env.register(GUIDE_ADDR, self.guide_addr)
def initialize(cls, is_master):
Broadcast.initialize(is_master)
sock = env.ctx.socket(zmq.REP)
sock.setsockopt(zmq.LINGER, 0)
port = sock.bind_to_random_port("tcp://0.0.0.0")
cls.tracker_addr = 'tcp://%s:%d' % (cls.host, port)
def run():
logger.debug("TreeBroadcast tracker started at %s",
cls.tracker_addr)
while True:
uuid = sock.recv_pyobj()
obj = cls.guides.get(uuid, '')
sock.send_pyobj(obj and (obj.guide_addr, len(obj.blocks)))
if not uuid:
break
sock.close()
logger.debug("TreeBroadcast tracker stopped")
if is_master:
cls.tracker_thread = spawn(run)
env.register('TreeBroadcastTrackerAddr', cls.tracker_addr)
else:
cls.tracker_addr = env.get('TreeBroadcastTrackerAddr')
logger.debug("TreeBroadcast initialized")
def initialize(cls, is_master):
Broadcast.initialize(is_master)
sock = env.ctx.socket(zmq.REP)
sock.setsockopt(zmq.LINGER, 0)
port = sock.bind_to_random_port("tcp://0.0.0.0")
cls.tracker_addr = 'tcp://%s:%d' % (cls.host, port)
def run():
logger.debug("TreeBroadcast tracker started at %s",
cls.tracker_addr)
while True:
uuid = sock.recv_pyobj()
obj = cls.guides.get(uuid, '')
sock.send_pyobj(obj and (obj.guide_addr, len(obj.blocks)))
if not uuid:
break
sock.close()
logger.debug("TreeBroadcast tracker stopped")
if is_master:
cls.tracker_thread = spawn(run)
env.register('TreeBroadcastTrackerAddr', cls.tracker_addr)
else:
cls.tracker_addr = env.get('TreeBroadcastTrackerAddr')
logger.debug("TreeBroadcast initialized")
def start(self, is_master):
if is_master:
self.guides = {}
self.guide_addr, self.guide_thread = self.start_guide()
env.register('BroadcastGuideAddr', self.guide_addr)
else:
self.guide_addr = env.get('BroadcastGuideAddr')
logger.debug("broadcast started: %s", self.guide_addr)
def start(self, is_master):
if is_master:
self.guides = {}
self.guide_addr, self.guide_thread = self.start_guide()
env.register('BroadcastGuideAddr', self.guide_addr)
else:
self.guide_addr = env.get('BroadcastGuideAddr')
logger.debug("broadcast started: %s", self.guide_addr)
def __init__(self, isMaster):
LocalMapOutputTracker.__init__(self, isMaster)
if isMaster:
self.server = MapOutputTrackerServer(self.serverUris)
self.server.start()
addr = self.server.addr
env.register('MapOutputTrackerAddr', addr)
else:
addr = env.get('MapOutputTrackerAddr')
self.client = MapOutputTrackerClient(addr)
logger.debug("MapOutputTracker started")
def __init__(self, isMaster):
LocalMapOutputTracker.__init__(self, isMaster)
if isMaster:
self.server = MapOutputTrackerServer(self.serverUris)
self.server.start()
addr = self.server.addr
env.register('MapOutputTrackerAddr', addr)
else:
addr = env.get('MapOutputTrackerAddr')
self.client = MapOutputTrackerClient(addr)
logger.debug("MapOutputTracker started")
def __init__(self, isMaster):
LocalCacheTracker.__init__(self, isMaster)
if isMaster:
self.server = CacheTrackerServer(self.locs)
self.server.start()
addr = self.server.addr
env.register('CacheTrackerAddr', addr)
else:
cachedir = os.path.join(env.get('WORKDIR')[0], 'cache')
self.cache = DiskCache(self, cachedir)
addr = env.get('CacheTrackerAddr')
self.client = CacheTrackerClient(addr)
def __init__(self, isMaster):
LocalCacheTracker.__init__(self, isMaster)
if isMaster:
self.server = CacheTrackerServer(self.locs)
self.server.start()
addr = self.server.addr
env.register("CacheTrackerAddr", addr)
else:
cachedir = os.path.join(env.get("WORKDIR")[0], "cache")
self.cache = DiskCache(self, cachedir)
addr = env.get("CacheTrackerAddr")
self.client = CacheTrackerClient(addr)
def __init__(self, isMaster):
LocalCacheTracker.__init__(self, isMaster)
if isMaster:
self.cache = Cache()
else:
self.cache = LocalCache(mmapCache).newKeySpace()
if isMaster:
self.server = CacheTrackerServer(self.locs)
self.server.start()
addr = self.server.addr
env.register('CacheTrackerAddr', addr)
else:
addr = env.get('CacheTrackerAddr')
self.client = CacheTrackerClient(addr)
def start(self):
if self._started:
return
self.manager = manager = Manager()
self.shared_uuid_fn_dict = manager.dict()
self.shared_uuid_map_dict = manager.dict()
self.shared_master_blocks = manager.dict()
self.download_cond = Condition()
self._started = True
self.ctx = zmq.Context()
self.host = socket.gethostname()
if GUIDE_ADDR not in env.environ:
start_guide_manager()
self.guide_addr = env.get(GUIDE_ADDR)
self.random_inst = random.SystemRandom()
self.server_addr, self.server_thread = self.start_server()
self.uuid_state_dict = {}
self.uuid_map_dict = {}
self.master_broadcast_blocks = {}
env.register(DOWNLOAD_ADDR, self.server_addr)
