def start(self): if self._started: return self._started = True start_download_manager() self.guide_addr = env.get(GUIDE_ADDR) self.download_addr = env.get(DOWNLOAD_ADDR) self.cache = Cache() self.ctx = zmq.Context() self.shared_uuid_fn_dict = _download_manager.shared_uuid_fn_dict self.shared_uuid_map_dict = _download_manager.shared_uuid_map_dict self.download_cond = _download_manager.download_cond
def __init__(self): self.published = {} self.cache = Cache() self.host = socket.gethostname() self.server_thread = None random.seed(os.getpid() + int(time.time() * 1000) % 1000)
class Broadcast: initialized = False is_master = False cache = Cache() broadcastFactory = None BlockSize = 1024 * 1024 def __init__(self, value, is_local): assert value is not None, 'broadcast object should not been None' self.uuid = str(uuid.uuid4()) self.value = value self.is_local = is_local self.bytes = 0 self.stopped = False if is_local: if not self.cache.put(self.uuid, value): raise Exception('object %s is too big to cache', repr(value)) else: self.send() def clear(self): self.stopped = True self.cache.put(self.uuid, None) if hasattr(self, 'value'): delattr(self, 'value') def __getstate__(self): return (self.uuid, self.bytes, self.value if self.bytes < self.BlockSize / 20 else None) def __setstate__(self, v): self.stopped = False self.uuid, self.bytes, value = v if value is not None: self.value = value def __getattr__(self, name): if name != 'value': return getattr(self.value, name) if self.stopped: raise SystemExit("broadcast has been cleared") # in the executor process, Broadcast is not initialized if not self.initialized: raise AttributeError(name) uuid = self.uuid value = self.cache.get(uuid) if value is not None: self.value = value return value oldtitle = getproctitle() setproctitle('dpark worker: broadcasting ' + uuid) value = self.recv() if value is None: raise Exception("recv broadcast failed") self.value = value self.cache.put(uuid, value) setproctitle(oldtitle) return value def send(self): raise NotImplementedError def recv(self): raise NotImplementedError def blockifyObject(self, obj): try: buf = marshal.dumps(obj) except Exception: buf = cPickle.dumps(obj, -1) N = self.BlockSize blockNum = len(buf) / N + 1 val = [ Block(i, compress(buf[i * N:i * N + N])) for i in range(blockNum) ] return val, len(buf) def unBlockifyObject(self, blocks): s = ''.join(decompress(b.data) for b in blocks) try: return marshal.loads(s) except Exception: return cPickle.loads(s) @classmethod def initialize(cls, is_master): if cls.initialized: return cls.initialized = True cls.is_master = is_master cls.host = socket.gethostname() logger.debug("Broadcast initialized") @classmethod def shutdown(cls): pass