def start(self):
if self._started:
return
self._started = True
start_download_manager()
self.guide_addr = env.get(GUIDE_ADDR)
self.download_addr = env.get(DOWNLOAD_ADDR)
self.cache = Cache()
self.ctx = zmq.Context()
self.shared_uuid_fn_dict = _download_manager.shared_uuid_fn_dict
self.shared_uuid_map_dict = _download_manager.shared_uuid_map_dict
self.download_cond = _download_manager.download_cond
def __init__(self):
self.published = {}
self.cache = Cache()
self.host = socket.gethostname()
self.server_thread = None
random.seed(os.getpid() + int(time.time() * 1000) % 1000)
class Broadcast:
initialized = False
is_master = False
cache = Cache()
broadcastFactory = None
BlockSize = 1024 * 1024
def __init__(self, value, is_local):
assert value is not None, 'broadcast object should not been None'
self.uuid = str(uuid.uuid4())
self.value = value
self.is_local = is_local
self.bytes = 0
self.stopped = False
if is_local:
if not self.cache.put(self.uuid, value):
raise Exception('object %s is too big to cache', repr(value))
else:
self.send()
def clear(self):
self.stopped = True
self.cache.put(self.uuid, None)
if hasattr(self, 'value'):
delattr(self, 'value')
def __getstate__(self):
return (self.uuid, self.bytes,
self.value if self.bytes < self.BlockSize / 20 else None)
def __setstate__(self, v):
self.stopped = False
self.uuid, self.bytes, value = v
if value is not None:
self.value = value
def __getattr__(self, name):
if name != 'value':
return getattr(self.value, name)
if self.stopped:
raise SystemExit("broadcast has been cleared")
# in the executor process, Broadcast is not initialized
if not self.initialized:
raise AttributeError(name)
uuid = self.uuid
value = self.cache.get(uuid)
if value is not None:
self.value = value
return value
oldtitle = getproctitle()
setproctitle('dpark worker: broadcasting ' + uuid)
value = self.recv()
if value is None:
raise Exception("recv broadcast failed")
self.value = value
self.cache.put(uuid, value)
setproctitle(oldtitle)
return value
def send(self):
raise NotImplementedError
def recv(self):
raise NotImplementedError
def blockifyObject(self, obj):
try:
buf = marshal.dumps(obj)
except Exception:
buf = cPickle.dumps(obj, -1)
N = self.BlockSize
blockNum = len(buf) / N + 1
val = [
Block(i, compress(buf[i * N:i * N + N])) for i in range(blockNum)
]
return val, len(buf)
def unBlockifyObject(self, blocks):
s = ''.join(decompress(b.data) for b in blocks)
try:
return marshal.loads(s)
except Exception:
return cPickle.loads(s)
@classmethod
def initialize(cls, is_master):
if cls.initialized:
return
cls.initialized = True
cls.is_master = is_master
cls.host = socket.gethostname()
logger.debug("Broadcast initialized")
@classmethod
def shutdown(cls):
pass
