def _fetch_missing(self, key):
result = {}
urls = env.trackerClient.call(GetValueMessage('mutable_dict:%s' % key))
for url in urls:
f = urllib.urlopen(url)
if f.code is not None and f.code != 200:
raise IOError('Open %s failed:%s' % (url, f.code))
data = f.read()
if len(data) < 4:
raise IOError('Transfer %s failed: %s received' % (url, len(data)))
length, = struct.unpack('<I', data[:4])
if length != len(data):
raise IOError('Transfer %s failed: %s received, %s expected' % (url,
len(data), length))
data = cPickle.loads(decompress(data[4:]))
for k,v in data.items():
if k in result:
r = result[k]
if v[1] == r[1]:
r = r.value if isinstance(r, ConflictValues) else [r]
r += v.value if isinstance(v, ConflictValues) else [v]
result[k] = ConflictValues(r)
else:
result[k] = v if v[1] > r[1] else r
else:
result[k] = v
return result
def getCacheUri(self, rdd_id, index):
return self.client.call(
GetValueMessage('cache:%s-%s' % (rdd_id, index)))
def getServerUris(self, shuffle_id):
locs = self.client.call(GetValueMessage('shuffle:%s' % shuffle_id))
logger.debug("Fetch done: %s", locs)
return locs
def get_locs(cls, shuffle_id):
key = cls.get_key(shuffle_id)
if env.trackerServer:
return env.trackerServer.get(key)
else:
return env.trackerClient.call(GetValueMessage(key))