class MapperServer(Member):
def __init__(self, domain):
Member.__init__(self)
mappers = get_mappers(domain)
if not mappers:
log_err(self, 'failed to initialize')
raise Exception(log_get(self, 'failed to initialize'))
self.set_members(mappers)
self._server = Member()
servers = get_servers(domain)
if not servers:
log_err(self, 'failed to initialize')
raise Exception(log_get(self, 'failed to initialize'))
self._server.set_members(servers)
self._cache = MapperCache(self._server, domain)
def set_servers(self, servers):
return self._server.set_members(servers, sort=True)
def add_servers(self, servers, pos):
return self._server.add_members(servers, pos, sort=True)
def check_servers(self, servers, pos):
return self._server.check_members(servers, pos)
def get_servers(self, pos):
return self._server.get_members(pos)
def get(self, key):
return self._cache.get(key)
class MapperCache(object):
def __init__(self, server, domain):
self._cache = {}
self._server = server
self._domain = domain
self._finder = Member()
finders = get_finders(domain)
if not finders:
log_err(self, 'failed to initialize')
raise Exception(log_get(self, 'failed to initialize'))
self._finder.set_members(finders)
self._thread = Thread(target=self._update_finders)
self._thread.start()
def _log(self, text):
if LOG_MAPPER:
log_debug(self, text)
def _get_finder(self, addr):
finder = zerorpc.Client()
if self._domain == DOMAIN_USR:
finder.connect(zmqaddr(addr, USR_FINDER_PORT))
else:
finder.connect(zmqaddr(addr, DEV_FINDER_PORT))
return finder
def _do_update_finders(self):
pos = self._finder.length()
n = randint(0, pos - 1)
addr = self._finder.get(n)
finder = self._get_finder(addr)
if finder:
try:
members = finder.get_members(pos)
if members:
self._log('new finders %s (pos=%d)' % (str(members), pos))
self._finder.add_members(members, pos)
finally:
finder.close()
def _update_finders(self):
while True:
try:
time.sleep(WAIT_TIME)
self._do_update_finders()
except:
log_err(self, 'failed to update finders')
def _get(self, key):
length = self._finder.length()
if length < GROUP_MAX:
groups = length
else:
groups = GROUP_MAX
g = abs(hash(key)) % groups
sz = length / groups
if sz <= 0:
log_err(self, 'failed to get, invalid group size')
raise Exception(log_get(self, 'failed to get, invalid group size'))
n = g * sz + randint(0, sz - 1)
addr = self._finder.get(n)
finder = self._get_finder(addr)
if finder:
try:
val = finder.get(key)
if type(val) == list and len(val) == 2 and type(val[0]) == float and type(val[1]) == int:
return val
finally:
finder.close()
def _map(self, key, t, area):
length = self._server.length()
if not length:
return
start = 0
end = length - 1
left = self._server.get(start)
right = self._server.get(end)
if not left or not right:
return
if 1 == length:
return left[6]
while True:
if length < 2:
if right[0] < t:
pos = end
else:
pos = start
break
length /= 2
middle = start + length
curr = self._server.get(middle)
if not curr:
return
if curr[0] < t:
start = middle
left = curr
else:
end = middle
right = curr
curr = self._server.get(pos)
if not curr:
return
if curr[3] == 1:
return curr[6]
elif curr[2] < curr[3]:
length = curr[3]
start = pos - curr[2]
end = pos + curr[3] - curr[2] - 1
left = self._server.get(start)
right = self._server.get(end)
if left and right:
while True:
if length < 2:
if left[1] != right[1]:
if abs(area - left[1]) <= abs(area - right[1]):
return left[6]
else:
return right[6]
else:
if left != right:
if left[5] <= 1 or left[4] >= left[5]:
return
n = abs(hash(key)) % left[5]
start += n - left[4]
left = self._server.get(start)
if not left:
return
return left[6]
length /= 2
middle = start + length
curr = self._server.get(middle)
if not curr:
return
if curr[1] < area:
start = middle
left = curr
else:
right = curr
def get(self, key):
res = self._cache.get(key)
if res:
return res
val = self._get(key)
if val:
res = self._map(key, val[0], val[1])
if not res:
log_err(self, 'failed to get')
raise Exception(log_get(self, 'failed to get'))
if len(self._cache) >= CACHE_MAX:
self._cache.popitem()
self._cache.update({key:res})
self._log('get, key=%s, val=%s' % (str(key), str(res)))
return res
