class WorksetMapper(object): """maps URLs to WorkSet""" def __init__(self, nworksets_bits): self.nworksets_bits = nworksets_bits self.nworksets = (1 << self.nworksets_bits) self._fp31 = FPGenerator(0xBA75BB4300000000, 31) def workset(self, curi): '''reutrns WorkSet id to which curi should be dispatched''' uc = urlsplit(curi['u']) h = uc.netloc p = h.find(':') if p > 0: h = h[:p] # Note: don't use % (mod) - FP hash much less even distribution in # lower bits. hosthash = int(self._fp31.fp(h) >> (64 - self.nworksets_bits)) return hosthash
class WorksetMapper(object): """maps URLs to WorkSet""" def __init__(self, nworksets_bits): self.nworksets_bits = nworksets_bits self.nworksets = (1 << self.nworksets_bits) self._fp31 = FPGenerator(0xBA75BB4300000000, 31) def hosthash(self, curi): prefix = tasprefix.prefix(curi) if isinstance(prefix, unicode): prefix = prefix.encode('utf-8') return int(self._fp31.fp(prefix)) def workset(self, curi): '''reutrns WorkSet id to which curi should be dispatched''' hosthash = self.hosthash(curi['u']) # Note: don't use % (mod) - FP hash is much less evenly distributed # in lower bits. ws = hosthash >> (64 - self.nworksets_bits) return ws
def __init__(self, nworksets_bits): self.nworksets_bits = nworksets_bits self.nworksets = (1 << self.nworksets_bits) self._fp31 = FPGenerator(0xBA75BB4300000000, 31)