def exists(self, hash, want_source=False): """Return nonempty if the object exists in the index files.""" global _total_searches, _total_steps _total_searches += 1 want = str(hash) el = extract_bits(want, self.bits) if el: start = self._fanget(el-1) startv = el << (32-self.bits) else: start = 0 startv = 0 end = self._fanget(el) endv = (el+1) << (32-self.bits) _total_steps += 1 # lookup table is a step hashv = _helpers.firstword(hash) #print '(%08x) %08x %08x %08x' % (extract_bits(want, 32), startv, hashv, endv) while start < end: _total_steps += 1 #print '! %08x %08x %08x %d - %d' % (startv, hashv, endv, start, end) mid = start + (hashv-startv)*(end-start-1)/(endv-startv) #print ' %08x %08x %08x %d %d %d' % (startv, hashv, endv, start, mid, end) v = self._get(mid) #print ' %08x' % self._num(v) if v < want: start = mid+1 startv = _helpers.firstword(v) elif v > want: end = mid endv = _helpers.firstword(v) else: # got it! return want_source and self._get_idxname(mid) or True return None
def exists(self, hash, want_source=False): """Return nonempty if the object exists in the index files.""" global _total_searches, _total_steps _total_searches += 1 want = hash el = extract_bits(want, self.bits) if el: start = self._fanget(el-1) startv = el << (32-self.bits) else: start = 0 startv = 0 end = self._fanget(el) endv = (el+1) << (32-self.bits) _total_steps += 1 # lookup table is a step hashv = _helpers.firstword(hash) #print '(%08x) %08x %08x %08x' % (extract_bits(want, 32), startv, hashv, endv) while start < end: _total_steps += 1 #print '! %08x %08x %08x %d - %d' % (startv, hashv, endv, start, end) mid = start + (hashv - startv) * (end - start - 1) // (endv - startv) #print ' %08x %08x %08x %d %d %d' % (startv, hashv, endv, start, mid, end) v = self._get(mid) #print ' %08x' % self._num(v) if v < want: start = mid+1 startv = _helpers.firstword(v) elif v > want: end = mid endv = _helpers.firstword(v) else: # got it! return want_source and self._get_idxname(mid) or True return None
def __init__(self, filename): self.name = filename self.force_keep = False self.map = None assert (filename.endswith('.midx')) self.map = mmap_read(open(filename)) if str(self.map[0:4]) != 'MIDX': log('Warning: skipping: invalid MIDX header in %r\n' % filename) self.force_keep = True return self._init_failed() ver = struct.unpack('!I', self.map[4:8])[0] if ver < MIDX_VERSION: log('Warning: ignoring old-style (v%d) midx %r\n' % (ver, filename)) self.force_keep = False # old stuff is boring return self._init_failed() if ver > MIDX_VERSION: log('Warning: ignoring too-new (v%d) midx %r\n' % (ver, filename)) self.force_keep = True # new stuff is exciting return self._init_failed() self.bits = _helpers.firstword(self.map[8:12]) self.entries = 2**self.bits self.fanout = buffer(self.map, 12, self.entries * 4) self.sha_ofs = 12 + self.entries * 4 self.nsha = nsha = self._fanget(self.entries - 1) self.shatable = buffer(self.map, self.sha_ofs, nsha * 20) self.which_ofs = self.sha_ofs + 20 * nsha self.whichlist = buffer(self.map, self.which_ofs, nsha * 4) self.idxnames = str(self.map[self.which_ofs + 4 * nsha:]).split('\0')
def __init__(self, filename): self.name = filename self.force_keep = False self.map = None assert(filename.endswith('.midx')) self.map = mmap_read(open(filename)) if str(self.map[0:4]) != 'MIDX': log('Warning: skipping: invalid MIDX header in %r\n' % filename) self.force_keep = True return self._init_failed() ver = struct.unpack('!I', self.map[4:8])[0] if ver < MIDX_VERSION: log('Warning: ignoring old-style (v%d) midx %r\n' % (ver, filename)) self.force_keep = False # old stuff is boring return self._init_failed() if ver > MIDX_VERSION: log('Warning: ignoring too-new (v%d) midx %r\n' % (ver, filename)) self.force_keep = True # new stuff is exciting return self._init_failed() self.bits = _helpers.firstword(self.map[8:12]) self.entries = 2**self.bits self.fanout = buffer(self.map, 12, self.entries*4) self.sha_ofs = 12 + self.entries*4 self.nsha = nsha = self._fanget(self.entries-1) self.shatable = buffer(self.map, self.sha_ofs, nsha*20) self.which_ofs = self.sha_ofs + 20*nsha self.whichlist = buffer(self.map, self.which_ofs, nsha*4) self.idxnames = str(self.map[self.which_ofs + 4*nsha:]).split('\0')
def __init__(self, filename): self.name = filename self.force_keep = False self.map = None assert(filename.endswith(b'.midx')) self.map = mmap_read(open(filename)) if self.map[0:4] != b'MIDX': log('Warning: skipping: invalid MIDX header in %r\n' % path_msg(filename)) self.force_keep = True return self._init_failed() ver = struct.unpack('!I', self.map[4:8])[0] if ver < MIDX_VERSION: log('Warning: ignoring old-style (v%d) midx %r\n' % (ver, path_msg(filename))) self.force_keep = False # old stuff is boring return self._init_failed() if ver > MIDX_VERSION: log('Warning: ignoring too-new (v%d) midx %r\n' % (ver, path_msg(filename))) self.force_keep = True # new stuff is exciting return self._init_failed() self.bits = _helpers.firstword(self.map[8:12]) self.entries = 2**self.bits self.fanout_ofs = 12 # fanout len is self.entries * 4 self.sha_ofs = self.fanout_ofs + self.entries * 4 self.nsha = self._fanget(self.entries - 1) # sha table len is self.nsha * 20 self.which_ofs = self.sha_ofs + 20 * self.nsha # which len is self.nsha * 4 self.idxnames = self.map[self.which_ofs + 4 * self.nsha:].split(b'\0')
def _fanget(self, i): if i >= self.entries * 4 or i < 0: raise IndexError('invalid midx index %d' % i) ofs = self.fanout_ofs + i * 4 return _helpers.firstword(self.map[ofs : ofs + 4])
def _fanget(self, i): start = i * 4 s = self.fanout[start:start + 4] return _helpers.firstword(s)
def _fanget(self, i): start = i*4 s = self.fanout[start:start+4] return _helpers.firstword(s)