def exists(self, hash, want_source=False):
"""Return nonempty if the object exists in the index files."""
global _total_searches, _total_steps
_total_searches += 1
want = str(hash)
el = extract_bits(want, self.bits)
if el:
start = self._fanget(el-1)
startv = el << (32-self.bits)
else:
start = 0
startv = 0
end = self._fanget(el)
endv = (el+1) << (32-self.bits)
_total_steps += 1 # lookup table is a step
hashv = _helpers.firstword(hash)
#print '(%08x) %08x %08x %08x' % (extract_bits(want, 32), startv, hashv, endv)
while start < end:
_total_steps += 1
#print '! %08x %08x %08x %d - %d' % (startv, hashv, endv, start, end)
mid = start + (hashv-startv)*(end-start-1)/(endv-startv)
#print ' %08x %08x %08x %d %d %d' % (startv, hashv, endv, start, mid, end)
v = self._get(mid)
#print ' %08x' % self._num(v)
if v < want:
start = mid+1
startv = _helpers.firstword(v)
elif v > want:
end = mid
endv = _helpers.firstword(v)
else: # got it!
return want_source and self._get_idxname(mid) or True
return None
def exists(self, hash, want_source=False):
"""Return nonempty if the object exists in the index files."""
global _total_searches, _total_steps
_total_searches += 1
want = hash
el = extract_bits(want, self.bits)
if el:
start = self._fanget(el-1)
startv = el << (32-self.bits)
else:
start = 0
startv = 0
end = self._fanget(el)
endv = (el+1) << (32-self.bits)
_total_steps += 1 # lookup table is a step
hashv = _helpers.firstword(hash)
#print '(%08x) %08x %08x %08x' % (extract_bits(want, 32), startv, hashv, endv)
while start < end:
_total_steps += 1
#print '! %08x %08x %08x %d - %d' % (startv, hashv, endv, start, end)
mid = start + (hashv - startv) * (end - start - 1) // (endv - startv)
#print ' %08x %08x %08x %d %d %d' % (startv, hashv, endv, start, mid, end)
v = self._get(mid)
#print ' %08x' % self._num(v)
if v < want:
start = mid+1
startv = _helpers.firstword(v)
elif v > want:
end = mid
endv = _helpers.firstword(v)
else: # got it!
return want_source and self._get_idxname(mid) or True
return None
def __init__(self, filename):
self.name = filename
self.force_keep = False
self.map = None
assert (filename.endswith('.midx'))
self.map = mmap_read(open(filename))
if str(self.map[0:4]) != 'MIDX':
log('Warning: skipping: invalid MIDX header in %r\n' % filename)
self.force_keep = True
return self._init_failed()
ver = struct.unpack('!I', self.map[4:8])[0]
if ver < MIDX_VERSION:
log('Warning: ignoring old-style (v%d) midx %r\n' %
(ver, filename))
self.force_keep = False # old stuff is boring
return self._init_failed()
if ver > MIDX_VERSION:
log('Warning: ignoring too-new (v%d) midx %r\n' % (ver, filename))
self.force_keep = True # new stuff is exciting
return self._init_failed()
self.bits = _helpers.firstword(self.map[8:12])
self.entries = 2**self.bits
self.fanout = buffer(self.map, 12, self.entries * 4)
self.sha_ofs = 12 + self.entries * 4
self.nsha = nsha = self._fanget(self.entries - 1)
self.shatable = buffer(self.map, self.sha_ofs, nsha * 20)
self.which_ofs = self.sha_ofs + 20 * nsha
self.whichlist = buffer(self.map, self.which_ofs, nsha * 4)
self.idxnames = str(self.map[self.which_ofs + 4 * nsha:]).split('\0')
def __init__(self, filename):
self.name = filename
self.force_keep = False
self.map = None
assert(filename.endswith('.midx'))
self.map = mmap_read(open(filename))
if str(self.map[0:4]) != 'MIDX':
log('Warning: skipping: invalid MIDX header in %r\n' % filename)
self.force_keep = True
return self._init_failed()
ver = struct.unpack('!I', self.map[4:8])[0]
if ver < MIDX_VERSION:
log('Warning: ignoring old-style (v%d) midx %r\n'
% (ver, filename))
self.force_keep = False # old stuff is boring
return self._init_failed()
if ver > MIDX_VERSION:
log('Warning: ignoring too-new (v%d) midx %r\n'
% (ver, filename))
self.force_keep = True # new stuff is exciting
return self._init_failed()
self.bits = _helpers.firstword(self.map[8:12])
self.entries = 2**self.bits
self.fanout = buffer(self.map, 12, self.entries*4)
self.sha_ofs = 12 + self.entries*4
self.nsha = nsha = self._fanget(self.entries-1)
self.shatable = buffer(self.map, self.sha_ofs, nsha*20)
self.which_ofs = self.sha_ofs + 20*nsha
self.whichlist = buffer(self.map, self.which_ofs, nsha*4)
self.idxnames = str(self.map[self.which_ofs + 4*nsha:]).split('\0')
def __init__(self, filename):
self.name = filename
self.force_keep = False
self.map = None
assert(filename.endswith(b'.midx'))
self.map = mmap_read(open(filename))
if self.map[0:4] != b'MIDX':
log('Warning: skipping: invalid MIDX header in %r\n'
% path_msg(filename))
self.force_keep = True
return self._init_failed()
ver = struct.unpack('!I', self.map[4:8])[0]
if ver < MIDX_VERSION:
log('Warning: ignoring old-style (v%d) midx %r\n'
% (ver, path_msg(filename)))
self.force_keep = False # old stuff is boring
return self._init_failed()
if ver > MIDX_VERSION:
log('Warning: ignoring too-new (v%d) midx %r\n'
% (ver, path_msg(filename)))
self.force_keep = True # new stuff is exciting
return self._init_failed()
self.bits = _helpers.firstword(self.map[8:12])
self.entries = 2**self.bits
self.fanout_ofs = 12
# fanout len is self.entries * 4
self.sha_ofs = self.fanout_ofs + self.entries * 4
self.nsha = self._fanget(self.entries - 1)
# sha table len is self.nsha * 20
self.which_ofs = self.sha_ofs + 20 * self.nsha
# which len is self.nsha * 4
self.idxnames = self.map[self.which_ofs + 4 * self.nsha:].split(b'\0')
def _fanget(self, i):
if i >= self.entries * 4 or i < 0:
raise IndexError('invalid midx index %d' % i)
ofs = self.fanout_ofs + i * 4
return _helpers.firstword(self.map[ofs : ofs + 4])
def _fanget(self, i):
start = i * 4
s = self.fanout[start:start + 4]
return _helpers.firstword(s)
def _fanget(self, i):
start = i*4
s = self.fanout[start:start+4]
return _helpers.firstword(s)
