def locate_key_iterate(db_homes, db_depth, key, ver_=None):
""" assume disk0 already have link,
Returns
True if key exists and valid
False if key not exist
"""
if isinstance(db_homes, (list, tuple)):
db_home = db_homes[0]
else:
db_home = db_homes
key_hash = get_khash(key)
if db_depth == 1:
sector = (key_hash >> 28) & 0xf
sector_path = "%x" % (sector)
g = glob.glob(os.path.join(db_home, sector_path, "*.data"))
elif db_depth == 2:
sector1 = (key_hash >> 28) & 0xf
sector2 = (key_hash >> 24) & 0xf
sector_path = "%x/%x" % (sector1, sector2)
g = glob.glob(os.path.join(db_home, sector_path, "*.data"))
else:
raise NotImplementedError()
for data_file in g:
print data_file
if check_data_with_key(data_file, key, ver_=ver_):
return True
return False
def __init__(self, servers, buckets_count=16, N=3, W=1, R=1):
self.buckets_count = buckets_count
self.bucket_size = self.hash_space / buckets_count
self.servers = {}
self.server_buckets = {}
self.buckets = [[] for i in range(buckets_count)]
for s, bs in servers.items():
server = DBClient(s)
self.servers[s] = server
self.server_buckets[s] = bs
for b in bs:
self.buckets[b].append(server)
for b in range(self.buckets_count):
self.buckets[b].sort(
key=lambda x: get_khash("%d:%s:%d" % (b, x, b)))
self.N = N
self.W = W
self.R = R
def check_data_first_key(file_path, bucket):
""" if ok , return first_key """
assert isinstance(bucket, tuple)
assert isinstance(bucket[0], int)
with DataFile(file_path, check_crc=False, decompress_value=False) as f:
for (_, rec) in f:
key = rec[R_KEY]
_hash = get_khash(key)
if len(bucket) == 1:
if _hash >> 28 == bucket[0]:
return key
raise ValueError("%s belongs to %x" % (file_path, _hash >> 28))
elif len(bucket) == 2:
if (_hash >> 28 == bucket[0]
and ((_hash >> 24) & 0xf) == bucket[1]):
return key
raise ValueError("%s belongs to %x" % (file_path, _hash >> 24))
else:
raise NotImplementedError()
def locate_key_with_hint(db_homes, db_depth, key, ver_=None):
""" assume disk0 already have link,
if key exists and valid return True, if key not exist return False
"""
if isinstance(db_homes, (list, tuple)):
db_home = db_homes[0]
else:
db_home = db_homes
key_hash = get_khash(key)
if db_depth == 1:
sector = (key_hash >> 28) & 0xf
sector_path = "%x" % (sector)
g = glob.glob(os.path.join(db_home, sector_path, "*.hint.s"))
elif db_depth == 2:
sector1 = (key_hash >> 28) & 0xf
sector2 = (key_hash >> 24) & 0xf
sector_path = "%x/%x" % (sector1, sector2)
g = glob.glob(os.path.join(db_home, sector_path, "*.hint.s"))
else:
raise NotImplementedError()
for hint_file in g:
r = get_keyinfo_from_hint(hint_file, key)
if r is not None:
pos, ver, hash_ = r
data_file = re.sub(r'(.+)\.hint.s', r'\1.data',
os.path.basename(hint_file))
data_file = os.path.join(db_home, sector_path, data_file)
print "file", data_file, "pos", pos, "ver", ver
if ver_ is not None and ver != ver_:
continue
assert check_data_with_key(data_file,
key,
ver_=ver,
hash_=hash_ if ver_ > 0 else None,
pos=pos)
return True
return False
def _get_servers(self, key):
hash_ = get_khash(key)
b = hash_ / self.bucket_size
return self.buckets[b]
def get_bucket_from_key(key, db_depth=1):
assert db_depth <= 2
hash_ = get_khash(key)
return hash_ >> (32 - db_depth * 4)