def locate_key_iterate(db_homes, db_depth, key, ver_=None): """ assume disk0 already have link, Returns True if key exists and valid False if key not exist """ if isinstance(db_homes, (list, tuple)): db_home = db_homes[0] else: db_home = db_homes key_hash = get_khash(key) if db_depth == 1: sector = (key_hash >> 28) & 0xf sector_path = "%x" % (sector) g = glob.glob(os.path.join(db_home, sector_path, "*.data")) elif db_depth == 2: sector1 = (key_hash >> 28) & 0xf sector2 = (key_hash >> 24) & 0xf sector_path = "%x/%x" % (sector1, sector2) g = glob.glob(os.path.join(db_home, sector_path, "*.data")) else: raise NotImplementedError() for data_file in g: print data_file if check_data_with_key(data_file, key, ver_=ver_): return True return False
def __init__(self, servers, buckets_count=16, N=3, W=1, R=1): self.buckets_count = buckets_count self.bucket_size = self.hash_space / buckets_count self.servers = {} self.server_buckets = {} self.buckets = [[] for i in range(buckets_count)] for s, bs in servers.items(): server = DBClient(s) self.servers[s] = server self.server_buckets[s] = bs for b in bs: self.buckets[b].append(server) for b in range(self.buckets_count): self.buckets[b].sort( key=lambda x: get_khash("%d:%s:%d" % (b, x, b))) self.N = N self.W = W self.R = R
def check_data_first_key(file_path, bucket): """ if ok , return first_key """ assert isinstance(bucket, tuple) assert isinstance(bucket[0], int) with DataFile(file_path, check_crc=False, decompress_value=False) as f: for (_, rec) in f: key = rec[R_KEY] _hash = get_khash(key) if len(bucket) == 1: if _hash >> 28 == bucket[0]: return key raise ValueError("%s belongs to %x" % (file_path, _hash >> 28)) elif len(bucket) == 2: if (_hash >> 28 == bucket[0] and ((_hash >> 24) & 0xf) == bucket[1]): return key raise ValueError("%s belongs to %x" % (file_path, _hash >> 24)) else: raise NotImplementedError()
def locate_key_with_hint(db_homes, db_depth, key, ver_=None): """ assume disk0 already have link, if key exists and valid return True, if key not exist return False """ if isinstance(db_homes, (list, tuple)): db_home = db_homes[0] else: db_home = db_homes key_hash = get_khash(key) if db_depth == 1: sector = (key_hash >> 28) & 0xf sector_path = "%x" % (sector) g = glob.glob(os.path.join(db_home, sector_path, "*.hint.s")) elif db_depth == 2: sector1 = (key_hash >> 28) & 0xf sector2 = (key_hash >> 24) & 0xf sector_path = "%x/%x" % (sector1, sector2) g = glob.glob(os.path.join(db_home, sector_path, "*.hint.s")) else: raise NotImplementedError() for hint_file in g: r = get_keyinfo_from_hint(hint_file, key) if r is not None: pos, ver, hash_ = r data_file = re.sub(r'(.+)\.hint.s', r'\1.data', os.path.basename(hint_file)) data_file = os.path.join(db_home, sector_path, data_file) print "file", data_file, "pos", pos, "ver", ver if ver_ is not None and ver != ver_: continue assert check_data_with_key(data_file, key, ver_=ver, hash_=hash_ if ver_ > 0 else None, pos=pos) return True return False
def _get_servers(self, key): hash_ = get_khash(key) b = hash_ / self.bucket_size return self.buckets[b]
def get_bucket_from_key(key, db_depth=1): assert db_depth <= 2 hash_ = get_khash(key) return hash_ >> (32 - db_depth * 4)