def get_versions_for_id_str_and_group(self, id_str, group_name): '''Given an id_str and a group, we should be able to figure out which of the group's registered schema versions used the identified schema. We ensure we are using an sha256-based id (which are used to populate the vid.* lists), then we step through the list for the group and note which list ids match. This lets you identify non-sequential re- registration of the same schema. This does not come up much, so we don't bother with a LUA script. ''' if not Group.validate_group_name(group_name): raise InvalidGroupException('Bad group name: %s' % group_name) base64_id = id_str[3:] if id_str.startswith('id.') else id_str sha256_key = None if len(base64_id) == 44: # we have an sha256 id already sha256_key = u'id.%s' % base64_id elif len(base64_id) == 24: # we have an md5 id, so get the sha256 id from redis md5_key = u'id.%s' % base64_id sha256_key = self.redis.hget(md5_key, 'sha256_id') vid_key = u'vid.%s' % group_name vlist = [] version = 0 for vid in self.redis.lrange(vid_key, 0, -1): version += 1 if vid == sha256_key: vlist.append(version) return vlist
def get_all_version_sha256_ids_for_group(self, group_name): '''Get the list of sha256_id values identifying group schema versions. ''' if not Group.validate_group_name(group_name): raise InvalidGroupException('Bad group name: %s' % group_name) vid_key = u'vid.%s' % group_name return self.redis.lrange(vid_key, 0, -1)
def get_schema_for_group_and_version(self, group_name, version): '''Gets the registered schema for a group_name and version using the registered LUA script. Note that version must be a whole integer greater than 0 or -1, which is a flag for the most current version. ''' if not Group.validate_group_name(group_name): raise InvalidGroupException('Bad group name: %s' % group_name) vid_key = u'vid.%s' % group_name index = int(version) if index == 0: # 0 is an invalid version here, we count from 1, not 0 return elif index < -1: # -1 is a flag for current, but other negative values are invalid return elif index > 0: index -= 1 # ver counts from 1, index from 0 rvals = self.lua_get_for_group_and_version(keys=[vid_key, index, ]) if rvals: rs_d = RedisSchemaRepository.pair_seq_2_dict(rvals) retrieved_rs = self.instantiate_registered_schema() retrieved_rs.update_from_dict(rs_d) return retrieved_rs
def get_latest_schema_versions_for_group(self, group_name, max_versions=5): '''This retrieves the n most recent schema versions for a group. If max_versions is set to -1, it will return ALL versions for the group. Note that this is iterative, and will generate a call for each version returned, so keep the depth reasonable. ''' if not Group.validate_group_name(group_name): raise InvalidGroupException('Bad group name: %s' % group_name) versions = [] rs = self.get_latest_schema_for_group(group_name) if rs: # there is a latest version versions.insert(0, rs) cur_ver = rs.current_version(group_name) if max_versions < 0: first_ver = 1 else: first_ver = cur_ver - max_versions + 1 # counting from 1 first_ver = 1 if first_ver < 1 else first_ver for ver_num in sorted(range(first_ver, cur_ver), reverse=True): schema = self.get_schema_for_group_and_version(group_name, ver_num) versions.insert(0, schema) return versions
def register_schema(self, group_name, schema_str): '''Register a schema string as a version for a group_name.''' if not Group.validate_group_name(group_name): raise InvalidGroupException('Bad group name: %s' % group_name) new_rs = self.instantiate_registered_schema() new_rs.schema_str = schema_str if not new_rs.validate_schema_str(): raise ValueError(u'Cannot register_schema invalid schema.') # make sure the group_name is registered first self.register_group(group_name) # the key values are what we use as Redis keys sha256_key = u'id.%s' % new_rs.sha256_id md5_key = u'id.%s' % new_rs.md5_id vid_key = u'vid.%s' % group_name vts_key = u'vts.%s' % group_name now = long(time.time()) # we also need to support the old topic.* lists as well for Vadim topic_key = u'topic.%s' % group_name old_d = self.get_schema_dict_for_sha256_id(new_rs.sha256_id) if old_d: # schema registered for some group already, so make sure gv_dict # and ts_dict are copied from old_d to new_rs so current_version() # call will work new_rs.update_from_dict(old_d) else: self.redis.hmset(sha256_key, new_rs.as_dict()) self.redis.hset(md5_key, 'sha256_id', sha256_key) new_rs.created = True # now that we know the schema is in the hashes, reg for the group_name if not new_rs.current_version(group_name): # no version for this group_name, so add it ver = self.redis.rpush(vid_key, sha256_key) # ensure we append the topic.* list as well for back-compatibility topic_ver = self.redis.rpush(topic_key, sha256_key) if ver != topic_ver: sys.stderr.write('vid.* and topic.* version mismatch') self.redis.hset(sha256_key, vid_key, ver) new_rs.gv_dict[group_name] = ver self.redis.rpush(vts_key, now) self.redis.hset(sha256_key, vts_key, now) new_rs.ts_dict[group_name] = now new_rs.created = True else: last_ver_sha256_key = self.redis.lrange(vid_key, -1, -1)[0] if not last_ver_sha256_key == sha256_key: # need to override outdated version entry with new one ver = self.redis.rpush(vid_key, sha256_key) # ensure we append the topic.* list as well (for now) topic_ver = self.redis.rpush(topic_key, sha256_key) if ver != topic_ver: sys.stderr.write('vid.* and topic.* version mismatch') self.redis.hset(sha256_key, vid_key, ver) new_rs.gv_dict[group_name] = ver self.redis.rpush(vts_key, now) self.redis.hset(sha256_key, vts_key, now) new_rs.ts_dict[group_name] = now # since we are creating a version entry, it counts as creation new_rs.created = True return new_rs
def get_group_key(self, group_name): '''A util method to get the redis key used for the group hash.''' if not Group.validate_group_name(group_name): raise InvalidGroupException('Bad group name: %s' % group_name) return 'g.%s' % group_name