def lookup_group(self, group_name):
'''Retrieve a Group object with the specified name or None. The field
names starting with "group_" should set group level attributes. The
field names starting with "default_" should set field defaults for the
group schemas.'''
group_key = self.get_group_key(group_name)
group_dict = self.redis.hgetall(group_key)
if group_dict:
group = Group(group_name, group_dict)
group.current_schema = self.get_latest_schema_for_group(group_name)
return group
def get_versions_for_id_str_and_group(self, id_str, group_name):
'''Given an id_str and a group, we should be able to figure out which
of the group's registered schema versions used the identified schema.
We ensure we are using an sha256-based id (which are used to populate
the vid.* lists), then we step through the list for the group and note
which list ids match. This lets you identify non-sequential re-
registration of the same schema. This does not come up much, so we
don't bother with a LUA script.
'''
if not Group.validate_group_name(group_name):
raise InvalidGroupException('Bad group name: %s' % group_name)
base64_id = id_str[3:] if id_str.startswith('id.') else id_str
sha256_key = None
if len(base64_id) == 44:
# we have an sha256 id already
sha256_key = u'id.%s' % base64_id
elif len(base64_id) == 24:
# we have an md5 id, so get the sha256 id from redis
md5_key = u'id.%s' % base64_id
sha256_key = self.redis.hget(md5_key, 'sha256_id')
vid_key = u'vid.%s' % group_name
vlist = []
version = 0
for vid in self.redis.lrange(vid_key, 0, -1):
version += 1
if vid == sha256_key:
vlist.append(version)
return vlist
def get_all_version_sha256_ids_for_group(self, group_name):
'''Get the list of sha256_id values identifying group schema versions.
'''
if not Group.validate_group_name(group_name):
raise InvalidGroupException('Bad group name: %s' % group_name)
vid_key = u'vid.%s' % group_name
return self.redis.lrange(vid_key, 0, -1)
def get_schema_for_group_and_version(self, group_name, version):
'''Gets the registered schema for a group_name and version using the
registered LUA script. Note that version must be a whole integer
greater than 0 or -1, which is a flag for the most current version.
'''
if not Group.validate_group_name(group_name):
raise InvalidGroupException('Bad group name: %s' % group_name)
vid_key = u'vid.%s' % group_name
index = int(version)
if index == 0:
# 0 is an invalid version here, we count from 1, not 0
return
elif index < -1:
# -1 is a flag for current, but other negative values are invalid
return
elif index > 0:
index -= 1 # ver counts from 1, index from 0
rvals = self.lua_get_for_group_and_version(keys=[vid_key, index, ])
if rvals:
rs_d = RedisSchemaRepository.pair_seq_2_dict(rvals)
retrieved_rs = self.instantiate_registered_schema()
retrieved_rs.update_from_dict(rs_d)
return retrieved_rs
def get_latest_schema_versions_for_group(self, group_name, max_versions=5):
'''This retrieves the n most recent schema versions for a group. If
max_versions is set to -1, it will return ALL versions for the group.
Note that this is iterative, and will generate a call for each version
returned, so keep the depth reasonable.
'''
if not Group.validate_group_name(group_name):
raise InvalidGroupException('Bad group name: %s' % group_name)
versions = []
rs = self.get_latest_schema_for_group(group_name)
if rs:
# there is a latest version
versions.insert(0, rs)
cur_ver = rs.current_version(group_name)
if max_versions < 0:
first_ver = 1
else:
first_ver = cur_ver - max_versions + 1 # counting from 1
first_ver = 1 if first_ver < 1 else first_ver
for ver_num in sorted(range(first_ver, cur_ver), reverse=True):
schema = self.get_schema_for_group_and_version(group_name,
ver_num)
versions.insert(0, schema)
return versions
def register_schema(self, group_name, schema_str):
'''Register a schema string as a version for a group_name.'''
if not Group.validate_group_name(group_name):
raise InvalidGroupException('Bad group name: %s' % group_name)
new_rs = self.instantiate_registered_schema()
new_rs.schema_str = schema_str
if not new_rs.validate_schema_str():
raise ValueError(u'Cannot register_schema invalid schema.')
# make sure the group_name is registered first
self.register_group(group_name)
# the key values are what we use as Redis keys
sha256_key = u'id.%s' % new_rs.sha256_id
md5_key = u'id.%s' % new_rs.md5_id
vid_key = u'vid.%s' % group_name
vts_key = u'vts.%s' % group_name
now = long(time.time())
# we also need to support the old topic.* lists as well for Vadim
topic_key = u'topic.%s' % group_name
old_d = self.get_schema_dict_for_sha256_id(new_rs.sha256_id)
if old_d:
# schema registered for some group already, so make sure gv_dict
# and ts_dict are copied from old_d to new_rs so current_version()
# call will work
new_rs.update_from_dict(old_d)
else:
self.redis.hmset(sha256_key, new_rs.as_dict())
self.redis.hset(md5_key, 'sha256_id', sha256_key)
new_rs.created = True
# now that we know the schema is in the hashes, reg for the group_name
if not new_rs.current_version(group_name):
# no version for this group_name, so add it
ver = self.redis.rpush(vid_key, sha256_key)
# ensure we append the topic.* list as well for back-compatibility
topic_ver = self.redis.rpush(topic_key, sha256_key)
if ver != topic_ver:
sys.stderr.write('vid.* and topic.* version mismatch')
self.redis.hset(sha256_key, vid_key, ver)
new_rs.gv_dict[group_name] = ver
self.redis.rpush(vts_key, now)
self.redis.hset(sha256_key, vts_key, now)
new_rs.ts_dict[group_name] = now
new_rs.created = True
else:
last_ver_sha256_key = self.redis.lrange(vid_key, -1, -1)[0]
if not last_ver_sha256_key == sha256_key:
# need to override outdated version entry with new one
ver = self.redis.rpush(vid_key, sha256_key)
# ensure we append the topic.* list as well (for now)
topic_ver = self.redis.rpush(topic_key, sha256_key)
if ver != topic_ver:
sys.stderr.write('vid.* and topic.* version mismatch')
self.redis.hset(sha256_key, vid_key, ver)
new_rs.gv_dict[group_name] = ver
self.redis.rpush(vts_key, now)
self.redis.hset(sha256_key, vts_key, now)
new_rs.ts_dict[group_name] = now
# since we are creating a version entry, it counts as creation
new_rs.created = True
return new_rs
def get_group_key(self, group_name):
'''A util method to get the redis key used for the group hash.'''
if not Group.validate_group_name(group_name):
raise InvalidGroupException('Bad group name: %s' % group_name)
return 'g.%s' % group_name
