def iterate_key(filepath, groups):
'''
Iterates key and sort each key key_infos by timestamp and size
'''
groups = set(groups)
for key, key_infos in load_key_data(filepath):
if len(key_infos) + len(groups) > 1:
key_infos = sorted(key_infos,
key=lambda x: (x.timestamp, x.size),
reverse=True)
missed_groups = tuple(
groups.difference([k.group_id for k in key_infos]))
# if all key_infos has the same timestamp and size and there is no missed groups -
# skip key, it is already up-to-date in all groups
same_meta = lambda lhs, rhs: (lhs.timestamp, lhs.size) == (
rhs.timestamp, rhs.size)
if same_meta(key_infos[0], key_infos[-1]) and not missed_groups:
continue
yield (key, key_infos, missed_groups)
else:
log.error(
"Invalid number of replicas for key: {0}: infos_count: {1}, groups_count: {2}"
.format(key, len(key_infos), len(groups)))
def iterate_key(filepath, groups):
'''
Iterates key and sort each key key_infos by timestamp and size
'''
groups = set(groups)
for key, key_infos in load_key_data(filepath):
if len(key_infos) + len(groups) > 1:
key_infos = sorted(key_infos, key=lambda x: (x.timestamp, x.size), reverse=True)
missed_groups = tuple(groups.difference([k.group_id for k in key_infos]))
#if all key_infos has the same timestamp and size and there is no missed groups - skip key, it is already up-to-date in all groups
if (key_infos[0].timestamp, key_infos[0].size) == (key_infos[-1].timestamp, key_infos[-1].size) and not missed_groups:
continue
yield (key, key_infos, missed_groups)
else:
log.error("Invalid number of replicas for key: {0}: infos_count: {1}, groups_count: {2}".format(key, len(key_infos), len(groups)))