def materialized_result_set_compare(oset, materialized_set):
"""
Compares the actual data (keys, values) between two result sets. 'oset'
must contain normal topics, while 'materaizlied_set' contains
materialized_topics
"""
if len(oset.rset.keys()) != len(materialized_set.rset.keys()):
return False
if oset.num_records() != materialized_set.num_records():
return False
def strip_topic(bkr):
bkr.topic = None
return bkr
for tp, records in materialized_set.rset.items():
if not is_materialized_topic(tp.topic):
raise Exception(
"'materialized_set' must contain only materialized topics")
input_data = oset.rset.get(
TopicPartition(topic=get_source_topic(tp.topic),
partition=tp.partition))
if input_data is None:
return False
mat_records = [strip_topic(x) for x in records]
src_recs = [strip_topic(x) for x in input_data]
if mat_records != src_recs:
return False
return True
def _materialized_topic_set_compare(a, b, comparator):
for tp, records in b.rset.items():
if not is_materialized_topic(tp.topic):
return CmpErr.DataInvalid
input_data = a.rset.get(
TopicPartition(topic=get_source_topic(tp.topic),
partition=tp.partition))
if input_data is None or not comparator(input_data, records):
return CmpErr.CmpFailed
return CmpErr.Success
def _materialized_topic_set_compare(a, b, comparator):
for tp, records in b.rset.items():
if not is_materialized_topic(tp.topic):
raise Exception(
"'materialized_set' must contain only materialized topics")
input_data = a.rset.get(
TopicPartition(topic=get_source_topic(tp.topic),
partition=tp.partition))
if input_data is None or not comparator(input_data, records):
return False
return True
def to_output_topic_spec(output_topics):
"""
Create a list of TopicPartitions for the set of output topics.
Must parse the materialzied topic for the input topic to determine
the number of partitions.
"""
input_lookup = dict([(x.name, x) for x in topic_spec])
result = []
for output_topic, _ in output_topics:
src = input_lookup.get(get_source_topic(output_topic))
if src is None:
raise Exception(
"Bad spec, materialized topics source must belong to "
"the input topic spec set")
result.append(
TopicSpec(name=output_topic,
partition_count=src.partition_count,
replication_factor=src.replication_factor,
cleanup_policy=src.cleanup_policy))
return result
def compare(topic):
iis = input_results.filter(lambda x: x.topic == topic)
oos = output_results.filter(
lambda x: get_source_topic(x.topic) == topic)
return iis.num_records() == oos.num_records()
def accrue(acc, output_topic):
src = get_source_topic(output_topic)
num_records = [x[1] for x in topic_spec if x[0].name == src]
assert (len(num_records) == 1)
return acc + num_records[0]