def comsumer():
# consumer = SimpleConsumer(client, group=None,
# topic=topic, partitions=[0, ],
# auto_commit=False)
# node_id = list(consumer.client.conns.keys())[0]
# print dir(consumer.client.conns[node_id])
# for i in consumer.get_messages(100):
# print i.offset
# consumer.commit()
# from pykafka import KafkaClient
#
# client = KafkaClient(hosts="127.0.0.1:9092")
# print client.topics
# topic1 = client.topics[topic]
# consumer = topic1.get_simple_consumer(auto_commit_enable=True, )
# for message in consumer:
# if message is not None:
# print message.offset, message.value
connect_str = '127.0.0.1:9092'
consumer = KafkaConsumer(topic, group_id='my-group', bootstrap_servers=[connect_str],
auto_offset_reset='largest',auto_commit_enable=True,
auto_commit_interval_messages=1000)# largest,smallest
consumer.set_topic_partitions((topic, 2, 50032),) # Optionally specify offsets to start from
# kafka.set_topic_partitions("topic1", ("topic2", 2), {"topic3": 0}) #partition只能被一个消费者消费,所以最好指定消费哪个partitions
# kafka.set_topic_partitions({ ("topic1", 0): 12, ("topic2", 1): 45 })
# print consumer.topics
for message in consumer:
print ("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
message.offset, message.key,
message.value))
consumer.commit()
class IndexedConsumer():
"""
A simple consumer to retrieve messages from the input queue when it is time to send them
"""
def __init__(self, input_topic, hosts):
self.input_topic = input_topic
self.consumer = KafkaConsumer(bootstrap_servers=hosts)
def retrieve_event(self, event_reference):
self.consumer.set_topic_partitions(
(self.input_topic, event_reference.partition,
event_reference.offset))
message = self.consumer.next()
event = ScheduledEvent.from_dict(json.loads(message.value))
return event
relations, childs, parents = caida_filter_annaunce(
"20160101.as-rel.txt", "20160101.ppdc-ases.txt")
print(len(relations), len(childs), len(parents))
parser = argparse.ArgumentParser(
description="get a feed of abnormal BGP conflicts")
parser.add_argument("--offset", type=int)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
consumer = KafkaConsumer("hijacks",
bootstrap_servers=["comet-17-08.sdsc.edu:9092"],
group_id="client")
if args.offset is not None:
topics = [("hijacks", i, args.offset) for i in PARTITIONS.values()]
consumer.set_topic_partitions(*topics)
hijacks = 0
total = 0
for item in consumer:
total += 1
if (is_legittimate(relations, childs, parents,
json.loads(item.value)) == 0):
hijacks += 1
#print(item.value)
if (total == 10000): print(total, hijacks)
if __name__ == "__main__":
import argparse
relations,childs,parents=caida_filter_annaunce("20160101.as-rel.txt","20160101.ppdc-ases.txt")
print(len(relations),len(childs),len(parents))
parser = argparse.ArgumentParser(description="get a feed of abnormal BGP conflicts")
parser.add_argument("--offset", type=int)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
consumer = KafkaConsumer("hijacks",
bootstrap_servers=["comet-17-08.sdsc.edu:9092"],
group_id="client")
if args.offset is not None:
topics = [("hijacks", i, args.offset) for i in PARTITIONS.values()]
consumer.set_topic_partitions(*topics)
hijacks=0
total=0
for item in consumer:
total+=1
if(is_legittimate(relations,childs,parents, json.loads(item.value))==0):
hijacks+=1
#print(item.value)
if(total==10000): print(total,hijacks)
args = parser.parse_args()
collectors = COLLECTORS if len(args.collector) == 0 else args.collector
logging.basicConfig(level=logging.INFO)
topics = ["rib-{}".format(c) for c in collectors]
logger.info("using topics %s", topics)
consumer = KafkaConsumer(*topics,
bootstrap_servers=args.our_servers.split(","),
group_id="follower")
if args.offset is not None:
consumer.set_topic_partitions({(t, 0): args.offset for t in topics})
# setup filters
filters = []
if args.anycast_file is not None:
anycast = Radix()
count = 0
with open(args.anycast_file, "r") as f:
for prefix in f:
if not prefix.startswith("#"):
anycast.add(prefix.strip())
count += 1
logger.info("loaded %s prefixes in the anycast list", count)
logger.info("filtering on prefixes from the file %s", args.anycast_file)
else:
fill_relation_struct(args.irr_org_file, relations_dict,
"organisations")
fill_relation_struct(args.irr_mnt_file, relations_dict, "maintainers")
funcs.append(partial(annotate_if_relation, relations_dict))
if args.as_rel_file is not None and args.ppdc_ases_file is not None and args.as2org_file is not None:
a, b,c,d = caida_filter_annaunce(args.as_rel_file, args.ppdc_ases_file, args.as2org_file)
funcs.append(partial(is_legittimate, a, b, c,d))
if args.from_timestamp is None:
consumer = KafkaConsumer("conflicts",
metadata_broker_list=args.our_servers.split(","),
group_id="detector",
auto_commit_enable=False)
offset, = consumer.get_partition_offsets("conflicts", PARTITIONS[args.collector], -1, 1)
consumer.set_topic_partitions({("conflicts", PARTITIONS[args.collector]): offset - 1})
last_message = next(iter(consumer))
last_data = json.loads(last_message.value)
last_ts = last_data["timestamp"]
logger.info("last detected event was at offset %s timestamp %s", offset, last_ts)
else:
last_ts = args.from_timestamp
logger.info("detecting conflicts newer than %s", datetime.utcfromtimestamp(last_ts))
start_http_server(4240 + PARTITIONS[args.collector])
client = KafkaClient(args.our_servers.split(","))
stats = defaultdict(int)
for msg in detect_conflicts(**kwargs):
ts = msg.get("timestamp", 0)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
if args.collector is not None:
topics = ["rib-{}".format(c) for c in args.collector]
else:
topics = ["rib-{}".format(c) for c in COLLECTORS]
consumer = KafkaConsumer(*topics,
bootstrap_servers=args.our_servers.split(","),
group_id="follower")
if args.offset is not None:
consumer.set_topic_partitions({(t, 0): args.offset for t in topics})
# setup filters
filters = []
if args.prefixes_file is not None:
filter_prefixes = Radix()
with open(args.prefixes_file, "r") as f:
for prefix in f:
filter_prefixes.add(prefix.strip())
def func(data):
return len(list(filter_prefixes.search_covering(
data["prefix"]))) > 0
logger.info("filtering on prefixes from the file %s",
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
start_http_server(4340 + PARTITIONS[args.collector])
logger.info("loading the stats server on %s", 4340 + PARTITIONS[args.collector])
consumer = KafkaConsumer("raw-{}".format(args.collector),
group_id='test_hackathon10',
bootstrap_servers=args.ripe_servers.split(","))
save_file = "offsets-{}".format(args.collector)
if args.from_beginning:
logger.info("starting from scratch")
offsets = {("raw-{}".format(args.collector), i): 0 for i in range(0, 10)}
consumer.set_topic_partitions(offsets)
elif os.path.exists(save_file):
with open(save_file, "r") as f:
offsets = cPickle.load(f)
logger.info("loading offsets from file: %s", offsets)
consumer.set_topic_partitions(offsets)
else:
logger.info("starting from last messages")
client = KafkaClient(args.our_servers.split(","))
count = 0
for batch in group_by_n(messages_from_internal(iterate_messages(consumer, args.collector)), 1000):
req = ProduceRequest("rib-{}".format(args.collector), 0, batch)
count += len(batch)
logger.info("sending %i", count)
res = client.send_produce_request([req])
