def list_ideas(count):
ideas = []
for i in range(count):
idea = Idea()
ideas.append(idea)
return select_idea("Choose your idea", ideas)
def parse_app(app, meta):
if app['name'] == "Atom":
return Atom(
folder=parse_field('folder', app, meta),
file=parse_field('file', app, meta)
)
elif app['name'] == "Google-chrome":
return GoogleChrome(
file=parse_field('file', app, meta)
)
elif app['name'] == "XTerm":
return XTerm(
command=app['cmd'],
cwd=parse_field('cwd', app, meta),
args=[meta[arg] for arg in app['args']]
)
elif app['name'] == "Slack":
return Slack()
elif app['name'] == "idea":
return Idea(folder=parse_field('folder', app, meta))
ssc = StreamingContext(sc, batch_size)
# Creating accumulator
accum = sc.accumulator({}, IDAccumulatorParam())
""" Kafka init producer and load messages from topic """
# Producer for topic_out
producer = KafkaProducer(bootstrap_servers=kafka_brokers_out)
# Kafka messages are input, groupId (id8641233) has to be unique - if 2 consumers are present
kafkaStream = KafkaUtils.createStream(ssc, zookeper_in, offset,
{topic_in: 1})
""" Mark duplicates """
# Build key:(category, source_ip, target_ip, node_name, detect_time) !Do not change order! and value:idea
tuples = kafkaStream.map(lambda message: Idea(message[1])). \
map(lambda idea: ((idea.category,
idea.source_ip4,
idea.target_ip4,
idea.node_name,
idea.detect_time), idea))
# Reduce current batch
batch_reduced = tuples.mapValues(lambda idea: (idea.id, idea.detect_time)). \
reduceByKey(lambda x, y: leaveOlder(x, y))
# Apply window and leave only needed idea values (id, detect_time) then reduce
reduced = batch_reduced.window(window_duplicate, slide_size). \
reduceByKey(lambda x, y: leaveOlder(x, y))
# Join reduced and raw IDEA with key, apply marking function (see def for tuple members) and filter None
