class OutputRedisTopology(Topology):
files_spout = IterFilesMailSpout.spec(name="iter-files-mails")
tokenizer = Tokenizer.spec(name="tokenizer", inputs=[files_spout], par=2)
attachments = Attachments.spec(
name="attachments",
inputs={tokenizer['attachments']: Grouping.fields('sha256_random')},
par=2)
urls = Urls.spec(name="urls",
inputs={
attachments: Grouping.fields('sha256_random'),
tokenizer['body']: Grouping.fields('sha256_random')
},
par=2)
phishing = Phishing.spec(name="phishing",
inputs={
attachments:
Grouping.fields('sha256_random'),
tokenizer['mail']:
Grouping.fields('sha256_random'),
urls:
Grouping.fields('sha256_random')
})
network = Network.spec(
name="network",
inputs={tokenizer['network']: Grouping.fields('sha256_random')},
par=1)
raw_mail = RawMail.spec(
name="raw_mail",
inputs={tokenizer['raw_mail']: Grouping.fields('sha256_random')},
par=2)
json_maker = JsonMaker.spec(name="json_maker",
inputs={
attachments:
Grouping.fields('sha256_random'),
network:
Grouping.fields('sha256_random'),
phishing:
Grouping.fields('sha256_random'),
raw_mail:
Grouping.fields('sha256_random'),
tokenizer['mail']:
Grouping.fields('sha256_random'),
urls:
Grouping.fields('sha256_random')
},
par=2)
output_redis = OutputRedis.spec(name="output-redis", inputs=[json_maker])
class OutputElasticsearchTopology(Topology):
files_spout = FilesMailSpout.spec(name="files-mails")
tokenizer = Tokenizer.spec(name="tokenizer", inputs=[files_spout], par=1)
attachments = Attachments.spec(
name="attachments",
inputs={tokenizer['attachments']: Grouping.fields('sha256_random')},
par=2)
urls_body = UrlsHandlerBody.spec(
name="urls-handler-body",
inputs={tokenizer['body']: Grouping.fields('sha256_random')})
urls_attachments = UrlsHandlerAttachments.spec(
name="urls-handler-attachments",
inputs={attachments: Grouping.fields('sha256_random')})
phishing = Phishing.spec(name="phishing",
inputs={
tokenizer['mail']:
Grouping.fields('sha256_random'),
attachments:
Grouping.fields('sha256_random'),
urls_body:
Grouping.fields('sha256_random'),
urls_attachments:
Grouping.fields('sha256_random')
})
forms = Forms.spec(
name="forms",
inputs={tokenizer['body']: Grouping.fields('sha256_random')})
network = Network.spec(
name="network",
inputs={tokenizer['network']: Grouping.fields('sha256_random')},
par=2)
json = JsonMaker.spec(name="json",
inputs={
tokenizer['mail']:
Grouping.fields('sha256_random'),
phishing: Grouping.fields('sha256_random'),
attachments: Grouping.fields('sha256_random'),
forms: Grouping.fields('sha256_random'),
network: Grouping.fields('sha256_random'),
urls_body: Grouping.fields('sha256_random'),
urls_attachments:
Grouping.fields('sha256_random')
})
output_elasticsearch = OutputElasticsearch.spec(
name="output-elasticsearch", inputs=[json], par=1)