forked from xujun10110/es_email_intel
-
Notifications
You must be signed in to change notification settings - Fork 0
/
es_query_ipv4_feed_tagged.py
148 lines (134 loc) · 2.38 KB
/
es_query_ipv4_feed_tagged.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
from elasticsearch import Elasticsearch
import common_functions
import re
'''
Generate a plaintext feed for a SIEM
I invoke mine like this:
#!/bin/sh
/usr/local/bin/python <mypath>/es_query_ipv4_feed.py > /tmp/es_feed.txt
scp /tmp/es_feed.txt <webserver dir>/es_feed.txt &>/dev/null
'''
keywords = '''APT 28
APT 30
RAT
SEA
Zeus
Adobe
Angler
anonopsaudix2
APT 12
Axiom
Backoff
Badur
BlackPOS
brute force
Bugat
codoso
Cool EK
Cridex
cross team
cryptowall
CVE-2015-1635
Cyber Berkut
DD4BC
DDoS
Deep Panda
Dexter
DPRK
Dridex
Driveby
Duqu
Dyre
Equation Group
Feodor
Fiesta
FIN4
FlashPack
FlimKit
Gamarue
GhostSec
Heartbleed
ISIL
ISIS
LastPass
Lotus Blossom
Magnitude
Malum
MalumPOS
MS15-034
Neutrino
Newscaster
Nitlove
Nuclear
obfuscat
Ocean Lotus
OPM
password guess
Phishing
Poodle
PoS
Poweliks
Ransomware
Sakula
Sakura
Sandworm
Shell Crew
ShellShock
SQLi
SQLMap
Styx
Sweet Orange
Syrian Electronic Army
TESLACYRPT
Titan Rain
Upatre
vawtrak
vSkimmer
'''
es_server = '192.168.3.208'
es = Elasticsearch([{'host': es_server, 'port': 9200}])
es_collection_name = 'mail2json'
then = common_functions.queryrange(30)
body = '''{
"size" : 10000,
"query": {
"constant_score": {
"filter": {
"range": {
"epoch": {
"from": '''+then+'''
}
}
}
}
}
}'''
def ipv4_matchkeywords(ipv4):
ret_match = ''
# Given an IPv4 address, pull 30 days worth of messages that contain the indicator and see what keywords are found
then = common_functions.queryrange(30)
json = '''
{"size" : 10000,
"query": {
"match": {
"ipv4":{"query":"'''+ipv4+'''"}
}
}}
'''
res = es.search(index=es_collection_name, body=json)
keywords_list = keywords.split('\n')
for keyword in keywords_list:
if keyword.strip() == '': continue
regex = r"\b(?=\w)" + re.escape(keyword) + r"\b(?!\w)"
#print 'Looking at keyword: '+keyword
#print 'Compiled this regex: '+regex
for hit in res['hits']['hits']:
rawtext = hit["_source"]['message_text']
if re.search(regex, rawtext, re.IGNORECASE):
if not re.search(r"\:?" + re.escape(keyword) + r"\:", ret_match):
ret_match += keyword+':'
if ret_match.endswith(":"): ret_match = ret_match[:-1]
return ret_match
ipv4s = common_functions.pull_ipv4_addresses(es, es_collection_name, body)
for ipv4 in ipv4s:
print ipv4+','+ipv4_matchkeywords(ipv4)