def gen_min_idea():
return {
"Format": "IDEA0",
"ID": str(uuid4()),
"DetectTime": format_timestamp(),
"Category": ["Test"],
}
def gen_min_idea():
return {
"Format": "IDEA0",
"ID": str(uuid4()),
"DetectTime": format_timestamp(),
"Category": ["Test"],
}
def connectionMade(self):
self._peer = self.transport.getPeer()
self._srcip = self._peer.host
self._srcport = self._peer.port
self._proto = self._peer.type.lower()
self._socket = self.transport.socket.getsockname()
self._dtime = format_timestamp()
self._data = []
self._sessionid = genId()
self.lastaction = "CONNECTED"
print "Connection #%s made: %s" % (self._sessionid, self._srcip)
self.setTimeout(NODATA_TIMEOUT)
def doRespond(self):
body_len = int(self.headers.getheader('content-length', 0))
body_data = self.rfile.read(body_len)
data2log = {
"detect_time" : format_timestamp(),
"src_ip" : self.client_address[0],
"src_port" : self.client_address[1],
"dst_ip" : self.request.getsockname()[0],
"dst_port" : port,
"proto" : ["tcp", "http"],
"data" : {"requestline": self.requestline,
"headers" : str(self.headers),
"body" : base64.b64encode(body_data),
"body_len" : body_len }
}
logger.info(json.dumps(data2log))
#process request
if self.path.endswith('/'):
file_name = self.path+'index.html'
else:
file_name = self.path
file_name = re.sub("^/", "", file_name)
file_name = re.sub("/", "AAAA", file_name)
file_name = re.sub("\?", "BBBB", file_name)
try:
absolute_path = content_base+file_name
normalized_path = os.path.normpath(absolute_path)
if not normalized_path.startswith(content_base):
return self.doAbort()
f = open(normalized_path, "rb")
mime_type = mimetypes.guess_type(normalized_path)[0]
output = f.read()
f.close()
except Exception as e:
return self.doAbort()
self.send_response(200)
self.send_header('Content-type', mime_type)
self.end_headers()
self.wfile.write(output)
return
def connectionMade(self):
try:
# Cisco IOS fingerprint
self.transport.transport.socket.send('\xff\xfb\x01\xff\xfb\x03\xff\xfd\x18\xff\xfd\x1f')
except:
pass
self.lastaction = "CONNECTED"
self._peer = self.transport.getPeer()
self._srcip = self._peer.host
self._srcport = self._peer.port
self._proto = self._peer.type
self._socket = self.transport.transport.socket.getsockname()
self._dstip = self._socket[0]
self._dstport = self._socket[1]
self._dtime = format_timestamp()
self._data = []
self.showBanner()
self.promptUsername()
self.state = 'User'
return 'User'
def connectionMade(self): self._peer = self.transport.getPeer() self._proto = self._peer.type.lower() self._socket = self.transport.socket.getsockname() self._dtime = format_timestamp() self._data = []
def gen_random_idea(client_name="cz.example.warden.test"):
def geniprange(gen):
def iprange():
u = v = 0
while u==v:
u, v = gen(), gen()
u, v = min(u, v), max(u, v)
return "%s-%s" % (u, v)
return iprange
def rand4ip():
return "%s%d" % ('192.0.2.', randint(1, 254))
def rand4cidr():
return "%s%d/%d" % ('192.0.2.', randint(1, 254), randint(24, 31))
def randip4():
return [rand4ip, geniprange(rand4ip), rand4cidr][randint(0, 2)]()
def rand6ip():
return "2001:DB8:%s" % ":".join("%x" % randint(0, 65535) for i in range(6))
def rand6cidr():
m = randint(0, 5)
return "2001:DB8%s%s::/%d" % (":" if m else "", ":".join("%x" % randint(0, 65535) for i in range(m)), (m+2)*16)
def randip6():
return [rand6ip, geniprange(rand6ip), rand6cidr][randint(0, 2)]()
def randstr(charlist=string.letters, maxlen=32, minlen=1):
return ''.join(choice(charlist) for i in range(randint(minlen, maxlen)))
event = {
"Format": "IDEA0",
"ID": str(uuid4()),
"CreateTime": format_timestamp(),
"DetectTime": format_timestamp(),
"WinStartTime": format_timestamp(),
"WinEndTime": format_timestamp(),
"EventTime": format_timestamp(),
"CeaseTime": format_timestamp(),
#"Category": ["Abusive.Spam","Abusive.Harassment","Malware","Fraud.Copyright","Test","Fraud.Phishing","Fraud.Scam"],
# "Category": ["Abusive.Spam","Fraud.Copyright"],
"Category": [choice(["Abusive.Spam","Abusive.Harassment","Malware","Fraud.Copyright","Test","Fraud.Phishing","Fraud.Scam"]) for dummy in range(randint(1, 3))],
"Ref": ["cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr()), "http://www.example.com/%s" % randstr()],
"Confidence": random(),
"Note": "Random event",
"ConnCount": randint(0, 65535),
# "ConnCount": choice([randint(0, 65535), "asdf"]), # Send wrong event sometimes
"Source": [
{
"Type": ["Phishing"],
"IP4": [randip4() for i in range(randrange(1, 5))],
"IP6": [randip6() for i in range(randrange(1, 5))],
"Hostname": ["example.com"],
"Port": [randint(1, 65535) for i in range(randrange(1, 3))],
"AttachHand": ["att1"],
"Netname": ["arin:TEST-NET-1"]
}
],
"Target": [
{
"IP4": [randip4() for i in range(randrange(1, 5))],
"IP6": [randip6() for i in range(randrange(1, 5))],
"URL": ["http://example.com/%s" % randstr()],
"Proto": ["tcp", "http"],
"Netname": ["arin:TEST-NET-1"]
}
],
"Attach": [
{
"Handle": "att1",
"FileName": [randstr()],
"Type": ["Malware"],
"ContentType": "application/octet-stream",
"Hash": ["sha1:%s" % randstr(string.hexdigits, 24)],
"Size": 46,
"Ref": ["cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr())],
"ContentEncoding": "base64",
"Content": b64encode(randstr())
}
],
"Node": [
{
"Name": client_name,
"Type": [choice(["Data", "Protocol", "Honeypot", "Heuristic", "Log"]) for dummy in range(randint(1, 3))],
"SW": ["Kippo"],
"AggrWin": "00:05:00"
},
{
"Name": "org.example.warden.client",
"Type": [choice(["Connection", "Datagram"]) for dummy in range(randint(1, 2))],
}
]
}
return event
def gen_random_idea(client_name="cz.example.warden.test"):
def geniprange(gen):
def iprange():
u = v = 0
while u == v:
u, v = gen(), gen()
u, v = min(u, v), max(u, v)
return "%s-%s" % (u, v)
return iprange
def rand4ip():
return "%s%d" % ('192.0.2.', randint(1, 254))
def rand4cidr():
return "%s%d/%d" % ('192.0.2.', randint(1, 254), randint(24, 31))
def randip4():
return [rand4ip, geniprange(rand4ip), rand4cidr][randint(0, 2)]()
def rand6ip():
return "2001:DB8:%s" % ":".join("%x" % randint(0, 65535)
for i in range(6))
def rand6cidr():
m = randint(0, 5)
return "2001:DB8%s%s::/%d" % (":" if m else "", ":".join(
"%x" % randint(0, 65535) for i in range(m)), (m + 2) * 16)
def randip6():
return [rand6ip, geniprange(rand6ip), rand6cidr][randint(0, 2)]()
def randstr(charlist=string.ascii_letters, maxlen=32, minlen=1):
return ''.join(
choice(charlist) for i in range(randint(minlen, maxlen)))
event = {
"Format":
"IDEA0",
"ID":
str(uuid4()),
"CreateTime":
format_timestamp(),
"DetectTime":
format_timestamp(),
"WinStartTime":
format_timestamp(),
"WinEndTime":
format_timestamp(),
"EventTime":
format_timestamp(),
"CeaseTime":
format_timestamp(),
#"Category": ["Abusive.Spam","Abusive.Harassment","Malware","Fraud.Copyright","Test","Fraud.Phishing","Fraud.Scam"],
# "Category": ["Abusive.Spam","Fraud.Copyright"],
"Category": [
choice([
"Abusive.Spam", "Abusive.Harassment", "Malware",
"Fraud.Copyright", "Test", "Fraud.Phishing", "Fraud.Scam"
]) for dummy in range(randint(1, 3))
],
"Ref": [
"cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr()),
"http://www.example.com/%s" % randstr()
],
"Confidence":
random(),
"Note":
"Random event",
"ConnCount":
randint(0, 65535),
# "ConnCount": choice([randint(0, 65535), "asdf"]), # Send wrong event sometimes
"Source": [{
"Type": ["Phishing"],
"IP4": [randip4() for i in range(randrange(1, 5))],
"IP6": [randip6() for i in range(randrange(1, 5))],
"Hostname": ["example.com"],
"Port": [randint(1, 65535) for i in range(randrange(1, 3))],
"AttachHand": ["att1"],
"Netname": ["arin:TEST-NET-1"]
}],
"Target": [{
"IP4": [randip4() for i in range(randrange(1, 5))],
"IP6": [randip6() for i in range(randrange(1, 5))],
"URL": ["http://example.com/%s" % randstr()],
"Proto": ["tcp", "http"],
"Netname": ["arin:TEST-NET-1"]
}],
"Attach": [{
"Handle":
"att1",
"FileName": [randstr()],
"Type": ["Malware"],
"ContentType":
"application/octet-stream",
"Hash": ["sha1:%s" % randstr(string.hexdigits, 24)],
"Size":
46,
"Ref": ["cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr())],
"ContentEncoding":
"base64",
"Content":
b64encode(randstr().encode('ascii')).decode("ascii")
}],
"Node": [{
"Name":
client_name,
"Type": [
choice(["Data", "Protocol", "Honeypot", "Heuristic", "Log"])
for dummy in range(randint(1, 3))
],
"SW": ["Kippo"],
"AggrWin":
"00:05:00"
}, {
"Name":
"org.example.warden.client",
"Type": [
choice(["Connection", "Datagram"])
for dummy in range(randint(1, 2))
],
}]
}
return event
crs.execute(query)
break
except sqlite3.Error, e:
attempts += 1
wclient.logger.info("Info: %s - attempt %d/%d." % (e.args[0], attempts, aconattempts))
if attempts == aconattempts:
wclient.logger.error("Error: %s (dbfile: %s)" % (e.args[0], adbfile))
sleep(aretryinterval)
rows = crs.fetchall()
if con:
con.close
etime = format_timestamp(time())
stime = format_timestamp(time() - awin)
for row in rows:
dtime = format_timestamp(row['timestamp'])
events.append(gen_event_idea_dio(logger = wclient.logger, binaries_path = abinpath, report_binaries = areportbinaries, client_name = aclient_name, anonymised = aanonymised, target_net = aanonymised_net, detect_time = dtime, win_start_time = stime, win_end_time = etime, aggr_win = awin, data = row))
print "=== Sending ==="
start = time()
ret = wclient.sendEvents(events)
if 'saved' in ret:
wclient.logger.info("%d event(s) successfully delivered." % ret['saved'])
print "Time: %f" % (time() - start)
try:
crs.execute(query)
break
except sqlite3.Error, e:
attempts += 1
wclient.logger.info("Info: %s - attempt %d/%d." % (e.args[0], attempts, aconattempts))
if attempts == aconattempts:
wclient.logger.error("Error: %s (dbfile: %s)" % (e.args[0], adbfile))
sleep(aretryinterval)
rows = crs.fetchall()
if con:
con.close
etime = format_timestamp(time())
stime = format_timestamp(time() - awin)
for row in rows:
#print row
dtime = format_timestamp(float(row['utime']))
source_info = row['source'].split(":")
a = gen_event_idea_gl(
detect_time = dtime,
src_ip = source_info[0],
src_port = int(source_info[1]),
request_url = row['request_url'],
request_raw = row['request_raw'],
pattern = row['pattern'],
filename = row['filename'],
)
}
event = w3u.IDEA_fill_addresses(event, src_ip, dst_ip, aanonymised, aanonymised_net)
return event
events = []
try:
for line in w3u.Pygtail(filename=aconfig.get('logfile'), wait_timeout=0):
#sys.stdout.write(line)
data = json.loads(line)
#import pdb; pdb.set_trace()
#yes gringo ;) text > object > unixtime > text again
dtime = format_timestamp( calendar.timegm( dateutil.parser.parse(data["@timestamp"]).utctimetuple() ) )
a = gen_event_idea_elastichoney(
detect_time = dtime,
src_ip = data['source'],
dst_ip = data['honeypot'],
data = data
)
#print json.dumps(a)
events.append(a)
except:
pass
#print json.dumps(events, indent=3)
print "=== Sending ==="
start = time()
ret = wclient.sendEvents(events)