class Neo4J():
DEST_RELS = 'sequence'
DOM_RELS = 'domain'
SRC_RELS = 'source'
SRCDST_RELS = 'sourcedest'
SRCLOGIN_RELS = 'srclogin'
SESSIONS_RELS = 'sessions'
FROM_SESSIONS = 'fromsessions'
USER_LIST = 'users'
DOM_LIST = 'domains'
SRV_LIST = 'servers'
SRVDOM_RELS = 'serverbelongsto'
def __init__(self, url, cache_data):
proto = "{}/".format('/'.join(url.split('/')[:2]))
userpwd = url.split('/')[2].split('@')[0]
uri = url.split('@')[1]
data = [
"{}{}".format(proto, uri),
userpwd.split(':')[0],
userpwd.split(':')[1]
]
# TODO: Store this relations in a redis-like cache
self.cache = Cache(cache_data)
self.cache.create_cache(self.DEST_RELS)
self.cache.create_cache(self.DOM_RELS)
self.cache.create_cache(self.SRC_RELS)
self.cache.create_cache(self.SRCDST_RELS)
self.cache.create_cache(self.SRCLOGIN_RELS)
self.cache.create_cache(self.SESSIONS_RELS)
self.cache.create_cache(self.FROM_SESSIONS)
self.cache.create_cache(self.USER_LIST)
self.cache.create_cache(self.DOM_LIST)
self.cache.create_cache(self.SRV_LIST)
self.cache.create_cache(self.SRVDOM_RELS)
# setup neo4j
self.drv = GraphDatabase.driver(data[0],
auth=basic_auth(data[1], data[2]))
self.neo = self.drv.session()
self.neo.run("CREATE INDEX ON :User(sid)")
self.neo.run("CREATE INDEX ON :Computer(name)")
self.neo.run("CREATE INDEX ON :Domain(name)")
def finish(self):
try:
self.neo.close()
except:
pass
def __genid_dict(self, value):
value = value.strip().upper()
return {
'id':
"_{}".format(
hashlib.sha1("{}".format(value).encode('utf-8')).hexdigest()),
'name':
value
}
def __gen_key(self, a, b):
return "{}-{}".format(a.upper(), b.upper())
def __get_logon_data(self, event):
tmp = ''
for i in CSV_FIELDS:
if type(event[i]) == type(int()):
val = event[i]
else:
val = "\"{}\"".format(event[i])
tmp = "{}, {}: {}".format(tmp, i.replace('.', '_'), val)
return "{{{}}}".format(tmp[1:])
def __create_node(self, key, node, query):
ret = False
if self.cache.get_key(key, node['id']) is None:
self.neo.run(query)
self.cache.set_key(key, node['id'], node['name'])
ret = True
return ret
def __add_domain(self, domain):
if domain['name'] in ['N/A', 'MicrosoftAccount', '-']:
return None
prev = None
for dom in domain['name'].split('.')[::-1]:
dom = self.__genid_dict(dom)
if prev is None:
self.__create_node(
self.DOM_LIST, dom,
"MERGE ({}:Domain {{name: '{}',label:'{}'}})".format(
dom['id'], dom['name'], dom['name']))
prev = dom
else:
new = self.__genid_dict("{}.{}".format(dom['name'],
prev['name']))
self.__create_node(
self.DOM_LIST, new,
"MERGE ({}:Domain {{name: '{}',label:'{}'}})".format(
new['id'], new['name'], new['name']))
if self.cache.get_key(self.DOM_LIST, "{}.{}".format(
new['id'], prev['id'])) is None:
self.cache.set_key(self.DOM_LIST,
"{}.{}".format(new['id'],
prev['id']), True)
self.neo.run(
"MATCH (subdomain:Domain {{name:'{}'}}),(domain:Domain {{name:'{}'}}) MERGE (subdomain)-[:BELONGS_TO]->(domain)"
.format(new['name'], prev['name']))
prev = new
def __add_computer(self, fqdn, ip=None):
data = fqdn['name'].split('.')
name = self.__genid_dict(data[0])
if ip is None:
self.__create_node(
self.SRV_LIST, fqdn,
"MERGE ({}:Computer {{name: '{}',label:'{}'}})".format(
name['id'], fqdn['name'], name['name']))
else:
fqdn = self.__genid_dict("{}({})".format(fqdn['name'], ip['name']))
self.__create_node(
self.SRV_LIST, fqdn,
"MERGE ({}:Computer {{name: '{}',label:'{}',ip: '{}'}})".
format(name['id'], fqdn['name'], name['name'], ip['name']))
if len(data[1:]) > 0:
dom = self.__genid_dict('.'.join(data[1:]))
self.__add_domain(dom)
if self.cache.get_key(self.SRVDOM_RELS, "{}@{}".format(
fqdn['id'], dom['id'])) is None:
self.cache.set_key(self.SRVDOM_RELS,
"{}@{}".format(fqdn['id'], dom['id']), True)
self.neo.run(
"MATCH (computer:Computer {{name:'{}'}}),(domain:Domain {{name:'{}'}}) MERGE (computer)-[:MEMBER_OF]->(domain)"
.format(fqdn['name'], dom['name']))
return fqdn
def __add_source_ip(self, ip):
self.__create_node(
self.SRV_LIST, ip,
"MERGE ({}:Computer {{name:'{}',label:'{}'}})".format(
ip['id'], ip['name'], ip['name']))
return ip
def __add_user(self, user, sid):
if user['name'] == 'N/A':
return None
self.__create_node(
self.USER_LIST, sid,
"MERGE ({}:User {{name: '{}',sid: '{}',label:'{}'}})".format(
sid['id'], user['name'], sid['name'], user['name']))
def add_sequence(self, event, fullinfo, uniquelogon):
self.uniquelogon = uniquelogon
# add logon source
username = self.__genid_dict(event['logon.username'])
usersid = self.__genid_dict(event['logon.dstsid'])
computer = self.__genid_dict(event['logon.computer'])
domain = self.__genid_dict(event['logon.domain'])
srccomputer = self.__genid_dict(event['logon.srccomputer'])
srcip = self.__genid_dict(event['logon.srcip'])
source = None
if len(event['logon.srcip']) > 0:
orig = event['logon.srcip']
if not orig in ["127.0.0.1", "LOCAL", "::1", "N/A"]:
if event['logon.srccomputer'] != 'N/A':
source = self.__add_computer(srccomputer, srcip)
else:
source = self.__add_source_ip(srcip)
self.__add_user(username, usersid)
self.__add_computer(computer)
self.__add_domain(domain)
# for future possible references, store the username
if username['name'] != 'N/A':
self.cache.set_key(self.SESSIONS_RELS, event['logon.trackingid'],
usersid['name'])
# check user-computer relation
exists = None
if self.uniquelogon is True:
exists = self.cache.get_key(
self.DEST_RELS, self.__gen_key(usersid['id'], computer['id']))
if exists is None:
self.cache.set_key(
self.DEST_RELS,
self.__gen_key(usersid['id'], computer['id']), True)
if exists is None:
if fullinfo is True:
logondata = self.__get_logon_data(event)
query = "MATCH (user:User {{sid:'{}'}}),(dest:Computer {{name:'{}'}}) MERGE (user)-[:LOGON_TO {}]->(dest)".format(
usersid['name'], computer['name'], logondata)
else:
query = "MATCH (user:User {{sid:'{}'}}),(dest:Computer {{name:'{}'}}) MERGE (user)-[:LOGON_TO {{date:'{}', type: '{}', logonid: '{}', srcid: '{}', trackingid:'{}', src:'{}'}}]->(dest)".format(
usersid['name'], computer['name'], event['logon.datetime'],
event['logon.type'], event['logon.id'],
event['logon.meta.id'], event['logon.trackingid'],
event['logon.srcip'])
self.neo.run(query)
# check user-domain relation
exists = self.cache.get_key(
self.DOM_RELS, self.__gen_key(usersid['id'], domain['id']))
if exists is None:
self.cache.set_key(self.DOM_RELS,
self.__gen_key(usersid['id'], domain['id']),
True)
self.neo.run(
"MATCH (user:User {{sid:'{}'}}),(domain:Domain {{name:'{}'}}) MERGE (user)-[:MEMBER_OF]->(domain)"
.format(usersid['name'], domain['name']))
# check src-dst relation
if source is not None:
exists = self.cache.get_key(
self.SRCDST_RELS, self.__gen_key(source['id'], computer['id']))
if exists is None:
self.cache.set_key(
self.SRCDST_RELS,
self.__gen_key(source['id'], computer['id']), True)
self.neo.run(
"MATCH (src:Computer {{name:'{}'}}),(dst:Computer {{name:'{}'}}) MERGE (src)-[:ACCESS_TO {{trackingid:'{}'}}]->(dst)"
.format(source['name'], computer['name'],
event['logon.trackingid']))
# check user-src relation
if source is not None:
exists = self.cache.get_key(
self.SRC_RELS, self.__gen_key(usersid['id'], source['id']))
if exists is None:
self.cache.set_key(self.SRC_RELS,
self.__gen_key(usersid['id'], source['id']),
True)
self.neo.run(
"MATCH (src:Computer {{name:'{}'}}),(user:User {{sid:'{}'}}) MERGE (user)-[:AUTH_FROM {{trackingid:'{}'}}]->(src)"
.format(source['name'], usersid['name'],
event['logon.trackingid']))
# srctrackingid
self.cache.set_key(self.FROM_SESSIONS, usersid['name'],
event['logon.srctrackingid'])
def finish(self):
sessions = self.cache.get_keys(self.FROM_SESSIONS)
if not sessions:
return
for sid in sessions.keys():
prev = self.cache.get_key(self.SESSIONS_RELS, sessions[sid])
if prev is not None:
exists = None
if self.uniquelogon is True:
exists = self.cache.get_key(
self.SRCLOGIN_RELS,
self.__gen_key(sessions[sid], prev))
if exists is None:
self.cache.set_key(self.SRCLOGIN_RELS,
self.__gen_key(sessions[sid], prev),
True)
self.neo.run(
"MATCH (cur:User {{sid:'{}'}}),(from:User {{sid:'{}'}}) MERGE (cur)-[:FROM_SESSION {{trackingid:'{}'}}]->(from)"
.format(sid, prev, sessions[sid]))
try:
# https://neo4j.com/developer/python/
# File "/usr/local/lib/python3.5/dist-packages/neo4j/v1/api.py", line 245, in _disconnect
# self._connection.in_use = False
#AttributeError: 'NoneType' object has no attribute 'in_use'
self.neo.close()
except:
pass
class Graphviz():
DEST_RELS = 'sequence'
DOM_RELS = 'domain'
SRC_RELS = 'source'
SRCDST_RELS = 'sourcedest'
SRCLOGIN_RELS = 'srclogin'
SESSIONS_RELS = 'sessions'
FROM_SESSIONS = 'fromsessions'
USER_LIST = 'users'
DOM_LIST = 'domains'
SRV_LIST = 'servers'
SRVDOM_RELS = 'serverbelongsto'
def __init__(self, output, cache_data):
# TODO: Store this relations in a redis-like cache
self.cache = Cache(cache_data)
self.cache.create_cache(self.DEST_RELS)
self.cache.create_cache(self.DOM_RELS)
self.cache.create_cache(self.SRC_RELS)
self.cache.create_cache(self.SRCDST_RELS)
self.cache.create_cache(self.SRCLOGIN_RELS)
self.cache.create_cache(self.SESSIONS_RELS)
self.cache.create_cache(self.FROM_SESSIONS)
self.cache.create_cache(self.USER_LIST)
self.cache.create_cache(self.DOM_LIST)
self.cache.create_cache(self.SRV_LIST)
self.cache.create_cache(self.SRVDOM_RELS)
self.output = output
self.graph = gv.Digraph()
def __genid_dict(self, value):
value = value.strip().upper()
return {
'id':
"_{}".format(
hashlib.sha1("{}".format(value).encode('utf-8')).hexdigest()),
'name':
value
}
def __gen_key(self, a, b):
return "{}-{}".format(a.upper(), b.upper())
def __get_logon_data(self, event):
tmp = ''
for i in CSV_FIELDS:
if type(event[i]) == type(int()):
val = event[i]
else:
val = "\"{}\"".format(event[i])
tmp = "{}, {}: {}".format(tmp, i.replace('.', '_'), val)
return "{{{}}}".format(tmp[1:])
def __create_node(self, key, node, label):
ret = False
if self.cache.get_key(key, node['id']) is None:
self.graph.node(node['id'], label=label)
self.cache.set_key(key, node['id'], node['name'])
ret = True
return ret
def __add_domain(self, domain):
if domain['name'] in ['N/A', 'MicrosoftAccount', '-']:
return None
prev = None
for dom in domain['name'].split('.')[::-1]:
dom = self.__genid_dict(dom)
if prev is None:
self.__create_node(self.DOM_LIST, dom, dom['name'])
prev = dom
else:
new = self.__genid_dict("{}.{}".format(dom['name'],
prev['name']))
self.__create_node(self.DOM_LIST, new, new['name'])
if self.cache.get_key(self.DOM_LIST, "{}.{}".format(
new['id'], prev['id'])) is None:
self.cache.set_key(self.DOM_LIST,
"{}.{}".format(new['id'],
prev['id']), True)
self.graph.edge(new['id'], prev['id'], label='BELONGS_TO')
prev = new
def __add_computer(self, fqdn, ip=None):
data = fqdn['name'].split('.')
name = self.__genid_dict(data[0])
if ip is None:
self.__create_node(self.SRV_LIST, fqdn, fqdn['name'])
else:
fqdn = self.__genid_dict("{}({})".format(fqdn['name'], ip['name']))
self.__create_node(self.SRV_LIST, fqdn, fqdn['name'])
if len(data[1:]) > 0:
dom = self.__genid_dict('.'.join(data[1:]))
self.__add_domain(dom)
if self.cache.get_key(self.SRVDOM_RELS, "{}@{}".format(
fqdn['id'], dom['id'])) is None:
self.cache.set_key(self.SRVDOM_RELS,
"{}@{}".format(fqdn['id'], dom['id']), True)
self.graph.edge(fqdn['id'], dom['id'], label='MEMBER_OF')
return fqdn
def __add_source_ip(self, ip):
self.__create_node(self.SRV_LIST, ip, ip['name'])
return ip
def __add_user(self, user, sid):
if user['name'] == 'N/A':
return None
self.__create_node(self.USER_LIST, sid, user['name'])
def add_sequence(self, event, fullinfo, uniquelogon):
self.uniquelogon = uniquelogon
# add logon source
username = self.__genid_dict(event['logon.username'])
usersid = self.__genid_dict(event['logon.dstsid'])
computer = self.__genid_dict(event['logon.computer'])
domain = self.__genid_dict(event['logon.domain'])
srccomputer = self.__genid_dict(event['logon.srccomputer'])
srcip = self.__genid_dict(event['logon.srcip'])
source = None
if len(event['logon.srcip']) > 0:
orig = event['logon.srcip']
if not orig in ["127.0.0.1", "LOCAL", "::1", "N/A"]:
if event['logon.srccomputer'] != 'N/A':
source = self.__add_computer(srccomputer, srcip)
else:
source = self.__add_source_ip(srcip)
self.__add_user(username, usersid)
self.__add_computer(computer)
self.__add_domain(domain)
# for future possible references, store the username
if username['name'] != 'N/A':
self.cache.set_key(self.SESSIONS_RELS, event['logon.trackingid'],
usersid['name'])
# check user-computer relation
exists = None
if uniquelogon is True:
exists = self.cache.get_key(
self.DEST_RELS, self.__gen_key(usersid['id'], computer['id']))
if exists is None:
self.cache.set_key(
self.DEST_RELS,
self.__gen_key(usersid['id'], computer['id']), True)
if exists is None:
self.graph.edge(usersid['id'], computer['id'], label='LOGON_TO')
# check user-domain relation
exists = self.cache.get_key(
self.DOM_RELS, self.__gen_key(usersid['id'], domain['id']))
if exists is None:
self.cache.set_key(self.DOM_RELS,
self.__gen_key(usersid['id'], domain['id']),
True)
self.graph.edge(usersid['id'], domain['id'], label='MEMBER_OF')
# check src-dst relation
if source is not None:
exists = self.cache.get_key(
self.SRCDST_RELS, self.__gen_key(source['id'], computer['id']))
if exists is None:
self.cache.set_key(
self.SRCDST_RELS,
self.__gen_key(source['id'], computer['id']), True)
self.graph.edge(source['id'],
computer['id'],
label='ACCESS_TO')
# check user-src relation
if source is not None:
exists = self.cache.get_key(
self.SRC_RELS, self.__gen_key(usersid['id'], source['id']))
if exists is None:
self.cache.set_key(self.SRC_RELS,
self.__gen_key(usersid['id'], source['id']),
True)
self.graph.edge(usersid['id'], source['id'], label='AUTH_FROM')
# srctrackingid
self.cache.set_key(self.FROM_SESSIONS, usersid['name'],
event['logon.srctrackingid'])
def finish(self):
sessions = self.cache.get_keys(self.FROM_SESSIONS)
if not sessions:
return
for sid in sessions.keys():
prev = self.cache.get_key(self.SESSIONS_RELS, sessions[sid])
if prev is not None:
exists = None
if self.uniquelogon is True:
exists = self.cache.get_key(
self.SRCLOGIN_RELS,
self.__gen_key(sessions[sid], prev))
if exists is None:
self.cache.set_key(self.SRCLOGIN_RELS,
self.__gen_key(sessions[sid], prev),
True)
self.graph.edge(sid, prev, label="FROM_SESSION")
self.output.write(self.graph.source)
self.output.close()
