def do_dataimport(self, args):
"""Imports data from source files using data source specific adapters. The location of source files and appropriate adapters are defined in the configuration file - "dataSources.xml")."""
importedAdapter = dict()
for i in imports:
sourceName = i.get("sourceName")
if ((tools.getAdapter(sourceName, adapterList)) != None):
dataLoc = i.get("location")
coverage = i.get("coverage")
mainip = i.get("mainip")
nodename = i.get("nodename")
adapter = tools.getAdapter(sourceName, adapterList)
ts = tools.creationDate(dataLoc)
# Importing adapters from adapters
if (sourceName not in importedAdapter):
spec = importlib.util.spec_from_file_location(
'run', 'adapters\\{}.py'.format(adapter))
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
#print('Importing adapters ',mod)
importedAdapter[sourceName] = mod
else:
mod = importedAdapter[sourceName]
if (mainip != None and nodename != None):
query = (
"mod.run('{0}', '{1}', '{2}','{3}','{4}','{5}',con )"
).format(sourceName, coverage, mainip, nodename, dataLoc,
ts)
elif (mainip == None and nodename == None):
query = ("mod.run('{0}', '{1}', '{2}','{3}',con)").format(
sourceName, coverage, dataLoc, ts)
exec(query)
storeAssocArango('elementAssoc', sourcezoneKey, sourceKey, 'zoneInterface', client)
if (dzone not in zoned):
destinKey = created.get(destin)
destinzoneKey = storeElementArango('element', 'zone', 'network', dzone, None, client)
storeAssocArango('elementAssoc', destinzoneKey, destinKey, 'zoneInterface', client)
storeAssocArango('metaelementAssoc', srcId, destinzoneKey, 'sourceNetwork', client)
zoned[dzone] = destinzoneKey
else:
destinKey = created.get(destin)
destinzoneKey = zoned.get(dzone)
storeAssocArango('elementAssoc', destinzoneKey, destinKey, 'zoneInterface', client)
def run(sourceName,scope, sourceIP,nodeName, dataFile, date, con):
date=str(date)
if(checkExists(dataFile, con)== False):
print('Importing :', dataFile, ' from ', sourceName)
hostsData = readFile(dataFile)
#hostsData=paperRUN.readMyCSV(dataFile, ',', False)
#hostsData=parseReadData(hostsData)
hash=tools.hashfile(open(dataFile, "rb"),dataFile)
store(sourceName, scope, sourceIP, nodeName, hostsData, hash, date, con)
else: print(dataFile + " already imported")
#run('winscanner', '10.1.1.112',"..\\data\\p0f\\p0f_output", tools.creationDate("..\\data\\p0f\\p0f_output"), createConArango('Lab'))
#run('winscanner', 'subnet', '10.1.1.112','Winscanner',"..\\data\\p0f\\p0f_output_allLANs", tools.creationDate("..\\data\\wireshark\\summary-packets_fixed.xml"), createConArango('Lab'))
run('Wireshark', 'subnet','10.1.1.112','Winscanner',"..\\data\\wireshark\\summary-packets_fixed.xml", tools.creationDate("..\\data\\wireshark\\summary-packets_fixed.xml"), createConArango('Lab'))
#run('Wireshark', 'subnet','10.1.1.112','Winscanner',"F:\\ongoingWork\\ModBusCaps\\unzipped\\converted\\packets_00001_20161115135616.xml", tools.creationDate("F:\\ongoingWork\\ModBusCaps\\unzipped\\converted\\packets_00001_20161115135616.xml"), createConArango('Lab'))
#run('Wireshark', 'subnet','199.206.2.31','Winscanner','F:\ongoingWork\FOI\pcap\snart-fw.cleanfurnace1.se\psml\pcap_00.csv', tools.creationDate("F:\ongoingWork\FOI\pcap\snart-fw.cleanfurnace1.se\psml\pcap_00.csv"), createConArango('Lab'))
"""
tektek=readFile("..\\data\\bldde01\\Software.xml")
for k, v in tektek.items():
print(v.get('vendor')+','+v.get('name')+','+v.get('version'))
"""
#store('Windows_Config', '192.168.109.5', readFile("..\\data\\bldad01\\Software.xml"), None, tools.creationDate("..\\data\\bldad01\\Software.xml"), createConArango('claimOmania'))
def run(sourceName, scope, ip, nodeName, dataFile, date, con):
date = str(date)
if (checkExists(dataFile, con) == False):
print('Importing :', dataFile, ' from ', sourceName)
hostsData = readFile(dataFile)
hash = tools.hashfile(open(dataFile, "rb"), dataFile)
store(sourceName, scope, ip, nodeName, hostsData, hash, date, con)
else:
print(dataFile + " already imported")
run('Powershell-Win32_Product', 'system', '192.168.109.21', 'bldad01',
"..\\data\\bldad01\\Software.xml",
tools.creationDate("..\\data\\bldad01\\Software.xml"),
createConArango('OntoLab'))
run('Powershell-Win32_Product', 'system', '192.168.109.23', 'bldde01',
"..\\data\\bldde01\\Software.xml",
tools.creationDate("..\\data\\bldde01\\Software.xml"),
createConArango('OntoLab'))
run('Powershell-Win32_Product', 'system', '192.168.109.110', 'winscanner',
"..\\data\\winscanner\\Software.xml",
tools.creationDate("..\\data\\winscanner\\Software.xml"),
createConArango('OntoLab'))
softPropertyVers = storeElementArango('element', 'property', 'version', softvers, None, client)
softPropertyChange = storeElementArango('element', 'property', 'lastChange', softChange, None, client)
softPropertyPackId = storeElementArango('element', 'property', 'packageCode', softPackageId, None, client)
#Associations
storeAssocArango('elementAssoc', nodeID, softInst, 'nodeSoftware', client)
storeAssocArango('elementAssoc', softInst, softPropertyVendor, 'softwareProperty', client)
storeAssocArango('elementAssoc', softInst, softPropertyVers, 'softwareProperty', client)
storeAssocArango('elementAssoc', softInst, softPropertyChange, 'softwareProperty', client)
storeAssocArango('elementAssoc', softInst, softPropertyPackId, 'softwareProperty', client)
else: print('No node to add data to')
"""
tektek=readFile("..\\data\\bldde01\\Software.xml")
for k, v in tektek.items():
print(v.get('vendor')+','+v.get('name')+','+v.get('version'))
"""
#store('Windows_Config', '192.168.109.5', readFile("..\\data\\bldad01\\Software.xml"), None, tools.creationDate("..\\data\\bldad01\\Software.xml"), createConArango('claimOmania'))
def run(sourceName, scope, ip,nodeName, dataFile, date, con):
date=str(date)
if(checkExists(dataFile, con)== False):
print('Importing :', dataFile, ' from ', sourceName)
hostsData = readFile(dataFile)
hash=tools.hashfile(open(dataFile, "rb"),dataFile)
store(sourceName,scope, ip, nodeName,hostsData, hash, date, con)
else: print(dataFile + " already imported")
run('Powershell-Win32_Product','system', '192.168.109.21','bldad01',"..\\data\\bldad01\\Software.xml", tools.creationDate("..\\data\\bldad01\\Software.xml"), createConArango('Lab'))
run('Powershell-Win32_Product','system', '192.168.109.23','bldde01',"..\\data\\bldde01\\Software.xml", tools.creationDate("..\\data\\bldde01\\Software.xml"), createConArango('Lab'))
run('Powershell-Win32_Product','system', '192.168.109.110','winscanner',"..\\data\\winscanner\\Software.xml", tools.creationDate("..\\data\\winscanner\\Software.xml"), createConArango('Lab'))
): #=getPortNames('adapters\\thirddata\\service-names-port-numbers.xml')):
date = str(date)
if (checkExists(dataFile, con) == False):
print('Importing :', dataFile, ' from ', sourceName)
hostsData = readFile(dataFile, portnames)
hash = tools.hashfile(open(dataFile, "rb"), dataFile)
store(sourceName, scope, sourceIP, nodeName, hostsData, hash, date,
con)
else:
print(dataFile + " already imported")
###CHECK THE FIRST DATA! >> edge error
#run('p0f', 'subnet','10.1.1.112','Winscanner',"..\\data\\p0f\\p0f_output", tools.creationDate("..\\data\\p0f\\p0f_output"), ontologyTools2.getPortNames('thirddata\\service-names-port-numbers.xml'), createConArango('Lab'))
#run('p0f', 'subnet','10.1.1.112','Winscanner',"..\\data\\p0f\\nextScanOutput", tools.creationDate("..\\data\\p0f\\nextScanOutput"), ontologyTools2.getPortNames('thirddata\\service-names-port-numbers.xml'), createConArango('Lab'))
run('p0f', 'subnet', '10.1.1.112', 'Winscanner',
"..\\data\\p0f\\p0f_output_allLANs",
tools.creationDate("..\\data\\p0f\\p0f_output_allLANs"),
ontologyTools2.getPortNames('thirddata\\service-names-port-numbers.xml'),
createConArango('Lab'))
#print(tools.creationDate("..\\data\\p0f\\nextScanOutput"))
#pprint(readFile("..\\data\\p0f\\\p0f_output", ontologyTools2.getPortNames('thirddata\\service-names-port-numbers.xml')))
#pprint(readFile("C:\\Temp\\foi\\pcaptxt_00.txt", ontologyTools2.getPortNames('thirddata\\service-names-port-numbers.xml')))
#2#run('p0f', 'subnet','199.206.2.23','Winscanner',"C:\\Temp\\foi\\pcaptxt_00.txt", tools.creationDate("C:\\Temp\\foi\\pcaptxt_00.txt"), ontologyTools2.getPortNames('thirddata\\service-names-port-numbers.xml'), createConArango('Lab'))
#print(getSourceClasses('metaelement/4506083',createConArango('claimOmania')))
if (protocol != None):
protKey = storeElementArango('element', 'property', 'protocol',
protocol, None, client)
storeAssocArango('elementAssoc', servKey, protKey,
'ServerProtocol', client)
#if (serverName != None):
#srvSpKey = storeElementArango('element', 'softwareInstance', serverName, None, client)
#storeAssocArango('elementAssoc', servKey, srvSpKey, 'serverSoftwareproduct', client)
def run(sourceName, scope, dataFile, date, con):
if (checkExists(dataFile, con) == False):
print('Importing :', dataFile, ' from ', sourceName)
hostsData = hosts(dataFile)
#timestamp = str(datetime.date.today())
hash = tools.hashfile(open(dataFile, "rb"), dataFile)
#storeBasic(con)
store(sourceName, scope, hostsData, hash, date, con)
else:
print(dataFile + " already imported")
###run('Nexpose', "..\data\\2014-03-28\\FullXMLReport_v2.xml", '2014-03-28', createConArango('claimOmania'))
#pprint(hosts("..\data\\nexpose\\nexpose04_12.xml"))
run('Nexpose', 'subnet', "..\data\\nexpose\\nexpose04_12.xml",
tools.creationDate("..\data\\nexpose\\nexpose04_12.xml"),
createConArango('OntoLab'))
#run('Nexpose','subnet', "..\data\\nexpose\\report.xml", tools.creationDate("..\data\\nexpose\\report.xml"), createConArango('OntoLab'))
sourcezoneKey = zoned.get(szone)
storeAssocArango('elementAssoc', sourcezoneKey, sourceKey, 'zoneInterface', client)
if (dzone not in zoned):
destinKey = created.get(destin)
destinzoneKey = storeElementArango('element', 'zone', 'network', dzone, None, client)
storeAssocArango('elementAssoc', destinzoneKey, destinKey, 'zoneInterface', client)
storeAssocArango('metaelementAssoc', srcId, destinzoneKey, 'sourceNetwork', client)
zoned[dzone] = destinzoneKey
else:
destinKey = created.get(destin)
destinzoneKey = zoned.get(dzone)
storeAssocArango('elementAssoc', destinzoneKey, destinKey, 'zoneInterface', client)
def run(sourceName,scope, sourceIP,nodeName, dataFile, date, con):
date=str(date)
if(checkExists(dataFile, con)== False):
print('Importing :', dataFile, ' from ', sourceName)
hostsData = readFile(dataFile)
hash=tools.hashfile(open(dataFile, "rb"),dataFile)
store(sourceName, scope, sourceIP, nodeName, hostsData, hash, date, con)
else: print(dataFile + " already imported")
#run('winscanner', '10.1.1.112',"..\\data\\p0f\\p0f_output", tools.creationDate("..\\data\\p0f\\p0f_output"), createConArango('OntoLab'))
#run('winscanner', '10.1.1.112',"..\\data\\p0f\\p0f_output_allLANs", tools.creationDate("..\\data\\p0f\\p0f_output_allLANs"), createConArango('claimOmania'))
#run('Wireshark', 'subnet','10.1.1.112','Winscanner',"..\\data\\wireshark\\summary-packets_fixed.xml", tools.creationDate("..\\data\\wireshark\\summary-packets_fixed.xml"), createConArango('OntoLab'))
run('Wireshark', 'subnet','10.1.1.112','Winscanner',"F:\\ongoingWork\\ModBusCaps\\unzipped\\converted\\packets_00001_20161115135616.xml", tools.creationDate("F:\\ongoingWork\\ModBusCaps\\unzipped\\converted\\packets_00001_20161115135616.xml"), createConArango('OntoLab'))
else:
serviceName = ('-'.join((port, protocol, serviceName)))
serverName = v.get("serviceSoftware")
servKey=storeElementArango('element','softwareInstance','server',serviceName, None, client)
storeAssocArango('elementAssoc', nodeKey, servKey,'nodeServer',client)
if(protocol!=None):
protKey = storeElementArango('element', 'property','protocol', protocol, None, client)
storeAssocArango('elementAssoc', servKey, protKey, 'property', client)
if(serverName!=None):
srvSpKey = storeElementArango('element', 'property','serverName', serverName, None, client)
storeAssocArango('elementAssoc', servKey, srvSpKey, 'serverSoftwareproduct', client)
def run(sourceName,scope, dataFile, date, con):
if(checkExists(dataFile, con)== False):
print('Importing :', dataFile, ' from ', sourceName)
hostsData = hosts(dataFile)
#timestamp = str(datetime.date.today())
hash=tools.hashfile(open(dataFile, "rb"),dataFile)
#storeBasic(con)
store(sourceName,scope, hostsData, hash, date, con)
else: print(dataFile + " already imported")
#store('Nmap', hosts("..\data\\2014-11-06\\nmap_20-10-2014.xml"), tools.hashfile(open("..\data\\2014-11-06\\nmap_20-10-2014.xml", "rb")), '2014-11-06', createConArango('claimOmania'))
run('Nmap','subnet', "..\\data\\nmap\\04_25_nmap_scan.xml", tools.creationDate("..\\data\\nmap\\04_25_nmap_scan.xml"), createConArango('Lab'))
#run('Nmap','subnet', "..\\data\\nmap\\all-subnets-SCADAscan.xml", tools.creationDate("..\\data\\nmap\\all-subnets-SCADAscan.xml"), createConArango('Lab'))
#pprint(hosts('..\\data\\nmap\\04_25_nmap_scan.xml'))
storeAssocArango('elementAssoc', destinzoneKey, destinKey, 'zoneInterface', client)
storeAssocArango('metaelementAssoc', srcId, destinzoneKey, 'sourceNetwork', client)
zoned[dzone] = destinzoneKey
else:
destinKey = created.get(destin)
destinzoneKey = zoned.get(dzone)
storeAssocArango('elementAssoc', destinzoneKey, destinKey, 'zoneInterface', client)
def run(sourceName,scope, sourceIP,nodeName, dataFile, date, portnames, con):#=getPortNames('adapters\\thirddata\\service-names-port-numbers.xml')):
date=str(date)
if(checkExists(dataFile, con)== False):
print('Importing :', dataFile, ' from ', sourceName)
hostsData = readFile(dataFile, portnames)
hash=tools.hashfile(open(dataFile, "rb"),dataFile)
store(sourceName, scope, sourceIP, nodeName, hostsData, hash, date, con)
else: print(dataFile + " already imported")
###CHECK THE FIRST DATA! >> edge error
run('p0f', 'subnet','10.1.1.112','Winscanner',"..\\data\\p0f\\p0f_output", tools.creationDate("..\\data\\p0f\\p0f_output"), ontologyTools2.getPortNames('thirddata\\service-names-port-numbers.xml'), createConArango('OntoLab'))
#run('p0f', 'subnet','10.1.1.112','Winscanner',"..\\data\\p0f\\nextScanOutput", tools.creationDate("..\\data\\p0f\\nextScanOutput"), ontologyTools2.getPortNames('thirddata\\service-names-port-numbers.xml'), createConArango('OntoLab'))
#run('p0f', 'subnet','10.1.1.112','Winscanner',"..\\data\\p0f\\p0f_output_allLANs", tools.creationDate("..\\data\\p0f\\p0f_output_allLANs"), ontologyTools2.getPortNames('thirddata\\service-names-port-numbers.xml'), createConArango('OntoLab'))
#print(tools.creationDate("..\\data\\p0f\\nextScanOutput"))
#pprint(readFile("..\\data\\p0f\\\p0f_output_allLANs", writeOntologyTrees.getPortNames('..\\ontology\\service-names-port-numbers.xml')))
#print(getSourceClasses('metaelement/4506083',createConArango('claimOmania')))
'server', serviceName, None, client)
storeAssocArango('elementAssoc', nodeKey, servKey,
'operatingsystemServer', client)
if (protocol != None):
protKey = storeElementArango('element', 'property', 'protocol',
protocol, None, client)
storeAssocArango('elementAssoc', servKey, protKey,
'ServerProtocol', client)
def run(sourceName, scope, dataFile, date, con):
if (checkExists(dataFile, con) == False):
print('Importing :', dataFile, ' from ', sourceName)
hostsData = hosts(dataFile)
#timestamp = str(datetime.date.today())
hash = tools.hashfile(open(dataFile, "rb"), dataFile)
#storeBasic(con)
store(sourceName, scope, hostsData, hash, date, con)
else:
print(dataFile + " already imported")
run(
'Nessus', 'subnet',
"..\data\\nessus\\Authenticated_SCADA_lab_scan_encn7b.xml",
tools.creationDate(
"..\data\\nessus\\Authenticated_SCADA_lab_scan_encn7b.xml"),
createConArango('OntoLab'))
#run('Nessus','subnet', "..\data\\nessus\\Authenticated_SCADA_lab_scan_mj3yvv.xml", tools.creationDate("..\data\\nessus\\Authenticated_SCADA_lab_scan_mj3yvv.xml"), createConArango('OntoLab'))
client)
storeAssocArango('elementAssoc', servKey, protKey,
'property', client)
if (serverName != None):
srvSpKey = storeElementArango('element', 'property',
'serverName', serverName,
None, client)
storeAssocArango('elementAssoc', servKey, srvSpKey,
'serverSoftwareproduct', client)
def run(sourceName, scope, dataFile, date, con):
if (checkExists(dataFile, con) == False):
print('Importing :', dataFile, ' from ', sourceName)
hostsData = hosts(dataFile)
#timestamp = str(datetime.date.today())
hash = tools.hashfile(open(dataFile, "rb"), dataFile)
#storeBasic(con)
store(sourceName, scope, hostsData, hash, date, con)
else:
print(dataFile + " already imported")
#store('Nmap', hosts("..\data\\2014-11-06\\nmap_20-10-2014.xml"), tools.hashfile(open("..\data\\2014-11-06\\nmap_20-10-2014.xml", "rb")), '2014-11-06', createConArango('claimOmania'))
run('Nmap', 'subnet', "..\\data\\nmap\\04_25_nmap_scan.xml",
tools.creationDate("..\\data\\nmap\\04_25_nmap_scan.xml"),
createConArango('OntoLab'))
#run('Nmap','subnet', "..\\data\\nmap\\all-subnets-SCADAscan.xml", tools.creationDate("..\\data\\nmap\\all-subnets-SCADAscan.xml"), createConArango('OntoLab'))
