def do_dataimport(self, args): """Imports data from source files using data source specific adapters. The location of source files and appropriate adapters are defined in the configuration file - "dataSources.xml").""" importedAdapter = dict() for i in imports: sourceName = i.get("sourceName") if ((tools.getAdapter(sourceName, adapterList)) != None): dataLoc = i.get("location") coverage = i.get("coverage") mainip = i.get("mainip") nodename = i.get("nodename") adapter = tools.getAdapter(sourceName, adapterList) ts = tools.creationDate(dataLoc) # Importing adapters from adapters if (sourceName not in importedAdapter): spec = importlib.util.spec_from_file_location( 'run', 'adapters\\{}.py'.format(adapter)) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) #print('Importing adapters ',mod) importedAdapter[sourceName] = mod else: mod = importedAdapter[sourceName] if (mainip != None and nodename != None): query = ( "mod.run('{0}', '{1}', '{2}','{3}','{4}','{5}',con )" ).format(sourceName, coverage, mainip, nodename, dataLoc, ts) elif (mainip == None and nodename == None): query = ("mod.run('{0}', '{1}', '{2}','{3}',con)").format( sourceName, coverage, dataLoc, ts) exec(query)
storeAssocArango('elementAssoc', sourcezoneKey, sourceKey, 'zoneInterface', client) if (dzone not in zoned): destinKey = created.get(destin) destinzoneKey = storeElementArango('element', 'zone', 'network', dzone, None, client) storeAssocArango('elementAssoc', destinzoneKey, destinKey, 'zoneInterface', client) storeAssocArango('metaelementAssoc', srcId, destinzoneKey, 'sourceNetwork', client) zoned[dzone] = destinzoneKey else: destinKey = created.get(destin) destinzoneKey = zoned.get(dzone) storeAssocArango('elementAssoc', destinzoneKey, destinKey, 'zoneInterface', client) def run(sourceName,scope, sourceIP,nodeName, dataFile, date, con): date=str(date) if(checkExists(dataFile, con)== False): print('Importing :', dataFile, ' from ', sourceName) hostsData = readFile(dataFile) #hostsData=paperRUN.readMyCSV(dataFile, ',', False) #hostsData=parseReadData(hostsData) hash=tools.hashfile(open(dataFile, "rb"),dataFile) store(sourceName, scope, sourceIP, nodeName, hostsData, hash, date, con) else: print(dataFile + " already imported") #run('winscanner', '10.1.1.112',"..\\data\\p0f\\p0f_output", tools.creationDate("..\\data\\p0f\\p0f_output"), createConArango('Lab')) #run('winscanner', 'subnet', '10.1.1.112','Winscanner',"..\\data\\p0f\\p0f_output_allLANs", tools.creationDate("..\\data\\wireshark\\summary-packets_fixed.xml"), createConArango('Lab')) run('Wireshark', 'subnet','10.1.1.112','Winscanner',"..\\data\\wireshark\\summary-packets_fixed.xml", tools.creationDate("..\\data\\wireshark\\summary-packets_fixed.xml"), createConArango('Lab')) #run('Wireshark', 'subnet','10.1.1.112','Winscanner',"F:\\ongoingWork\\ModBusCaps\\unzipped\\converted\\packets_00001_20161115135616.xml", tools.creationDate("F:\\ongoingWork\\ModBusCaps\\unzipped\\converted\\packets_00001_20161115135616.xml"), createConArango('Lab')) #run('Wireshark', 'subnet','199.206.2.31','Winscanner','F:\ongoingWork\FOI\pcap\snart-fw.cleanfurnace1.se\psml\pcap_00.csv', tools.creationDate("F:\ongoingWork\FOI\pcap\snart-fw.cleanfurnace1.se\psml\pcap_00.csv"), createConArango('Lab'))
""" tektek=readFile("..\\data\\bldde01\\Software.xml") for k, v in tektek.items(): print(v.get('vendor')+','+v.get('name')+','+v.get('version')) """ #store('Windows_Config', '192.168.109.5', readFile("..\\data\\bldad01\\Software.xml"), None, tools.creationDate("..\\data\\bldad01\\Software.xml"), createConArango('claimOmania')) def run(sourceName, scope, ip, nodeName, dataFile, date, con): date = str(date) if (checkExists(dataFile, con) == False): print('Importing :', dataFile, ' from ', sourceName) hostsData = readFile(dataFile) hash = tools.hashfile(open(dataFile, "rb"), dataFile) store(sourceName, scope, ip, nodeName, hostsData, hash, date, con) else: print(dataFile + " already imported") run('Powershell-Win32_Product', 'system', '192.168.109.21', 'bldad01', "..\\data\\bldad01\\Software.xml", tools.creationDate("..\\data\\bldad01\\Software.xml"), createConArango('OntoLab')) run('Powershell-Win32_Product', 'system', '192.168.109.23', 'bldde01', "..\\data\\bldde01\\Software.xml", tools.creationDate("..\\data\\bldde01\\Software.xml"), createConArango('OntoLab')) run('Powershell-Win32_Product', 'system', '192.168.109.110', 'winscanner', "..\\data\\winscanner\\Software.xml", tools.creationDate("..\\data\\winscanner\\Software.xml"), createConArango('OntoLab'))
softPropertyVers = storeElementArango('element', 'property', 'version', softvers, None, client) softPropertyChange = storeElementArango('element', 'property', 'lastChange', softChange, None, client) softPropertyPackId = storeElementArango('element', 'property', 'packageCode', softPackageId, None, client) #Associations storeAssocArango('elementAssoc', nodeID, softInst, 'nodeSoftware', client) storeAssocArango('elementAssoc', softInst, softPropertyVendor, 'softwareProperty', client) storeAssocArango('elementAssoc', softInst, softPropertyVers, 'softwareProperty', client) storeAssocArango('elementAssoc', softInst, softPropertyChange, 'softwareProperty', client) storeAssocArango('elementAssoc', softInst, softPropertyPackId, 'softwareProperty', client) else: print('No node to add data to') """ tektek=readFile("..\\data\\bldde01\\Software.xml") for k, v in tektek.items(): print(v.get('vendor')+','+v.get('name')+','+v.get('version')) """ #store('Windows_Config', '192.168.109.5', readFile("..\\data\\bldad01\\Software.xml"), None, tools.creationDate("..\\data\\bldad01\\Software.xml"), createConArango('claimOmania')) def run(sourceName, scope, ip,nodeName, dataFile, date, con): date=str(date) if(checkExists(dataFile, con)== False): print('Importing :', dataFile, ' from ', sourceName) hostsData = readFile(dataFile) hash=tools.hashfile(open(dataFile, "rb"),dataFile) store(sourceName,scope, ip, nodeName,hostsData, hash, date, con) else: print(dataFile + " already imported") run('Powershell-Win32_Product','system', '192.168.109.21','bldad01',"..\\data\\bldad01\\Software.xml", tools.creationDate("..\\data\\bldad01\\Software.xml"), createConArango('Lab')) run('Powershell-Win32_Product','system', '192.168.109.23','bldde01',"..\\data\\bldde01\\Software.xml", tools.creationDate("..\\data\\bldde01\\Software.xml"), createConArango('Lab')) run('Powershell-Win32_Product','system', '192.168.109.110','winscanner',"..\\data\\winscanner\\Software.xml", tools.creationDate("..\\data\\winscanner\\Software.xml"), createConArango('Lab'))
): #=getPortNames('adapters\\thirddata\\service-names-port-numbers.xml')): date = str(date) if (checkExists(dataFile, con) == False): print('Importing :', dataFile, ' from ', sourceName) hostsData = readFile(dataFile, portnames) hash = tools.hashfile(open(dataFile, "rb"), dataFile) store(sourceName, scope, sourceIP, nodeName, hostsData, hash, date, con) else: print(dataFile + " already imported") ###CHECK THE FIRST DATA! >> edge error #run('p0f', 'subnet','10.1.1.112','Winscanner',"..\\data\\p0f\\p0f_output", tools.creationDate("..\\data\\p0f\\p0f_output"), ontologyTools2.getPortNames('thirddata\\service-names-port-numbers.xml'), createConArango('Lab')) #run('p0f', 'subnet','10.1.1.112','Winscanner',"..\\data\\p0f\\nextScanOutput", tools.creationDate("..\\data\\p0f\\nextScanOutput"), ontologyTools2.getPortNames('thirddata\\service-names-port-numbers.xml'), createConArango('Lab')) run('p0f', 'subnet', '10.1.1.112', 'Winscanner', "..\\data\\p0f\\p0f_output_allLANs", tools.creationDate("..\\data\\p0f\\p0f_output_allLANs"), ontologyTools2.getPortNames('thirddata\\service-names-port-numbers.xml'), createConArango('Lab')) #print(tools.creationDate("..\\data\\p0f\\nextScanOutput")) #pprint(readFile("..\\data\\p0f\\\p0f_output", ontologyTools2.getPortNames('thirddata\\service-names-port-numbers.xml'))) #pprint(readFile("C:\\Temp\\foi\\pcaptxt_00.txt", ontologyTools2.getPortNames('thirddata\\service-names-port-numbers.xml'))) #2#run('p0f', 'subnet','199.206.2.23','Winscanner',"C:\\Temp\\foi\\pcaptxt_00.txt", tools.creationDate("C:\\Temp\\foi\\pcaptxt_00.txt"), ontologyTools2.getPortNames('thirddata\\service-names-port-numbers.xml'), createConArango('Lab')) #print(getSourceClasses('metaelement/4506083',createConArango('claimOmania')))
if (protocol != None): protKey = storeElementArango('element', 'property', 'protocol', protocol, None, client) storeAssocArango('elementAssoc', servKey, protKey, 'ServerProtocol', client) #if (serverName != None): #srvSpKey = storeElementArango('element', 'softwareInstance', serverName, None, client) #storeAssocArango('elementAssoc', servKey, srvSpKey, 'serverSoftwareproduct', client) def run(sourceName, scope, dataFile, date, con): if (checkExists(dataFile, con) == False): print('Importing :', dataFile, ' from ', sourceName) hostsData = hosts(dataFile) #timestamp = str(datetime.date.today()) hash = tools.hashfile(open(dataFile, "rb"), dataFile) #storeBasic(con) store(sourceName, scope, hostsData, hash, date, con) else: print(dataFile + " already imported") ###run('Nexpose', "..\data\\2014-03-28\\FullXMLReport_v2.xml", '2014-03-28', createConArango('claimOmania')) #pprint(hosts("..\data\\nexpose\\nexpose04_12.xml")) run('Nexpose', 'subnet', "..\data\\nexpose\\nexpose04_12.xml", tools.creationDate("..\data\\nexpose\\nexpose04_12.xml"), createConArango('OntoLab')) #run('Nexpose','subnet', "..\data\\nexpose\\report.xml", tools.creationDate("..\data\\nexpose\\report.xml"), createConArango('OntoLab'))
sourcezoneKey = zoned.get(szone) storeAssocArango('elementAssoc', sourcezoneKey, sourceKey, 'zoneInterface', client) if (dzone not in zoned): destinKey = created.get(destin) destinzoneKey = storeElementArango('element', 'zone', 'network', dzone, None, client) storeAssocArango('elementAssoc', destinzoneKey, destinKey, 'zoneInterface', client) storeAssocArango('metaelementAssoc', srcId, destinzoneKey, 'sourceNetwork', client) zoned[dzone] = destinzoneKey else: destinKey = created.get(destin) destinzoneKey = zoned.get(dzone) storeAssocArango('elementAssoc', destinzoneKey, destinKey, 'zoneInterface', client) def run(sourceName,scope, sourceIP,nodeName, dataFile, date, con): date=str(date) if(checkExists(dataFile, con)== False): print('Importing :', dataFile, ' from ', sourceName) hostsData = readFile(dataFile) hash=tools.hashfile(open(dataFile, "rb"),dataFile) store(sourceName, scope, sourceIP, nodeName, hostsData, hash, date, con) else: print(dataFile + " already imported") #run('winscanner', '10.1.1.112',"..\\data\\p0f\\p0f_output", tools.creationDate("..\\data\\p0f\\p0f_output"), createConArango('OntoLab')) #run('winscanner', '10.1.1.112',"..\\data\\p0f\\p0f_output_allLANs", tools.creationDate("..\\data\\p0f\\p0f_output_allLANs"), createConArango('claimOmania')) #run('Wireshark', 'subnet','10.1.1.112','Winscanner',"..\\data\\wireshark\\summary-packets_fixed.xml", tools.creationDate("..\\data\\wireshark\\summary-packets_fixed.xml"), createConArango('OntoLab')) run('Wireshark', 'subnet','10.1.1.112','Winscanner',"F:\\ongoingWork\\ModBusCaps\\unzipped\\converted\\packets_00001_20161115135616.xml", tools.creationDate("F:\\ongoingWork\\ModBusCaps\\unzipped\\converted\\packets_00001_20161115135616.xml"), createConArango('OntoLab'))
else: serviceName = ('-'.join((port, protocol, serviceName))) serverName = v.get("serviceSoftware") servKey=storeElementArango('element','softwareInstance','server',serviceName, None, client) storeAssocArango('elementAssoc', nodeKey, servKey,'nodeServer',client) if(protocol!=None): protKey = storeElementArango('element', 'property','protocol', protocol, None, client) storeAssocArango('elementAssoc', servKey, protKey, 'property', client) if(serverName!=None): srvSpKey = storeElementArango('element', 'property','serverName', serverName, None, client) storeAssocArango('elementAssoc', servKey, srvSpKey, 'serverSoftwareproduct', client) def run(sourceName,scope, dataFile, date, con): if(checkExists(dataFile, con)== False): print('Importing :', dataFile, ' from ', sourceName) hostsData = hosts(dataFile) #timestamp = str(datetime.date.today()) hash=tools.hashfile(open(dataFile, "rb"),dataFile) #storeBasic(con) store(sourceName,scope, hostsData, hash, date, con) else: print(dataFile + " already imported") #store('Nmap', hosts("..\data\\2014-11-06\\nmap_20-10-2014.xml"), tools.hashfile(open("..\data\\2014-11-06\\nmap_20-10-2014.xml", "rb")), '2014-11-06', createConArango('claimOmania')) run('Nmap','subnet', "..\\data\\nmap\\04_25_nmap_scan.xml", tools.creationDate("..\\data\\nmap\\04_25_nmap_scan.xml"), createConArango('Lab')) #run('Nmap','subnet', "..\\data\\nmap\\all-subnets-SCADAscan.xml", tools.creationDate("..\\data\\nmap\\all-subnets-SCADAscan.xml"), createConArango('Lab')) #pprint(hosts('..\\data\\nmap\\04_25_nmap_scan.xml'))
storeAssocArango('elementAssoc', destinzoneKey, destinKey, 'zoneInterface', client) storeAssocArango('metaelementAssoc', srcId, destinzoneKey, 'sourceNetwork', client) zoned[dzone] = destinzoneKey else: destinKey = created.get(destin) destinzoneKey = zoned.get(dzone) storeAssocArango('elementAssoc', destinzoneKey, destinKey, 'zoneInterface', client) def run(sourceName,scope, sourceIP,nodeName, dataFile, date, portnames, con):#=getPortNames('adapters\\thirddata\\service-names-port-numbers.xml')): date=str(date) if(checkExists(dataFile, con)== False): print('Importing :', dataFile, ' from ', sourceName) hostsData = readFile(dataFile, portnames) hash=tools.hashfile(open(dataFile, "rb"),dataFile) store(sourceName, scope, sourceIP, nodeName, hostsData, hash, date, con) else: print(dataFile + " already imported") ###CHECK THE FIRST DATA! >> edge error run('p0f', 'subnet','10.1.1.112','Winscanner',"..\\data\\p0f\\p0f_output", tools.creationDate("..\\data\\p0f\\p0f_output"), ontologyTools2.getPortNames('thirddata\\service-names-port-numbers.xml'), createConArango('OntoLab')) #run('p0f', 'subnet','10.1.1.112','Winscanner',"..\\data\\p0f\\nextScanOutput", tools.creationDate("..\\data\\p0f\\nextScanOutput"), ontologyTools2.getPortNames('thirddata\\service-names-port-numbers.xml'), createConArango('OntoLab')) #run('p0f', 'subnet','10.1.1.112','Winscanner',"..\\data\\p0f\\p0f_output_allLANs", tools.creationDate("..\\data\\p0f\\p0f_output_allLANs"), ontologyTools2.getPortNames('thirddata\\service-names-port-numbers.xml'), createConArango('OntoLab')) #print(tools.creationDate("..\\data\\p0f\\nextScanOutput")) #pprint(readFile("..\\data\\p0f\\\p0f_output_allLANs", writeOntologyTrees.getPortNames('..\\ontology\\service-names-port-numbers.xml'))) #print(getSourceClasses('metaelement/4506083',createConArango('claimOmania')))
'server', serviceName, None, client) storeAssocArango('elementAssoc', nodeKey, servKey, 'operatingsystemServer', client) if (protocol != None): protKey = storeElementArango('element', 'property', 'protocol', protocol, None, client) storeAssocArango('elementAssoc', servKey, protKey, 'ServerProtocol', client) def run(sourceName, scope, dataFile, date, con): if (checkExists(dataFile, con) == False): print('Importing :', dataFile, ' from ', sourceName) hostsData = hosts(dataFile) #timestamp = str(datetime.date.today()) hash = tools.hashfile(open(dataFile, "rb"), dataFile) #storeBasic(con) store(sourceName, scope, hostsData, hash, date, con) else: print(dataFile + " already imported") run( 'Nessus', 'subnet', "..\data\\nessus\\Authenticated_SCADA_lab_scan_encn7b.xml", tools.creationDate( "..\data\\nessus\\Authenticated_SCADA_lab_scan_encn7b.xml"), createConArango('OntoLab')) #run('Nessus','subnet', "..\data\\nessus\\Authenticated_SCADA_lab_scan_mj3yvv.xml", tools.creationDate("..\data\\nessus\\Authenticated_SCADA_lab_scan_mj3yvv.xml"), createConArango('OntoLab'))
client) storeAssocArango('elementAssoc', servKey, protKey, 'property', client) if (serverName != None): srvSpKey = storeElementArango('element', 'property', 'serverName', serverName, None, client) storeAssocArango('elementAssoc', servKey, srvSpKey, 'serverSoftwareproduct', client) def run(sourceName, scope, dataFile, date, con): if (checkExists(dataFile, con) == False): print('Importing :', dataFile, ' from ', sourceName) hostsData = hosts(dataFile) #timestamp = str(datetime.date.today()) hash = tools.hashfile(open(dataFile, "rb"), dataFile) #storeBasic(con) store(sourceName, scope, hostsData, hash, date, con) else: print(dataFile + " already imported") #store('Nmap', hosts("..\data\\2014-11-06\\nmap_20-10-2014.xml"), tools.hashfile(open("..\data\\2014-11-06\\nmap_20-10-2014.xml", "rb")), '2014-11-06', createConArango('claimOmania')) run('Nmap', 'subnet', "..\\data\\nmap\\04_25_nmap_scan.xml", tools.creationDate("..\\data\\nmap\\04_25_nmap_scan.xml"), createConArango('OntoLab')) #run('Nmap','subnet', "..\\data\\nmap\\all-subnets-SCADAscan.xml", tools.creationDate("..\\data\\nmap\\all-subnets-SCADAscan.xml"), createConArango('OntoLab'))