def run(
self
): # it is necessary to get the qprocess because we need to send it back to the scheduler when we're done importing
try:
print "[+] Parsing nmap xml file: " + self.filename
starttime = time.time()
try:
parser = Parser(self.filename)
except:
print '\t[-] Giving up on import due to previous errors.'
print "\t[-] Unexpected error:", sys.exc_info()[0]
self.done.emit()
return
self.db.dbsemaphore.acquire(
) # ensure that while this thread is running, no one else can write to the DB
s = parser.get_session() # nmap session info
if s:
nmap_session(self.filename, s.start_time, s.finish_time,
s.nmap_version, s.scan_args, s.total_hosts,
s.up_hosts, s.down_hosts)
hostCount = len(parser.all_hosts())
if hostCount == 0: # to fix a division by zero if we ran nmap on one host
hostCount = 1
progress = 100.0 / hostCount
totalprogress = 0
self.tick.emit(int(totalprogress))
for h in parser.all_hosts(
): # create all the hosts that need to be created
db_host = nmap_host.query.filter_by(ip=h.ip).first()
if not db_host: # if host doesn't exist in DB, create it first
hid = nmap_host('', '', h.ip, h.ipv4, h.ipv6, h.macaddr,
h.status, h.hostname, h.vendor, h.uptime,
h.lastboot, h.distance, h.state, h.count)
note(hid, '')
session.commit()
for h in parser.all_hosts(
): # create all OS, service and port objects that need to be created
db_host = nmap_host.query.filter_by(
ip=h.ip).first() # fetch the host
os_nodes = h.get_OS() # parse and store all the OS nodes
for os in os_nodes:
db_os = nmap_os.query.filter_by(
host_id=db_host.id).filter_by(name=os.name).filter_by(
family=os.family).filter_by(
generation=os.generation).filter_by(
os_type=os.os_type).filter_by(
vendor=os.vendor).first()
if not db_os:
nmap_os(os.name, os.family, os.generation, os.os_type,
os.vendor, os.accuracy, db_host)
for p in h.all_ports(): # parse the ports
s = p.get_service()
if not (
s is None
): # check if service already exists to avoid adding duplicates
db_service = nmap_service.query.filter_by(
name=s.name).filter_by(
product=s.product).filter_by(
version=s.version).filter_by(
extrainfo=s.extrainfo).filter_by(
fingerprint=s.fingerprint).first()
if not db_service:
db_service = nmap_service(s.name, s.product,
s.version, s.extrainfo,
s.fingerprint)
else: # else, there is no service info to parse
db_service = None
# fetch the port
db_port = nmap_port.query.filter_by(
host_id=db_host.id).filter_by(
port_id=p.portId).filter_by(
protocol=p.protocol).first()
if not db_port:
db_port = nmap_port(p.portId, p.protocol, p.state,
db_host, db_service)
session.commit()
totalprogress += progress
self.tick.emit(int(totalprogress))
for h in parser.all_hosts(
): # create all script objects that need to be created
db_host = nmap_host.query.filter_by(ip=h.ip).first()
for p in h.all_ports():
for scr in p.get_scripts():
db_port = nmap_port.query.filter_by(
host_id=db_host.id).filter_by(
port_id=p.portId).filter_by(
protocol=p.protocol).first()
db_script = nmap_script.query.filter_by(
script_id=scr.scriptId).filter_by(
port_id=db_port.id).first()
if not db_script: # if this script object doesn't exist, create it
nmap_script(scr.scriptId, scr.output, db_port,
db_host)
for hs in h.get_hostscripts():
db_script = nmap_script.query.filter_by(
script_id=hs.scriptId).filter_by(
host_id=db_host.id).first()
if not db_script:
nmap_script(hs.scriptId, hs.output, None, db_host)
session.commit()
for h in parser.all_hosts(): # update everything
db_host = nmap_host.query.filter_by(ip=h.ip).first(
) # get host from DB (if any with the same IP address)
if db_host.ipv4 == '' and not h.ipv4 == '':
db_host.ipv4 = h.ipv4
if db_host.ipv6 == '' and not h.ipv6 == '':
db_host.ipv6 = h.ipv6
if db_host.macaddr == '' and not h.macaddr == '':
db_host.macaddr = h.macaddr
if not h.status == '':
db_host.status = h.status
if db_host.hostname == '' and not h.hostname == '':
db_host.hostname = h.hostname
if db_host.vendor == '' and not h.vendor == '':
db_host.vendor = h.vendor
if db_host.uptime == '' and not h.uptime == '':
db_host.uptime = h.uptime
if db_host.lastboot == '' and not h.lastboot == '':
db_host.lastboot = h.lastboot
if db_host.distance == '' and not h.distance == '':
db_host.distance = h.distance
if db_host.state == '' and not h.state == '':
db_host.state = h.state
if db_host.count == '' and not h.count == '':
db_host.count = h.count
tmp_name = ''
tmp_accuracy = '0' # TODO: check if better to convert to int for comparison
os_nodes = h.get_OS()
for os in os_nodes:
db_os = nmap_os.query.filter_by(
host_id=db_host.id).filter_by(name=os.name).filter_by(
family=os.family).filter_by(
generation=os.generation).filter_by(
os_type=os.os_type).filter_by(
vendor=os.vendor).first()
db_os.os_accuracy = os.accuracy # update the accuracy
if not os.name == '': # get the most accurate OS match/accuracy to store it in the host table for easier access
if os.accuracy > tmp_accuracy:
tmp_name = os.name
tmp_accuracy = os.accuracy
if os_nodes: # if there was operating system info to parse
if not tmp_name == '' and not tmp_accuracy == '0': # update the current host with the most accurate OS match
db_host.os_match = tmp_name
db_host.os_accuracy = tmp_accuracy
for p in h.all_ports():
s = p.get_service()
if not (s is None):
# fetch the service for this port
db_service = nmap_service.query.filter_by(
name=s.name).filter_by(
product=s.product).filter_by(
version=s.version).filter_by(
extrainfo=s.extrainfo).filter_by(
fingerprint=s.fingerprint).first()
else:
db_service = None
# fetch the port
db_port = nmap_port.query.filter_by(
host_id=db_host.id).filter_by(
port_id=p.portId).filter_by(
protocol=p.protocol).first()
db_port.state = p.state
if not (
db_service is None
): # if there is some new service information, update it
db_port.service_id = db_service.id
for scr in p.get_scripts(
): # store the script results (note that existing script outputs are also kept)
db_script = nmap_script.query.filter_by(
script_id=scr.scriptId).filter_by(
port_id=db_port.id).first()
if not scr.output == '':
db_script.output = scr.output
totalprogress += progress
self.tick.emit(int(totalprogress))
session.commit()
self.db.dbsemaphore.release() # we are done with the DB
print '\t[+] Finished in ' + str(time.time() -
starttime) + ' seconds.'
self.done.emit()
self.schedule.emit(
parser, self.output == ''
) # call the scheduler (if there is no terminal output it means we imported nmap)
except:
print '\t[-] Something went wrong when parsing the nmap file..'
print "\t[-] Unexpected error:", sys.exc_info()[0]
self.done.emit()
def run(self): # it is necessary to get the qprocess because we need to send it back to the scheduler when we're done importing
try:
print "[+] Parsing nmap xml file: " + self.filename
starttime = time.time()
try:
parser = Parser(self.filename)
except:
print '\t[-] Giving up on import due to previous errors.'
print "\t[-] Unexpected error:", sys.exc_info()[0]
self.done.emit()
return
self.db.dbsemaphore.acquire() # ensure that while this thread is running, no one else can write to the DB
s = parser.get_session() # nmap session info
if s:
nmap_session(self.filename, s.start_time, s.finish_time, s.nmap_version, s.scan_args, s.total_hosts, s.up_hosts, s.down_hosts)
hostCount = len(parser.all_hosts())
if hostCount==0: # to fix a division by zero if we ran nmap on one host
hostCount=1
progress = 100.0 / hostCount
totalprogress = 0
self.tick.emit(int(totalprogress))
for h in parser.all_hosts(): # create all the hosts that need to be created
db_host = nmap_host.query.filter_by(ip=h.ip).first()
if not db_host: # if host doesn't exist in DB, create it first
hid = nmap_host('', '', h.ip, h.ipv4, h.ipv6, h.macaddr, h.status, h.hostname, h.vendor, h.uptime, h.lastboot, h.distance, h.state, h.count)
note(hid, '')
session.commit()
for h in parser.all_hosts(): # create all OS, service and port objects that need to be created
db_host = nmap_host.query.filter_by(ip=h.ip).first() # fetch the host
os_nodes = h.get_OS() # parse and store all the OS nodes
for os in os_nodes:
db_os = nmap_os.query.filter_by(host_id=db_host.id).filter_by(name=os.name).filter_by(family=os.family).filter_by(generation=os.generation).filter_by(os_type=os.os_type).filter_by(vendor=os.vendor).first()
if not db_os:
nmap_os(os.name, os.family, os.generation, os.os_type, os.vendor, os.accuracy, db_host)
for p in h.all_ports(): # parse the ports
s = p.get_service()
if not (s is None): # check if service already exists to avoid adding duplicates
db_service = nmap_service.query.filter_by(name=s.name).filter_by(product=s.product).filter_by(version=s.version).filter_by(extrainfo=s.extrainfo).filter_by(fingerprint=s.fingerprint).first()
if not db_service:
db_service = nmap_service(s.name, s.product, s.version, s.extrainfo, s.fingerprint)
else: # else, there is no service info to parse
db_service = None
# fetch the port
db_port = nmap_port.query.filter_by(host_id=db_host.id).filter_by(port_id=p.portId).filter_by(protocol=p.protocol).first()
if not db_port:
db_port = nmap_port(p.portId, p.protocol, p.state, db_host, db_service)
session.commit()
totalprogress += progress
self.tick.emit(int(totalprogress))
for h in parser.all_hosts(): # create all script objects that need to be created
db_host = nmap_host.query.filter_by(ip=h.ip).first()
for p in h.all_ports():
for scr in p.get_scripts():
db_port = nmap_port.query.filter_by(host_id=db_host.id).filter_by(port_id=p.portId).filter_by(protocol=p.protocol).first()
db_script = nmap_script.query.filter_by(script_id=scr.scriptId).filter_by(port_id=db_port.id).first()
if not db_script: # if this script object doesn't exist, create it
nmap_script(scr.scriptId, scr.output, db_port, db_host)
for hs in h.get_hostscripts():
db_script = nmap_script.query.filter_by(script_id=hs.scriptId).filter_by(host_id=db_host.id).first()
if not db_script:
nmap_script(hs.scriptId, hs.output, None, db_host)
session.commit()
for h in parser.all_hosts(): # update everything
db_host = nmap_host.query.filter_by(ip=h.ip).first() # get host from DB (if any with the same IP address)
if db_host.ipv4 == '' and not h.ipv4 == '':
db_host.ipv4 = h.ipv4
if db_host.ipv6 == '' and not h.ipv6 == '':
db_host.ipv6 = h.ipv6
if db_host.macaddr == '' and not h.macaddr == '':
db_host.macaddr = h.macaddr
if not h.status == '':
db_host.status = h.status
if db_host.hostname == '' and not h.hostname == '':
db_host.hostname = h.hostname
if db_host.vendor == '' and not h.vendor == '':
db_host.vendor = h.vendor
if db_host.uptime == '' and not h.uptime == '':
db_host.uptime = h.uptime
if db_host.lastboot == '' and not h.lastboot == '':
db_host.lastboot = h.lastboot
if db_host.distance == '' and not h.distance == '':
db_host.distance = h.distance
if db_host.state == '' and not h.state == '':
db_host.state = h.state
if db_host.count == '' and not h.count == '':
db_host.count = h.count
tmp_name = ''
tmp_accuracy = '0' # TODO: check if better to convert to int for comparison
os_nodes = h.get_OS()
for os in os_nodes:
db_os = nmap_os.query.filter_by(host_id=db_host.id).filter_by(name=os.name).filter_by(family=os.family).filter_by(generation=os.generation).filter_by(os_type=os.os_type).filter_by(vendor=os.vendor).first()
db_os.os_accuracy = os.accuracy # update the accuracy
if not os.name == '': # get the most accurate OS match/accuracy to store it in the host table for easier access
if os.accuracy > tmp_accuracy:
tmp_name = os.name
tmp_accuracy = os.accuracy
if os_nodes: # if there was operating system info to parse
if not tmp_name == '' and not tmp_accuracy == '0': # update the current host with the most accurate OS match
db_host.os_match = tmp_name
db_host.os_accuracy = tmp_accuracy
for p in h.all_ports():
s = p.get_service()
if not (s is None):
# fetch the service for this port
db_service = nmap_service.query.filter_by(name=s.name).filter_by(product=s.product).filter_by(version=s.version).filter_by(extrainfo=s.extrainfo).filter_by(fingerprint=s.fingerprint).first()
else:
db_service = None
# fetch the port
db_port = nmap_port.query.filter_by(host_id=db_host.id).filter_by(port_id=p.portId).filter_by(protocol=p.protocol).first()
db_port.state = p.state
if not (db_service is None): # if there is some new service information, update it
db_port.service_id = db_service.id
for scr in p.get_scripts(): # store the script results (note that existing script outputs are also kept)
db_script = nmap_script.query.filter_by(script_id=scr.scriptId).filter_by(port_id=db_port.id).first()
if not scr.output == '':
db_script.output = scr.output
totalprogress += progress
self.tick.emit(int(totalprogress))
session.commit()
self.db.dbsemaphore.release() # we are done with the DB
print '\t[+] Finished in '+ str(time.time()-starttime) + ' seconds.'
self.done.emit()
self.schedule.emit(parser, self.output == '') # call the scheduler (if there is no terminal output it means we imported nmap)
except:
print '\t[-] Something went wrong when parsing the nmap file..'
print "\t[-] Unexpected error:", sys.exc_info()[0]
self.done.emit()
def run(
self
): # it is necessary to get the qprocess because we need to send it back to the scheduler when we're done importing
try:
self.importProgressWidget.show()
session = self.db.session()
self.tsLog("Parsing nmap xml file: " + self.filename)
starttime = time()
try:
parser = Parser(self.filename)
except:
self.tsLog('Giving up on import due to previous errors.')
self.tsLog("Unexpected error: {0}".format(sys.exc_info()[0]))
self.done.emit()
return
self.db.dbsemaphore.acquire(
) # ensure that while this thread is running, no one else can write to the DB
s = parser.get_session() # nmap session info
if s:
n = nmap_session(self.filename, s.start_time, s.finish_time,
s.nmap_version, s.scan_args, s.total_hosts,
s.up_hosts, s.down_hosts)
session.add(n)
hostCount = len(parser.all_hosts())
if hostCount == 0: # to fix a division by zero if we ran nmap on one host
hostCount = 1
totalprogress = 0
self.importProgressWidget.setProgress(int(totalprogress))
self.importProgressWidget.show()
createProgress = 0
createOsNodesProgress = 0
createPortsProgress = 0
for h in parser.all_hosts(
): # create all the hosts that need to be created
db_host = session.query(nmap_host).filter_by(ip=h.ip).first()
if not db_host: # if host doesn't exist in DB, create it first
hid = nmap_host(os_match='',
os_accuracy='',
ip=h.ip,
ipv4=h.ipv4,
ipv6=h.ipv6,
macaddr=h.macaddr,
status=h.status,
hostname=h.hostname,
vendor=h.vendor,
uptime=h.uptime,
lastboot=h.lastboot,
distance=h.distance,
state=h.state,
count=h.count)
self.tsLog("Adding db_host")
session.add(hid)
t_note = note(h.ip, 'Added by nmap')
session.add(t_note)
else:
self.tsLog("Found db_host already in db")
createProgress = createProgress + ((100.0 / hostCount) / 5)
totalprogress = totalprogress + createProgress
self.importProgressWidget.setProgress(int(totalprogress))
self.importProgressWidget.show()
session.commit()
for h in parser.all_hosts(
): # create all OS, service and port objects that need to be created
self.tsLog("Processing h {ip}".format(ip=h.ip))
db_host = session.query(nmap_host).filter_by(ip=h.ip).first()
if db_host:
self.tsLog(
"Found db_host during os/ports/service processing")
else:
self.log(
"Did not find db_host during os/ports/service processing"
)
os_nodes = h.get_OS() # parse and store all the OS nodes
self.tsLog(" 'os_nodes' to process: {os_nodes}".format(
os_nodes=str(len(os_nodes))))
for os in os_nodes:
self.tsLog(
" Processing os obj {os}".format(os=str(os.name)))
db_os = session.query(nmap_os).filter_by(
host_id=db_host.id).filter_by(name=os.name).filter_by(
family=os.family).filter_by(
generation=os.generation).filter_by(
os_type=os.os_type).filter_by(
vendor=os.vendor).first()
if not db_os:
t_nmap_os = nmap_os(os.name, os.family, os.generation,
os.os_type, os.vendor, os.accuracy,
db_host.id)
session.add(t_nmap_os)
createOsNodesProgress = createOsNodesProgress + (
(100.0 / hostCount) / 5)
totalprogress = totalprogress + createOsNodesProgress
self.importProgressWidget.setProgress(int(totalprogress))
self.importProgressWidget.show()
session.commit()
all_ports = h.all_ports()
self.tsLog(" 'ports' to process: {all_ports}".format(
all_ports=str(len(all_ports))))
for p in all_ports: # parse the ports
self.tsLog(" Processing port obj {port}".format(
port=str(p.portId)))
s = p.get_service()
if not (
s is None
): # check if service already exists to avoid adding duplicates
#print(" Found service {service} for port {port}".format(service=str(s.name),port=str(p.portId)))
#db_service = session.query(nmap_service).filter_by(name=s.name).filter_by(product=s.product).filter_by(version=s.version).filter_by(extrainfo=s.extrainfo).filter_by(fingerprint=s.fingerprint).first()
db_service = session.query(nmap_service).filter_by(
name=s.name).first()
if not db_service:
#print("Did not find service *********** name={0} prod={1} ver={2} extra={3} fing={4}".format(s.name, s.product, s.version, s.extrainfo, s.fingerprint))
db_service = nmap_service(s.name, s.product,
s.version, s.extrainfo,
s.fingerprint)
session.add(db_service)
# else:
#print("FOUND service *************** name={0}".format(db_service.name))
else: # else, there is no service info to parse
db_service = None
# fetch the port
db_port = session.query(nmap_port).filter_by(
host_id=db_host.id).filter_by(
port_id=p.portId).filter_by(
protocol=p.protocol).first()
if not db_port:
#print("Did not find port *********** portid={0} proto={1}".format(p.portId, p.protocol))
if db_service:
db_port = nmap_port(p.portId, p.protocol, p.state,
db_host.id, db_service.id)
else:
db_port = nmap_port(p.portId, p.protocol, p.state,
db_host.id, '')
session.add(db_port)
#else:
#print('FOUND port *************** portid={0}'.format(db_port.port_id))
createPortsProgress = createPortsProgress + (
(100.0 / hostCount) / 5)
totalprogress = totalprogress + createPortsProgress
self.importProgressWidget.setProgress(totalprogress)
self.importProgressWidget.show()
session.commit()
#totalprogress += progress
#self.tick.emit(int(totalprogress))
for h in parser.all_hosts(
): # create all script objects that need to be created
db_host = session.query(nmap_host).filter_by(ip=h.ip).first()
for p in h.all_ports():
for scr in p.get_scripts():
self.tsLog(
" Processing script obj {scr}".format(
scr=str(scr)))
db_port = session.query(nmap_port).filter_by(
host_id=db_host.id).filter_by(
port_id=p.portId).filter_by(
protocol=p.protocol).first()
db_script = session.query(nmap_script).filter_by(
script_id=scr.scriptId).filter_by(
port_id=db_port.id).first()
cveResults = scr.get_cves()
for cveEntry in cveResults:
t_cve = cve(name=cveEntry.name,
url=cveEntry.url,
source=cveEntry.source,
severity=cveEntry.severity,
product=cveEntry.product,
version=cveEntry.version,
hostId=db_host.id)
session.add(t_cve)
if not db_script: # if this script object doesn't exist, create it
t_nmap_script = nmap_script(
scr.scriptId, scr.output, db_port.id,
db_host.id)
self.tsLog(
" Adding nmap_script obj {script}".
format(script=scr.scriptId))
session.add(t_nmap_script)
for hs in h.get_hostscripts():
db_script = session.query(nmap_script).filter_by(
script_id=hs.scriptId).filter_by(
host_id=db_host.id).first()
if not db_script:
t_nmap_script = nmap_script(hs.scriptId, hs.output,
None, db_host.id)
session.add(t_nmap_script)
session.commit()
for h in parser.all_hosts(): # update everything
db_host = session.query(nmap_host).filter_by(ip=h.ip).first()
if db_host.ipv4 == '' and not h.ipv4 == '':
db_host.ipv4 = h.ipv4
if db_host.ipv6 == '' and not h.ipv6 == '':
db_host.ipv6 = h.ipv6
if db_host.macaddr == '' and not h.macaddr == '':
db_host.macaddr = h.macaddr
if not h.status == '':
db_host.status = h.status
if db_host.hostname == '' and not h.hostname == '':
db_host.hostname = h.hostname
if db_host.vendor == '' and not h.vendor == '':
db_host.vendor = h.vendor
if db_host.uptime == '' and not h.uptime == '':
db_host.uptime = h.uptime
if db_host.lastboot == '' and not h.lastboot == '':
db_host.lastboot = h.lastboot
if db_host.distance == '' and not h.distance == '':
db_host.distance = h.distance
if db_host.state == '' and not h.state == '':
db_host.state = h.state
if db_host.count == '' and not h.count == '':
db_host.count = h.count
session.add(db_host)
tmp_name = ''
tmp_accuracy = '0' # TODO: check if better to convert to int for comparison
os_nodes = h.get_OS()
for os in os_nodes:
db_os = session.query(nmap_os).filter_by(
host_id=db_host.id).filter_by(name=os.name).filter_by(
family=os.family).filter_by(
generation=os.generation).filter_by(
os_type=os.os_type).filter_by(
vendor=os.vendor).first()
db_os.os_accuracy = os.accuracy # update the accuracy
if not os.name == '': # get the most accurate OS match/accuracy to store it in the host table for easier access
if os.accuracy > tmp_accuracy:
tmp_name = os.name
tmp_accuracy = os.accuracy
if os_nodes: # if there was operating system info to parse
if not tmp_name == '' and not tmp_accuracy == '0': # update the current host with the most accurate OS match
db_host.os_match = tmp_name
db_host.os_accuracy = tmp_accuracy
session.add(db_host)
for p in h.all_ports():
s = p.get_service()
if not (s is None):
#db_service = session.query(nmap_service).filter_by(name=s.name).filter_by(product=s.product).filter_by(version=s.version).filter_by(extrainfo=s.extrainfo).filter_by(fingerprint=s.fingerprint).first()
db_service = session.query(nmap_service).filter_by(
name=s.name).first()
else:
db_service = None
# fetch the port
db_port = session.query(nmap_port).filter_by(
host_id=db_host.id).filter_by(
port_id=p.portId).filter_by(
protocol=p.protocol).first()
if db_port:
#print("************************ Found {0}".format(db_port))
if db_port.state != p.state:
db_port.state = p.state
session.add(db_port)
if not (
db_service is None
) and db_port.service_id != db_service.id: # if there is some new service information, update it
db_port.service_id = db_service.id
session.add(db_port)
for scr in p.get_scripts(
): # store the script results (note that existing script outputs are also kept)
db_script = session.query(nmap_script).filter_by(
script_id=scr.scriptId).filter_by(
port_id=db_port.id).first()
if not scr.output == '' and scr.output is not None:
db_script.output = scr.output
session.add(db_script)
totalprogress = 100
self.importProgressWidget.setProgress(int(totalprogress))
self.importProgressWidget.show()
session.commit()
self.db.dbsemaphore.release() # we are done with the DB
self.tsLog('Finished in ' + str(time() - starttime) + ' seconds.')
self.done.emit()
self.importProgressWidget.hide()
self.schedule.emit(
parser, self.output == ''
) # call the scheduler (if there is no terminal output it means we imported nmap)
except Exception as e:
self.tsLog('Something went wrong when parsing the nmap file..')
self.tsLog("Unexpected error: {0}".format(sys.exc_info()[0]))
self.tsLog(e)
raise
self.done.emit()