def get(self, spec, **kargs): """Queries the active index.""" for rec in helpers.scan(self.db_client, query={"query": spec}, index=self.indexes[0], ignore_unavailable=True): host = dict(rec['_source'], _id=rec['_id']) if 'coordinates' in host.get('infos', {}): host['infos']['coordinates'] = host['infos'][ 'coordinates'][::-1] for field in self.datetime_fields: if field in host: host[field] = utils.all2datetime(host[field]) yield host
def fix_result(value): return utils.all2datetime(value / 1000)
def _store_host(self, host): addr = self.convert_ip(host['addr']) info = host.get('infos') if 'coordinates' in (info or {}).get('loc', {}): info['coordinates'] = info.pop('loc')['coordinates'][::-1] source = host.get('source', []) host_tstart = utils.all2datetime(host['starttime']) host_tstop = utils.all2datetime(host['endtime']) insrt = postgresql.insert(self.tables.scan) scanid, scan_tstop = self.db.execute( insrt.values( addr=addr, source=source, info=info, time_start=host_tstart, time_stop=host_tstop, **dict( (key, host.get(key)) for key in ['state', 'state_reason', 'state_reason_ttl'] if key in host)).on_conflict_do_update( index_elements=['addr'], set_={ 'source': self.tables.scan.source + insrt.excluded.source, 'time_start': func.least( self.tables.scan.time_start, insrt.excluded.time_start, ), 'time_stop': func.greatest( self.tables.scan.time_stop, insrt.excluded.time_stop, ), }, ).returning(self.tables.scan.id, self.tables.scan.time_stop)).fetchone() newest = scan_tstop <= host_tstop for category in host.get("categories", []): insrt = postgresql.insert(self.tables.category) catid = self.db.execute( insrt.values(name=category).on_conflict_do_update( index_elements=['name'], set_={ 'name': insrt.excluded.name }).returning(self.tables.category.id)).fetchone()[0] self.db.execute( postgresql.insert( self.tables.association_scan_category).values( scan=scanid, category=catid).on_conflict_do_nothing()) for port in host.get('ports', []): scripts = port.pop('scripts', []) # FIXME: handle screenshots for fld in [ 'screendata', 'screenshot', 'screenwords', 'service_method' ]: try: del port[fld] except KeyError: pass if 'service_servicefp' in port: port['service_fp'] = port.pop('service_servicefp') if 'state_state' in port: port['state'] = port.pop('state_state') if 'state_reason_ip' in port: port['state_reason_ip'] = self.convert_ip( port['state_reason_ip']) insrt = postgresql.insert(self.tables.port) portid = self.db.execute( insrt.values(scan=scanid, **port).on_conflict_do_update( index_elements=['scan', 'port', 'protocol'], set_=dict(scan=scanid, **(port if newest else {}))).returning( self.tables.port.id)).fetchone()[0] for script in scripts: name, output = script.pop('id'), script.pop('output') if newest: insrt = postgresql.insert(self.tables.script) self.bulk.append( insrt.values(port=portid, name=name, output=output, data=script).on_conflict_do_update( index_elements=['port', 'name'], set_={ "output": insrt.excluded.output, "data": insrt.excluded.data, }, )) else: insrt = postgresql.insert(self.tables.script) self.bulk.append( insrt.values(port=portid, name=name, output=output, data=script).on_conflict_do_nothing()) for trace in host.get('traces', []): traceid = self.db.execute( postgresql.insert(self.tables.trace).values( scan=scanid, port=trace.get('port'), protocol=trace['protocol']).on_conflict_do_nothing(). returning(self.tables.trace.id)).fetchone()[0] for hop in trace.get('hops'): hop['ipaddr'] = self.convert_ip(hop['ipaddr']) self.bulk.append( postgresql.insert(self.tables.hop).values( trace=traceid, ipaddr=self.convert_ip(hop['ipaddr']), ttl=hop["ttl"], rtt=None if hop["rtt"] == '--' else hop["rtt"], host=hop.get("host"), domains=hop.get("domains"), )) for hostname in host.get('hostnames', []): self.bulk.append( postgresql.insert(self.tables.hostname).values( scan=scanid, domains=hostname.get('domains'), name=hostname.get('name'), type=hostname.get('type'), ).on_conflict_do_nothing()) utils.LOGGER.debug("VIEW STORED: %r", scanid) return scanid
def _store_host(self, host): addr = self.ip2internal(host['addr']) info = host.get('infos') source = host.get('source', '') host_tstart = utils.all2datetime(host['starttime']) host_tstop = utils.all2datetime(host['endtime']) scanid = self.db.execute( postgresql.insert(self.tables.scan).values( addr=addr, source=source, info=info, time_start=host_tstart, time_stop=host_tstop, # FIXME: masscan results may lack 'state' and 'state_reason' state=host.get('state'), state_reason=host.get('state_reason'), state_reason_ttl=host.get('state_reason_ttl'), ).on_conflict_do_nothing().returning( self.tables.scan.id)).fetchone()[0] for category in host.get("categories", []): insrt = postgresql.insert(self.tables.category) catid = self.db.execute( insrt.values(name=category).on_conflict_do_update( index_elements=['name'], set_={ 'name': insrt.excluded.name }).returning(self.tables.category.id)).fetchone()[0] self.db.execute( postgresql.insert( self.tables.association_scan_category).values( scan=scanid, category=catid).on_conflict_do_nothing()) for port in host.get('ports', []): scripts = port.pop('scripts', []) # FIXME: handle screenshots for fld in [ 'screendata', 'screenshot', 'screenwords', 'service_method' ]: try: del port[fld] except KeyError: pass if 'service_servicefp' in port: port['service_fp'] = port.pop('service_servicefp') if 'state_state' in port: port['state'] = port.pop('state_state') if 'state_reason_ip' in port: port['state_reason_ip'] = self.ip2internal( port['state_reason_ip']) portid = self.db.execute( insert(self.tables.port).values(scan=scanid, **port).returning( self.tables.port.id)).fetchone()[0] for script in scripts: name, output = script.pop('id'), script.pop('output') self.bulk.append( insert(self.tables.script).values(port=portid, name=name, output=output, data=script)) for trace in host.get('traces', []): traceid = self.db.execute( insert(self.tables.trace).values( scan=scanid, port=trace.get('port'), protocol=trace['protocol']).returning( self.tables.trace.id)).fetchone()[0] for hop in trace.get('hops'): hop['ipaddr'] = self.ip2internal(hop['ipaddr']) self.bulk.append( insert(self.tables.hop).values( trace=traceid, ipaddr=self.ip2internal(hop['ipaddr']), ttl=hop["ttl"], rtt=None if hop["rtt"] == '--' else hop["rtt"], host=hop.get("host"), domains=hop.get("domains"), )) for hostname in host.get('hostnames', []): self.bulk.append( insert(self.tables.hostname).values( scan=scanid, domains=hostname.get('domains'), name=hostname.get('name'), type=hostname.get('type'), )) utils.LOGGER.debug("HOST STORED: %r", scanid) return scanid