def test_windowed_query(app, excl_network): # pylint: disable=unused-argument
"""test windowed query"""
assert list(windowed_query(Excl.query, Excl.id, 1))
assert list(
windowed_query(
db.session.query(Excl.id, Excl.id).select_from(Excl), Excl.id, 1))
def rescan_services(_, interval, queue):
"""rescan services from storage; update known services info"""
qref = Queue.query.filter(Queue.name == queue).one()
now = datetime.utcnow()
rescan_horizont = now - timedelta(seconds=timeparse(interval))
query = Service.query.filter(or_(Service.rescan_time < rescan_horizont, Service.rescan_time == None)) # noqa: E501,E711 pylint: disable=singleton-comparison
rescan, ids = [], []
for service in windowed_query(query, Service.id):
item = f'{service.proto}://{format_host_address(service.host.address)}:{service.port}'
rescan.append(item)
ids.append(service.id)
# orm is bypassed for performance reasons in case of large rescans
update_statement = Service.__table__.update().where(Service.id.in_(ids)).values(rescan_time=now)
db.session.execute(update_statement)
db.session.commit()
db.session.expire_all()
rescan = filter_already_queued(qref, rescan)
queue_enqueue(qref, rescan)
if rescan:
current_app.logger.info(f'rescan_services, rescan {len(rescan)} items')
def rescan_hosts(ctx, interval):
"""rescan hosts from storage; discovers new services on hosts"""
now = datetime.utcnow()
rescan_horizont = now - timedelta(seconds=timeparse(interval))
query = Host.query.filter(or_(Host.rescan_time < rescan_horizont, Host.rescan_time == None)) # noqa: E711 pylint: disable=singleton-comparison
rescan, ids = [], []
for host in windowed_query(query, Host.id):
rescan.append(host.address)
ids.append(host.id)
# orm is bypassed for performance reasons in case of large rescans
update_statement = Host.__table__.update().where(Host.id.in_(ids)).values(rescan_time=now)
db.session.execute(update_statement)
db.session.commit()
db.session.expire_all()
ctx.data = rescan
if rescan:
current_app.logger.info(f'rescan_hosts, rescan {len(rescan)} items')
return ctx
def sync_es_index(cvesearch_url, esd_url, namelen):
"""
synchronize vulnsearch esd index with cvesearch data for all cpe notes in storage
"""
esclient = Elasticsearch([esd_url])
indexer = BulkIndexer(esclient)
# create new index. there can be new or updated services or cpe notes of
# existing services, also there might be new or updated cves for cpes the
# easiest way to handle such complex situation is to create new index and
# update alias used by the vulnsearch elk ui objects
current_index = f'{ES_INDEX}-{time()}'
for note in windowed_query(Note.query.filter(Note.xtype == 'cpe'), Note.id):
for icpe in json.loads(note.data):
try:
parsed_cpe = CPE(icpe)
except Exception: # pylint: disable=broad-except ; library does not provide own core exception class
current_app.logger.warning(f'invalid cpe, note_id:{note.id} {icpe}')
continue
if not parsed_cpe.get_version()[0]:
continue
for cve in cvefor(icpe, cvesearch_url):
data_id, data = vulndata(note, parsed_cpe, cve, namelen)
indexer.index(current_index, data_id, data)
indexer.flush()
# update alias and prune old indexes
update_managed_indices(esclient, current_index)
# print cache stats
current_app.logger.debug(f'cvefor cache: {cvefor.cache_info()}') # pylint: disable=no-value-for-parameter ; lru decorator side-effect
def filter_already_queued(queue, targets):
"""filters already queued targets"""
current_targets = {x[0]: 0 for x in windowed_query(db.session.query(Target.target).filter(Target.queue == queue), Target.id)}
targets = [tgt for tgt in targets if tgt not in current_targets]
return targets