def json_get_running_times():
now = util.utcnow() # .replace(tzinfo=None) # everything is implicitly UTC
running_times = current_app.db.HostDoc.find({"status": "RUNNING"}, {
"_id": False,
"last_change": True,
"stage": True
})
running_times = list(running_times)
df = DataFrame(running_times)
df["age"] = ((now - df["last_change"]) / np.timedelta64(1, "s")).astype(
int) # timedelta to seconds
df["tally"] = 1
df = df.set_index(["stage", "age"])
g = df.groupby(level=["stage", "age"])
df2 = g.sum()
df2 = df2.fillna(0)
results = OrderedDict() # the order matters to c3js
results["x"] = list(df2.loc["NETSCAN1"].index)
# TODO stages not guaranteed to be in index
results["NETSCAN1"] = list(df2.loc["NETSCAN1"]["tally"].astype(int))
results["NETSCAN2"] = list(df2.loc["NETSCAN2"]["tally"].astype(int))
results["PORTSCAN"] = list(df2.loc["PORTSCAN"]["tally"].astype(int))
results["VULNSCAN"] = list(df2.loc["VULNSCAN"]["tally"].astype(int))
# import IPython; IPython.embed() #<<< BREAKPOINT >>>
return json.dumps(results, default=util.custom_json_handler)
def __check_database(self):
now = util.utcnow()
tickets = self.__db.tickets.find(
{
"source": "nessus",
"last_change": {
"$gt": self.__since
}
},
{
"_id": 1,
"owner": 1,
"details": 1,
"events.action": 1,
"events": {
"$slice": -1
},
},
).sort([("last_change", -1)])
tickets = list(tickets)
for (
x
) in tickets: # convert _ids to string since we can't set custom json handler
x["_id"] = str(x["_id"])
self.__since = now
return tickets
def close_tickets(self, closing_time=None):
if closing_time is None:
closing_time = util.utcnow()
not_up_ips = self.__ips - self.__seen_ips
ip_ints = [int(i) for i in not_up_ips]
# find tickets with ips that were not up and are open
tickets = self.__db.TicketDoc.find({"ip_int": {"$in": ip_ints}, "open": True})
for ticket in tickets:
# don't close tickets that are false_positives, just add event
reason = "host down"
self.__check_false_positive_expiration(
ticket, closing_time.replace(tzinfo=tz.tzutc())
) # explicitly set to UTC (see CYHY-286)
if ticket["false_positive"] is True:
event = {
"time": closing_time,
"action": TICKET_EVENT.UNVERIFIED,
"reason": reason,
"reference": None,
}
else:
ticket["open"] = False
ticket["time_closed"] = closing_time
event = {
"time": closing_time,
"action": TICKET_EVENT.CLOSED,
"reason": reason,
"reference": None,
}
ticket["events"].append(event)
ticket.save()
def __init__(self, db, emitter, logger, sleep_time=3000, history_size=100):
self.__db = db
self.__emitter = emitter
self.__logger = logger
self.__sleep_time = sleep_time
self.__since = util.utcnow() - datetime.timedelta(minutes=30)
self.__history = deque(maxlen=history_size)
schedule.every(sleep_time).seconds.do(self.__work)
def get_cybex_dataframe(db, start_date, ticket_severity):
now = util.utcnow()
tomorrow = now + datetime.timedelta(days=1)
days_to_graph = pd.to_datetime(pd.date_range(start_date, now), utc=True)
fed_executive_owners = db.RequestDoc.get_all_descendants("EXECUTIVE")
# Calculate Buckets
tix = db.TicketDoc.find(
{
"source": "nessus",
"details.severity": ticket_severity,
"false_positive": False,
"owner": {
"$in": fed_executive_owners
},
"$or": [{
"time_closed": {
"$gte": start_date
}
}, {
"time_closed": None
}],
},
{
"_id": False,
"time_opened": True,
"time_closed": True
},
)
tix = list(tix)
df = DataFrame(tix)
results_df = DataFrame(index=days_to_graph,
columns=["young", "old", "total"])
if not df.empty:
df.time_closed = df.time_closed.fillna(
tomorrow
) # for accounting purposes, say all open tix will close tomorrow
# convert times to datetime64
df.time_closed = pd.to_datetime(df.time_closed, utc=True)
df.time_opened = pd.to_datetime(df.time_opened, utc=True)
old_delta = np.timedelta64(TICKETS_CLOSED_PAST_DAYS, "D")
for start_of_day, values in results_df.iterrows():
end_of_day = start_of_day + np.timedelta64(
1, "D") - np.timedelta64(1, "ns")
open_on_day_mask = (df.time_opened <=
end_of_day) & (df.time_closed > start_of_day)
age_on_date = start_of_day - df.time_opened
age_on_date_masked = age_on_date.mask(open_on_day_mask == False)
values["total"] = open_on_day_mask.value_counts().get(True, 0)
values["young"] = ((age_on_date_masked <
old_delta).value_counts().get(True, 0))
values["old"] = ((age_on_date_masked >=
old_delta).value_counts().get(True, 0))
return results_df
def __check_database(self):
now = util.utcnow()
tickets = self.__db.new_hire.find({"latest": True})
tickets = list(tickets)
for (
x
) in tickets: # convert _ids to string since we can't set custom json handler
x["_id"] = str(x["_id"])
self.__since = now
return tickets
def close_tickets(self):
if self.__closing_time is None:
# You don't have to go home but you can't stay here
self.__closing_time = util.utcnow()
ip_ints = [int(i) for i in self.__ips]
# find tickets that are covered by this scan, but weren't just touched
# TODO: this is the way I wanted to do it, but it blows up mongo
# tickets = self.__db.TicketDoc.find({'ip_int':{'$in':ip_ints},
# 'port':{'$in':self.__ports},
# 'source_id':{'$in':self.__source_ids},
# '_id':{'$nin':list(self.__seen_ticket_ids)},
# 'source':self.__source,
# 'open':True})
# work-around using a pipeline
tickets = database.run_pipeline_cursor(
close_tickets_pl(
ip_ints,
list(self.__ports),
list(self.__source_ids),
list(self.__seen_ticket_ids),
self.__source,
),
self.__db,
)
for raw_ticket in tickets:
ticket = self.__db.TicketDoc(raw_ticket) # make it managed
# don't close tickets that are false_positives, just add event
reason = "vulnerability not detected"
self.__check_false_positive_expiration(
ticket, self.__closing_time.replace(tzinfo=tz.tzutc())
) # explicitly set to UTC (see CYHY-286)
if ticket["false_positive"] is True:
event = {
"time": self.__closing_time,
"action": TICKET_EVENT.UNVERIFIED,
"reason": reason,
"reference": None,
}
else:
ticket["open"] = False
ticket["time_closed"] = self.__closing_time
event = {
"time": self.__closing_time,
"action": TICKET_EVENT.CLOSED,
"reason": reason,
"reference": None,
}
if self.__manual_scan:
event["manual"] = True
ticket["events"].append(event)
ticket.save()
def transition_host(self,
ip,
up=None,
reason=None,
has_open_ports=None,
was_failure=False):
"""Attempts to move host from one state to another.
returns (HostDoc, state_changed)
- HostDoc: host that was transitioned
- state_changed: True if the state changed, False otherwise."""
host = self.__db.HostDoc.get_by_ip(ip)
if host == None:
self.__logger.warning(
"Could not find %s in database during transition_host call" %
ip)
return (None, False)
prev_stage = host["stage"]
prev_status = host["status"]
host_transitioned, host_finished_stage = self.__state_manager.transition(
host, up, has_open_ports, was_failure)
# Calculating the state of a HostDoc is non-trivial.
host.set_state(up, has_open_ports, reason)
# If host finished a stage, update timestamp for latest_scan.<stage_that_just_finished>
current_time = util.utcnow()
if host_finished_stage:
host["latest_scan"][prev_stage] = current_time
if host["status"] == STATUS.DONE:
host["latest_scan"][
STATUS.
DONE] = current_time # Update timestamp for reaching DONE status
# check to see if owner should use a scheduler
request = self.__db.requests.find_one({"_id": host["owner"]},
{"scheduler": True})
if request and request.get("scheduler") != None:
self.__scheduler.schedule(host)
# save all changes made to the host by the state manager and the scheduler
host.save()
if host_transitioned:
new_stage = host["stage"]
new_status = host["status"]
owner = host["owner"]
self.tally_update(owner, prev_stage, prev_status, new_stage,
new_status)
return (host, host_transitioned)
def close_tickets(self, closing_time=None):
if closing_time is None:
closing_time = util.utcnow()
ip_ints = [int(i) for i in self.__ips]
all_ports_scanned = len(self.__ports) == MAX_PORTS_COUNT
if all_ports_scanned:
# If all the ports were scanned we have an opportunity to close port 0
# tickets. This can only be done if no ports are open for an IP.
# Otherwise they can be closed in the VULNSCAN stage.
ips_with_no_open_ports = self.__ips - IPSet(self.__seen_ip_port.keys())
ips_with_no_open_ports_ints = [int(i) for i in ips_with_no_open_ports]
# Close all tickets regardless of protocol for ips_with_no_open_ports
tickets_to_close = self.__db.TicketDoc.find(
{"ip_int": {"$in": ips_with_no_open_ports_ints}, "open": True}
)
for ticket in tickets_to_close:
self.__handle_ticket_port_closed(ticket, closing_time)
# handle ips that had at least one port open
# next query optimized for all_ports_scanned
tickets = self.__db.TicketDoc.find(
{
"ip_int": {"$in": ip_ints},
"port": {"$ne": 0},
"protocol": {"$in": list(self.__protocols)},
"open": True,
}
)
else:
# not all ports scanned
tickets = self.__db.TicketDoc.find(
{
"ip_int": {"$in": ip_ints},
"port": {"$in": list(self.__ports)},
"protocol": {"$in": list(self.__protocols)},
"open": True,
}
)
for ticket in tickets:
if ticket["port"] in self.__seen_ip_port[ticket["ip"]]:
# this ticket's ip:port was open, so we skip closing it
continue
self.__handle_ticket_port_closed(ticket, closing_time)
def request_limits(self, when=None):
# returns {owner: {stage:limit, stage:limit, ...}, ...}
if when == None:
when = util.utcnow()
requests = self.__db.RequestDoc.find({"scan_types": SCAN_TYPE.CYHY})
results = {}
for request in requests:
if request["period_start"] < when and time_calc.in_windows(
request["windows"], when):
limits = copy.copy(self.TEMP_MAX_CONCURRENCY)
for limit in request.get("scan_limits", []):
limits[limit["scanType"]] = limit["concurrent"]
else:
limits = self.TEMP_OFF_CONCURRENCY
results[request["_id"]] = limits
return results
def csv_get_open_tickets(ticket_severity):
if ticket_severity == CRITICAL_SEVERITY:
severity_name = "_critical"
elif ticket_severity == HIGH_SEVERITY:
severity_name = "_high"
else:
severity_name = ""
csv = ncats_webd.cybex_queries.csv_get_open_tickets(
current_app.db, ticket_severity)
response = Response(csv, mimetype="text/csv")
response.headers[
"Content-Disposition"] = "attachment; filename=cybex_open_tickets{!s}_{!s}.csv".format(
severity_name,
util.utcnow().strftime("%Y%m%d"))
return response
def schedule(self, host):
super(DefaultScheduler, self).schedule(host)
# determine the new priority for the host
if host["state"]["up"] == False:
self.__process_down_host(host)
else:
# host was up
max_severity = self.__host_max_severity(host)
if max_severity > 0:
self.__process_vuln_host(host, max_severity)
else:
self.__process_vuln_free_host(host)
# determine the next scan time based on the priority
d = self.__timedelta_for_priority(host["priority"])
host["next_scan"] = util.utcnow() + d
def get_bod_open_tickets_dataframe(bod_start_date):
bod_owners = current_app.db.RequestDoc.get_all_descendants("EXECUTIVE")
tix = current_app.db.TicketDoc.find(
{
"source": "nessus",
"details.severity": 4,
"false_positive": False,
"owner": {"$in": bod_owners},
"open": True,
},
{
"_id": False,
"owner": True,
"time_opened": True,
"ip": True,
"port": True,
"details.name": True,
"details.cve": True,
},
)
tix = list(tix)
for x in tix:
x.update(x["details"])
del x["details"]
df = DataFrame(tix)
if not df.empty:
bsd_utc = pd.to_datetime(
bod_start_date
) # Store bod_start_date as pandas Timestamp
df["time_opened"] = df.time_opened.dt.tz_convert(
"UTC"
) # Mark df.time_opened as UTC
df["bod_time_opened"] = df.time_opened.apply(
lambda x, bsd_utc=bsd_utc: max(x, bsd_utc)
)
now = pd.to_datetime(
util.utcnow()
) # Store current UTC time as pandas Timestamp
df["bod_age_td"] = now - df.bod_time_opened
df["bod_age"] = df["bod_age_td"].astype("timedelta64[D]")
del df["bod_time_opened"]
del df["bod_age_td"]
df.sort_values(by="bod_age", ascending=False, inplace=True)
return df
def get_stats(db):
orgs = db.RequestDoc.get_owner_types(as_lists=True,
stakeholders_only=False,
include_retired=True)
orgs["SLTT"] = (orgs[AGENCY_TYPE.STATE] + orgs[AGENCY_TYPE.LOCAL] +
orgs[AGENCY_TYPE.TRIBAL] + orgs[AGENCY_TYPE.TERRITORIAL])
now = util.utcnow()
rd = util.report_dates(now)
ticket = dict()
ticket["open_non_FP_tix"], ticket["open_FP_tix"] = total_open(db, orgs)
(
ticket["FY_ticket_counts_by_year"],
ticket["FY_ticket_count_totals"],
) = ticket_counts_by_fiscal_year(db, rd["fy_start"])
ticket["currentFYactivity"] = return_activity(db, orgs,
"Current FY Activity",
rd["fy_start"], now)
ticket["previousFYactivity"] = return_activity(db, orgs,
"Previous FY Activity",
rd["prev_fy_start"],
rd["prev_fy_end"])
ticket["currentMonthActivity"] = return_activity(db, orgs,
"Current Month Activity",
rd["month_start"], now)
ticket["previousMonthActivity"] = return_activity(
db,
orgs,
"Previous Month Activity",
rd["prev_month_start"],
rd["prev_month_end"],
)
ticket["currentWeekActivity"] = return_activity(db, orgs,
"Current Week Activity",
rd["week_start"], now)
ticket["previousWeekActivity"] = return_activity(db, orgs,
"Previous Week Activity",
rd["prev_week_start"],
rd["prev_week_end"])
ticket["stakeholders_scanned"] = scanning_breakdown(db)
ticket["all_time_tickets"] = all_time_opened_and_closed_breakdown(db)
return rd, ticket
def get_closed_tickets_dataframe(db, ticket_severity):
closed_since_date = util.utcnow() - datetime.timedelta(
days=TICKETS_CLOSED_PAST_DAYS)
fed_executive_owners = db.RequestDoc.get_all_descendants("EXECUTIVE")
tix = db.TicketDoc.find(
{
"source": "nessus",
"time_closed": {
"$gte": closed_since_date
},
"details.severity": ticket_severity,
"owner": {
"$in": fed_executive_owners
},
"open": False,
},
{
"_id": False,
"owner": True,
"time_opened": True,
"time_closed": True,
"ip": True,
"port": True,
"details.name": True,
"details.cve": True,
},
)
tix = list(tix)
for x in tix:
x.update(x["details"])
del x["details"]
df = DataFrame(tix)
if not df.empty:
df["days_to_close"] = ((df.time_closed - df.time_opened).apply(
lambda x: x.total_seconds() / 86400.0).round(1))
df.sort_values(by="time_closed", ascending=True, inplace=True)
return df
def check_host_next_scans(self):
"""Moves hosts to WAITING status based on their "next_scan" field.
jump_starts previously "up" hosts, and ignores the owner init_stage (to be deprecated)
returns the number of modified hosts"""
now = util.utcnow()
# move "up" hosts to STAGE.PORTSCAN, STATUS.WAITING
up_hosts_cursor = self.__db.HostDoc.get_scheduled_hosts(
True, now, self.__next_scan_limit)
self.__logger.debug(
'Updating previous "up" hosts that are now due to be scanned.')
hosts_processed = self.__update_hosts_next_scans(
up_hosts_cursor, STAGE.PORTSCAN, STATUS.WAITING)
self.__logger.debug('Updated %d "up" hosts.' % hosts_processed)
# move "down" hosts to STAGE.NETSCAN1, STATUS.WAITING
down_hosts_cursor = self.__db.HostDoc.get_scheduled_hosts(
False, now, self.__next_scan_limit)
self.__logger.debug(
'Updating previous "down" hosts that are now due to be scanned.')
hosts_processed = self.__update_hosts_next_scans(
down_hosts_cursor, STAGE.NETSCAN1, STATUS.WAITING)
self.__logger.debug('Updated %d "down" hosts.' % hosts_processed)
def in_windows(windows, time=None):
if time == None:
time = util.utcnow()
for w in windows:
parse_me = "%s %s" % (w["day"], w["start"])
dt = parser.parse(parse_me)
dow = dt.weekday()
relative_weekday = dateutil.relativedelta.weekday(dow)
duration = int(w["duration"])
delta = relativedelta(
weekday=relative_weekday(-1),
hour=dt.hour,
minute=dt.minute,
second=dt.second,
microsecond=dt.microsecond,
)
window_start = time + delta
window_duration = relativedelta(hours=+duration)
window_close = window_start + window_duration
if time > window_start and time < window_close:
return True
return False
def open_ticket(self, vuln, reason):
if self.__closing_time is None or self.__closing_time < vuln["time"]:
self.__closing_time = vuln["time"]
# search for previous open ticket that matches
prev_open_ticket = self.__db.TicketDoc.find_one(
{
"ip_int": long(vuln["ip"]),
"port": vuln["port"],
"protocol": vuln["protocol"],
"source": vuln["source"],
"source_id": vuln["plugin_id"],
"open": True,
}
)
if prev_open_ticket:
self.__generate_ticket_details(vuln, prev_open_ticket)
self.__check_false_positive_expiration(
prev_open_ticket, vuln["time"].replace(tzinfo=tz.tzutc())
) # explicitly set to UTC (see CYHY-286)
# add an entry to the existing open ticket
event = {
"time": vuln["time"],
"action": TICKET_EVENT.VERIFIED,
"reason": reason,
"reference": vuln["_id"],
}
if self.__manual_scan:
event["manual"] = True
prev_open_ticket["events"].append(event)
prev_open_ticket.save()
self.__mark_seen(prev_open_ticket)
return
# no matching tickets are currently open
# search for a previously closed ticket that was closed before the cutoff
cutoff_date = util.utcnow() + self.__reopen_delta
reopen_ticket = self.__db.TicketDoc.find_one(
{
"ip_int": long(vuln["ip"]),
"port": vuln["port"],
"protocol": vuln["protocol"],
"source": vuln["source"],
"source_id": vuln["plugin_id"],
"open": False,
"time_closed": {"$gt": cutoff_date},
}
)
if reopen_ticket:
self.__generate_ticket_details(vuln, reopen_ticket)
event = {
"time": vuln["time"],
"action": TICKET_EVENT.REOPENED,
"reason": reason,
"reference": vuln["_id"],
}
if self.__manual_scan:
event["manual"] = True
reopen_ticket["events"].append(event)
reopen_ticket["time_closed"] = None
reopen_ticket["open"] = True
reopen_ticket.save()
self.__mark_seen(reopen_ticket)
return
# time to open a new ticket
new_ticket = self.__db.TicketDoc()
new_ticket.ip = vuln["ip"]
new_ticket["port"] = vuln["port"]
new_ticket["protocol"] = vuln["protocol"]
new_ticket["source"] = vuln["source"]
new_ticket["source_id"] = vuln["plugin_id"]
new_ticket["owner"] = vuln["owner"]
new_ticket["time_opened"] = vuln["time"]
self.__generate_ticket_details(vuln, new_ticket, check_for_changes=False)
host = self.__db.HostDoc.get_by_ip(vuln["ip"])
if host is not None:
new_ticket["loc"] = host["loc"]
event = {
"time": vuln["time"],
"action": TICKET_EVENT.OPENED,
"reason": reason,
"reference": vuln["_id"],
}
if self.__manual_scan:
event["manual"] = True
new_ticket["events"].append(event)
if (
new_ticket["owner"] == UNKNOWN_OWNER
): # close tickets with no owner immediately
event = {
"time": vuln["time"],
"action": TICKET_EVENT.CLOSED,
"reason": "No associated owner",
"reference": None,
}
if self.__manual_scan:
event["manual"] = True
new_ticket["events"].append(event)
new_ticket["open"] = False
new_ticket["time_closed"] = self.__closing_time
new_ticket.save()
self.__mark_seen(new_ticket)
# Create notifications for Highs (3) or Criticals (4)
if new_ticket["details"]["severity"] > 2:
self.__create_notification(new_ticket)
def get_bod_dataframe(bod_start_date):
now = util.utcnow()
tomorrow = now + datetime.timedelta(days=1)
days_of_the_bod = pd.to_datetime(pd.date_range(bod_start_date, now), utc=True)
bod_owners = current_app.db.RequestDoc.get_all_descendants("EXECUTIVE")
backlog_tix = current_app.db.TicketDoc.find(
{
"source": "nessus",
"details.severity": 4,
"owner": {"$in": bod_owners},
"time_opened": {"$lte": bod_start_date},
"false_positive": False,
"$or": [{"time_closed": {"$gte": bod_start_date}}, {"time_closed": None}],
},
{"_id": False, "time_closed": True},
)
results_df = DataFrame(
index=days_of_the_bod, columns=["young", "mid", "old", "total"]
)
backlog_tix = list(backlog_tix)
df = DataFrame(backlog_tix)
if not df.empty:
df["tally"] = 1
df.time_closed = df.time_closed.fillna(
tomorrow
) # assume they'll close tomorrow
df.time_closed = df.time_closed.apply(
pd.to_datetime
) # if fillna is noop we have wrong types still
df_backlog = df.set_index("time_closed")
df_backlog = df_backlog.resample("1D").sum()
df_backlog = df_backlog.reindex(days_of_the_bod) # does not include tomorrow
df_backlog.tally = df_backlog.tally.fillna(0).astype(np.int)
df_backlog["csum"] = df_backlog.tally.cumsum()
df_backlog["remaining"] = len(backlog_tix) - df_backlog.csum
else:
df_backlog = DataFrame()
# Calculate Buckets
tix = current_app.db.TicketDoc.find(
{
"source": "nessus",
"details.severity": 4,
"false_positive": False,
"owner": {"$in": bod_owners},
"$or": [{"time_closed": {"$gte": bod_start_date}}, {"time_closed": None}],
},
{"_id": False, "time_opened": True, "time_closed": True},
)
df = DataFrame(list(tix))
if not df.empty:
# Convert times to pandas Timestamp
df.time_opened = pd.to_datetime(df.time_opened, utc=True)
# For tickets that haven't closed yet, also set time_closed to tomorrow
# df.time_closed = pd.to_datetime(df.time_closed.fillna(tomorrow), utc=True).dt.tz_localize('UTC')
df.time_closed = pd.to_datetime(
df.time_closed.fillna(tomorrow), utc=True
).dt.tz_convert("UTC")
bsd_utc = pd.to_datetime(
bod_start_date
) # Store bod_start_date as pandas Timestamp
df["bod_time_opened"] = df.time_opened.apply(
lambda x, bsd_utc=bsd_utc: max(x, bsd_utc)
)
df["bod_age"] = df.time_closed - df.bod_time_opened
mid_delta = np.timedelta64(BOD_CUTOFF_1, "D")
old_delta = np.timedelta64(BOD_CUTOFF_2, "D")
for start_of_day, values in results_df.iterrows():
end_of_day = start_of_day + np.timedelta64(1, "D") - np.timedelta64(1, "ns")
open_on_day_mask = (df.time_opened <= end_of_day) & (
df.time_closed > start_of_day
)
bod_age_on_date = start_of_day - df.bod_time_opened
bod_age_on_date_masked = bod_age_on_date.mask(open_on_day_mask == False)
values["total"] = open_on_day_mask.value_counts().get(True, 0)
values["young"] = (
(bod_age_on_date_masked < mid_delta).value_counts().get(True, 0)
)
values["mid"] = (
(
(bod_age_on_date_masked >= mid_delta)
& (bod_age_on_date_masked < old_delta)
)
.value_counts()
.get(True, 0)
)
values["old"] = (
(bod_age_on_date_masked >= old_delta).value_counts().get(True, 0)
)
if not df_backlog.empty:
# combine previous calculations
results_df["backlog"] = df_backlog.remaining
return results_df
def open_ticket(self, portscan, reason):
if self.__closing_time is None or self.__closing_time < portscan["time"]:
self.__closing_time = portscan["time"]
# search for previous open ticket that matches
prev_open_ticket = self.__db.TicketDoc.find_one(
{
"ip_int": portscan["ip_int"],
"port": portscan["port"],
"protocol": portscan["protocol"],
"source": portscan["source"],
"source_id": portscan["source_id"],
"open": True,
}
)
if prev_open_ticket:
self.__check_false_positive_expiration(
prev_open_ticket, portscan["time"].replace(tzinfo=tz.tzutc())
) # explicitly set to UTC (see CYHY-286)
# add an entry to the existing open ticket
event = {
"time": portscan["time"],
"action": TICKET_EVENT.VERIFIED,
"reason": reason,
"reference": portscan["_id"],
}
prev_open_ticket["events"].append(event)
prev_open_ticket.save()
return
# no matching tickets are currently open
# search for a previously closed ticket that was closed before the cutoff
cutoff_date = util.utcnow() + self.__reopen_delta
reopen_ticket = self.__db.TicketDoc.find_one(
{
"ip_int": portscan["ip_int"],
"port": portscan["port"],
"protocol": portscan["protocol"],
"source": portscan["source"],
"source_id": portscan["source_id"],
"open": False,
"time_closed": {"$gt": cutoff_date},
}
)
if reopen_ticket:
event = {
"time": portscan["time"],
"action": TICKET_EVENT.REOPENED,
"reason": reason,
"reference": portscan["_id"],
}
reopen_ticket["events"].append(event)
reopen_ticket["time_closed"] = None
reopen_ticket["open"] = True
reopen_ticket.save()
return
# time to open a new ticket
new_ticket = self.__db.TicketDoc()
new_ticket.ip = portscan["ip"]
new_ticket["port"] = portscan["port"]
new_ticket["protocol"] = portscan["protocol"]
new_ticket["source"] = portscan["source"]
new_ticket["source_id"] = portscan["source_id"]
new_ticket["owner"] = portscan["owner"]
new_ticket["time_opened"] = portscan["time"]
new_ticket["details"] = {
"cve": None,
"score_source": None,
"cvss_base_score": None,
"severity": 0,
"name": portscan["name"],
"service": portscan["service"],
}
host = self.__db.HostDoc.get_by_ip(portscan["ip"])
if host is not None:
new_ticket["loc"] = host["loc"]
event = {
"time": portscan["time"],
"action": TICKET_EVENT.OPENED,
"reason": reason,
"reference": portscan["_id"],
}
new_ticket["events"].append(event)
if (
new_ticket["owner"] == UNKNOWN_OWNER
): # close tickets with no owner immediately
event = {
"time": portscan["time"],
"action": TICKET_EVENT.CLOSED,
"reason": "No associated owner",
"reference": None,
}
new_ticket["events"].append(event)
new_ticket["open"] = False
new_ticket["time_closed"] = self.__closing_time
new_ticket.save()
# Create a notification for this ticket
self.__create_notification(new_ticket)
def congressional_data(db, start_date, end_date):
if start_date == None:
return "Must input a start date. Received %s & %s" % (start_date,
end_date)
if end_date == None:
end_date = util.utcnow()
# Needed when run as a standalone script:
# start_date = parser.parse(start_date)
# end_date = parser.parse(end_date)
all_stakeholders = db.RequestDoc.get_owner_types(as_lists=True,
stakeholders_only=True)
active_fed_stakeholders = db.ReportDoc.find({
"owner": {
"$in": all_stakeholders[AGENCY_TYPE.FEDERAL]
},
"generated_time": {
"$gte": start_date,
"$lt": end_date
},
}).distinct("owner")
active_fed_stakeholders.sort()
active_fed_owners = list()
for org in active_fed_stakeholders:
active_fed_owners.append(org)
active_fed_owners += db.RequestDoc.get_all_descendants(org)
fed_executive_stakeholders = db.RequestDoc.get_by_owner(
"EXECUTIVE")["children"]
active_fed_executive_stakeholders = db.ReportDoc.find({
"owner": {
"$in": fed_executive_stakeholders
},
"generated_time": {
"$gte": start_date,
"$lt": end_date
},
}).distinct("owner")
active_fed_executive_stakeholders.sort()
active_fed_executive_owners = list()
for org in active_fed_executive_stakeholders:
active_fed_executive_owners.append(org)
active_fed_executive_owners += db.RequestDoc.get_all_descendants(org)
fed_cfo_stakeholders = db.RequestDoc.get_by_owner(
"FED_CFO_ACT")["children"]
active_fed_cfo_stakeholders = db.ReportDoc.find({
"owner": {
"$in": fed_cfo_stakeholders
},
"generated_time": {
"$gte": start_date,
"$lt": end_date
},
}).distinct("owner")
active_fed_cfo_stakeholders.sort()
active_fed_cfo_owners = list()
for org in active_fed_cfo_stakeholders:
active_fed_cfo_owners.append(org)
active_fed_cfo_owners += db.RequestDoc.get_all_descendants(org)
fed_exec_non_cfo_stakeholders = list(
set(fed_executive_stakeholders) - set(fed_cfo_stakeholders))
active_fed_exec_non_cfo_stakeholders = db.ReportDoc.find({
"owner": {
"$in": fed_exec_non_cfo_stakeholders
},
"generated_time": {
"$gte": start_date,
"$lt": end_date
},
}).distinct("owner")
active_fed_exec_non_cfo_stakeholders.sort()
active_fed_exec_non_cfo_owners = list()
for org in active_fed_exec_non_cfo_stakeholders:
active_fed_exec_non_cfo_owners.append(org)
active_fed_exec_non_cfo_owners += db.RequestDoc.get_all_descendants(
org)
SLTT_stakeholders = (all_stakeholders[AGENCY_TYPE.STATE] +
all_stakeholders[AGENCY_TYPE.LOCAL] +
all_stakeholders[AGENCY_TYPE.TRIBAL] +
all_stakeholders[AGENCY_TYPE.TERRITORIAL])
active_SLTT_stakeholders = db.ReportDoc.find({
"owner": {
"$in": SLTT_stakeholders
},
"generated_time": {
"$gte": start_date,
"$lt": end_date
},
}).distinct("owner")
# active_SLTT_stakeholders.sort()
# active_SLTT_owners = list()
# for org in active_SLTT_stakeholders:
# active_SLTT_owners.append(org)
# active_SLTT_owners += db.RequestDoc.get_all_descendants(org)
active_private_stakeholders = db.ReportDoc.find({
"owner": {
"$in": all_stakeholders[AGENCY_TYPE.PRIVATE]
},
"generated_time": {
"$gte": start_date,
"$lt": end_date
},
}).distinct("owner")
# active_private_stakeholders.sort()
# active_private_owners = list()
# for org in active_private_stakeholders:
# active_private_owners.append(org)
# active_private_owners += db.RequestDoc.get_all_descendants(org)
print "Congressional Cyber Hygiene Metrics Report"
mylist = {}
print "Date Range: {} - {}".format(
start_date.strftime("%Y-%m-%d %H:%M UTC"),
end_date.strftime("%Y-%m-%d %H:%M UTC"),
)
print "\n--------------------------------------------------"
print "Critical Vulnerability Remediation Performance:\n"
# Use "_owners" lists here because we DO want to include descendent orgs in these metrics
for (stakeholder_list, group_name) in (
(active_fed_owners, "FEDERAL"),
(active_fed_executive_owners, "FEDERAL EXECUTIVE"),
(active_fed_cfo_owners, "FEDERAL EXECUTIVE - CFO ACT"),
(active_fed_exec_non_cfo_owners, "FEDERAL EXECUTIVE - NON-CFO ACT"),
):
print "{}:".format(group_name)
(pipeline, collection) = closed_in_date_range_closed_ticket_age_pl(
stakeholder_list, start_date, end_date)
output = db[collection].aggregate(pipeline, cursor={})
df = DataFrame(list(output))
median_days_to_mitigate_criticals = round(
df.loc[df["severity"] == 4]["duration_to_close"].median() /
(24 * 60 * 60 * 1000.0))
if isnull(median_days_to_mitigate_criticals):
print " Median time to mitigate Critical vulnerabilities: No Critical vulnerabilities mitigated"
myarg = {
group_name.replace(" ", "_").replace("-", "_") + "_median_time_to_mitigate":
"No Critical Vulnerabilities Mitigated"
}
else:
print " Median time to mitigate Critical vulnerabilities: {0:,g} days".format(
median_days_to_mitigate_criticals)
myarg = {
group_name.replace(" ", "_").replace("-", "_") + "_median_time_to_mitigate":
"{0:,g}".format(median_days_to_mitigate_criticals)
}
mylist.update(myarg)
(pipeline, collection) = opened_in_date_range_open_ticket_age_pl(
stakeholder_list, start_date, end_date)
output = db[collection].aggregate(pipeline, cursor={})
df = DataFrame(list(output))
median_days_active_criticals = round(
df.loc[df["severity"] == 4]["open_ticket_duration"].median() /
(24 * 60 * 60 * 1000.0))
if isnull(median_days_active_criticals):
print " Median time Critical vulnerabilities currently active: No Critical vulnerabilities currently active"
myarg = {
group_name.replace(" ", "_").replace("-", "_") + "_currently_active":
"No Critical Vulnerabilities Currently Active"
}
else:
print " Median time Critical vulnerabilities currently active: {0:,g} days".format(
median_days_active_criticals)
myarg = {
group_name.replace(" ", "_").replace("-", "_") + "_currently_active":
"{0:,g}".format(median_days_active_criticals)
}
mylist.update(myarg)
print
print "--------------------------------------------------"
print "Number of Active CyHy Stakeholders by Segment:\n"
# Use "_stakeholders" lists here because we DO NOT want to include descendent orgs in these metrics
for (stakeholder_list, group_name) in (
(active_fed_stakeholders, "FEDERAL"),
(active_fed_executive_stakeholders, "FEDERAL EXECUTIVE"),
(active_fed_cfo_stakeholders, "FEDERAL EXECUTIVE - CFO ACT"),
(active_fed_exec_non_cfo_stakeholders,
"FEDERAL EXECUTIVE - NON-CFO ACT"),
(active_SLTT_stakeholders, "SLTT"),
(active_private_stakeholders, "PRIVATE"),
):
print "{}: {} stakeholders".format(group_name, len(stakeholder_list))
myarg = {
group_name.replace(" ", "_").replace("-", "_") + "_active_stakeholders":
"{0:,g}".format(len(stakeholder_list))
}
mylist.update(myarg)
print "\n--------------------------------------------------"
print "Number of New Vulnerabilities Detected:\n"
# Use "_owners" lists here because we DO want to include descendent orgs in these metrics
for (stakeholder_list, group_name) in (
(active_fed_owners, "FEDERAL"),
(active_fed_executive_owners, "FEDERAL EXECUTIVE"),
(active_fed_cfo_owners, "FEDERAL EXECUTIVE - CFO ACT"),
(active_fed_exec_non_cfo_owners, "FEDERAL EXECUTIVE - NON-CFO ACT"),
):
(pipeline,
collection) = tickets_opened_count_pl(stakeholder_list, start_date,
end_date)
output = list(db[collection].aggregate(pipeline, cursor={}))
print "{}:".format(group_name)
for i in ("critical", "high", "medium", "low"):
try:
print " {}: {:,}".format(i.title(), output[0][i])
myarg = {
group_name.replace(" ", "_").replace("-", "_") + "_{}_New_Vulns_Detected".format(i.title(
)):
"{:,}".format(output[0][i])
}
except IndexError:
print " {}: 0".format(i.title())
myarg = {
group_name.replace(" ", "_").replace("-", "_") + "_{}_New_Vulns_Detected".format(i.title(
)):
"0"
}
mylist.update(myarg)
print
print "--------------------------------------------------"
print "Number of Vulnerabilities Mitigated:\n"
# Use "_owners" lists here because we DO want to include descendent orgs in these metrics
for (stakeholder_list, group_name) in (
(active_fed_owners, "FEDERAL"),
(active_fed_executive_owners, "FEDERAL EXECUTIVE"),
(active_fed_cfo_owners, "FEDERAL EXECUTIVE - CFO ACT"),
(active_fed_exec_non_cfo_owners, "FEDERAL EXECUTIVE - NON-CFO ACT"),
):
(pipeline,
collection) = closed_ticket_count_pl(stakeholder_list, start_date,
end_date)
output = list(db[collection].aggregate(pipeline, cursor={}))
print "{}:".format(group_name)
for i in ("critical", "high", "medium", "low"):
try:
print " {}: {:,}".format(i.title(), output[0][i])
myarg = {
group_name.replace(" ", "_").replace("-", "_") + "_{}_New_Vulns_Mitigated".format(i.title(
)):
"{:,}".format(output[0][i])
}
except IndexError:
print " {}: 0".format(i.title())
myarg = {
group_name.replace(" ", "_").replace("-", "_") + "_{}_New_Vulns_Mitigated".format(i.title(
)):
"0"
}
mylist.update(myarg)
print
print "=================================================="
print "Active CyHy Federal Stakeholders:\n"
# Use "_stakeholders" lists here because we DO NOT want to include descendent orgs in these metrics
for (stakeholder_list, group_name) in (
(active_fed_stakeholders, "FEDERAL"),
(active_fed_executive_stakeholders, "FEDERAL EXECUTIVE"),
(active_fed_cfo_stakeholders, "FEDERAL EXECUTIVE - CFO ACT"),
(active_fed_exec_non_cfo_stakeholders,
"FEDERAL EXECUTIVE - NON-CFO ACT"),
):
myarg = {
group_name.replace(" ", "_").replace("-", "_") + "_Active_CyHy_Stakeholders":
stakeholder_list
}
mylist.update(myarg)
print "{} ({} agencies): ".format(group_name, len(stakeholder_list)),
for org in stakeholder_list:
if org != stakeholder_list[-1]:
print "{},".format(org),
else:
print "{}\n".format(org)
# import IPython; IPython.embed() #<<< BREAKPOINT >>>
# sys.exit(0)
print mylist
return mylist
def create_snapshot(
self,
owner,
snapshot_oid,
parent_oid=None,
descendants_included=[],
exclude_from_world_stats=False,
progress_callback=None,
):
"""creates a new snapshot document with the oid returned from one of the tagging methods.
Returns the snapshot if the snapshot is created successfully.
Returns None if the snapshot is not unique. In this case reports should be untagged."""
snapshot_doc = self.__db.SnapshotDoc()
snapshot_doc["_id"] = snapshot_oid
snapshot_doc["latest"] = True
snapshot_doc["owner"] = owner
snapshot_doc["descendants_included"] = descendants_included
if parent_oid:
snapshot_doc["parents"] = [parent_oid]
else:
snapshot_doc["parents"] = [
snapshot_oid
] # If you don't have a parent snapshot, you are your own parent; this prevents deletion of
# this snap if it ever becomes a child of another snapshot that later gets deleted
snapshot_doc["networks"] = self.__db.RequestDoc.get_by_owner(
owner).networks.iter_cidrs()
for descendant in descendants_included:
snapshot_doc["networks"] += self.__db.RequestDoc.get_by_owner(
descendant).networks.iter_cidrs()
if exclude_from_world_stats:
snapshot_doc["exclude_from_world_stats"] = True
current_time = util.utcnow()
snapshot_doc["last_change"] = current_time
start_time, end_time = self.__get_tag_timespan(
snapshot_oid
) # Try to get start/end time from host_scan/port_scan/vuln_scan docs
if (
start_time == None
): # If org has no latest=true host_scans, port_scans or vuln_scans, start_time will be None
start_time, end_time = self.__get_host_timespan(
[owner] + descendants_included
) # Try to get start/end time from host docs (not ideal, but better than nothing)
if (
start_time == None
): # If org(s) have no host docs (or hosts that have not been netscanned yet), start_time will be None
start_time = (
end_time
) = current_time # All else has failed- just set start/end time to current time
snapshot_doc["start_time"] = start_time
snapshot_doc["end_time"] = end_time
if progress_callback:
progress_callback()
if snapshot_doc.will_conflict():
snapshot_doc[
"end_time"] = current_time # Avoid conflicts by setting end_time to current time
# Not ideal, but will only happen in rare cases and should have minimal impact
pipeline_collection = queries.addresses_scanned_pl(
[owner] + descendants_included)
results = database.run_pipeline_cursor(pipeline_collection, self.__db)
if progress_callback:
progress_callback()
database.combine_results(snapshot_doc, results)
pipeline_collection = queries.cvss_sum_pl(snapshot_oid)
results = database.run_pipeline_cursor(pipeline_collection, self.__db)
if progress_callback:
progress_callback()
if results:
cvss_sum = float(results[0].get("cvss_sum", 0.0))
else:
cvss_sum = 0.0
pipeline_collection = queries.host_count_pl([owner] +
descendants_included)
results = database.run_pipeline_cursor(pipeline_collection, self.__db)
if progress_callback:
progress_callback()
database.combine_results(snapshot_doc, results)
pipeline_collection = queries.vulnerable_host_count_pl(snapshot_oid)
results = database.run_pipeline_cursor(pipeline_collection, self.__db)
if progress_callback:
progress_callback()
database.combine_results(snapshot_doc, results)
snapshot_doc["cvss_average_all"] = util.safe_divide(
cvss_sum, snapshot_doc["host_count"])
snapshot_doc["cvss_average_vulnerable"] = util.safe_divide(
cvss_sum, snapshot_doc["vulnerable_host_count"])
pipeline_collection = queries.unique_operating_system_count_pl(
snapshot_oid)
results = database.run_pipeline_cursor(pipeline_collection, self.__db)
if progress_callback:
progress_callback()
database.combine_results(snapshot_doc, results)
pipeline_collection = queries.port_count_pl(snapshot_oid)
results = database.run_pipeline_cursor(pipeline_collection, self.__db)
if progress_callback:
progress_callback()
database.combine_results(snapshot_doc, results)
pipeline_collection = queries.unique_port_count_pl(snapshot_oid)
results = database.run_pipeline_cursor(pipeline_collection, self.__db)
if progress_callback:
progress_callback()
database.combine_results(snapshot_doc, results)
pipeline_collection = queries.silent_port_count_pl(
[owner] + descendants_included)
results = database.run_pipeline_cursor(pipeline_collection, self.__db)
if progress_callback:
progress_callback()
database.combine_results(snapshot_doc, results)
pipeline_collection = queries.severity_count_pl(snapshot_oid)
results = database.run_pipeline_cursor(pipeline_collection, self.__db)
if progress_callback:
progress_callback()
database.combine_results(snapshot_doc, results, "vulnerabilities")
pipeline_collection = queries.unique_severity_count_pl(snapshot_oid)
results = database.run_pipeline_cursor(pipeline_collection, self.__db)
if progress_callback:
progress_callback()
database.combine_results(snapshot_doc, results,
"unique_vulnerabilities")
pipeline_collection = queries.false_positives_pl(snapshot_oid)
results = database.run_pipeline_cursor(pipeline_collection, self.__db)
if progress_callback:
progress_callback()
database.combine_results(snapshot_doc, results, "false_positives")
pipeline_collection = queries.service_counts_simple_pl(snapshot_oid)
results = database.run_pipeline_cursor(pipeline_collection, self.__db)
if progress_callback:
progress_callback()
services = self.__process_services(results)
snapshot_doc["services"] = services
pipeline_collection = queries.open_ticket_age_in_snapshot_pl(
current_time, snapshot_oid)
results = database.run_pipeline_cursor(pipeline_collection, self.__db)
if progress_callback:
progress_callback()
snapshot_doc["tix_msec_open"] = self.__process_open_ticket_age(
results, current_time)
tix_closed_since_date = current_time - datetime.timedelta(
self.SNAPSHOT_CLOSED_TICKET_HISTORY_DAYS)
pipeline_collection = queries.closed_ticket_age_for_orgs_pl(
tix_closed_since_date, [owner] + descendants_included)
results = database.run_pipeline_cursor(pipeline_collection, self.__db)
if progress_callback:
progress_callback()
snapshot_doc["tix_msec_to_close"] = self.__process_closed_ticket_age(
results, tix_closed_since_date)
# reset previous latest flag
self.__db.SnapshotDoc.reset_latest_flag_by_owner(owner)
snapshot_doc.save()
# now calculate the world statistics and update the snapshot
# Since parent snapshots include data for their descendants, we don't want to count descendant snapshots when calculating world stats
# NOTE: This won't work if a snapshot is created for a descendant org on it's own after the parent org snapshot was created, but
# it's good enough for now. The world statistics process should get updated as part of CYHY-145.
snaps_to_exclude_from_world_stats = list()
all_latest_snapshots = list(
self.__db.SnapshotDoc.collection.find(
{"latest": True},
{
"_id": 1,
"parents": 1,
"exclude_from_world_stats": 1
},
))
for snap in all_latest_snapshots:
if (snap["_id"] not in snap["parents"]
) or snap.get("exclude_from_world_stats"):
# NOTE: A descendant snapshot has a different parent id than itself
snaps_to_exclude_from_world_stats.append(snap["_id"])
pipeline_collection = queries.world_pl(
snaps_to_exclude_from_world_stats)
results = database.run_pipeline_cursor(pipeline_collection, self.__db)
if progress_callback:
progress_callback()
database.combine_results(snapshot_doc, results, "world")
snapshot_doc.save()
return snapshot_doc
def change_ownership(self, orig_owner, new_owner, networks, reason):
# Change owner on all relevant documents for a given list of networks.
# Special case for tickets collection; add a CHANGED event to the events list of each ticket
change_event = {
"time": util.utcnow(),
"action": TICKET_EVENT.CHANGED,
"reason": reason,
"reference": None,
"delta": [{
"from": orig_owner,
"to": new_owner,
"key": "owner"
}],
}
for net in networks.iter_cidrs():
print "Changing owner of network %s to %s" % (net, new_owner)
for collection, ip_key, update_cmd in (
(self.__db.hosts, "_id", {
"$set": {
"owner": new_owner
}
}),
(self.__db.host_scans, "ip_int", {
"$set": {
"owner": new_owner
}
}),
(self.__db.port_scans, "ip_int", {
"$set": {
"owner": new_owner
}
}),
(self.__db.vuln_scans, "ip_int", {
"$set": {
"owner": new_owner
}
}),
(
self.__db.tickets,
"ip_int",
{
"$set": {
"owner": new_owner
},
"$push": {
"events": change_event
}
},
),
):
result = collection.update(
{ip_key: {
"$gte": net.first,
"$lte": net.last
}},
update_cmd,
upsert=False,
multi=True,
safe=True,
)
result["collection"] = collection.name
print " {nModified} {collection} documents modified".format(
**result)
PS_1 = {
"ip": IPS[0],
"ip_int": long(IPS[0]),
"port": PORTSCAN_PORTS[1],
"state": "open",
"protocol": "tcp",
"service": "telnet",
"source": SOURCE_NMAP,
"source_id": SOURCE_IDS[0],
"owner": OWNER,
"severity": 0,
"cvss_base_score": None,
"name": "Potentially Risky Service Detected: telnet",
"_id": ObjectId(),
"time": util.utcnow(),
"latest": True,
}
PS_2 = {
"ip": IPS[1],
"ip_int": long(IPS[1]),
"port": PORTSCAN_PORTS[2],
"state": "open",
"protocol": "tcp",
"service": "ldap",
"source": SOURCE_NMAP,
"source_id": SOURCE_IDS[0],
"owner": OWNER,
"severity": 0,
"cvss_base_score": None,
"name": "Potentially Risky Service Detected: ldap",
import subprocess
import sys
import threading
import time
from bson import ObjectId
from collections import defaultdict
from docopt import docopt
from cyhy.core import SCAN_TYPE
from cyhy.core.common import REPORT_TYPE, REPORT_PERIOD
from cyhy.db import database, CHDatabase
from cyhy.util import util
from ncats_webd import cybex_queries
current_time = util.utcnow()
LOGGING_LEVEL = logging.INFO
LOG_FILE = "snapshots_reports_scorecard_automation.log"
REPORT_THREADS = 18
SNAPSHOT_THREADS = 32
NCATS_DHUB_URL = "dhub.ncats.cyber.dhs.gov:5001"
NCATS_WEB_URL = "web.data.ncats.cyber.dhs.gov"
WEEKLY_REPORT_BASE_DIR = "/var/cyhy/reports/output"
SCORECARD_OUTPUT_DIR = "scorecards"
SCORECARD_JSON_OUTPUT_DIR = "JSONfiles"
CYBEX_CSV_DIR = "cybex_csvs"
CYHY_REPORT_DIR = os.path.join(
"report_archive", "reports{}".format(current_time.strftime("%Y%m%d")))
def get_open_tickets_dataframe(db, ticket_severity):
now = util.utcnow()
first_report_time_cache = (
dict()) # Cache generated_time of first report for each ticket
fed_executive_owners = db.RequestDoc.get_all_descendants("EXECUTIVE")
tix = db.TicketDoc.find(
{
"source": "nessus",
"details.severity": ticket_severity,
"false_positive": False,
"owner": {
"$in": fed_executive_owners
},
"open": True,
},
{
"_id": False,
"owner": True,
"time_opened": True,
"ip": True,
"port": True,
"details.name": True,
"details.cve": True,
"snapshots": True,
},
)
tix = list(tix)
for x in tix:
x.update(x["details"])
del x["details"]
x["days_since_first_detected"] = (
now - x["time_opened"]).total_seconds() / (60 * 60 * 24)
x["first_reported"] = None
x["days_since_first_reported"] = None
x["days_to_report"] = None
for snap_id in x.get("snapshots", []):
x["first_reported"] = first_report_time_cache.get(snap_id)
if x["first_reported"]:
# Found this snapshot's report time in the cache
break
else:
# Not found in the cache, so make a database call
first_report = db.reports.find_one({"snapshot_oid": snap_id})
if first_report:
x["first_reported"] = first_report.get("generated_time")
first_report_time_cache[snap_id] = x["first_reported"]
break
if x["first_reported"]:
x["days_since_first_reported"] = (
now - x["first_reported"]).total_seconds() / (60 * 60 * 24)
x["days_to_report"] = (x["days_since_first_detected"] -
x["days_since_first_reported"])
if x.get("snapshots"):
del x["snapshots"]
df = DataFrame(tix)
if not df.empty:
df.sort_values(by="days_since_first_detected",
ascending=False,
inplace=True)
return df
