def _get_timespan(self, timespan=None, **kwargs):
if timespan:
if isinstance(timespan, TimeSpan):
self.timespan = timespan
else:
self.timespan = TimeSpan(timespan=timespan)
elif "start" in kwargs and "end" in kwargs:
self.timespan = TimeSpan(start=kwargs["start"], end=kwargs["end"])
def test_timespan_invalid_params():
"""Test error handling for invalid params."""
period = timedelta(days=1)
with pytest.raises(ValueError):
TimeSpan()
with pytest.raises(ValueError):
TimeSpan(start="foo", period=period)
with pytest.raises(ValueError):
TimeSpan(start=None, end=None)
with pytest.raises(ValueError):
TimeSpan(period="some length")
with pytest.raises(ValueError):
TimeSpan(period=1)
def test_timespan_eq():
"""Test creating Timespan from another Timespan."""
period = timedelta(days=1)
tspan = TimeSpan(period=period)
# Timespan object as a parameter
tspan2 = TimeSpan(timespan=tspan)
check.equal(tspan2, tspan)
check.equal(hash(tspan2), hash(tspan))
tspan2 = TimeSpan(timespan=(tspan.start, tspan.end))
check.equal(tspan2, tspan)
tspan2 = TimeSpan(timespan=(str(tspan.start), str(tspan.end)))
check.equal(tspan2, tspan)
def test_timespan_parms():
"""Test standard parameters."""
end = datetime.utcnow()
period = timedelta(days=1)
start = end - period
tspan = TimeSpan(start=start, end=end)
_validate_timespan(tspan, start, end)
tspan = TimeSpan(end=end, period=period)
_validate_timespan(tspan, start, end)
tspan = TimeSpan(end=end, period="1D")
_validate_timespan(tspan, start, end)
tspan = TimeSpan(end=str(end), period="1D")
_validate_timespan(tspan, start, end)
tspan = TimeSpan(start=str(start), end=str(end))
_validate_timespan(tspan, start, end)
tspan = TimeSpan(start=str(start), period="1D")
_validate_timespan(tspan, start, end)
# end is set to utcnow()
tspan = TimeSpan(start=start)
_validate_timespan(tspan, start)
# end is set to utcnow()
tspan = TimeSpan(period=period)
_validate_timespan(tspan, period=period)
def test_local_data(monkeypatch):
"""Test nblt output types and values using LocalData provider."""
test_data = str(Path.cwd().joinpath(TEST_DATA_PATH))
monkeypatch.setattr(data_providers, "GeoLiteLookup", GeoIPLiteMock)
data_providers.init(
query_provider="LocalData",
LocalData_data_paths=[test_data],
LocalData_query_paths=[test_data],
providers=["tilookup", "geolitelookup"],
)
test_nblt = nblts.azsent.host.HostLogonsSummary()
tspan = TimeSpan(start=datetime(2020, 6, 23, 4, 20),
end=datetime(2020, 6, 29, 21, 32))
nbltlocaldata = test_nblt.run(value="WinAttackSim", timespan=tspan)
assert isinstance(nbltlocaldata.logon_sessions, pd.DataFrame)
assert nbltlocaldata.logon_sessions["SubjectUserName"].iloc[
0] == "WinAttackSim$"
assert nbltlocaldata.logon_sessions["LogonProcessName"].iloc[
3] == "Advapi "
assert "User Pie Chart" in nbltlocaldata.plots.keys()
assert isinstance(nbltlocaldata.plots["Process Bar Chart"], Figure)
assert isinstance(nbltlocaldata.logon_matrix, pd.io.formats.style.Styler)
assert nbltlocaldata.logon_matrix.index[0][0] == "Font Driver Host\\UMFD-0"
assert isinstance(nbltlocaldata.logon_map, FoliumMap)
assert isinstance(nbltlocaldata.timeline, Column)
def test_network_flow_summary_notebooklet(monkeypatch):
"""Test basic run of notebooklet."""
test_data = str(Path(TEST_DATA_PATH).absolute())
monkeypatch.setattr(data_providers, "GeoLiteLookup", GeoIPLiteMock)
monkeypatch.setattr(data_providers, "TILookup", TILookupMock)
data_providers.init(
query_provider="LocalData",
LocalData_data_paths=[test_data],
LocalData_query_paths=[test_data],
)
test_nb = nblts.azsent.network.NetworkFlowSummary()
tspan = TimeSpan(period="1D")
test_nb.query_provider.schema.update({tab: {} for tab in DEF_PROV_TABLES})
options = ["+geo_map"]
result = test_nb.run(value="myhost", timespan=tspan, options=options)
check.is_not_none(result.host_entity)
check.is_not_none(result.network_flows)
check.is_instance(result.network_flows, pd.DataFrame)
check.is_not_none(result.plot_flows_by_protocol)
check.is_instance(result.plot_flows_by_protocol, LayoutDOM)
check.is_not_none(result.plot_flows_by_direction)
check.is_instance(result.plot_flows_by_direction, LayoutDOM)
check.is_not_none(result.plot_flow_values)
check.is_instance(result.plot_flow_values, LayoutDOM)
check.is_not_none(result.flow_index)
check.is_instance(result.flow_summary, pd.DataFrame)
result.select_asns()
result.lookup_ti_for_asn_ips()
result.show_selected_asn_map()
def test_template_notebooklet(monkeypatch):
"""Test basic run of notebooklet."""
test_data = str(Path(TEST_DATA_PATH).absolute())
monkeypatch.setattr(data_providers, "GeoLiteLookup", GeoIPLiteMock)
data_providers.init(
query_provider="LocalData",
LocalData_data_paths=[test_data],
LocalData_query_paths=[test_data],
)
test_nb = TemplateNB()
tspan = TimeSpan(period="1D")
result = test_nb.run(value="myhost", timespan=tspan)
check.is_not_none(result.all_events)
check.is_not_none(result.description)
check.is_not_none(result.plot)
result = test_nb.run(value="myhost",
timespan=tspan,
options=["+get_metadata"])
check.is_not_none(result.additional_info)
evts = test_nb.run_additional_operation(
["4679", "5058", "5061", "5059", "4776"])
check.is_instance(evts, pd.DataFrame)
def test_ip_summary_notebooklet(monkeypatch):
"""Test basic run of notebooklet."""
test_data = str(Path(TEST_DATA_PATH).absolute())
monkeypatch.setattr(data_providers, "GeoLiteLookup", GeoIPLiteMock)
monkeypatch.setattr(data_providers, "TILookup", TILookupMock)
data_providers.init(
query_provider="LocalData",
LocalData_data_paths=[test_data],
LocalData_query_paths=[test_data],
providers=["tilookup", "geolitelookup"],
)
test_nb = nblts.azsent.network.IpAddressSummary()
tspan = TimeSpan(period="1D")
result = test_nb.run(value="11.1.2.3", timespan=tspan)
check.is_not_none(result.ip_entity)
check.equal(result.ip_type, "Public")
check.equal(result.ip_origin, "External")
check.is_in("CountryCode", result.geoip)
check.is_not_none(result.location)
check.is_not_none(result.notebooklet)
check.is_not_none(result.whois)
check.is_instance(result.related_alerts, pd.DataFrame)
check.is_not_none(test_nb.browse_alerts())
check.is_instance(result.passive_dns, pd.DataFrame)
check.is_instance(result.ti_results, pd.DataFrame)
def test_winhostevents_notebooklet(monkeypatch):
"""Test basic run of notebooklet."""
test_data = str(Path(TEST_DATA_PATH).absolute())
monkeypatch.setattr(data_providers, "GeoLiteLookup", GeoIPLiteMock)
data_providers.init(
query_provider="LocalData",
LocalData_data_paths=[test_data],
LocalData_query_paths=[test_data],
)
test_nb = nblts.azsent.host.WinHostEvents()
tspan = TimeSpan(period="1D")
result = test_nb.run(value="myhost", timespan=tspan)
check.is_not_none(result.all_events)
check.is_instance(result.all_events, pd.DataFrame)
check.is_not_none(result.event_pivot)
check.is_instance(result.event_pivot, pd.DataFrame)
check.is_not_none(result.account_events)
check.is_instance(result.account_events, pd.DataFrame)
check.is_not_none(result.event_pivot)
check.is_instance(result.event_pivot, pd.DataFrame)
# check.is_not_none(result.account_timeline)
exp_events = test_nb.expand_events(["5058", "5061"])
check.is_instance(exp_events, pd.DataFrame)
def test_ip_summary_notebooklet_internal(monkeypatch):
"""Test basic run of notebooklet."""
test_data = str(Path(TEST_DATA_PATH).absolute())
monkeypatch.setattr(data_providers, "GeoLiteLookup", GeoIPLiteMock)
monkeypatch.setattr(data_providers, "TILookup", TILookupMock)
data_providers.init(
query_provider="LocalData",
LocalData_data_paths=[test_data],
LocalData_query_paths=[test_data],
providers=["tilookup", "geolitelookup"],
)
test_nb = nblts.azsent.network.IpAddressSummary()
tspan = TimeSpan(period="1D")
test_nb.query_provider.schema.update({tab: {} for tab in DEF_PROV_TABLES})
result = test_nb.run(value="40.76.43.124", timespan=tspan)
check.is_not_none(result.ip_entity)
check.equal(result.ip_type, "Public")
check.equal(result.ip_origin, "Internal")
check.is_not_none(result.whois)
check.is_instance(result.related_alerts, pd.DataFrame)
check.is_instance(result.heartbeat, pd.DataFrame)
check.is_instance(result.az_network_if, pd.DataFrame)
check.is_none(result.passive_dns)
check.is_none(result.ti_results)
def test_misc_functions(_create_pivot_ns):
"""Test some additional methods of pivot.py."""
check.greater(len(_create_pivot_ns.providers), 2)
t_span = TimeSpan(end=datetime.utcnow(), period="1D")
_create_pivot_ns.edit_query_time(timespan=t_span)
check.equal(_create_pivot_ns.start, t_span.start)
check.equal(_create_pivot_ns.end, t_span.end)
check.equal(_create_pivot_ns.timespan, t_span)
def test_notebooklet_params(monkeypatch):
"""Test supplying timespan param."""
monkeypatch.setattr(data_providers, "GeoLiteLookup", GeoIPLiteMock)
data_providers.init(query_provider="LocalData",
providers=["tilookup", "geolitelookup"])
test_nb = TstNBSummary()
tspan = TimeSpan(period="1D")
test_nb.run(timespan=tspan)
check.equal(tspan, test_nb.timespan)
test_nb.run(start=tspan.start, end=tspan.end)
check.equal(tspan, test_nb.timespan)
def test_timespan_timeselector():
"""Test timespan with a time selector object."""
end = datetime.utcnow()
period = timedelta(days=1)
start = end - period
tspan = TimeSpan(period=period)
# pylint: disable=too-few-public-methods
class _TestTime:
"""Class to emulate QueryTimes widget. etc."""
start = None
end = None
period = None
test_t = _TestTime()
test_t.start = start
test_t.end = str(end)
test_t.period = "1D"
tspan = TimeSpan(timespan=test_t)
_validate_timespan(tspan, start, end)
def test_pivot_time(data_providers):
"""Function_docstring."""
providers = data_providers.values()
end = datetime.utcnow()
start = end - timedelta(1)
timespan = TimeSpan(start=start, end=end)
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UserWarning)
pivot = Pivot(providers=providers, timespan=timespan)
check.equal(pivot.start, start)
check.equal(pivot.end, end)
end = end - timedelta(1)
start = start - timedelta(1)
# Test different ways of setting the time
timespan = TimeSpan(start=start, end=end)
pivot.timespan = timespan
check.equal(pivot.start, start)
check.equal(pivot.end, end)
pivot.timespan = _TimeObj(start=timespan.start, end=timespan.end)
check.equal(pivot.start, start)
check.equal(pivot.end, end)
pivot.set_timespan(timespan)
check.equal(pivot.start, start)
check.equal(pivot.end, end)
pivot.set_timespan(start=timespan.start, end=timespan.end)
check.equal(pivot.start, start)
check.equal(pivot.end, end)
# Make sure the values provided to queries match.
_fake_provider_connected(data_providers["az_sent_prov"])
query = entities.Host.AzureSentinel.SecurityEvent_list_host_processes(
host_name="test", print=True)
check.is_in(start.isoformat(), query)
check.is_in(end.isoformat(), query)
def test_pivot_time(data_providers):
"""Function_docstring."""
providers = data_providers.values()
end = datetime.utcnow()
start = end - timedelta(1)
timespan = TimeSpan(start=start, end=end)
pivot = Pivot(providers=providers, timespan=timespan)
check.equal(pivot.start, start)
check.equal(pivot.end, end)
end = end - timedelta(1)
start = start - timedelta(1)
timespan = TimeSpan(start=start, end=end)
pivot.timespan = timespan
check.equal(pivot.start, start)
check.equal(pivot.end, end)
_fake_provider_connected(data_providers["az_sent_prov"])
query = entities.Host.AzureSentinel.list_host_processes(host_name="test",
print_query=True)
check.is_in(start.isoformat(), query)
check.is_in(end.isoformat(), query)
def test_host_summary_notebooklet(monkeypatch):
"""Test basic run of notebooklet."""
monkeypatch.setattr(data_providers, "GeoLiteLookup", GeoIPLiteMock)
test_data = str(Path(TEST_DATA_PATH).absolute())
data_providers.init(
query_provider="LocalData",
LocalData_data_paths=[test_data],
LocalData_query_paths=[test_data],
)
test_nb = nblts.azsent.host.HostSummary()
tspan = TimeSpan(period="1D")
result = test_nb.run(value="myhost", timespan=tspan)
check.is_not_none(result.host_entity)
check.is_not_none(result.related_alerts)
check.is_instance(result.related_alerts, pd.DataFrame)
check.is_not_none(result.alert_timeline)
check.is_not_none(result.related_bookmarks)
check.is_instance(result.related_bookmarks, pd.DataFrame)
def test_ip_summary_notebooklet_all(monkeypatch):
"""Test basic run of notebooklet."""
test_data = str(Path(TEST_DATA_PATH).absolute())
monkeypatch.setattr(data_providers, "GeoLiteLookup", GeoIPLiteMock)
monkeypatch.setattr(data_providers, "TILookup", TILookupMock)
data_providers.init(
query_provider="LocalData",
LocalData_data_paths=[test_data],
LocalData_query_paths=[test_data],
providers=["tilookup", "geolitelookup"],
)
opts = ["+az_netflow", "+passive_dns", "+az_activity", "+office_365", "+ti"]
test_nb = nblts.azsent.network.IpAddressSummary()
tspan = TimeSpan(period="1D")
test_nb.query_provider.schema.update({tab: {} for tab in DEF_PROV_TABLES})
result = test_nb.run(value="40.76.43.124", timespan=tspan, options=opts)
check.is_not_none(result.ip_entity)
check.is_not_none(result.host_entity)
check.equal(result.host_entity.HostName, "MSTICAlertsWin1")
check.equal(result.host_entity.OSFamily.name, "Linux")
check.equal(result.ip_type, "Public")
check.equal(result.ip_origin, "Internal")
check.is_instance(result.heartbeat, pd.DataFrame)
check.is_instance(result.az_network_if, pd.DataFrame)
check.is_instance(result.az_network_flows, pd.DataFrame)
check.is_instance(result.az_network_flow_summary, pd.DataFrame)
check.is_instance(result.az_network_flows_timeline, LayoutDOM)
check.is_instance(result.aad_signins, pd.DataFrame)
check.is_instance(result.office_activity, pd.DataFrame)
check.is_instance(result.vmcomputer, pd.DataFrame)
check.is_instance(test_nb.netflow_total_by_protocol(), LayoutDOM)
check.is_instance(test_nb.netflow_by_direction(), LayoutDOM)
check.is_not_none(result.whois)
check.is_instance(result.related_alerts, pd.DataFrame)
check.is_instance(result.passive_dns, pd.DataFrame)
check.is_instance(result.ti_results, pd.DataFrame)
def __init__(self,
data_providers: Optional[DataProviders] = None,
**kwargs):
"""
Intialize a new instance of the notebooklet class.
Parameters
----------
data_providers : DataProviders, Optional
Optional DataProviders instance to query data.
Most classes require this.
Raises
------
MsticnbDataProviderError
If DataProviders has not been initialized.
If required providers are specified by the notebooklet
but are not available.
"""
self._kwargs = kwargs
self.options: List[str] = self.default_options()
self._set_tqdm_notebook(get_opt("verbose"))
self._last_result: Any = None
self.timespan = TimeSpan(period="1d")
self._inst_default_silent: Optional[bool] = kwargs.get("silent")
self._current_run_silent: Optional[bool] = None
set_opt("temp_silent", self.silent)
# update "run" function documentation on first run
self._add_run_doc_options()
# Check required data providers are loaded.
# pylint: disable=no-member
self.data_providers = data_providers or DataProviders.current(
) # type: ignore
# pylint: enable=no-member
self._check_nb_providers(**kwargs)
def _get_timespan():
end = datetime.utcnow()
return TimeSpan(start=(end - timedelta(1)), end=end)
def test_account_summary_notebooklet(monkeypatch):
"""Test basic run of notebooklet."""
test_data = str(Path(TEST_DATA_PATH).absolute())
monkeypatch.setattr(data_providers, "GeoLiteLookup", GeoIPLiteMock)
data_providers.init(
"LocalData",
providers=["-tilookup"],
LocalData_data_paths=[test_data],
LocalData_query_paths=[test_data],
)
test_nb = nblts.azsent.account.AccountSummary()
tspan = TimeSpan(period="1D")
result = test_nb.run(value="accountname", timespan=tspan)
check.is_not_none(result.account_selector)
acct_select = test_nb.browse_accounts()
check.is_instance(acct_select, nbwidgets.SelectItem)
select_opts = result.account_selector._item_dict
disp_account = result.account_selector.item_action
for acct_item in select_opts.values():
# Programatically select the item list control
select_item = [
key for key, value in select_opts.items() if value == acct_item
]
if select_item:
result.account_selector._wgt_select.value = select_item[0]
disp_account(acct_item)
check.is_instance(result.account_activity, pd.DataFrame)
check.is_instance(result.related_alerts, pd.DataFrame)
check.is_instance(result.related_bookmarks, pd.DataFrame)
check.is_instance(result.alert_timeline, LayoutDOM)
check.is_instance(result.account_entity, entities.Account)
alert_select = test_nb.browse_alerts()
check.is_instance(alert_select, nbwidgets.SelectAlert)
bm_select = test_nb.browse_bookmarks()
assert isinstance(bm_select, nbwidgets.SelectItem)
test_nb.get_additional_data()
check.is_instance(result.account_timeline_by_ip, LayoutDOM)
if "Windows" in acct_item or "Linux" in acct_item:
check.is_instance(result.host_logons, pd.DataFrame)
check.is_instance(result.host_logon_summary, pd.DataFrame)
check.is_none(result.azure_activity)
check.is_none(result.azure_activity_summary)
check.is_none(result.azure_timeline_by_provider)
check.is_none(result.azure_timeline_by_operation)
vwr = result.view_events(
attrib="host_logons",
summary_cols=["Computer", "LogonResult", "LogonType"],
)
else:
check.is_none(result.host_logons)
check.is_none(result.host_logon_summary)
check.is_instance(result.azure_activity, pd.DataFrame)
check.is_instance(result.azure_activity_summary, pd.DataFrame)
check.is_instance(result.azure_timeline_by_provider, LayoutDOM)
check.is_instance(result.azure_timeline_by_operation, LayoutDOM)
vwr = result.view_events(
attrib="azure_activity",
summary_cols=["Source", "Operation", "IPAddress"],
)
check.is_instance(vwr, nbwidgets.SelectItem)
result.display_alert_timeline()
result.browse_accounts()
result.browse_alerts()
result.browse_bookmarks()
result.az_activity_timeline_by_provider()
result.az_activity_timeline_by_ip()
result.az_activity_timeline_by_operation()
result.host_logon_timeline()
check.is_not_none(result.get_geoip_map())
def run(
self,
value: Any = None,
data: Optional[pd.DataFrame] = None,
timespan: Optional[TimeSpan] = None,
options: Optional[Iterable[str]] = None,
**kwargs,
) -> NotebookletResult:
"""
Notebooklet abstract base class.
Parameters
----------
value : Any, optional
value to process, by default None
data : Optional[pd.DataFrame], optional
Input data to process, by default None
timespan : Optional[TimeSpan, Any], optional
Timespan over which operations such as queries will be
performed, by default None.
This can be a TimeStamp object or another object that
has valid `start`, `end`, or `period` attributes.
options :Optional[Iterable[str]], optional
List of options to use, by default None
A value of None means use default options.
Options prefixed with "+" will be added to the default options.
Options prefixed with "-" will be removed from the default options.
To see the list of available options type `help(cls)` where
"cls" is the notebooklet class or an instance of this class.
Other Parameters
----------------
start : Union[datetime, datelike-string]
Alternative to specifying timespan parameter.
end : Union[datetime, datelike-string]
Alternative to specifying timespan parameter.
Returns
-------
NotebookletResult
Result class from the notebooklet
See Also
--------
TimeSpan
"""
self._current_run_silent = kwargs.get("silent")
set_opt("temp_silent", self.silent)
if not options:
self.options = self.default_options()
else:
def_options = self.default_options()
add_options = {opt[1:] for opt in options if opt.startswith("+")}
sub_options = {opt[1:] for opt in options if opt.startswith("-")}
std_options = {opt for opt in options if opt[0] not in ("+", "-")}
if std_options and (add_options or sub_options):
raise MsticnbError(
"Option list must be either a list of options to use",
"or options to add/remove from the default set.",
"You cannot mix these.",
)
invalid_opts = (sub_options | add_options | std_options) - set(
self.all_options())
if invalid_opts:
print(f"Invalid options {list(invalid_opts)} ignored.")
if sub_options:
self.options = list(set(def_options) - sub_options)
if add_options:
self.options = list(set(def_options) | add_options)
if not (add_options or sub_options):
self.options = list(options)
self._set_tqdm_notebook(get_opt("verbose"))
if timespan:
self.timespan = TimeSpan(timespan=timespan)
elif "start" in kwargs and "end" in kwargs:
self.timespan = TimeSpan(start=kwargs.get("start"),
end=kwargs.get("end"))
return NotebookletResult(notebooklet=self)