def _label_search(label: Text,
return_fields: Optional[Text] = '',
max_entries: Optional[int] = None) -> pd.DataFrame:
"""Returns a DataFrame with all events that have a certain label.
Args:
label (str): a string representing the label to search for.
return_fields (str): a comma separated string with all the fields that
should be returned. If not provided all fields will be returned.
max_entries (int): an integer with the maximum number of entries to return.
Returns:
A pandas DataFrame with the search results.
"""
connect()
state_obj = state.state()
sketch = state_obj.get_from_cache('timesketch_sketch')
if not sketch:
print('No data, not connected to a sketch.')
return pd.DataFrame()
if not return_fields:
return_fields = '*'
search_obj = api_search.Search(sketch)
label_chip = api_search.LabelChip()
label_chip.label = label
search_obj.add_chip(label_chip)
search_obj.return_fields = _fix_return_fields(return_fields)
if max_entries:
search_obj.max_entries = max_entries
return search_obj.table
def test_wildcard_query(self):
"""Wildcard query over all data in the sketch."""
search_obj = search.Search(self.sketch)
search_obj.query_string = '*'
data_frame = search_obj.table
count = len(data_frame)
self.assertions.assertEqual(count, 3205)
def test_get_sigma_rule(self):
"""Client Sigma object tests."""
rule = self.api.get_sigma_rule(
rule_uuid="5266a592-b793-11ea-b3de-0242ac130004")
rule.from_rule_uuid("5266a592-b793-11ea-b3de-0242ac130004")
self.assertions.assertGreater(len(rule.attributes), 5)
self.assertions.assertIsNotNone(rule)
self.assertions.assertIn("Alexander", rule.author)
self.assertions.assertIn("Alexander", rule.get_attribute("author"))
self.assertions.assertIn("b793-11ea-b3de-0242ac130004", rule.id)
self.assertions.assertIn("Installation of ZMap", rule.title)
self.assertions.assertIn("zmap", rule.es_query)
self.assertions.assertIn("shell:zsh:history", rule.es_query)
self.assertions.assertIn("lnx_susp_zmap.yml", rule.file_relpath)
self.assertions.assertIn("sigma/rule/5266a592", rule.resource_uri)
self.assertions.assertIn("installation of ZMap", rule.description)
self.assertions.assertIn("high", rule.level)
self.assertions.assertEqual(len(rule.falsepositives), 1)
self.assertions.assertIn("Unknown", rule.falsepositives[0])
self.assertions.assertIn("susp_zmap", rule.file_name)
self.assertions.assertIn("2020/06/26", rule.date)
self.assertions.assertIn("2020/06/26", rule.modified)
self.assertions.assertIn("high", rule.level)
self.assertions.assertIn("rmusser.net", rule.references[0])
self.assertions.assertEqual(len(rule.detection), 2)
self.assertions.assertEqual(len(rule.logsource), 2)
# Test an actual query
self.import_timeline("sigma_events.csv")
search_obj = search.Search(self.sketch)
search_obj.query_string = rule.es_query
data_frame = search_obj.table
count = len(data_frame)
self.assertions.assertEqual(count, 1)
def test_get_sigma_rule(self):
"""Client Sigma object tests."""
rule = self.api.get_sigma_rule(
rule_uuid='5266a592-b793-11ea-b3de-0242ac130004')
rule.from_rule_uuid('5266a592-b793-11ea-b3de-0242ac130004')
self.assertions.assertGreater(len(rule.attributes), 5)
self.assertions.assertIsNotNone(rule)
self.assertions.assertIn('Alexander', rule.author)
self.assertions.assertIn('Alexander', rule.get_attribute('author'))
self.assertions.assertIn('b793-11ea-b3de-0242ac130004', rule.id)
self.assertions.assertIn('Installation of ZMap', rule.title)
self.assertions.assertIn('zmap', rule.es_query)
self.assertions.assertIn('shell:zsh:history', rule.es_query)
self.assertions.assertIn('lnx_susp_zmap.yml', rule.file_relpath)
self.assertions.assertIn('sigma/rule/5266a592', rule.resource_uri)
self.assertions.assertIn('installation of ZMap', rule.description)
self.assertions.assertIn('high', rule.level)
self.assertions.assertEqual(len(rule.falsepositives), 1)
self.assertions.assertIn('Unknown', rule.falsepositives[0])
self.assertions.assertIn('susp_zmap', rule.file_name)
self.assertions.assertIn('2020/06/26', rule.date)
self.assertions.assertIn('2020/06/26', rule.modified)
self.assertions.assertIn('high', rule.level)
self.assertions.assertIn('rmusser.net', rule.references[0])
self.assertions.assertEqual(len(rule.detection), 2)
self.assertions.assertEqual(len(rule.logsource), 2)
# Test an actual query
self.import_timeline('sigma_events.csv')
search_obj = search.Search(self.sketch)
search_obj.query_string = rule.es_query
data_frame = search_obj.table
count = len(data_frame)
self.assertions.assertEqual(count, 1)
def test_specific_queries(self):
"""Test few specific queries."""
search_obj = search.Search(self.sketch)
search_obj.query_string = 'message_identifier: "1073748864"'
search_obj.return_fields = "computer_name,data_type,strings,user_sid"
data_frame = search_obj.table
self.assertions.assertEqual(len(data_frame), 204)
computers = list(data_frame.computer_name.unique())
self.assertions.assertEqual(len(computers), 1)
self.assertions.assertEqual(computers[0], "WKS-WIN764BITB.shieldbase.local")
def extract_strings(row):
strings = row.strings
return pd.Series(
{
"service": strings[0],
"state_from": strings[1],
"state_to": strings[2],
"by": strings[3],
}
)
strings_frame = data_frame.apply(extract_strings, axis=1)
services = set(strings_frame.service.unique())
expected_set = set(
["Background Intelligent Transfer Service", "Windows Modules Installer"]
)
self.assertions.assertSetEqual(services, expected_set)
search_name = "My First Search"
search_obj.name = search_name
search_obj.description = "Can it be, is it really?"
search_obj.save()
_ = self.sketch.lazyload_data(refresh_cache=True)
saved_search = None
for search_obj in self.sketch.list_saved_searches():
if search_obj.name == search_name:
saved_search = search_obj
break
if search_obj is None:
raise RuntimeError("Unable to find the saved search.")
self.assertions.assertEqual(
saved_search.return_fields,
"computer_name,data_type,strings,user_sid,datetime"
)
self.assertions.assertEqual(
saved_search.query_string, 'message_identifier: "1073748864"'
)
def _click_function(_):
sketch = timesketch_get_sketch_func()
search_obj = search.Search(sketch)
if start_time_form.value and end_time_form.value:
date_chip = search.DateRangeChip()
date_chip.start_time = start_time_form.value.strftime(
'%Y-%m-%dT%H:%M:%S')
date_chip.end_time = end_time_form.value.strftime(
'%Y-%m-%dT%H:%M:%S')
search_obj.add_chip(date_chip)
search_obj.query_string = query_string_form.value
display(
Markdown(f'Query **executed** - returned: {len(search_obj.table)} '
'records'))
utils.ipython_bind_global('search_obj', search_obj)
display(Markdown('Results are stored in the **search_obj**'))
def query_timesketch(
query: Optional[Text] = None,
query_dsl: Optional[Text] = None,
query_filter: Optional[Dict[Text, Any]] = None,
return_fields: Optional[Text] = None,
start_date: Optional[Text] = '',
end_date: Optional[Text] = '',
max_entries: Optional[int] = None,
indices: Optional[List[Text]] = None) -> api_search.Search:
"""Return back a search object from a Timesketch query.
Args:
query (str): the query string to send to Timesketch.
query_dsl (str): the query DSL to send to Timesketch.
return_fields (str): string with comma separated names of fields to
return back.
start_date (str): a timestamp in the form of
'YYYY-MM-DDTHH:MM:SS+00:00' of when to start the search query.
end_date (str): a timestamp in the form of
'YYYY-MM-DDTHH:MM:SS+00:00' of when to end the search query.
max_entries (int): Optional integer denoting a best effort to limit
the output size to the number of events. Events are read in,
10k at a time so there may be more events in the answer back
than this number denotes, this is a best effort. This value defaults
to 40k events. Setting max_entries to zero will return all entries
back (as in no limit).
indices (list): a list of indices to query, if not provided _all
will be used.
Returns:
A search object (api_search.Search) that is pre-configured.
Raises:
KeyError: if the query is sent in without a query or query_dsl.
"""
connect()
state_obj = state.state()
sketch = state_obj.get_from_cache('timesketch_sketch')
if all(x is None for x in [query, query_dsl]):
raise KeyError('Need to provide a query or query_dsl')
chip = None
if start_date or end_date:
if start_date and end_date:
chip = api_search.DateRangeChip()
chip.start_time = start_date
chip.end_time = end_date
else:
chip = api_search.DateIntervalChip()
if end_date:
chip.date = end_date
else:
chip.date = start_date
search_obj = api_search.Search(sketch)
search_obj.from_manual(query_string=query,
query_dsl=query_dsl,
query_filter=query_filter,
max_entries=max_entries)
return_fields = _fix_return_fields(return_fields)
if return_fields:
search_obj.return_fields = return_fields
if chip:
search_obj.add_chip(chip)
return search_obj
def search_group(ctx, query, times, time_ranges, labels, header, output,
return_fields, order, limit, saved_search, describe):
"""Search and explore."""
sketch = ctx.obj.sketch
output_format = ctx.obj.output_format
search_obj = search.Search(sketch=sketch)
if output:
output_format = output
new_line = True
if output_format == 'csv':
new_line = False
# Construct query from saved search and return early.
if saved_search:
search_obj.from_saved(saved_search)
if describe:
describe_query(search_obj)
return
click.echo(format_output(
search_obj, output_format, header), nl=new_line)
return
# Construct the query from flags.
# TODO (berggren): Add support for query DSL.
search_obj.query_string = query
if return_fields:
search_obj.return_fields = return_fields
if limit:
search_obj.max_entries = limit
if order == 'asc':
search_obj.order_ascending()
elif order == 'desc':
search_obj.order_descending()
# TODO: Add term chips.
if time_ranges:
for time_range in time_ranges:
try:
range_chip = search.DateRangeChip()
range_chip.add_start_time = time_range[0]
range_chip.add_end_time = time_range[1]
search_obj.add_chip(range_chip)
except ValueError:
click.echo(
"Error parsing date (make sure it is ISO formatted)")
sys.exit(1)
# TODO (berggren): This should support dates like 2021-02-12 and then
# convert to ISO format.
if times:
for time in times:
try:
range_chip = search.DateRangeChip()
range_chip.add_start_time = time
range_chip.add_end_time = time
search_obj.add_chip(range_chip)
except ValueError:
click.echo(
"Error parsing date (make sure it is ISO formatted)")
sys.exit(1)
if labels:
for label in labels:
label_chip = search.LabelChip()
if label == 'star':
label_chip.use_star_label()
elif label == 'comment':
label_chip.use_comment_label()
else:
label_chip.label = label
search_obj.add_chip(label_chip)
if describe:
describe_query(search_obj)
return
click.echo(format_output(search_obj, output_format, header), nl=new_line)
