def _prepare_yaml_file(filename, obj_type, include_all_score_objs):
"""
Prepare the YAML file such that it can be used for EQL
:param filename: file location of the YAML file
:param obj_type: technique administration file ('techniques') or data source administration file ('data_sources')
:return: A dict with date fields compatible for JSON and a new key-value pair event-type
for the EQL engine
"""
_yaml = init_yaml()
with open(filename, 'r') as yaml_file:
yaml_content = _yaml.load(yaml_file)
yaml_content_eql = _traverse_modify_date(yaml_content)
yaml_eql_events = []
# create EQL events from the list of dictionaries
if obj_type == 'data_sources':
for item in yaml_content_eql[obj_type]:
yaml_eql_events.append(eql.Event(obj_type, 0, item))
# flatten the technique administration file to EQL events
elif obj_type in ['visibility', 'detection']:
yaml_content_eql = _techniques_to_events(yaml_content_eql, obj_type,
include_all_score_objs)
for e in yaml_content_eql:
yaml_eql_events.append(eql.Event('techniques', 0, e))
return yaml_eql_events, yaml_content
def _prepare_yaml_file(filename, obj_type, include_all_score_objs):
"""
Prepare the YAML file such that it can be used for EQL
:param filename: file location of the YAML file
:param obj_type: technique administration file ('visibility' or 'detection') or data source administration file ('data_source')
:return: A dict with date fields compatible for JSON and a new key-value pair event-type
for the EQL engine
"""
if isinstance(filename, dict):
# file is a dict created due to the use of an EQL query by the user
yaml_content = filename
else:
# file is a file location on disk
_yaml = init_yaml()
with open(filename, 'r') as yaml_file:
yaml_content = _yaml.load(yaml_file)
yaml_content_eql = _traverse_modify_date(yaml_content)
yaml_eql_events = []
# create EQL events from the list of dictionaries
if obj_type == 'data_sources':
yaml_content_eql, _, _, _, _ = load_data_sources(yaml_content, filter_empty_scores=False)
yaml_content_eql = _data_sources_to_events(yaml_content_eql)
for e in yaml_content_eql:
yaml_eql_events.append(eql.Event(obj_type, 0, e))
# flatten the technique administration file to EQL events
elif obj_type in ['visibility', 'detection']:
yaml_content_eql = _techniques_to_events(yaml_content_eql, obj_type, include_all_score_objs)
for e in yaml_content_eql:
yaml_eql_events.append(eql.Event('techniques', 0, e))
return yaml_eql_events, yaml_content
def normalize_callback(data):
"""Normalize an event to the common schema."""
scoped = scoper(data) if scoper else data
output = {} if self.strict else scoped.copy()
if self.time_field not in data:
raise ValueError(
"Unable to normalize. Check that the input schema matches {}"
.format(self.name))
ts = data[self.time_field]
if self.time_format != 'filetime':
ts = int((datetime.datetime.strptime(ts, self.time_format) -
FILETIME_BASE).total_seconds() * 1e7)
# Determine the event type first
evt = eql.Event(None, None, data)
if data.get('event_type') in event_updates:
event_type = data['event_type']
else:
for name, check_type in event_updates:
if check_type(evt):
event_type = name
break
else:
event_type = EVENT_TYPE_GENERIC
# Convert the global fields
scoped_evt = eql.Event(None, None, scoped)
for normalized, converter in global_mapping.items():
value = converter(scoped_evt)
if value is not None:
output[normalized] = value
# check the enums, but not against the scoped fields
for enum_name, enum_options in enum_converters.get(event_type, []):
for enum_option, enum_checker in enum_options:
if enum_checker(evt):
output[enum_name] = enum_option
break
# check the mappings against the scoped fields
for normalized, converter in event_mapping.get(event_type,
{}).items():
value = converter(scoped_evt)
if value is not None:
output[normalized] = value
output['event_type'] = event_type
output['timestamp'] = ts
converted_event = eql.Event(event_type, ts, output)
return converted_event
def _create_events(self, data, data_type, event_type, timestamp_key):
"""
Create EQL Events from the provided data.
:param data: list of dictionaries to be transformed to EQL Events
:param data_type: if 'yaml', serialize all 'Datetime.date' key-value pairs
:param event_type: the value to be used as event_type for the provided data
:param timestamp_key: name of the key-value pair to be used as timestamp
:return: EQL Events or data
"""
eql_events = []
if data_type == 'yaml':
data = self._serialize_date(data)
# this result in EQL trying the derive the event_type and timestamp from the contents of 'data'
if not event_type:
return data
# create EQL Events from 'data'
for item in data:
eql_events.append(eql.Event(event_type, timestamp_key, item))
return eql_events