def _parse_time_range(self, qualifier, time_range):
"""
Format the input time range UTC timestamp to Unix time
:param time_range: int, value available from main.py in options variable
:return: str, format_string
"""
format_string = ''
try:
compile_timestamp_regex = re.compile(TIMESTAMP_PATTERN)
transformer = TimestampToMilliseconds()
if qualifier and compile_timestamp_regex.search(qualifier):
time_range_iterator = map(
lambda x: int(transformer.transform(x.group()) / 1000),
compile_timestamp_regex.finditer(qualifier))
# Default time range Start time = Now - 5 minutes and Stop time = Now
else:
stop_time = datetime.now()
start_time = int(
round((stop_time -
timedelta(minutes=time_range)).timestamp()))
stop_time = int(round(stop_time.timestamp()))
time_range_iterator = [start_time, stop_time]
self.time_range_lst.append([each for each in time_range_iterator])
return format_string
except (KeyError, IndexError, TypeError) as e:
raise e
def _convert_timestamps_to_milliseconds(query_parts):
# grab time stamps from array
start_time = _test_or_add_milliseconds(query_parts[2])
stop_time = _test_or_add_milliseconds(query_parts[4])
transformer = TimestampToMilliseconds()
millisecond_start_time = transformer.transform(start_time)
millisecond_stop_time = transformer.transform(stop_time)
return query_parts[0] + " " + query_parts[1] + " " + str(millisecond_start_time) + " " + query_parts[3] + " " + str(millisecond_stop_time)
def _parse_comparison_expression(self, expression, qualifier=None):
# Resolve STIX Object Path to a field in the target Data Model
stix_object, stix_field = expression.object_path.split(':')
# Multiple QRadar fields may map to the same STIX Object
mapped_fields_array = self.dmm.map_field(stix_object, stix_field)
# Resolve the comparison symbol to use in the query string (usually just ':')
comparator = self._lookup_comparison_operator(self,
expression.comparator)
if stix_field == 'protocols[*]':
map_data = _fetch_network_protocol_mapping()
try:
expression.value = map_data[expression.value.lower()]
except Exception as protocol_key:
raise KeyError("Network protocol {} is not supported.".format(
protocol_key))
elif stix_field == 'start' or stix_field == 'end':
transformer = TimestampToMilliseconds()
expression.value = transformer.transform(expression.value)
# Some values are formatted differently based on how they're being compared
if expression.comparator == ComparisonComparators.Matches: # needs forward slashes
value = self._format_match(expression.value)
# should be (x, y, z, ...)
elif expression.comparator == ComparisonComparators.In:
value = self._format_set(expression.value)
elif expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual:
# Should be in single-quotes
value = self._format_equality(expression.value)
# '%' -> '*' wildcard, '_' -> '?' single wildcard
elif expression.comparator == ComparisonComparators.Like and not (
expression.object_path == 'x-readable-payload:value'):
value = self._format_like(expression.value)
else:
value = self._escape_value(expression.value)
comparison_string = self._parse_mapped_fields(self, expression, value,
comparator, stix_field,
mapped_fields_array)
if (len(mapped_fields_array) > 1
and not self._is_reference_value(stix_field)):
# More than one AQL field maps to the STIX attribute so group the ORs.
comparison_string = "({})".format(comparison_string)
if expression.negated:
comparison_string = self._negate_comparison(comparison_string)
if qualifier:
self.qualified_queries.append("{} limit {} {}".format(
comparison_string, self.result_limit, qualifier))
return ''
else:
return "{}".format(comparison_string)
def _parse_comparison_expression(self,
expression,
qualifier=None,
calling_object_type=None):
# Resolve STIX Object Path to a field in the target Data Model
stix_object, stix_field = expression.object_path.split(':')
# Multiple QRadar fields may map to the same STIX Object
mapped_fields_array = self.dmm.map_field(stix_object, stix_field)
# Resolve the comparison symbol to use in the query string (usually just ':')
comparator = self.comparator_lookup[expression.comparator]
if stix_field == 'protocols[*]':
map_data = _fetch_network_protocol_mapping()
try:
expression.value = map_data[expression.value.lower()]
except Exception as protocol_key:
raise KeyError("Network protocol {} is not supported.".format(
protocol_key))
elif stix_field == 'start' or stix_field == 'end':
transformer = TimestampToMilliseconds()
expression.value = transformer.transform(expression.value)
# Some values are formatted differently based on how they're being compared
value = self._format_value(self, expression)
comparison_string = self._parse_mapped_fields(self, expression, value,
comparator, stix_field,
mapped_fields_array)
if (len(mapped_fields_array) > 1
and not self._is_reference_value(stix_field)):
# More than one AQL field maps to the STIX attribute so group the ORs.
comparison_string = "({})".format(comparison_string)
if expression.comparator == ComparisonComparators.NotEqual:
comparison_string = self._negate_comparison(comparison_string)
if expression.negated:
comparison_string = self._negate_comparison(comparison_string)
if calling_object_type in [
'CombinedObservationExpression', 'ObservationExpression',
'CombinedComparisonExpression'
]:
return comparison_string
elif qualifier is not None:
self.qualified_queries.append("{} limit {} {}".format(
comparison_string, self.result_limit, qualifier))
else:
self.unqualified_queries.append(comparison_string)
return comparison_string
def _format_datetime(value) -> list:
"""
Formating value in the event of value is datetime
:param value: datetime value
:return: list, timestamp in milliseconds or timestamp itself
"""
transformer = TimestampToMilliseconds()
values = value.values if hasattr(value, 'values') else [value]
milli_secs_lst = list(map(transformer.transform, values))
values = list(map(lambda x: '{}'.format(str(x)[:-3]), milli_secs_lst))
return values
def _parse_expression(self, expression, qualifier=None) -> str:
if isinstance(expression, ComparisonExpression): # Base Case
# Resolve STIX Object Path to a field in the target Data Model
stix_object, stix_field = expression.object_path.split(':')
# Multiple data source fields may map to the same STIX Object
mapped_fields_array = self.dmm.map_field(stix_object, stix_field)
# Resolve the comparison symbol to use in the query string (usually just ':')
comparator = self._lookup_comparison_operator(
self, expression.comparator)
if stix_field == 'start' or stix_field == 'end':
transformer = TimestampToMilliseconds()
expression.value = transformer.transform(expression.value)
# Some values are formatted differently based on how they're being compared
if expression.comparator == ComparisonComparators.Matches: # needs forward slashes
value = self._format_match(expression.value)
# should be (x, y, z, ...)
elif expression.comparator == ComparisonComparators.In:
value = self._format_set(expression.value)
elif expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual:
# Should be in single-quotes
value = self._format_equality(expression.value)
# '%' -> '*' wildcard, '_' -> '?' single wildcard
elif expression.comparator == ComparisonComparators.Like:
value = self._format_like(expression.value)
else:
value = self._escape_value(expression.value)
comparison_string = self._parse_mapped_fields(
self, expression, value, comparator, stix_field,
mapped_fields_array)
if (len(mapped_fields_array) > 1
and not self._is_reference_value(stix_field)):
# More than one data source field maps to the STIX attribute, so group comparisons together.
grouped_comparison_string = "(" + comparison_string + ")"
comparison_string = grouped_comparison_string
if expression.negated:
comparison_string = self._negate_comparison(comparison_string)
if qualifier is not None:
return "{} {}".format(comparison_string, qualifier)
else:
return "{}".format(comparison_string)
elif isinstance(expression, CombinedComparisonExpression):
operator = self._lookup_comparison_operator(
self, expression.operator)
expression_01 = self._parse_expression(expression.expr1)
expression_02 = self._parse_expression(expression.expr2)
if not expression_01 or not expression_02:
return ''
if isinstance(expression.expr1, CombinedComparisonExpression):
expression_01 = "({})".format(expression_01)
if isinstance(expression.expr2, CombinedComparisonExpression):
expression_02 = "({})".format(expression_02)
query_string = "{} {} {}".format(expression_01, operator,
expression_02)
if qualifier is not None:
return "{} {}".format(query_string, qualifier)
else:
return "{}".format(query_string)
elif isinstance(expression, ObservationExpression):
return self._parse_expression(expression.comparison_expression,
qualifier)
elif hasattr(expression, 'qualifier') and hasattr(
expression, 'observation_expression'):
if isinstance(expression.observation_expression,
CombinedObservationExpression):
operator = self._lookup_comparison_operator(
self, expression.observation_expression.operator)
expression_01 = self._parse_expression(
expression.observation_expression.expr1)
# qualifier only needs to be passed into the parse expression once since it will be the same for both expressions
expression_02 = self._parse_expression(
expression.observation_expression.expr2,
expression.qualifier)
return "{} {} {}".format(expression_01, operator,
expression_02)
else:
return self._parse_expression(
expression.observation_expression.comparison_expression,
expression.qualifier)
elif isinstance(expression, CombinedObservationExpression):
operator = self._lookup_comparison_operator(
self, expression.operator)
expression_01 = self._parse_expression(expression.expr1)
expression_02 = self._parse_expression(expression.expr2)
if expression_01 and expression_02:
return "({}) {} ({})".format(expression_01, operator,
expression_02)
elif expression_01:
return "{}".format(expression_01)
elif expression_02:
return "{}".format(expression_02)
else:
return ''
elif isinstance(expression, Pattern):
return "{expr}".format(
expr=self._parse_expression(expression.expression))
else:
raise RuntimeError(
"Unknown Recursion Case for expression={}, type(expression)={}"
.format(expression, type(expression)))
def _parse_expression(self, expression, qualifier=None) -> str:
if isinstance(expression, ComparisonExpression): # Base Case
# Resolve STIX Object Path to a field in the target Data Model
stix_object, stix_field = expression.object_path.split(':')
# Multiple QRadar fields may map to the same STIX Object
mapped_fields_array = self.dmm.map_field(stix_object, stix_field)
# Resolve the comparison symbol to use in the query string (usually just ':')
comparator = self.comparator_lookup[expression.comparator]
original_stix_value = expression.value
if stix_field == 'protocols[*]':
map_data = _fetch_network_protocol_mapping()
try:
expression.value = map_data[expression.value.lower()]
except Exception as protocol_key:
raise KeyError(
"Network protocol {} is not supported.".format(
protocol_key))
elif stix_field == 'start' or stix_field == 'end':
transformer = TimestampToMilliseconds()
# TODO Skydive uses seconds for timestamps, but this is something we should configure
expression.value = int(
transformer.transform(expression.value) / 1000)
# Some values are formatted differently based on how they're being compared
if expression.comparator == ComparisonComparators.Matches: # needs forward slashes
value = self._format_match(expression.value)
# should be (x, y, z, ...)
elif expression.comparator == ComparisonComparators.In:
value = self._format_set(expression.value)
elif expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual:
# Should be in single-quotes
value = self._format_equality(expression.value)
# '%' -> '*' wildcard, '_' -> '?' single wildcard
elif expression.comparator == ComparisonComparators.Like:
value = self._format_like(expression.value)
else:
value = self._escape_value(expression.value)
self.parsed_pattern.append({
'attribute': expression.object_path,
'comparison_operator': comparator,
'value': original_stix_value
})
comparison_string = ""
mapped_fields_count = len(mapped_fields_array)
if len(mapped_fields_array) == 0:
comparison_string += "false"
for mapped_field in mapped_fields_array:
comparison_string += "{mapped_field} {comparator} {value}".format(
mapped_field=mapped_field,
comparator=comparator,
value=value)
if (mapped_fields_count > 1):
comparison_string += " OR "
mapped_fields_count -= 1
if (len(mapped_fields_array) > 1):
# More than one SQL field maps to the STIX attribute so group the ORs.
grouped_comparison_string = "(" + comparison_string + ")"
comparison_string = grouped_comparison_string
if expression.comparator == ComparisonComparators.NotEqual:
comparison_string = self._negate_comparison(comparison_string)
if expression.negated:
comparison_string = self._negate_comparison(comparison_string)
if qualifier is not None:
return "{comparison} {qualifier} split".format(
comparison=comparison_string, qualifier=qualifier)
else:
return "{comparison}".format(comparison=comparison_string)
elif isinstance(expression, CombinedComparisonExpression):
query_string = "{} {} {}".format(
self._parse_expression(expression.expr1),
self.comparator_lookup[expression.operator],
self._parse_expression(expression.expr2))
if qualifier is not None:
return "{query_string} {qualifier} split".format(
query_string=query_string, qualifier=qualifier)
else:
return "{query_string}".format(query_string=query_string)
elif isinstance(expression, ObservationExpression):
return self._parse_expression(expression.comparison_expression,
qualifier)
elif hasattr(expression, 'qualifier') and hasattr(
expression, 'observation_expression'):
if isinstance(expression.observation_expression,
CombinedObservationExpression):
operator = self.comparator_lookup[
expression.observation_expression.operator]
# qualifier only needs to be passed into the parse expression once since it will be the same for both expressions
return "{expr1} {operator} {expr2}".format(
expr1=self._parse_expression(
expression.observation_expression.expr1),
operator=operator,
expr2=self._parse_expression(
expression.observation_expression.expr2,
expression.qualifier))
else:
return self._parse_expression(
expression.observation_expression.comparison_expression,
expression.qualifier)
elif isinstance(expression, CombinedObservationExpression):
operator = self.comparator_lookup[expression.operator]
return "{expr1} {operator} {expr2}".format(
expr1=self._parse_expression(expression.expr1),
operator=operator,
expr2=self._parse_expression(expression.expr2))
elif isinstance(expression, Pattern):
return "{expr}".format(
expr=self._parse_expression(expression.expression))
else:
raise RuntimeError(
"Unknown Recursion Case for expression={}, type(expression)={}"
.format(expression, type(expression)))
