def _parse_time_range(self, qualifier, time_range):
"""
Format the input time range UTC timestamp to Unix time
:param time_range: int, value available from main.py in options variable
:return: str, format_string
"""
format_string = ''
try:
compile_timestamp_regex = re.compile(TIMESTAMP_PATTERN)
transformer = TimestampToMilliseconds()
if qualifier and compile_timestamp_regex.search(qualifier):
time_range_iterator = map(
lambda x: int(transformer.transform(x.group()) / 1000),
compile_timestamp_regex.finditer(qualifier))
# Default time range Start time = Now - 5 minutes and Stop time = Now
else:
stop_time = datetime.now()
start_time = int(
round((stop_time -
timedelta(minutes=time_range)).timestamp()))
stop_time = int(round(stop_time.timestamp()))
time_range_iterator = [start_time, stop_time]
self.time_range_lst.append([each for each in time_range_iterator])
return format_string
except (KeyError, IndexError, TypeError) as e:
raise e
def _parse_comparison_expression(self, expression, qualifier=None):
# Resolve STIX Object Path to a field in the target Data Model
stix_object, stix_field = expression.object_path.split(':')
# Multiple QRadar fields may map to the same STIX Object
mapped_fields_array = self.dmm.map_field(stix_object, stix_field)
# Resolve the comparison symbol to use in the query string (usually just ':')
comparator = self._lookup_comparison_operator(self,
expression.comparator)
# Special case for artifact:payload_bin object with Like operator where we apply aql TEXT SEARCH
if expression.comparator == ComparisonComparators.Like and (
expression.object_path == 'artifact:payload_bin'):
return "TEXT SEARCH '{}'".format(expression.value)
# Special case where we want the risk finding
if stix_object == 'x-ibm-finding' and stix_field == 'name' and expression.value == "*":
return "devicetype = 18"
if stix_field == 'protocols[*]':
map_data = read_json('network_protocol_map', self.options)
try:
expression.value = map_data[expression.value.lower()]
except Exception as protocol_key:
raise KeyError("Network protocol {} is not supported.".format(
protocol_key))
elif stix_field == 'start' or stix_field == 'end':
transformer = TimestampToMilliseconds()
expression.value = transformer.transform(expression.value)
# Some values are formatted differently based on how they're being compared
if expression.comparator == ComparisonComparators.Matches: # needs forward slashes
value = self._format_match(expression.value)
# should be (x, y, z, ...)
elif expression.comparator == ComparisonComparators.In:
value = self._format_in(expression.value)
elif expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual:
# Should be in single-quotes
value = self._format_equality(expression.value)
# '%' -> '*' wildcard, '_' -> '?' single wildcard
elif expression.comparator == ComparisonComparators.Like and not (
expression.object_path == 'artifact:payload_bin'):
value = self._format_like(expression.value)
else:
value = self._escape_value(expression.value)
comparison_string = self._parse_mapped_fields(self, expression, value,
comparator, stix_field,
mapped_fields_array)
if (len(mapped_fields_array) > 1
and not self._is_reference_value(stix_field)):
# More than one AQL field maps to the STIX attribute so group the ORs.
comparison_string = "({})".format(comparison_string)
if expression.negated:
comparison_string = self._negate_comparison(comparison_string)
if qualifier:
self.qualified_queries.append("{} limit {} {}".format(
comparison_string, self.result_limit, qualifier))
return ''
else:
return "{}".format(comparison_string)
def _convert_timestamps_to_milliseconds(query_parts):
# grab time stamps from array
start_time = _test_or_add_milliseconds(query_parts[2])
stop_time = _test_or_add_milliseconds(query_parts[4])
transformer = TimestampToMilliseconds()
millisecond_start_time = transformer.transform(start_time)
millisecond_stop_time = transformer.transform(stop_time)
return query_parts[0] + " " + query_parts[1] + " " + str(millisecond_start_time) + " " + query_parts[3] + " " + str(millisecond_stop_time)
def _format_start_stop_qualifier(self, expression, qualifier) -> str:
"""Convert a STIX start stop qualifier into a query string.
"""
transformer = TimestampToMilliseconds()
qualifier_split = qualifier.split("'")
start = transformer.transform(qualifier_split[1]) // 1000
stop = transformer.transform(qualifier_split[3]) // 1000
qualified_query = "%s AND 'start' : %s AND 'end' : %s" % (expression,
start, stop)
return qualified_query
def _format_start_stop_qualifier(self, expression, qualifier) -> str:
"""Convert a STIX start stop qualifier into a query string.
The sample MySQL schema included in this connector defines a timerange with a start and stop value
based on the entry_time field.
"""
transformer = TimestampToMilliseconds()
qualifier_split = qualifier.split("'")
start = transformer.transform(qualifier_split[1])
stop = transformer.transform(qualifier_split[3])
qualified_query = "%s AND (entry_time >= %s OR entry_time <= %s)" % (
expression, start, stop)
return qualified_query
def _convert_timestamps_to_milliseconds(query_parts):
# grab time stamps from array
start_time = _test_or_add_milliseconds(query_parts[2])
stop_time = _test_or_add_milliseconds(query_parts[4])
transformer = TimestampToMilliseconds()
millisecond_start_time = transformer.transform(start_time)
millisecond_stop_time = transformer.transform(stop_time)
payload = dict()
payload['offset'] = 0
payload['from'] = millisecond_start_time
payload['to'] = millisecond_stop_time
payload['query'] = query_parts[0]
return payload
def _format_start_stop_qualifier(self, expression, qualifier) -> str:
"""
Convert a STIX start stop qualifier into a query string.
"""
transformer = TimestampToMilliseconds()
qualifier_split = qualifier.split("'")
start = qualifier_split[1]
stop = qualifier_split[3]
# convert timepestamp to millisecond which will be passed to rest service
start_epoach = transformer.transform(start)
stop_epoach = transformer.transform(stop)
qualified_query = "%s&from=%s&to=%s" % (expression, start_epoach,
stop_epoach)
return qualified_query
def _format_datetime(value) -> list:
"""
Formating value in the event of value is datetime
:param value: datetime value
:return: list, timestamp in milliseconds or timestamp itself
"""
transformer = TimestampToMilliseconds()
values = value.values if hasattr(value, 'values') else [value]
milli_secs_lst = list(map(transformer.transform, values))
values = list(map(lambda x: '{}'.format(str(x)[:-3]), milli_secs_lst))
return values
def _parse_expression(self, expression, qualifier=None) -> str:
if isinstance(expression, ComparisonExpression): # Base Case
# Resolve STIX Object Path to a field in the target Data Model
stix_object, stix_field = expression.object_path.split(':')
# Multiple data source fields may map to the same STIX Object
mapped_fields_array = self.dmm.map_field(stix_object, stix_field)
# Resolve the comparison symbol to use in the query string (usually just ':')
comparator = self._lookup_comparison_operator(
self, expression.comparator)
if stix_field == 'start' or stix_field == 'end':
transformer = TimestampToMilliseconds()
expression.value = transformer.transform(expression.value)
# Some values are formatted differently based on how they're being compared
if expression.comparator == ComparisonComparators.Matches: # needs forward slashes
value = self._format_match(expression.value)
elif expression.comparator == ComparisonComparators.Equal or \
expression.comparator == ComparisonComparators.NotEqual:
# Should be in single-quotes
value = self._format_equality(expression.value)
# '%' -> '*' wildcard, '_' -> '?' single wildcard
elif expression.comparator == ComparisonComparators.Like:
value = self._format_like(expression.value)
else:
value = self._escape_value(expression.value)
comparison_string = self._parse_mapped_fields(
self, expression, value, comparator, stix_field,
mapped_fields_array)
if len(mapped_fields_array) > 1 and not self._is_reference_value(
stix_field):
# More than one data source field maps to the STIX attribute, so group comparisons together.
grouped_comparison_string = "(" + comparison_string + ")"
comparison_string = grouped_comparison_string
if expression.negated:
comparison_string = self._negate_comparison(comparison_string)
if qualifier is not None:
return "{} {}".format(comparison_string, qualifier)
else:
return "{}".format(comparison_string)
elif isinstance(expression, CombinedComparisonExpression):
operator = self._lookup_comparison_operator(
self, expression.operator)
expression_01 = self._parse_expression(expression.expr1)
expression_02 = self._parse_expression(expression.expr2)
if not expression_01 or not expression_02:
return ''
if isinstance(expression.expr1, CombinedComparisonExpression):
expression_01 = "({})".format(expression_01)
if isinstance(expression.expr2, CombinedComparisonExpression):
expression_02 = "({})".format(expression_02)
query_string = "{} {} {}".format(expression_01, operator,
expression_02)
if qualifier is not None:
return "{} {}".format(query_string, qualifier)
else:
return "{}".format(query_string)
elif isinstance(expression, ObservationExpression):
return self._parse_expression(expression.comparison_expression,
qualifier)
elif hasattr(expression, 'qualifier') and hasattr(
expression, 'observation_expression'):
if isinstance(expression.observation_expression,
CombinedObservationExpression):
operator = self._lookup_comparison_operator(
self, expression.observation_expression.operator)
expression_01 = self._parse_expression(
expression.observation_expression.expr1)
# qualifier only needs to be passed into the parse expression once since it will be the same for both
# expressions
expression_02 = self._parse_expression(
expression.observation_expression.expr2,
expression.qualifier)
return "{} {} {}".format(expression_01, operator,
expression_02)
else:
return self._parse_expression(
expression.observation_expression.comparison_expression,
expression.qualifier)
elif isinstance(expression, CombinedObservationExpression):
operator = self._lookup_comparison_operator(
self, expression.operator)
expression_01 = self._parse_expression(expression.expr1)
expression_02 = self._parse_expression(expression.expr2)
if expression_01 and expression_02:
return "({}) {} ({})".format(expression_01, operator,
expression_02)
elif expression_01:
return "{}".format(expression_01)
elif expression_02:
return "{}".format(expression_02)
else:
return ''
elif isinstance(expression, Pattern):
return "{expr}".format(
expr=self._parse_expression(expression.expression))
else:
raise RuntimeError(
"Unknown Recursion Case for expression={}, type(expression)={}"
.format(expression, type(expression)))
def _parse_expression(self, expression, qualifier=None) -> str:
if isinstance(expression, ComparisonExpression): # Base Case
# Resolve STIX Object Path to a field in the target Data Model
stix_object, stix_field = expression.object_path.split(':')
# Multiple QRadar fields may map to the same STIX Object
mapped_fields_array = self.dmm.map_field(stix_object, stix_field)
# Resolve the comparison symbol to use in the query string (usually just ':')
comparator = self.comparator_lookup[expression.comparator]
original_stix_value = expression.value
if stix_field == 'protocols[*]':
map_data = _fetch_network_protocol_mapping()
try:
expression.value = map_data[expression.value.lower()]
except Exception as protocol_key:
raise KeyError(
"Network protocol {} is not supported.".format(
protocol_key))
elif stix_field == 'start' or stix_field == 'end':
transformer = TimestampToMilliseconds()
# TODO Skydive uses seconds for timestamps, but this is something we should configure
expression.value = int(
transformer.transform(expression.value) / 1000)
# Some values are formatted differently based on how they're being compared
if expression.comparator == ComparisonComparators.Matches: # needs forward slashes
value = self._format_match(expression.value)
# should be (x, y, z, ...)
elif expression.comparator == ComparisonComparators.In:
value = self._format_set(expression.value)
elif expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual:
# Should be in single-quotes
value = self._format_equality(expression.value)
# '%' -> '*' wildcard, '_' -> '?' single wildcard
elif expression.comparator == ComparisonComparators.Like:
value = self._format_like(expression.value)
else:
value = self._escape_value(expression.value)
comparison_string = ""
mapped_fields_count = len(mapped_fields_array)
if len(mapped_fields_array) == 0:
comparison_string += "false"
for mapped_field in mapped_fields_array:
comparison_string += "{mapped_field} {comparator} {value}".format(
mapped_field=mapped_field,
comparator=comparator,
value=value)
if (mapped_fields_count > 1):
comparison_string += " OR "
mapped_fields_count -= 1
if (len(mapped_fields_array) > 1):
# More than one SQL field maps to the STIX attribute so group the ORs.
grouped_comparison_string = "(" + comparison_string + ")"
comparison_string = grouped_comparison_string
if expression.comparator == ComparisonComparators.NotEqual:
comparison_string = self._negate_comparison(comparison_string)
if expression.negated:
comparison_string = self._negate_comparison(comparison_string)
if qualifier is not None:
return "{comparison} {qualifier} split".format(
comparison=comparison_string, qualifier=qualifier)
else:
return "{comparison}".format(comparison=comparison_string)
elif isinstance(expression, CombinedComparisonExpression):
query_string = "{} {} {}".format(
self._parse_expression(expression.expr1),
self.comparator_lookup[expression.operator],
self._parse_expression(expression.expr2))
if qualifier is not None:
return "{query_string} {qualifier} split".format(
query_string=query_string, qualifier=qualifier)
else:
return "{query_string}".format(query_string=query_string)
elif isinstance(expression, ObservationExpression):
return self._parse_expression(expression.comparison_expression,
qualifier)
elif hasattr(expression, 'qualifier') and hasattr(
expression, 'observation_expression'):
if isinstance(expression.observation_expression,
CombinedObservationExpression):
operator = self.comparator_lookup[
expression.observation_expression.operator]
# qualifier only needs to be passed into the parse expression once since it will be the same for both expressions
return "{expr1} {operator} {expr2}".format(
expr1=self._parse_expression(
expression.observation_expression.expr1),
operator=operator,
expr2=self._parse_expression(
expression.observation_expression.expr2,
expression.qualifier))
else:
return self._parse_expression(
expression.observation_expression.comparison_expression,
expression.qualifier)
elif isinstance(expression, CombinedObservationExpression):
operator = self.comparator_lookup[expression.operator]
return "{expr1} {operator} {expr2}".format(
expr1=self._parse_expression(expression.expr1),
operator=operator,
expr2=self._parse_expression(expression.expr2))
elif isinstance(expression, Pattern):
return "{expr}".format(
expr=self._parse_expression(expression.expression))
else:
raise RuntimeError(
"Unknown Recursion Case for expression={}, type(expression)={}"
.format(expression, type(expression)))
def _format_timestamp(self, value):
transformer = TimestampToMilliseconds()
value = re.sub("'", "", value)
return transformer.transform(value)
def _parse_expression(self, expression, dialect, qualifier=None) -> str:
if isinstance(expression, ComparisonExpression): # Base Case
# Resolve STIX Object Path to a field in the target Data Model
stix_object, stix_field = expression.object_path.split(':')
# Multiple data source fields may map to the same STIX Object
mapped_fields_array = self.dmm.map_field(stix_object, stix_field)
if dialect == 'messageActivityData':
mapped_fields_set = set(mapped_fields_array)
intersection = self.assigned_fields.intersection(
mapped_fields_set)
if intersection:
logger.error(
f"[{', '.join(intersection)}] mapped from {stix_field} has multiple criteria"
)
raise NotImplementedError(
"Multiple criteria for one field is not support in MDL"
)
else:
self.assigned_fields |= mapped_fields_set
# Resolve the comparison symbol to use in the query string (usually just ':')
comparator = self._lookup_comparison_operator(
expression.comparator, dialect)
if stix_field in ('start', 'end'):
transformer = TimestampToMilliseconds()
expression.value = transformer.transform(expression.value)
# Some values are formatted differently based on how they're being compared
if expression.comparator == ComparisonComparators.Equal:
# Should be in single-quotes
value = self._format_equality(expression.value)
elif expression.comparator == ComparisonComparators.NotEqual:
value = self._format_equality(expression.value)
expression.negated = True
# '%' -> '*' wildcard, '_' -> '?' single wildcard
elif expression.comparator == ComparisonComparators.Like:
value = self._format_like(expression.value)
else:
value = self._escape_value(expression.value)
comparison_string = self._parse_mapped_fields(
expression, value, comparator, stix_field, mapped_fields_array)
if len(mapped_fields_array) > 1 and not self._is_reference_value(
stix_field):
# More than one data source field maps to the STIX attribute, so group comparisons together.
grouped_comparison_string = "(" + comparison_string + ")"
comparison_string = grouped_comparison_string
if expression.negated:
comparison_string = self._negate_comparison(comparison_string)
if qualifier is not None:
return "{} {}".format(comparison_string, qualifier)
else:
return "{}".format(comparison_string)
elif isinstance(expression, CombinedComparisonExpression):
operator = self._lookup_comparison_operator(
expression.operator, dialect)
expression_01 = self._parse_expression(expression.expr1, dialect)
expression_02 = self._parse_expression(expression.expr2, dialect)
if not expression_01 or not expression_02:
return ''
if isinstance(expression.expr1, CombinedComparisonExpression):
expression_01 = "({})".format(expression_01)
if isinstance(expression.expr2, CombinedComparisonExpression):
expression_02 = "({})".format(expression_02)
query_string = "{} {} {}".format(expression_01, operator,
expression_02)
if qualifier is not None:
return "{} {}".format(query_string, qualifier)
else:
return "{}".format(query_string)
elif isinstance(expression, ObservationExpression):
return self._parse_expression(expression.comparison_expression,
dialect, qualifier)
elif hasattr(expression, 'qualifier') and hasattr(
expression, 'observation_expression'):
if isinstance(expression.observation_expression,
CombinedObservationExpression):
operator = self._lookup_comparison_operator(
expression.observation_expression.operator, dialect)
expression_01 = self._parse_expression(
expression.observation_expression.expr1, dialect)
# qualifier only needs to be passed into the parse expression once since it will be the same for both expressions
expression_02 = self._parse_expression(
expression.observation_expression.expr2, dialect,
expression.qualifier)
return "{} {} {}".format(expression_01, operator,
expression_02)
else:
return self._parse_expression(
expression.observation_expression.comparison_expression,
dialect, expression.qualifier)
elif isinstance(expression, CombinedObservationExpression):
operator = self._lookup_comparison_operator(
expression.operator, dialect)
expression_01 = self._parse_expression(expression.expr1, dialect)
expression_02 = self._parse_expression(expression.expr2, dialect)
if expression_01 and expression_02:
return "({}) {} ({})".format(expression_01, operator,
expression_02)
elif expression_01:
return "{}".format(expression_01)
elif expression_02:
return "{}".format(expression_02)
else:
return ''
elif isinstance(expression, Pattern):
return "{expr}".format(
expr=self._parse_expression(expression.expression, dialect))
else:
raise RuntimeError(
"Unknown Recursion Case for expression={}, type(expression)={}"
.format(expression, type(expression)))
def _parse_expression(self,
expression,
qualifier=None) -> Union[str, list]:
if isinstance(expression, ComparisonExpression): # Base Case
# Resolve STIX Object Path to a field in the target Data Model
stix_object, stix_field = expression.object_path.split(':')
# Multiple data source fields may map to the same STIX Object
mapped_fields_array = self.dmm.map_field(stix_object, stix_field)
# Resolve the comparison symbol to use in the query string (usually just ':')
comparator = self._lookup_comparison_operator(
self, expression.comparator)
if stix_field in ['start', 'end', "time_observed"]:
transformer = TimestampToMilliseconds()
expression.value = transformer.transform(
expression.value) // 1000
# Some values are formatted differently based on how they're being compared
# should be (x, y, z, ...)
if expression.comparator == ComparisonComparators.In:
value = self._format_set(expression.value)
elif expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual:
# Should be in single-quotes
value = self._format_equality(expression.value)
else:
value = self._escape_value(expression.value)
comparison_string = self._parse_mapped_fields(
self, expression, value, comparator, stix_field,
mapped_fields_array)
# if(len(mapped_fields_array) > 1 and not self._is_reference_value(stix_field)):
# # More than one data source field maps to the STIX attribute, so group comparisons together.
# grouped_comparison_string = comparison_string
# comparison_string = grouped_comparison_string
if expression.negated:
comparison_string = self._negate_comparison(comparison_string)
if qualifier is not None:
return self._format_start_stop_qualifier(
comparison_string, qualifier)
else:
return "{}".format(comparison_string)
elif isinstance(expression, CombinedComparisonExpression):
operator = self._lookup_comparison_operator(
self, expression.operator)
expression_01 = self._parse_expression(expression.expr1)
expression_02 = self._parse_expression(expression.expr2)
if not expression_01 or not expression_02:
return ''
if isinstance(expression.expr1, CombinedComparisonExpression):
expression_01 = "{}".format(expression_01)
if isinstance(expression.expr2, CombinedComparisonExpression):
expression_02 = "{}".format(expression_02)
query_string = "{} {} {}".format(expression_01, operator,
expression_02)
if qualifier is not None:
return self._format_start_stop_qualifier(
query_string, qualifier)
else:
return "{}".format(query_string)
elif isinstance(expression, ObservationExpression):
return self._parse_expression(expression.comparison_expression,
qualifier)
elif hasattr(expression, 'qualifier') and hasattr(
expression, 'observation_expression'):
if isinstance(expression.observation_expression,
CombinedObservationExpression):
operator = self._lookup_comparison_operator(
self, expression.observation_expression.operator)
expression_01 = self._parse_expression(
expression.observation_expression.expr1)
# qualifier only needs to be passed into the parse expression once since it will be the same for both expressions
expression_02 = self._parse_expression(
expression.observation_expression.expr2,
expression.qualifier)
return "{} {} {}".format(expression_01, operator,
expression_02)
else:
return self._parse_expression(
expression.observation_expression.comparison_expression,
expression.qualifier)
elif isinstance(expression, CombinedObservationExpression):
queries = []
operator = self._lookup_comparison_operator(
self, expression.operator)
expression_01 = self._parse_expression(expression.expr1)
expression_02 = self._parse_expression(expression.expr2)
if not isinstance(expression_01, list):
queries.extend([expression_01])
if not isinstance(expression_02, list):
queries.extend([expression_02])
return queries
elif isinstance(expression, Pattern):
return self._parse_expression(expression.expression)
else:
raise RuntimeError(
"Unknown Recursion Case for expression={}, type(expression)={}"
.format(expression, type(expression)))
