def _convert_timestamps_to_milliseconds(query_parts): # grab time stamps from array start_time = _test_or_add_milliseconds(query_parts[2]) stop_time = _test_or_add_milliseconds(query_parts[4]) transformer = TimestampToMilliseconds() millisecond_start_time = transformer.transform(start_time) millisecond_stop_time = transformer.transform(stop_time) return query_parts[0] + " " + query_parts[1] + " " + str(millisecond_start_time) + " " + query_parts[3] + " " + str(millisecond_stop_time)
def _format_start_stop_qualifier(self, expression, qualifier) -> str: """Convert a STIX start stop qualifier into a query string. """ transformer = TimestampToMilliseconds() qualifier_split = qualifier.split("'") start = transformer.transform(qualifier_split[1]) // 1000 stop = transformer.transform(qualifier_split[3]) // 1000 qualified_query = "%s AND 'start' : %s AND 'end' : %s" % (expression, start, stop) return qualified_query
def _format_start_stop_qualifier(self, expression, qualifier) -> str: """Convert a STIX start stop qualifier into a query string. The sample MySQL schema included in this connector defines a timerange with a start and stop value based on the entry_time field. """ transformer = TimestampToMilliseconds() qualifier_split = qualifier.split("'") start = transformer.transform(qualifier_split[1]) stop = transformer.transform(qualifier_split[3]) qualified_query = "%s AND (entry_time >= %s OR entry_time <= %s)" % ( expression, start, stop) return qualified_query
def _convert_timestamps_to_milliseconds(query_parts): # grab time stamps from array start_time = _test_or_add_milliseconds(query_parts[2]) stop_time = _test_or_add_milliseconds(query_parts[4]) transformer = TimestampToMilliseconds() millisecond_start_time = transformer.transform(start_time) millisecond_stop_time = transformer.transform(stop_time) payload = dict() payload['offset'] = 0 payload['from'] = millisecond_start_time payload['to'] = millisecond_stop_time payload['query'] = query_parts[0] return payload
def _format_start_stop_qualifier(self, expression, qualifier) -> str: """ Convert a STIX start stop qualifier into a query string. """ transformer = TimestampToMilliseconds() qualifier_split = qualifier.split("'") start = qualifier_split[1] stop = qualifier_split[3] # convert timepestamp to millisecond which will be passed to rest service start_epoach = transformer.transform(start) stop_epoach = transformer.transform(stop) qualified_query = "%s&from=%s&to=%s" % (expression, start_epoach, stop_epoach) return qualified_query
def _parse_time_range(self, qualifier, time_range): """ Format the input time range UTC timestamp to Unix time :param time_range: int, value available from main.py in options variable :return: str, format_string """ format_string = '' try: compile_timestamp_regex = re.compile(TIMESTAMP_PATTERN) transformer = TimestampToMilliseconds() if qualifier and compile_timestamp_regex.search(qualifier): time_range_iterator = map( lambda x: int(transformer.transform(x.group()) / 1000), compile_timestamp_regex.finditer(qualifier)) # Default time range Start time = Now - 5 minutes and Stop time = Now else: stop_time = datetime.now() start_time = int( round((stop_time - timedelta(minutes=time_range)).timestamp())) stop_time = int(round(stop_time.timestamp())) time_range_iterator = [start_time, stop_time] self.time_range_lst.append([each for each in time_range_iterator]) return format_string except (KeyError, IndexError, TypeError) as e: raise e
def _parse_comparison_expression(self, expression, qualifier=None): # Resolve STIX Object Path to a field in the target Data Model stix_object, stix_field = expression.object_path.split(':') # Multiple QRadar fields may map to the same STIX Object mapped_fields_array = self.dmm.map_field(stix_object, stix_field) # Resolve the comparison symbol to use in the query string (usually just ':') comparator = self._lookup_comparison_operator(self, expression.comparator) # Special case for artifact:payload_bin object with Like operator where we apply aql TEXT SEARCH if expression.comparator == ComparisonComparators.Like and ( expression.object_path == 'artifact:payload_bin'): return "TEXT SEARCH '{}'".format(expression.value) # Special case where we want the risk finding if stix_object == 'x-ibm-finding' and stix_field == 'name' and expression.value == "*": return "devicetype = 18" if stix_field == 'protocols[*]': map_data = read_json('network_protocol_map', self.options) try: expression.value = map_data[expression.value.lower()] except Exception as protocol_key: raise KeyError("Network protocol {} is not supported.".format( protocol_key)) elif stix_field == 'start' or stix_field == 'end': transformer = TimestampToMilliseconds() expression.value = transformer.transform(expression.value) # Some values are formatted differently based on how they're being compared if expression.comparator == ComparisonComparators.Matches: # needs forward slashes value = self._format_match(expression.value) # should be (x, y, z, ...) elif expression.comparator == ComparisonComparators.In: value = self._format_in(expression.value) elif expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual: # Should be in single-quotes value = self._format_equality(expression.value) # '%' -> '*' wildcard, '_' -> '?' single wildcard elif expression.comparator == ComparisonComparators.Like and not ( expression.object_path == 'artifact:payload_bin'): value = self._format_like(expression.value) else: value = self._escape_value(expression.value) comparison_string = self._parse_mapped_fields(self, expression, value, comparator, stix_field, mapped_fields_array) if (len(mapped_fields_array) > 1 and not self._is_reference_value(stix_field)): # More than one AQL field maps to the STIX attribute so group the ORs. comparison_string = "({})".format(comparison_string) if expression.negated: comparison_string = self._negate_comparison(comparison_string) if qualifier: self.qualified_queries.append("{} limit {} {}".format( comparison_string, self.result_limit, qualifier)) return '' else: return "{}".format(comparison_string)
def _parse_expression(self, expression, qualifier=None) -> str: if isinstance(expression, ComparisonExpression): # Base Case # Resolve STIX Object Path to a field in the target Data Model stix_object, stix_field = expression.object_path.split(':') # Multiple data source fields may map to the same STIX Object mapped_fields_array = self.dmm.map_field(stix_object, stix_field) # Resolve the comparison symbol to use in the query string (usually just ':') comparator = self._lookup_comparison_operator( self, expression.comparator) if stix_field == 'start' or stix_field == 'end': transformer = TimestampToMilliseconds() expression.value = transformer.transform(expression.value) # Some values are formatted differently based on how they're being compared if expression.comparator == ComparisonComparators.Matches: # needs forward slashes value = self._format_match(expression.value) elif expression.comparator == ComparisonComparators.Equal or \ expression.comparator == ComparisonComparators.NotEqual: # Should be in single-quotes value = self._format_equality(expression.value) # '%' -> '*' wildcard, '_' -> '?' single wildcard elif expression.comparator == ComparisonComparators.Like: value = self._format_like(expression.value) else: value = self._escape_value(expression.value) comparison_string = self._parse_mapped_fields( self, expression, value, comparator, stix_field, mapped_fields_array) if len(mapped_fields_array) > 1 and not self._is_reference_value( stix_field): # More than one data source field maps to the STIX attribute, so group comparisons together. grouped_comparison_string = "(" + comparison_string + ")" comparison_string = grouped_comparison_string if expression.negated: comparison_string = self._negate_comparison(comparison_string) if qualifier is not None: return "{} {}".format(comparison_string, qualifier) else: return "{}".format(comparison_string) elif isinstance(expression, CombinedComparisonExpression): operator = self._lookup_comparison_operator( self, expression.operator) expression_01 = self._parse_expression(expression.expr1) expression_02 = self._parse_expression(expression.expr2) if not expression_01 or not expression_02: return '' if isinstance(expression.expr1, CombinedComparisonExpression): expression_01 = "({})".format(expression_01) if isinstance(expression.expr2, CombinedComparisonExpression): expression_02 = "({})".format(expression_02) query_string = "{} {} {}".format(expression_01, operator, expression_02) if qualifier is not None: return "{} {}".format(query_string, qualifier) else: return "{}".format(query_string) elif isinstance(expression, ObservationExpression): return self._parse_expression(expression.comparison_expression, qualifier) elif hasattr(expression, 'qualifier') and hasattr( expression, 'observation_expression'): if isinstance(expression.observation_expression, CombinedObservationExpression): operator = self._lookup_comparison_operator( self, expression.observation_expression.operator) expression_01 = self._parse_expression( expression.observation_expression.expr1) # qualifier only needs to be passed into the parse expression once since it will be the same for both # expressions expression_02 = self._parse_expression( expression.observation_expression.expr2, expression.qualifier) return "{} {} {}".format(expression_01, operator, expression_02) else: return self._parse_expression( expression.observation_expression.comparison_expression, expression.qualifier) elif isinstance(expression, CombinedObservationExpression): operator = self._lookup_comparison_operator( self, expression.operator) expression_01 = self._parse_expression(expression.expr1) expression_02 = self._parse_expression(expression.expr2) if expression_01 and expression_02: return "({}) {} ({})".format(expression_01, operator, expression_02) elif expression_01: return "{}".format(expression_01) elif expression_02: return "{}".format(expression_02) else: return '' elif isinstance(expression, Pattern): return "{expr}".format( expr=self._parse_expression(expression.expression)) else: raise RuntimeError( "Unknown Recursion Case for expression={}, type(expression)={}" .format(expression, type(expression)))
def _parse_expression(self, expression, qualifier=None) -> str: if isinstance(expression, ComparisonExpression): # Base Case # Resolve STIX Object Path to a field in the target Data Model stix_object, stix_field = expression.object_path.split(':') # Multiple QRadar fields may map to the same STIX Object mapped_fields_array = self.dmm.map_field(stix_object, stix_field) # Resolve the comparison symbol to use in the query string (usually just ':') comparator = self.comparator_lookup[expression.comparator] original_stix_value = expression.value if stix_field == 'protocols[*]': map_data = _fetch_network_protocol_mapping() try: expression.value = map_data[expression.value.lower()] except Exception as protocol_key: raise KeyError( "Network protocol {} is not supported.".format( protocol_key)) elif stix_field == 'start' or stix_field == 'end': transformer = TimestampToMilliseconds() # TODO Skydive uses seconds for timestamps, but this is something we should configure expression.value = int( transformer.transform(expression.value) / 1000) # Some values are formatted differently based on how they're being compared if expression.comparator == ComparisonComparators.Matches: # needs forward slashes value = self._format_match(expression.value) # should be (x, y, z, ...) elif expression.comparator == ComparisonComparators.In: value = self._format_set(expression.value) elif expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual: # Should be in single-quotes value = self._format_equality(expression.value) # '%' -> '*' wildcard, '_' -> '?' single wildcard elif expression.comparator == ComparisonComparators.Like: value = self._format_like(expression.value) else: value = self._escape_value(expression.value) comparison_string = "" mapped_fields_count = len(mapped_fields_array) if len(mapped_fields_array) == 0: comparison_string += "false" for mapped_field in mapped_fields_array: comparison_string += "{mapped_field} {comparator} {value}".format( mapped_field=mapped_field, comparator=comparator, value=value) if (mapped_fields_count > 1): comparison_string += " OR " mapped_fields_count -= 1 if (len(mapped_fields_array) > 1): # More than one SQL field maps to the STIX attribute so group the ORs. grouped_comparison_string = "(" + comparison_string + ")" comparison_string = grouped_comparison_string if expression.comparator == ComparisonComparators.NotEqual: comparison_string = self._negate_comparison(comparison_string) if expression.negated: comparison_string = self._negate_comparison(comparison_string) if qualifier is not None: return "{comparison} {qualifier} split".format( comparison=comparison_string, qualifier=qualifier) else: return "{comparison}".format(comparison=comparison_string) elif isinstance(expression, CombinedComparisonExpression): query_string = "{} {} {}".format( self._parse_expression(expression.expr1), self.comparator_lookup[expression.operator], self._parse_expression(expression.expr2)) if qualifier is not None: return "{query_string} {qualifier} split".format( query_string=query_string, qualifier=qualifier) else: return "{query_string}".format(query_string=query_string) elif isinstance(expression, ObservationExpression): return self._parse_expression(expression.comparison_expression, qualifier) elif hasattr(expression, 'qualifier') and hasattr( expression, 'observation_expression'): if isinstance(expression.observation_expression, CombinedObservationExpression): operator = self.comparator_lookup[ expression.observation_expression.operator] # qualifier only needs to be passed into the parse expression once since it will be the same for both expressions return "{expr1} {operator} {expr2}".format( expr1=self._parse_expression( expression.observation_expression.expr1), operator=operator, expr2=self._parse_expression( expression.observation_expression.expr2, expression.qualifier)) else: return self._parse_expression( expression.observation_expression.comparison_expression, expression.qualifier) elif isinstance(expression, CombinedObservationExpression): operator = self.comparator_lookup[expression.operator] return "{expr1} {operator} {expr2}".format( expr1=self._parse_expression(expression.expr1), operator=operator, expr2=self._parse_expression(expression.expr2)) elif isinstance(expression, Pattern): return "{expr}".format( expr=self._parse_expression(expression.expression)) else: raise RuntimeError( "Unknown Recursion Case for expression={}, type(expression)={}" .format(expression, type(expression)))
def _format_timestamp(self, value): transformer = TimestampToMilliseconds() value = re.sub("'", "", value) return transformer.transform(value)
def _parse_expression(self, expression, dialect, qualifier=None) -> str: if isinstance(expression, ComparisonExpression): # Base Case # Resolve STIX Object Path to a field in the target Data Model stix_object, stix_field = expression.object_path.split(':') # Multiple data source fields may map to the same STIX Object mapped_fields_array = self.dmm.map_field(stix_object, stix_field) if dialect == 'messageActivityData': mapped_fields_set = set(mapped_fields_array) intersection = self.assigned_fields.intersection( mapped_fields_set) if intersection: logger.error( f"[{', '.join(intersection)}] mapped from {stix_field} has multiple criteria" ) raise NotImplementedError( "Multiple criteria for one field is not support in MDL" ) else: self.assigned_fields |= mapped_fields_set # Resolve the comparison symbol to use in the query string (usually just ':') comparator = self._lookup_comparison_operator( expression.comparator, dialect) if stix_field in ('start', 'end'): transformer = TimestampToMilliseconds() expression.value = transformer.transform(expression.value) # Some values are formatted differently based on how they're being compared if expression.comparator == ComparisonComparators.Equal: # Should be in single-quotes value = self._format_equality(expression.value) elif expression.comparator == ComparisonComparators.NotEqual: value = self._format_equality(expression.value) expression.negated = True # '%' -> '*' wildcard, '_' -> '?' single wildcard elif expression.comparator == ComparisonComparators.Like: value = self._format_like(expression.value) else: value = self._escape_value(expression.value) comparison_string = self._parse_mapped_fields( expression, value, comparator, stix_field, mapped_fields_array) if len(mapped_fields_array) > 1 and not self._is_reference_value( stix_field): # More than one data source field maps to the STIX attribute, so group comparisons together. grouped_comparison_string = "(" + comparison_string + ")" comparison_string = grouped_comparison_string if expression.negated: comparison_string = self._negate_comparison(comparison_string) if qualifier is not None: return "{} {}".format(comparison_string, qualifier) else: return "{}".format(comparison_string) elif isinstance(expression, CombinedComparisonExpression): operator = self._lookup_comparison_operator( expression.operator, dialect) expression_01 = self._parse_expression(expression.expr1, dialect) expression_02 = self._parse_expression(expression.expr2, dialect) if not expression_01 or not expression_02: return '' if isinstance(expression.expr1, CombinedComparisonExpression): expression_01 = "({})".format(expression_01) if isinstance(expression.expr2, CombinedComparisonExpression): expression_02 = "({})".format(expression_02) query_string = "{} {} {}".format(expression_01, operator, expression_02) if qualifier is not None: return "{} {}".format(query_string, qualifier) else: return "{}".format(query_string) elif isinstance(expression, ObservationExpression): return self._parse_expression(expression.comparison_expression, dialect, qualifier) elif hasattr(expression, 'qualifier') and hasattr( expression, 'observation_expression'): if isinstance(expression.observation_expression, CombinedObservationExpression): operator = self._lookup_comparison_operator( expression.observation_expression.operator, dialect) expression_01 = self._parse_expression( expression.observation_expression.expr1, dialect) # qualifier only needs to be passed into the parse expression once since it will be the same for both expressions expression_02 = self._parse_expression( expression.observation_expression.expr2, dialect, expression.qualifier) return "{} {} {}".format(expression_01, operator, expression_02) else: return self._parse_expression( expression.observation_expression.comparison_expression, dialect, expression.qualifier) elif isinstance(expression, CombinedObservationExpression): operator = self._lookup_comparison_operator( expression.operator, dialect) expression_01 = self._parse_expression(expression.expr1, dialect) expression_02 = self._parse_expression(expression.expr2, dialect) if expression_01 and expression_02: return "({}) {} ({})".format(expression_01, operator, expression_02) elif expression_01: return "{}".format(expression_01) elif expression_02: return "{}".format(expression_02) else: return '' elif isinstance(expression, Pattern): return "{expr}".format( expr=self._parse_expression(expression.expression, dialect)) else: raise RuntimeError( "Unknown Recursion Case for expression={}, type(expression)={}" .format(expression, type(expression)))
def _parse_expression(self, expression, qualifier=None) -> Union[str, list]: if isinstance(expression, ComparisonExpression): # Base Case # Resolve STIX Object Path to a field in the target Data Model stix_object, stix_field = expression.object_path.split(':') # Multiple data source fields may map to the same STIX Object mapped_fields_array = self.dmm.map_field(stix_object, stix_field) # Resolve the comparison symbol to use in the query string (usually just ':') comparator = self._lookup_comparison_operator( self, expression.comparator) if stix_field in ['start', 'end', "time_observed"]: transformer = TimestampToMilliseconds() expression.value = transformer.transform( expression.value) // 1000 # Some values are formatted differently based on how they're being compared # should be (x, y, z, ...) if expression.comparator == ComparisonComparators.In: value = self._format_set(expression.value) elif expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual: # Should be in single-quotes value = self._format_equality(expression.value) else: value = self._escape_value(expression.value) comparison_string = self._parse_mapped_fields( self, expression, value, comparator, stix_field, mapped_fields_array) # if(len(mapped_fields_array) > 1 and not self._is_reference_value(stix_field)): # # More than one data source field maps to the STIX attribute, so group comparisons together. # grouped_comparison_string = comparison_string # comparison_string = grouped_comparison_string if expression.negated: comparison_string = self._negate_comparison(comparison_string) if qualifier is not None: return self._format_start_stop_qualifier( comparison_string, qualifier) else: return "{}".format(comparison_string) elif isinstance(expression, CombinedComparisonExpression): operator = self._lookup_comparison_operator( self, expression.operator) expression_01 = self._parse_expression(expression.expr1) expression_02 = self._parse_expression(expression.expr2) if not expression_01 or not expression_02: return '' if isinstance(expression.expr1, CombinedComparisonExpression): expression_01 = "{}".format(expression_01) if isinstance(expression.expr2, CombinedComparisonExpression): expression_02 = "{}".format(expression_02) query_string = "{} {} {}".format(expression_01, operator, expression_02) if qualifier is not None: return self._format_start_stop_qualifier( query_string, qualifier) else: return "{}".format(query_string) elif isinstance(expression, ObservationExpression): return self._parse_expression(expression.comparison_expression, qualifier) elif hasattr(expression, 'qualifier') and hasattr( expression, 'observation_expression'): if isinstance(expression.observation_expression, CombinedObservationExpression): operator = self._lookup_comparison_operator( self, expression.observation_expression.operator) expression_01 = self._parse_expression( expression.observation_expression.expr1) # qualifier only needs to be passed into the parse expression once since it will be the same for both expressions expression_02 = self._parse_expression( expression.observation_expression.expr2, expression.qualifier) return "{} {} {}".format(expression_01, operator, expression_02) else: return self._parse_expression( expression.observation_expression.comparison_expression, expression.qualifier) elif isinstance(expression, CombinedObservationExpression): queries = [] operator = self._lookup_comparison_operator( self, expression.operator) expression_01 = self._parse_expression(expression.expr1) expression_02 = self._parse_expression(expression.expr2) if not isinstance(expression_01, list): queries.extend([expression_01]) if not isinstance(expression_02, list): queries.extend([expression_02]) return queries elif isinstance(expression, Pattern): return self._parse_expression(expression.expression) else: raise RuntimeError( "Unknown Recursion Case for expression={}, type(expression)={}" .format(expression, type(expression)))