def _parse_time_range(self, qualifier, time_range): """ Format the input time range UTC timestamp to Unix time :param time_range: int, value available from main.py in options variable :return: str, format_string """ format_string = '' try: compile_timestamp_regex = re.compile(TIMESTAMP_PATTERN) transformer = TimestampToMilliseconds() if qualifier and compile_timestamp_regex.search(qualifier): time_range_iterator = map( lambda x: int(transformer.transform(x.group()) / 1000), compile_timestamp_regex.finditer(qualifier)) # Default time range Start time = Now - 5 minutes and Stop time = Now else: stop_time = datetime.now() start_time = int( round((stop_time - timedelta(minutes=time_range)).timestamp())) stop_time = int(round(stop_time.timestamp())) time_range_iterator = [start_time, stop_time] self.time_range_lst.append([each for each in time_range_iterator]) return format_string except (KeyError, IndexError, TypeError) as e: raise e
def _convert_timestamps_to_milliseconds(query_parts): # grab time stamps from array start_time = _test_or_add_milliseconds(query_parts[2]) stop_time = _test_or_add_milliseconds(query_parts[4]) transformer = TimestampToMilliseconds() millisecond_start_time = transformer.transform(start_time) millisecond_stop_time = transformer.transform(stop_time) return query_parts[0] + " " + query_parts[1] + " " + str(millisecond_start_time) + " " + query_parts[3] + " " + str(millisecond_stop_time)
def _parse_comparison_expression(self, expression, qualifier=None): # Resolve STIX Object Path to a field in the target Data Model stix_object, stix_field = expression.object_path.split(':') # Multiple QRadar fields may map to the same STIX Object mapped_fields_array = self.dmm.map_field(stix_object, stix_field) # Resolve the comparison symbol to use in the query string (usually just ':') comparator = self._lookup_comparison_operator(self, expression.comparator) if stix_field == 'protocols[*]': map_data = _fetch_network_protocol_mapping() try: expression.value = map_data[expression.value.lower()] except Exception as protocol_key: raise KeyError("Network protocol {} is not supported.".format( protocol_key)) elif stix_field == 'start' or stix_field == 'end': transformer = TimestampToMilliseconds() expression.value = transformer.transform(expression.value) # Some values are formatted differently based on how they're being compared if expression.comparator == ComparisonComparators.Matches: # needs forward slashes value = self._format_match(expression.value) # should be (x, y, z, ...) elif expression.comparator == ComparisonComparators.In: value = self._format_set(expression.value) elif expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual: # Should be in single-quotes value = self._format_equality(expression.value) # '%' -> '*' wildcard, '_' -> '?' single wildcard elif expression.comparator == ComparisonComparators.Like and not ( expression.object_path == 'x-readable-payload:value'): value = self._format_like(expression.value) else: value = self._escape_value(expression.value) comparison_string = self._parse_mapped_fields(self, expression, value, comparator, stix_field, mapped_fields_array) if (len(mapped_fields_array) > 1 and not self._is_reference_value(stix_field)): # More than one AQL field maps to the STIX attribute so group the ORs. comparison_string = "({})".format(comparison_string) if expression.negated: comparison_string = self._negate_comparison(comparison_string) if qualifier: self.qualified_queries.append("{} limit {} {}".format( comparison_string, self.result_limit, qualifier)) return '' else: return "{}".format(comparison_string)
def _parse_comparison_expression(self, expression, qualifier=None, calling_object_type=None): # Resolve STIX Object Path to a field in the target Data Model stix_object, stix_field = expression.object_path.split(':') # Multiple QRadar fields may map to the same STIX Object mapped_fields_array = self.dmm.map_field(stix_object, stix_field) # Resolve the comparison symbol to use in the query string (usually just ':') comparator = self.comparator_lookup[expression.comparator] if stix_field == 'protocols[*]': map_data = _fetch_network_protocol_mapping() try: expression.value = map_data[expression.value.lower()] except Exception as protocol_key: raise KeyError("Network protocol {} is not supported.".format( protocol_key)) elif stix_field == 'start' or stix_field == 'end': transformer = TimestampToMilliseconds() expression.value = transformer.transform(expression.value) # Some values are formatted differently based on how they're being compared value = self._format_value(self, expression) comparison_string = self._parse_mapped_fields(self, expression, value, comparator, stix_field, mapped_fields_array) if (len(mapped_fields_array) > 1 and not self._is_reference_value(stix_field)): # More than one AQL field maps to the STIX attribute so group the ORs. comparison_string = "({})".format(comparison_string) if expression.comparator == ComparisonComparators.NotEqual: comparison_string = self._negate_comparison(comparison_string) if expression.negated: comparison_string = self._negate_comparison(comparison_string) if calling_object_type in [ 'CombinedObservationExpression', 'ObservationExpression', 'CombinedComparisonExpression' ]: return comparison_string elif qualifier is not None: self.qualified_queries.append("{} limit {} {}".format( comparison_string, self.result_limit, qualifier)) else: self.unqualified_queries.append(comparison_string) return comparison_string
def _format_datetime(value) -> list: """ Formating value in the event of value is datetime :param value: datetime value :return: list, timestamp in milliseconds or timestamp itself """ transformer = TimestampToMilliseconds() values = value.values if hasattr(value, 'values') else [value] milli_secs_lst = list(map(transformer.transform, values)) values = list(map(lambda x: '{}'.format(str(x)[:-3]), milli_secs_lst)) return values
def _parse_expression(self, expression, qualifier=None) -> str: if isinstance(expression, ComparisonExpression): # Base Case # Resolve STIX Object Path to a field in the target Data Model stix_object, stix_field = expression.object_path.split(':') # Multiple data source fields may map to the same STIX Object mapped_fields_array = self.dmm.map_field(stix_object, stix_field) # Resolve the comparison symbol to use in the query string (usually just ':') comparator = self._lookup_comparison_operator( self, expression.comparator) if stix_field == 'start' or stix_field == 'end': transformer = TimestampToMilliseconds() expression.value = transformer.transform(expression.value) # Some values are formatted differently based on how they're being compared if expression.comparator == ComparisonComparators.Matches: # needs forward slashes value = self._format_match(expression.value) # should be (x, y, z, ...) elif expression.comparator == ComparisonComparators.In: value = self._format_set(expression.value) elif expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual: # Should be in single-quotes value = self._format_equality(expression.value) # '%' -> '*' wildcard, '_' -> '?' single wildcard elif expression.comparator == ComparisonComparators.Like: value = self._format_like(expression.value) else: value = self._escape_value(expression.value) comparison_string = self._parse_mapped_fields( self, expression, value, comparator, stix_field, mapped_fields_array) if (len(mapped_fields_array) > 1 and not self._is_reference_value(stix_field)): # More than one data source field maps to the STIX attribute, so group comparisons together. grouped_comparison_string = "(" + comparison_string + ")" comparison_string = grouped_comparison_string if expression.negated: comparison_string = self._negate_comparison(comparison_string) if qualifier is not None: return "{} {}".format(comparison_string, qualifier) else: return "{}".format(comparison_string) elif isinstance(expression, CombinedComparisonExpression): operator = self._lookup_comparison_operator( self, expression.operator) expression_01 = self._parse_expression(expression.expr1) expression_02 = self._parse_expression(expression.expr2) if not expression_01 or not expression_02: return '' if isinstance(expression.expr1, CombinedComparisonExpression): expression_01 = "({})".format(expression_01) if isinstance(expression.expr2, CombinedComparisonExpression): expression_02 = "({})".format(expression_02) query_string = "{} {} {}".format(expression_01, operator, expression_02) if qualifier is not None: return "{} {}".format(query_string, qualifier) else: return "{}".format(query_string) elif isinstance(expression, ObservationExpression): return self._parse_expression(expression.comparison_expression, qualifier) elif hasattr(expression, 'qualifier') and hasattr( expression, 'observation_expression'): if isinstance(expression.observation_expression, CombinedObservationExpression): operator = self._lookup_comparison_operator( self, expression.observation_expression.operator) expression_01 = self._parse_expression( expression.observation_expression.expr1) # qualifier only needs to be passed into the parse expression once since it will be the same for both expressions expression_02 = self._parse_expression( expression.observation_expression.expr2, expression.qualifier) return "{} {} {}".format(expression_01, operator, expression_02) else: return self._parse_expression( expression.observation_expression.comparison_expression, expression.qualifier) elif isinstance(expression, CombinedObservationExpression): operator = self._lookup_comparison_operator( self, expression.operator) expression_01 = self._parse_expression(expression.expr1) expression_02 = self._parse_expression(expression.expr2) if expression_01 and expression_02: return "({}) {} ({})".format(expression_01, operator, expression_02) elif expression_01: return "{}".format(expression_01) elif expression_02: return "{}".format(expression_02) else: return '' elif isinstance(expression, Pattern): return "{expr}".format( expr=self._parse_expression(expression.expression)) else: raise RuntimeError( "Unknown Recursion Case for expression={}, type(expression)={}" .format(expression, type(expression)))
def _parse_expression(self, expression, qualifier=None) -> str: if isinstance(expression, ComparisonExpression): # Base Case # Resolve STIX Object Path to a field in the target Data Model stix_object, stix_field = expression.object_path.split(':') # Multiple QRadar fields may map to the same STIX Object mapped_fields_array = self.dmm.map_field(stix_object, stix_field) # Resolve the comparison symbol to use in the query string (usually just ':') comparator = self.comparator_lookup[expression.comparator] original_stix_value = expression.value if stix_field == 'protocols[*]': map_data = _fetch_network_protocol_mapping() try: expression.value = map_data[expression.value.lower()] except Exception as protocol_key: raise KeyError( "Network protocol {} is not supported.".format( protocol_key)) elif stix_field == 'start' or stix_field == 'end': transformer = TimestampToMilliseconds() # TODO Skydive uses seconds for timestamps, but this is something we should configure expression.value = int( transformer.transform(expression.value) / 1000) # Some values are formatted differently based on how they're being compared if expression.comparator == ComparisonComparators.Matches: # needs forward slashes value = self._format_match(expression.value) # should be (x, y, z, ...) elif expression.comparator == ComparisonComparators.In: value = self._format_set(expression.value) elif expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual: # Should be in single-quotes value = self._format_equality(expression.value) # '%' -> '*' wildcard, '_' -> '?' single wildcard elif expression.comparator == ComparisonComparators.Like: value = self._format_like(expression.value) else: value = self._escape_value(expression.value) self.parsed_pattern.append({ 'attribute': expression.object_path, 'comparison_operator': comparator, 'value': original_stix_value }) comparison_string = "" mapped_fields_count = len(mapped_fields_array) if len(mapped_fields_array) == 0: comparison_string += "false" for mapped_field in mapped_fields_array: comparison_string += "{mapped_field} {comparator} {value}".format( mapped_field=mapped_field, comparator=comparator, value=value) if (mapped_fields_count > 1): comparison_string += " OR " mapped_fields_count -= 1 if (len(mapped_fields_array) > 1): # More than one SQL field maps to the STIX attribute so group the ORs. grouped_comparison_string = "(" + comparison_string + ")" comparison_string = grouped_comparison_string if expression.comparator == ComparisonComparators.NotEqual: comparison_string = self._negate_comparison(comparison_string) if expression.negated: comparison_string = self._negate_comparison(comparison_string) if qualifier is not None: return "{comparison} {qualifier} split".format( comparison=comparison_string, qualifier=qualifier) else: return "{comparison}".format(comparison=comparison_string) elif isinstance(expression, CombinedComparisonExpression): query_string = "{} {} {}".format( self._parse_expression(expression.expr1), self.comparator_lookup[expression.operator], self._parse_expression(expression.expr2)) if qualifier is not None: return "{query_string} {qualifier} split".format( query_string=query_string, qualifier=qualifier) else: return "{query_string}".format(query_string=query_string) elif isinstance(expression, ObservationExpression): return self._parse_expression(expression.comparison_expression, qualifier) elif hasattr(expression, 'qualifier') and hasattr( expression, 'observation_expression'): if isinstance(expression.observation_expression, CombinedObservationExpression): operator = self.comparator_lookup[ expression.observation_expression.operator] # qualifier only needs to be passed into the parse expression once since it will be the same for both expressions return "{expr1} {operator} {expr2}".format( expr1=self._parse_expression( expression.observation_expression.expr1), operator=operator, expr2=self._parse_expression( expression.observation_expression.expr2, expression.qualifier)) else: return self._parse_expression( expression.observation_expression.comparison_expression, expression.qualifier) elif isinstance(expression, CombinedObservationExpression): operator = self.comparator_lookup[expression.operator] return "{expr1} {operator} {expr2}".format( expr1=self._parse_expression(expression.expr1), operator=operator, expr2=self._parse_expression(expression.expr2)) elif isinstance(expression, Pattern): return "{expr}".format( expr=self._parse_expression(expression.expression)) else: raise RuntimeError( "Unknown Recursion Case for expression={}, type(expression)={}" .format(expression, type(expression)))