Ejemplo n.º 1
0
 def _parse_time_range(self, qualifier, time_range):
     """
     Format the input time range UTC timestamp to Unix time
     :param time_range: int, value available from main.py in options variable
     :return: str, format_string
     """
     format_string = ''
     try:
         compile_timestamp_regex = re.compile(TIMESTAMP_PATTERN)
         transformer = TimestampToMilliseconds()
         if qualifier and compile_timestamp_regex.search(qualifier):
             time_range_iterator = map(
                 lambda x: int(transformer.transform(x.group()) / 1000),
                 compile_timestamp_regex.finditer(qualifier))
         # Default time range Start time = Now - 5 minutes and Stop time  = Now
         else:
             stop_time = datetime.now()
             start_time = int(
                 round((stop_time -
                        timedelta(minutes=time_range)).timestamp()))
             stop_time = int(round(stop_time.timestamp()))
             time_range_iterator = [start_time, stop_time]
         self.time_range_lst.append([each for each in time_range_iterator])
         return format_string
     except (KeyError, IndexError, TypeError) as e:
         raise e
Ejemplo n.º 2
0
def _convert_timestamps_to_milliseconds(query_parts):
    # grab time stamps from array
    start_time = _test_or_add_milliseconds(query_parts[2])
    stop_time = _test_or_add_milliseconds(query_parts[4])
    transformer = TimestampToMilliseconds()
    millisecond_start_time = transformer.transform(start_time)
    millisecond_stop_time = transformer.transform(stop_time)
    return query_parts[0] + " " + query_parts[1] + " " + str(millisecond_start_time) + " " + query_parts[3] + " " + str(millisecond_stop_time)
Ejemplo n.º 3
0
    def _parse_comparison_expression(self, expression, qualifier=None):
        # Resolve STIX Object Path to a field in the target Data Model
        stix_object, stix_field = expression.object_path.split(':')
        # Multiple QRadar fields may map to the same STIX Object
        mapped_fields_array = self.dmm.map_field(stix_object, stix_field)
        # Resolve the comparison symbol to use in the query string (usually just ':')
        comparator = self._lookup_comparison_operator(self,
                                                      expression.comparator)

        if stix_field == 'protocols[*]':
            map_data = _fetch_network_protocol_mapping()
            try:
                expression.value = map_data[expression.value.lower()]
            except Exception as protocol_key:
                raise KeyError("Network protocol {} is not supported.".format(
                    protocol_key))
        elif stix_field == 'start' or stix_field == 'end':
            transformer = TimestampToMilliseconds()
            expression.value = transformer.transform(expression.value)

        # Some values are formatted differently based on how they're being compared
        if expression.comparator == ComparisonComparators.Matches:  # needs forward slashes
            value = self._format_match(expression.value)
        # should be (x, y, z, ...)
        elif expression.comparator == ComparisonComparators.In:
            value = self._format_set(expression.value)
        elif expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual:
            # Should be in single-quotes
            value = self._format_equality(expression.value)
        # '%' -> '*' wildcard, '_' -> '?' single wildcard
        elif expression.comparator == ComparisonComparators.Like and not (
                expression.object_path == 'x-readable-payload:value'):
            value = self._format_like(expression.value)
        else:
            value = self._escape_value(expression.value)

        comparison_string = self._parse_mapped_fields(self, expression, value,
                                                      comparator, stix_field,
                                                      mapped_fields_array)

        if (len(mapped_fields_array) > 1
                and not self._is_reference_value(stix_field)):
            # More than one AQL field maps to the STIX attribute so group the ORs.
            comparison_string = "({})".format(comparison_string)
        if expression.negated:
            comparison_string = self._negate_comparison(comparison_string)
        if qualifier:
            self.qualified_queries.append("{} limit {} {}".format(
                comparison_string, self.result_limit, qualifier))
            return ''
        else:
            return "{}".format(comparison_string)
    def _parse_comparison_expression(self,
                                     expression,
                                     qualifier=None,
                                     calling_object_type=None):
        # Resolve STIX Object Path to a field in the target Data Model
        stix_object, stix_field = expression.object_path.split(':')
        # Multiple QRadar fields may map to the same STIX Object
        mapped_fields_array = self.dmm.map_field(stix_object, stix_field)
        # Resolve the comparison symbol to use in the query string (usually just ':')
        comparator = self.comparator_lookup[expression.comparator]

        if stix_field == 'protocols[*]':
            map_data = _fetch_network_protocol_mapping()
            try:
                expression.value = map_data[expression.value.lower()]
            except Exception as protocol_key:
                raise KeyError("Network protocol {} is not supported.".format(
                    protocol_key))
        elif stix_field == 'start' or stix_field == 'end':
            transformer = TimestampToMilliseconds()
            expression.value = transformer.transform(expression.value)

        # Some values are formatted differently based on how they're being compared
        value = self._format_value(self, expression)
        comparison_string = self._parse_mapped_fields(self, expression, value,
                                                      comparator, stix_field,
                                                      mapped_fields_array)

        if (len(mapped_fields_array) > 1
                and not self._is_reference_value(stix_field)):
            # More than one AQL field maps to the STIX attribute so group the ORs.
            comparison_string = "({})".format(comparison_string)
        if expression.comparator == ComparisonComparators.NotEqual:
            comparison_string = self._negate_comparison(comparison_string)
        if expression.negated:
            comparison_string = self._negate_comparison(comparison_string)
        if calling_object_type in [
                'CombinedObservationExpression', 'ObservationExpression',
                'CombinedComparisonExpression'
        ]:
            return comparison_string
        elif qualifier is not None:
            self.qualified_queries.append("{} limit {} {}".format(
                comparison_string, self.result_limit, qualifier))
        else:
            self.unqualified_queries.append(comparison_string)
            return comparison_string
Ejemplo n.º 5
0
 def _format_datetime(value) -> list:
     """
     Formating value in the event of value is datetime
     :param value: datetime value
     :return: list, timestamp in milliseconds or timestamp itself
     """
     transformer = TimestampToMilliseconds()
     values = value.values if hasattr(value, 'values') else [value]
     milli_secs_lst = list(map(transformer.transform, values))
     values = list(map(lambda x: '{}'.format(str(x)[:-3]), milli_secs_lst))
     return values
Ejemplo n.º 6
0
    def _parse_expression(self, expression, qualifier=None) -> str:
        if isinstance(expression, ComparisonExpression):  # Base Case
            # Resolve STIX Object Path to a field in the target Data Model
            stix_object, stix_field = expression.object_path.split(':')
            # Multiple data source fields may map to the same STIX Object
            mapped_fields_array = self.dmm.map_field(stix_object, stix_field)
            # Resolve the comparison symbol to use in the query string (usually just ':')
            comparator = self._lookup_comparison_operator(
                self, expression.comparator)

            if stix_field == 'start' or stix_field == 'end':
                transformer = TimestampToMilliseconds()
                expression.value = transformer.transform(expression.value)

            # Some values are formatted differently based on how they're being compared
            if expression.comparator == ComparisonComparators.Matches:  # needs forward slashes
                value = self._format_match(expression.value)
            # should be (x, y, z, ...)
            elif expression.comparator == ComparisonComparators.In:
                value = self._format_set(expression.value)
            elif expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual:
                # Should be in single-quotes
                value = self._format_equality(expression.value)
            # '%' -> '*' wildcard, '_' -> '?' single wildcard
            elif expression.comparator == ComparisonComparators.Like:
                value = self._format_like(expression.value)
            else:
                value = self._escape_value(expression.value)

            comparison_string = self._parse_mapped_fields(
                self, expression, value, comparator, stix_field,
                mapped_fields_array)
            if (len(mapped_fields_array) > 1
                    and not self._is_reference_value(stix_field)):
                # More than one data source field maps to the STIX attribute, so group comparisons together.
                grouped_comparison_string = "(" + comparison_string + ")"
                comparison_string = grouped_comparison_string

            if expression.negated:
                comparison_string = self._negate_comparison(comparison_string)
            if qualifier is not None:
                return "{} {}".format(comparison_string, qualifier)
            else:
                return "{}".format(comparison_string)

        elif isinstance(expression, CombinedComparisonExpression):
            operator = self._lookup_comparison_operator(
                self, expression.operator)
            expression_01 = self._parse_expression(expression.expr1)
            expression_02 = self._parse_expression(expression.expr2)
            if not expression_01 or not expression_02:
                return ''
            if isinstance(expression.expr1, CombinedComparisonExpression):
                expression_01 = "({})".format(expression_01)
            if isinstance(expression.expr2, CombinedComparisonExpression):
                expression_02 = "({})".format(expression_02)
            query_string = "{} {} {}".format(expression_01, operator,
                                             expression_02)
            if qualifier is not None:
                return "{} {}".format(query_string, qualifier)
            else:
                return "{}".format(query_string)
        elif isinstance(expression, ObservationExpression):
            return self._parse_expression(expression.comparison_expression,
                                          qualifier)
        elif hasattr(expression, 'qualifier') and hasattr(
                expression, 'observation_expression'):
            if isinstance(expression.observation_expression,
                          CombinedObservationExpression):
                operator = self._lookup_comparison_operator(
                    self, expression.observation_expression.operator)
                expression_01 = self._parse_expression(
                    expression.observation_expression.expr1)
                # qualifier only needs to be passed into the parse expression once since it will be the same for both expressions
                expression_02 = self._parse_expression(
                    expression.observation_expression.expr2,
                    expression.qualifier)
                return "{} {} {}".format(expression_01, operator,
                                         expression_02)
            else:
                return self._parse_expression(
                    expression.observation_expression.comparison_expression,
                    expression.qualifier)
        elif isinstance(expression, CombinedObservationExpression):
            operator = self._lookup_comparison_operator(
                self, expression.operator)
            expression_01 = self._parse_expression(expression.expr1)
            expression_02 = self._parse_expression(expression.expr2)
            if expression_01 and expression_02:
                return "({}) {} ({})".format(expression_01, operator,
                                             expression_02)
            elif expression_01:
                return "{}".format(expression_01)
            elif expression_02:
                return "{}".format(expression_02)
            else:
                return ''
        elif isinstance(expression, Pattern):
            return "{expr}".format(
                expr=self._parse_expression(expression.expression))
        else:
            raise RuntimeError(
                "Unknown Recursion Case for expression={}, type(expression)={}"
                .format(expression, type(expression)))
Ejemplo n.º 7
0
    def _parse_expression(self, expression, qualifier=None) -> str:
        if isinstance(expression, ComparisonExpression):  # Base Case
            # Resolve STIX Object Path to a field in the target Data Model
            stix_object, stix_field = expression.object_path.split(':')
            # Multiple QRadar fields may map to the same STIX Object
            mapped_fields_array = self.dmm.map_field(stix_object, stix_field)
            # Resolve the comparison symbol to use in the query string (usually just ':')
            comparator = self.comparator_lookup[expression.comparator]
            original_stix_value = expression.value

            if stix_field == 'protocols[*]':
                map_data = _fetch_network_protocol_mapping()
                try:
                    expression.value = map_data[expression.value.lower()]
                except Exception as protocol_key:
                    raise KeyError(
                        "Network protocol {} is not supported.".format(
                            protocol_key))
            elif stix_field == 'start' or stix_field == 'end':
                transformer = TimestampToMilliseconds()
                # TODO Skydive uses seconds for timestamps, but this is something we should configure
                expression.value = int(
                    transformer.transform(expression.value) / 1000)

            # Some values are formatted differently based on how they're being compared
            if expression.comparator == ComparisonComparators.Matches:  # needs forward slashes
                value = self._format_match(expression.value)
            # should be (x, y, z, ...)
            elif expression.comparator == ComparisonComparators.In:
                value = self._format_set(expression.value)
            elif expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual:
                # Should be in single-quotes
                value = self._format_equality(expression.value)
            # '%' -> '*' wildcard, '_' -> '?' single wildcard
            elif expression.comparator == ComparisonComparators.Like:
                value = self._format_like(expression.value)
            else:
                value = self._escape_value(expression.value)

            self.parsed_pattern.append({
                'attribute': expression.object_path,
                'comparison_operator': comparator,
                'value': original_stix_value
            })

            comparison_string = ""
            mapped_fields_count = len(mapped_fields_array)
            if len(mapped_fields_array) == 0:
                comparison_string += "false"
            for mapped_field in mapped_fields_array:
                comparison_string += "{mapped_field} {comparator} {value}".format(
                    mapped_field=mapped_field,
                    comparator=comparator,
                    value=value)
                if (mapped_fields_count > 1):
                    comparison_string += " OR "
                    mapped_fields_count -= 1

            if (len(mapped_fields_array) > 1):
                # More than one SQL field maps to the STIX attribute so group the ORs.
                grouped_comparison_string = "(" + comparison_string + ")"
                comparison_string = grouped_comparison_string

            if expression.comparator == ComparisonComparators.NotEqual:
                comparison_string = self._negate_comparison(comparison_string)

            if expression.negated:
                comparison_string = self._negate_comparison(comparison_string)
            if qualifier is not None:
                return "{comparison} {qualifier} split".format(
                    comparison=comparison_string, qualifier=qualifier)
            else:
                return "{comparison}".format(comparison=comparison_string)

        elif isinstance(expression, CombinedComparisonExpression):
            query_string = "{} {} {}".format(
                self._parse_expression(expression.expr1),
                self.comparator_lookup[expression.operator],
                self._parse_expression(expression.expr2))
            if qualifier is not None:
                return "{query_string} {qualifier} split".format(
                    query_string=query_string, qualifier=qualifier)
            else:
                return "{query_string}".format(query_string=query_string)
        elif isinstance(expression, ObservationExpression):
            return self._parse_expression(expression.comparison_expression,
                                          qualifier)
        elif hasattr(expression, 'qualifier') and hasattr(
                expression, 'observation_expression'):
            if isinstance(expression.observation_expression,
                          CombinedObservationExpression):
                operator = self.comparator_lookup[
                    expression.observation_expression.operator]
                # qualifier only needs to be passed into the parse expression once since it will be the same for both expressions
                return "{expr1} {operator} {expr2}".format(
                    expr1=self._parse_expression(
                        expression.observation_expression.expr1),
                    operator=operator,
                    expr2=self._parse_expression(
                        expression.observation_expression.expr2,
                        expression.qualifier))
            else:
                return self._parse_expression(
                    expression.observation_expression.comparison_expression,
                    expression.qualifier)
        elif isinstance(expression, CombinedObservationExpression):
            operator = self.comparator_lookup[expression.operator]
            return "{expr1} {operator} {expr2}".format(
                expr1=self._parse_expression(expression.expr1),
                operator=operator,
                expr2=self._parse_expression(expression.expr2))
        elif isinstance(expression, Pattern):
            return "{expr}".format(
                expr=self._parse_expression(expression.expression))
        else:
            raise RuntimeError(
                "Unknown Recursion Case for expression={}, type(expression)={}"
                .format(expression, type(expression)))