Example #1
0
    def sanitize(self, username=None, sources=None, rels=None):
        """
        Sanitize the source list down to only those a user has access to see.
        This was sniped from core/crits_mongoengine.

        :param username: The user requesting this data.
        :type username: str
        :param sources: A list of sources the user has access to.
        :type sources: list
        """

        if username and hasattr(self, 'source'):
            length = len(self.source)
            if not sources:
                sources = user_sources(username)
            # use slice to modify in place in case any code is referencing
            # the source already will reflect the changes as well
            self.source[:] = [s for s in self.source if s.name in sources]
            # a bit of a hack but we add a poorly formatted source to the
            # source list which has an instances length equal to the amount
            # of sources that were sanitized out of the user's list.
            # not tested but this has the added benefit of throwing a
            # ValidationError if someone were to try and save() this.
            new_length = len(self.source)
            if length > new_length:
                i_length = length - new_length
                s = EmbeddedSource()
                s.name = "Other"
                s.instances = [0] * i_length
                self.source.append(s)
Example #2
0
    def sanitize(self, username=None, sources=None, rels=None):
        """
        Sanitize the source list down to only those a user has access to see.
        This was sniped from core/crits_mongoengine.

        :param username: The user requesting this data.
        :type username: str
        :param sources: A list of sources the user has access to.
        :type sources: list
        """

        if username and hasattr(self, 'source'):
            length = len(self.source)
            if not sources:
                sources = user_sources(username)
            # use slice to modify in place in case any code is referencing
            # the source already will reflect the changes as well
            self.source[:] = [s for s in self.source if s.name in sources]
            # a bit of a hack but we add a poorly formatted source to the
            # source list which has an instances length equal to the amount
            # of sources that were sanitized out of the user's list.
            # not tested but this has the added benefit of throwing a
            # ValidationError if someone were to try and save() this.
            new_length = len(self.source)
            if length > new_length:
                i_length = length - new_length
                s = EmbeddedSource()
                s.name = "Other"
                s.instances = [0] * i_length
                self.source.append(s)
Example #3
0
def upsert_domain(sdomain,
                  domain,
                  source,
                  username=None,
                  campaign=None,
                  confidence=None,
                  bucket_list=None,
                  ticket=None,
                  cache={}):
    """
    Add or update a domain/FQDN. Campaign is assumed to be a list of campaign
    dictionary objects.

    :param sdomain: Response from parsing the domain for a root domain. Will
                    either be an error message or the root domain itself.
    :type sdomain: str
    :param domain: The domain to add/update.
    :type domain: str
    :param source: The name of the source.
    :type source: str
    :param username: The user adding/updating the domain.
    :type username: str
    :param campaign: The campaign to attribute to this domain.
    :type campaign: list, str
    :param confidence: Confidence for the campaign attribution.
    :type confidence: str
    :param bucket_list: List of buckets to add to this domain.
    :type bucket_list: list, str
    :param ticket: The ticket for this domain.
    :type ticket: str
    :param cache: Cached data, typically for performance enhancements
                  during bulk uperations.
    :type cache: dict
    :returns: dict with keys:
              "success" (boolean),
              "object" the domain that was added,
              "is_domain_new" (boolean)
    """

    if sdomain == "no_tld_found_error":  #oops...
        return {'success': False, 'message': "Invalid domain: %s " % sdomain}

    is_fqdn_domain_new = False
    is_root_domain_new = False

    if not campaign:
        campaign = []
    # assume it's a list, but check if it's a string
    elif isinstance(campaign, basestring):
        c = EmbeddedCampaign(name=campaign,
                             confidence=confidence,
                             analyst=username)
        campaign = [c]

    # assume it's a list, but check if it's a string
    if isinstance(source, basestring):
        s = EmbeddedSource()
        s.name = source
        instance = EmbeddedSource.SourceInstance()
        instance.reference = ''
        instance.method = ''
        instance.analyst = username
        instance.date = datetime.datetime.now()
        s.instances = [instance]
        source = [s]

    fqdn_domain = None
    root_domain = None
    cached_results = cache.get(form_consts.Domain.CACHED_RESULTS)

    if cached_results != None:
        if domain != sdomain:
            fqdn_domain = cached_results.get(domain)
            root_domain = cached_results.get(sdomain)
        else:
            root_domain = cached_results.get(sdomain)
    else:
        #first find the domain(s) if it/they already exist
        root_domain = Domain.objects(domain=sdomain).first()
        if domain != sdomain:
            fqdn_domain = Domain.objects(domain=domain).first()

    #if they don't exist, create them
    if not root_domain:
        root_domain = Domain()
        root_domain.domain = sdomain.strip()
        root_domain.source = []
        root_domain.record_type = 'A'
        is_root_domain_new = True

        if cached_results != None:
            cached_results[sdomain] = root_domain
    if domain != sdomain and not fqdn_domain:
        fqdn_domain = Domain()
        fqdn_domain.domain = domain.strip()
        fqdn_domain.source = []
        fqdn_domain.record_type = 'A'
        is_fqdn_domain_new = True

        if cached_results != None:
            cached_results[domain] = fqdn_domain

    # if new or found, append the new source(s)
    for s in source:
        if root_domain:
            root_domain.add_source(s)
        if fqdn_domain:
            fqdn_domain.add_source(s)

    #campaigns
    #both root and fqdn get campaigns updated
    for c in campaign:
        if root_domain:
            root_domain.add_campaign(c)
        if fqdn_domain:
            fqdn_domain.add_campaign(c)
    if username:
        if root_domain:
            root_domain.analyst = username
        if fqdn_domain:
            fqdn_domain.analyst = username

    if bucket_list:
        if root_domain:
            root_domain.add_bucket_list(bucket_list, username)
        if fqdn_domain:
            fqdn_domain.add_bucket_list(bucket_list, username)

    if ticket:
        if root_domain:
            root_domain.add_ticket(ticket, username)
        if fqdn_domain:
            fqdn_domain.add_ticket(ticket, username)

    # save
    try:
        if root_domain:
            root_domain.save(username=username)
        if fqdn_domain:
            fqdn_domain.save(username=username)
    except Exception, e:
        return {'success': False, 'message': e}
Example #4
0
def upsert_domain(domain, source, username=None, campaign=None,
                  confidence=None, bucket_list=None, ticket=None, cache={}):
    """
    Add or update a domain/FQDN. Campaign is assumed to be a list of campaign
    dictionary objects.

    :param domain: The domain to add/update.
    :type domain: str
    :param source: The name of the source.
    :type source: str
    :param username: The user adding/updating the domain.
    :type username: str
    :param campaign: The campaign to attribute to this domain.
    :type campaign: list, str
    :param confidence: Confidence for the campaign attribution.
    :type confidence: str
    :param bucket_list: List of buckets to add to this domain.
    :type bucket_list: list, str
    :param ticket: The ticket for this domain.
    :type ticket: str
    :param cache: Cached data, typically for performance enhancements
                  during bulk uperations.
    :type cache: dict
    :returns: dict with keys:
              "success" (boolean),
              "object" the domain that was added,
              "is_domain_new" (boolean)
    """

    # validate domain and grab root domain
    (root, domain, error) = get_valid_root_domain(domain)
    if error:
        return {'success': False, 'message': error}

    is_fqdn_domain_new = False
    is_root_domain_new = False

    if not campaign:
        campaign = []
    # assume it's a list, but check if it's a string
    elif isinstance(campaign, basestring):
        c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=username)
        campaign = [c]

    # assume it's a list, but check if it's a string
    if isinstance(source, basestring):
        s = EmbeddedSource()
        s.name = source
        instance = EmbeddedSource.SourceInstance()
        instance.reference = ''
        instance.method = ''
        instance.analyst = username
        instance.date = datetime.datetime.now()
        s.instances = [instance]
        source = [s]

    fqdn_domain = None
    root_domain = None
    cached_results = cache.get(form_consts.Domain.CACHED_RESULTS)

    if cached_results != None:
        if domain != root:
            fqdn_domain = cached_results.get(domain)
            root_domain = cached_results.get(root)
        else:
            root_domain = cached_results.get(root)
    else:
        #first find the domain(s) if it/they already exist
        root_domain = Domain.objects(domain=root).first()
        if domain != root:
            fqdn_domain = Domain.objects(domain=domain).first()

    #if they don't exist, create them
    if not root_domain:
        root_domain = Domain()
        root_domain.domain = root
        root_domain.source = []
        root_domain.record_type = 'A'
        is_root_domain_new = True

        if cached_results != None:
            cached_results[root] = root_domain
    if domain != root and not fqdn_domain:
        fqdn_domain = Domain()
        fqdn_domain.domain = domain
        fqdn_domain.source = []
        fqdn_domain.record_type = 'A'
        is_fqdn_domain_new = True

        if cached_results != None:
            cached_results[domain] = fqdn_domain

    # if new or found, append the new source(s)
    for s in source:
        if root_domain:
            root_domain.add_source(s)
        if fqdn_domain:
            fqdn_domain.add_source(s)

    #campaigns
    #both root and fqdn get campaigns updated
    for c in campaign:
        if root_domain:
            root_domain.add_campaign(c)
        if fqdn_domain:
            fqdn_domain.add_campaign(c)
    if username:
        if root_domain:
            root_domain.analyst = username
        if fqdn_domain:
            fqdn_domain.analyst = username

    if bucket_list:
        if root_domain:
            root_domain.add_bucket_list(bucket_list, username)
        if fqdn_domain:
            fqdn_domain.add_bucket_list(bucket_list, username)

    if ticket:
        if root_domain:
            root_domain.add_ticket(ticket, username)
        if fqdn_domain:
            fqdn_domain.add_ticket(ticket, username)

    # save
    try:
        if root_domain:
            root_domain.save(username=username)
        if fqdn_domain:
            fqdn_domain.save(username=username)
    except Exception, e:
        return {'success': False, 'message': e}
Example #5
0
def handle_indicator_insert(ind, source, reference='', analyst='', method='',
                            add_domain=False, add_relationship=False, cache={}):
    """
    Insert an individual indicator into the database.

    NOTE: Setting add_domain to True will always create a relationship as well.
    However, to create a relationship with an object that already exists before
    this function was called, set add_relationship to True. This will assume
    that the domain or IP object to create the relationship with already exists
    and will avoid infinite mutual calls between, for example, add_update_ip
    and this function. add domain/IP objects.

    :param ind: Information about the indicator.
    :type ind: dict
    :param source: The source for this indicator.
    :type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
    :param reference: The reference to the data.
    :type reference: str
    :param analyst: The user adding this indicator.
    :type analyst: str
    :param method: Method of acquiring this indicator.
    :type method: str
    :param add_domain: If this indicator is also a top-level object, try to add
                       it.
    :type add_domain: boolean
    :param add_relationship: Attempt to add relationships if applicable.
    :type add_relationship: boolean
    :param cache: Cached data, typically for performance enhancements
                  during bulk uperations.
    :type cache: dict
    :returns: dict with keys:
              "success" (boolean),
              "message" str) if failed,
              "objectid" (str) if successful,
              "is_new_indicator" (boolean) if successful.
    """

    if ind['type'] == "URI - URL" and "://" not in ind['value'].split('.')[0]:
        return {"success": False, "message": "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://) "}

    is_new_indicator = False
    dmain = None
    ip = None
    rank = {
        'unknown': 0,
        'benign': 1,
        'low': 2,
        'medium': 3,
        'high': 4,
    }

    indicator = Indicator.objects(ind_type=ind['type'],
                                  value=ind['value']).first()
    if not indicator:
        indicator = Indicator()
        indicator.ind_type = ind['type']
        indicator.value = ind['value']
        indicator.created = datetime.datetime.now()
        indicator.confidence = EmbeddedConfidence(analyst=analyst)
        indicator.impact = EmbeddedImpact(analyst=analyst)
        is_new_indicator = True

    if 'campaign' in ind:
        if isinstance(ind['campaign'], basestring) and len(ind['campaign']) > 0:
            confidence = ind.get('campaign_confidence', 'low')
            ind['campaign'] = EmbeddedCampaign(name=ind['campaign'],
                                               confidence=confidence,
                                               description="",
                                               analyst=analyst,
                                               date=datetime.datetime.now())
        if isinstance(ind['campaign'], EmbeddedCampaign):
            indicator.add_campaign(ind['campaign'])
        elif isinstance(ind['campaign'], list):
            for campaign in ind['campaign']:
                if isinstance(campaign, EmbeddedCampaign):
                    indicator.add_campaign(campaign)

    if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(indicator.confidence.rating, 0):
        indicator.confidence.rating = ind['confidence']
        indicator.confidence.analyst = analyst

    if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(indicator.impact.rating, 0):
        indicator.impact.rating = ind['impact']
        indicator.impact.analyst = analyst

    bucket_list = None
    if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
        bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
        if bucket_list:
            indicator.add_bucket_list(bucket_list, analyst)

    ticket = None
    if form_consts.Common.TICKET_VARIABLE_NAME in ind:
        ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
        if ticket:
            indicator.add_ticket(ticket, analyst)

    if isinstance(source, list):
        for s in source:
            indicator.add_source(source_item=s, method=method, reference=reference)
    elif isinstance(source, EmbeddedSource):
        indicator.add_source(source_item=source, method=method, reference=reference)
    elif isinstance(source, basestring):
        s = EmbeddedSource()
        s.name = source
        instance = EmbeddedSource.SourceInstance()
        instance.reference = reference
        instance.method = method
        instance.analyst = analyst
        instance.date = datetime.datetime.now()
        s.instances = [instance]
        indicator.add_source(s)

    if add_domain or add_relationship:
        ind_type = indicator.ind_type
        ind_value = indicator.value
        url_contains_ip = False
        if ind_type in ("URI - Domain Name", "URI - URL"):
            if ind_type == "URI - URL":
                domain_or_ip = urlparse.urlparse(ind_value).hostname
            elif ind_type == "URI - Domain Name":
                domain_or_ip = ind_value
            (sdomain, fqdn) = get_domain(domain_or_ip)
            if sdomain == "no_tld_found_error" and ind_type == "URI - URL":
                try:
                    validate_ipv46_address(domain_or_ip)
                    url_contains_ip = True
                except DjangoValidationError:
                    pass
            if not url_contains_ip:
                success = None
                if add_domain:
                    success = upsert_domain(sdomain, fqdn, indicator.source,
                                            '%s' % analyst, None,
                                            bucket_list=bucket_list, cache=cache)
                    if not success['success']:
                        return {'success': False, 'message': success['message']}

                if not success or not 'object' in success:
                    dmain = Domain.objects(domain=domain_or_ip).first()
                else:
                    dmain = success['object']

        if ind_type.startswith("Address - ip") or ind_type == "Address - cidr" or url_contains_ip:
            if url_contains_ip:
                ind_value = domain_or_ip
                try:
                    validate_ipv4_address(domain_or_ip)
                    ind_type = 'Address - ipv4-addr'
                except DjangoValidationError:
                    ind_type = 'Address - ipv6-addr'
            success = None
            if add_domain:
                success = ip_add_update(ind_value,
                                        ind_type,
                                        source=indicator.source,
                                        campaign=indicator.campaign,
                                        analyst=analyst,
                                        bucket_list=bucket_list,
                                        ticket=ticket,
                                        indicator_reference=reference,
                                        cache=cache)
                if not success['success']:
                    return {'success': False, 'message': success['message']}

            if not success or not 'object' in success:
                ip = IP.objects(ip=indicator.value).first()
            else:
                ip = success['object']

    indicator.save(username=analyst)

    if dmain:
        dmain.add_relationship(rel_item=indicator,
                               rel_type='Related_To',
                               analyst="%s" % analyst,
                               get_rels=False)
        dmain.save(username=analyst)
    if ip:
        ip.add_relationship(rel_item=indicator,
                            rel_type='Related_To',
                            analyst="%s" % analyst,
                            get_rels=False)
        ip.save(username=analyst)

    indicator.save(username=analyst)

    # run indicator triage
    if is_new_indicator:
        indicator.reload()
        run_triage(indicator, analyst)

    return {'success': True, 'objectid': str(indicator.id),
            'is_new_indicator': is_new_indicator, 'object': indicator}
Example #6
0
def handle_indicator_insert(ind,
                            source,
                            reference='',
                            analyst='',
                            method='',
                            add_domain=False,
                            add_relationship=False,
                            cache={}):
    """
    Insert an individual indicator into the database.

    NOTE: Setting add_domain to True will always create a relationship as well.
    However, to create a relationship with an object that already exists before
    this function was called, set add_relationship to True. This will assume
    that the domain or IP object to create the relationship with already exists
    and will avoid infinite mutual calls between, for example, add_update_ip
    and this function. add domain/IP objects.

    :param ind: Information about the indicator.
    :type ind: dict
    :param source: The source for this indicator.
    :type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
    :param reference: The reference to the data.
    :type reference: str
    :param analyst: The user adding this indicator.
    :type analyst: str
    :param method: Method of acquiring this indicator.
    :type method: str
    :param add_domain: If this indicator is also a top-level object, try to add
                       it.
    :type add_domain: boolean
    :param add_relationship: Attempt to add relationships if applicable.
    :type add_relationship: boolean
    :param cache: Cached data, typically for performance enhancements
                  during bulk uperations.
    :type cache: dict
    :returns: dict with keys:
              "success" (boolean),
              "message" str) if failed,
              "objectid" (str) if successful,
              "is_new_indicator" (boolean) if successful.
    """

    is_new_indicator = False
    rank = {'unknown': 0, 'benign': 1, 'low': 2, 'medium': 3, 'high': 4}

    indicator = Indicator.objects(ind_type=ind['type'],
                                  value=ind['value']).first()
    if not indicator:
        indicator = Indicator()
        indicator.ind_type = ind['type']
        indicator.value = ind['value']
        indicator.created = datetime.datetime.now()
        indicator.confidence = EmbeddedConfidence(analyst=analyst)
        indicator.impact = EmbeddedImpact(analyst=analyst)
        is_new_indicator = True

    ec = None
    if 'campaign' in ind:
        confidence = 'low'

        if 'campaign_confidence' in ind:
            confidence = ind['campaign_confidence']

        ec = EmbeddedCampaign(name=ind['campaign'],
                              confidence=confidence,
                              description="",
                              analyst=analyst,
                              date=datetime.datetime.now())

    if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(
            indicator.confidence.rating, 0):
        indicator.confidence.rating = ind['confidence']
        indicator.confidence.analyst = analyst

    if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(
            indicator.impact.rating, 0):
        indicator.impact.rating = ind['impact']
        indicator.impact.analyst = analyst

    bucket_list = None
    if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
        bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
        indicator.add_bucket_list(bucket_list, analyst)

    ticket = None
    if form_consts.Common.TICKET_VARIABLE_NAME in ind:
        ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
        indicator.add_ticket(ticket, analyst)

    if isinstance(source, list):
        for s in source:
            indicator.add_source(source_item=s)
    elif isinstance(source, EmbeddedSource):
        indicator.add_source(source_item=source)
    elif isinstance(source, basestring):
        s = EmbeddedSource()
        s.name = source
        instance = EmbeddedSource.SourceInstance()
        instance.reference = reference
        instance.method = method
        instance.analyst = analyst
        instance.date = datetime.datetime.now()
        s.instances = [instance]
        indicator.add_source(s)
    if ec:
        indicator.add_campaign(ec)
    indicator.save(username=analyst)

    if add_domain or add_relationship:
        ind_type = indicator.ind_type
        ind_value = indicator.value
        if ind_type in ("URI - Domain Name", "URI - URL"):
            if ind_type == "URI - URL":
                domain = ind_value.split("/")[2]
            elif ind_type == "URI - Domain Name":
                domain = ind_value
            #try:
            (sdomain, fqdn) = get_domain(domain)
            success = None
            if add_domain:
                success = upsert_domain(sdomain,
                                        fqdn,
                                        indicator.source,
                                        '%s' % analyst,
                                        None,
                                        bucket_list=bucket_list,
                                        cache=cache)
                if not success['success']:
                    return {'success': False, 'message': success['message']}

            if not success or not 'object' in success:
                dmain = Domain.objects(domain=domain).first()
            else:
                dmain = success['object']
            if dmain:
                dmain.add_relationship(rel_item=indicator,
                                       rel_type='Related_To',
                                       analyst="%s" % analyst,
                                       get_rels=False)
                dmain.save(username=analyst)
                indicator.save(username=analyst)

        elif ind_type.startswith(
                "Address - ip") or ind_type == "Address - cidr":
            success = None
            if add_domain:
                success = ip_add_update(indicator.value,
                                        ind_type,
                                        source=indicator.source,
                                        campaign=indicator.campaign,
                                        analyst=analyst,
                                        bucket_list=bucket_list,
                                        ticket=ticket,
                                        indicator_reference=reference,
                                        cache=cache)
                if not success['success']:
                    return {'success': False, 'message': success['message']}

            if not success or not 'object' in success:
                ip = IP.objects(ip=indicator.value).first()
            else:
                ip = success['object']
            if ip:
                ip.add_relationship(rel_item=indicator,
                                    rel_type='Related_To',
                                    analyst="%s" % analyst,
                                    get_rels=False)
                ip.save(username=analyst)
                indicator.save(username=analyst)

    # run indicator triage
    if is_new_indicator:
        indicator.reload()
        run_triage(None, indicator, analyst)

    return {
        'success': True,
        'objectid': indicator.id,
        'is_new_indicator': is_new_indicator,
        'object': indicator
    }
Example #7
0
def upsert_domain(domain, source, username=None, campaign=None,
                  confidence=None, bucket_list=None, ticket=None, cache={}, related_id=None, related_type=None, relationship_type=None):
    """
    Add or update a domain/FQDN. Campaign is assumed to be a list of campaign
    dictionary objects.

    :param domain: The domain to add/update.
    :type domain: str
    :param source: The name of the source.
    :type source: str
    :param username: The user adding/updating the domain.
    :type username: str
    :param campaign: The campaign to attribute to this domain.
    :type campaign: list, str
    :param confidence: Confidence for the campaign attribution.
    :type confidence: str
    :param bucket_list: List of buckets to add to this domain.
    :type bucket_list: list, str
    :param ticket: The ticket for this domain.
    :type ticket: str
    :param cache: Cached data, typically for performance enhancements
                  during bulk uperations.
    :type cache: dict
    :param related_id: ID of object to create relationship with
    :type related_id: str
    :param related_type: Type of object to create relationship with
    :type related_id: str
    :param relationship_type: Type of relationship to create.
    :type relationship_type: str
    :returns: dict with keys:
              "success" (boolean),
              "object" the domain that was added,
              "is_domain_new" (boolean)
    """


    # validate domain and grab root domain
    (root, domain, error) = get_valid_root_domain(domain)
    if error:
        return {'success': False, 'message': error}

    is_fqdn_domain_new = False
    is_root_domain_new = False

    if not campaign:
        campaign = []
    # assume it's a list, but check if it's a string
    elif isinstance(campaign, basestring):
        c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=username)
        campaign = [c]

    # assume it's a list, but check if it's a string
    if isinstance(source, basestring):
        s = EmbeddedSource()
        s.name = source
        instance = EmbeddedSource.SourceInstance()
        instance.reference = ''
        instance.method = ''
        instance.analyst = username
        instance.date = datetime.datetime.now()
        s.instances = [instance]
        source = [s]

    fqdn_domain = None
    root_domain = None
    cached_results = cache.get(form_consts.Domain.CACHED_RESULTS)

    if cached_results != None:
        if domain != root:
            fqdn_domain = cached_results.get(domain)
            root_domain = cached_results.get(root)
        else:
            root_domain = cached_results.get(root)
    else:
        #first find the domain(s) if it/they already exist
        root_domain = Domain.objects(domain=root).first()
        if domain != root:
            fqdn_domain = Domain.objects(domain=domain).first()

    #if they don't exist, create them
    if not root_domain:
        root_domain = Domain()
        root_domain.domain = root
        root_domain.source = []
        root_domain.record_type = 'A'
        is_root_domain_new = True

        if cached_results != None:
            cached_results[root] = root_domain
    if domain != root and not fqdn_domain:
        fqdn_domain = Domain()
        fqdn_domain.domain = domain
        fqdn_domain.source = []
        fqdn_domain.record_type = 'A'
        is_fqdn_domain_new = True

        if cached_results != None:
            cached_results[domain] = fqdn_domain

    # if new or found, append the new source(s)
    for s in source:
        if root_domain:
            root_domain.add_source(s)
        if fqdn_domain:
            fqdn_domain.add_source(s)

    #campaigns
    #both root and fqdn get campaigns updated
    for c in campaign:
        if root_domain:
            root_domain.add_campaign(c)
        if fqdn_domain:
            fqdn_domain.add_campaign(c)
    if username:
        if root_domain:
            root_domain.analyst = username
        if fqdn_domain:
            fqdn_domain.analyst = username

    if bucket_list:
        if root_domain:
            root_domain.add_bucket_list(bucket_list, username)
        if fqdn_domain:
            fqdn_domain.add_bucket_list(bucket_list, username)

    if ticket:
        if root_domain:
            root_domain.add_ticket(ticket, username)
        if fqdn_domain:
            fqdn_domain.add_ticket(ticket, username)

    related_obj = None
    if related_id:
        related_obj = class_from_id(related_type, related_id)
        if not related_obj:
            retVal['success'] = False
            retVal['message'] = 'Related Object not found.'
            return retVal

    # save
    try:
        if root_domain:
            root_domain.save(username=username)
        if fqdn_domain:
            fqdn_domain.save(username=username)
    except Exception, e:
        return {'success': False, 'message': e}
Example #8
0
def handle_indicator_insert(
    ind, source, reference="", analyst="", method="", add_domain=False, add_relationship=False, cache={}
):
    """
    Insert an individual indicator into the database.

    NOTE: Setting add_domain to True will always create a relationship as well.
    However, to create a relationship with an object that already exists before
    this function was called, set add_relationship to True. This will assume
    that the domain or IP object to create the relationship with already exists
    and will avoid infinite mutual calls between, for example, add_update_ip
    and this function. add domain/IP objects.

    :param ind: Information about the indicator.
    :type ind: dict
    :param source: The source for this indicator.
    :type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
    :param reference: The reference to the data.
    :type reference: str
    :param analyst: The user adding this indicator.
    :type analyst: str
    :param method: Method of acquiring this indicator.
    :type method: str
    :param add_domain: If this indicator is also a top-level object, try to add
                       it.
    :type add_domain: boolean
    :param add_relationship: Attempt to add relationships if applicable.
    :type add_relationship: boolean
    :param cache: Cached data, typically for performance enhancements
                  during bulk uperations.
    :type cache: dict
    :returns: dict with keys:
              "success" (boolean),
              "message" (str) if failed,
              "objectid" (str) if successful,
              "is_new_indicator" (boolean) if successful.
    """

    if ind["type"] not in IndicatorTypes.values():
        return {"success": False, "message": "Not a valid Indicator Type: %s" % ind["type"]}
    if ind["threat_type"] not in IndicatorThreatTypes.values():
        return {"success": False, "message": "Not a valid Indicator Threat Type: %s" % ind["threat_type"]}
    if ind["attack_type"] not in IndicatorAttackTypes.values():
        return {"success": False, "message": "Not a valid Indicator Attack Type: " % ind["attack_type"]}

    (ind["value"], error) = validate_indicator_value(ind["value"], ind["type"])
    if error:
        return {"success": False, "message": error}

    is_new_indicator = False
    dmain = None
    ip = None
    rank = {"unknown": 0, "benign": 1, "low": 2, "medium": 3, "high": 4}

    if ind.get("status", None) is None or len(ind.get("status", "")) < 1:
        ind["status"] = Status.NEW

    indicator = Indicator.objects(
        ind_type=ind["type"], lower=ind["lower"], threat_type=ind["threat_type"], attack_type=ind["attack_type"]
    ).first()
    if not indicator:
        indicator = Indicator()
        indicator.ind_type = ind["type"]
        indicator.threat_type = ind["threat_type"]
        indicator.attack_type = ind["attack_type"]
        indicator.value = ind["value"]
        indicator.lower = ind["lower"]
        indicator.description = ind["description"]
        indicator.created = datetime.datetime.now()
        indicator.confidence = EmbeddedConfidence(analyst=analyst)
        indicator.impact = EmbeddedImpact(analyst=analyst)
        indicator.status = ind["status"]
        is_new_indicator = True
    else:
        if ind["status"] != Status.NEW:
            indicator.status = ind["status"]
        add_desc = "\nSeen on %s as: %s" % (str(datetime.datetime.now()), ind["value"])
        if indicator.description is None:
            indicator.description = add_desc
        else:
            indicator.description += add_desc

    if "campaign" in ind:
        if isinstance(ind["campaign"], basestring) and len(ind["campaign"]) > 0:
            confidence = ind.get("campaign_confidence", "low")
            ind["campaign"] = EmbeddedCampaign(
                name=ind["campaign"],
                confidence=confidence,
                description="",
                analyst=analyst,
                date=datetime.datetime.now(),
            )
        if isinstance(ind["campaign"], EmbeddedCampaign):
            indicator.add_campaign(ind["campaign"])
        elif isinstance(ind["campaign"], list):
            for campaign in ind["campaign"]:
                if isinstance(campaign, EmbeddedCampaign):
                    indicator.add_campaign(campaign)

    if "confidence" in ind and rank.get(ind["confidence"], 0) > rank.get(indicator.confidence.rating, 0):
        indicator.confidence.rating = ind["confidence"]
        indicator.confidence.analyst = analyst

    if "impact" in ind and rank.get(ind["impact"], 0) > rank.get(indicator.impact.rating, 0):
        indicator.impact.rating = ind["impact"]
        indicator.impact.analyst = analyst

    bucket_list = None
    if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
        bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
        if bucket_list:
            indicator.add_bucket_list(bucket_list, analyst)

    ticket = None
    if form_consts.Common.TICKET_VARIABLE_NAME in ind:
        ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
        if ticket:
            indicator.add_ticket(ticket, analyst)

    if isinstance(source, list):
        for s in source:
            indicator.add_source(source_item=s, method=method, reference=reference)
    elif isinstance(source, EmbeddedSource):
        indicator.add_source(source_item=source, method=method, reference=reference)
    elif isinstance(source, basestring):
        s = EmbeddedSource()
        s.name = source
        instance = EmbeddedSource.SourceInstance()
        instance.reference = reference
        instance.method = method
        instance.analyst = analyst
        instance.date = datetime.datetime.now()
        s.instances = [instance]
        indicator.add_source(s)

    if add_domain or add_relationship:
        ind_type = indicator.ind_type
        ind_value = indicator.lower
        url_contains_ip = False
        if ind_type in (IndicatorTypes.DOMAIN, IndicatorTypes.URI):
            if ind_type == IndicatorTypes.URI:
                domain_or_ip = urlparse.urlparse(ind_value).hostname
                try:
                    validate_ipv46_address(domain_or_ip)
                    url_contains_ip = True
                except DjangoValidationError:
                    pass
            else:
                domain_or_ip = ind_value
            if not url_contains_ip:
                success = None
                if add_domain:
                    success = upsert_domain(
                        domain_or_ip,
                        indicator.source,
                        username="******" % analyst,
                        campaign=indicator.campaign,
                        bucket_list=bucket_list,
                        cache=cache,
                    )
                    if not success["success"]:
                        return {"success": False, "message": success["message"]}

                if not success or not "object" in success:
                    dmain = Domain.objects(domain=domain_or_ip).first()
                else:
                    dmain = success["object"]

        if ind_type in IPTypes.values() or url_contains_ip:
            if url_contains_ip:
                ind_value = domain_or_ip
                try:
                    validate_ipv4_address(domain_or_ip)
                    ind_type = IndicatorTypes.IPV4_ADDRESS
                except DjangoValidationError:
                    ind_type = IndicatorTypes.IPV6_ADDRESS
            success = None
            if add_domain:
                success = ip_add_update(
                    ind_value,
                    ind_type,
                    source=indicator.source,
                    campaign=indicator.campaign,
                    analyst=analyst,
                    bucket_list=bucket_list,
                    ticket=ticket,
                    indicator_reference=reference,
                    cache=cache,
                )
                if not success["success"]:
                    return {"success": False, "message": success["message"]}

            if not success or not "object" in success:
                ip = IP.objects(ip=indicator.value).first()
            else:
                ip = success["object"]

    indicator.save(username=analyst)

    if dmain:
        dmain.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
        dmain.save(username=analyst)
    if ip:
        ip.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
        ip.save(username=analyst)

    # run indicator triage
    if is_new_indicator:
        indicator.reload()
        run_triage(indicator, analyst)

    return {"success": True, "objectid": str(indicator.id), "is_new_indicator": is_new_indicator, "object": indicator}