Ejemplo n.º 1
0
    def obj_create(self, bundle, **kwargs):
        """
        Handles creating IPs through the API.

        :param bundle: Bundle containing the information to create the IP.
        :type bundle: Tastypie Bundle object.
        :returns: HttpResponse.
        """

        analyst = bundle.request.user.username
        data = bundle.data
        ip = data.get('ip', None)
        name = data.get('source', None)
        reference = data.get('reference', None)
        method = data.get('method', None)
        campaign = data.get('campaign', None)
        description = data.get('description', None)
        confidence = data.get('confidence', None)
        ip_type = data.get('ip_type', None)
        add_indicator = data.get('add_indicator', False)
        indicator_reference = data.get('indicator_reference', None)
        bucket_list = data.get('bucket_list', None)
        ticket = data.get('ticket', None)

        content = {'return_code': 1, 'type': 'IP'}

        if not ip or not name or not ip_type:
            content['message'] = "Must provide an IP, IP Type, and Source."
            self.crits_response(content)

        result = ip_add_update(ip,
                               ip_type,
                               source=name,
                               source_method=method,
                               source_reference=reference,
                               campaign=campaign,
                               confidence=confidence,
                               description=description,
                               analyst=analyst,
                               bucket_list=bucket_list,
                               ticket=ticket,
                               is_add_indicator=add_indicator,
                               indicator_reference=indicator_reference)

        if result.get('message'):
            content['message'] = result.get('message')
        if result.get('object'):
            content['id'] = str(result.get('object').id)
        if content.get('id'):
            url = reverse('api_dispatch_detail',
                          kwargs={
                              'resource_name': 'ips',
                              'api_name': 'v1',
                              'pk': content.get('id')
                          })
            content['url'] = url
        if result['success']:
            content['return_code'] = 0
        self.crits_response(content)
Ejemplo n.º 2
0
    def obj_create(self, bundle, **kwargs):
        """
        Handles creating IPs through the API.

        :param bundle: Bundle containing the information to create the IP.
        :type bundle: Tastypie Bundle object.
        :returns: HttpResponse.
        """

        analyst = bundle.request.user.username
        data = bundle.data
        ip = data.get('ip', None)
        name = data.get('source', None)
        reference = data.get('reference', None)
        method = data.get('method', None)
        campaign = data.get('campaign', None)
        description = data.get('description', None)
        confidence = data.get('confidence', None)
        ip_type = data.get('ip_type', None)
        add_indicator = data.get('add_indicator', False)
        indicator_reference = data.get('indicator_reference', None)
        bucket_list = data.get('bucket_list', None)
        ticket = data.get('ticket', None)

        content = {'return_code': 1,
                   'type': 'IP'}

        if not ip or not name or not ip_type:
            content['message'] = "Must provide an IP, IP Type, and Source."
            self.crits_response(content)

        result = ip_add_update(ip,
                               ip_type,
                               source=name,
                               source_method=method,
                               source_reference=reference,
                               campaign=campaign,
                               confidence=confidence,
                               description=description,
                               analyst=analyst,
                               bucket_list=bucket_list,
                               ticket=ticket,
                               is_add_indicator=add_indicator,
                               indicator_reference=indicator_reference)

        if result.get('message'):
            content['message'] = result.get('message')
        if result.get('object'):
            content['id'] = str(result.get('object').id)
        if content.get('id'):
            url = reverse('api_dispatch_detail',
                          kwargs={'resource_name': 'ips',
                                  'api_name': 'v1',
                                  'pk': content.get('id')})
            content['url'] = url
        if result['success']:
            content['return_code'] = 0
        self.crits_response(content)
Ejemplo n.º 3
0
    def obj_create(self, bundle, **kwargs):
        """
        Handles creating IPs through the API.

        :param bundle: Bundle containing the information to create the IP.
        :type bundle: Tastypie Bundle object.
        :returns: Bundle object.
        :raises BadRequest: If creation fails.
        """

        analyst = bundle.request.user.username
        data = bundle.data
        ip = data.get('ip', None)
        name = data.get('source', None)
        reference = data.get('reference', None)
        method = data.get('method', None)
        campaign = data.get('campaign', None)
        confidence = data.get('confidence', None)
        ip_type = data.get('ip_type', None)
        add_indicator = data.get('add_indicator', False)
        indicator_reference = data.get('indicator_reference', None)
        bucket_list = data.get('bucket_list', None)
        ticket = data.get('ticket', None)

        if not ip or not name or not ip_type:
            raise BadRequest("Must provide an IP, IP Type, and Source.")

        result = ip_add_update(ip,
                               ip_type,
                               source=name,
                               source_method=method,
                               source_reference=reference,
                               campaign=campaign,
                               confidence=confidence,
                               analyst=analyst,
                               bucket_list=bucket_list,
                               ticket=ticket,
                               is_add_indicator=add_indicator,
                               indicator_reference=indicator_reference)
        if not result['success']:
            raise BadRequest(result['message'])
        else:
            return bundle
Ejemplo n.º 4
0
    def process_ips(self, incident_report):
        ips = incident_report.findall("./contacted/ips/ip")
        for ip in ips:
            ret = ip_add_update(ip.text, self._ip_type(ip.text),
                                source=get_user_organization(self.current_task.user),
                                source_method=self.name,
                                source_tlp=self.obj.tlp,
                                user=self.current_task.user,
                                related_id=str(self.obj.id),
                                related_type=self.obj._meta['crits_type'],
                                relationship_type=RelationshipTypes.CONNECTED_TO)

            if ret['success']:
                malicious = ip.get('malicious', 'unknown')
                self._add_result("IPs", ip.text, {'malicious': malicious})
            else:
                self._warning(ret["message"])

        self._notify()
Ejemplo n.º 5
0
    def obj_create(self, bundle, **kwargs):
        """
        Handles creating IPs through the API.

        :param bundle: Bundle containing the information to create the IP.
        :type bundle: Tastypie Bundle object.
        :returns: Bundle object.
        :raises BadRequest: If creation fails.
        """

        analyst = bundle.request.user.username
        data = bundle.data
        ip = data['ip']
        name = data['source']
        reference = data['reference']
        method = data['method']
        campaign = data['campaign']
        confidence = data['confidence']
        ip_type = data['ip_type']
        add_indicator = data.get('add_indicator', False)
        indicator_reference = data.get('indicator_reference')
        bucket_list = data.get('bucket_list', None)
        ticket = data.get('ticket', None)

        result = ip_add_update(ip,
                               ip_type,
                               source=name,
                               source_method=method,
                               source_reference=reference,
                               campaign=campaign,
                               confidence=confidence,
                               analyst=analyst,
                               bucket_list=bucket_list,
                               ticket=ticket,
                               is_add_indicator=add_indicator,
                               indicator_reference=indicator_reference)
        if 'message' in result:
            raise BadRequest(result['message'])
        else:
            return bundle
Ejemplo n.º 6
0
    def parse_observables(self, observables):
        """
        Parse list of observables in STIX doc.

        :param observables: List of STIX observables.
        :type observables: List of STIX observables.
        """

        analyst = self.source_instance.analyst

        for obs in observables:  # for each STIX observable

            if obs.observable_composition:
                object_list = obs.observable_composition.observables
            else:
                object_list = [obs]

            for obs_comp in object_list:

                if not obs_comp.object_ or not obs_comp.object_.properties:
                    self.failed.append(
                        ("No valid object_properties was found!",
                         type(obs_comp).__name__,
                         obs_comp.id_))  # note for display in UI
                    continue
                try:  # try to create CRITs object from observable
                    item = obs_comp.object_.properties

                    if isinstance(item, Address):
                        if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
                                             'ipv4-netmask', 'ipv6-addr',
                                             'ipv6-net', 'ipv6-netmask',
                                             'ipv6-subnet'):
                            imp_type = "IP"
                            for value in item.address_value.values:
                                ip = str(value).strip()
                                iptype = get_crits_ip_type(item.category)
                                if iptype:
                                    res = ip_add_update(ip,
                                                        iptype, [self.source],
                                                        analyst=analyst,
                                                        id=self.package.id_)
                                    self.parse_res(imp_type, obs, res)
                    if isinstance(item, DomainName):
                        imp_type = "Domain"
                        for value in item.value.values:
                            res = upsert_domain(str(value), [self.source],
                                                username=analyst,
                                                id=self.package.id_)
                            self.parse_res(imp_type, obs, res)
                    elif isinstance(item, Artifact):
                        # Not sure if this is right, and I believe these can be
                        # encoded in a couple different ways.
                        imp_type = "RawData"
                        rawdata = item.data.decode('utf-8')
                        description = "None"
                        # TODO: find out proper ways to determine title, datatype,
                        #       tool_name, tool_version
                        title = "Artifact for Event: STIX Document %s" % self.package.id_
                        res = handle_raw_data_file(
                            rawdata,
                            self.source.name,
                            user=analyst,
                            description=description,
                            title=title,
                            data_type="Text",
                            tool_name="STIX",
                            tool_version=None,
                            method=self.source_instance.method,
                            reference=self.source_instance.reference)
                        self.parse_res(imp_type, obs, res)
                    elif (isinstance(item, File) and item.custom_properties
                          and item.custom_properties[0].name == "crits_type"
                          and item.custom_properties[0]._value
                          == "Certificate"):
                        imp_type = "Certificate"
                        description = "None"
                        filename = str(item.file_name)
                        data = None
                        for obj in item.parent.related_objects:
                            if isinstance(obj.properties, Artifact):
                                data = obj.properties.data
                        res = handle_cert_file(filename,
                                               data,
                                               self.source,
                                               user=analyst,
                                               description=description)
                        self.parse_res(imp_type, obs, res)
                    elif isinstance(item,
                                    File) and self.has_network_artifact(item):
                        imp_type = "PCAP"
                        description = "None"
                        filename = str(item.file_name)
                        data = None
                        for obj in item.parent.related_objects:
                            if (isinstance(obj.properties, Artifact)
                                    and obj.properties.type_
                                    == Artifact.TYPE_NETWORK):
                                data = obj.properties.data
                        res = handle_pcap_file(filename,
                                               data,
                                               self.source,
                                               user=analyst,
                                               description=description)
                        self.parse_res(imp_type, obs, res)
                    elif isinstance(item, File):
                        imp_type = "Sample"
                        filename = str(item.file_name)
                        md5 = item.md5
                        data = None
                        for obj in item.parent.related_objects:
                            if (isinstance(obj.properties, Artifact)
                                    and obj.properties.type_
                                    == Artifact.TYPE_FILE):
                                data = obj.properties.data
                        res = handle_file(filename,
                                          data,
                                          self.source,
                                          user=analyst,
                                          md5_digest=md5,
                                          is_return_only_md5=False,
                                          id=self.package.id_)
                        self.parse_res(imp_type, obs, res)
                        if item.extracted_features:
                            self.parse_filenames(item.extracted_features,
                                                 res['object'].id)
                    elif isinstance(item, EmailMessage):

                        imp_type = "Email"
                        data = {}
                        data['source'] = self.source.name
                        data['source_method'] = self.source_instance.method
                        data[
                            'source_reference'] = self.source_instance.reference
                        data['raw_body'] = str(item.raw_body)
                        data['raw_header'] = str(item.raw_header)
                        data['helo'] = str(item.email_server)
                        if item.header:
                            data['message_id'] = str(item.header.message_id)
                            data['subject'] = str(item.header.subject)
                            data['sender'] = str(item.header.sender)
                            data['reply_to'] = str(item.header.reply_to)
                            data['x_originating_ip'] = str(
                                item.header.x_originating_ip)
                            data['x_mailer'] = str(item.header.x_mailer)
                            data['boundary'] = str(item.header.boundary)
                            data['from_address'] = str(item.header.from_)
                            data['date'] = item.header.date.value
                            if item.header.to:
                                data['to'] = [str(r) for r in item.header.to]
                            if item.header.cc:
                                data['cc'] = [str(r) for r in item.header.cc]
                        res = handle_email_fields(data,
                                                  analyst,
                                                  "STIX",
                                                  id=self.package.id_)

                        # Should check for attachments and add them here.
                        self.parse_res(imp_type, obs, res)

                        if res.get('status') and item.attachments:
                            for attach in item.attachments:
                                rel_id = attach.to_dict()['object_reference']
                                self.relationships.append(
                                    (obs.id_, "Contains", rel_id, "High"))
                    else:  # try to parse all other possibilities as Indicator
                        imp_type = "Indicator"
                        obj = make_crits_object(item)
                        if obj.object_type == 'Address':
                            # This was already caught above
                            continue
                        else:
                            ind_type = obj.object_type
                            for value in obj.value:
                                if value and ind_type:
                                    res = handle_indicator_ind(
                                        value.strip(),
                                        self.source,
                                        ind_type,
                                        IndicatorThreatTypes.UNKNOWN,
                                        IndicatorAttackTypes.UNKNOWN,
                                        analyst,
                                        add_domain=True,
                                        add_relationship=True)
                                    self.parse_res(imp_type, obs, res)
                except Exception, e:  # probably caused by cybox object we don't handle
                    self.failed.append(
                        (e.message, type(item).__name__,
                         item.parent.id_))  # note for display in UI
Ejemplo n.º 7
0
def add_new_domain(data,
                   request,
                   errors,
                   rowData=None,
                   is_validate_only=False,
                   cache={}):
    """
    Add a new domain to CRITs.

    :param data: The data about the domain.
    :type data: dict
    :param request: The Django request.
    :type request: :class:`django.http.HttpRequest`
    :param errors: A list of current errors to append to.
    :type errors: list
    :param rowData: Any objects that need to be added to the domain.
    :type rowData: dict
    :param is_validate_only: Only validate the data and return any errors.
    :type is_validate_only: boolean
    :param cache: Cached data, typically for performance enhancements
                  during bulk operations.
    :type cache: dict
    :returns: tuple (<result>, <errors>, <retVal>)
    """

    result = False
    retVal = {}
    domain = data['domain']
    add_ip = data.get('add_ip')
    ip = data.get('ip')
    ip_type = data.get('ip_type')

    if add_ip:
        error = validate_and_normalize_ip(ip, ip_type)[1]
        if error:
            errors.append(error)

    if is_validate_only:
        error = get_valid_root_domain(domain)[2]
        if error:
            errors.append(error)

        # check for duplicate domains
        fqdn_domain = retrieve_domain(domain, cache)

        if fqdn_domain:
            if isinstance(fqdn_domain, Domain):
                resp_url = reverse('crits.domains.views.domain_detail',
                                   args=[domain])
                message = ('Warning: Domain already exists: '
                           '<a href="%s">%s</a>' % (resp_url, domain))
                retVal['message'] = message
                retVal['status'] = form_consts.Status.DUPLICATE
                retVal['warning'] = message
        else:
            result_cache = cache.get(form_consts.Domain.CACHED_RESULTS)
            result_cache[domain.lower()] = True

    elif not errors:
        username = request.user.username
        reference = data.get('domain_reference')
        source_name = data.get('domain_source')
        method = data.get('domain_method')
        source = [
            create_embedded_source(source_name,
                                   reference=reference,
                                   method=method,
                                   analyst=username)
        ]
        bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
        ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)

        if data.get('campaign') and data.get('confidence'):
            campaign = [
                EmbeddedCampaign(name=data.get('campaign'),
                                 confidence=data.get('confidence'),
                                 analyst=username)
            ]
        else:
            campaign = []

        retVal = upsert_domain(domain,
                               source,
                               username,
                               campaign,
                               bucket_list=bucket_list,
                               ticket=ticket,
                               cache=cache)

        if not retVal['success']:
            errors.append(retVal.get('message'))
            retVal['message'] = ""

        else:
            new_domain = retVal['object']
            ip_result = {}
            if add_ip:
                if data.get('same_source'):
                    ip_source = source_name
                    ip_method = method
                    ip_reference = reference
                else:
                    ip_source = data.get('ip_source')
                    ip_method = data.get('ip_method')
                    ip_reference = data.get('ip_reference')
                from crits.ips.handlers import ip_add_update
                ip_result = ip_add_update(ip,
                                          ip_type,
                                          ip_source,
                                          ip_method,
                                          ip_reference,
                                          campaign=campaign,
                                          analyst=username,
                                          bucket_list=bucket_list,
                                          ticket=ticket,
                                          cache=cache)
                if not ip_result['success']:
                    errors.append(ip_result['message'])
                else:
                    #add a relationship with the new IP address
                    new_ip = ip_result['object']
                    if new_domain and new_ip:
                        new_domain.add_relationship(rel_item=new_ip,
                                                    rel_type='Resolved_To',
                                                    analyst=username,
                                                    get_rels=False)
                        new_domain.save(username=username)
                        new_ip.save(username=username)

            #set the URL for viewing the new data
            resp_url = reverse('crits.domains.views.domain_detail',
                               args=[domain])

            if retVal['is_domain_new'] == True:
                retVal['message'] = (
                    'Success! Click here to view the new domain: '
                    '<a href="%s">%s</a>' % (resp_url, domain))
            else:
                message = ('Updated existing domain: <a href="%s">%s</a>' %
                           (resp_url, domain))
                retVal['message'] = message
                retVal[form_consts.Status.
                       STATUS_FIELD] = form_consts.Status.DUPLICATE
                retVal['warning'] = message

            #add indicators
            if data.get('add_indicators'):
                from crits.indicators.handlers import create_indicator_from_tlo
                # If we have an IP object, add an indicator for that.
                if ip_result.get('success'):
                    ip = ip_result['object']
                    result = create_indicator_from_tlo('IP',
                                                       ip,
                                                       username,
                                                       ip_source,
                                                       add_domain=False)
                    ip_ind = result.get('indicator')
                    if not result['success']:
                        errors.append(result['message'])

                # Add an indicator for the domain.
                result = create_indicator_from_tlo('Domain',
                                                   new_domain,
                                                   username,
                                                   source_name,
                                                   add_domain=False)

                if not result['success']:
                    errors.append(result['message'])
                elif ip_result.get('success') and ip_ind:
                    forge_relationship(left_class=result['indicator'],
                                       right_class=ip_ind,
                                       rel_type='Resolved_To',
                                       analyst=username)
            result = True

    # This block validates, and may also add, objects to the Domain
    if retVal.get('success') or is_validate_only == True:
        if rowData:
            objectsData = rowData.get(form_consts.Common.OBJECTS_DATA)

            # add new objects if they exist
            if objectsData:
                objectsData = json.loads(objectsData)
                current_domain = retrieve_domain(domain, cache)
                for object_row_counter, objectData in enumerate(
                        objectsData, 1):
                    if current_domain != None:
                        # if the domain exists then try to add objects to it
                        if isinstance(current_domain, Domain) == True:
                            objectDict = object_array_to_dict(
                                objectData, "Domain", current_domain.id)
                        else:
                            objectDict = object_array_to_dict(
                                objectData, "Domain", "")
                            current_domain = None
                    else:
                        objectDict = object_array_to_dict(
                            objectData, "Domain", "")

                    (obj_result, errors,
                     obj_retVal) = validate_and_add_new_handler_object(
                         None,
                         objectDict,
                         request,
                         errors,
                         object_row_counter,
                         is_validate_only=is_validate_only,
                         cache=cache,
                         obj=current_domain)
                    if not obj_result:
                        retVal['success'] = False

    return result, errors, retVal
Ejemplo n.º 8
0
Archivo: views.py Proyecto: vsbca/crits
def add_update_ip(request, method):
    """
    Add/update an IP address. Should be an AJAX POST.

    :param request: Django request.
    :type request: :class:`django.http.HttpRequest`
    :param method: If this is an "add" or an "update".
    :type method: str
    :returns: :class:`django.http.HttpResponse`
    """

    if request.method == "POST" and request.is_ajax():
        request.user._setup()
        data = request.POST
        form = AddIPForm(request.user, None, data)

        if form.is_valid():
            cleaned_data = form.cleaned_data
            ip = cleaned_data['ip']
            source_name = cleaned_data['source_name']
            source_reference = cleaned_data['source_reference']
            source_method = cleaned_data['source_method']
            source_tlp = cleaned_data['source_tlp']
            campaign = cleaned_data['campaign']
            confidence = cleaned_data['confidence']
            ip_type = cleaned_data['ip_type']
            add_indicator = False
            if cleaned_data.get('add_indicator'):
                add_indicator = True
            indicator_reference = cleaned_data.get('indicator_reference')
            bucket_list = cleaned_data.get(
                form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
            ticket = cleaned_data.get(form_consts.Common.TICKET_VARIABLE_NAME)
            related_id = cleaned_data['related_id']
            related_type = cleaned_data['related_type']
            relationship_type = cleaned_data['relationship_type']

            result = ip_add_update(ip,
                                   ip_type,
                                   source=source_name,
                                   source_method=source_method,
                                   source_reference=source_reference,
                                   source_tlp=source_tlp,
                                   campaign=campaign,
                                   confidence=confidence,
                                   user=request.user,
                                   bucket_list=bucket_list,
                                   ticket=ticket,
                                   is_add_indicator=add_indicator,
                                   indicator_reference=indicator_reference,
                                   related_id=related_id,
                                   related_type=related_type,
                                   relationship_type=relationship_type)
            if 'message' in result:
                if not isinstance(result['message'], list):
                    result['message'] = [result['message']]
            else:
                result['message'] = []
                message = (
                    '<div>Success! Click here to view the new IP: <a '
                    'href="%s">%s</a></div>' %
                    (reverse('crits.ips.views.ip_detail', args=[ip]), ip))
                result['message'].insert(0, message)
            return HttpResponse(json.dumps(result, default=json_handler),
                                content_type="application/json")

        return HttpResponse(json.dumps({
            'success': False,
            'form': form.as_table()
        }),
                            content_type="application/json")
    return render_to_response("error.html", {'error': 'Expected AJAX/POST'},
                              RequestContext(request))
Ejemplo n.º 9
0
def add_new_domain(data, request, errors, rowData=None, is_validate_only=False, cache={}):
    """
    Add a new domain to CRITs.

    :param data: The data about the domain.
    :type data: dict
    :param request: The Django request.
    :type request: :class:`django.http.HttpRequest`
    :param errors: A list of current errors to append to.
    :type errors: list
    :param rowData: Any objects that need to be added to the domain.
    :type rowData: dict
    :param is_validate_only: Only validate the data and return any errors.
    :type is_validate_only: boolean
    :param cache: Cached data, typically for performance enhancements
                  during bulk operations.
    :type cache: dict
    :returns: tuple (<result>, <errors>, <retVal>)
    """

    result = False
    retVal = {}
    domain = data['domain']
    add_ip = data.get('add_ip')
    ip = data.get('ip')
    ip_type = data.get('ip_type')

    if add_ip:
        error = validate_and_normalize_ip(ip, ip_type)[1]
        if error:
             errors.append(error)

    if is_validate_only:
        error = get_valid_root_domain(domain)[2]
        if error:
            errors.append(error)

        # check for duplicate domains
        fqdn_domain = retrieve_domain(domain, cache)

        if fqdn_domain:
            if isinstance(fqdn_domain, Domain):
                resp_url = reverse('crits.domains.views.domain_detail', args=[domain])
                message = ('Warning: Domain already exists: '
                                     '<a href="%s">%s</a>' % (resp_url, domain))
                retVal['message'] = message
                retVal['status'] = form_consts.Status.DUPLICATE
                retVal['warning'] = message
        else:
            result_cache = cache.get(form_consts.Domain.CACHED_RESULTS);
            result_cache[domain.lower()] = True

    elif not errors:
        username = request.user.username
        reference = data.get('domain_reference')
        source_name = data.get('domain_source')
        method = data.get('domain_method')
        source = [create_embedded_source(source_name, reference=reference,
                                         method=method, analyst=username)]
        bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
        ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)

        if data.get('campaign') and data.get('confidence'):
            campaign = [EmbeddedCampaign(name=data.get('campaign'),
                                         confidence=data.get('confidence'),
                                         analyst=username)]
        else:
            campaign = []

        retVal = upsert_domain(domain, source, username, campaign,
                               bucket_list=bucket_list, ticket=ticket, cache=cache)

        if not retVal['success']:
            errors.append(retVal.get('message'))
            retVal['message'] = ""

        else:
            new_domain = retVal['object']
            ip_result = {}
            if add_ip:
                if data.get('same_source'):
                    ip_source = source_name
                    ip_method = method
                    ip_reference = reference
                else:
                    ip_source = data.get('ip_source')
                    ip_method = data.get('ip_method')
                    ip_reference = data.get('ip_reference')
                from crits.ips.handlers import ip_add_update
                ip_result = ip_add_update(ip,
                                          ip_type,
                                          ip_source,
                                          ip_method,
                                          ip_reference,
                                          campaign=campaign,
                                          analyst=username,
                                          bucket_list=bucket_list,
                                          ticket=ticket,
                                          cache=cache)
                if not ip_result['success']:
                    errors.append(ip_result['message'])
                else:
                    #add a relationship with the new IP address
                    new_ip = ip_result['object']
                    if new_domain and new_ip:
                        new_domain.add_relationship(new_ip,
                                                    'Resolved_To',
                                                    analyst=username,
                                                    get_rels=False)
                        new_domain.save(username=username)

            #set the URL for viewing the new data
            resp_url = reverse('crits.domains.views.domain_detail', args=[domain])

            if retVal['is_domain_new'] == True:
                retVal['message'] = ('Success! Click here to view the new domain: '
                                     '<a href="%s">%s</a>' % (resp_url, domain))
            else:
                message = ('Updated existing domain: <a href="%s">%s</a>' % (resp_url, domain))
                retVal['message'] = message
                retVal[form_consts.Status.STATUS_FIELD] = form_consts.Status.DUPLICATE
                retVal['warning'] = message

            #add indicators
            if data.get('add_indicators'):
                from crits.indicators.handlers import create_indicator_from_tlo
                # If we have an IP object, add an indicator for that.
                if ip_result.get('success'):
                    ip = ip_result['object']
                    result = create_indicator_from_tlo('IP',
                                                       ip,
                                                       username,
                                                       ip_source,
                                                       add_domain=False)
                    ip_ind = result.get('indicator')
                    if not result['success']:
                        errors.append(result['message'])

                # Add an indicator for the domain.
                result = create_indicator_from_tlo('Domain',
                                                   new_domain,
                                                   username,
                                                   source_name,
                                                   add_domain=False)

                if not result['success']:
                    errors.append(result['message'])
                elif ip_result.get('success') and ip_ind:
                    forge_relationship(left_class=result['indicator'],
                                       right_class=ip_ind,
                                       rel_type='Resolved_To',
                                       analyst=username)
            result = True

    # This block validates, and may also add, objects to the Domain
    if retVal.get('success') or is_validate_only == True:
        if rowData:
            objectsData = rowData.get(form_consts.Common.OBJECTS_DATA)

            # add new objects if they exist
            if objectsData:
                objectsData = json.loads(objectsData)
                current_domain = retrieve_domain(domain, cache)
                for object_row_counter, objectData in enumerate(objectsData, 1):
                    if current_domain != None:
                        # if the domain exists then try to add objects to it
                        if isinstance(current_domain, Domain) == True:
                            objectDict = object_array_to_dict(objectData,
                                                              "Domain",
                                                              current_domain.id)
                        else:
                            objectDict = object_array_to_dict(objectData,
                                                              "Domain",
                                                              "")
                            current_domain = None;
                    else:
                        objectDict = object_array_to_dict(objectData,
                                                          "Domain",
                                                          "")

                    (obj_result,
                     errors,
                     obj_retVal) = validate_and_add_new_handler_object(
                        None, objectDict, request, errors, object_row_counter,
                        is_validate_only=is_validate_only,
                        cache=cache, obj=current_domain)
                    if not obj_result:
                        retVal['success'] = False

    return result, errors, retVal
Ejemplo n.º 10
0
def handle_indicator_insert(ind, source, reference='', analyst='', method='',
                            add_domain=False, add_relationship=False, cache={}):
    """
    Insert an individual indicator into the database.

    NOTE: Setting add_domain to True will always create a relationship as well.
    However, to create a relationship with an object that already exists before
    this function was called, set add_relationship to True. This will assume
    that the domain or IP object to create the relationship with already exists
    and will avoid infinite mutual calls between, for example, add_update_ip
    and this function. add domain/IP objects.

    :param ind: Information about the indicator.
    :type ind: dict
    :param source: The source for this indicator.
    :type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
    :param reference: The reference to the data.
    :type reference: str
    :param analyst: The user adding this indicator.
    :type analyst: str
    :param method: Method of acquiring this indicator.
    :type method: str
    :param add_domain: If this indicator is also a top-level object, try to add
                       it.
    :type add_domain: boolean
    :param add_relationship: Attempt to add relationships if applicable.
    :type add_relationship: boolean
    :param cache: Cached data, typically for performance enhancements
                  during bulk uperations.
    :type cache: dict
    :returns: dict with keys:
              "success" (boolean),
              "message" str) if failed,
              "objectid" (str) if successful,
              "is_new_indicator" (boolean) if successful.
    """

    if ind['type'] == "URI - URL" and "://" not in ind['value'].split('.')[0]:
        return {"success": False, "message": "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://) "}

    is_new_indicator = False
    dmain = None
    ip = None
    rank = {
        'unknown': 0,
        'benign': 1,
        'low': 2,
        'medium': 3,
        'high': 4,
    }

    indicator = Indicator.objects(ind_type=ind['type'],
                                  value=ind['value']).first()
    if not indicator:
        indicator = Indicator()
        indicator.ind_type = ind['type']
        indicator.value = ind['value']
        indicator.created = datetime.datetime.now()
        indicator.confidence = EmbeddedConfidence(analyst=analyst)
        indicator.impact = EmbeddedImpact(analyst=analyst)
        is_new_indicator = True

    if 'campaign' in ind:
        if isinstance(ind['campaign'], basestring) and len(ind['campaign']) > 0:
            confidence = ind.get('campaign_confidence', 'low')
            ind['campaign'] = EmbeddedCampaign(name=ind['campaign'],
                                               confidence=confidence,
                                               description="",
                                               analyst=analyst,
                                               date=datetime.datetime.now())
        if isinstance(ind['campaign'], EmbeddedCampaign):
            indicator.add_campaign(ind['campaign'])
        elif isinstance(ind['campaign'], list):
            for campaign in ind['campaign']:
                if isinstance(campaign, EmbeddedCampaign):
                    indicator.add_campaign(campaign)

    if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(indicator.confidence.rating, 0):
        indicator.confidence.rating = ind['confidence']
        indicator.confidence.analyst = analyst

    if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(indicator.impact.rating, 0):
        indicator.impact.rating = ind['impact']
        indicator.impact.analyst = analyst

    bucket_list = None
    if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
        bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
        if bucket_list:
            indicator.add_bucket_list(bucket_list, analyst)

    ticket = None
    if form_consts.Common.TICKET_VARIABLE_NAME in ind:
        ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
        if ticket:
            indicator.add_ticket(ticket, analyst)

    if isinstance(source, list):
        for s in source:
            indicator.add_source(source_item=s, method=method, reference=reference)
    elif isinstance(source, EmbeddedSource):
        indicator.add_source(source_item=source, method=method, reference=reference)
    elif isinstance(source, basestring):
        s = EmbeddedSource()
        s.name = source
        instance = EmbeddedSource.SourceInstance()
        instance.reference = reference
        instance.method = method
        instance.analyst = analyst
        instance.date = datetime.datetime.now()
        s.instances = [instance]
        indicator.add_source(s)

    if add_domain or add_relationship:
        ind_type = indicator.ind_type
        ind_value = indicator.value
        url_contains_ip = False
        if ind_type in ("URI - Domain Name", "URI - URL"):
            if ind_type == "URI - URL":
                domain_or_ip = urlparse.urlparse(ind_value).hostname
            elif ind_type == "URI - Domain Name":
                domain_or_ip = ind_value
            (sdomain, fqdn) = get_domain(domain_or_ip)
            if sdomain == "no_tld_found_error" and ind_type == "URI - URL":
                try:
                    validate_ipv46_address(domain_or_ip)
                    url_contains_ip = True
                except DjangoValidationError:
                    pass
            if not url_contains_ip:
                success = None
                if add_domain:
                    success = upsert_domain(sdomain, fqdn, indicator.source,
                                            '%s' % analyst, None,
                                            bucket_list=bucket_list, cache=cache)
                    if not success['success']:
                        return {'success': False, 'message': success['message']}

                if not success or not 'object' in success:
                    dmain = Domain.objects(domain=domain_or_ip).first()
                else:
                    dmain = success['object']

        if ind_type.startswith("Address - ip") or ind_type == "Address - cidr" or url_contains_ip:
            if url_contains_ip:
                ind_value = domain_or_ip
                try:
                    validate_ipv4_address(domain_or_ip)
                    ind_type = 'Address - ipv4-addr'
                except DjangoValidationError:
                    ind_type = 'Address - ipv6-addr'
            success = None
            if add_domain:
                success = ip_add_update(ind_value,
                                        ind_type,
                                        source=indicator.source,
                                        campaign=indicator.campaign,
                                        analyst=analyst,
                                        bucket_list=bucket_list,
                                        ticket=ticket,
                                        indicator_reference=reference,
                                        cache=cache)
                if not success['success']:
                    return {'success': False, 'message': success['message']}

            if not success or not 'object' in success:
                ip = IP.objects(ip=indicator.value).first()
            else:
                ip = success['object']

    indicator.save(username=analyst)

    if dmain:
        dmain.add_relationship(rel_item=indicator,
                               rel_type='Related_To',
                               analyst="%s" % analyst,
                               get_rels=False)
        dmain.save(username=analyst)
    if ip:
        ip.add_relationship(rel_item=indicator,
                            rel_type='Related_To',
                            analyst="%s" % analyst,
                            get_rels=False)
        ip.save(username=analyst)

    indicator.save(username=analyst)

    # run indicator triage
    if is_new_indicator:
        indicator.reload()
        run_triage(indicator, analyst)

    return {'success': True, 'objectid': str(indicator.id),
            'is_new_indicator': is_new_indicator, 'object': indicator}
Ejemplo n.º 11
0
    def parse_observables(self, observables):
        """
        Parse list of observables in STIX doc.

        :param observables: List of STIX observables.
        :type observables: List of STIX observables.
        """

        analyst = self.source_instance.analyst
        for obs in observables: # for each STIX observable
            if not obs.object_ or not obs.object_.properties:
                self.failed.append(("No valid object_properties was found!",
                                    type(obs).__name__,
                                    obs.id_)) # note for display in UI
                continue
            try: # try to create CRITs object from observable
                item = obs.object_.properties
                if isinstance(item, Address):
                    if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
                                         'ipv4-netmask', 'ipv6-addr',
                                         'ipv6-net', 'ipv6-netmask'):
                        imp_type = "IP"
                        for value in item.address_value.values:
                            ip = str(value).strip()
                            iptype = get_crits_ip_type(item.category)
                            if iptype:
                                res = ip_add_update(ip,
                                                    iptype,
                                                    [self.source],
                                                    analyst=analyst,
                                                    is_add_indicator=True)
                                self.parse_res(imp_type, obs, res)
                if isinstance(item, DomainName):
                    imp_type = "Domain"
                    for value in item.value.values:
                        res = upsert_domain(str(value),
                                            [self.source],
                                            username=analyst)
                        self.parse_res(imp_type, obs, res)
                elif isinstance(item, Artifact):
                    # Not sure if this is right, and I believe these can be
                    # encoded in a couple different ways.
                    imp_type = "RawData"
                    rawdata = item.data.decode('utf-8')
                    description = "None"
                    # TODO: find out proper ways to determine title, datatype,
                    #       tool_name, tool_version
                    title = "Artifact for Event: STIX Document %s" % self.package.id_
                    res = handle_raw_data_file(rawdata,
                                            self.source.name,
                                            user=analyst,
                                            description=description,
                                            title=title,
                                            data_type="Text",
                                            tool_name="STIX",
                                            tool_version=None,
                                            method=self.source_instance.method,
                                            reference=self.source_instance.reference)
                    self.parse_res(imp_type, obs, res)
                elif (isinstance(item, File) and
                      item.custom_properties and
                      item.custom_properties[0].name == "crits_type" and
                      item.custom_properties[0]._value == "Certificate"):
                    imp_type = "Certificate"
                    description = "None"
                    filename = str(item.file_name)
                    data = None
                    for obj in item.parent.related_objects:
                        if isinstance(obj.properties, Artifact):
                            data = obj.properties.data
                    res = handle_cert_file(filename,
                                           data,
                                           self.source,
                                           user=analyst,
                                           description=description)
                    self.parse_res(imp_type, obs, res)
                elif isinstance(item, File) and self.has_network_artifact(item):
                    imp_type = "PCAP"
                    description = "None"
                    filename = str(item.file_name)
                    data = None
                    for obj in item.parent.related_objects:
                        if (isinstance(obj.properties, Artifact) and
                            obj.properties.type_ == Artifact.TYPE_NETWORK):
                            data = obj.properties.data
                    res = handle_pcap_file(filename,
                                           data,
                                           self.source,
                                           user=analyst,
                                           description=description)
                    self.parse_res(imp_type, obs, res)
                elif isinstance(item, File):
                    imp_type = "Sample"
                    filename = str(item.file_name)
                    md5 = item.md5
                    data = None
                    for obj in item.parent.related_objects:
                        if (isinstance(obj.properties, Artifact) and
                            obj.properties.type_ == Artifact.TYPE_FILE):
                            data = obj.properties.data
                    res = handle_file(filename,
                                      data,
                                      self.source,
                                      user=analyst,
                                      md5_digest=md5,
                                      is_return_only_md5=False)
                    self.parse_res(imp_type, obs, res)
                elif isinstance(item, EmailMessage):
                    imp_type = "Email"
                    data = {}
                    data['source'] = self.source.name
                    data['source_method'] = self.source_instance.method
                    data['source_reference'] = self.source_instance.reference
                    data['raw_body'] = str(item.raw_body)
                    data['raw_header'] = str(item.raw_header)
                    data['helo'] = str(item.email_server)
                    if item.header:
                        data['message_id'] = str(item.header.message_id)
                        data['subject'] = str(item.header.subject)
                        data['sender'] = str(item.header.sender)
                        data['reply_to'] = str(item.header.reply_to)
                        data['x_originating_ip'] = str(item.header.x_originating_ip)
                        data['x_mailer'] = str(item.header.x_mailer)
                        data['boundary'] = str(item.header.boundary)
                        data['from_address'] = str(item.header.from_)
                        data['date'] = item.header.date.value
                        if item.header.to:
                            data['to'] = [str(r) for r in item.header.to.to_list()]
                    res = handle_email_fields(data,
                                            analyst,
                                            "STIX")
                    # Should check for attachments and add them here.
                    self.parse_res(imp_type, obs, res)
                    if res.get('status') and item.attachments:
                        for attach in item.attachments:
                            rel_id = attach.to_dict()['object_reference']
                            self.relationships.append((obs.id_,
                                                       "Contains",
                                                       rel_id, "High"))
                else: # try to parse all other possibilities as Indicator
                    imp_type = "Indicator"
                    obj = make_crits_object(item)
                    if obj.object_type == 'Address':
                        # This was already caught above
                        continue
                    else:
                        ind_type = obj.object_type
                        for value in obj.value:
                            if value and ind_type:
                                res = handle_indicator_ind(value.strip(),
                                                        self.source,
                                                        ind_type,
                                                        IndicatorThreatTypes.UNKNOWN,
                                                        IndicatorAttackTypes.UNKNOWN,
                                                        analyst,
                                                        add_domain=True,
                                                        add_relationship=True)
                                self.parse_res(imp_type, obs, res)
            except Exception, e: # probably caused by cybox object we don't handle
                self.failed.append((e.message,
                                    type(item).__name__,
                                    item.parent.id_)) # note for display in UI
Ejemplo n.º 12
0
def handle_indicator_insert(
    ind, source, reference="", analyst="", method="", add_domain=False, add_relationship=False, cache={}
):
    """
    Insert an individual indicator into the database.

    NOTE: Setting add_domain to True will always create a relationship as well.
    However, to create a relationship with an object that already exists before
    this function was called, set add_relationship to True. This will assume
    that the domain or IP object to create the relationship with already exists
    and will avoid infinite mutual calls between, for example, add_update_ip
    and this function. add domain/IP objects.

    :param ind: Information about the indicator.
    :type ind: dict
    :param source: The source for this indicator.
    :type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
    :param reference: The reference to the data.
    :type reference: str
    :param analyst: The user adding this indicator.
    :type analyst: str
    :param method: Method of acquiring this indicator.
    :type method: str
    :param add_domain: If this indicator is also a top-level object, try to add
                       it.
    :type add_domain: boolean
    :param add_relationship: Attempt to add relationships if applicable.
    :type add_relationship: boolean
    :param cache: Cached data, typically for performance enhancements
                  during bulk uperations.
    :type cache: dict
    :returns: dict with keys:
              "success" (boolean),
              "message" (str) if failed,
              "objectid" (str) if successful,
              "is_new_indicator" (boolean) if successful.
    """

    if ind["type"] not in IndicatorTypes.values():
        return {"success": False, "message": "Not a valid Indicator Type: %s" % ind["type"]}
    if ind["threat_type"] not in IndicatorThreatTypes.values():
        return {"success": False, "message": "Not a valid Indicator Threat Type: %s" % ind["threat_type"]}
    if ind["attack_type"] not in IndicatorAttackTypes.values():
        return {"success": False, "message": "Not a valid Indicator Attack Type: " % ind["attack_type"]}

    (ind["value"], error) = validate_indicator_value(ind["value"], ind["type"])
    if error:
        return {"success": False, "message": error}

    is_new_indicator = False
    dmain = None
    ip = None
    rank = {"unknown": 0, "benign": 1, "low": 2, "medium": 3, "high": 4}

    if ind.get("status", None) is None or len(ind.get("status", "")) < 1:
        ind["status"] = Status.NEW

    indicator = Indicator.objects(
        ind_type=ind["type"], lower=ind["lower"], threat_type=ind["threat_type"], attack_type=ind["attack_type"]
    ).first()
    if not indicator:
        indicator = Indicator()
        indicator.ind_type = ind["type"]
        indicator.threat_type = ind["threat_type"]
        indicator.attack_type = ind["attack_type"]
        indicator.value = ind["value"]
        indicator.lower = ind["lower"]
        indicator.description = ind["description"]
        indicator.created = datetime.datetime.now()
        indicator.confidence = EmbeddedConfidence(analyst=analyst)
        indicator.impact = EmbeddedImpact(analyst=analyst)
        indicator.status = ind["status"]
        is_new_indicator = True
    else:
        if ind["status"] != Status.NEW:
            indicator.status = ind["status"]
        add_desc = "\nSeen on %s as: %s" % (str(datetime.datetime.now()), ind["value"])
        if indicator.description is None:
            indicator.description = add_desc
        else:
            indicator.description += add_desc

    if "campaign" in ind:
        if isinstance(ind["campaign"], basestring) and len(ind["campaign"]) > 0:
            confidence = ind.get("campaign_confidence", "low")
            ind["campaign"] = EmbeddedCampaign(
                name=ind["campaign"],
                confidence=confidence,
                description="",
                analyst=analyst,
                date=datetime.datetime.now(),
            )
        if isinstance(ind["campaign"], EmbeddedCampaign):
            indicator.add_campaign(ind["campaign"])
        elif isinstance(ind["campaign"], list):
            for campaign in ind["campaign"]:
                if isinstance(campaign, EmbeddedCampaign):
                    indicator.add_campaign(campaign)

    if "confidence" in ind and rank.get(ind["confidence"], 0) > rank.get(indicator.confidence.rating, 0):
        indicator.confidence.rating = ind["confidence"]
        indicator.confidence.analyst = analyst

    if "impact" in ind and rank.get(ind["impact"], 0) > rank.get(indicator.impact.rating, 0):
        indicator.impact.rating = ind["impact"]
        indicator.impact.analyst = analyst

    bucket_list = None
    if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
        bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
        if bucket_list:
            indicator.add_bucket_list(bucket_list, analyst)

    ticket = None
    if form_consts.Common.TICKET_VARIABLE_NAME in ind:
        ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
        if ticket:
            indicator.add_ticket(ticket, analyst)

    if isinstance(source, list):
        for s in source:
            indicator.add_source(source_item=s, method=method, reference=reference)
    elif isinstance(source, EmbeddedSource):
        indicator.add_source(source_item=source, method=method, reference=reference)
    elif isinstance(source, basestring):
        s = EmbeddedSource()
        s.name = source
        instance = EmbeddedSource.SourceInstance()
        instance.reference = reference
        instance.method = method
        instance.analyst = analyst
        instance.date = datetime.datetime.now()
        s.instances = [instance]
        indicator.add_source(s)

    if add_domain or add_relationship:
        ind_type = indicator.ind_type
        ind_value = indicator.lower
        url_contains_ip = False
        if ind_type in (IndicatorTypes.DOMAIN, IndicatorTypes.URI):
            if ind_type == IndicatorTypes.URI:
                domain_or_ip = urlparse.urlparse(ind_value).hostname
                try:
                    validate_ipv46_address(domain_or_ip)
                    url_contains_ip = True
                except DjangoValidationError:
                    pass
            else:
                domain_or_ip = ind_value
            if not url_contains_ip:
                success = None
                if add_domain:
                    success = upsert_domain(
                        domain_or_ip,
                        indicator.source,
                        username="******" % analyst,
                        campaign=indicator.campaign,
                        bucket_list=bucket_list,
                        cache=cache,
                    )
                    if not success["success"]:
                        return {"success": False, "message": success["message"]}

                if not success or not "object" in success:
                    dmain = Domain.objects(domain=domain_or_ip).first()
                else:
                    dmain = success["object"]

        if ind_type in IPTypes.values() or url_contains_ip:
            if url_contains_ip:
                ind_value = domain_or_ip
                try:
                    validate_ipv4_address(domain_or_ip)
                    ind_type = IndicatorTypes.IPV4_ADDRESS
                except DjangoValidationError:
                    ind_type = IndicatorTypes.IPV6_ADDRESS
            success = None
            if add_domain:
                success = ip_add_update(
                    ind_value,
                    ind_type,
                    source=indicator.source,
                    campaign=indicator.campaign,
                    analyst=analyst,
                    bucket_list=bucket_list,
                    ticket=ticket,
                    indicator_reference=reference,
                    cache=cache,
                )
                if not success["success"]:
                    return {"success": False, "message": success["message"]}

            if not success or not "object" in success:
                ip = IP.objects(ip=indicator.value).first()
            else:
                ip = success["object"]

    indicator.save(username=analyst)

    if dmain:
        dmain.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
        dmain.save(username=analyst)
    if ip:
        ip.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
        ip.save(username=analyst)

    # run indicator triage
    if is_new_indicator:
        indicator.reload()
        run_triage(indicator, analyst)

    return {"success": True, "objectid": str(indicator.id), "is_new_indicator": is_new_indicator, "object": indicator}
Ejemplo n.º 13
0
def handle_indicator_insert(ind,
                            source,
                            reference='',
                            analyst='',
                            method='',
                            add_domain=False,
                            add_relationship=False,
                            cache={}):
    """
    Insert an individual indicator into the database.

    NOTE: Setting add_domain to True will always create a relationship as well.
    However, to create a relationship with an object that already exists before
    this function was called, set add_relationship to True. This will assume
    that the domain or IP object to create the relationship with already exists
    and will avoid infinite mutual calls between, for example, add_update_ip
    and this function. add domain/IP objects.

    :param ind: Information about the indicator.
    :type ind: dict
    :param source: The source for this indicator.
    :type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
    :param reference: The reference to the data.
    :type reference: str
    :param analyst: The user adding this indicator.
    :type analyst: str
    :param method: Method of acquiring this indicator.
    :type method: str
    :param add_domain: If this indicator is also a top-level object, try to add
                       it.
    :type add_domain: boolean
    :param add_relationship: Attempt to add relationships if applicable.
    :type add_relationship: boolean
    :param cache: Cached data, typically for performance enhancements
                  during bulk uperations.
    :type cache: dict
    :returns: dict with keys:
              "success" (boolean),
              "message" str) if failed,
              "objectid" (str) if successful,
              "is_new_indicator" (boolean) if successful.
    """

    is_new_indicator = False
    rank = {'unknown': 0, 'benign': 1, 'low': 2, 'medium': 3, 'high': 4}

    indicator = Indicator.objects(ind_type=ind['type'],
                                  value=ind['value']).first()
    if not indicator:
        indicator = Indicator()
        indicator.ind_type = ind['type']
        indicator.value = ind['value']
        indicator.created = datetime.datetime.now()
        indicator.confidence = EmbeddedConfidence(analyst=analyst)
        indicator.impact = EmbeddedImpact(analyst=analyst)
        is_new_indicator = True

    ec = None
    if 'campaign' in ind:
        confidence = 'low'

        if 'campaign_confidence' in ind:
            confidence = ind['campaign_confidence']

        ec = EmbeddedCampaign(name=ind['campaign'],
                              confidence=confidence,
                              description="",
                              analyst=analyst,
                              date=datetime.datetime.now())

    if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(
            indicator.confidence.rating, 0):
        indicator.confidence.rating = ind['confidence']
        indicator.confidence.analyst = analyst

    if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(
            indicator.impact.rating, 0):
        indicator.impact.rating = ind['impact']
        indicator.impact.analyst = analyst

    bucket_list = None
    if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
        bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
        indicator.add_bucket_list(bucket_list, analyst)

    ticket = None
    if form_consts.Common.TICKET_VARIABLE_NAME in ind:
        ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
        indicator.add_ticket(ticket, analyst)

    if isinstance(source, list):
        for s in source:
            indicator.add_source(source_item=s)
    elif isinstance(source, EmbeddedSource):
        indicator.add_source(source_item=source)
    elif isinstance(source, basestring):
        s = EmbeddedSource()
        s.name = source
        instance = EmbeddedSource.SourceInstance()
        instance.reference = reference
        instance.method = method
        instance.analyst = analyst
        instance.date = datetime.datetime.now()
        s.instances = [instance]
        indicator.add_source(s)
    if ec:
        indicator.add_campaign(ec)
    indicator.save(username=analyst)

    if add_domain or add_relationship:
        ind_type = indicator.ind_type
        ind_value = indicator.value
        if ind_type in ("URI - Domain Name", "URI - URL"):
            if ind_type == "URI - URL":
                domain = ind_value.split("/")[2]
            elif ind_type == "URI - Domain Name":
                domain = ind_value
            #try:
            (sdomain, fqdn) = get_domain(domain)
            success = None
            if add_domain:
                success = upsert_domain(sdomain,
                                        fqdn,
                                        indicator.source,
                                        '%s' % analyst,
                                        None,
                                        bucket_list=bucket_list,
                                        cache=cache)
                if not success['success']:
                    return {'success': False, 'message': success['message']}

            if not success or not 'object' in success:
                dmain = Domain.objects(domain=domain).first()
            else:
                dmain = success['object']
            if dmain:
                dmain.add_relationship(rel_item=indicator,
                                       rel_type='Related_To',
                                       analyst="%s" % analyst,
                                       get_rels=False)
                dmain.save(username=analyst)
                indicator.save(username=analyst)

        elif ind_type.startswith(
                "Address - ip") or ind_type == "Address - cidr":
            success = None
            if add_domain:
                success = ip_add_update(indicator.value,
                                        ind_type,
                                        source=indicator.source,
                                        campaign=indicator.campaign,
                                        analyst=analyst,
                                        bucket_list=bucket_list,
                                        ticket=ticket,
                                        indicator_reference=reference,
                                        cache=cache)
                if not success['success']:
                    return {'success': False, 'message': success['message']}

            if not success or not 'object' in success:
                ip = IP.objects(ip=indicator.value).first()
            else:
                ip = success['object']
            if ip:
                ip.add_relationship(rel_item=indicator,
                                    rel_type='Related_To',
                                    analyst="%s" % analyst,
                                    get_rels=False)
                ip.save(username=analyst)
                indicator.save(username=analyst)

    # run indicator triage
    if is_new_indicator:
        indicator.reload()
        run_triage(None, indicator, analyst)

    return {
        'success': True,
        'objectid': indicator.id,
        'is_new_indicator': is_new_indicator,
        'object': indicator
    }
Ejemplo n.º 14
0
def add_new_domain(data, request, errors, rowData=None, is_validate_only=False, cache={}):
    """
    Add a new domain to CRITs.

    :param data: The data about the domain.
    :type data: dict
    :param request: The Django request.
    :type request: :class:`django.http.HttpRequest`
    :param errors: A list of current errors to append to.
    :type errors: list
    :param rowData: Any objects that need to be added to the domain.
    :type rowData: dict
    :param is_validate_only: Only validate the data and return any errors.
    :type is_validate_only: boolean
    :param cache: Cached data, typically for performance enhancements
                  during bulk uperations.
    :type cache: dict
    :returns: tuple
    """

    username = request.user.username
    result = False
    retVal = {}
    reference = data.get('domain_reference')
    name = data.get('domain_source')
    method = data.get('domain_method')
    source = [create_embedded_source(name, reference=reference, method=method,
                                     analyst=username)]
    bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
    ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)

    if data.get('campaign') and data.get('confidence'):
        campaign = [EmbeddedCampaign(name=data.get('campaign'),
                                     confidence=data.get('confidence'),
                                     analyst=username)]
    else:
        campaign = []

    (sdomain, fqdn) = get_domain(data['domain'])
    if sdomain == "no_tld_found_error":
        errors.append(u"Error: Invalid domain: " + data['domain'])
    elif is_validate_only == False:
        retVal = upsert_domain(sdomain, fqdn, source, username, campaign,
                               bucket_list=bucket_list, ticket=ticket, cache=cache)
        ip_result = None

        if retVal['success']:
            add_ip = data.get('add_ip')
            if add_ip:
                ip = data.get('ip')
                if data.get('same_source'):
                    ip_source = data.get('domain_source')
                    ip_method = data.get('domain_method')
                    ip_reference = reference
                else:
                    ip_source = data.get('ip_source')
                    ip_method = data.get('ip_method')
                    ip_reference = data.get('ip_reference')
                from crits.ips.handlers import ip_add_update
                ip_result = ip_add_update(ip,
                                          None,
                                          ip_source,
                                          ip_method,
                                          ip_reference,
                                          campaign=campaign,
                                          analyst=username,
                                          bucket_list=bucket_list,
                                          ticket=ticket,
                                          cache=cache)
                if not ip_result['success']:
                    errors.append(ip_result['message'])
                else:
                    #add a relationship with the new IP address
                    new_domain = retVal['object']
                    new_ip = ip_result['object']
                    if new_domain and new_ip:
                        new_domain.add_relationship(rel_item=new_ip,
                                                    rel_type='Resolved_To',
                                                    analyst=username,
                                                    get_rels=False)
                        new_domain.save(username=username)
                        new_ip.save(username=username)

            #set the URL for viewing the new data
            resp_url = reverse('crits.domains.views.domain_detail', args=[fqdn])

            if retVal['is_domain_new'] == True:
                retVal['message'] = ('Success! Click here to view the new domain: '
                                     '<a href="%s">%s</a>' % (resp_url, fqdn))
            else:
                message = ('Updated existing domain: <a href="%s">%s</a>' % (resp_url, fqdn))
                retVal['message'] = message
                retVal[form_consts.Status.STATUS_FIELD] = form_consts.Status.DUPLICATE
                retVal['warning'] = message

            #add indicators
            # add_indicators_for_domain handles adding relationships as well
            if data.get('add_indicators'):
                analyst = username
                from crits.indicators.handlers import add_indicators_for_domain
                if ip_result and ip_result['success']:
                    obj = ip_result['object']
                    errors += add_indicators_for_domain(sdomain,
                                                        fqdn,
                                                        source,
                                                        analyst,
                                                        reference,
                                                        obj,
                                                        bucket_list=bucket_list,
                                                        ticket=ticket)['errors']
                else:
                    errors += add_indicators_for_domain(sdomain,
                                                        fqdn,
                                                        source,
                                                        analyst,
                                                        reference,
                                                        bucket_list=bucket_list,
                                                        ticket=ticket)['errors']
            result = True

        elif 'message' in retVal: #database error? (!c_dom)
            errors.append(retVal['message']) #u"Unknown error: unable to add domain")

    elif is_validate_only == True:
        domain = data['domain']
        fqdn_domain = retrieve_domain(domain, cache);

        if fqdn_domain:
            if isinstance(fqdn_domain, Domain):
                resp_url = reverse('crits.domains.views.domain_detail', args=[fqdn])
                message = ('Warning: Domain already exists: '
                                     '<a href="%s">%s</a>' % (resp_url, fqdn))
                retVal['message'] = message;
                retVal['status'] = form_consts.Status.DUPLICATE;
                retVal['warning'] = message;
        else:
            result_cache = cache.get(form_consts.Domain.CACHED_RESULTS);
            result_cache[domain.lower()] = True;

    # This block tries to add objects to the item
    if retVal.get('success') or is_validate_only == True:
        if rowData:
            objectsData = rowData.get(form_consts.Common.OBJECTS_DATA)

            # add new objects if they exist
            if objectsData:
                objectsData = json.loads(objectsData)
                object_row_counter = 1
                current_domain = retrieve_domain(fqdn, cache)
                for objectData in objectsData:
                    if current_domain != None:
                        # if the domain exists then try to add objects to it
                        if isinstance(current_domain, Domain) == True:
                            objectDict = object_array_to_dict(objectData, "Domain", current_domain.id)
                        else:
                            objectDict = object_array_to_dict(objectData, "Domain", "")
                            current_domain = None;
                    else:
                        objectDict = object_array_to_dict(objectData, "Domain", "")

                    (object_result, object_errors, object_retVal) = validate_and_add_new_handler_object(
                            None, objectDict, request, errors, object_row_counter,
                            is_validate_only=is_validate_only,
                            cache=cache, obj=current_domain)

                    if object_retVal.get('success') == False:
                        retVal['success'] = False
                    if object_retVal.get('message'):
                        errors.append(object_retVal['message'])

                    object_row_counter += 1

    return result, errors, retVal
Ejemplo n.º 15
0
def add_update_ip(request, method):
    """
    Add/update an IP address. Should be an AJAX POST.

    :param request: Django request.
    :type request: :class:`django.http.HttpRequest`
    :param method: If this is an "add" or an "update".
    :type method: str
    :returns: :class:`django.http.HttpResponse`
    """

    if request.method == "POST" and request.is_ajax():
        request.user._setup()
        data = request.POST
        form = AddIPForm(request.user, None, data)

        if form.is_valid():
            cleaned_data = form.cleaned_data
            ip = cleaned_data['ip']
            source_name = cleaned_data['source_name']
            source_reference = cleaned_data['source_reference']
            source_method = cleaned_data['source_method']
            source_tlp= cleaned_data['source_tlp']
            campaign = cleaned_data['campaign']
            confidence = cleaned_data['confidence']
            ip_type = cleaned_data['ip_type']
            add_indicator = False
            if cleaned_data.get('add_indicator'):
                add_indicator = True
            indicator_reference = cleaned_data.get('indicator_reference')
            bucket_list = cleaned_data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
            ticket = cleaned_data.get(form_consts.Common.TICKET_VARIABLE_NAME)
            related_id = cleaned_data['related_id']
            related_type = cleaned_data['related_type']
            relationship_type = cleaned_data['relationship_type']

            result = ip_add_update(ip,
                                   ip_type,
                                   source=source_name,
                                   source_method=source_method,
                                   source_reference=source_reference,
                                   source_tlp=source_tlp,
                                   campaign=campaign,
                                   confidence=confidence,
                                   user=request.user,
                                   bucket_list=bucket_list,
                                   ticket=ticket,
                                   is_add_indicator=add_indicator,
                                   indicator_reference=indicator_reference,
                                   related_id=related_id,
                                   related_type=related_type,
                                   relationship_type=relationship_type)
            if 'message' in result:
                if not isinstance(result['message'], list):
                    result['message'] = [result['message']]
            else:
                result['message'] = []
                message = ('<div>Success! Click here to view the new IP: <a '
                           'href="%s">%s</a></div>'
                           % (reverse('crits.ips.views.ip_detail',
                                      args=[ip]),
                              ip))
                result['message'].insert(0, message)
            return HttpResponse(json.dumps(result,
                                           default=json_handler),
                                content_type="application/json")

        return HttpResponse(json.dumps({'success': False,
                                        'form':form.as_table()}),
                            content_type="application/json")
    return render_to_response("error.html",
                              {'error': 'Expected AJAX/POST'},
                              RequestContext(request))
Ejemplo n.º 16
0
    def parse_cybox_object(self, cbx_obj, description='', ind_id=None):
        """
        Parse a CybOX object form a STIX doc. An object can contain
        multiple related_objects, which in turn can have their own
        related_objects, so this handles those recursively.

        :param cbx_obj: The CybOX object to parse.
        :type cbx_obj: A CybOX object.
        :param description: Parent-level (e.g. Observable) description.
        :type description: str
        :param ind_id: The ID of a parent STIX Indicator.
        :type ind_id: str
        """

        # check for missing attributes
        if not cbx_obj or not cbx_obj.properties:
            if cbx_obj.idref:  # just a reference, so nothing to parse
                return
            else:
                cbx_id = getattr(cbx_obj, 'id_', 'None')
                self.failed.append(("No valid object_properties was found!",
                                    "Observable (%s)" % cbx_id,
                                    cbx_id))  # note for display in UI
                return

        # Don't parse if already been parsed
        # This is for artifacts that are related to CybOX File Objects
        if cbx_obj.id_ in self.parsed:
            return

        try:  # try to create CRITs object from Cybox Object
            analyst = self.source_instance.analyst
            item = cbx_obj.properties
            val = cbx_obj.id_
            if isinstance(item, Address) and not ind_id:
                if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
                                     'ipv4-netmask', 'ipv6-addr', 'ipv6-net',
                                     'ipv6-netmask'):
                    imp_type = "IP"
                    for value in item.address_value.values:
                        val = str(value).strip()
                        if self.preview:
                            res = None
                        else:
                            iptype = get_crits_ip_type(item.category)
                            if iptype:
                                res = ip_add_update(val,
                                                    iptype, [self.source],
                                                    analyst=analyst,
                                                    is_add_indicator=True)
                            else:
                                res = {
                                    'success': False,
                                    'reason': 'No IP Type'
                                }
                        self.parse_res(imp_type, val, cbx_obj, res, ind_id)
            if (not ind_id and
                (isinstance(item, DomainName) or
                 (isinstance(item, URI) and item.type_ == 'Domain Name'))):
                imp_type = "Domain"
                for val in item.value.values:
                    if self.preview:
                        res = None
                    else:
                        res = upsert_domain(str(val), [self.source],
                                            username=analyst)
                    self.parse_res(imp_type, str(val), cbx_obj, res, ind_id)

            elif isinstance(item, HTTPSession):
                imp_type = "RawData"
                val = cbx_obj.id_
                try:
                    c_req = item.http_request_response[0].http_client_request
                    hdr = c_req.http_request_header
                    if hdr.raw_header:
                        data = hdr.raw_header.value
                        title = "HTTP Header from STIX: %s" % self.package.id_
                        method = self.source_instance.method
                        ref = self.source_instance.reference
                        if self.preview:
                            res = None
                            val = title
                        else:
                            res = handle_raw_data_file(data,
                                                       self.source.name,
                                                       user=analyst,
                                                       description=description,
                                                       title=title,
                                                       data_type="HTTP Header",
                                                       tool_name="STIX",
                                                       tool_version=None,
                                                       method=method,
                                                       reference=ref)
                    else:
                        imp_type = "Indicator"
                        ind_type = "HTTP Request Header Fields - User-Agent"
                        val = hdr.parsed_header.user_agent.value
                        val = ','.join(val) if isinstance(val, list) else val
                        if self.preview:
                            res = None
                        else:
                            res = handle_indicator_ind(
                                val,
                                self.source,
                                ind_type,
                                IndicatorThreatTypes.UNKNOWN,
                                IndicatorAttackTypes.UNKNOWN,
                                analyst,
                                add_relationship=True,
                                description=description)
                except:
                    msg = "Unsupported use of 'HTTPSession' object."
                    res = {'success': False, 'reason': msg}

                self.parse_res(imp_type, val, cbx_obj, res, ind_id)
            elif isinstance(item, WhoisEntry):
                # No sure where else to put this
                imp_type = "RawData"
                val = cbx_obj.id_
                if item.remarks:
                    data = item.remarks.value
                    title = "WHOIS Entry from STIX: %s" % self.package.id_
                    if self.preview:
                        res = None
                        val = title
                    else:
                        res = handle_raw_data_file(
                            data,
                            self.source.name,
                            user=analyst,
                            description=description,
                            title=title,
                            data_type="Text",
                            tool_name="WHOIS",
                            tool_version=None,
                            method=self.source_instance.method,
                            reference=self.source_instance.reference)
                else:
                    msg = "Unsupported use of 'WhoisEntry' object."
                    res = {'success': False, 'reason': msg}

                self.parse_res(imp_type, val, cbx_obj, res, ind_id)
            elif isinstance(item, Artifact):
                # Not sure if this is right, and I believe these can be
                # encoded in a couple different ways.
                imp_type = "RawData"
                val = cbx_obj.id_
                rawdata = item.data.decode('utf-8')
                # TODO: find out proper ways to determine title, datatype,
                #       tool_name, tool_version
                title = "Artifact for Event: STIX Document %s" % self.package.id_
                if self.preview:
                    res = None
                    val = title
                else:
                    res = handle_raw_data_file(
                        rawdata,
                        self.source.name,
                        user=analyst,
                        description=description,
                        title=title,
                        data_type="Text",
                        tool_name="STIX",
                        tool_version=None,
                        method=self.source_instance.method,
                        reference=self.source_instance.reference)
                self.parse_res(imp_type, val, cbx_obj, res, ind_id)
            elif (isinstance(item, File) and item.custom_properties
                  and item.custom_properties[0].name == "crits_type"
                  and item.custom_properties[0]._value == "Certificate"):
                imp_type = "Certificate"
                val = str(item.file_name)
                data = None
                if self.preview:
                    res = None
                else:
                    for rel_obj in item.parent.related_objects:
                        if isinstance(rel_obj.properties, Artifact):
                            data = rel_obj.properties.data
                            self.parsed.append(rel_obj.id_)
                    res = handle_cert_file(val,
                                           data,
                                           self.source,
                                           user=analyst,
                                           description=description)
                self.parse_res(imp_type, val, cbx_obj, res, ind_id)
            elif isinstance(item, File) and self.has_network_artifact(item):
                imp_type = "PCAP"
                val = str(item.file_name)
                data = None
                if self.preview:
                    res = None
                else:
                    for rel_obj in item.parent.related_objects:
                        if (isinstance(rel_obj.properties, Artifact)
                                and rel_obj.properties.type_
                                == Artifact.TYPE_NETWORK):
                            data = rel_obj.properties.data
                            self.parsed.append(rel_obj.id_)
                    res = handle_pcap_file(val,
                                           data,
                                           self.source,
                                           user=analyst,
                                           description=description)
                self.parse_res(imp_type, val, cbx_obj, res, ind_id)
            elif isinstance(item, File):
                imp_type = "Sample"
                md5 = item.md5
                if md5:
                    md5 = md5.lower()
                val = str(item.file_name or md5)
                # add sha1/sha256/ssdeep once handle_file supports it
                size = item.size_in_bytes
                data = None
                if item.file_path:
                    path = "File Path: " + str(item.file_path)
                    description += "\n" + path
                for rel_obj in item.parent.related_objects:
                    if (isinstance(rel_obj.properties, Artifact) and
                            rel_obj.properties.type_ == Artifact.TYPE_FILE):
                        data = rel_obj.properties.data
                        self.parsed.append(rel_obj.id_)
                if not md5 and not data and val and val != "None":
                    imp_type = "Indicator"
                    if self.preview:
                        res = None
                    else:
                        res = handle_indicator_ind(
                            val,
                            self.source,
                            "Win File",
                            IndicatorThreatTypes.UNKNOWN,
                            IndicatorAttackTypes.UNKNOWN,
                            analyst,
                            add_domain=True,
                            add_relationship=True,
                            description=description)
                elif md5 or data:
                    if self.preview:
                        res = None
                    else:
                        res = handle_file(val,
                                          data,
                                          self.source,
                                          user=analyst,
                                          md5_digest=md5,
                                          is_return_only_md5=False,
                                          size=size,
                                          description=description)
                else:
                    val = cbx_obj.id_
                    msg = "CybOX 'File' object has no MD5, data, or filename"
                    res = {'success': False, 'reason': msg}
                self.parse_res(imp_type, val, cbx_obj, res, ind_id)
            elif isinstance(item, EmailMessage):
                imp_type = 'Email'
                id_list = []
                data = {}
                val = cbx_obj.id_
                get_attach = False
                data['raw_body'] = str(item.raw_body)
                data['raw_header'] = str(item.raw_header)
                data['helo'] = str(item.email_server)
                if item.header:
                    data['subject'] = str(item.header.subject)
                    if item.header.date:
                        data['date'] = item.header.date.value
                    val = "Date: %s, Subject: %s" % (data.get(
                        'date', 'None'), data['subject'])
                    data['message_id'] = str(item.header.message_id)
                    data['sender'] = str(item.header.sender)
                    data['reply_to'] = str(item.header.reply_to)
                    data['x_originating_ip'] = str(
                        item.header.x_originating_ip)
                    data['x_mailer'] = str(item.header.x_mailer)
                    data['boundary'] = str(item.header.boundary)
                    data['from_address'] = str(item.header.from_)
                    if item.header.to:
                        data['to'] = [str(r) for r in item.header.to.to_list()]

                if data.get('date'):  # Email TLOs must have a date
                    data['source'] = self.source.name
                    data['source_method'] = self.source_instance.method
                    data['source_reference'] = self.source_instance.reference
                    if self.preview:
                        res = None
                    else:
                        res = handle_email_fields(data, analyst, "STIX")
                    self.parse_res(imp_type, val, cbx_obj, res, ind_id)
                    if not self.preview and res.get('status'):
                        id_list.append(cbx_obj.id_)  # save ID for atchmnt rels
                        get_attach = True
                else:  # Can't be an Email TLO, so save fields
                    for x, key in enumerate(data):
                        if data[key] and data[key] != "None":
                            if key in ('raw_header', 'raw_body'):
                                if key == 'raw_header':
                                    title = "Email Header from STIX Email: %s"
                                    d_type = "Email Header"
                                else:
                                    title = "Email Body from STIX Email: %s"
                                    d_type = "Email Body"
                                imp_type = 'RawData'
                                title = title % cbx_obj.id_
                                if self.preview:
                                    res = None
                                else:
                                    res = handle_raw_data_file(
                                        data[key], self.source, analyst,
                                        description, title, d_type, "STIX",
                                        self.stix_version)
                                self.parse_res(imp_type, title, cbx_obj, res,
                                               ind_id)
                            elif key == 'to':
                                imp_type = 'Target'
                                for y, addr in enumerate(data[key]):
                                    tgt_dict = {'email_address': addr}
                                    if self.preview:
                                        res = None
                                    else:
                                        res = upsert_target(tgt_dict, analyst)
                                        if res['success']:
                                            get_attach = True
                                    tmp_obj = copy(cbx_obj)
                                    tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_, x,
                                                                y)
                                    self.parse_res(imp_type, addr, tmp_obj,
                                                   res, ind_id)
                                    self.ind2obj.setdefault(
                                        cbx_obj.id_, []).append(tmp_obj.id_)
                                    id_list.append(tmp_obj.id_)
                            else:
                                imp_type = 'Indicator'
                                if key in ('sender', 'reply_to',
                                           'from_address'):
                                    ind_type = "Address - e-mail"
                                elif 'ip' in key:
                                    ind_type = "Address - ipv4-addr"
                                elif key == 'raw_body':
                                    ind_type = "Email Message"
                                else:
                                    ind_type = "String"
                                if self.preview:
                                    res = None
                                else:
                                    res = handle_indicator_ind(
                                        data[key],
                                        self.source,
                                        ind_type,
                                        IndicatorThreatTypes.UNKNOWN,
                                        IndicatorAttackTypes.UNKNOWN,
                                        analyst,
                                        add_domain=True,
                                        add_relationship=True,
                                        description=description)
                                    if res['success']:
                                        get_attach = True
                                tmp_obj = copy(cbx_obj)
                                tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x)
                                self.parse_res(imp_type, data[key], tmp_obj,
                                               res, ind_id)
                                self.ind2obj.setdefault(cbx_obj.id_,
                                                        []).append(tmp_obj.id_)
                                id_list.append(tmp_obj.id_)

                if not self.preview:
                    # Setup relationships between all Email attributes
                    for oid in id_list:
                        for oid2 in id_list:
                            if oid != oid2:
                                self.relationships.append(
                                    (oid, RelationshipTypes.RELATED_TO, oid2,
                                     "High"))

                    # Should check for attachments and add them here.
                    if get_attach and item.attachments:
                        for attach in item.attachments:
                            rel_id = attach.to_dict()['object_reference']
                            for oid in id_list:
                                self.relationships.append(
                                    (oid, RelationshipTypes.CONTAINS, rel_id,
                                     "High"))

            else:  # try to parse all other possibilities as Indicator
                imp_type = "Indicator"
                val = cbx_obj.id_
                c_obj = make_crits_object(item)

                # Ignore what was already caught above
                if (ind_id or c_obj.object_type not in IPTypes.values()):
                    ind_type = c_obj.object_type
                    for val in [str(v).strip() for v in c_obj.value if v]:
                        if ind_type:
                            # handle domains mislabeled as URLs
                            if c_obj.object_type == 'URI' and '/' not in val:
                                ind_type = "Domain"

                            if self.preview:
                                res = None
                            else:
                                res = handle_indicator_ind(
                                    val,
                                    self.source,
                                    ind_type,
                                    IndicatorThreatTypes.UNKNOWN,
                                    IndicatorAttackTypes.UNKNOWN,
                                    analyst,
                                    add_domain=True,
                                    add_relationship=True,
                                    description=description)
                            self.parse_res(imp_type, val, cbx_obj, res, ind_id)

        except Exception, e:  # probably caused by cybox object we don't handle
            self.failed.append((e.message, "%s (%s)" % (imp_type, val),
                                cbx_obj.id_))  # note for display in UI
Ejemplo n.º 17
0
def add_new_domain(data,
                   request,
                   errors,
                   rowData=None,
                   is_validate_only=False,
                   cache={}):
    """
    Add a new domain to CRITs.

    :param data: The data about the domain.
    :type data: dict
    :param request: The Django request.
    :type request: :class:`django.http.HttpRequest`
    :param errors: A list of current errors to append to.
    :type errors: list
    :param rowData: Any objects that need to be added to the domain.
    :type rowData: dict
    :param is_validate_only: Only validate the data and return any errors.
    :type is_validate_only: boolean
    :param cache: Cached data, typically for performance enhancements
                  during bulk uperations.
    :type cache: dict
    :returns: tuple
    """

    username = request.user.username
    result = False
    retVal = {}
    reference = data.get('domain_reference')
    name = data.get('domain_source')
    method = data.get('domain_method')
    source = [
        create_embedded_source(name,
                               reference=reference,
                               method=method,
                               analyst=username)
    ]
    bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
    ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)

    if data.get('campaign') and data.get('confidence'):
        campaign = [
            EmbeddedCampaign(name=data.get('campaign'),
                             confidence=data.get('confidence'),
                             analyst=username)
        ]
    else:
        campaign = []

    (sdomain, fqdn) = get_domain(data['domain'])
    if sdomain == "no_tld_found_error":
        errors.append(u"Error: Invalid domain: " + data['domain'])
    elif is_validate_only == False:
        retVal = upsert_domain(sdomain,
                               fqdn,
                               source,
                               username,
                               campaign,
                               bucket_list=bucket_list,
                               ticket=ticket,
                               cache=cache)
        ip_result = None

        if retVal['success']:
            new_domain = retVal['object']
            add_ip = data.get('add_ip')
            if add_ip:
                ip = data.get('ip')
                ip_type = data.get('ip_type')
                if data.get('same_source'):
                    ip_source = data.get('domain_source')
                    ip_method = data.get('domain_method')
                    ip_reference = reference
                else:
                    ip_source = data.get('ip_source')
                    ip_method = data.get('ip_method')
                    ip_reference = data.get('ip_reference')
                from crits.ips.handlers import ip_add_update
                ip_result = ip_add_update(ip,
                                          ip_type,
                                          ip_source,
                                          ip_method,
                                          ip_reference,
                                          campaign=campaign,
                                          analyst=username,
                                          bucket_list=bucket_list,
                                          ticket=ticket,
                                          cache=cache)
                if not ip_result['success']:
                    errors.append(ip_result['message'])
                else:
                    #add a relationship with the new IP address
                    new_ip = ip_result['object']
                    if new_domain and new_ip:
                        new_domain.add_relationship(rel_item=new_ip,
                                                    rel_type='Resolved_To',
                                                    analyst=username,
                                                    get_rels=False)
                        new_domain.save(username=username)
                        new_ip.save(username=username)

            #set the URL for viewing the new data
            resp_url = reverse('crits.domains.views.domain_detail',
                               args=[fqdn])

            if retVal['is_domain_new'] == True:
                retVal['message'] = (
                    'Success! Click here to view the new domain: '
                    '<a href="%s">%s</a>' % (resp_url, fqdn))
            else:
                message = ('Updated existing domain: <a href="%s">%s</a>' %
                           (resp_url, fqdn))
                retVal['message'] = message
                retVal[form_consts.Status.
                       STATUS_FIELD] = form_consts.Status.DUPLICATE
                retVal['warning'] = message

            #add indicators
            if data.get('add_indicators'):
                from crits.indicators.handlers import create_indicator_from_obj
                # If we have an IP object, add an indicator for that.
                if ip_result and ip_result['success']:
                    obj = ip_result['object']
                    result = create_indicator_from_obj('Address - ipv4-addr',
                                                       'IP', obj.id, obj.ip,
                                                       username)
                    if result['success'] == False:
                        errors.append(result['message'])

                # Add an indicator for the domain.
                result = create_indicator_from_obj('URI - Domain Name',
                                                   'Domain', new_domain.id,
                                                   sdomain, username)
                if result['success'] == False:
                    errors.append(result['message'])
                # If we have an FQDN (ie: it is not the same as sdomain)
                # then add that also.
                if fqdn != sdomain:
                    result = create_indicator_from_obj('URI - Domain Name',
                                                       'Domain', new_domain.id,
                                                       fqdn, username)
                    if result['success'] == False:
                        errors.append(result['message'])
            result = True

        elif 'message' in retVal:  #database error? (!c_dom)
            errors.append(
                retVal['message'])  #u"Unknown error: unable to add domain")

    elif is_validate_only == True:
        domain = data['domain']
        fqdn_domain = retrieve_domain(domain, cache)

        if fqdn_domain:
            if isinstance(fqdn_domain, Domain):
                resp_url = reverse('crits.domains.views.domain_detail',
                                   args=[fqdn])
                message = ('Warning: Domain already exists: '
                           '<a href="%s">%s</a>' % (resp_url, fqdn))
                retVal['message'] = message
                retVal['status'] = form_consts.Status.DUPLICATE
                retVal['warning'] = message
        else:
            result_cache = cache.get(form_consts.Domain.CACHED_RESULTS)
            result_cache[domain.lower()] = True

    # This block tries to add objects to the item
    if retVal.get('success') or is_validate_only == True:
        if rowData:
            objectsData = rowData.get(form_consts.Common.OBJECTS_DATA)

            # add new objects if they exist
            if objectsData:
                objectsData = json.loads(objectsData)
                current_domain = retrieve_domain(fqdn, cache)
                for object_row_counter, objectData in enumerate(
                        objectsData, 1):
                    if current_domain != None:
                        # if the domain exists then try to add objects to it
                        if isinstance(current_domain, Domain) == True:
                            objectDict = object_array_to_dict(
                                objectData, "Domain", current_domain.id)
                        else:
                            objectDict = object_array_to_dict(
                                objectData, "Domain", "")
                            current_domain = None
                    else:
                        objectDict = object_array_to_dict(
                            objectData, "Domain", "")

                    (object_result, object_errors,
                     object_retVal) = validate_and_add_new_handler_object(
                         None,
                         objectDict,
                         request,
                         errors,
                         object_row_counter,
                         is_validate_only=is_validate_only,
                         cache=cache,
                         obj=current_domain)

                    if object_retVal.get('success') == False:
                        retVal['success'] = False
                    if object_retVal.get('message'):
                        errors.append(object_retVal['message'])

    return result, errors, retVal
Ejemplo n.º 18
0
    def parse_cybox_object(self, cbx_obj, description='', ind_id=None):
        """
        Parse a CybOX object form a STIX doc. An object can contain
        multiple related_objects, which in turn can have their own
        related_objects, so this handles those recursively.

        :param cbx_obj: The CybOX object to parse.
        :type cbx_obj: A CybOX object.
        :param description: Parent-level (e.g. Observable) description.
        :type description: str
        :param ind_id: The ID of a parent STIX Indicator.
        :type ind_id: str
        """

        # check for missing attributes
        if not cbx_obj or not cbx_obj.properties:
            if cbx_obj.idref: # just a reference, so nothing to parse
                return
            else:
                cbx_id = getattr(cbx_obj, 'id_', 'None')
                self.failed.append(("No valid object_properties was found!",
                                    "Observable (%s)" % cbx_id,
                                    cbx_id)) # note for display in UI
                return

        # Don't parse if already been parsed
        # This is for artifacts that are related to CybOX File Objects
        if cbx_obj.id_ in self.parsed:
            return

        try: # try to create CRITs object from Cybox Object
            analyst = self.source_instance.analyst
            item = cbx_obj.properties
            val = cbx_obj.id_
            if isinstance(item, Address) and not ind_id:
                if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
                                     'ipv4-netmask', 'ipv6-addr',
                                     'ipv6-net', 'ipv6-netmask'):
                    imp_type = "IP"
                    for value in item.address_value.values:
                        val = str(value).strip()
                        if self.preview:
                            res = None
                        else:
                            iptype = get_crits_ip_type(item.category)
                            if iptype:
                                res = ip_add_update(val,
                                                    iptype,
                                                    [self.source],
                                                    analyst=analyst,
                                                    is_add_indicator=True)
                            else:
                                res = {'success': False, 'reason': 'No IP Type'}
                        self.parse_res(imp_type, val, cbx_obj, res, ind_id)
            if (not ind_id and (isinstance(item, DomainName) or
                (isinstance(item, URI) and item.type_ == 'Domain Name'))):
                imp_type = "Domain"
                for val in item.value.values:
                    if self.preview:
                        res = None
                    else:
                        res = upsert_domain(str(val),
                                            [self.source],
                                            username=analyst)
                    self.parse_res(imp_type, str(val), cbx_obj, res, ind_id)

            elif isinstance(item, HTTPSession):
                imp_type = "RawData"
                val = cbx_obj.id_
                try:
                    c_req = item.http_request_response[0].http_client_request
                    hdr = c_req.http_request_header
                    if hdr.raw_header:
                        data = hdr.raw_header.value
                        title = "HTTP Header from STIX: %s" % self.package.id_
                        method = self.source_instance.method
                        ref = self.source_instance.reference
                        if self.preview:
                            res = None
                            val = title
                        else:
                            res = handle_raw_data_file(data,
                                                    self.source.name,
                                                    user=analyst,
                                                    description=description,
                                                    title=title,
                                                    data_type="HTTP Header",
                                                    tool_name="STIX",
                                                    tool_version=None,
                                                    method=method,
                                                    reference=ref)
                    else:
                        imp_type = "Indicator"
                        ind_type = "HTTP Request Header Fields - User-Agent"
                        val = hdr.parsed_header.user_agent.value
                        val = ','.join(val) if isinstance(val, list) else val
                        if self.preview:
                            res = None
                        else:
                            res = handle_indicator_ind(val,
                                                    self.source,
                                                    ind_type,
                                                    IndicatorThreatTypes.UNKNOWN,
                                                    IndicatorAttackTypes.UNKNOWN,
                                                    analyst,
                                                    add_relationship=True,
                                                    description=description)
                except:
                    msg = "Unsupported use of 'HTTPSession' object."
                    res = {'success': False, 'reason': msg}

                self.parse_res(imp_type, val, cbx_obj, res, ind_id)
            elif isinstance(item, WhoisEntry):
                # No sure where else to put this
                imp_type = "RawData"
                val = cbx_obj.id_
                if item.remarks:
                    data = item.remarks.value
                    title = "WHOIS Entry from STIX: %s" % self.package.id_
                    if self.preview:
                        res = None
                        val = title
                    else:
                        res = handle_raw_data_file(data,
                                                self.source.name,
                                                user=analyst,
                                                description=description,
                                                title=title,
                                                data_type="Text",
                                                tool_name="WHOIS",
                                                tool_version=None,
                                                method=self.source_instance.method,
                                                reference=self.source_instance.reference)
                else:
                    msg = "Unsupported use of 'WhoisEntry' object."
                    res = {'success': False, 'reason': msg}

                self.parse_res(imp_type, val, cbx_obj, res, ind_id)
            elif isinstance(item, Artifact):
                # Not sure if this is right, and I believe these can be
                # encoded in a couple different ways.
                imp_type = "RawData"
                val = cbx_obj.id_
                rawdata = item.data.decode('utf-8')
                # TODO: find out proper ways to determine title, datatype,
                #       tool_name, tool_version
                title = "Artifact for Event: STIX Document %s" % self.package.id_
                if self.preview:
                    res = None
                    val = title
                else:
                    res = handle_raw_data_file(rawdata,
                                            self.source.name,
                                            user=analyst,
                                            description=description,
                                            title=title,
                                            data_type="Text",
                                            tool_name="STIX",
                                            tool_version=None,
                                            method=self.source_instance.method,
                                            reference=self.source_instance.reference)
                self.parse_res(imp_type, val, cbx_obj, res, ind_id)
            elif (isinstance(item, File) and
                  item.custom_properties and
                  item.custom_properties[0].name == "crits_type" and
                  item.custom_properties[0]._value == "Certificate"):
                imp_type = "Certificate"
                val = str(item.file_name)
                data = None
                if self.preview:
                    res = None
                else:
                    for rel_obj in item.parent.related_objects:
                        if isinstance(rel_obj.properties, Artifact):
                            data = rel_obj.properties.data
                            self.parsed.append(rel_obj.id_)
                    res = handle_cert_file(val,
                                           data,
                                           self.source,
                                           user=analyst,
                                           description=description)
                self.parse_res(imp_type, val, cbx_obj, res, ind_id)
            elif isinstance(item, File) and self.has_network_artifact(item):
                imp_type = "PCAP"
                val = str(item.file_name)
                data = None
                if self.preview:
                    res = None
                else:
                    for rel_obj in item.parent.related_objects:
                        if (isinstance(rel_obj.properties, Artifact) and
                            rel_obj.properties.type_ == Artifact.TYPE_NETWORK):
                            data = rel_obj.properties.data
                            self.parsed.append(rel_obj.id_)
                    res = handle_pcap_file(val,
                                           data,
                                           self.source,
                                           user=analyst,
                                           description=description)
                self.parse_res(imp_type, val, cbx_obj, res, ind_id)
            elif isinstance(item, File):
                imp_type = "Sample"
                md5 = item.md5
                if md5:
                    md5 = md5.lower()
                val = str(item.file_name or md5)
                # add sha1/sha256/ssdeep once handle_file supports it
                size = item.size_in_bytes
                data = None
                if item.file_path:
                    path = "File Path: " + str(item.file_path)
                    description += "\n" + path
                for rel_obj in item.parent.related_objects:
                    if (isinstance(rel_obj.properties, Artifact) and
                        rel_obj.properties.type_ == Artifact.TYPE_FILE):
                        data = rel_obj.properties.data
                        self.parsed.append(rel_obj.id_)
                if not md5 and not data and val and val != "None":
                    imp_type = "Indicator"
                    if self.preview:
                        res = None
                    else:
                        res = handle_indicator_ind(val,
                                                   self.source,
                                                   "Win File",
                                                   IndicatorThreatTypes.UNKNOWN,
                                                   IndicatorAttackTypes.UNKNOWN,
                                                   analyst,
                                                   add_domain=True,
                                                   add_relationship=True,
                                                   description=description)
                elif md5 or data:
                    if self.preview:
                        res = None
                    else:
                        res = handle_file(val,
                                          data,
                                          self.source,
                                          user=analyst,
                                          md5_digest=md5,
                                          is_return_only_md5=False,
                                          size=size,
                                          description=description)
                else:
                    val = cbx_obj.id_
                    msg = "CybOX 'File' object has no MD5, data, or filename"
                    res = {'success': False, 'reason': msg}
                self.parse_res(imp_type, val, cbx_obj, res, ind_id)
            elif isinstance(item, EmailMessage):
                imp_type = 'Email'
                id_list = []
                data = {}
                val = cbx_obj.id_
                get_attach = False
                data['raw_body'] = str(item.raw_body)
                data['raw_header'] = str(item.raw_header)
                data['helo'] = str(item.email_server)
                if item.header:
                    data['subject'] = str(item.header.subject)
                    if item.header.date:
                        data['date'] = item.header.date.value
                    val = "Date: %s, Subject: %s" % (data.get('date', 'None'),
                                                     data['subject'])
                    data['message_id'] = str(item.header.message_id)
                    data['sender'] = str(item.header.sender)
                    data['reply_to'] = str(item.header.reply_to)
                    data['x_originating_ip'] = str(item.header.x_originating_ip)
                    data['x_mailer'] = str(item.header.x_mailer)
                    data['boundary'] = str(item.header.boundary)
                    data['from_address'] = str(item.header.from_)
                    if item.header.to:
                        data['to'] = [str(r) for r in item.header.to.to_list()]

                if data.get('date'): # Email TLOs must have a date
                    data['source'] = self.source.name
                    data['source_method'] = self.source_instance.method
                    data['source_reference'] = self.source_instance.reference
                    if self.preview:
                        res = None
                    else:
                        res = handle_email_fields(data,
                                                  analyst,
                                                  "STIX")
                    self.parse_res(imp_type, val, cbx_obj, res, ind_id)
                    if not self.preview and res.get('status'):
                        id_list.append(cbx_obj.id_) # save ID for atchmnt rels
                        get_attach = True
                else: # Can't be an Email TLO, so save fields
                    for x, key in enumerate(data):
                        if data[key] and data[key] != "None":
                            if key in ('raw_header', 'raw_body'):
                                if key == 'raw_header':
                                    title = "Email Header from STIX Email: %s"
                                    d_type = "Email Header"
                                else:
                                    title = "Email Body from STIX Email: %s"
                                    d_type = "Email Body"
                                imp_type = 'RawData'
                                title = title % cbx_obj.id_
                                if self.preview:
                                    res = None
                                else:
                                    res = handle_raw_data_file(data[key],
                                                               self.source,
                                                               analyst,
                                                               description,
                                                               title,
                                                               d_type,
                                                               "STIX",
                                                               self.stix_version)
                                self.parse_res(imp_type, title, cbx_obj,
                                               res, ind_id)
                            elif key == 'to':
                                imp_type = 'Target'
                                for y, addr in enumerate(data[key]):
                                    tgt_dict = {'email_address': addr}
                                    if self.preview:
                                        res = None
                                    else:
                                        res = upsert_target(tgt_dict, analyst)
                                        if res['success']:
                                            get_attach = True
                                    tmp_obj = copy(cbx_obj)
                                    tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_,
                                                                x, y)
                                    self.parse_res(imp_type, addr, tmp_obj,
                                                   res, ind_id)
                                    self.ind2obj.setdefault(cbx_obj.id_,
                                                            []).append(tmp_obj.id_)
                                    id_list.append(tmp_obj.id_)
                            else:
                                imp_type = 'Indicator'
                                if key in ('sender', 'reply_to', 'from_address'):
                                    ind_type = "Address - e-mail"
                                elif 'ip' in key:
                                    ind_type = "Address - ipv4-addr"
                                elif key == 'raw_body':
                                    ind_type = "Email Message"
                                else:
                                    ind_type = "String"
                                if self.preview:
                                    res = None
                                else:
                                    res = handle_indicator_ind(data[key],
                                                          self.source,
                                                          ind_type,
                                                          IndicatorThreatTypes.UNKNOWN,
                                                          IndicatorAttackTypes.UNKNOWN,
                                                          analyst,
                                                          add_domain=True,
                                                          add_relationship=True,
                                                          description=description)
                                    if res['success']:
                                        get_attach = True
                                tmp_obj = copy(cbx_obj)
                                tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x)
                                self.parse_res(imp_type, data[key], tmp_obj,
                                               res, ind_id)
                                self.ind2obj.setdefault(cbx_obj.id_,
                                                        []).append(tmp_obj.id_)
                                id_list.append(tmp_obj.id_)

                if not self.preview:
                    # Setup relationships between all Email attributes
                    for oid in id_list:
                        for oid2 in id_list:
                            if oid != oid2:
                                self.relationships.append((oid,
                                                           RelationshipTypes.RELATED_TO,
                                                           oid2, "High"))

                    # Should check for attachments and add them here.
                    if get_attach and item.attachments:
                        for attach in item.attachments:
                            rel_id = attach.to_dict()['object_reference']
                            for oid in id_list:
                                self.relationships.append((oid,
                                                           RelationshipTypes.CONTAINS,
                                                           rel_id, "High"))

            else: # try to parse all other possibilities as Indicator
                imp_type = "Indicator"
                val = cbx_obj.id_
                c_obj = make_crits_object(item)

                # Ignore what was already caught above
                if (ind_id or c_obj.object_type not in IPTypes.values()):
                    ind_type = c_obj.object_type
                    for val in [str(v).strip() for v in c_obj.value if v]:
                        if ind_type:
                            # handle domains mislabeled as URLs
                            if c_obj.object_type == 'URI' and '/' not in val:
                                ind_type = "Domain"

                            if self.preview:
                                res = None
                            else:
                                res = handle_indicator_ind(val,
                                                        self.source,
                                                        ind_type,
                                                        IndicatorThreatTypes.UNKNOWN,
                                                        IndicatorAttackTypes.UNKNOWN,
                                                        analyst,
                                                        add_domain=True,
                                                        add_relationship=True,
                                                        description=description)
                            self.parse_res(imp_type, val, cbx_obj, res, ind_id)

        except Exception, e: # probably caused by cybox object we don't handle
            self.failed.append((e.message,
                                "%s (%s)" % (imp_type, val),
                                cbx_obj.id_)) # note for display in UI