Exemplo n.º 1
0
Arquivo: tests.py Projeto: 0x3a/crits
 def setUp(self):
     prep_db()
     self.user = CRITsUser.objects(username=TUSER_NAME).first()
     self.user2 = CRITsUser.objects(username=TUSER2_NAME).first()
     self.campaign1 = Campaign.objects(name=TCAMPAIGN1).first()
     self.campaign2 = Campaign.objects(name=TCAMPAIGN2).first()
     forge_relationship(left_class=self.campaign1,
                        right_class=self.campaign2,
                        rel_type=TRELATIONSHIP_TYPE,
                        analyst=self.user.username,
                        rel_confidence=TRELATIONSHIP_CONFIDENCE)
Exemplo n.º 2
0
 def setUp(self):
     prep_db()
     self.user = CRITsUser.objects(username=TUSER_NAME).first()
     self.user2 = CRITsUser.objects(username=TUSER2_NAME).first()
     self.campaign1 = Campaign.objects(name=TCAMPAIGN1).first()
     self.campaign2 = Campaign.objects(name=TCAMPAIGN2).first()
     forge_relationship(class_=self.campaign1,
                        right_class=self.campaign2,
                        rel_type=TRELATIONSHIP_TYPE,
                        user=self.user.username,
                        rel_confidence=TRELATIONSHIP_CONFIDENCE)
Exemplo n.º 3
0
    def obj_create(self, bundle, **kwargs):
        """
        Handles forging relationships through the API.

        :param bundle: Bundle containing the relationship information.
        :type bundle: Tastypie Bundle object.
        :returns: Bundle object.
        :raises BadRequest: If necessary data is not provided or creation fails.

        """
        analyst = bundle.request.user.username
        left_type = bundle.data.get('left_type', None)
        left_id = bundle.data.get('left_id', None)
        right_type = bundle.data.get('right_type', None)
        right_id = bundle.data.get('right_id', None)
        rel_type = bundle.data.get('rel_type', None)
        rel_date = bundle.data.get('rel_date', None)

        if (not left_type or not left_id or not right_type or not right_id
                or not rel_type):
            raise BadRequest('Need all of the relationship information.')
        result = forge_relationship(left_type=left_type,
                                    left_id=left_id,
                                    right_type=right_type,
                                    right_id=right_id,
                                    rel_type=rel_type,
                                    rel_date=rel_date,
                                    analyst=analyst,
                                    get_rels=False)
        if not result['success']:
            raise BadRequest(result['message'])
        else:
            return bundle
Exemplo n.º 4
0
Arquivo: api.py Projeto: 0x3a/crits
    def obj_create(self, bundle, **kwargs):
        """
        Handles forging relationships through the API.

        :param bundle: Bundle containing the relationship information.
        :type bundle: Tastypie Bundle object.
        :returns: HttpResponse.

        """
        analyst = bundle.request.user.username
        left_type = bundle.data.get('left_type', None)
        left_id = bundle.data.get('left_id', None)
        right_type = bundle.data.get('right_type', None)
        right_id = bundle.data.get('right_id', None)
        rel_type = bundle.data.get('rel_type', None)
        rel_date = bundle.data.get('rel_date', None)
        rel_confidence = bundle.data.get('rel_confidence', 'unknown')
        rel_reason = bundle.data.get('rel_reason', 'N/A')

        content = {'return_code': 1,
                   'type': left_type}

        if (not left_type
            or not left_id
            or not right_type
            or not right_id
            or not rel_type):
            content['message'] = 'Need all of the relationship information.'
            self.crits_response(content)

        if rel_confidence not in ('unknown', 'low', 'medium', 'high'):
            content['message'] = 'Bad relationship confidence.'
            self.crits_response(content)

        result = forge_relationship(left_type=left_type,
                                    left_id=left_id,
                                    right_type=right_type,
                                    right_id=right_id,
                                    rel_type=rel_type,
                                    rel_date=rel_date,
                                    analyst=analyst,
                                    rel_confidence=rel_confidence,
                                    rel_reason=rel_reason,
                                    get_rels=False)

        if result.get('message'):
            content['message'] = result.get('message')

        content['id'] = left_id
        rname = self.resource_name_from_type(left_type)
        url = reverse('api_dispatch_detail',
                        kwargs={'resource_name': rname,
                                'api_name': 'v1',
                                'pk': left_id})
        content['url'] = url
        if result['success']:
            content['return_code'] = 0
        self.crits_response(content)
Exemplo n.º 5
0
    def obj_create(self, bundle, **kwargs):
        """
        Handles forging relationships through the API.

        :param bundle: Bundle containing the relationship information.
        :type bundle: Tastypie Bundle object.
        :returns: HttpResponse.

        """
        analyst = bundle.request.user.username
        left_type = bundle.data.get('left_type', None)
        left_id = bundle.data.get('left_id', None)
        right_type = bundle.data.get('right_type', None)
        right_id = bundle.data.get('right_id', None)
        rel_type = bundle.data.get('rel_type', None)
        rel_date = bundle.data.get('rel_date', None)
        rel_confidence = bundle.data.get('rel_confidence', 'unknown')
        rel_reason = bundle.data.get('rel_reason', 'N/A')

        content = {'return_code': 1, 'type': left_type}

        if (not left_type or not left_id or not right_type or not right_id
                or not rel_type):
            content['message'] = 'Need all of the relationship information.'
            self.crits_response(content)

        if rel_confidence not in ('unknown', 'low', 'medium', 'high'):
            content['message'] = 'Bad relationship confidence.'
            self.crits_response(content)

        result = forge_relationship(left_type=left_type,
                                    left_id=left_id,
                                    right_type=right_type,
                                    right_id=right_id,
                                    rel_type=rel_type,
                                    rel_date=rel_date,
                                    analyst=analyst,
                                    rel_confidence=rel_confidence,
                                    rel_reason=rel_reason,
                                    get_rels=False)

        if result.get('message'):
            content['message'] = result.get('message')

        content['id'] = left_id
        rname = self.resource_name_from_type(left_type)
        url = reverse('api_dispatch_detail',
                      kwargs={
                          'resource_name': rname,
                          'api_name': 'v1',
                          'pk': left_id
                      })
        content['url'] = url
        if result['success']:
            content['return_code'] = 0
        self.crits_response(content)
Exemplo n.º 6
0
Arquivo: views.py Projeto: vsbca/crits
def add_new_relationship(request):
    """
    Add a new relationship. Should be an AJAX POST.

    :param request: Django request object (Required)
    :type request: :class:`django.http.HttpRequest`
    :returns: :class:`django.http.HttpResponse`
    """

    if request.method == 'POST' and request.is_ajax():
        form = ForgeRelationshipForm(request.POST)
        user = request.user
        choices = [(c,c) for c in RelationshipTypes.values(sort=True)]
        form.fields['forward_relationship'].choices = choices
        if form.is_valid():
            cleaned_data = form.cleaned_data
            # Get user permission to verify the user can forge relationships...
            # Should we check permission on both the forward and reverse TLO for this?
            acl = get_acl_object(cleaned_data.get('forward_type'))

            if user.has_access_to(acl.RELATIONSHIPS_ADD):
                results = forge_relationship(type_=cleaned_data.get('forward_type'),
                                             id_=cleaned_data.get('forward_value'),
                                             right_type=cleaned_data.get('reverse_type'),
                                             right_id=cleaned_data.get('dest_id'),
                                             rel_type=cleaned_data.get('forward_relationship'),
                                             rel_date=cleaned_data.get('relationship_date'),
                                             user=request.user.username,
                                             rel_reason=cleaned_data.get('rel_reason'),
                                             rel_confidence=cleaned_data.get('rel_confidence'),
                                             get_rels=True)
            else:
                results = {"success":False,
                           "message":"User does not have permission to forge relationships"}
            if results['success'] == True:
                relationship = {'type': cleaned_data.get('forward_type'),
                                'value': cleaned_data.get('forward_value')}
                message = render_to_string('relationships_listing_widget.html',
                                           {'relationship': relationship,
                                            'nohide': True,
                                            'relationships': results['relationships']},
                                           RequestContext(request))
                result = {'success': True, 'message': message}
            else:
                message = "Error adding relationship: %s" % results['message']
                result = {'success': False, 'message': message}
        else:
            message = "Invalid Form: %s" % form.errors
            form = form.as_table()
            result = {'success': False, 'form': form, 'message': message}
        return HttpResponse(json.dumps(result), content_type="application/json")
    else:
        error = "Expected AJAX POST"
        return render_to_response("error.html",
                                  {"error" : error },
                                  RequestContext(request))
Exemplo n.º 7
0
def add_new_relationship(request):
    """
    Add a new relationship. Should be an AJAX POST.

    :param request: Django request object (Required)
    :type request: :class:`django.http.HttpRequest`
    :returns: :class:`django.http.HttpResponse`
    """

    if request.method == 'POST' and request.is_ajax():
        form = ForgeRelationshipForm(request.POST)
        choices = [(c, c) for c in RelationshipTypes.values(sort=True)]
        form.fields['forward_relationship'].choices = choices
        if form.is_valid():
            cleaned_data = form.cleaned_data
            results = forge_relationship(
                type_=cleaned_data.get('forward_type'),
                id_=cleaned_data.get('forward_value'),
                right_type=cleaned_data.get('reverse_type'),
                right_id=cleaned_data.get('dest_id'),
                rel_type=cleaned_data.get('forward_relationship'),
                rel_date=cleaned_data.get('relationship_date'),
                user=request.user.username,
                rel_reason=cleaned_data.get('rel_reason'),
                rel_confidence=cleaned_data.get('rel_confidence'),
                get_rels=True)
            if results['success'] == True:
                set_releasability_flag(cleaned_data.get('forward_type'),
                                       cleaned_data.get('forward_value'),
                                       request.user.username)
                relationship = {
                    'type': cleaned_data.get('forward_type'),
                    'url_key': cleaned_data.get('forward_value')
                }
                message = render_to_string(
                    'relationships_listing_widget.html', {
                        'relationship': relationship,
                        'nohide': True,
                        'relationships': results['relationships']
                    }, RequestContext(request))
                result = {'success': True, 'message': message}
            else:
                message = "Error adding relationship: %s" % results['message']
                result = {'success': False, 'message': message}
        else:
            message = "Invalid Form: %s" % form.errors
            form = form.as_table()
            result = {'success': False, 'form': form, 'message': message}
        return HttpResponse(json.dumps(result), mimetype="application/json")
    else:
        error = "Expected AJAX POST"
        return render_to_response("error.html", {"error": error},
                                  RequestContext(request))
Exemplo n.º 8
0
def add_new_relationship(request):
    """
    Add a new relationship. Should be an AJAX POST.

    :param request: Django request object (Required)
    :type request: :class:`django.http.HttpRequest`
    :returns: :class:`django.http.HttpResponse`
    """

    if request.method == 'POST' and request.is_ajax():
        form = ForgeRelationshipForm(request.POST)
        choices = [(c,c) for c in RelationshipTypes.values(sort=True)]
        form.fields['forward_relationship'].choices = choices
        if form.is_valid():
            cleaned_data = form.cleaned_data;
            results = forge_relationship(type_=cleaned_data.get('forward_type'),
                                         id_=cleaned_data.get('forward_value'),
                                         right_type=cleaned_data.get('reverse_type'),
                                         right_id=cleaned_data.get('dest_id'),
                                         rel_type=cleaned_data.get('forward_relationship'),
                                         rel_date=cleaned_data.get('relationship_date'),
                                         user=request.user.username,
                                         rel_reason=cleaned_data.get('rel_reason'),
                                         rel_confidence=cleaned_data.get('rel_confidence'),
                                         get_rels=True)
            if results['success'] == True:
                relationship = {'type': cleaned_data.get('forward_type'),
                                'value': cleaned_data.get('forward_value')}
                message = render_to_string('relationships_listing_widget.html',
                                           {'relationship': relationship,
                                            'nohide': True,
                                            'relationships': results['relationships']},
                                           RequestContext(request))
                result = {'success': True, 'message': message}
            else:
                message = "Error adding relationship: %s" % results['message']
                result = {'success': False, 'message': message}
        else:
            message = "Invalid Form: %s" % form.errors
            form = form.as_table()
            result = {'success': False, 'form': form, 'message': message}
        return HttpResponse(json.dumps(result), mimetype="application/json")
    else:
        error = "Expected AJAX POST"
        return render_to_response("error.html",
                                  {"error" : error },
                                  RequestContext(request))
Exemplo n.º 9
0
def add_new_relationship(request):
    """
    Add a new relationship. Should be an AJAX POST.

    :param request: Django request object (Required)
    :type request: :class:`django.http.HttpRequest`
    :returns: :class:`django.http.HttpResponse`
    """

    if request.method == "POST" and request.is_ajax():
        form = ForgeRelationshipForm(request.POST)
        choices = [(c, c) for c in get_relationship_types(False)]
        form.fields["forward_relationship"].choices = choices
        if form.is_valid():
            cleaned_data = form.cleaned_data
            results = forge_relationship(
                left_type=cleaned_data.get("forward_type"),
                left_id=cleaned_data.get("forward_value"),
                right_type=cleaned_data.get("reverse_type"),
                right_id=cleaned_data.get("dest_id"),
                rel_type=cleaned_data.get("forward_relationship"),
                rel_date=cleaned_data.get("relationship_date"),
                analyst=request.user.username,
                get_rels=True,
            )
            if results["success"] == True:
                relationship = {"type": cleaned_data.get("forward_type"), "value": cleaned_data.get("forward_value")}
                message = render_to_string(
                    "relationships_listing_widget.html",
                    {"relationship": relationship, "nohide": True, "relationships": results["relationships"]},
                    RequestContext(request),
                )
                result = {"success": True, "message": message}
            else:
                message = "Error adding relationship: %s" % results["message"]
                result = {"success": False, "message": message}
        else:
            message = "Invalid Form: %s" % form.errors
            form = form.as_table()
            result = {"success": False, "form": form, "message": message}
        return HttpResponse(json.dumps(result), mimetype="application/json")
    else:
        error = "Expected AJAX POST"
        return render_to_response("error.html", {"error": error}, RequestContext(request))
Exemplo n.º 10
0
    def parse_relationship(self, indicators):
        from crits.relationships.handlers import forge_relationship

        for indicator in indicators:
            if self.was_saved(indicator):
                obj = class_from_id(str(self.imported[indicator.id_][0]),
                                    str(self.imported[indicator.id_][1].id))

                for rel in getattr(indicator, 'related_indicators', ()):
                    if rel.item.title in 'MARTI Relation':
                        result = forge_relationship(
                            type_=obj._meta['crits_type'],
                            id_=obj.get_url_key(),
                            right_type=self.get_marti_type(
                                rel.item.indicator_types),
                            right_id=str(rel.item.id_),
                            rel_type=str(rel.item.short_description),
                            rel_date=rel.item.timestamp,
                            user='******',
                            rel_reason=str(rel.item.description),
                            rel_confidence=str(rel.item.confidence.value),
                            get_rels=True)
Exemplo n.º 11
0
    def obj_create(self, bundle, **kwargs):
        """
        Handles forging relationships through the API.

        :param bundle: Bundle containing the relationship information.
        :type bundle: Tastypie Bundle object.
        :returns: Bundle object.
        :raises BadRequest: If necessary data is not provided or creation fails.

        """
        analyst = bundle.request.user.username
        left_type = bundle.data.get('left_type', None)
        left_id = bundle.data.get('left_id', None)
        right_type = bundle.data.get('right_type', None)
        right_id = bundle.data.get('right_id', None)
        rel_type = bundle.data.get('rel_type', None)
        rel_date = bundle.data.get('rel_date', None)

        if (not left_type
            or not left_id
            or not right_type
            or not right_id
            or not rel_type):
            raise BadRequest('Need all of the relationship information.')
        result = forge_relationship(left_type=left_type,
                                    left_id=left_id,
                                    right_type=right_type,
                                    right_id=right_id,
                                    rel_type=rel_type,
                                    rel_date=rel_date,
                                    analyst=analyst,
                                    get_rels=False)
        if not result['success']:
            raise BadRequest(result['message'])
        else:
            return bundle
Exemplo n.º 12
0
def add_object(type_, oid, object_type, name, source, method,
               reference, analyst, value=None, file_=None,
               add_indicator=False, get_objects=True,
               obj=None, is_sort_relationships=False,
               is_validate_only=False, is_validate_locally=False, cache={}):
    """
    Add an object to the database.

    :param type_: The top-level object type.
    :type type_: str
    :param oid: The ObjectId of the top-level object.
    :type oid: str
    :param object_type: The type of the ObjectType being added.
    :type object_type: str
    :param name: The name of the ObjectType being added.
    :type name: str
    :param source: The name of the source adding this object.
    :type source: str
    :param method: The method for this object.
    :type method: str
    :param reference: The reference for this object.
    :type reference: str
    :param analyst: The user adding this object.
    :type analyst: str
    :param value: The value of the object.
    :type value: str
    :param file_: The file if the object is a file upload.
    :type file_: file handle.
    :param add_indicator: Also add an indicator for this object.
    :type add_indicator: bool
    :param get_objects: Return the formatted list of objects when completed.
    :type get_object: bool
    :param obj: The CRITs top-level object we are adding objects to.
                This is an optional parameter used mainly for performance
                reasons (by not querying mongo if we already have the
                top level-object).
    :type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
    :param is_validate_only: Only validate, do not add.
    :type is_validate_only: bool
    :param is_validate_locally: Only validate, do not add.
    :type is_validate_locally: bool
    :param cache: Cached data, typically for performance enhancements
                  during bulk operations.
    :type cache: dict
    :returns: dict with keys:
              "success" (boolean),
              "message" (str),
              "objects" (list),
              "relationships" (list)
    """

    results = {}

    if oid == None:
        oid = ""

    if obj == None:
        obj = class_from_id(type_, oid)

    if not obj:
        if is_validate_locally == True:
            # TODO: Perform some form of validation
            results['success'] = True
            return results
        else:
            results['message'] = "Could not find item to add object to."
            results['success'] = False
            return results

    if name == "URL" and "://" not in value.split('.')[0]:
        return {"success" : False, "message" : "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://)"}
    elif object_type == "Address":
        if "ipv4" in name:
            try:
                validate_ipv4_address(value)
            except DjangoValidationError:
                return {"success" : False, "message" : "Invalid IPv4 address. "}
        elif "ipv6" in name:
            try:
                validate_ipv6_address(value)
            except DjangoValidationError:
                return {"success" : False, "message" : "Invalid IPv6 address. "}
        elif "cidr" in name:
            try:
                if '/' not in value:
                    raise ValidationError("")
                cidr_parts = value.split('/')
                if int(cidr_parts[1]) < 0 or int(cidr_parts[1]) > 128:
                    raise ValidationError("")
                if ':' not in cidr_parts[0] and int(cidr_parts[1]) > 32:
                    raise ValidationError("")
                validate_ipv46_address(cidr_parts[0])
            except (ValidationError, ValueError) as cidr_error:
                return {"success" : False, "message" : "Invalid CIDR address. "}

    try:
        cur_len = len(obj.obj)
        if file_:
            data = file_.read()
            filename = file_.name
            md5sum = md5(data).hexdigest()
            value = md5sum
            reference = filename
        obj.add_object(object_type,
                       name,
                       value,
                       source,
                       method,
                       reference,
                       analyst)

        if is_validate_only == False:
            obj.save(username=analyst)

        new_len = len(obj.obj)
        if new_len > cur_len:
            results['message'] = "Object added successfully!"
            results['success'] = True
            if file_:
                # do we have a pcap?
                if data[:4] in ('\xa1\xb2\xc3\xd4',
                                '\xd4\xc3\xb2\xa1',
                                '\x0a\x0d\x0d\x0a'):
                    handle_pcap_file(filename,
                                     data,
                                     source,
                                     user=analyst,
                                     related_id=oid,
                                     related_type=type_)
                else:
                    #XXX: MongoEngine provides no direct GridFS access so we
                    #     need to use pymongo directly.
                    col = settings.COL_OBJECTS
                    grid = mongo_connector("%s.files" % col)
                    if grid.find({'md5': md5sum}).count() == 0:
                        put_file(filename, data, collection=col)
            if add_indicator and is_validate_only == False:
                from crits.indicators.handlers import handle_indicator_ind

                if object_type != name:
                    object_type = "%s - %s" % (object_type, name)

                campaign = obj.campaign if hasattr(obj, 'campaign') else None
                ind_res = handle_indicator_ind(value,
                                               source,
                                               reference,
                                               object_type,
                                               analyst,
                                               method,
                                               add_domain=True,
                                               campaign=campaign,
                                               cache=cache)

                if ind_res['success']:
                    ind = ind_res['object']
                    forge_relationship(left_class=obj,
                                       right_class=ind,
                                       rel_type="Related_To",
                                       analyst=analyst,
                                       get_rels=is_sort_relationships)
                else:
                    results['message'] = "Object was added, but failed to add Indicator." \
                                         "<br>Error: " + ind_res.get('message')

            if is_sort_relationships == True:
                if file_ or add_indicator:
                    # does this line need to be here?
                    # obj.reload()
                    results['relationships'] = obj.sort_relationships(analyst, meta=True)
                else:
                    results['relationships'] = obj.sort_relationships(analyst, meta=True)

        else:
            results['message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
            results['success'] = False
        if (get_objects):
            results['objects'] = obj.sort_objects()

        results['id'] = str(obj.id)
        return results
    except ValidationError, e:
        return {'success': False,
                'message': str(e)}
Exemplo n.º 13
0
def add_new_domain(data,
                   request,
                   errors,
                   rowData=None,
                   is_validate_only=False,
                   cache={}):
    """
    Add a new domain to CRITs.

    :param data: The data about the domain.
    :type data: dict
    :param request: The Django request.
    :type request: :class:`django.http.HttpRequest`
    :param errors: A list of current errors to append to.
    :type errors: list
    :param rowData: Any objects that need to be added to the domain.
    :type rowData: dict
    :param is_validate_only: Only validate the data and return any errors.
    :type is_validate_only: boolean
    :param cache: Cached data, typically for performance enhancements
                  during bulk operations.
    :type cache: dict
    :returns: tuple (<result>, <errors>, <retVal>)
    """

    result = False
    retVal = {}
    domain = data['domain']
    add_ip = data.get('add_ip')
    ip = data.get('ip')
    ip_type = data.get('ip_type')

    if add_ip:
        error = validate_and_normalize_ip(ip, ip_type)[1]
        if error:
            errors.append(error)

    if is_validate_only:
        error = get_valid_root_domain(domain)[2]
        if error:
            errors.append(error)

        # check for duplicate domains
        fqdn_domain = retrieve_domain(domain, cache)

        if fqdn_domain:
            if isinstance(fqdn_domain, Domain):
                resp_url = reverse('crits.domains.views.domain_detail',
                                   args=[domain])
                message = ('Warning: Domain already exists: '
                           '<a href="%s">%s</a>' % (resp_url, domain))
                retVal['message'] = message
                retVal['status'] = form_consts.Status.DUPLICATE
                retVal['warning'] = message
        else:
            result_cache = cache.get(form_consts.Domain.CACHED_RESULTS)
            result_cache[domain.lower()] = True

    elif not errors:
        username = request.user.username
        reference = data.get('domain_reference')
        source_name = data.get('domain_source')
        method = data.get('domain_method')
        source = [
            create_embedded_source(source_name,
                                   reference=reference,
                                   method=method,
                                   analyst=username)
        ]
        bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
        ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)

        if data.get('campaign') and data.get('confidence'):
            campaign = [
                EmbeddedCampaign(name=data.get('campaign'),
                                 confidence=data.get('confidence'),
                                 analyst=username)
            ]
        else:
            campaign = []

        retVal = upsert_domain(domain,
                               source,
                               username,
                               campaign,
                               bucket_list=bucket_list,
                               ticket=ticket,
                               cache=cache)

        if not retVal['success']:
            errors.append(retVal.get('message'))
            retVal['message'] = ""

        else:
            new_domain = retVal['object']
            ip_result = {}
            if add_ip:
                if data.get('same_source'):
                    ip_source = source_name
                    ip_method = method
                    ip_reference = reference
                else:
                    ip_source = data.get('ip_source')
                    ip_method = data.get('ip_method')
                    ip_reference = data.get('ip_reference')
                from crits.ips.handlers import ip_add_update
                ip_result = ip_add_update(ip,
                                          ip_type,
                                          ip_source,
                                          ip_method,
                                          ip_reference,
                                          campaign=campaign,
                                          analyst=username,
                                          bucket_list=bucket_list,
                                          ticket=ticket,
                                          cache=cache)
                if not ip_result['success']:
                    errors.append(ip_result['message'])
                else:
                    #add a relationship with the new IP address
                    new_ip = ip_result['object']
                    if new_domain and new_ip:
                        new_domain.add_relationship(rel_item=new_ip,
                                                    rel_type='Resolved_To',
                                                    analyst=username,
                                                    get_rels=False)
                        new_domain.save(username=username)
                        new_ip.save(username=username)

            #set the URL for viewing the new data
            resp_url = reverse('crits.domains.views.domain_detail',
                               args=[domain])

            if retVal['is_domain_new'] == True:
                retVal['message'] = (
                    'Success! Click here to view the new domain: '
                    '<a href="%s">%s</a>' % (resp_url, domain))
            else:
                message = ('Updated existing domain: <a href="%s">%s</a>' %
                           (resp_url, domain))
                retVal['message'] = message
                retVal[form_consts.Status.
                       STATUS_FIELD] = form_consts.Status.DUPLICATE
                retVal['warning'] = message

            #add indicators
            if data.get('add_indicators'):
                from crits.indicators.handlers import create_indicator_from_tlo
                # If we have an IP object, add an indicator for that.
                if ip_result.get('success'):
                    ip = ip_result['object']
                    result = create_indicator_from_tlo('IP',
                                                       ip,
                                                       username,
                                                       ip_source,
                                                       add_domain=False)
                    ip_ind = result.get('indicator')
                    if not result['success']:
                        errors.append(result['message'])

                # Add an indicator for the domain.
                result = create_indicator_from_tlo('Domain',
                                                   new_domain,
                                                   username,
                                                   source_name,
                                                   add_domain=False)

                if not result['success']:
                    errors.append(result['message'])
                elif ip_result.get('success') and ip_ind:
                    forge_relationship(left_class=result['indicator'],
                                       right_class=ip_ind,
                                       rel_type='Resolved_To',
                                       analyst=username)
            result = True

    # This block validates, and may also add, objects to the Domain
    if retVal.get('success') or is_validate_only == True:
        if rowData:
            objectsData = rowData.get(form_consts.Common.OBJECTS_DATA)

            # add new objects if they exist
            if objectsData:
                objectsData = json.loads(objectsData)
                current_domain = retrieve_domain(domain, cache)
                for object_row_counter, objectData in enumerate(
                        objectsData, 1):
                    if current_domain != None:
                        # if the domain exists then try to add objects to it
                        if isinstance(current_domain, Domain) == True:
                            objectDict = object_array_to_dict(
                                objectData, "Domain", current_domain.id)
                        else:
                            objectDict = object_array_to_dict(
                                objectData, "Domain", "")
                            current_domain = None
                    else:
                        objectDict = object_array_to_dict(
                            objectData, "Domain", "")

                    (obj_result, errors,
                     obj_retVal) = validate_and_add_new_handler_object(
                         None,
                         objectDict,
                         request,
                         errors,
                         object_row_counter,
                         is_validate_only=is_validate_only,
                         cache=cache,
                         obj=current_domain)
                    if not obj_result:
                        retVal['success'] = False

    return result, errors, retVal
Exemplo n.º 14
0
def add_new_domain(data, request, errors, rowData=None, is_validate_only=False, cache={}):
    """
    Add a new domain to CRITs.

    :param data: The data about the domain.
    :type data: dict
    :param request: The Django request.
    :type request: :class:`django.http.HttpRequest`
    :param errors: A list of current errors to append to.
    :type errors: list
    :param rowData: Any objects that need to be added to the domain.
    :type rowData: dict
    :param is_validate_only: Only validate the data and return any errors.
    :type is_validate_only: boolean
    :param cache: Cached data, typically for performance enhancements
                  during bulk operations.
    :type cache: dict
    :returns: tuple (<result>, <errors>, <retVal>)
    """

    result = False
    retVal = {}
    domain = data['domain']
    add_ip = data.get('add_ip')
    ip = data.get('ip')
    ip_type = data.get('ip_type')

    if add_ip:
        error = validate_and_normalize_ip(ip, ip_type)[1]
        if error:
             errors.append(error)

    if is_validate_only:
        error = get_valid_root_domain(domain)[2]
        if error:
            errors.append(error)

        # check for duplicate domains
        fqdn_domain = retrieve_domain(domain, cache)

        if fqdn_domain:
            if isinstance(fqdn_domain, Domain):
                resp_url = reverse('crits.domains.views.domain_detail', args=[domain])
                message = ('Warning: Domain already exists: '
                                     '<a href="%s">%s</a>' % (resp_url, domain))
                retVal['message'] = message
                retVal['status'] = form_consts.Status.DUPLICATE
                retVal['warning'] = message
        else:
            result_cache = cache.get(form_consts.Domain.CACHED_RESULTS);
            result_cache[domain.lower()] = True

    elif not errors:
        username = request.user.username
        reference = data.get('domain_reference')
        source_name = data.get('domain_source')
        method = data.get('domain_method')
        source = [create_embedded_source(source_name, reference=reference,
                                         method=method, analyst=username)]
        bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
        ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)

        if data.get('campaign') and data.get('confidence'):
            campaign = [EmbeddedCampaign(name=data.get('campaign'),
                                         confidence=data.get('confidence'),
                                         analyst=username)]
        else:
            campaign = []

        retVal = upsert_domain(domain, source, username, campaign,
                               bucket_list=bucket_list, ticket=ticket, cache=cache)

        if not retVal['success']:
            errors.append(retVal.get('message'))
            retVal['message'] = ""

        else:
            new_domain = retVal['object']
            ip_result = {}
            if add_ip:
                if data.get('same_source'):
                    ip_source = source_name
                    ip_method = method
                    ip_reference = reference
                else:
                    ip_source = data.get('ip_source')
                    ip_method = data.get('ip_method')
                    ip_reference = data.get('ip_reference')
                from crits.ips.handlers import ip_add_update
                ip_result = ip_add_update(ip,
                                          ip_type,
                                          ip_source,
                                          ip_method,
                                          ip_reference,
                                          campaign=campaign,
                                          analyst=username,
                                          bucket_list=bucket_list,
                                          ticket=ticket,
                                          cache=cache)
                if not ip_result['success']:
                    errors.append(ip_result['message'])
                else:
                    #add a relationship with the new IP address
                    new_ip = ip_result['object']
                    if new_domain and new_ip:
                        new_domain.add_relationship(new_ip,
                                                    'Resolved_To',
                                                    analyst=username,
                                                    get_rels=False)
                        new_domain.save(username=username)

            #set the URL for viewing the new data
            resp_url = reverse('crits.domains.views.domain_detail', args=[domain])

            if retVal['is_domain_new'] == True:
                retVal['message'] = ('Success! Click here to view the new domain: '
                                     '<a href="%s">%s</a>' % (resp_url, domain))
            else:
                message = ('Updated existing domain: <a href="%s">%s</a>' % (resp_url, domain))
                retVal['message'] = message
                retVal[form_consts.Status.STATUS_FIELD] = form_consts.Status.DUPLICATE
                retVal['warning'] = message

            #add indicators
            if data.get('add_indicators'):
                from crits.indicators.handlers import create_indicator_from_tlo
                # If we have an IP object, add an indicator for that.
                if ip_result.get('success'):
                    ip = ip_result['object']
                    result = create_indicator_from_tlo('IP',
                                                       ip,
                                                       username,
                                                       ip_source,
                                                       add_domain=False)
                    ip_ind = result.get('indicator')
                    if not result['success']:
                        errors.append(result['message'])

                # Add an indicator for the domain.
                result = create_indicator_from_tlo('Domain',
                                                   new_domain,
                                                   username,
                                                   source_name,
                                                   add_domain=False)

                if not result['success']:
                    errors.append(result['message'])
                elif ip_result.get('success') and ip_ind:
                    forge_relationship(left_class=result['indicator'],
                                       right_class=ip_ind,
                                       rel_type='Resolved_To',
                                       analyst=username)
            result = True

    # This block validates, and may also add, objects to the Domain
    if retVal.get('success') or is_validate_only == True:
        if rowData:
            objectsData = rowData.get(form_consts.Common.OBJECTS_DATA)

            # add new objects if they exist
            if objectsData:
                objectsData = json.loads(objectsData)
                current_domain = retrieve_domain(domain, cache)
                for object_row_counter, objectData in enumerate(objectsData, 1):
                    if current_domain != None:
                        # if the domain exists then try to add objects to it
                        if isinstance(current_domain, Domain) == True:
                            objectDict = object_array_to_dict(objectData,
                                                              "Domain",
                                                              current_domain.id)
                        else:
                            objectDict = object_array_to_dict(objectData,
                                                              "Domain",
                                                              "")
                            current_domain = None;
                    else:
                        objectDict = object_array_to_dict(objectData,
                                                          "Domain",
                                                          "")

                    (obj_result,
                     errors,
                     obj_retVal) = validate_and_add_new_handler_object(
                        None, objectDict, request, errors, object_row_counter,
                        is_validate_only=is_validate_only,
                        cache=cache, obj=current_domain)
                    if not obj_result:
                        retVal['success'] = False

    return result, errors, retVal
Exemplo n.º 15
0
def add_object(type_, id_, object_type, source, method, reference, tlp, user,
               value=None, file_=None, add_indicator=False, get_objects=True,
               tlo=None, is_sort_relationships=False, is_validate_only=False,
               is_validate_locally=False, cache={}, **kwargs):
    """
    Add an object to the database.

    :param type_: The top-level object type.
    :type type_: str
    :param id_: The ObjectId of the top-level object.
    :type id_: str
    :param object_type: The type of the ObjectType being added.
    :type object_type: str
    :param source: The name of the source adding this object.
    :type source: str
    :param method: The method for this object.
    :type method: str
    :param reference: The reference for this object.
    :type reference: str
    :param user: The user adding this object.
    :type user: str
    :param value: The value of the object.
    :type value: str
    :param file_: The file if the object is a file upload.
    :type file_: file handle.
    :param add_indicator: Also add an indicator for this object.
    :type add_indicator: bool
    :param get_objects: Return the formatted list of objects when completed.
    :type get_objects: bool
    :param tlo: The CRITs top-level object we are adding objects to.
                This is an optional parameter used mainly for performance
                reasons (by not querying mongo if we already have the
                top level-object).
    :type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
    :param is_sort_relationships: Return all relationships and meta, sorted
    :type is_sort_relationships: bool
    :param is_validate_only: Validate, but do not add to TLO.
    :type is_validate_only: bool
    :param is_validate_locally: Validate, but do not add b/c there is no TLO.
    :type is_validate_locally: bool
    :param cache: Cached data, typically for performance enhancements
                  during bulk operations.
    :type cache: dict
    :returns: dict with keys:
              "success" (boolean),
              "message" (str),
              "objects" (list),
              "relationships" (list)
    """

    # if object_type is a validated indicator type, then validate value
    if value:
        from crits.indicators.handlers import validate_indicator_value
        (value, error) = validate_indicator_value(value, object_type)
        if error:
            return {"success": False, "message": error}

    if is_validate_locally: # no TLO provided
        return {"success": True}

    if not tlo:
        if type_ and id_:
            tlo = class_from_id(type_, id_)
        if not tlo:
            return {'success': False, 'message': "Failed to find TLO"}

    try:
        if file_:
            data = file_.read()
            filename = file_.name
            md5sum = md5(data).hexdigest()
            value = md5sum
            reference = filename
        ret = tlo.add_object(object_type, value,
                             source, method, reference, user)

        if not ret['success']:
            msg = '%s! [Type: "%s"][Value: "%s"]'
            return {"success": False,
                    "message": msg % (ret['message'], object_type, value)}
        else:
            results = {'success': True}

        if not is_validate_only: # save the object
            tlo.update(add_to_set__obj=ret['object'])
            results['message'] = "Object added successfully"

        if file_:
            # do we have a pcap?
            if detect_pcap(data):
                handle_pcap_file(filename,
                                 data,
                                 source,
                                 user=user,
                                 related_id=id_,
                                 related_type=type_)
            else:
                #XXX: MongoEngine provides no direct GridFS access so we
                #     need to use pymongo directly.
                col = settings.COL_OBJECTS
                grid = mongo_connector("%s.files" % col)
                if grid.find({'md5': md5sum}).count() == 0:
                    put_file(filename, data, collection=col)

        if add_indicator and not is_validate_only:
            campaign = tlo.campaign if hasattr(tlo, 'campaign') else None
            from crits.indicators.handlers import handle_indicator_ind
            ind_res = handle_indicator_ind(value,
                                           source,
                                           object_type,
                                           IndicatorThreatTypes.UNKNOWN,
                                           IndicatorAttackTypes.UNKNOWN,
                                           user,
                                           source_method=method,
                                           source_reference=reference,
                                           source_tlp=tlp,
                                           add_domain=True,
                                           campaign=campaign,
                                           cache=cache)

            if ind_res['success']:
                forge_relationship(class_=tlo,
                                   right_class=ind_res['object'],
                                   rel_type=RelationshipTypes.RELATED_TO,
                                   user=user)
            else:
                msg = "Object added, but failed to add Indicator.<br>Error: %s"
                results['message'] = msg % ind_res.get('message')

        if is_sort_relationships == True:
            results['relationships'] = tlo.sort_relationships(user, meta=True)

        if get_objects:
            results['objects'] = tlo.sort_objects()

        results['id'] = str(tlo.id)
        return results
    except ValidationError as e:
        return {'success': False, 'message': str(e)}
Exemplo n.º 16
0
def add_object(type_,
               id_,
               object_type,
               source,
               method,
               reference,
               tlp,
               user,
               value=None,
               file_=None,
               add_indicator=False,
               get_objects=True,
               tlo=None,
               is_sort_relationships=False,
               is_validate_only=False,
               is_validate_locally=False,
               cache={},
               **kwargs):
    """
    Add an object to the database.

    :param type_: The top-level object type.
    :type type_: str
    :param id_: The ObjectId of the top-level object.
    :type id_: str
    :param object_type: The type of the ObjectType being added.
    :type object_type: str
    :param source: The name of the source adding this object.
    :type source: str
    :param method: The method for this object.
    :type method: str
    :param reference: The reference for this object.
    :type reference: str
    :param user: The user adding this object.
    :type user: str
    :param value: The value of the object.
    :type value: str
    :param file_: The file if the object is a file upload.
    :type file_: file handle.
    :param add_indicator: Also add an indicator for this object.
    :type add_indicator: bool
    :param get_objects: Return the formatted list of objects when completed.
    :type get_objects: bool
    :param tlo: The CRITs top-level object we are adding objects to.
                This is an optional parameter used mainly for performance
                reasons (by not querying mongo if we already have the
                top level-object).
    :type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
    :param is_sort_relationships: Return all relationships and meta, sorted
    :type is_sort_relationships: bool
    :param is_validate_only: Validate, but do not add to TLO.
    :type is_validate_only: bool
    :param is_validate_locally: Validate, but do not add b/c there is no TLO.
    :type is_validate_locally: bool
    :param cache: Cached data, typically for performance enhancements
                  during bulk operations.
    :type cache: dict
    :returns: dict with keys:
              "success" (boolean),
              "message" (str),
              "objects" (list),
              "relationships" (list)
    """

    # if object_type is a validated indicator type, then validate value
    if value:
        from crits.indicators.handlers import validate_indicator_value
        (value, error) = validate_indicator_value(value, object_type)
        if error:
            return {"success": False, "message": error}

    if is_validate_locally:  # no TLO provided
        return {"success": True}

    if not tlo:
        if type_ and id_:
            tlo = class_from_id(type_, id_)
        if not tlo:
            return {'success': False, 'message': "Failed to find TLO"}

    try:
        if file_:
            data = file_.read()
            filename = file_.name
            md5sum = md5(data).hexdigest()
            value = md5sum
            reference = filename
        ret = tlo.add_object(object_type, value, source, method, reference,
                             user)

        if not ret['success']:
            msg = '%s! [Type: "%s"][Value: "%s"]'
            return {
                "success": False,
                "message": msg % (ret['message'], object_type, value)
            }
        else:
            results = {'success': True}

        if not is_validate_only:  # save the object
            tlo.update(add_to_set__obj=ret['object'])
            results['message'] = "Object added successfully"

        if file_:
            # do we have a pcap?
            if detect_pcap(data):
                handle_pcap_file(filename,
                                 data,
                                 source,
                                 user=user,
                                 related_id=id_,
                                 related_type=type_)
            else:
                #XXX: MongoEngine provides no direct GridFS access so we
                #     need to use pymongo directly.
                col = settings.COL_OBJECTS
                grid = mongo_connector("%s.files" % col)
                if grid.find({'md5': md5sum}).count() == 0:
                    put_file(filename, data, collection=col)

        if add_indicator and not is_validate_only:
            campaign = tlo.campaign if hasattr(tlo, 'campaign') else None
            from crits.indicators.handlers import handle_indicator_ind
            ind_res = handle_indicator_ind(value,
                                           source,
                                           object_type,
                                           IndicatorThreatTypes.UNKNOWN,
                                           IndicatorAttackTypes.UNKNOWN,
                                           user,
                                           source_method=method,
                                           source_reference=reference,
                                           source_tlp=tlp,
                                           add_domain=True,
                                           campaign=campaign,
                                           cache=cache)

            if ind_res['success']:
                forge_relationship(class_=tlo,
                                   right_class=ind_res['object'],
                                   rel_type=RelationshipTypes.RELATED_TO,
                                   user=user)
            else:
                msg = "Object added, but failed to add Indicator.<br>Error: %s"
                results['message'] = msg % ind_res.get('message')

        if is_sort_relationships == True:
            results['relationships'] = tlo.sort_relationships(user, meta=True)

        if get_objects:
            results['objects'] = tlo.sort_objects()

        results['id'] = str(tlo.id)
        return results
    except ValidationError as e:
        return {'success': False, 'message': str(e)}
Exemplo n.º 17
0
def add_object(type_,
               oid,
               object_type,
               name,
               source,
               method,
               reference,
               analyst,
               value=None,
               file_=None,
               add_indicator=False,
               get_objects=True,
               indicator_campaign=None,
               indicator_campaign_confidence=None,
               obj=None,
               is_sort_relationships=False,
               is_validate_only=False,
               is_validate_locally=False,
               cache={}):
    """
    Add an object to the database.

    :param type_: The top-level object type.
    :type type_: str
    :param oid: The ObjectId of the top-level object.
    :type oid: str
    :param object_type: The type of the ObjectType being added.
    :type object_type: str
    :param name: The name of the ObjectType being added.
    :type name: str
    :param source: The name of the source adding this object.
    :type source: str
    :param method: The method for this object.
    :type method: str
    :param reference: The reference for this object.
    :type reference: str
    :param analyst: The user adding this object.
    :type analyst: str
    :param value: The value of the object.
    :type value: str
    :param file_: The file if the object is a file upload.
    :type file_: file handle.
    :param add_indicator: Also add an indicator for this object.
    :type add_indicator: bool
    :param get_objects: Return the formatted list of objects when completed.
    :type get_object: bool
    :param is_validate_only: Only validate, do not add.
    :type is_validate_only: bool
    :param is_validate_locally: Only validate, do not add.
    :type is_validate_locally: bool
    :param cache: Cached data, typically for performance enhancements
                  during bulk operations.
    :type cache: dict
    :param obj: The CRITs top-level object we are adding objects to.
                This is an optional parameter used mainly for performance
                reasons (by not querying mongo if we already have the
                top level-object).
    :type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
    :returns: dict with keys:
              "success" (boolean),
              "message" (str),
              "objects" (list),
              "relationships" (list)
    """

    results = {}

    if oid == None:
        oid = ""

    if obj == None:
        obj = class_from_id(type_, oid)

    if not obj:
        if is_validate_locally == True:
            # TODO: Perform some form of validation
            results['success'] = True
            return results
        else:
            results['message'] = "Could not find item to add object to."
            results['success'] = False
            return results
    try:
        cur_len = len(obj.obj)
        if file_:
            data = file_.read()
            filename = file_.name
            md5sum = md5(data).hexdigest()
            value = md5sum
            reference = filename
        obj.add_object(object_type, name, value, source, method, reference,
                       analyst)

        if is_validate_only == False:
            obj.save(username=analyst)

        new_len = len(obj.obj)
        if new_len > cur_len:
            results['message'] = "Object added successfully!"
            results['success'] = True
            if file_:
                # do we have a pcap?
                if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1',
                                '\x0a\x0d\x0d\x0a'):
                    handle_pcap_file(filename,
                                     data,
                                     source,
                                     user=analyst,
                                     related_id=oid,
                                     related_type=type_)
                else:
                    #XXX: MongoEngine provides no direct GridFS access so we
                    #     need to use pymongo directly.
                    col = settings.COL_OBJECTS
                    grid = mongo_connector("%s.files" % col)
                    if grid.find({'md5': md5sum}).count() == 0:
                        put_file(filename, data, collection=col)
            if add_indicator and is_validate_only == False:
                from crits.indicators.handlers import handle_indicator_ind

                if object_type != name:
                    object_type = "%s - %s" % (object_type, name)

                ind_res = handle_indicator_ind(
                    value,
                    source,
                    reference,
                    object_type,
                    analyst,
                    method=method,
                    add_domain=True,
                    campaign=indicator_campaign,
                    campaign_confidence=indicator_campaign_confidence,
                    cache=cache)

                if ind_res['success']:
                    ind = ind_res['object']

                    # Inherit campaigns from top level item when creating
                    # an indicator from an object if no campaigns were specified
                    if indicator_campaign == None and ind != None:
                        for campaign in obj.campaign:
                            ec = EmbeddedCampaign(
                                name=campaign.name,
                                confidence=campaign.confidence,
                                description="",
                                analyst=analyst,
                                date=datetime.datetime.now())
                            ind.add_campaign(ec)

                        ind.save(username=analyst)

                    forge_relationship(left_class=obj,
                                       right_class=ind,
                                       rel_type="Related_To",
                                       analyst=analyst,
                                       get_rels=is_sort_relationships)

            if is_sort_relationships == True:
                if file_ or add_indicator:
                    # does this line need to be here?
                    # obj.reload()
                    results['relationships'] = obj.sort_relationships(
                        analyst, meta=True)
                else:
                    results['relationships'] = obj.sort_relationships(
                        analyst, meta=True)

        else:
            results[
                'message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
            results['success'] = False
        if (get_objects):
            results['objects'] = obj.sort_objects()

        return results
    except ValidationError, e:
        return {'success': False, 'message': e}
Exemplo n.º 18
0
def add_object(type_, id_, object_type, source, method,
               reference, user, value=None, file_=None,
               add_indicator=False, get_objects=True,
               obj=None, is_sort_relationships=False,
               is_validate_only=False, is_validate_locally=False, cache={},
               **kwargs):
    """
    Add an object to the database.

    :param type_: The top-level object type.
    :type type_: str
    :param id_: The ObjectId of the top-level object.
    :type id_: str
    :param object_type: The type of the ObjectType being added.
    :type object_type: str
    :param source: The name of the source adding this object.
    :type source: str
    :param method: The method for this object.
    :type method: str
    :param reference: The reference for this object.
    :type reference: str
    :param user: The user adding this object.
    :type user: str
    :param value: The value of the object.
    :type value: str
    :param file_: The file if the object is a file upload.
    :type file_: file handle.
    :param add_indicator: Also add an indicator for this object.
    :type add_indicator: bool
    :param get_objects: Return the formatted list of objects when completed.
    :type get_object: bool
    :param obj: The CRITs top-level object we are adding objects to.
                This is an optional parameter used mainly for performance
                reasons (by not querying mongo if we already have the
                top level-object).
    :type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
    :param is_validate_only: Validate, but do not add to TLO.
    :type is_validate_only: bool
    :param is_validate_locally: Validate, but do not add b/c there is no TLO.
    :type is_validate_locally: bool
    :param cache: Cached data, typically for performance enhancements
                  during bulk operations.
    :type cache: dict
    :returns: dict with keys:
              "success" (boolean),
              "message" (str),
              "objects" (list),
              "relationships" (list)
    """

    results = {}

    if id_ == None:
        id_ = ""

    if obj == None:
        obj = class_from_id(type_, id_)

    from crits.indicators.handlers import validate_indicator_value
    if value is not None:
        (value, error) = validate_indicator_value(value, object_type)
        if error:
            return {"success": False, "message": error}

    if is_validate_locally: # no obj provided
        results['success'] = True
        return results

    if not obj:
        results['message'] = "TLO could not be found"
        results['success'] = False
        return results

    try:
        cur_len = len(obj.obj)
        if file_:
            data = file_.read()
            filename = file_.name
            md5sum = md5(data).hexdigest()
            value = md5sum
            reference = filename
        obj.add_object(object_type,
                       value,
                       source,
                       method,
                       reference,
                       user)

        if is_validate_only == False:
            obj.save(username=user)

        new_len = len(obj.obj)
        if new_len > cur_len:
            if not is_validate_only:
                results['message'] = "Object added successfully!"
            results['success'] = True
            if file_:
                # do we have a pcap?
                if data[:4] in ('\xa1\xb2\xc3\xd4',
                                '\xd4\xc3\xb2\xa1',
                                '\x0a\x0d\x0d\x0a'):
                    handle_pcap_file(filename,
                                     data,
                                     source,
                                     user=user,
                                     related_id=id_,
                                     related_type=type_)
                else:
                    #XXX: MongoEngine provides no direct GridFS access so we
                    #     need to use pymongo directly.
                    col = settings.COL_OBJECTS
                    grid = mongo_connector("%s.files" % col)
                    if grid.find({'md5': md5sum}).count() == 0:
                        put_file(filename, data, collection=col)
            if add_indicator and is_validate_only == False:
                from crits.indicators.handlers import handle_indicator_ind

                campaign = obj.campaign if hasattr(obj, 'campaign') else None
                ind_res = handle_indicator_ind(value,
                                               source,
                                               object_type,
                                               IndicatorThreatTypes.UNKNOWN,
                                               IndicatorAttackTypes.UNKNOWN,
                                               user,
                                               method,
                                               reference,
                                               add_domain=True,
                                               campaign=campaign,
                                               cache=cache)

                if ind_res['success']:
                    ind = ind_res['object']
                    forge_relationship(class_=obj,
                                       right_class=ind,
                                       rel_type=RelationshipTypes.RELATED_TO,
                                       user=user,
                                       get_rels=is_sort_relationships)
                else:
                    results['message'] = "Object was added, but failed to add Indicator." \
                                         "<br>Error: " + ind_res.get('message')

            if is_sort_relationships == True:
                if file_ or add_indicator:
                    # does this line need to be here?
                    # obj.reload()
                    results['relationships'] = obj.sort_relationships(user, meta=True)
                else:
                    results['relationships'] = obj.sort_relationships(user, meta=True)

        else:
            results['message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
            results['success'] = False
        if (get_objects):
            results['objects'] = obj.sort_objects()

        results['id'] = str(obj.id)
        return results
    except ValidationError, e:
        return {'success': False,
                'message': str(e)}
Exemplo n.º 19
0
def add_object(type_, oid, object_type, name, source, method,
               reference, analyst, value=None, file_=None,
               add_indicator=False, get_objects=True,
               indicator_campaign=None, indicator_campaign_confidence=None,
               obj=None, is_sort_relationships=False,
               is_validate_only=False, is_validate_locally=False, cache={}):
    """
    Add an object to the database.

    :param type_: The top-level object type.
    :type type_: str
    :param oid: The ObjectId of the top-level object.
    :type oid: str
    :param object_type: The type of the ObjectType being added.
    :type object_type: str
    :param name: The name of the ObjectType being added.
    :type name: str
    :param source: The name of the source adding this object.
    :type source: str
    :param method: The method for this object.
    :type method: str
    :param reference: The reference for this object.
    :type reference: str
    :param analyst: The user adding this object.
    :type analyst: str
    :param value: The value of the object.
    :type value: str
    :param file_: The file if the object is a file upload.
    :type file_: file handle.
    :param add_indicator: Also add an indicator for this object.
    :type add_indicator: bool
    :param get_objects: Return the formatted list of objects when completed.
    :type get_object: bool
    :param is_validate_only: Only validate, do not add.
    :type is_validate_only: bool
    :param is_validate_locally: Only validate, do not add.
    :type is_validate_locally: bool
    :param cache: Cached data, typically for performance enhancements
                  during bulk operations.
    :type cache: dict
    :param obj: The CRITs top-level object we are adding objects to.
                This is an optional parameter used mainly for performance
                reasons (by not querying mongo if we already have the
                top level-object).
    :type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
    :returns: dict with keys:
              "success" (boolean),
              "message" (str),
              "objects" (list),
              "relationships" (list)
    """

    results = {}

    if oid == None:
        oid = ""

    if obj == None:
        obj = class_from_id(type_, oid)

    if not obj:
        if is_validate_locally == True:
            # TODO: Perform some form of validation
            results['success'] = True
            return results
        else:
            results['message'] = "Could not find item to add object to."
            results['success'] = False
            return results
    try:
        cur_len = len(obj.obj)
        if file_:
            data = file_.read()
            filename = file_.name
            md5sum = md5(data).hexdigest()
            value = md5sum
            reference = filename
        obj.add_object(object_type,
                       name,
                       value,
                       source,
                       method,
                       reference,
                       analyst)

        if is_validate_only == False:
            obj.save(username=analyst)

        new_len = len(obj.obj)
        if new_len > cur_len:
            results['message'] = "Object added successfully!"
            results['success'] = True
            if file_:
                # do we have a pcap?
                if data[:4] in ('\xa1\xb2\xc3\xd4',
                                '\xd4\xc3\xb2\xa1',
                                '\x0a\x0d\x0d\x0a'):
                    handle_pcap_file(filename,
                                     data,
                                     source,
                                     user=analyst,
                                     parent_id=oid,
                                     parent_type=type_)
                else:
                    #XXX: MongoEngine provides no direct GridFS access so we
                    #     need to use pymongo directly.
                    col = settings.COL_OBJECTS
                    grid = mongo_connector("%s.files" % col)
                    if grid.find({'md5': md5sum}).count() == 0:
                        put_file(filename, data, collection=col)
            if add_indicator and is_validate_only == False:
                from crits.indicators.handlers import handle_indicator_ind

                if object_type != name:
                    object_type = "%s - %s" % (object_type, name)

                ind_res = handle_indicator_ind(value,
                                               source,
                                               reference,
                                               object_type,
                                               analyst,
                                               method=method,
                                               add_domain=True,
                                               campaign=indicator_campaign,
                                               campaign_confidence=indicator_campaign_confidence,
                                               cache=cache)

                if ind_res['success']:
                    ind = ind_res['object']

                    # Inherit campaigns from top level item when creating
                    # an indicator from an object if no campaigns were specified
                    if indicator_campaign == None and ind != None:
                        for campaign in obj.campaign:
                            ec = EmbeddedCampaign(name=campaign.name,
                                                  confidence=campaign.confidence,
                                                  description="",
                                                  analyst=analyst,
                                                  date=datetime.datetime.now())
                            ind.add_campaign(ec)

                        ind.save(username=analyst)

                    forge_relationship(left_class=obj,
                                       right_class=ind,
                                       rel_type="Related_To",
                                       analyst=analyst,
                                       get_rels=is_sort_relationships)

            if is_sort_relationships == True:
                if file_ or add_indicator:
                    # does this line need to be here?
                    # obj.reload()
                    results['relationships'] = obj.sort_relationships(analyst, meta=True)
                else:
                    results['relationships'] = obj.sort_relationships(analyst, meta=True)

        else:
            results['message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
            results['success'] = False
        if (get_objects):
            results['objects'] = obj.sort_objects()

        return results
    except ValidationError, e:
        return {'success': False,
                'message': e}
Exemplo n.º 20
0
def add_object(type_,
               id_,
               object_type,
               source,
               method,
               reference,
               user,
               value=None,
               file_=None,
               add_indicator=False,
               get_objects=True,
               tlo=None,
               is_sort_relationships=False,
               is_validate_only=False,
               is_validate_locally=False,
               cache={},
               **kwargs):
    """
    Add an object to the database.

    :param type_: The top-level object type.
    :type type_: str
    :param id_: The ObjectId of the top-level object.
    :type id_: str
    :param object_type: The type of the ObjectType being added.
    :type object_type: str
    :param source: The name of the source adding this object.
    :type source: str
    :param method: The method for this object.
    :type method: str
    :param reference: The reference for this object.
    :type reference: str
    :param user: The user adding this object.
    :type user: str
    :param value: The value of the object.
    :type value: str
    :param file_: The file if the object is a file upload.
    :type file_: file handle.
    :param add_indicator: Also add an indicator for this object.
    :type add_indicator: bool
    :param get_objects: Return the formatted list of objects when completed.
    :type get_object: bool
    :param tlo: The CRITs top-level object we are adding objects to.
                This is an optional parameter used mainly for performance
                reasons (by not querying mongo if we already have the
                top level-object).
    :type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
    :param is_validate_only: Validate, but do not add to TLO.
    :type is_validate_only: bool
    :param is_validate_locally: Validate, but do not add b/c there is no TLO.
    :type is_validate_locally: bool
    :param cache: Cached data, typically for performance enhancements
                  during bulk operations.
    :type cache: dict
    :returns: dict with keys:
              "success" (boolean),
              "message" (str),
              "objects" (list),
              "relationships" (list)
    """

    results = {}
    obj = tlo
    if id_ == None:
        id_ = ""

    if obj == None:
        obj = class_from_id(type_, id_)

    from crits.indicators.handlers import validate_indicator_value
    if value is not None:
        (value, error) = validate_indicator_value(value, object_type)
        if error:
            return {"success": False, "message": error}

    if is_validate_locally:  # no obj provided
        results['success'] = True
        return results

    if not obj:
        results['message'] = "TLO could not be found"
        results['success'] = False
        return results

    try:
        cur_len = len(obj.obj)
        if file_:
            data = file_.read()
            filename = file_.name
            md5sum = md5(data).hexdigest()
            value = md5sum
            reference = filename
        obj.add_object(object_type, value, source, method, reference, user)

        if is_validate_only == False:
            obj.save(username=user)

        new_len = len(obj.obj)
        if new_len > cur_len:
            if not is_validate_only:
                results['message'] = "Object added successfully!"
            results['success'] = True
            if file_:
                # do we have a pcap?
                if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1',
                                '\x0a\x0d\x0d\x0a'):
                    handle_pcap_file(filename,
                                     data,
                                     source,
                                     user=user,
                                     related_id=id_,
                                     related_type=type_)
                else:
                    #XXX: MongoEngine provides no direct GridFS access so we
                    #     need to use pymongo directly.
                    col = settings.COL_OBJECTS
                    grid = mongo_connector("%s.files" % col)
                    if grid.find({'md5': md5sum}).count() == 0:
                        put_file(filename, data, collection=col)
            if add_indicator and is_validate_only == False:
                from crits.indicators.handlers import handle_indicator_ind

                campaign = obj.campaign if hasattr(obj, 'campaign') else None
                ind_res = handle_indicator_ind(value,
                                               source,
                                               object_type,
                                               IndicatorThreatTypes.UNKNOWN,
                                               IndicatorAttackTypes.UNKNOWN,
                                               user,
                                               method,
                                               reference,
                                               add_domain=True,
                                               campaign=campaign,
                                               cache=cache)

                if ind_res['success']:
                    ind = ind_res['object']
                    forge_relationship(class_=obj,
                                       right_class=ind,
                                       rel_type=RelationshipTypes.RELATED_TO,
                                       user=user,
                                       get_rels=is_sort_relationships)
                else:
                    results['message'] = "Object was added, but failed to add Indicator." \
                                         "<br>Error: " + ind_res.get('message')

            if is_sort_relationships == True:
                if file_ or add_indicator:
                    # does this line need to be here?
                    # obj.reload()
                    results['relationships'] = obj.sort_relationships(
                        user, meta=True)
                else:
                    results['relationships'] = obj.sort_relationships(
                        user, meta=True)

        else:
            results[
                'message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
            results['success'] = False
        if (get_objects):
            results['objects'] = obj.sort_objects()

        results['id'] = str(obj.id)
        return results
    except ValidationError, e:
        return {'success': False, 'message': str(e)}
Exemplo n.º 21
0
def add_object(type_,
               oid,
               object_type,
               name,
               source,
               method,
               reference,
               analyst,
               value=None,
               file_=None,
               add_indicator=False,
               get_objects=True,
               obj=None,
               is_sort_relationships=False,
               is_validate_only=False,
               is_validate_locally=False,
               cache={}):
    """
    Add an object to the database.

    :param type_: The top-level object type.
    :type type_: str
    :param oid: The ObjectId of the top-level object.
    :type oid: str
    :param object_type: The type of the ObjectType being added.
    :type object_type: str
    :param name: The name of the ObjectType being added.
    :type name: str
    :param source: The name of the source adding this object.
    :type source: str
    :param method: The method for this object.
    :type method: str
    :param reference: The reference for this object.
    :type reference: str
    :param analyst: The user adding this object.
    :type analyst: str
    :param value: The value of the object.
    :type value: str
    :param file_: The file if the object is a file upload.
    :type file_: file handle.
    :param add_indicator: Also add an indicator for this object.
    :type add_indicator: bool
    :param get_objects: Return the formatted list of objects when completed.
    :type get_object: bool
    :param obj: The CRITs top-level object we are adding objects to.
                This is an optional parameter used mainly for performance
                reasons (by not querying mongo if we already have the
                top level-object).
    :type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
    :param is_validate_only: Only validate, do not add.
    :type is_validate_only: bool
    :param is_validate_locally: Only validate, do not add.
    :type is_validate_locally: bool
    :param cache: Cached data, typically for performance enhancements
                  during bulk operations.
    :type cache: dict
    :returns: dict with keys:
              "success" (boolean),
              "message" (str),
              "objects" (list),
              "relationships" (list)
    """

    results = {}

    if oid == None:
        oid = ""

    if obj == None:
        obj = class_from_id(type_, oid)

    if not obj:
        if is_validate_locally == True:
            # TODO: Perform some form of validation
            results['success'] = True
            return results
        else:
            results['message'] = "Could not find item to add object to."
            results['success'] = False
            return results

    if name == "URL" and "://" not in value.split('.')[0]:
        return {
            "success":
            False,
            "message":
            "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://)"
        }
    elif object_type == "Address":
        if "ipv4" in name:
            try:
                validate_ipv4_address(value)
            except DjangoValidationError:
                return {"success": False, "message": "Invalid IPv4 address. "}
        elif "ipv6" in name:
            try:
                validate_ipv6_address(value)
            except DjangoValidationError:
                return {"success": False, "message": "Invalid IPv6 address. "}
        elif "cidr" in name:
            try:
                if '/' not in value:
                    raise ValidationError("")
                cidr_parts = value.split('/')
                if int(cidr_parts[1]) < 0 or int(cidr_parts[1]) > 128:
                    raise ValidationError("")
                if ':' not in cidr_parts[0] and int(cidr_parts[1]) > 32:
                    raise ValidationError("")
                validate_ipv46_address(cidr_parts[0])
            except (ValidationError, ValueError) as cidr_error:
                return {"success": False, "message": "Invalid CIDR address. "}

    try:
        cur_len = len(obj.obj)
        if file_:
            data = file_.read()
            filename = file_.name
            md5sum = md5(data).hexdigest()
            value = md5sum
            reference = filename
        obj.add_object(object_type, name, value, source, method, reference,
                       analyst)

        if is_validate_only == False:
            obj.save(username=analyst)

        new_len = len(obj.obj)
        if new_len > cur_len:
            results['message'] = "Object added successfully!"
            results['success'] = True
            if file_:
                # do we have a pcap?
                if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1',
                                '\x0a\x0d\x0d\x0a'):
                    handle_pcap_file(filename,
                                     data,
                                     source,
                                     user=analyst,
                                     related_id=oid,
                                     related_type=type_)
                else:
                    #XXX: MongoEngine provides no direct GridFS access so we
                    #     need to use pymongo directly.
                    col = settings.COL_OBJECTS
                    grid = mongo_connector("%s.files" % col)
                    if grid.find({'md5': md5sum}).count() == 0:
                        put_file(filename, data, collection=col)
            if add_indicator and is_validate_only == False:
                from crits.indicators.handlers import handle_indicator_ind

                if object_type != name:
                    object_type = "%s - %s" % (object_type, name)

                campaign = obj.campaign if hasattr(obj, 'campaign') else None
                ind_res = handle_indicator_ind(value,
                                               source,
                                               reference,
                                               object_type,
                                               analyst,
                                               method,
                                               add_domain=True,
                                               campaign=campaign,
                                               cache=cache)

                if ind_res['success']:
                    ind = ind_res['object']
                    forge_relationship(left_class=obj,
                                       right_class=ind,
                                       rel_type="Related_To",
                                       analyst=analyst,
                                       get_rels=is_sort_relationships)
                else:
                    results['message'] = "Object was added, but failed to add Indicator." \
                                         "<br>Error: " + ind_res.get('message')

            if is_sort_relationships == True:
                if file_ or add_indicator:
                    # does this line need to be here?
                    # obj.reload()
                    results['relationships'] = obj.sort_relationships(
                        analyst, meta=True)
                else:
                    results['relationships'] = obj.sort_relationships(
                        analyst, meta=True)

        else:
            results[
                'message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
            results['success'] = False
        if (get_objects):
            results['objects'] = obj.sort_objects()

        results['id'] = str(obj.id)
        return results
    except ValidationError, e:
        return {'success': False, 'message': str(e)}