Пример #1
0
def handle_raw_data_file(data, source_name, user=None,
                         description=None, title=None, data_type=None,
                         tool_name=None, tool_version=None, tool_details=None,
                         link_id=None, method='', reference='', tlp='',
                         copy_rels=False, bucket_list=None, ticket=None,
                         related_id=None, related_type=None, relationship_type=None):
    """
    Add RawData.

    :param data: The data of the RawData.
    :type data: str
    :param source_name: The source which provided this RawData.
    :type source_name: str,
                       :class:`crits.core.crits_mongoengine.EmbeddedSource`,
                       list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
    :param user: The user adding the RawData.
    :type user: str
    :param description: Description of the RawData.
    :type description: str
    :param title: Title of the RawData.
    :type title: str
    :param data_type: Datatype of the RawData.
    :type data_type: str
    :param tool_name: Name of the tool used to acquire/generate the RawData.
    :type tool_name: str
    :param tool_version: Version of the tool.
    :type tool_version: str
    :param tool_details: Details about the tool.
    :type tool_details: str
    :param link_id: LinkId to tie this to another RawData as a new version.
    :type link_id: str
    :param method: The method of acquiring this RawData.
    :type method: str
    :param reference: A reference to the source of this RawData.
    :type reference: str
    :param tlp: TLP for the source.
    :type tlp: str
    :param copy_rels: Copy relationships from the previous version to this one.
    :type copy_rels: bool
    :param bucket_list: Bucket(s) to add to this RawData
    :type bucket_list: str(comma separated) or list.
    :param ticket: Ticket(s) to add to this RawData
    :type ticket: str(comma separated) or list.
    :param related_id: ID of object to create relationship with
    :type related_id: str
    :param related_type: Type of object to create relationship with
    :type related_type: str
    :param relationship_type: Type of relationship to create.
    :type relationship_type: str
    :returns: dict with keys:
              'success' (boolean),
              'message' (str),
              '_id' (str) if successful.
    """

    if not data or not title or not data_type:
        status = {
            'success':   False,
            'message':  'No data object, title, or data type passed in'
        }
        return status

    if not source_name:
        return {"success" : False, "message" : "Missing source information."}

    rdt = RawDataType.objects(name=data_type).first()
    if not rdt:
        status = {
            'success':   False,
            'message':  'Invalid data type passed in'
        }
        return status

    if len(data) <= 0:
        status = {
            'success':   False,
            'message':  'Data length <= 0'
        }
        return status

    if isinstance(data, unicode):
        data=data.encode('utf-8')
    # generate md5 and timestamp
    md5 = hashlib.md5(data).hexdigest()
    timestamp = datetime.datetime.now()

    # generate raw_data
    is_rawdata_new = False
    raw_data = RawData.objects(md5=md5).first()
    if not raw_data:
        raw_data = RawData()
        raw_data.created = timestamp
        raw_data.description = description
        raw_data.md5 = md5
        # raw_data.source = [source_name]
        raw_data.data = data
        raw_data.title = title
        raw_data.data_type = data_type
        raw_data.add_tool(name=tool_name,
                          version=tool_version,
                          details=tool_details)
        is_rawdata_new = True

    # generate new source information and add to sample
    if isinstance(source_name, basestring) and len(source_name) > 0:
        if user.check_source_write(source_name):
            source = create_embedded_source(source_name,
                                       method=method,
                                       reference=reference,
                                       tlp=tlp,
                                       analyst=user.username)
            raw_data.add_source(source)

        else:
            return {"success":False,
                    "message": "User does not have permission to add object using source %s." % source_name}
        # this will handle adding a new source, or an instance automatically

    elif isinstance(source_name, EmbeddedSource):
        raw_data.add_source(source_name, method=method, reference=reference, tlp=tlp, analyst=user.usrname)
    elif isinstance(source_name, list) and len(source_name) > 0:
        for s in source_name:
            if isinstance(s, EmbeddedSource):
                raw_data.add_source(s, method=method, reference=reference, tlp=tlp, analyst=user.username)

    #XXX: need to validate this is a UUID
    if link_id:
        raw_data.link_id = link_id
        if copy_rels:
            rd2 = RawData.objects(link_id=link_id).first()
            if rd2:
                if len(rd2.relationships):
                    raw_data.save(username=user)
                    raw_data.reload()
                    for rel in rd2.relationships:
                        # Get object to relate to.
                        rel_item = class_from_id(rel.rel_type, rel.object_id)
                        if rel_item:
                            raw_data.add_relationship(rel_item,
                                                      rel.relationship,
                                                      rel_date=rel.relationship_date,
                                                      analyst=user.username)


    raw_data.version = len(RawData.objects(link_id=link_id)) + 1

    if bucket_list:
        raw_data.add_bucket_list(bucket_list, user)

    if ticket:
        raw_data.add_ticket(ticket, user);

    related_obj = None
    if related_id and related_type:
        related_obj = class_from_id(related_type, related_id)
        if not related_obj:
            retVal['success'] = False
            retVal['message'] = 'Related Object not found.'
            return retVal

    raw_data.save(username=user.username)

    if related_obj and relationship_type and raw_data:
        relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
        raw_data.add_relationship(related_obj,
                              relationship_type,
                              analyst=user.username,
                              get_rels=False)
        raw_data.save(username=user.username)
        raw_data.reload()

    # save raw_data
    raw_data.save(username=user.username)

    # run raw_data triage
    if is_rawdata_new:
        raw_data.reload()
        run_triage(raw_data, user)

    status = {
        'success':      True,
        'message':      'Uploaded raw_data',
        '_id':          raw_data.id,
        'object':       raw_data
    }

    return status
Пример #2
0
def handle_raw_data_file(data, source_name, user=None,
                         description=None, title=None, data_type=None,
                         tool_name=None, tool_version=None, tool_details=None,
                         link_id=None, method=None, copy_rels=False,
                         bucket_list=None, ticket=None):
    """
    Add RawData.

    :param data: The data of the RawData.
    :type data: str
    :param source_name: The source which provided this RawData.
    :type source_name: str,
                       :class:`crits.core.crits_mongoengine.EmbeddedSource`,
                       list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
    :param user: The user adding the RawData.
    :type user: str
    :param description: Description of the RawData.
    :type description: str
    :param title: Title of the RawData.
    :type title: str
    :param data_type: Datatype of the RawData.
    :type data_type: str
    :param tool_name: Name of the tool used to acquire/generate the RawData.
    :type tool_name: str
    :param tool_version: Version of the tool.
    :type tool_version: str
    :param tool_details: Details about the tool.
    :type tool_details: str
    :param link_id: LinkId to tie this to another RawData as a new version.
    :type link_id: str
    :param method: The method of acquiring this RawData.
    :type method: str
    :param copy_rels: Copy relationships from the previous version to this one.
    :type copy_rels: bool
    :param bucket_list: Bucket(s) to add to this RawData
    :type bucket_list: str(comma separated) or list.
    :param ticket: Ticket(s) to add to this RawData
    :type ticket: str(comma separated) or list.
    :returns: dict with keys:
              'success' (boolean),
              'message' (str),
              'md5' (str) if successful.
    """

    if not data or not title or not data_type:
        status = {
            'success':   False,
            'message':  'No data object, title, or data type passed in'
        }
        return status

    rdt = RawDataType.objects(name=data_type).first()
    if not rdt:
        status = {
            'success':   False,
            'message':  'Invalid data type passed in'
        }
        return status

    data = data.encode('utf-8')

    if len(data) <= 0:
        status = {
            'success':   False,
            'message':  'Data length <= 0'
        }
        return status

    # generate md5 and timestamp
    md5 = hashlib.md5(data).hexdigest()
    timestamp = datetime.datetime.now()

    # create source
    source = create_embedded_source(source_name,
                                    date=timestamp,
                                    reference='',
                                    method=method,
                                    analyst=user)

    # generate raw_data
    is_rawdata_new = False
    raw_data = RawData.objects(md5=md5).first()
    if raw_data:
        raw_data.add_source(source)
    else:
        raw_data = RawData()
        raw_data.created = timestamp
        raw_data.description = description
        raw_data.md5 = md5
        raw_data.source = [source]
        raw_data.data = data
        raw_data.title = title
        raw_data.data_type = data_type
        raw_data.add_tool(name=tool_name,
                          version=tool_version,
                          details=tool_details)
        is_rawdata_new = True
    #XXX: need to validate this is a UUID
    if link_id:
        raw_data.link_id = link_id
        if copy_rels:
            rd2 = RawData.objects(link_id=link_id).first()
            if rd2:
                if len(rd2.relationships):
                    raw_data.save(username=user)
                    raw_data.reload()
                    for rel in rd2.relationships:
                        raw_data.add_relationship(rel_id=rel.object_id,
                                                  type_=rel.rel_type,
                                                  rel_type=rel.relationship,
                                                  rel_date=rel.relationship_date,
                                                  analyst=user)


    raw_data.version = len(RawData.objects(link_id=link_id)) + 1

    if bucket_list:
        raw_data.add_bucket_list(bucket_list, user)

    if ticket:
        raw_data.add_ticket(ticket, user);

    # save raw_data
    raw_data.save(username=user)

    # run raw_data triage
    if is_rawdata_new:
        raw_data.reload()
        run_triage(raw_data, user)

    status = {
        'success':      True,
        'message':      'Uploaded raw_data',
        '_id':          raw_data.id,
    }

    return status