def populate_raw_data_types(drop): """ Populate default set of raw data types into the system. :param drop: Drop the existing collection before trying to populate. :type: boolean """ # define your raw data types here data_types = ['Text', 'JSON'] if drop: RawDataType.drop_collection() if len(RawDataType.objects()) < 1: for data_type in data_types: dt = RawDataType() dt.name = data_type dt.save() print "Raw Data Types: added %s types!" % len(data_types) else: print "Raw Data Types: existing documents detected. skipping!"
def add_new_raw_data_type(data_type, analyst): """ Add a new RawData datatype to CRITs. :param data_type: The new datatype to add. :type data_type: str :param analyst: The user adding the new datatype. :type analyst: str :returns: bool """ data_type = data_type.strip() try: raw_data_type = RawDataType.objects(name=data_type).first() if raw_data_type: return False raw_data_type = RawDataType() raw_data_type.name = data_type raw_data_type.save(username=analyst) return True except ValidationError: return False
def update_raw_data_type(_id, data_type, analyst): """ Update the RawData data type. :param _id: ObjectId of the RawData to update. :type _id: str :param data_type: The data type to set. :type data_type: str :param analyst: The user updating the data type. :type analyst: str :returns: dict with keys "success" (boolean) and "message" (str) if failed. """ raw_data = RawData.objects(id=_id).first() data_type = RawDataType.objects(name=data_type).first() if not data_type: return None else: raw_data.data_type = data_type.name try: raw_data.save(username=analyst) return {'success': True} except ValidationError, e: return {'success': False, 'message': str(e)}
def handle_raw_data_file(data, source_name, user=None, description=None, title=None, data_type=None, tool_name=None, tool_version=None, tool_details=None, link_id=None, method='', reference='', tlp='', copy_rels=False, bucket_list=None, ticket=None, related_id=None, related_type=None, relationship_type=None): """ Add RawData. :param data: The data of the RawData. :type data: str :param source_name: The source which provided this RawData. :type source_name: str, :class:`crits.core.crits_mongoengine.EmbeddedSource`, list of :class:`crits.core.crits_mongoengine.EmbeddedSource` :param user: The user adding the RawData. :type user: str :param description: Description of the RawData. :type description: str :param title: Title of the RawData. :type title: str :param data_type: Datatype of the RawData. :type data_type: str :param tool_name: Name of the tool used to acquire/generate the RawData. :type tool_name: str :param tool_version: Version of the tool. :type tool_version: str :param tool_details: Details about the tool. :type tool_details: str :param link_id: LinkId to tie this to another RawData as a new version. :type link_id: str :param method: The method of acquiring this RawData. :type method: str :param reference: A reference to the source of this RawData. :type reference: str :param tlp: TLP for the source. :type tlp: str :param copy_rels: Copy relationships from the previous version to this one. :type copy_rels: bool :param bucket_list: Bucket(s) to add to this RawData :type bucket_list: str(comma separated) or list. :param ticket: Ticket(s) to add to this RawData :type ticket: str(comma separated) or list. :param related_id: ID of object to create relationship with :type related_id: str :param related_type: Type of object to create relationship with :type related_type: str :param relationship_type: Type of relationship to create. :type relationship_type: str :returns: dict with keys: 'success' (boolean), 'message' (str), '_id' (str) if successful. """ if not data or not title or not data_type: status = { 'success': False, 'message': 'No data object, title, or data type passed in' } return status if not source_name: return {"success" : False, "message" : "Missing source information."} rdt = RawDataType.objects(name=data_type).first() if not rdt: status = { 'success': False, 'message': 'Invalid data type passed in' } return status if len(data) <= 0: status = { 'success': False, 'message': 'Data length <= 0' } return status if isinstance(data, unicode): data=data.encode('utf-8') # generate md5 and timestamp md5 = hashlib.md5(data).hexdigest() timestamp = datetime.datetime.now() # generate raw_data is_rawdata_new = False raw_data = RawData.objects(md5=md5).first() if not raw_data: raw_data = RawData() raw_data.created = timestamp raw_data.description = description raw_data.md5 = md5 # raw_data.source = [source_name] raw_data.data = data raw_data.title = title raw_data.data_type = data_type raw_data.add_tool(name=tool_name, version=tool_version, details=tool_details) is_rawdata_new = True # generate new source information and add to sample if isinstance(source_name, basestring) and len(source_name) > 0: if user.check_source_write(source_name): source = create_embedded_source(source_name, method=method, reference=reference, tlp=tlp, analyst=user.username) raw_data.add_source(source) else: return {"success":False, "message": "User does not have permission to add object using source %s." % source_name} # this will handle adding a new source, or an instance automatically elif isinstance(source_name, EmbeddedSource): raw_data.add_source(source_name, method=method, reference=reference, tlp=tlp, analyst=user.usrname) elif isinstance(source_name, list) and len(source_name) > 0: for s in source_name: if isinstance(s, EmbeddedSource): raw_data.add_source(s, method=method, reference=reference, tlp=tlp, analyst=user.username) #XXX: need to validate this is a UUID if link_id: raw_data.link_id = link_id if copy_rels: rd2 = RawData.objects(link_id=link_id).first() if rd2: if len(rd2.relationships): raw_data.save(username=user) raw_data.reload() for rel in rd2.relationships: # Get object to relate to. rel_item = class_from_id(rel.rel_type, rel.object_id) if rel_item: raw_data.add_relationship(rel_item, rel.relationship, rel_date=rel.relationship_date, analyst=user.username) raw_data.version = len(RawData.objects(link_id=link_id)) + 1 if bucket_list: raw_data.add_bucket_list(bucket_list, user) if ticket: raw_data.add_ticket(ticket, user); related_obj = None if related_id and related_type: related_obj = class_from_id(related_type, related_id) if not related_obj: retVal['success'] = False retVal['message'] = 'Related Object not found.' return retVal raw_data.save(username=user.username) if related_obj and relationship_type and raw_data: relationship_type=RelationshipTypes.inverse(relationship=relationship_type) raw_data.add_relationship(related_obj, relationship_type, analyst=user.username, get_rels=False) raw_data.save(username=user.username) raw_data.reload() # save raw_data raw_data.save(username=user.username) # run raw_data triage if is_rawdata_new: raw_data.reload() run_triage(raw_data, user) status = { 'success': True, 'message': 'Uploaded raw_data', '_id': raw_data.id, 'object': raw_data } return status
def handle_raw_data_file(data, source_name, user=None, description=None, title=None, data_type=None, tool_name=None, tool_version=None, tool_details=None, link_id=None, method=None, copy_rels=False, bucket_list=None, ticket=None): """ Add RawData. :param data: The data of the RawData. :type data: str :param source_name: The source which provided this RawData. :type source_name: str, :class:`crits.core.crits_mongoengine.EmbeddedSource`, list of :class:`crits.core.crits_mongoengine.EmbeddedSource` :param user: The user adding the RawData. :type user: str :param description: Description of the RawData. :type description: str :param title: Title of the RawData. :type title: str :param data_type: Datatype of the RawData. :type data_type: str :param tool_name: Name of the tool used to acquire/generate the RawData. :type tool_name: str :param tool_version: Version of the tool. :type tool_version: str :param tool_details: Details about the tool. :type tool_details: str :param link_id: LinkId to tie this to another RawData as a new version. :type link_id: str :param method: The method of acquiring this RawData. :type method: str :param copy_rels: Copy relationships from the previous version to this one. :type copy_rels: bool :param bucket_list: Bucket(s) to add to this RawData :type bucket_list: str(comma separated) or list. :param ticket: Ticket(s) to add to this RawData :type ticket: str(comma separated) or list. :returns: dict with keys: 'success' (boolean), 'message' (str), 'md5' (str) if successful. """ if not data or not title or not data_type: status = { 'success': False, 'message': 'No data object, title, or data type passed in' } return status rdt = RawDataType.objects(name=data_type).first() if not rdt: status = { 'success': False, 'message': 'Invalid data type passed in' } return status data = data.encode('utf-8') if len(data) <= 0: status = { 'success': False, 'message': 'Data length <= 0' } return status # generate md5 and timestamp md5 = hashlib.md5(data).hexdigest() timestamp = datetime.datetime.now() # create source source = create_embedded_source(source_name, date=timestamp, reference='', method=method, analyst=user) # generate raw_data is_rawdata_new = False raw_data = RawData.objects(md5=md5).first() if raw_data: raw_data.add_source(source) else: raw_data = RawData() raw_data.created = timestamp raw_data.description = description raw_data.md5 = md5 raw_data.source = [source] raw_data.data = data raw_data.title = title raw_data.data_type = data_type raw_data.add_tool(name=tool_name, version=tool_version, details=tool_details) is_rawdata_new = True #XXX: need to validate this is a UUID if link_id: raw_data.link_id = link_id if copy_rels: rd2 = RawData.objects(link_id=link_id).first() if rd2: if len(rd2.relationships): raw_data.save(username=user) raw_data.reload() for rel in rd2.relationships: raw_data.add_relationship(rel_id=rel.object_id, type_=rel.rel_type, rel_type=rel.relationship, rel_date=rel.relationship_date, analyst=user) raw_data.version = len(RawData.objects(link_id=link_id)) + 1 if bucket_list: raw_data.add_bucket_list(bucket_list, user) if ticket: raw_data.add_ticket(ticket, user); # save raw_data raw_data.save(username=user) # run raw_data triage if is_rawdata_new: raw_data.reload() run_triage(raw_data, user) status = { 'success': True, 'message': 'Uploaded raw_data', '_id': raw_data.id, } return status
def class_from_id(type_, _id): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param _id: The ObjectId to search for. :type _id: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.actors.actor import ActorThreatIdentifier, Actor from crits.backdoors.backdoor import Backdoor from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.core.crits_mongoengine import Action from crits.core.source_access import SourceAccess from crits.core.user_role import UserRole from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event from crits.exploits.exploit import Exploit from crits.indicators.indicator import Indicator from crits.ips.ip import IP from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData, RawDataType from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.signatures.signature import Signature, SignatureType, SignatureDependency from crits.targets.target import Target if not _id: return None # make sure it's a string _id = str(_id) # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise # the queries below will raise a ValidationError exception. if not ObjectId.is_valid(_id.decode('utf8')): return None if type_ == 'Actor': return Actor.objects(id=_id).first() elif type_ == 'Backdoor': return Backdoor.objects(id=_id).first() elif type_ == 'ActorThreatIdentifier': return ActorThreatIdentifier.objects(id=_id).first() elif type_ == 'Campaign': return Campaign.objects(id=_id).first() elif type_ == 'Certificate': return Certificate.objects(id=_id).first() elif type_ == 'Comment': return Comment.objects(id=_id).first() elif type_ == 'Domain': return Domain.objects(id=_id).first() elif type_ == 'Email': return Email.objects(id=_id).first() elif type_ == 'Event': return Event.objects(id=_id).first() elif type_ == 'Exploit': return Exploit.objects(id=_id).first() elif type_ == 'Indicator': return Indicator.objects(id=_id).first() elif type_ == 'Action': return Action.objects(id=_id).first() elif type_ == 'IP': return IP.objects(id=_id).first() elif type_ == 'PCAP': return PCAP.objects(id=_id).first() elif type_ == 'RawData': return RawData.objects(id=_id).first() elif type_ == 'RawDataType': return RawDataType.objects(id=_id).first() elif type_ == 'Sample': return Sample.objects(id=_id).first() elif type_ == 'Signature': return Signature.objects(id=_id).first() elif type_ == 'SignatureType': return SignatureType.objects(id=_id).first() elif type_ == 'SignatureDependency': return SignatureDependency.objects(id=_id).first() elif type_ == 'SourceAccess': return SourceAccess.objects(id=_id).first() elif type_ == 'Screenshot': return Screenshot.objects(id=_id).first() elif type_ == 'Target': return Target.objects(id=_id).first() elif type_ == 'UserRole': return UserRole.objects(id=_id).first() else: return None
def class_from_id(type_, _id): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param _id: The ObjectId to search for. :type _id: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.actors.actor import ActorThreatIdentifier, Actor from crits.backdoors.backdoor import Backdoor from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.core.source_access import SourceAccess from crits.core.user_role import UserRole from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event from crits.exploits.exploit import Exploit from crits.indicators.indicator import Indicator, IndicatorAction from crits.ips.ip import IP from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData, RawDataType from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.targets.target import Target if not _id: return None # make sure it's a string _id = str(_id) # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise # the queries below will raise a ValidationError exception. if not ObjectId.is_valid(_id.decode('utf8')): return None if type_ == 'Actor': return Actor.objects(id=_id).first() elif type_ == 'Backdoor': return Backdoor.objects(id=_id).first() elif type_ == 'ActorThreatIdentifier': return ActorThreatIdentifier.objects(id=_id).first() elif type_ == 'Campaign': return Campaign.objects(id=_id).first() elif type_ == 'Certificate': return Certificate.objects(id=_id).first() elif type_ == 'Comment': return Comment.objects(id=_id).first() elif type_ == 'Domain': return Domain.objects(id=_id).first() elif type_ == 'Email': return Email.objects(id=_id).first() elif type_ == 'Event': return Event.objects(id=_id).first() elif type_ == 'Exploit': return Exploit.objects(id=_id).first() elif type_ == 'Indicator': return Indicator.objects(id=_id).first() elif type_ == 'IndicatorAction': return IndicatorAction.objects(id=_id).first() elif type_ == 'IP': return IP.objects(id=_id).first() elif type_ == 'PCAP': return PCAP.objects(id=_id).first() elif type_ == 'RawData': return RawData.objects(id=_id).first() elif type_ == 'RawDataType': return RawDataType.objects(id=_id).first() elif type_ == 'Sample': return Sample.objects(id=_id).first() elif type_ == 'SourceAccess': return SourceAccess.objects(id=_id).first() elif type_ == 'Screenshot': return Screenshot.objects(id=_id).first() elif type_ == 'Target': return Target.objects(id=_id).first() elif type_ == 'UserRole': return UserRole.objects(id=_id).first() else: return None
def class_from_id(type_, _id): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param _id: The ObjectId to search for. :type _id: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.core.crits_mongoengine import RelationshipType from crits.core.source_access import SourceAccess from crits.core.user_role import UserRole from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event, EventType from crits.indicators.indicator import Indicator, IndicatorAction from crits.ips.ip import IP from crits.objects.object_type import ObjectType from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData, RawDataType from crits.samples.backdoor import Backdoor from crits.samples.exploit import Exploit from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.targets.target import Target if not _id: return None # make sure it's a string _id = str(_id) if type_ == 'Backdoor': return Backdoor.objects(id=_id).first() if type_ == 'Campaign': return Campaign.objects(id=_id).first() elif type_ == 'Certificate': return Certificate.objects(id=_id).first() elif type_ == 'Comment': return Comment.objects(id=_id).first() elif type_ == 'Domain': return Domain.objects(id=_id).first() elif type_ == 'Email': return Email.objects(id=_id).first() elif type_ == 'Event': return Event.objects(id=_id).first() elif type_ == 'EventType': return EventType.objects(id=_id).first() elif type_ == 'Exploit': return Exploit.objects(id=_id).first() elif type_ == 'Indicator': return Indicator.objects(id=_id).first() elif type_ == 'IndicatorAction': return IndicatorAction.objects(id=_id).first() elif type_ == 'IP': return IP.objects(id=_id).first() elif type_ == 'ObjectType': return ObjectType.objects(id=_id).first() elif type_ == 'PCAP': return PCAP.objects(id=_id).first() elif type_ == 'RawData': return RawData.objects(id=_id).first() elif type_ == 'RawDataType': return RawDataType.objects(id=_id).first() elif type_ == 'RelationshipType': return RelationshipType.objects(id=_id).first() elif type_ == 'Sample': return Sample.objects(id=_id).first() elif type_ == 'SourceAccess': return SourceAccess.objects(id=_id).first() elif type_ == 'Screenshot': return Screenshot.objects(id=_id).first() elif type_ == 'Target': return Target.objects(id=_id).first() elif type_ == 'UserRole': return UserRole.objects(id=_id).first() else: return None
def class_from_id(type_, _id): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param _id: The ObjectId to search for. :type _id: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # Quick fail if not _id or not type_: return None # doing this to avoid circular imports from crits.actors.actor import ActorThreatIdentifier, Actor from crits.backdoors.backdoor import Backdoor from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.core.crits_mongoengine import Action from crits.core.source_access import SourceAccess from crits.core.user_role import UserRole from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event from crits.exploits.exploit import Exploit from crits.indicators.indicator import Indicator from crits.ips.ip import IP from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData, RawDataType from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.signatures.signature import Signature, SignatureType, SignatureDependency from crits.targets.target import Target # make sure it's a string _id = str(_id) # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise # the queries below will raise a ValidationError exception. if not ObjectId.is_valid(_id.decode("utf8")): return None if type_ == "Actor": return Actor.objects(id=_id).first() elif type_ == "Backdoor": return Backdoor.objects(id=_id).first() elif type_ == "ActorThreatIdentifier": return ActorThreatIdentifier.objects(id=_id).first() elif type_ == "Campaign": return Campaign.objects(id=_id).first() elif type_ == "Certificate": return Certificate.objects(id=_id).first() elif type_ == "Comment": return Comment.objects(id=_id).first() elif type_ == "Domain": return Domain.objects(id=_id).first() elif type_ == "Email": return Email.objects(id=_id).first() elif type_ == "Event": return Event.objects(id=_id).first() elif type_ == "Exploit": return Exploit.objects(id=_id).first() elif type_ == "Indicator": return Indicator.objects(id=_id).first() elif type_ == "Action": return Action.objects(id=_id).first() elif type_ == "IP": return IP.objects(id=_id).first() elif type_ == "PCAP": return PCAP.objects(id=_id).first() elif type_ == "RawData": return RawData.objects(id=_id).first() elif type_ == "RawDataType": return RawDataType.objects(id=_id).first() elif type_ == "Sample": return Sample.objects(id=_id).first() elif type_ == "Signature": return Signature.objects(id=_id).first() elif type_ == "SignatureType": return SignatureType.objects(id=_id).first() elif type_ == "SignatureDependency": return SignatureDependency.objects(id=_id).first() elif type_ == "SourceAccess": return SourceAccess.objects(id=_id).first() elif type_ == "Screenshot": return Screenshot.objects(id=_id).first() elif type_ == "Target": return Target.objects(id=_id).first() elif type_ == "UserRole": return UserRole.objects(id=_id).first() else: return None