def run(self, argv): parser = OptionParser() parser.add_option('-l', '--list', dest='list_services', action='store_true', default=False, help='List available services') parser.add_option('-t', '--triage', dest='triage', action='store_true', default=False, help='Run all triage services') parser.add_option('-e', '--enabled', dest='enabled', action='store_true', default=False, help='Run all enabled services') parser.add_option('-s', '--services', dest='services', help='Service list') parser.add_option('-v', '--verbose', dest='verbose', action='store_true', default=False, help='Verbose mode') parser.add_option('-f', '--filter', dest='sample_filter', help='Sample query filter') parser.add_option('-m', '--md5', dest='md5', help='md5 of sample') parser.add_option('-F', '--force', dest='force', action='store_true', default=False, help='Force run') (opts, args) = parser.parse_args(argv) service_list = [] sample_list = [] if opts.list_services: self.list_available_services() if (opts.triage or opts.enabled): service_list = self.get_service_list(opts.triage, opts.enabled) if opts.verbose: self.print_running_services(service_list) elif (opts.services): if len(opts.services) > 0: service_list = opts.services.split(',') if opts.verbose: self.print_running_services(service_list) if (opts.sample_filter): query = ast.literal_eval(opts.sample_filter) query_results = mongo_find(settings.COL_SAMPLES, query, {'md5': 1}) sample_list = [(sample["md5"], str(sample["_id"])) for sample in query_results] if opts.verbose: self.print_sample_stats(sample_list, opts.sample_filter) if (opts.md5): # Given an MD5 we have to get the sample ID. # # XXX: This should be extended so we can pass an MD5 of a PCAP. # The entire script also needs to have an option for ID, so we # can work with other object types that support services. obj = class_from_value('Sample', opts.md5) if not obj: print "[-] Unable to find object." return sample_list = [(opts.md5, obj.id)] if opts.verbose: self.print_sample_stats(sample_list) if sample_list and service_list: self.run_services(service_list, sample_list, opts.verbose, opts.force)
def run(self, argv): parser = OptionParser() parser.add_option('-d', '--domain', dest='domain', help='domain') parser.add_option('-l', '--domain_list', dest='domain_list', help='domain list') parser.add_option('-i', '--id_list', dest='id_list', help='id list') parser.add_option('-v', '--verbose', dest='verbose', action='store_true', default=False, help='Verbose mode') parser.add_option('-T', '--type', dest='type_', default='Domain', help='CRITs type query for (default: Domain)') parser.add_option('--delete', dest='delete', action='store_true', default=False, help='Delete Domains') parser.add_option('--csv', dest='csv', action='store_true', default=False, help='Treat file as CSV') (opts, args) = parser.parse_args(argv) domain_list = [] if opts.domain_list: with open(opts.domain_list) as infile: for line in infile: if opts.csv: domain_list = [x.strip() for x in line.split(',')] else: domain_list.append(line.strip()) elif opts.domain: domain_list = opts.domain.split(',') if opts.verbose: self.print_delete_objects(domain_list) error_list = [] obj_list = [] for domain in domain_list: #domain_obj = Domain.objects(domain_iexact=domain).first() obj = class_from_value(opts.type_, domain) if not obj: error_list.append(domain) else: obj_list.append(obj) if opts.verbose: self.print_found_objects(obj_list, error_list) if opts.id_list: with open(opts.id_list) as infile: for line in infile: result = json.loads(line.strip()) obj = class_from_id(opts.type_, result['object_id']) obj_list.append(obj) if opts.delete: self.delete_domains(obj_list, opts.type_, 0.5) else: self.run_analysis_cleanup(obj_list, opts.type_, 0.5) print("Done!")
def relate_objects(self): """ for now we are relating all objects to the event with a common relationship type that is most likely inaccurate. Need to get actual relationship out of the cybox document once we are storing it there. """ for (type_, event_id) in self.events: event_obj = class_from_value(type_, event_id) finished_objects = [event_obj] for (t, v) in self.samples + self.emails + self.indicators: obj = class_from_value(t, v) obj.add_relationship(event_obj, rel_type="Related_To", analyst=self.source_instance.analyst) finished_objects.append(obj) for f in finished_objects: f.save(username=self.source_instance.analyst) # If we have no event relate every object to every other object. if not self.events: finished_objects = [] for (t, v) in self.samples + self.emails + self.indicators: obj = class_from_value(t, v) # Prime the list... if not finished_objects: finished_objects.append(obj) continue for right in finished_objects: obj.add_relationship(right, rel_type="Related_To", analyst=self.source_instance.analyst) finished_objects.append(obj) for f in finished_objects: f.save(username=self.source_instance.analyst)
def run(self, argv): parser = OptionParser() parser.add_option('-s', '--services', dest='services', help='Service list') parser.add_option('-v', '--verbose', dest='verbose', action='store_true', default=False, help='Verbose mode') parser.add_option('-f', '--filter', dest='query_filter', help='Query filter') parser.add_option('-T', '--type', dest='type_', default='Sample', help='CRITs type query for (default: Sample)') parser.add_option('-i', '--identifier', dest='identifier', help='Identifier for type (NOT OBJECT ID)') parser.add_option('-c', '--config', dest='config', default={}, help='Service configuration') (opts, args) = parser.parse_args(argv) service_list = [] if opts.services: service_list = opts.services.split(',') if opts.verbose: self.print_running_services(service_list) if opts.query_filter: query = ast.literal_eval(opts.query_filter) klass = class_from_type(opts.type_) if not klass: print "[-] Invalid type." obj_list = klass.objects(__raw__=query) if opts.verbose: self.print_object_stats(obj_list, opts.query_filter) if opts.identifier: obj = class_from_value(opts.type_, opts.identifier) if not obj: print "[-] Unable to find object." return obj_list = [obj] if opts.verbose: self.print_object_stats(obj_list) config = {} if opts.config: config = ast.literal_eval(opts.config) if obj_list and service_list: self.run_services(service_list, obj_list, opts.verbose, config)
def target_details(request, email_address=None): """ Target modification form generation. :param request: Django request. :type request: :class:`django.http.HttpRequest` :param email_address: The email address of the Target to get details for.. :type email_address: str :returns: :class:`django.http.HttpResponse` """ if email_address is None: form = TargetInfoForm() else: target = class_from_value("Target", email_address) if not target: form = TargetInfoForm(initial={"email_address": email_address}) else: form = TargetInfoForm(initial=target.to_dict()) return render_to_response("target_form.html", {"form": form}, RequestContext(request))
def target_details(request, email_address=None): """ Target modification form generation. :param request: Django request. :type request: :class:`django.http.HttpRequest` :param email_address: The email address of the Target to get details for.. :type email_address: str :returns: :class:`django.http.HttpResponse` """ if email_address is None: form = TargetInfoForm() else: target = class_from_value('Target', email_address) if not target: form = TargetInfoForm(initial={'email_address': email_address}) else: form = TargetInfoForm(initial=target.to_dict()) return render_to_response('target_form.html', {'form': form}, RequestContext(request))
def handle_cert_file(filename, data, source_name, user=None, description=None, related_md5=None, method='', reference='', tlp=None, relationship=None, bucket_list=None, ticket=None, related_id=None, related_type=None, relationship_type=None): """ Add a Certificate. :param filename: The filename of the Certificate. :type filename: str :param data: The filedata of the Certificate. :type data: str :param source_name: The source which provided this Certificate. :type source_name: str, :class:`crits.core.crits_mongoengine.EmbeddedSource`, list of :class:`crits.core.crits_mongoengine.EmbeddedSource` :param user: The user adding the Certificate. :type user: str :param description: Description of the Certificate. :type description: str :param related_md5: MD5 of a top-level object related to this Certificate. :type related_md5: str :param related_type: The CRITs type of the related top-level object. :type related_type: str :param method: The method of acquiring this Certificate. :type method: str :param reference: A reference to the source of this Certificate. :type reference: str :param tlp: The TLP for this certificate. :type tlp: str :param relationship: The relationship between the parent and the Certificate. :type relationship: str :param bucket_list: Bucket(s) to add to this Certificate :type bucket_list: str(comma separated) or list. :param ticket: Ticket(s) to add to this Certificate :type ticket: str(comma separated) or list. :param related_id: ID of object to create relationship with :type related_id: str :param related_type: Type of object to create relationship with :type related_id: str :param relationship_type: Type of relationship to create. :type relationship_type: str :returns: dict with keys: 'success' (boolean), 'message' (str), 'md5' (str) if successful. """ if not data: status = {'success': False, 'message': 'No data object passed in'} return status if len(data) <= 0: status = {'success': False, 'message': 'Data length <= 0'} return status if ((related_type and not (related_id or related_md5)) or (not related_type and (related_id or related_md5))): status = { 'success': False, 'message': 'Must specify both related_type and related_id or related_md5.' } return status related_obj = None if related_id or related_md5: if related_id: related_obj = class_from_id(related_type, related_id) else: related_obj = class_from_value(related_type, related_md5) if not related_obj: status = {'success': False, 'message': 'Related object not found.'} return status # generate md5 and timestamp md5 = hashlib.md5(data).hexdigest() timestamp = datetime.datetime.now() # generate Certificate cert = Certificate.objects(md5=md5).first() if not cert: cert = Certificate() cert.filename = filename cert.created = timestamp cert.size = len(data) cert.description = description cert.md5 = md5 # generate source information and add to certificate if isinstance(source_name, basestring) and len(source_name) > 0: if user.check_source_write(source_name): s = create_embedded_source(source_name, reference=reference, method=method, tlp=tlp, analyst=user.username) else: return { "success": False, "message": "User does not have permission to add objects \ using source %s." % str(source_name) } cert.add_source(s) elif isinstance(source_name, EmbeddedSource): cert.add_source(source_name, method=method, reference=reference, tlp=tlp) elif isinstance(source_name, list) and len(source_name) > 0: for s in source_name: if isinstance(s, EmbeddedSource): cert.add_source(s, method=method, reference=reference, tlp=tlp) if bucket_list: cert.add_bucket_list(bucket_list, user) if ticket: cert.add_ticket(ticket, user) # add file to GridFS if not isinstance(cert.filedata.grid_id, ObjectId): cert.add_file_data(data) # save cert cert.save(username=user) cert.reload() # run certificate triage if AnalysisResult.objects(object_id=str(cert.id)).count() < 1 and data: run_triage(cert, user) # update relationship if a related top-level object is supplied if related_obj and cert: if relationship_type: relationship = RelationshipTypes.inverse( relationship=relationship_type) if not relationship: relationship = RelationshipTypes.RELATED_TO cert.add_relationship(related_obj, relationship, analyst=user, get_rels=False) cert.save(username=user) status = { 'success': True, 'message': 'Uploaded certificate', 'md5': md5, 'id': str(cert.id), 'object': cert } return status
def handle_pcap_file( filename, data, source_name, user=None, description=None, related_id=None, related_md5=None, related_type=None, method=None, relationship=None, bucket_list=None, ticket=None, ): """ Add a PCAP. :param filename: The filename of the PCAP. :type filename: str :param data: The filedata of the PCAP. :type data: str :param source_name: The source which provided this PCAP. :type source_name: str, :class:`crits.core.crits_mongoengine.EmbeddedSource`, list of :class:`crits.core.crits_mongoengine.EmbeddedSource` :param user: The user adding the PCAP. :type user: str :param description: Description of the PCAP. :type description: str :param related_id: ObjectId of a top-level object related to this PCAP. :type related_id: str :param related_md5: MD5 of a top-level object related to this PCAP. :type related_md5: str :param related_type: The CRITs type of the related top-level object. :type related_type: str :param method: The method of acquiring this PCAP. :type method: str :param relationship: The relationship between the parent and the PCAP. :type relationship: str :param bucket_list: Bucket(s) to add to this PCAP. :type bucket_list: str(comma separated) or list. :param ticket: Ticket(s) to add to this PCAP. :type ticket: str(comma separated) or list. :returns: dict with keys: 'success' (boolean), 'message' (str), 'md5' (str) if successful. """ if not data: status = {"success": False, "message": "No data object passed in"} return status if len(data) <= 0: status = {"success": False, "message": "Data length <= 0"} return status if (related_type and not (related_id or related_md5)) or (not related_type and (related_id or related_md5)): status = {"success": False, "message": "Must specify both related_type and related_id or related_md5."} return status related_obj = None if related_id or related_md5: if related_id: related_obj = class_from_id(related_type, related_id) else: related_obj = class_from_value(related_type, related_md5) if not related_obj: status = {"success": False, "message": "Related object not found."} return status # generate md5 and timestamp md5 = hashlib.md5(data).hexdigest() timestamp = datetime.datetime.now() # generate PCAP is_pcap_new = False pcap = PCAP.objects(md5=md5).first() if not pcap: pcap = PCAP() pcap.filename = filename pcap.created = timestamp pcap.length = len(data) pcap.description = description pcap.md5 = md5 is_pcap_new = True # generate source information and add to pcap if isinstance(source_name, basestring) and len(source_name) > 0: s = create_embedded_source(source_name, method=method, reference="", analyst=user) pcap.add_source(s) elif isinstance(source_name, EmbeddedSource): pcap.add_source(source_name) elif isinstance(source_name, list) and len(source_name) > 0: for s in source_name: if isinstance(s, EmbeddedSource): pcap.add_source(s) # add file to GridFS if not isinstance(pcap.filedata.grid_id, ObjectId): pcap.add_file_data(data) if bucket_list: pcap.add_bucket_list(bucket_list, user) if ticket: pcap.add_ticket(ticket, user) # save pcap pcap.save(username=user) # update relationship if a related top-level object is supplied if related_obj and pcap: if not relationship: relationship = "Related_To" pcap.add_relationship(rel_item=related_obj, rel_type=relationship, analyst=user, get_rels=False) related_obj.save(username=user) pcap.save(username=user) # run pcap triage if is_pcap_new and data: pcap.reload() run_triage(data, pcap, user) status = {"success": True, "message": "Uploaded pcap", "md5": md5, "id": str(pcap.id)} return status
def handle_pcap_file(filename, data, source_name, user=None, description=None, related_id=None, related_md5=None, related_type=None, method='', reference='', relationship=None, bucket_list=None, ticket=None): """ Add a PCAP. :param filename: The filename of the PCAP. :type filename: str :param data: The filedata of the PCAP. :type data: str :param source_name: The source which provided this PCAP. :type source_name: str, :class:`crits.core.crits_mongoengine.EmbeddedSource`, list of :class:`crits.core.crits_mongoengine.EmbeddedSource` :param user: The user adding the PCAP. :type user: str :param description: Description of the PCAP. :type description: str :param related_id: ObjectId of a top-level object related to this PCAP. :type related_id: str :param related_md5: MD5 of a top-level object related to this PCAP. :type related_md5: str :param related_type: The CRITs type of the related top-level object. :type related_type: str :param method: The method of acquiring this PCAP. :type method: str :param reference: A reference to the source of this PCAP. :type reference: str :param relationship: The relationship between the parent and the PCAP. :type relationship: str :param bucket_list: Bucket(s) to add to this PCAP. :type bucket_list: str(comma separated) or list. :param ticket: Ticket(s) to add to this PCAP. :type ticket: str(comma separated) or list. :returns: dict with keys: 'success' (boolean), 'message' (str), 'md5' (str) if successful. """ if not data: status = {'success': False, 'message': 'No data object passed in'} return status if len(data) <= 0: status = {'success': False, 'message': 'Data length <= 0'} return status if ((related_type and not (related_id or related_md5)) or (not related_type and (related_id or related_md5))): status = { 'success': False, 'message': 'Must specify both related_type and related_id or related_md5.' } return status if not source_name: return {"success": False, "message": "Missing source information."} related_obj = None if related_id or related_md5: if related_id: related_obj = class_from_id(related_type, related_id) else: related_obj = class_from_value(related_type, related_md5) if not related_obj: status = {'success': False, 'message': 'Related object not found.'} return status # generate md5 and timestamp md5 = hashlib.md5(data).hexdigest() timestamp = datetime.datetime.now() # generate PCAP is_pcap_new = False pcap = PCAP.objects(md5=md5).first() if not pcap: pcap = PCAP() pcap.filename = filename pcap.created = timestamp pcap.length = len(data) pcap.description = description pcap.md5 = md5 is_pcap_new = True # generate source information and add to pcap if isinstance(source_name, basestring) and len(source_name) > 0: s = create_embedded_source(source_name, method=method, reference=reference, analyst=user) pcap.add_source(s) elif isinstance(source_name, EmbeddedSource): pcap.add_source(source_name, method=method, reference=reference) elif isinstance(source_name, list) and len(source_name) > 0: for s in source_name: if isinstance(s, EmbeddedSource): pcap.add_source(s, method=method, reference=reference) # add file to GridFS if not isinstance(pcap.filedata.grid_id, ObjectId): pcap.add_file_data(data) if bucket_list: pcap.add_bucket_list(bucket_list, user) if ticket: pcap.add_ticket(ticket, user) # save pcap pcap.save(username=user) # update relationship if a related top-level object is supplied if related_obj and pcap: if not relationship: relationship = "Related_To" pcap.add_relationship(rel_item=related_obj, rel_type=relationship, analyst=user, get_rels=False) related_obj.save(username=user) pcap.save(username=user) # run pcap triage if is_pcap_new and data: pcap.reload() run_triage(pcap, user) status = { 'success': True, 'message': 'Uploaded pcap', 'md5': md5, 'id': str(pcap.id), 'object': pcap } return status
def handle_cert_file(filename, data, source_name, user=None, description=None, related_id=None, related_md5=None, related_type=None, method=None, relationship=None, bucket_list=None, ticket=None): """ Add a Certificate. :param filename: The filename of the Certificate. :type filename: str :param data: The filedata of the Certificate. :type data: str :param source_name: The source which provided this Certificate. :type source_name: str, :class:`crits.core.crits_mongoengine.EmbeddedSource`, list of :class:`crits.core.crits_mongoengine.EmbeddedSource` :param user: The user adding the Certificate. :type user: str :param description: Description of the Certificate. :type description: str :param related_id: ObjectId of a top-level object related to this Certificate. :type related_id: str :param related_md5: MD5 of a top-level object related to this Certificate. :type related_md5: str :param related_type: The CRITs type of the related top-level object. :type related_type: str :param method: The method of acquiring this Certificate. :type method: str :param relationship: The relationship between the parent and the Certificate. :type relationship: str :param bucket_list: Bucket(s) to add to this Certificate :type bucket_list: str(comma separated) or list. :param ticket: Ticket(s) to add to this Certificate :type ticket: str(comma separated) or list. :returns: dict with keys: 'success' (boolean), 'message' (str), 'md5' (str) if successful. """ if not data: status = { 'success': False, 'message': 'No data object passed in' } return status if len(data) <= 0: status = { 'success': False, 'message': 'Data length <= 0' } return status if ((related_type and not (related_id or related_md5)) or (not related_type and (related_id or related_md5))): status = { 'success': False, 'message': 'Must specify both related_type and related_id or related_md5.' } return status related_obj = None if related_id or related_md5: if related_id: related_obj = class_from_id(related_type, related_id) else: related_obj = class_from_value(related_type, related_md5) if not related_obj: status = { 'success': False, 'message': 'Related object not found.' } return status # generate md5 and timestamp md5 = hashlib.md5(data).hexdigest() timestamp = datetime.datetime.now() # generate Certificate cert = Certificate.objects(md5=md5).first() if not cert: cert = Certificate() cert.filename = filename cert.created = timestamp cert.size = len(data) cert.description = description cert.md5 = md5 # generate source information and add to certificate if isinstance(source_name, basestring) and len(source_name) > 0: s = create_embedded_source(source_name, method=method, reference='', analyst=user) cert.add_source(s) elif isinstance(source_name, EmbeddedSource): cert.add_source(source_name) elif isinstance(source_name, list) and len(source_name) > 0: for s in source_name: if isinstance(s, EmbeddedSource): cert.add_source(s) if bucket_list: cert.add_bucket_list(bucket_list, user) if ticket: cert.add_ticket(ticket, user) # add file to GridFS if not isinstance(cert.filedata.grid_id, ObjectId): cert.add_file_data(data) # save cert cert.save(username=user) cert.reload() # run certificate triage if len(cert.analysis) < 1 and data: run_triage(cert, user) # update relationship if a related top-level object is supplied if related_obj and cert: if not relationship: relationship = "Related_To" cert.add_relationship(rel_item=related_obj, rel_type=relationship, analyst=user, get_rels=False) related_obj.save(username=user) cert.save(username=user) status = { 'success': True, 'message': 'Uploaded certificate', 'md5': md5, 'id': str(cert.id) } return status
def format_object(obj_type, obj_id, data_format="yaml", cleanse=True, obj_sources=[], remove_source=False, remove_rels=False, remove_schema_version=False, remove_campaign=False, remove_buckets=False, remove_releasability=False, remove_unsupported=False): """ Formats a top-level object for utilization in certain conditions. Removes CRITs-internal necessary data so users editing the document via the interface don't alter or have the ability to overwrite things they should not. :param obj_type: The CRITs type of the top-level object to format. :type obj_type: str :param obj_id: The ObjectId to search for. :type obj_id: str :param data_format: The format of the returned data. :type data_format: str of "yaml" or "json" :param cleanse: Remove "to", "actions", "releasability", and "bucket_list" if this is an Email or Indicator. :type cleanse: boolean :param obj_sources: The sources to overwrite into the document or to set the source list to an empty list if remove_source is False. :type obj_sources: list :param remove_source: Remove the source key from the document. :type remove_source: boolean :param remove_rels: Remove the relationships key from the document. :type remove_rels: boolean :param remove_schema_version: Remove the schema_version key from the document. :type remove_schema_version: boolean :param remove_campaign: Remove the campaign key from the document. :type remove_campaign: boolean :param remove_buckets: Remove the bucket_list key from the document. :type remove_buckets: boolean :param remove_releasability: Remove the releasability key from the document. :type remove_releasability: boolean :param remove_unsupported: Remove the unsupported_attrs key from the document. :type remove_unsupported: boolean :returns: str """ collection = settings.CRITS_TYPES[obj_type] obj_class = class_from_value(obj_type, obj_id) if not obj_class: return "" data = obj_class.to_dict() if data is None: return "" # Emails use raw_header (singular) as the attribute but store it as # raw_headers (plural) in the database. When viewing an email in YAML # or JSON convert from plural to singular. This will allow a copy/paste # of these views to be imported correctly. if 'raw_headers' in data: data['raw_header'] = data['raw_headers'] del data['raw_headers'] if cleanse and collection in [settings.COL_EMAIL, settings.COL_INDICATORS]: if "to" in data: del data["to"] if "actions" in data: del data["actions"] if "releasability" in data: del data["releasability"] if "bucket_list" in data: del data["bucket_list"] if remove_source and 'source' in data: del data["source"] elif 'source' in data: data['source'] = obj_sources if remove_rels and 'relationships' in data: del data["relationships"] if remove_rels and 'objects' in data: del data["objects"] if remove_schema_version and 'schema_version' in data: del data["schema_version"] if remove_campaign and 'campaign' in data: del data["campaign"] del data["_id"] if data.has_key("modified"): del data["modified"] if remove_buckets and 'bucket_list' in data: del data['bucket_list'] if remove_releasability and 'releasability' in data: del data['releasability'] if remove_unsupported and 'unsupported_attrs' in data: del data['unsupported_attrs'] data = json.dumps(convert_datetimes_to_string(data), default=json_util.default) if data_format == "yaml": data = yaml.dump(yaml.load(data), default_flow_style=False) elif data_format == "json": data = json.dumps(json.loads(data)) return data
def handle_pcap_file(filename, data, source_name, user=None, description=None, related_id=None, related_md5=None, related_type=None, method='', reference='', tlp='', relationship=None, bucket_list=None, ticket=None): """ Add a PCAP. :param filename: The filename of the PCAP. :type filename: str :param data: The filedata of the PCAP. :type data: str :param source_name: The source which provided this PCAP. :type source_name: str, :class:`crits.core.crits_mongoengine.EmbeddedSource`, list of :class:`crits.core.crits_mongoengine.EmbeddedSource` :param user: The user adding the PCAP. :type user: str :param description: Description of the PCAP. :type description: str :param related_id: ObjectId of a top-level object related to this PCAP. :type related_id: str :param related_md5: MD5 of a top-level object related to this PCAP. :type related_md5: str :param related_type: The CRITs type of the related top-level object. :type related_type: str :param method: The method of acquiring this PCAP. :type method: str :param reference: A reference to the source of this PCAP. :type reference: str :param relationship: The relationship between the parent and the PCAP. :type relationship: str :param bucket_list: Bucket(s) to add to this PCAP. :type bucket_list: str(comma separated) or list. :param ticket: Ticket(s) to add to this PCAP. :type ticket: str(comma separated) or list. :param related_id: ID of object to create relationship with :type related_id: str :param related_type: Type of object to create relationship with :type related_type: str :param relationship_type: Type of relationship to create. :type relationship_type: str :returns: dict with keys: 'success' (boolean), 'message' (str), 'md5' (str) if successful. """ if not data: status = { 'success': False, 'message': 'No data object passed in' } return status if len(data) <= 0: status = { 'success': False, 'message': 'Data length <= 0' } return status if ((related_id or related_md5) and not related_type): status = { 'success': False, 'message': 'Must specify both related_type and related_id or related_md5.' } return status if not source_name: return {"success" : False, "message" : "Missing source information."} related_obj = None if related_id or related_md5: if related_id: related_obj = class_from_id(related_type, related_id) else: related_obj = class_from_value(related_type, related_md5) if not related_obj: status = { 'success': False, 'message': 'Related object not found.' } return status # generate md5 and timestamp md5 = hashlib.md5(data).hexdigest() timestamp = datetime.datetime.now() # generate PCAP is_pcap_new = False pcap = PCAP.objects(md5=md5).first() if not pcap: pcap = PCAP() pcap.filename = filename pcap.created = timestamp pcap.length = len(data) pcap.description = description pcap.md5 = md5 is_pcap_new = True # generate source information and add to pcap if isinstance(source_name, basestring) and len(source_name) > 0: if user.check_source_write(source_name): s = create_embedded_source(source_name, method=method, reference=reference, tlp=tlp, analyst=user.username) else: return {"success":False, "message": "User does not have permission to add object \ using source %s." % source_name} pcap.add_source(s) elif isinstance(source_name, EmbeddedSource): pcap.add_source(source_name, method=method, reference=reference) elif isinstance(source_name, list) and len(source_name) > 0: for s in source_name: if isinstance(s, EmbeddedSource): pcap.add_source(s, method=method, reference=reference) # add file to GridFS if not isinstance(pcap.filedata.grid_id, ObjectId): pcap.add_file_data(data) if bucket_list: pcap.add_bucket_list(bucket_list, user) if ticket: pcap.add_ticket(ticket, user) # save pcap pcap.save(username=user) # update relationship if a related top-level object is supplied if related_obj and pcap: if relationship: relationship=RelationshipTypes.inverse(relationship=relationship) else: relationship = RelationshipTypes.RELATED_TO pcap.add_relationship(related_obj, relationship, analyst=user, get_rels=False) pcap.save(username=user) # run pcap triage if is_pcap_new and data: pcap.reload() run_triage(pcap, user) status = { 'success': True, 'message': 'Uploaded pcap', 'md5': md5, 'id': str(pcap.id), 'object': pcap } return status
def handle_cert_file(filename, data, source_name, user=None, description=None, related_id=None, related_md5=None, related_type=None, method=None, relationship=None, bucket_list=None, ticket=None): """ Add a Certificate. :param filename: The filename of the Certificate. :type filename: str :param data: The filedata of the Certificate. :type data: str :param source_name: The source which provided this Certificate. :type source_name: str, :class:`crits.core.crits_mongoengine.EmbeddedSource`, list of :class:`crits.core.crits_mongoengine.EmbeddedSource` :param user: The user adding the Certificate. :type user: str :param description: Description of the Certificate. :type description: str :param related_id: ObjectId of a top-level object related to this Certificate. :type related_id: str :param related_md5: MD5 of a top-level object related to this Certificate. :type related_md5: str :param related_type: The CRITs type of the related top-level object. :type related_type: str :param method: The method of acquiring this Certificate. :type method: str :param relationship: The relationship between the parent and the Certificate. :type relationship: str :param bucket_list: Bucket(s) to add to this Certificate :type bucket_list: str(comma separated) or list. :param ticket: Ticket(s) to add to this Certificate :type ticket: str(comma separated) or list. :returns: dict with keys: 'success' (boolean), 'message' (str), 'md5' (str) if successful. """ if not data: status = {'success': False, 'message': 'No data object passed in'} return status if len(data) <= 0: status = {'success': False, 'message': 'Data length <= 0'} return status # generate md5 and timestamp md5 = hashlib.md5(data).hexdigest() timestamp = datetime.datetime.now() # generate Certificate cert = Certificate.objects(md5=md5).first() if not cert: cert = Certificate() cert.filename = filename cert.created = timestamp cert.size = len(data) cert.description = description cert.md5 = md5 # generate source information and add to certificate if isinstance(source_name, basestring) and len(source_name) > 0: s = create_embedded_source(source_name, method=method, reference='', analyst=user) cert.add_source(s) elif isinstance(source_name, EmbeddedSource): cert.add_source(source_name) elif isinstance(source_name, list) and len(source_name) > 0: for s in source_name: if isinstance(s, EmbeddedSource): cert.add_source(s) if bucket_list: cert.add_bucket_list(bucket_list, user) if ticket: cert.add_ticket(ticket, user) # add file to GridFS if not isinstance(cert.filedata.grid_id, ObjectId): cert.add_file_data(data) # save cert cert.save(username=user) cert.reload() # run certificate triage if len(cert.analysis) < 1 and data: run_triage(data, cert, user) # update relationship if a related top-level object is supplied if related_id or related_md5: if related_id: related_obj = class_from_id(related_type, related_id) else: related_obj = class_from_value(related_type, related_md5) if related_obj and cert: if not relationship: relationship = "Related_To" cert.add_relationship(rel_item=related_obj, rel_type=relationship, analyst=user, get_rels=False) related_obj.save(username=user) cert.save(username=user) status = { 'success': True, 'message': 'Uploaded certificate', 'md5': md5, } return status
def forge_relationship(type_=None, id_=None, class_=None, right_type=None, right_id=None, right_class=None, rel_type=None, rel_date=None, user=None, rel_reason="N/A", rel_confidence='unknown', get_rels=False, **kwargs): """ Forge a relationship between two top-level objects. :param type_: The type of first top-level object to relate to. :type type_: str :param id_: The ObjectId of the first top-level object. :type id_: str :param class_: The first top-level object to relate to. :type class_: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param right_type: The type of second top-level object to relate to. :type right_type: str :param right_id: The ObjectId of the second top-level object. :type right_id: str :param right_class: The second top-level object to relate to. :type right_class: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param rel_type: The type of relationship. :type rel_type: str :param rel_date: The date this relationship applies. :type rel_date: datetime.datetime :param user: The user forging this relationship. :type user: str :param rel_reason: The reason for the relationship. :type rel_reason: str :param rel_confidence: The confidence of the relationship. :type rel_confidence: str :param get_rels: Return the relationships after forging. :type get_rels: boolean :returns: dict with keys "success" (boolean) and "message" (str if failed, dict if successful) """ if rel_date is None or rel_date == 'None': rel_date = None elif isinstance(rel_date, basestring) and rel_date != '': rel_date = parse(rel_date, fuzzy=True) elif not isinstance(rel_date, datetime.datetime): rel_date = None if not class_: if type_ and id_: class_ = class_from_value(type_, id_) if not class_: class_ = class_from_id(type_, id_) if not class_: return {'success': False, 'message': "Unable to get left object."} else: return {'success': False, 'message': "Need a valid left type and id"} try: # forge relationship if right_class: results = class_.add_relationship(right_class, rel_type, rel_date=rel_date, analyst=user, rel_confidence=rel_confidence, rel_reason=rel_reason) else: if right_type and right_id: if user in 'taxii': results = class_.set_relationship(right_id, rel_type, right_type, rel_date=rel_date, analyst=user, rel_confidence=rel_confidence, rel_reason=rel_reason) else: #Always check for class from value first rel_item = class_from_value(right_type, right_id) if not rel_item: rel_item = class_from_id(right_type, right_id) if rel_item: results = class_.add_relationship(rel_item, rel_type, rel_date=rel_date, analyst=user, rel_confidence=rel_confidence, rel_reason=rel_reason) else: return {'success': False, 'message': "Failed to get right object"} else: return {'success': False, 'message': "Need a valid right type and id"} except Exception, e: return {'success': False, 'message': e}
def update_relationship_confidences(left_class=None, right_class=None, left_type=None, left_id=None, right_type=None, right_id=None, rel_type=None, rel_date=None, new_type=None,analyst=None, new_confidence='unknown'): """ Update the relationship type between two top-level objects. :param left_class: The first top-level object. :type left_class: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param right_class: The second top-level object. :type right_class: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param left_type: The type of first top-level object. :type left_type: str :param left_id: The ObjectId of the first top-level object. :type left_id: str :param right_type: The type of second top-level object. :type right_type: str :param right_id: The ObjectId of the second top-level object. :type right_id: str :param rel_type: The type of relationship. :type rel_type: str :param rel_date: The date this relationship applies. :type rel_date: datetime.datetime :param analyst: The user updating this relationship. :type analyst: str :param new_confidence: The new confidence level. :type new_confidence: str :returns: dict with keys "success" (boolean) and "message" (str) """ if rel_date is None or rel_date == 'None': rel_date = None elif isinstance(rel_date, basestring) and rel_date != '': rel_date = parse(rel_date, fuzzy=True) elif not isinstance(rel_date, datetime.datetime): rel_date = None if not left_class: if left_type and left_id: left_class = class_from_value(left_type, left_id) else: return {'success': False, 'message': "Need a valid left type and id"} # update relationship if right_class: results = left_class.edit_relationship_confidence(rel_item=right_class, rel_type=rel_type, rel_date=rel_date, new_confidence=new_confidence, analyst=analyst) left_class.save(username=analyst) right_class.save(username=analyst) else: if right_type and right_id: results = left_class.edit_relationship_confidence(type_=right_type, rel_id=right_id, rel_type=rel_type, rel_date=rel_date, new_confidence=new_confidence, analyst=analyst) left_class.save(username=analyst) else: return {'success': False, 'message': "Need a valid right type and id"} return results
def delete_relationship(left_class=None, right_class=None, left_type=None, left_id=None, right_type=None, right_id=None, rel_type=None, rel_date=None, analyst=None, get_rels=True): """ Delete a relationship between two top-level objects. :param left_class: The first top-level object. :type left_class: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param right_class: The second top-level object. :type right_class: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param left_type: The type of first top-level object. :type left_type: str :param left_id: The ObjectId of the first top-level object. :type left_id: str :param right_type: The type of second top-level object. :type right_type: str :param right_id: The ObjectId of the second top-level object. :type right_id: str :param rel_type: The type of relationship. :type rel_type: str :param rel_date: The date this relationship applies. :type rel_date: datetime.datetime :param analyst: The user deleting this relationship. :type analyst: str :param get_rels: Return the relationships after forging. :type get_rels: boolean :returns: dict with keys "success" (boolean) and "message" (str if failed, dict if successful) """ if rel_date is None or rel_date == 'None': rel_date = None elif isinstance(rel_date, basestring) and rel_date != '': rel_date = parse(rel_date, fuzzy=True) elif not isinstance(rel_date, datetime.datetime): rel_date = None if not left_class: if left_type and left_id: left_class = class_from_value(left_type, left_id) if not left_class: return {'success': False, 'message': "Unable to get object."} else: return {'success': False, 'message': "Need a valid left type and id"} # delete relationship if not right_class: if right_type and right_id: right_class = class_from_value(right_type, right_id) else: return {'success': False, 'message': "Need a valid right type and id"} if not right_class: results = left_class.delete_relationship(rel_type=rel_type, rel_date=rel_date, analyst=analyst, rel_id=right_id, type_=right_type) else: results = left_class.delete_relationship(rel_item=right_class, rel_type=rel_type, rel_date=rel_date, analyst=analyst) if results['success']: left_class.save(username=analyst) left_class.reload() if get_rels: results['relationships'] = left_class.sort_relationships("%s" % analyst, meta=True) return results