def get_diffie_results(first, second): """ Retrieve requested analysis results. :param first: analysis_id of first result. :type first: str :param second: analysis_id of second result. :type second: str :returns: Dictionary with first and second keys. """ results = {'success': False} first_result = AnalysisResult.objects(analysis_id=first).first() if not first_result: results['message'] = "Unable to find first result." return results second_result = AnalysisResult.objects(analysis_id=second).first() if not second_result: results['message'] = "Unable to find second result." return results results['success'] = True results['first'] = first_result results['second'] = second_result return results
def add_results(object_type, object_id, analysis_id, result, type_, subtype, analyst): """ Add multiple results to an analysis task. :param object_type: The top-level object type. :type object_type: str :param object_id: The ObjectId to search for. :type object_id: str :param analysis_id: The ID of the task to update. :type analysis_id: str :param result: The list of result to append. :type result: list of str :param type_: The list of result types. :type type_: list of str :param subtype: The list of result subtypes. :type subtype: list of str :param analyst: The user updating the results. :type analyst: str :returns: dict with keys "success" (boolean) and "message" (str) if failed. """ res = {'success': False} if not object_type or not object_id or not analysis_id: res['message'] = "Must supply object id/type and analysis id." return res # Validate user can add service results to this TLO. klass = class_from_type(object_type) sources = user_sources(analyst) obj = klass.objects(id=object_id, source__name__in=sources).first() if not obj: res['message'] = "Could not find object to add results to." return res if not(result and type_ and subtype): res['message'] = "Need a result, type, and subtype to add a result." return res if not(len(result) == len(type_) == len(subtype)): res['message'] = "result, type, and subtype need to be the same length." return res # Update analysis results final_list = [] for key, r in enumerate(result): final = {} final['subtype'] = subtype[key] final['result'] = r tmp = ast.literal_eval(type_[key]) for k in tmp: final[k] = tmp[k] final_list.append(final) ar = AnalysisResult.objects(analysis_id=analysis_id).first() if ar: AnalysisResult.objects(id=ar.id).update_one(push_all__results=final_list) res['success'] = True return res
def update_analysis_results(task): """ Update analysis results for this task. """ # If the task does not currently exist for the given sample in the # database, add it. found = False ar = AnalysisResult.objects(analysis_id=task.task_id).first() if ar: found = True if not found: logger.warning("Tried to update a task that didn't exist.") insert_analysis_results(task) else: # Otherwise, update it. tdict = task.to_dict() tdict['analysis_id'] = tdict['id'] del tdict['id'] #TODO: find a better way to do this. new_dict = {} for k in tdict.iterkeys(): new_dict['set__%s' % k] = tdict[k] AnalysisResult.objects(id=ar.id).update_one(**new_dict)
def migrate_analysis_results(self): from crits.services.analysis_result import (AnalysisResult, AnalysisConfig, EmbeddedAnalysisResultLog) old_results = getattr(self.unsupported_attrs, 'analysis', None) if old_results: for result in old_results: ar = AnalysisResult() ar.analysis_id = result.get('id') if ar.analysis_id: del result['id'] config = result.get('config', {}) ar.config = AnalysisConfig(**config) if 'config' in result: del result['config'] logs = result.get('log', None) if logs: for l in logs: le = EmbeddedAnalysisResultLog(**l) ar.log.append(le) del result['log'] ar.merge(arg_dict=result) ar.object_type = self._meta['crits_type'] ar.object_id = str(self.id) ar.save() try: del self.unsupported_attrs['analysis'] except: pass
def insert_analysis_results(task): """ Insert analysis results for this task. """ ar = AnalysisResult() tdict = task.to_dict() tdict['analysis_id'] = tdict['id'] del tdict['id'] ar.merge(arg_dict=tdict) ar.save()
def run(self, obj, config): my_md5 = obj.md5 my_results = AnalysisResult.objects(object_type=obj._meta['crits_type'], object_id=str(obj.id)) completed_results = [] for result_doc in my_results: # skip our own results so we don't get nasty feedback if result_doc["service_name"] == self.name: continue for result in result_doc["results"]: if "md5" in result: res_type = "md5" else: res_type = "result" res_hash = "{0}-{1}".format(result_doc["service_name"], result[res_type]) if result[res_type] and res_hash not in completed_results: total_count = self._get_meta_count(res_type, result[res_type]) count_result = { 'service': result_doc["service_name"], 'type': res_type, res_type: result[res_type], 'count': total_count, } self._add_result("meta_count_{0}".format(res_type), result["result"], count_result) completed_results.append(res_hash)
def valid_for(obj): # Check if already running in case of triage re-run rezs = AnalysisResult.objects(object_id=str(obj.id), status='started', service_name='VirusTotal_Download') if rezs: raise ServiceConfigError("Service is already running")
def _check_triage(self): sample = Sample.objects(md5=self.test_md5).first() results = False if sample and sample.filedata: if len(AnalysisResult.objects(object_id=str(sample.id))) > 0: results = True print "[?] sample analysis executed == %s" % results return results
def delete_analysis(task_id, analyst): """ Delete analysis results. """ ar = AnalysisResult.objects(id=task_id).first() if ar: ar.delete(username=analyst)
def finish_task(object_type, object_id, analysis_id, status, analyst): """ Finish a task by setting its status to "completed" and setting the finish date. :param object_type: The top-level object type. :type object_type: str :param object_id: The ObjectId to search for. :type object_id: str :param analysis_id: The ID of the task to update. :type analysis_id: str :param status: The status of the task. :type status: str ("error", "completed") :param analyst: The user updating the log. :type analyst: str :returns: dict with keys "success" (boolean) and "message" (str) if failed. """ results = {'success': False} if not status: status = "completed" if status not in ('error', 'completed'): status = "completed" if not object_type or not object_id or not analysis_id: results['message'] = "Must supply object id/type and analysis id." return results # Validate user can add service results to this TLO. klass = class_from_type(object_type) params = {'id': object_id} if hasattr(klass, 'source'): params['source__name__in'] = user_sources(analyst) obj = klass.objects(**params).first() if not obj: results['message'] = "Could not find object to add results to." return results # Update analysis log date = str(datetime.datetime.now()) ar = AnalysisResult.objects(analysis_id=analysis_id).first() if ar: AnalysisResult.objects(id=ar.id).update_one(set__status=status, set__finish_date=date) results['success'] = True return results
def add_log(object_type, object_id, analysis_id, log_message, level, analyst): """ Add a log entry to an analysis task. :param object_type: The top-level object type. :type object_type: str :param object_id: The ObjectId to search for. :type object_id: str :param analysis_id: The ID of the task to update. :type analysis_id: str :param log_message: The log entry to append. :type log_message: dict :param level: The log level. :type level: str :param analyst: The user updating the log. :type analyst: str :returns: dict with keys "success" (boolean) and "message" (str) if failed. """ results = {'success': False} if not object_type or not object_id or not analysis_id: results['message'] = "Must supply object id/type and analysis id." return results # Validate user can add service results to this TLO. klass = class_from_type(object_type) sources = user_sources(analyst) obj = klass.objects(id=object_id, source__name__in=sources).first() if not obj: results['message'] = "Could not find object to add results to." return results # Update analysis log le = EmbeddedAnalysisResultLog() le.message = log_message le.level = level le.datetime = str(datetime.datetime.now()) ar = AnalysisResult.objects(analysis_id=analysis_id).first() if ar: AnalysisResult.objects(id=ar.id).update_one(push__log=le) results['success'] = True else: results['message'] = "Could not find task to add log to." return results
def finish_task(object_type, object_id, analysis_id, status, analyst): """ Finish a task by setting its status to "completed" and setting the finish date. :param object_type: The top-level object type. :type object_type: str :param object_id: The ObjectId to search for. :type object_id: str :param analysis_id: The ID of the task to update. :type analysis_id: str :param status: The status of the task. :type status: str ("error", "completed") :param analyst: The user updating the log. :type analyst: str :returns: dict with keys "success" (boolean) and "message" (str) if failed. """ results = {'success': False} if not status: status = "completed" if status not in ('error', 'completed'): status = "completed" if not object_type or not object_id or not analysis_id: results['message'] = "Must supply object id/type and analysis id." return results # Validate user can add service results to this TLO. klass = class_from_type(object_type) sources = user_sources(analyst) obj = klass.objects(id=object_id, source__name__in=sources).first() if not obj: results['message'] = "Could not find object to add results to." return results # Update analysis log date = str(datetime.datetime.now()) ar = AnalysisResult.objects(analysis_id=analysis_id).first() if ar: AnalysisResult.objects(id=ar.id).update_one(set__status=status, set__finish_date=date) results['success'] = True return results
def run_analysis_cleanup(self, obj_list, type_, delay): print "\nCleaning Analysis for:\n---------------" for obj in obj_list: results = AnalysisResult.objects(object_type=type_, object_id=str(obj.id)) print(" [+] {0}".format(obj.id)) for result in results: result.delete() time.sleep(float(delay)) run_triage(obj, self.username)
def refresh_services(request, crits_type, identifier): """ Refresh the Analysis tab with the latest information. """ response = {} # Verify user can see results. sources = user_sources(request.user.username) klass = class_from_type(crits_type) if not klass: msg = 'Could not find object to refresh!' response['success'] = False response['html'] = msg return HttpResponse(json.dumps(response), mimetype="application/json") if hasattr(klass, 'source'): obj = klass.objects(id=identifier, source__name__in=sources).first() else: obj = klass.objects(id=identifier).first() if not obj: msg = 'Could not find object to refresh!' response['success'] = False response['html'] = msg return HttpResponse(json.dumps(response), mimetype="application/json") # Get analysis results. results = AnalysisResult.objects(object_type=crits_type, object_id=identifier) relationship = { 'type': crits_type, 'value': identifier, 'url_key': obj.get_url_key() } subscription = {'type': crits_type, 'id': identifier} service_list = get_supported_services(crits_type) response['success'] = True response['html'] = render_to_string( "services_analysis_listing.html", { 'relationship': relationship, 'subscription': subscription, 'service_results': results, 'crits_type': crits_type, 'identifier': identifier, 'service_list': service_list }, RequestContext(request)) return HttpResponse(json.dumps(response), mimetype="application/json")
def refresh_services(request, crits_type, identifier): """ Refresh the Analysis tab with the latest information. """ response = {} request.user._setup() # Verify user can see results. sources = request.user.get_sources_list() klass = class_from_type(crits_type) if not klass: msg = 'Could not find object to refresh!' response['success'] = False response['html'] = msg return HttpResponse(json.dumps(response), content_type="application/json") if hasattr(klass, 'source'): obj = klass.objects(id=identifier,source__name__in=sources).first() else: obj = klass.objects(id=identifier).first() if not obj: msg = 'Could not find object to refresh!' response['success'] = False response['html'] = msg return HttpResponse(json.dumps(response), content_type="application/json") # Get analysis results. results = AnalysisResult.objects(object_type=crits_type, object_id=identifier) relationship = {'type': crits_type, 'value': identifier} subscription = {'type': crits_type, 'id': identifier} service_list = get_supported_services(crits_type) response['success'] = True response['html'] = render_to_string("services_analysis_listing.html", {'relationship': relationship, 'subscription': subscription, 'service_results': results, 'crits_type': crits_type, 'identifier': identifier, 'service_list': service_list}, request=request) return HttpResponse(json.dumps(response), content_type="application/json")
def __init__(self, type_, id_, *args, **kwargs): super(DiffieConfigForm, self).__init__(*args, **kwargs) # Take each analysis result passed in as a kwarg and turn it into # a tuple for the form: ('id', 'service_name: start_date'). Only # take the ones with a status that is 'completed'. analysisresults = AnalysisResult.objects(object_type=type_, object_id=id_) choices = [] for ar in analysisresults: if ar.status == AnalysisTask.STATUS_COMPLETED: choices.append((ar.analysis_id, '%s: %s' % (ar.service_name, ar.start_date))) self.fields['first'].choices = choices self.fields['second'].choices = choices self.fields['type_'].initial = type_ self.fields['id_'].initial = id_
def analysis_result(request, analysis_id): """ Get the TLO type and object_id and redirect to the details page for that TLO. :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :param analysis_id: The ObjectId of the AnalysisResult :type analysis_id: str :returns: :class:`django.http.HttpResponse` """ ar = AnalysisResult.objects(id=analysis_id).first() if ar: return HttpResponseRedirect(reverse('crits.core.views.details', args=(ar.object_type,ar.object_id))) else: return render_to_response('error.html', {'error': "No TLO found to redirect to."})
def update_analysis_results(task): """ Update analysis results for this task. """ # If the task does not currently exist for the given sample in the # database, add it. found = False ar = AnalysisResult.objects(analysis_id=task.task_id).first() if ar: found = True if not found: logger.warning("Tried to update a task that didn't exist.") insert_analysis_results(task) else: # Otherwise, update it. tdict = task.to_dict() tdict['analysis_id'] = tdict['id'] del tdict['id'] #TODO: find a better way to do this. new_dict = {} for k in tdict.iterkeys(): new_dict['set__%s' % k] = tdict[k] try: AnalysisResult.objects(id=ar.id).update_one(**new_dict) except Exception as e: # assume bad data in 'results' task.status = 'error' new_dict['set__results'] = [] le = EmbeddedAnalysisResultLog() le.message = 'DB Update Failed: %s' % e le.level = 'error' le.datetime = str(datetime.datetime.now()) new_dict['set__log'].append(le) try: AnalysisResult.objects(id=ar.id).update_one(**new_dict) except: # don't know what's wrong, try writing basic log only AnalysisResult.objects(id=ar.id).update_one(set__log=[le])
def _get_meta_count(self, meta_type, meta_val): query_field = "results.{0}".format(meta_type) query = {query_field: meta_val} total_count = AnalysisResult.objects(object_type='Sample', __raw__=query).only('id').count() return total_count
def handle_cert_file(filename, data, source_name, user=None, description=None, related_id=None, related_md5=None, related_type=None, method=None, reference=None, relationship=None, bucket_list=None, ticket=None): """ Add a Certificate. :param filename: The filename of the Certificate. :type filename: str :param data: The filedata of the Certificate. :type data: str :param source_name: The source which provided this Certificate. :type source_name: str, :class:`crits.core.crits_mongoengine.EmbeddedSource`, list of :class:`crits.core.crits_mongoengine.EmbeddedSource` :param user: The user adding the Certificate. :type user: str :param description: Description of the Certificate. :type description: str :param related_id: ObjectId of a top-level object related to this Certificate. :type related_id: str :param related_md5: MD5 of a top-level object related to this Certificate. :type related_md5: str :param related_type: The CRITs type of the related top-level object. :type related_type: str :param method: The method of acquiring this Certificate. :type method: str :param reference: A reference to the source of this Certificate. :type reference: str :param relationship: The relationship between the parent and the Certificate. :type relationship: str :param bucket_list: Bucket(s) to add to this Certificate :type bucket_list: str(comma separated) or list. :param ticket: Ticket(s) to add to this Certificate :type ticket: str(comma separated) or list. :returns: dict with keys: 'success' (boolean), 'message' (str), 'md5' (str) if successful. """ if not data: status = { 'success': False, 'message': 'No data object passed in' } return status if len(data) <= 0: status = { 'success': False, 'message': 'Data length <= 0' } return status if ((related_type and not (related_id or related_md5)) or (not related_type and (related_id or related_md5))): status = { 'success': False, 'message': 'Must specify both related_type and related_id or related_md5.' } return status related_obj = None if related_id or related_md5: if related_id: related_obj = class_from_id(related_type, related_id) else: related_obj = class_from_value(related_type, related_md5) if not related_obj: status = { 'success': False, 'message': 'Related object not found.' } return status # generate md5 and timestamp md5 = hashlib.md5(data).hexdigest() timestamp = datetime.datetime.now() # generate Certificate cert = Certificate.objects(md5=md5).first() if not cert: cert = Certificate() cert.filename = filename cert.created = timestamp cert.size = len(data) cert.description = description cert.md5 = md5 # generate source information and add to certificate if isinstance(source_name, basestring) and len(source_name) > 0: s = create_embedded_source(source_name, method=method, reference=reference, analyst=user) cert.add_source(s) elif isinstance(source_name, EmbeddedSource): cert.add_source(source_name, method=method, reference=reference) elif isinstance(source_name, list) and len(source_name) > 0: for s in source_name: if isinstance(s, EmbeddedSource): cert.add_source(s, method=method, reference=reference) if bucket_list: cert.add_bucket_list(bucket_list, user) if ticket: cert.add_ticket(ticket, user) # add file to GridFS if not isinstance(cert.filedata.grid_id, ObjectId): cert.add_file_data(data) # save cert cert.save(username=user) cert.reload() # run certificate triage if len(AnalysisResult.objects(object_id=str(cert.id))) < 1 and data: run_triage(cert, user) # update relationship if a related top-level object is supplied if related_obj and cert: if not relationship: relationship = "Related_To" cert.add_relationship(rel_item=related_obj, rel_type=relationship, analyst=user, get_rels=False) related_obj.save(username=user) cert.save(username=user) status = { 'success': True, 'message': 'Uploaded certificate', 'md5': md5, 'id': str(cert.id), 'object': cert } return status
def valid_for(obj): # Make sure there are AnalysisResults for this object. results = AnalysisResult.objects(object_type=obj._meta['crits_type'], object_id=str(obj.id)) if len(results) == 0: raise ServiceConfigError("Object must have analysis results.")
def add_results(object_type, object_id, analysis_id, result, type_, subtype, analyst): """ Add multiple results to an analysis task. :param object_type: The top-level object type. :type object_type: str :param object_id: The ObjectId to search for. :type object_id: str :param analysis_id: The ID of the task to update. :type analysis_id: str :param result: The list of result to append. :type result: list of str :param type_: The list of result types. :type type_: list of str :param subtype: The list of result subtypes. :type subtype: list of str :param analyst: The user updating the results. :type analyst: str :returns: dict with keys "success" (boolean) and "message" (str) if failed. """ res = {'success': False} if not object_type or not object_id or not analysis_id: res['message'] = "Must supply object id/type and analysis id." return res # Validate user can add service results to this TLO. klass = class_from_type(object_type) sources = user_sources(analyst) obj = klass.objects(id=object_id, source__name__in=sources).first() if not obj: res['message'] = "Could not find object to add results to." return res if not (result and type_ and subtype): res['message'] = "Need a result, type, and subtype to add a result." return res if not (len(result) == len(type_) == len(subtype)): res['message'] = "result, type, and subtype need to be the same length." return res # Update analysis results final_list = [] for key, r in enumerate(result): final = {} final['subtype'] = subtype[key] final['result'] = r tmp = ast.literal_eval(type_[key]) for k in tmp: final[k] = tmp[k] final_list.append(final) ar = AnalysisResult.objects(analysis_id=analysis_id).first() if ar: AnalysisResult.objects(id=ar.id).update_one( push_all__results=final_list) res['success'] = True return res
def handle_cert_file(filename, data, source_name, user=None, description=None, related_md5=None, method='', reference='', tlp=None, relationship=None, bucket_list=None, ticket=None, related_id=None, related_type=None, relationship_type=None): """ Add a Certificate. :param filename: The filename of the Certificate. :type filename: str :param data: The filedata of the Certificate. :type data: str :param source_name: The source which provided this Certificate. :type source_name: str, :class:`crits.core.crits_mongoengine.EmbeddedSource`, list of :class:`crits.core.crits_mongoengine.EmbeddedSource` :param user: The user adding the Certificate. :type user: str :param description: Description of the Certificate. :type description: str :param related_md5: MD5 of a top-level object related to this Certificate. :type related_md5: str :param related_type: The CRITs type of the related top-level object. :type related_type: str :param method: The method of acquiring this Certificate. :type method: str :param reference: A reference to the source of this Certificate. :type reference: str :param tlp: The TLP for this certificate. :type tlp: str :param relationship: The relationship between the parent and the Certificate. :type relationship: str :param bucket_list: Bucket(s) to add to this Certificate :type bucket_list: str(comma separated) or list. :param ticket: Ticket(s) to add to this Certificate :type ticket: str(comma separated) or list. :param related_id: ID of object to create relationship with :type related_id: str :param related_type: Type of object to create relationship with :type related_id: str :param relationship_type: Type of relationship to create. :type relationship_type: str :returns: dict with keys: 'success' (boolean), 'message' (str), 'md5' (str) if successful. """ if not data: status = {'success': False, 'message': 'No data object passed in'} return status if len(data) <= 0: status = {'success': False, 'message': 'Data length <= 0'} return status if ((related_type and not (related_id or related_md5)) or (not related_type and (related_id or related_md5))): status = { 'success': False, 'message': 'Must specify both related_type and related_id or related_md5.' } return status related_obj = None if related_id or related_md5: if related_id: related_obj = class_from_id(related_type, related_id) else: related_obj = class_from_value(related_type, related_md5) if not related_obj: status = {'success': False, 'message': 'Related object not found.'} return status # generate md5 and timestamp md5 = hashlib.md5(data).hexdigest() timestamp = datetime.datetime.now() # generate Certificate cert = Certificate.objects(md5=md5).first() if not cert: cert = Certificate() cert.filename = filename cert.created = timestamp cert.size = len(data) cert.description = description cert.md5 = md5 # generate source information and add to certificate if isinstance(source_name, basestring) and len(source_name) > 0: if user.check_source_write(source_name): s = create_embedded_source(source_name, reference=reference, method=method, tlp=tlp, analyst=user.username) else: return { "success": False, "message": "User does not have permission to add objects \ using source %s." % str(source_name) } cert.add_source(s) elif isinstance(source_name, EmbeddedSource): cert.add_source(source_name, method=method, reference=reference, tlp=tlp) elif isinstance(source_name, list) and len(source_name) > 0: for s in source_name: if isinstance(s, EmbeddedSource): cert.add_source(s, method=method, reference=reference, tlp=tlp) if bucket_list: cert.add_bucket_list(bucket_list, user) if ticket: cert.add_ticket(ticket, user) # add file to GridFS if not isinstance(cert.filedata.grid_id, ObjectId): cert.add_file_data(data) # save cert cert.save(username=user) cert.reload() # run certificate triage if AnalysisResult.objects(object_id=str(cert.id)).count() < 1 and data: run_triage(cert, user) # update relationship if a related top-level object is supplied if related_obj and cert: if relationship_type: relationship = RelationshipTypes.inverse( relationship=relationship_type) if not relationship: relationship = RelationshipTypes.RELATED_TO cert.add_relationship(related_obj, relationship, analyst=user, get_rels=False) cert.save(username=user) status = { 'success': True, 'message': 'Uploaded certificate', 'md5': md5, 'id': str(cert.id), 'object': cert } return status