def testSourceDataQuery(self): objs = TestSourceObject # User does not have source, should not return results resp = handlers.data_query(objs, self.user.username) self.assertEqual(resp['count'], 0) self.assertEqual(resp['result'], 'OK') self.assertEqual(resp['crits_type'], 'TestSourceBase') self.assertEqual(resp['msg'], '') self.assertTrue(isinstance(resp['data'], CritsQuerySet)) # Add source for user and query again data = { 'username': self.user.username, 'first_name': self.user.first_name, 'last_name': self.user.last_name, 'email': self.user.email, 'role': self.user.role, 'sources': [ TSRC, ], 'secret': '', 'organization': TSRC, 'subscriptions': [], 'totp': False, } handlers.modify_source_access(self.user.username, data) resp = handlers.data_query(objs, self.user.username) # Now we should get one result, but not the UnknownSource object self.assertEqual(resp['count'], 1) self.assertEqual(resp['result'], 'OK') self.assertEqual(resp['crits_type'], 'TestSourceBase') self.assertEqual(resp['msg'], '') self.assertEqual(resp['data'][0].name, TOBJS_NAME) self.assertEqual(resp['data'][0].value, TOBJS_VALUE) self.assertEqual(resp['data'][0]._meta['crits_type'], "TestSourceBase")
def testSourceDataQuery(self): objs = TestSourceObject # User does not have source, should not return results resp = handlers.data_query(objs, self.user.username) self.assertEqual(resp['count'], 0) self.assertEqual(resp['result'], 'OK') self.assertEqual(resp['crits_type'], 'TestSourceBase') self.assertEqual(resp['msg'], '') self.assertTrue(isinstance(resp['data'], CritsQuerySet)) # Add source for user and query again data = {'username': self.user.username, 'first_name': self.user.first_name, 'last_name': self.user.last_name, 'email': self.user.email, 'role': self.user.role, 'sources': [TSRC, ], 'secret': '', 'organization': TSRC, 'subscriptions': [], 'totp': False, } handlers.modify_source_access(self.user.username, data) resp = handlers.data_query(objs, self.user.username) # Now we should get one result, but not the UnknownSource object self.assertEqual(resp['count'], 1) self.assertEqual(resp['result'], 'OK') self.assertEqual(resp['crits_type'], 'TestSourceBase') self.assertEqual(resp['msg'], '') self.assertEqual(resp['data'][0].name, TOBJS_NAME) self.assertEqual(resp['data'][0].value, TOBJS_VALUE) self.assertEqual(resp['data'][0]._meta['crits_type'], "TestSourceBase")
def getRecordsForDefaultDashboardTable(username, tableName): """ Called by createTableObject to retrieve the proper records from the database for the default dashboard tables. These queries are different then the saved searches which is why it is needed. This is also called via ajax on the saved_search.html page by get_dashboard_table_data in Views.py. This is to get the records when editing the default tables. """ from crits.core.handlers import data_query, generate_counts_jtable if tableName == "Recent_Samples" or tableName == "Recent Samples": obj_type = "Sample" response = data_query(Sample, username, query={}, sort=["-created"], limit=5) elif tableName == "Recent_Emails" or tableName == "Recent Emails": obj_type = "Email" response = data_query(Email, username, query={}, sort=["-isodate"], limit=5) elif tableName == "Recent_Indicators" or tableName == "Recent Indicators": obj_type = "Indicator" response = data_query(Indicator, username, query={}, sort=["-created"], limit=5) elif tableName == "Top_Campaigns" or tableName == "Top Campaigns": obj_type = "Campaign" response = data_query(Campaign, username, query={}, limit=5) elif tableName == "Top_Backdoors" or tableName == "Top Backdoors": obj_type = "Backdoor" response = data_query(Backdoor, username, query={}, limit=5) elif tableName == "Counts": response = generate_counts_jtable(None, "jtlist") records = json.loads(response.content)["Records"] for record in records: record["recid"] = record.pop("id") return records return parseDocumentsForW2ui(response, obj_type)
def getRecordsForDefaultDashboardTable(user, tableName): """ Called by createTableObject to retrieve the proper records from the database for the default dashboard tables. These queries are different then the saved searches which is why it is needed. This is also called via ajax on the saved_search.html page by get_dashboard_table_data in Views.py. This is to get the records when editing the default tables. """ from crits.core.handlers import data_query, generate_counts_jtable if tableName == "Recent_Samples" or tableName == "Recent Samples" and user.has_access_to( SampleACL.READ): obj_type = "Sample" response = data_query(Sample, user, query={}, sort=["-created"], limit=5) elif tableName == "Recent_Emails" or tableName == "Recent Emails" and user.has_access_to( EmailACL.READ): obj_type = "Email" response = data_query(Email, user, query={}, sort=["-isodate"], limit=5) elif tableName == "Recent_Indicators" or tableName == "Recent Indicators" and user.has_access_to( IndicatorACL.READ): obj_type = "Indicator" response = data_query(Indicator, user, query={}, sort=["-created"], limit=5) elif tableName == "Top_Campaigns" or tableName == "Top Campaigns" and user.has_access_to( CampaignACL.READ): obj_type = "Campaign" response = data_query(Campaign, user, query={}, limit=5) elif tableName == "Counts": response = generate_counts_jtable(None, "jtlist") records = json.loads(response.content)["Records"] for record in records: record["recid"] = record.pop("id") return records else: # This only happens if we have a dashboard which is no longer valid. # For example, after Backdoor and Exploit were added the "Top_Backdoors" # dashboard is no longer valid. Produce an "empty" response. response = {'data': []} obj_type = None return parseDocumentsForW2ui(response, obj_type)
def get_table_data(request=None,obj=None,user=None,searchTerm="", search_type=None, includes=[], excludes=[], maxRows=25, sort={}, pageNumber=1): """ gets the records needed for the table, can be called via ajax on the saved_search.html or the above ConstructTable function """ from crits.core.handlers import get_query, data_query response = {"Result": "ERROR"} obj_type = get_obj_type_from_string(obj) # Build the query term = "" #if its being called from saved_search.html if request and request.is_ajax(): resp = get_query(obj_type, request) #if its calling to get data for the dashbaord elif user and search_type: resp = get_query_without_request(obj_type, user.username, searchTerm, search_type) else: return HttpResponse(json.dumps(response, default=json_handler), mimetype='application/json') if resp['Result'] in ["ERROR", "IGNORE"]: return resp query = resp['query'] term = resp['term'] sortBy = [] if 'direction' in sort: if sort['direction'] == 'asc': sortBy.append(sort['field']) elif sort['direction'] == 'desc': sortBy.append("-"+sort['field']) skip = (int(pageNumber)-1)*25 if request: response = data_query(obj_type, user=request.user.username, query=query, projection=includes, limit=int(maxRows), sort=sortBy, skip=skip) else: response = data_query(obj_type, user=user.username, query=query, projection=includes, limit=maxRows, sort=sortBy,skip=skip) if response['result'] == "ERROR": return {'Result': "ERROR", 'Message': response['msg']} response['crits_type'] = obj_type # Escape term for rendering in the UI. response['term'] = cgi.escape(term) response['data'] = response['data'].to_dict(excludes, includes) response['Records'] = parseDocObjectsToStrings(response.pop('data'), obj) response['TotalRecordCount'] = response.pop('count') response['Result'] = response.pop('result') if request: return HttpResponse(json.dumps(response, default=json_handler), mimetype='application/json') else: return response
def get_table_data(request=None,obj=None,user=None,searchTerm="", search_type=None, includes=[], excludes=[], maxRows=25, sort={}, pageNumber=1): """ gets the records needed for the table, can be called via ajax on the saved_search.html or the above ConstructTable function """ from crits.core.handlers import get_query, data_query response = {"Result": "ERROR"} obj_type = get_obj_type_from_string(obj) # Build the query term = "" #if its being called from saved_search.html if request and request.is_ajax(): resp = get_query(obj_type, request) #if its calling to get data for the dashbaord elif user and search_type: resp = get_query_without_request(obj_type, user.username, searchTerm, search_type) else: return HttpResponse(json.dumps(response, default=json_handler), content_type="application/json") if resp['Result'] in ["ERROR", "IGNORE"]: return resp query = resp['query'] term = resp['term'] sortBy = [] if 'direction' in sort: if sort['direction'] == 'asc': sortBy.append(sort['field']) elif sort['direction'] == 'desc': sortBy.append("-"+sort['field']) skip = (int(pageNumber)-1)*25 if request: response = data_query(obj_type, user=request.user.username, query=query, projection=includes, limit=int(maxRows), sort=sortBy, skip=skip) else: response = data_query(obj_type, user=user.username, query=query, projection=includes, limit=maxRows, sort=sortBy,skip=skip) if response['result'] == "ERROR": return {'Result': "ERROR", 'Message': response['msg']} response['crits_type'] = obj_type # Escape term for rendering in the UI. response['term'] = cgi.escape(term) response['data'] = response['data'].to_dict(excludes, includes) response['Records'] = parseDocObjectsToStrings(response.pop('data'), obj) response['TotalRecordCount'] = response.pop('count') response['Result'] = response.pop('result') if request: return HttpResponse(json.dumps(response, default=json_handler), content_type="application/json") else: return response
def generate_search_for_saved_table(user, id=None, request=None): """ Called by edit_save_search in views.py. This is for editing a previously saved table or one of the default dashboard tables """ from crits.core.handlers import data_query response = {} savedSearch = None try: savedSearch = SavedSearch.objects(id=id).first() if not savedSearch: response['Result'] = "ERROR" response[ 'Message'] = "Error finding table, please try again later." return response except: savedSearch = SavedSearch() savedSearch.isDefaultOnDashboard = True savedSearch.name = id.replace("_", " ") id = None results = [] records = [] term = "" url = "" if not savedSearch.isDefaultOnDashboard: objType = get_obj_type_from_string(savedSearch.objType) resp = get_query_without_request(objType, user, savedSearch.searchTerm, "global") if resp['Result'] == "ERROR": return resp formatted_query = resp['query'] term = resp['term'] resp = data_query(objType, user, query=formatted_query, count=True) results.append({'count': resp['count'], 'name': savedSearch.objType}) else: results = { "name": savedSearch.name, "count": str(len(records)), "type": get_obj_name_from_title(savedSearch.name) } #special url to get the records of a default dashboard since their queries are different url = reverse( "crits.dashboards.views.get_dashboard_table_data", kwargs={"tableName": str(savedSearch.name.replace(" ", "_"))}) args = {'term': term, 'results': results, 'dataUrl': url, 'Result': "OK"} if savedSearch: args.update({ 'tableId': id, 'tableName': savedSearch.name, 'columns': savedSearch.tableColumns, 'sortBy': savedSearch.sortBy, 'sizex': savedSearch.sizex, 'maxRows': savedSearch.maxRows, 'isDefaultOnDashboard': savedSearch.isDefaultOnDashboard, }) if savedSearch.dashboard: args["currentDash"] = str(savedSearch.dashboard) args["dashtheme"] = Dashboard.objects( id=savedSearch.dashboard).first().theme return args
def generate_search_for_saved_table(user, id=None,request=None): """ Called by edit_save_search in views.py. This is for editing a previously saved table or one of the default dashboard tables """ from crits.core.handlers import data_query response = {} savedSearch = None try: savedSearch = SavedSearch.objects(id=id).first() if not savedSearch: response['Result'] = "ERROR" response['Message'] = "Error finding table, please try again later." return response except: savedSearch = SavedSearch() savedSearch.isDefaultOnDashboard = True savedSearch.name = id.replace("_", " ") id = None results = [] records = [] term = "" url = "" if not savedSearch.isDefaultOnDashboard: objType = get_obj_type_from_string(savedSearch.objType) resp = get_query_without_request(objType, user, savedSearch.searchTerm, "global") if resp['Result'] == "ERROR": return resp formatted_query = resp['query'] term = resp['term'] resp = data_query(objType, user, query=formatted_query, count=True) results.append({'count': resp['count'], 'name': savedSearch.objType}) else: results = {"name":savedSearch.name, "count":str(len(records)), "type":get_obj_name_from_title(savedSearch.name)} #special url to get the records of a default dashboard since their queries are different url = reverse("crits-dashboards-views-get_dashboard_table_data", kwargs={"tableName":str(savedSearch.name.replace(" ", "_"))}) args = {'term': term, 'results': results, 'dataUrl':url, 'Result': "OK" } if savedSearch: args.update({'tableId':id, 'tableName': savedSearch.name, 'columns': savedSearch.tableColumns, 'sortBy': savedSearch.sortBy, 'sizex' : savedSearch.sizex, 'maxRows': savedSearch.maxRows, 'isDefaultOnDashboard': savedSearch.isDefaultOnDashboard, }) if savedSearch.dashboard: args["currentDash"] = str(savedSearch.dashboard) args["dashtheme"] = Dashboard.objects(id=savedSearch.dashboard).first().theme return args
def getRecordsForDefaultDashboardTable(user, tableName): """ Called by createTableObject to retrieve the proper records from the database for the default dashboard tables. These queries are different then the saved searches which is why it is needed. This is also called via ajax on the saved_search.html page by get_dashboard_table_data in Views.py. This is to get the records when editing the default tables. """ from crits.core.handlers import data_query, generate_counts_jtable if tableName == "Recent_Samples" or tableName == "Recent Samples" and user.has_access_to(SampleACL.READ): obj_type = "Sample" response = data_query(Sample, user, query={}, sort=["-created"], limit=5) elif tableName == "Recent_Emails" or tableName == "Recent Emails" and user.has_access_to(EmailACL.READ): obj_type = "Email" response = data_query(Email, user, query={}, sort=["-isodate"], limit=5) elif tableName == "Recent_Indicators" or tableName == "Recent Indicators" and user.has_access_to(IndicatorACL.READ): obj_type = "Indicator" response = data_query(Indicator, user, query={}, sort=["-created"], limit=5) elif tableName == "Top_Campaigns" or tableName == "Top Campaigns" and user.has_access_to(CampaignACL.READ): obj_type = "Campaign" response = data_query(Campaign, user, query={}, limit=5) elif tableName == "Counts": response = generate_counts_jtable(None, "jtlist") records = json.loads(response.content)["Records"] for record in records: record["recid"] = record.pop("id") return records else: # This only happens if we have a dashboard which is no longer valid. # For example, after Backdoor and Exploit were added the "Top_Backdoors" # dashboard is no longer valid. Produce an "empty" response. response = {'data': []} obj_type = None return parseDocumentsForW2ui(response, obj_type)
def testDataQuery(self): """ Test data_query from handlers.py data_query(col_obj,user[,limit,skip,sort,query,projection]) """ obj = TestObject resp = handlers.data_query(obj, self.user.username) self.assertEqual(resp['count'], 1) self.assertEqual(resp['result'], 'OK') self.assertEqual(resp['crits_type'], 'TestBase') self.assertEqual(resp['msg'], '') self.assertTrue(isinstance(resp['data'], CritsQuerySet)) self.assertEqual(resp['data'][0].name, TOBJ_NAME) self.assertEqual(resp['data'][0].value, TOBJ_VALUE) self.assertEqual(resp['data'][0]._meta['crits_type'], "TestBase")