def testSourceDataQuery(self):
objs = TestSourceObject
# User does not have source, should not return results
resp = handlers.data_query(objs, self.user.username)
self.assertEqual(resp['count'], 0)
self.assertEqual(resp['result'], 'OK')
self.assertEqual(resp['crits_type'], 'TestSourceBase')
self.assertEqual(resp['msg'], '')
self.assertTrue(isinstance(resp['data'], CritsQuerySet))
# Add source for user and query again
data = {
'username': self.user.username,
'first_name': self.user.first_name,
'last_name': self.user.last_name,
'email': self.user.email,
'role': self.user.role,
'sources': [
TSRC,
],
'secret': '',
'organization': TSRC,
'subscriptions': [],
'totp': False,
}
handlers.modify_source_access(self.user.username, data)
resp = handlers.data_query(objs, self.user.username)
# Now we should get one result, but not the UnknownSource object
self.assertEqual(resp['count'], 1)
self.assertEqual(resp['result'], 'OK')
self.assertEqual(resp['crits_type'], 'TestSourceBase')
self.assertEqual(resp['msg'], '')
self.assertEqual(resp['data'][0].name, TOBJS_NAME)
self.assertEqual(resp['data'][0].value, TOBJS_VALUE)
self.assertEqual(resp['data'][0]._meta['crits_type'], "TestSourceBase")
def testSourceDataQuery(self):
objs = TestSourceObject
# User does not have source, should not return results
resp = handlers.data_query(objs, self.user.username)
self.assertEqual(resp['count'], 0)
self.assertEqual(resp['result'], 'OK')
self.assertEqual(resp['crits_type'], 'TestSourceBase')
self.assertEqual(resp['msg'], '')
self.assertTrue(isinstance(resp['data'], CritsQuerySet))
# Add source for user and query again
data = {'username': self.user.username,
'first_name': self.user.first_name,
'last_name': self.user.last_name,
'email': self.user.email,
'role': self.user.role,
'sources': [TSRC, ],
'secret': '',
'organization': TSRC,
'subscriptions': [],
'totp': False,
}
handlers.modify_source_access(self.user.username, data)
resp = handlers.data_query(objs, self.user.username)
# Now we should get one result, but not the UnknownSource object
self.assertEqual(resp['count'], 1)
self.assertEqual(resp['result'], 'OK')
self.assertEqual(resp['crits_type'], 'TestSourceBase')
self.assertEqual(resp['msg'], '')
self.assertEqual(resp['data'][0].name, TOBJS_NAME)
self.assertEqual(resp['data'][0].value, TOBJS_VALUE)
self.assertEqual(resp['data'][0]._meta['crits_type'], "TestSourceBase")
def getRecordsForDefaultDashboardTable(username, tableName):
"""
Called by createTableObject to retrieve the proper records from the
database for the default dashboard tables. These queries are different then
the saved searches which is why it is needed.
This is also called via ajax on the saved_search.html page by
get_dashboard_table_data in Views.py. This is to get the records when
editing the default tables.
"""
from crits.core.handlers import data_query, generate_counts_jtable
if tableName == "Recent_Samples" or tableName == "Recent Samples":
obj_type = "Sample"
response = data_query(Sample, username, query={}, sort=["-created"], limit=5)
elif tableName == "Recent_Emails" or tableName == "Recent Emails":
obj_type = "Email"
response = data_query(Email, username, query={}, sort=["-isodate"], limit=5)
elif tableName == "Recent_Indicators" or tableName == "Recent Indicators":
obj_type = "Indicator"
response = data_query(Indicator, username, query={}, sort=["-created"], limit=5)
elif tableName == "Top_Campaigns" or tableName == "Top Campaigns":
obj_type = "Campaign"
response = data_query(Campaign, username, query={}, limit=5)
elif tableName == "Top_Backdoors" or tableName == "Top Backdoors":
obj_type = "Backdoor"
response = data_query(Backdoor, username, query={}, limit=5)
elif tableName == "Counts":
response = generate_counts_jtable(None, "jtlist")
records = json.loads(response.content)["Records"]
for record in records:
record["recid"] = record.pop("id")
return records
return parseDocumentsForW2ui(response, obj_type)
def getRecordsForDefaultDashboardTable(user, tableName):
"""
Called by createTableObject to retrieve the proper records from the
database for the default dashboard tables. These queries are different then
the saved searches which is why it is needed.
This is also called via ajax on the saved_search.html page by
get_dashboard_table_data in Views.py. This is to get the records when
editing the default tables.
"""
from crits.core.handlers import data_query, generate_counts_jtable
if tableName == "Recent_Samples" or tableName == "Recent Samples" and user.has_access_to(
SampleACL.READ):
obj_type = "Sample"
response = data_query(Sample,
user,
query={},
sort=["-created"],
limit=5)
elif tableName == "Recent_Emails" or tableName == "Recent Emails" and user.has_access_to(
EmailACL.READ):
obj_type = "Email"
response = data_query(Email,
user,
query={},
sort=["-isodate"],
limit=5)
elif tableName == "Recent_Indicators" or tableName == "Recent Indicators" and user.has_access_to(
IndicatorACL.READ):
obj_type = "Indicator"
response = data_query(Indicator,
user,
query={},
sort=["-created"],
limit=5)
elif tableName == "Top_Campaigns" or tableName == "Top Campaigns" and user.has_access_to(
CampaignACL.READ):
obj_type = "Campaign"
response = data_query(Campaign, user, query={}, limit=5)
elif tableName == "Counts":
response = generate_counts_jtable(None, "jtlist")
records = json.loads(response.content)["Records"]
for record in records:
record["recid"] = record.pop("id")
return records
else:
# This only happens if we have a dashboard which is no longer valid.
# For example, after Backdoor and Exploit were added the "Top_Backdoors"
# dashboard is no longer valid. Produce an "empty" response.
response = {'data': []}
obj_type = None
return parseDocumentsForW2ui(response, obj_type)
def get_table_data(request=None,obj=None,user=None,searchTerm="",
search_type=None, includes=[], excludes=[], maxRows=25,
sort={}, pageNumber=1):
"""
gets the records needed for the table, can be called via ajax on the
saved_search.html or the above ConstructTable function
"""
from crits.core.handlers import get_query, data_query
response = {"Result": "ERROR"}
obj_type = get_obj_type_from_string(obj)
# Build the query
term = ""
#if its being called from saved_search.html
if request and request.is_ajax():
resp = get_query(obj_type, request)
#if its calling to get data for the dashbaord
elif user and search_type:
resp = get_query_without_request(obj_type, user.username, searchTerm, search_type)
else:
return HttpResponse(json.dumps(response, default=json_handler),
mimetype='application/json')
if resp['Result'] in ["ERROR", "IGNORE"]:
return resp
query = resp['query']
term = resp['term']
sortBy = []
if 'direction' in sort:
if sort['direction'] == 'asc':
sortBy.append(sort['field'])
elif sort['direction'] == 'desc':
sortBy.append("-"+sort['field'])
skip = (int(pageNumber)-1)*25
if request:
response = data_query(obj_type, user=request.user.username, query=query,
projection=includes, limit=int(maxRows), sort=sortBy, skip=skip)
else:
response = data_query(obj_type, user=user.username, query=query,
projection=includes, limit=maxRows, sort=sortBy,skip=skip)
if response['result'] == "ERROR":
return {'Result': "ERROR", 'Message': response['msg']}
response['crits_type'] = obj_type
# Escape term for rendering in the UI.
response['term'] = cgi.escape(term)
response['data'] = response['data'].to_dict(excludes, includes)
response['Records'] = parseDocObjectsToStrings(response.pop('data'), obj)
response['TotalRecordCount'] = response.pop('count')
response['Result'] = response.pop('result')
if request:
return HttpResponse(json.dumps(response, default=json_handler),
mimetype='application/json')
else:
return response
def get_table_data(request=None,obj=None,user=None,searchTerm="",
search_type=None, includes=[], excludes=[], maxRows=25,
sort={}, pageNumber=1):
"""
gets the records needed for the table, can be called via ajax on the
saved_search.html or the above ConstructTable function
"""
from crits.core.handlers import get_query, data_query
response = {"Result": "ERROR"}
obj_type = get_obj_type_from_string(obj)
# Build the query
term = ""
#if its being called from saved_search.html
if request and request.is_ajax():
resp = get_query(obj_type, request)
#if its calling to get data for the dashbaord
elif user and search_type:
resp = get_query_without_request(obj_type, user.username, searchTerm, search_type)
else:
return HttpResponse(json.dumps(response, default=json_handler),
content_type="application/json")
if resp['Result'] in ["ERROR", "IGNORE"]:
return resp
query = resp['query']
term = resp['term']
sortBy = []
if 'direction' in sort:
if sort['direction'] == 'asc':
sortBy.append(sort['field'])
elif sort['direction'] == 'desc':
sortBy.append("-"+sort['field'])
skip = (int(pageNumber)-1)*25
if request:
response = data_query(obj_type, user=request.user.username, query=query,
projection=includes, limit=int(maxRows), sort=sortBy, skip=skip)
else:
response = data_query(obj_type, user=user.username, query=query,
projection=includes, limit=maxRows, sort=sortBy,skip=skip)
if response['result'] == "ERROR":
return {'Result': "ERROR", 'Message': response['msg']}
response['crits_type'] = obj_type
# Escape term for rendering in the UI.
response['term'] = cgi.escape(term)
response['data'] = response['data'].to_dict(excludes, includes)
response['Records'] = parseDocObjectsToStrings(response.pop('data'), obj)
response['TotalRecordCount'] = response.pop('count')
response['Result'] = response.pop('result')
if request:
return HttpResponse(json.dumps(response, default=json_handler),
content_type="application/json")
else:
return response
def generate_search_for_saved_table(user, id=None, request=None):
"""
Called by edit_save_search in views.py. This is for editing a previously
saved table or one of the default dashboard tables
"""
from crits.core.handlers import data_query
response = {}
savedSearch = None
try:
savedSearch = SavedSearch.objects(id=id).first()
if not savedSearch:
response['Result'] = "ERROR"
response[
'Message'] = "Error finding table, please try again later."
return response
except:
savedSearch = SavedSearch()
savedSearch.isDefaultOnDashboard = True
savedSearch.name = id.replace("_", " ")
id = None
results = []
records = []
term = ""
url = ""
if not savedSearch.isDefaultOnDashboard:
objType = get_obj_type_from_string(savedSearch.objType)
resp = get_query_without_request(objType, user, savedSearch.searchTerm,
"global")
if resp['Result'] == "ERROR":
return resp
formatted_query = resp['query']
term = resp['term']
resp = data_query(objType, user, query=formatted_query, count=True)
results.append({'count': resp['count'], 'name': savedSearch.objType})
else:
results = {
"name": savedSearch.name,
"count": str(len(records)),
"type": get_obj_name_from_title(savedSearch.name)
}
#special url to get the records of a default dashboard since their queries are different
url = reverse(
"crits.dashboards.views.get_dashboard_table_data",
kwargs={"tableName": str(savedSearch.name.replace(" ", "_"))})
args = {'term': term, 'results': results, 'dataUrl': url, 'Result': "OK"}
if savedSearch:
args.update({
'tableId': id,
'tableName': savedSearch.name,
'columns': savedSearch.tableColumns,
'sortBy': savedSearch.sortBy,
'sizex': savedSearch.sizex,
'maxRows': savedSearch.maxRows,
'isDefaultOnDashboard': savedSearch.isDefaultOnDashboard,
})
if savedSearch.dashboard:
args["currentDash"] = str(savedSearch.dashboard)
args["dashtheme"] = Dashboard.objects(
id=savedSearch.dashboard).first().theme
return args
def generate_search_for_saved_table(user, id=None,request=None):
"""
Called by edit_save_search in views.py. This is for editing a previously
saved table or one of the default dashboard tables
"""
from crits.core.handlers import data_query
response = {}
savedSearch = None
try:
savedSearch = SavedSearch.objects(id=id).first()
if not savedSearch:
response['Result'] = "ERROR"
response['Message'] = "Error finding table, please try again later."
return response
except:
savedSearch = SavedSearch()
savedSearch.isDefaultOnDashboard = True
savedSearch.name = id.replace("_", " ")
id = None
results = []
records = []
term = ""
url = ""
if not savedSearch.isDefaultOnDashboard:
objType = get_obj_type_from_string(savedSearch.objType)
resp = get_query_without_request(objType, user, savedSearch.searchTerm, "global")
if resp['Result'] == "ERROR":
return resp
formatted_query = resp['query']
term = resp['term']
resp = data_query(objType, user, query=formatted_query, count=True)
results.append({'count': resp['count'],
'name': savedSearch.objType})
else:
results = {"name":savedSearch.name,
"count":str(len(records)),
"type":get_obj_name_from_title(savedSearch.name)}
#special url to get the records of a default dashboard since their queries are different
url = reverse("crits-dashboards-views-get_dashboard_table_data",
kwargs={"tableName":str(savedSearch.name.replace(" ", "_"))})
args = {'term': term,
'results': results,
'dataUrl':url,
'Result': "OK"
}
if savedSearch:
args.update({'tableId':id,
'tableName': savedSearch.name,
'columns': savedSearch.tableColumns,
'sortBy': savedSearch.sortBy,
'sizex' : savedSearch.sizex,
'maxRows': savedSearch.maxRows,
'isDefaultOnDashboard': savedSearch.isDefaultOnDashboard,
})
if savedSearch.dashboard:
args["currentDash"] = str(savedSearch.dashboard)
args["dashtheme"] = Dashboard.objects(id=savedSearch.dashboard).first().theme
return args
def getRecordsForDefaultDashboardTable(username, tableName):
"""
Called by createTableObject to retrieve the proper records from the
database for the default dashboard tables. These queries are different then
the saved searches which is why it is needed.
This is also called via ajax on the saved_search.html page by
get_dashboard_table_data in Views.py. This is to get the records when
editing the default tables.
"""
from crits.core.handlers import data_query, generate_counts_jtable
if tableName == "Recent_Samples" or tableName == "Recent Samples":
obj_type = "Sample"
response = data_query(Sample,
username,
query={},
sort=["-created"],
limit=5)
elif tableName == "Recent_Emails" or tableName == "Recent Emails":
obj_type = "Email"
response = data_query(Email,
username,
query={},
sort=["-isodate"],
limit=5)
elif tableName == "Recent_Indicators" or tableName == "Recent Indicators":
obj_type = "Indicator"
response = data_query(Indicator,
username,
query={},
sort=["-created"],
limit=5)
elif tableName == "Top_Campaigns" or tableName == "Top Campaigns":
obj_type = "Campaign"
response = data_query(Campaign, username, query={}, limit=5)
elif tableName == "Top_Backdoors" or tableName == "Top Backdoors":
obj_type = "Backdoor"
response = data_query(Backdoor, username, query={}, limit=5)
elif tableName == "Counts":
response = generate_counts_jtable(None, "jtlist")
records = json.loads(response.content)["Records"]
for record in records:
record["recid"] = record.pop("id")
return records
return parseDocumentsForW2ui(response, obj_type)
def getRecordsForDefaultDashboardTable(user, tableName):
"""
Called by createTableObject to retrieve the proper records from the
database for the default dashboard tables. These queries are different then
the saved searches which is why it is needed.
This is also called via ajax on the saved_search.html page by
get_dashboard_table_data in Views.py. This is to get the records when
editing the default tables.
"""
from crits.core.handlers import data_query, generate_counts_jtable
if tableName == "Recent_Samples" or tableName == "Recent Samples" and user.has_access_to(SampleACL.READ):
obj_type = "Sample"
response = data_query(Sample, user, query={}, sort=["-created"], limit=5)
elif tableName == "Recent_Emails" or tableName == "Recent Emails" and user.has_access_to(EmailACL.READ):
obj_type = "Email"
response = data_query(Email, user, query={}, sort=["-isodate"], limit=5)
elif tableName == "Recent_Indicators" or tableName == "Recent Indicators" and user.has_access_to(IndicatorACL.READ):
obj_type = "Indicator"
response = data_query(Indicator, user, query={}, sort=["-created"], limit=5)
elif tableName == "Top_Campaigns" or tableName == "Top Campaigns" and user.has_access_to(CampaignACL.READ):
obj_type = "Campaign"
response = data_query(Campaign, user, query={}, limit=5)
elif tableName == "Counts":
response = generate_counts_jtable(None, "jtlist")
records = json.loads(response.content)["Records"]
for record in records:
record["recid"] = record.pop("id")
return records
else:
# This only happens if we have a dashboard which is no longer valid.
# For example, after Backdoor and Exploit were added the "Top_Backdoors"
# dashboard is no longer valid. Produce an "empty" response.
response = {'data': []}
obj_type = None
return parseDocumentsForW2ui(response, obj_type)
def testDataQuery(self):
"""
Test data_query from handlers.py
data_query(col_obj,user[,limit,skip,sort,query,projection])
"""
obj = TestObject
resp = handlers.data_query(obj, self.user.username)
self.assertEqual(resp['count'], 1)
self.assertEqual(resp['result'], 'OK')
self.assertEqual(resp['crits_type'], 'TestBase')
self.assertEqual(resp['msg'], '')
self.assertTrue(isinstance(resp['data'], CritsQuerySet))
self.assertEqual(resp['data'][0].name, TOBJ_NAME)
self.assertEqual(resp['data'][0].value, TOBJ_VALUE)
self.assertEqual(resp['data'][0]._meta['crits_type'], "TestBase")
def testDataQuery(self):
"""
Test data_query from handlers.py
data_query(col_obj,user[,limit,skip,sort,query,projection])
"""
obj = TestObject
resp = handlers.data_query(obj, self.user.username)
self.assertEqual(resp['count'], 1)
self.assertEqual(resp['result'], 'OK')
self.assertEqual(resp['crits_type'], 'TestBase')
self.assertEqual(resp['msg'], '')
self.assertTrue(isinstance(resp['data'], CritsQuerySet))
self.assertEqual(resp['data'][0].name, TOBJ_NAME)
self.assertEqual(resp['data'][0].value, TOBJ_VALUE)
self.assertEqual(resp['data'][0]._meta['crits_type'], "TestBase")
