def generate_alerts(self, search_start_time):
alerts = {}
#alerts["HIDDEN ELEMENT"] = self.get_hidden_iframes_css(search_start_time)
alerts["HIDDEN ELEMENT"] = self.get_hidden_iframes(search_start_time)
alerts["PROFILING SCRIPT"] = self.get_cart_id_injections(
search_start_time)
alerts["SCANBOX FRAMEWORK"] = self.get_scanbox_injections(
search_start_time)
for alert in alerts:
for elem in alerts[alert]:
if hasattr(elem,
'uri') and (elem.domain == None
or not self.check_whitelist(elem.uri)):
print "alert for ", elem.raw
infected_page = Page.objects.get(
Q(event_time__gte=search_start_time),
page_id=elem.page_id)
infected_page_url = ""
if infected_page is not None:
infected_page_url = infected_page.uri
a = Alert(reason=alert,
raw=elem.raw,
uri=elem.uri,
page=infected_page_url,
page_id=elem.page_id,
org_id=elem.org_id,
event_time=elem.event_time)
a.save()
alerts_nocheck = {}
if settings.ENABLE_EMAIL_ALERTS:
alerts_nocheck["EMAIL DISCLOSURE"] = self.get_email_disclosures(
search_start_time)
print "here"
alerts_nocheck["SUSPICIOUS SCRIPT"] = self.get_pastebin_injections(
search_start_time)
alerts_nocheck["WEBSHELL INJECTION"] = self.get_shell_injections(
search_start_time)
alerts_nocheck["VBSCRIPT INJECTION"] = self.get_vbscript_injections(
search_start_time)
alerts_nocheck["EVERCOOKIE SCRIPT"] = self.get_evercookie_scripts(
search_start_time)
for alert in alerts_nocheck:
for elem in alerts_nocheck[alert]:
print elem.raw
infected_page = Page.objects.get(
Q(event_time__gte=search_start_time), page_id=elem.page_id)
infected_page_url = ""
if infected_page is not None:
infected_page_url = infected_page.uri
a = Alert(reason=alert,
raw=elem.raw,
uri=elem.uri,
page=infected_page_url,
page_id=elem.page_id,
org_id=elem.org_id,
event_time=elem.event_time)
a.save()
def test_correct_admin_levels(self):
survey = HouseholdSurveyJSON.objects.create(
team_lead=self.team_member,
team_assistant=self.team_member,
team_anthropometrist=self.team_member,
household_number=13,
location=Point(52.503713, 13.424559),
first_admin_level='State 1',
second_admin_level='County 1',
cluster=1
)
survey.save()
result_first_admin = list(Alert.mapping_check_wrong_location_first_admin_level(survey))
self.assertEqual(result_first_admin, [])
result_second_admin = list(Alert.mapping_check_wrong_location_second_admin_level(survey))
self.assertEqual(result_second_admin, [])
def test_correct_admin_levels(self):
survey = HouseholdSurveyJSON.objects.create(
team_lead=self.team_member,
team_assistant=self.team_member,
team_anthropometrist=self.team_member,
household_number=13,
location=Point(52.503713, 13.424559),
first_admin_level='State 1',
second_admin_level='County 1',
cluster=1
)
survey.save()
result_first_admin = list(Alert.mapping_check_wrong_location_first_admin_level(survey))
self.assertEqual(result_first_admin, [])
result_second_admin = list(Alert.mapping_check_wrong_location_second_admin_level(survey))
self.assertEqual(result_second_admin, [])
def test_incorrect_second_admin_level(self):
survey = HouseholdSurveyJSON.objects.create(
team_lead=self.team_member,
team_assistant=self.team_member,
team_anthropometrist=self.team_member,
household_number=14,
location=Point(16.629403, 145.876453),
first_admin_level='State 1',
second_admin_level='County 1',
cluster=2
)
survey.save()
result_first_admin = list(Alert.mapping_check_wrong_location_first_admin_level(survey))
self.assertEqual(result_first_admin, [])
result_second_admin = list(Alert.mapping_check_wrong_location_second_admin_level(survey))
self.assertEqual(len(result_second_admin), 1)
self.assertEqual(result_second_admin[0]['alert_type'], 'mapping_check_wrong_location_second_admin_level')
def test_incorrect_second_admin_level(self):
survey = HouseholdSurveyJSON.objects.create(
team_lead=self.team_member,
team_assistant=self.team_member,
team_anthropometrist=self.team_member,
household_number=14,
location=Point(16.629403, 145.876453),
first_admin_level='State 1',
second_admin_level='County 1',
cluster=2
)
survey.save()
result_first_admin = list(Alert.mapping_check_wrong_location_first_admin_level(survey))
self.assertEqual(result_first_admin, [])
result_second_admin = list(Alert.mapping_check_wrong_location_second_admin_level(survey))
self.assertEqual(len(result_second_admin), 1)
self.assertEqual(result_second_admin[0]['alert_type'], 'mapping_check_wrong_location_second_admin_level')
def generate_alerts(self, search_start_time):
alerts = {}
#alerts["HIDDEN ELEMENT"] = self.get_hidden_iframes_css(search_start_time)
alerts["HIDDEN ELEMENT"] = self.get_hidden_iframes(search_start_time)
alerts["PROFILING SCRIPT"] = self.get_cart_id_injections(search_start_time)
alerts["SCANBOX FRAMEWORK"] = self.get_scanbox_injections(search_start_time)
for alert in alerts:
for elem in alerts[alert]:
if hasattr(elem, 'uri') and (elem.domain == None or not self.check_whitelist(elem.uri)):
print "alert for ", elem.raw
infected_page = Page.objects.get(Q(event_time__gte=search_start_time),page_id=elem.page_id)
infected_page_url = ""
if infected_page is not None:
infected_page_url = infected_page.uri
a = Alert(reason=alert, raw=elem.raw, uri=elem.uri, page=infected_page_url, page_id=elem.page_id, org_id=elem.org_id, event_time=elem.event_time)
a.save()
alerts_nocheck = {}
if settings.ENABLE_EMAIL_ALERTS:
alerts_nocheck["EMAIL DISCLOSURE"] = self.get_email_disclosures(search_start_time)
print "here"
alerts_nocheck["SUSPICIOUS SCRIPT"] = self.get_pastebin_injections(search_start_time)
alerts_nocheck["WEBSHELL INJECTION"] = self.get_shell_injections(search_start_time)
alerts_nocheck["VBSCRIPT INJECTION"] = self.get_vbscript_injections(search_start_time)
alerts_nocheck["EVERCOOKIE SCRIPT"] = self.get_evercookie_scripts(search_start_time)
for alert in alerts_nocheck:
for elem in alerts_nocheck[alert]:
print elem.raw
infected_page = Page.objects.get(Q(event_time__gte=search_start_time),page_id=elem.page_id)
infected_page_url = ""
if infected_page is not None:
infected_page_url = infected_page.uri
a = Alert(reason=alert, raw=elem.raw, uri=elem.uri, page=infected_page_url, page_id=elem.page_id, org_id=elem.org_id, event_time=elem.event_time)
a.save()
def test_missing_data_alert(self):
survey = HouseholdSurveyJSON.objects.create(
team_lead=self.team_member,
team_assistant=self.team_member,
team_anthropometrist=self.team_member,
household_number=12,
location=Point(52.503713, 13.424559),
start_time=datetime(2010, 1, 1),
)
survey.save()
self.woman, created_woman = HouseholdMember.objects.get_or_create(
index=1,
household_survey=survey,
birthdate=datetime(1970, 1, 1),
gender='F',
height=23,
weight=199,
)
self.woman2, created_woman2 = HouseholdMember.objects.get_or_create(
index=2,
household_survey=survey,
birthdate=datetime(1971, 1, 1),
gender='F',
height=24,
muac=198,
)
result = list(Alert.missing_data_alert(survey))
self.assertEqual(len(result), 3)
alert_titles = [a['text'] for a in result]
self.maxDiff = 2322
self.assertEqual(alert_titles, [
'Missing data issue on field muac for women in team {}'.format(
self.team_member.pk),
'Missing data issue on field edema for women in team {}'.format(
self.team_member.pk),
'Missing data issue on field weight for women in team {}'.format(
self.team_member.pk),
])
self.assertTrue(
'Missing data issue on field height for women in team {}'.format(
self.team_member.pk) not in alert_titles)
def test_missing_data_alert(self):
survey = HouseholdSurveyJSON.objects.create(
team_lead=self.team_member,
team_assistant=self.team_member,
team_anthropometrist=self.team_member,
household_number=12,
location=Point(52.503713, 13.424559),
start_time=datetime(2010, 1, 1),
)
survey.save()
self.woman, created_woman = HouseholdMember.objects.get_or_create(
index=1,
household_survey=survey,
birthdate=datetime(1970, 1, 1),
gender='F',
height=23,
weight=199,
)
self.woman2, created_woman2 = HouseholdMember.objects.get_or_create(
index=2,
household_survey=survey,
birthdate=datetime(1971, 1, 1),
gender='F',
height=24,
muac=198,
)
result = list(Alert.missing_data_alert(survey))
self.assertEqual(len(result), 3)
alert_titles = [a['text'] for a in result]
self.maxDiff = 2322
self.assertEqual(alert_titles, [
'Missing data issue on field muac for women in team {}'.format(self.team_member.pk),
'Missing data issue on field edema for women in team {}'.format(self.team_member.pk),
'Missing data issue on field weight for women in team {}'.format(self.team_member.pk),
])
self.assertTrue(
'Missing data issue on field height for women in team {}'.format(
self.team_member.pk) not in alert_titles
)
def test_mapping_check_missing_cluster(self):
survey = HouseholdSurveyJSON.objects.create(
team_lead=self.team_member,
team_assistant=self.team_member,
team_anthropometrist=self.team_member,
household_number=12,
location=Point(52.503713, 13.424559),
)
survey.save()
result = list(Alert.mapping_check_missing_cluster(survey))
self.assertEqual(result[0]['category'], 'map')
self.assertEqual(result[0]['text'], 'No cluster ID for survey of team {} (survey {})'.format(self.team_member.pk, survey.pk))
self.assertEqual(result[0]['survey'], survey)
self.assertEqual(result[0]['team_lead'], self.team_member)
def test_mapping_check_missing_cluster(self):
survey = HouseholdSurveyJSON.objects.create(
team_lead=self.team_member,
team_assistant=self.team_member,
team_anthropometrist=self.team_member,
household_number=12,
location=Point(52.503713, 13.424559),
)
survey.save()
result = list(Alert.mapping_check_missing_cluster(survey))
self.assertEqual(result[0]['category'], 'map')
self.assertEqual(result[0]['text'], 'No cluster ID for survey of team {} (survey {})'.format(self.team_member.pk, survey.pk))
self.assertEqual(result[0]['survey'], survey)
self.assertEqual(result[0]['team_lead'], self.team_member)
def import_csvfile(self, csvfile):
last_10_seconds = None
imported_last_10_seconds = 0
headers = None
for row_no, row in enumerate(csv.reader(csvfile, delimiter=',')):
if not headers:
# first row is the headers
headers = row
continue
rawdata = get_rawdata(headers, row)
parsed = parse_flat_formhub_csv(rawdata)
members = find_household_members(parsed)
try:
household_survey = HouseholdSurveyJSON(
uuid=parsed['_uuid'],
household_number=parsed['hh_number'],
first_admin_level=parsed['state'],
second_admin_level=parsed['lga'],
cluster=parsed['cluster'],
cluster_name=parsed['cluster_name'],
start_time=parsed['starttime'],
end_time=parsed['endtime'],
location=Point(parsed['_gps_latitude'], parsed['_gps_longitude']),
json={
"uuid": parsed['_uuid'],
"syncDate": parsed['_submission_time'] + ".000Z",
"startTime": parsed['starttime'],
"endTime": parsed['endtime'],
"created": parsed['_submission_time'] + ".000Z",
"modified": parsed['_submission_time'],
"householdID": parsed['hh_number'],
"cluster": parsed['cluster'],
"cluster_name": parsed['cluster_name'],
"first_admin_level": parsed['state'],
"second_admin_level": parsed['lga'],
"location": [
parsed['_gps_latitude'],
parsed['_gps_longitude']
],
"members": (household_member_to_legacy_format(dict(member.items())) for member in members),
"team_num": parsed['team_num'],
"team": FakeTeams.objects.get_or_create(
team_id=parsed['team_num']
)[0].json,
"_id": parsed['_uuid'],
"tools": {},
"history": []
}
)
household_survey.parse_and_set_team_members()
household_survey.save()
startTime = datetime.strptime(parsed['starttime'].split('.')[0], '%Y-%m-%dT%H:%M:%S')
create_household_member_models(household_survey, members, startTime)
except (KeyError, IntegrityError) as e:
logging.error('%r', parsed)
logging.exception(e)
update_mapping_documents_from_new_survey(parsed)
Alert.run_alert_checks_on_document(household_survey)
if datetime.now().second / 10 != last_10_seconds:
print dedent("""
===> imported {} records in the last 10 seconds, that is {} records/s
""").format(imported_last_10_seconds, imported_last_10_seconds / 10.0)
last_10_seconds = datetime.now().second / 10
imported_last_10_seconds = 0
else:
imported_last_10_seconds += 1
print '[{}]'.format(datetime.now()), row_no, 'created', parsed['_uuid'], len(members), 'household members'
def import_csvfile(self, csvfile):
last_10_seconds = None
imported_last_10_seconds = 0
headers = None
for row_no, row in enumerate(csv.reader(csvfile, delimiter=',')):
if not headers:
# first row is the headers
headers = row
continue
rawdata = get_rawdata(headers, row)
parsed = parse_flat_formhub_csv(rawdata)
members = find_household_members(parsed)
try:
household_survey = HouseholdSurveyJSON(
uuid=parsed['_uuid'],
household_number=parsed['hh_number'],
first_admin_level=parsed['state'],
second_admin_level=parsed['lga'],
cluster=parsed['cluster'],
cluster_name=parsed['cluster_name'],
start_time=parsed['starttime'],
end_time=parsed['endtime'],
location=Point(parsed['_gps_latitude'],
parsed['_gps_longitude']),
json={
"uuid":
parsed['_uuid'],
"syncDate":
parsed['_submission_time'] + ".000Z",
"startTime":
parsed['starttime'],
"endTime":
parsed['endtime'],
"created":
parsed['_submission_time'] + ".000Z",
"modified":
parsed['_submission_time'],
"householdID":
parsed['hh_number'],
"cluster":
parsed['cluster'],
"cluster_name":
parsed['cluster_name'],
"first_admin_level":
parsed['state'],
"second_admin_level":
parsed['lga'],
"location":
[parsed['_gps_latitude'], parsed['_gps_longitude']],
"members": (household_member_to_legacy_format(
dict(member.items())) for member in members),
"team_num":
parsed['team_num'],
"team":
FakeTeams.objects.get_or_create(
team_id=parsed['team_num'])[0].json,
"_id":
parsed['_uuid'],
"tools": {},
"history": []
})
household_survey.parse_and_set_team_members()
household_survey.save()
startTime = datetime.strptime(
parsed['starttime'].split('.')[0], '%Y-%m-%dT%H:%M:%S')
create_household_member_models(household_survey, members,
startTime)
except (KeyError, IntegrityError) as e:
logging.error('%r', parsed)
logging.exception(e)
update_mapping_documents_from_new_survey(parsed)
Alert.run_alert_checks_on_document(household_survey)
if datetime.now().second / 10 != last_10_seconds:
print dedent("""
===> imported {} records in the last 10 seconds, that is {} records/s
""").format(imported_last_10_seconds,
imported_last_10_seconds / 10.0)
last_10_seconds = datetime.now().second / 10
imported_last_10_seconds = 0
else:
imported_last_10_seconds += 1
print '[{}]'.format(
datetime.now()), row_no, 'created', parsed['_uuid'], len(
members), 'household members'
