def test_correct_complaint_cap (self, testapp):
''' New complaint data from the extractor is processed as expected.
'''
# Set up the extractor
department = Department.create(name="IM Police Department", short_name="IMPD", load_defaults=False)
extractor, envs = Extractor.from_department_and_password(department=department, password="******")
# Set the correct authorization
testapp.authorization = ('Basic', (extractor.username, 'password'))
# Get a generated list of complaint descriptions from the JSON test client
test_client = JSONTestClient()
complaint_count = 1
complaint_data = test_client.get_prebaked_complaints(last=complaint_count)
complaint_data[0]["allegation"] = "Rude, demeaning, or affronting language"
# post the json to the complaint URL
response = testapp.post_json("/data/complaints", params={'month': 0, 'year': 0, 'data': complaint_data})
# assert that we got the expected reponse
assert response.status_code == 200
assert response.json_body['updated'] == 0
assert response.json_body['added'] == complaint_count
# check the complaint incident in the database against the data that was sent
cleaner = Cleaners()
sent_complaint = cleaner.capitalize_incident(complaint_data[0])
check_complaint = CitizenComplaintIMPD.query.filter_by(opaque_id=sent_complaint['opaqueId']).first()
assert check_complaint.allegation == "Rude, Demeaning, or Affronting Language"
with open("scratch.txt", "w") as text_file:
text_file.write("Complaint Data: {} ".format(check_complaint.allegation))
def test_post_assaults_data(self, testapp):
''' New assaults data from the extractor is processed as expected.
'''
# Set up the extractor
department = Department.create(name="IM Police Department", short_name="IMPD", load_defaults=False)
extractor, envs = Extractor.from_department_and_password(department=department, password="******")
# Set the correct authorization
testapp.authorization = ('Basic', (extractor.username, 'password'))
# Get a generated list of assault descriptions from the JSON test client
test_client = JSONTestClient()
assault_count = 1
assault_data = test_client.get_prebaked_assaults(last=assault_count)
# post the json to the assault URL
response = testapp.post_json("/data/assaults", params={'month': 0, 'year': 0, 'data': assault_data})
# assert that we got the expected reponse
assert response.status_code == 200
assert response.json_body['updated'] == 0
assert response.json_body['added'] == assault_count
# check the assault incident in the database against the data that was sent
cleaner = Cleaners()
sent_assault = cleaner.capitalize_incident(assault_data[0])
check_assault = AssaultOnOfficerIMPD.query.filter_by(opaque_id=sent_assault['opaqueId']).first()
assert check_assault.service_type == sent_assault['serviceType']
assert check_assault.force_type == sent_assault['forceType']
assert check_assault.assignment == sent_assault['assignment']
assert check_assault.arrest_made == sent_assault['arrestMade']
assert check_assault.officer_injured == sent_assault['officerInjured']
assert check_assault.officer_killed == sent_assault['officerKilled']
assert check_assault.report_filed == sent_assault['reportFiled']
def test_post_mistyped_uof_data(self, testapp):
''' New UOF data from the extractor is processed as expected.
'''
# Set up the extractor
department = Department.create(name="Good Police Department", short_name="GPD", load_defaults=False)
extractor, envs = Extractor.from_department_and_password(department=department, password="******")
# Set the correct authorization
testapp.authorization = ('Basic', (extractor.username, 'password'))
# Get a generated list of UOF descriptions from the JSON test client
test_client = JSONTestClient()
uof_count = 1
uof_data = test_client.get_prebaked_uof(last=uof_count)
# The app expects number values to be transmitted as strings. Let's change them to integers.
uof_data[0]['residentAge'] = 28
uof_data[0]['officerAge'] = 46
uof_data[0]['officerYearsOfService'] = 17
# post the json to the UOF URL
response = testapp.post_json("/data/UOF", params={'month': 0, 'year': 0, 'data': uof_data})
# assert that we got the expected reponse
assert response.status_code == 200
assert response.json_body['updated'] == 0
assert response.json_body['added'] == uof_count
# check the uof incident in the database against the data that was sent
cleaner = Cleaners()
sent_uof = uof_data[0]
check_uof = UseOfForceIncident.query.filter_by(opaque_id=sent_uof['opaqueId']).first()
assert check_uof.occured_date.strftime('%Y-%m-%d %-H:%-M:%S') == sent_uof['occuredDate']
assert check_uof.division == cleaner.capitalize(sent_uof['division'])
assert check_uof.precinct == cleaner.capitalize(sent_uof['precinct'])
assert check_uof.shift == cleaner.capitalize(sent_uof['shift'])
assert check_uof.beat == cleaner.capitalize(sent_uof['beat'])
assert check_uof.disposition == sent_uof['disposition']
assert check_uof.officer_force_type == cleaner.officer_force_type(sent_uof['officerForceType'])
assert check_uof.use_of_force_reason == sent_uof['useOfForceReason']
assert check_uof.service_type == sent_uof['serviceType']
assert check_uof.arrest_made == sent_uof['arrestMade']
assert check_uof.arrest_charges == sent_uof['arrestCharges']
assert check_uof.resident_weapon_used == sent_uof['residentWeaponUsed']
assert check_uof.resident_injured == sent_uof['residentInjured']
assert check_uof.resident_hospitalized == sent_uof['residentHospitalized']
assert check_uof.officer_injured == sent_uof['officerInjured']
assert check_uof.officer_hospitalized == sent_uof['officerHospitalized']
assert check_uof.resident_race == cleaner.race(sent_uof['residentRace'])
assert check_uof.resident_sex == cleaner.sex(sent_uof['residentSex'])
assert check_uof.resident_age == cleaner.number_to_string(sent_uof['residentAge'])
assert check_uof.resident_condition == sent_uof['residentCondition']
assert check_uof.officer_identifier == sent_uof['officerIdentifier']
assert check_uof.officer_race == cleaner.race(sent_uof['officerRace'])
assert check_uof.officer_sex == cleaner.sex(sent_uof['officerSex'])
assert check_uof.officer_age == cleaner.number_to_string(sent_uof['officerAge'])
assert check_uof.officer_years_of_service == cleaner.number_to_string(sent_uof['officerYearsOfService'])
assert check_uof.officer_condition == sent_uof['officerCondition']
def test_post_mistyped_ois_data(self, testapp):
''' New OIS data from the extractor is processed as expected.
'''
# Set up the extractor
department = Department.create(name="Good Police Department", short_name="GPD", load_defaults=False)
extractor, envs = Extractor.from_department_and_password(department=department, password="******")
# Set the correct authorization
testapp.authorization = ('Basic', (extractor.username, 'password'))
# Get a generated list of OIS descriptions from the JSON test client
test_client = JSONTestClient()
ois_count = 1
ois_data = test_client.get_prebaked_ois(last=ois_count)
# The app expects number values to be transmitted as strings. Let's change them to integers.
ois_data[0]['residentAge'] = 28
ois_data[0]['officerAge'] = 46
# And it expects this number value to be transmitted as a number, so let's make it a string.
ois_data[0]['officerYearsOfService'] = "17"
# post the json to the OIS URL
response = testapp.post_json("/data/OIS", params={'month': 0, 'year': 0, 'data': ois_data})
# assert that we got the expected reponse
assert response.status_code == 200
assert response.json_body['updated'] == 0
assert response.json_body['added'] == ois_count
# check the ois incident in the database against the data that was sent
cleaner = Cleaners()
sent_ois = ois_data[0]
check_ois = OfficerInvolvedShooting.query.filter_by(opaque_id=sent_ois['opaqueId']).first()
assert check_ois.occured_date.strftime('%Y-%m-%d %-H:%-M:%S') == sent_ois['occuredDate']
assert check_ois.division == cleaner.capitalize(sent_ois['division'])
assert check_ois.precinct == cleaner.capitalize(sent_ois['precinct'])
assert check_ois.shift == cleaner.capitalize(sent_ois['shift'])
assert check_ois.beat == cleaner.capitalize(sent_ois['beat'])
assert check_ois.disposition == sent_ois['disposition']
assert check_ois.resident_race == cleaner.race(sent_ois['residentRace'])
assert check_ois.resident_sex == cleaner.sex(sent_ois['residentSex'])
assert check_ois.resident_age == cleaner.number_to_string(sent_ois['residentAge'])
assert check_ois.resident_weapon_used == cleaner.resident_weapon_used(sent_ois['residentWeaponUsed'])
assert check_ois.resident_condition == sent_ois['residentCondition']
assert check_ois.officer_identifier == sent_ois['officerIdentifier']
assert check_ois.officer_weapon_used == sent_ois['officerForceType']
assert check_ois.officer_race == cleaner.race(sent_ois['officerRace'])
assert check_ois.officer_sex == cleaner.sex(sent_ois['officerSex'])
assert check_ois.officer_age == cleaner.number_to_string(sent_ois['officerAge'])
assert check_ois.officer_years_of_service == cleaner.string_to_integer(sent_ois['officerYearsOfService'])
assert check_ois.officer_condition == sent_ois['officerCondition']
def test_post_mistyped_complaint_data(self, testapp):
''' New complaint data from the extractor with wrongly typed data is processed as expected.
'''
# Set up the extractor
department = Department.create(name="Good Police Department", short_name="GPD", load_defaults=False)
extractor, envs = Extractor.from_department_and_password(department=department, password="******")
# Set the correct authorization
testapp.authorization = ('Basic', (extractor.username, 'password'))
# Get a generated list of complaint descriptions from the JSON test client
test_client = JSONTestClient()
complaint_count = 1
complaint_data = test_client.get_prebaked_complaints(last=complaint_count)
# The app expects number values to be transmitted as strings. Let's change them to integers.
complaint_data[0]['residentAge'] = 28
complaint_data[0]['officerAge'] = 46
complaint_data[0]['officerYearsOfService'] = 17
# post the json to the complaint URL
response = testapp.post_json("/data/complaints", params={'month': 0, 'year': 0, 'data': complaint_data})
# assert that we got the expected reponse
assert response.status_code == 200
assert response.json_body['updated'] == 0
assert response.json_body['added'] == complaint_count
# check the complaint incident in the database against the data that was sent
cleaner = Cleaners()
sent_complaint = cleaner.capitalize_incident(complaint_data[0])
check_complaint = CitizenComplaint.query.filter_by(opaque_id=sent_complaint['opaqueId']).first()
assert check_complaint.occured_date.strftime('%Y-%m-%d %-H:%-M:%S') == sent_complaint['occuredDate']
assert check_complaint.division == sent_complaint['division']
assert check_complaint.precinct == sent_complaint['precinct']
assert check_complaint.shift == sent_complaint['shift']
assert check_complaint.beat == sent_complaint['beat']
assert check_complaint.disposition == sent_complaint['disposition']
assert check_complaint.service_type == sent_complaint['serviceType']
assert check_complaint.source == sent_complaint['source']
assert check_complaint.allegation_type == sent_complaint['allegationType']
assert check_complaint.allegation == sent_complaint['allegation']
assert check_complaint.resident_race == cleaner.race(sent_complaint['residentRace'])
assert check_complaint.resident_sex == cleaner.sex(sent_complaint['residentSex'])
assert check_complaint.resident_age == cleaner.number_to_string(sent_complaint['residentAge'])
assert check_complaint.officer_identifier == sent_complaint['officerIdentifier']
assert check_complaint.officer_race == cleaner.race(sent_complaint['officerRace'])
assert check_complaint.officer_sex == cleaner.sex(sent_complaint['officerSex'])
assert check_complaint.officer_age == cleaner.number_to_string(sent_complaint['officerAge'])
assert check_complaint.officer_years_of_service == cleaner.number_to_string(sent_complaint['officerYearsOfService'])
def test_client():
delete_everything()
department = Department.create(name="Busy Town Police Department", short_name="BTPD", load_defaults=True)
user = User.create(username="******", email="*****@*****.**", password="******", active=True, is_admin=True)
user.departments.append(department)
user.save()
test_client = JSONTestClient()
# missing_data_mutator = MissingDataMutator()
# fuzzed_data_mutator = FuzzedDataMutator()
# known_bad_data_mutator = KnownBadDataMutator()
# empty_data_mutator = EmptyDataMutator()
# casing_mutator = CasingMutator()
# condenisng_date_mutator = CondenisngDateMutator()
# gap_date_mutator = GapDateMutator()
test_client.run(department, [])
def test_skip_multiple_complaint_data(self, testapp):
''' Multiple complaint data from the extractor is skipped.
'''
# Set up the extractor
department = Department.create(name="Good Police Department", short_name="GPD", load_defaults=False)
extractor, envs = Extractor.from_department_and_password(department=department, password="******")
# Set the correct authorization
testapp.authorization = ('Basic', (extractor.username, 'password'))
# Get a generated list of complaint descriptions from the JSON test client
test_client = JSONTestClient()
complaint_data = test_client.get_prebaked_complaints(last=1)
# post the json to the complaint URL
response = testapp.post_json("/data/complaints", params={'month': 0, 'year': 0, 'data': complaint_data})
# assert that we got the expected reponse
assert response.status_code == 200
assert response.json_body['updated'] == 0
assert response.json_body['added'] == 1
# Get the second pre-baked complaint
multiple_complaint_data = test_client.get_prebaked_complaints(first=1, last=2)
# Swap in the opaque ID from the first complaint
multiple_complaint_data[0]["opaqueId"] = complaint_data[0]["opaqueId"]
# The complaint will be skipped as a 'multiple' if these fields are the same
multiple_complaint_data[0]["allegationType"] = complaint_data[0]["allegationType"]
multiple_complaint_data[0]["allegation"] = complaint_data[0]["allegation"]
multiple_complaint_data[0]["officerIdentifier"] = complaint_data[0]["officerIdentifier"]
# post the json to the complaint URL
response = testapp.post_json("/data/complaints", params={'month': 0, 'year': 0, 'data': multiple_complaint_data})
# assert that we got the expected reponse
assert response.status_code == 200
assert response.json_body['updated'] == 0
assert response.json_body['added'] == 0
# There is one complaint in the database.
all_complaints = CitizenComplaint.query.all()
assert len(all_complaints) == 1
def test_post_ois_data_near_match_does_not_update(self, testapp):
''' OIS data with the same ID but different details creates a new record.
'''
# Set up the extractor
department = Department.create(name="Good Police Department", short_name="GPD", load_defaults=False)
extractor, envs = Extractor.from_department_and_password(department=department, password="******")
# Set the correct authorization
testapp.authorization = ('Basic', (extractor.username, 'password'))
# Get a generated list of OIS descriptions from the JSON test client
test_client = JSONTestClient()
ois_data = test_client.get_prebaked_ois(last=1)
# post the json to the OIS URL
response = testapp.post_json("/data/OIS", params={'month': 0, 'year': 0, 'data': ois_data})
# assert that we got the expected reponse
assert response.status_code == 200
assert response.json_body['updated'] == 0
assert response.json_body['added'] == 1
# Get the second pre-baked ois incident
updated_ois_data = test_client.get_prebaked_ois(first=1, last=2)
# Swap in the opaque ID from the first ois incident
updated_ois_data[0]["opaqueId"] = ois_data[0]["opaqueId"]
# post the json to the ois URL
response = testapp.post_json("/data/OIS", params={'month': 0, 'year': 0, 'data': updated_ois_data})
# assert that we got the expected reponse
assert response.status_code == 200
assert response.json_body['updated'] == 0
assert response.json_body['added'] == 1
# There's only one complaint in the database.
all_ois = OfficerInvolvedShooting.query.all()
assert len(all_ois) == 2
def test_post_uof_data(self, testapp):
""" New and updated UOF data from the extractor is processed as expected.
"""
# Set up the extractor
department = Department.create(name="B Police Department", short_name="LMPD", load_defaults=False)
extractor, _ = Extractor.from_department_and_password(department=department, password="******")
# Set the correct authorization
testapp.authorization = ("Basic", (extractor.username, "password"))
# post to the heartbeat URL to start the update
response = testapp.post_json("/data/heartbeat", params={"heartbeat": "heartbeat"})
# Post 5 fake incidents to the UOF endpoint
uof_count = 5
test_client = JSONTestClient()
uof_data = test_client.make_uof(count=uof_count, short_name=department.short_name)
response = testapp.post_json("/data/UOF", params={"month": 0, "year": 0, "data": uof_data})
# assert that we got the expected reponse
assert response.status_code == 200
# there are 5 incident rows in the database
check_uofs = UseOfForceIncidentLMPD.query.all()
assert len(check_uofs) == uof_count
for incident in uof_data:
# verify that the opaqueIDs posted match those in the database
assert UseOfForceIncidentLMPD.query.filter_by(opaque_id=incident["opaqueId"]).first() is not None
# verify that the opaqueIds are recorded in IncidentsUpdated tables
record_updated = IncidentsUpdated.query.filter_by(opaque_id=incident["opaqueId"]).first()
assert record_updated is not None
assert record_updated.department_id == department.id
assert record_updated.incident_type == "uof"
# post to the heartbeat URL to start the new update
response = testapp.post_json("/data/heartbeat", params={"heartbeat": "heartbeat"})
# Create 5 more fake incidents
new_data = test_client.make_uof(count=uof_count, short_name=department.short_name)
# give them the same opaqueIds as the first batch
for idx, _ in enumerate(new_data):
new_data[idx]["opaqueId"] = uof_data[idx]["opaqueId"]
# post the new incident rows
response = testapp.post_json("/data/UOF", params={"month": 0, "year": 0, "data": new_data})
# assert that we got the expected reponse
assert response.status_code == 200
# there are 5 incident rows in the database
check_uofs = UseOfForceIncidentLMPD.query.all()
assert len(check_uofs) == uof_count
# verify that the opaqueIDs posted match those in the database
for incident in uof_data:
assert UseOfForceIncidentLMPD.query.filter_by(opaque_id=incident["opaqueId"]).first() is not None
# Create 5 more fake incidents
new_data = test_client.make_uof(count=uof_count, short_name=department.short_name)
# give them the same opaqueIds as the first batch
for idx, _ in enumerate(new_data):
new_data[idx]["opaqueId"] = uof_data[idx]["opaqueId"]
# post the new incident rows without starting a new update
response = testapp.post_json("/data/UOF", params={"month": 0, "year": 0, "data": new_data})
# assert that we got the expected reponse
assert response.status_code == 200
# there are 10 incident rows in the database
check_uofs = UseOfForceIncidentLMPD.query.all()
assert len(check_uofs) == uof_count * 2
def test_all_uof_records_destroyed_when_new_record_posted(self, testapp):
""" Posting a new record with an id that matches a set of past records destroys all of them.
"""
# Set up the extractor
department = Department.create(name="B Police Department", short_name="LMPD", load_defaults=False)
extractor, _ = Extractor.from_department_and_password(department=department, password="******")
# Set the correct authorization
testapp.authorization = ("Basic", (extractor.username, "password"))
# post to the heartbeat URL to start the update
response = testapp.post_json("/data/heartbeat", params={"heartbeat": "heartbeat"})
# Post 5 fake incidents with an identical opaqueId to the UOF endpoint
uof_count = 5
test_client = JSONTestClient()
uof_data = test_client.make_uof(count=uof_count, short_name=department.short_name)
use_id = uof_data[0]["opaqueId"]
for idx, _ in enumerate(uof_data):
uof_data[idx]["opaqueId"] = use_id
response = testapp.post_json("/data/UOF", params={"month": 0, "year": 0, "data": uof_data})
# assert that we got the expected reponse
assert response.status_code == 200
# there are 5 incident rows in the database
check_uofs = UseOfForceIncidentLMPD.query.all()
assert len(check_uofs) == uof_count
# all the records in the database have the same opaqueId
uof_records = UseOfForceIncidentLMPD.query.filter_by(opaque_id=use_id).all()
assert len(uof_records) == uof_count
# verify that the opaqueId is recorded in an IncidentsUpdated table
record_updated = IncidentsUpdated.query.filter_by(opaque_id=use_id).first()
assert record_updated is not None
assert record_updated.incident_type == "uof"
assert record_updated.department_id == department.id
# post to the heartbeat URL to start a new update
response = testapp.post_json("/data/heartbeat", params={"heartbeat": "heartbeat"})
# Create 1 new fake incident
new_data = test_client.make_uof(count=1, short_name=department.short_name)
# give it the same opaqueId as the first batch
new_data[0]["opaqueId"] = use_id
# post the new incident row
response = testapp.post_json("/data/UOF", params={"month": 0, "year": 0, "data": new_data})
# assert that we got the expected reponse
assert response.status_code == 200
# there is now only 1 incident row in the database
check_uofs = UseOfForceIncidentLMPD.query.all()
assert len(check_uofs) == 1
# verify that the opaqueID posted matches that in the database
assert check_uofs[0].opaque_id == use_id
# verify that the opaqueId is recorded in an IncidentsUpdated table
record_updated = IncidentsUpdated.query.filter_by(opaque_id=use_id).first()
assert record_updated is not None
assert record_updated.incident_type == "uof"
assert record_updated.department_id == department.id
def test_all_ois_records_destroyed_when_new_record_posted(self, testapp):
''' Posting a new record with an id that matches a set of past records destroys all of them.
'''
# Set up the extractor
department = Department.create(name="B Police Department", short_name="BPD", load_defaults=False)
extractor, _ = Extractor.from_department_and_password(department=department, password="******")
# Set the correct authorization
testapp.authorization = ('Basic', (extractor.username, 'password'))
# post to the heartbeat URL to start the update
response = testapp.post_json("/data/heartbeat", params={"heartbeat": "heartbeat"})
# Post 5 fake incidents with an identical opaqueId to the OIS endpoint
ois_count = 5
test_client = JSONTestClient()
ois_data = test_client.make_ois(count=ois_count, short_name=department.short_name)
use_id = ois_data[0]['opaqueId']
for idx, _ in enumerate(ois_data):
ois_data[idx]['opaqueId'] = use_id
response = testapp.post_json("/data/OIS", params={'month': 0, 'year': 0, 'data': ois_data})
# assert that we got the expected reponse
assert response.status_code == 200
# there are 5 incident rows in the database
check_ois = OfficerInvolvedShootingBPD.query.all()
assert len(check_ois) == ois_count
# all the records in the database have the same id
ois_records = OfficerInvolvedShootingBPD.query.filter_by(opaque_id=use_id).all()
assert len(ois_records) == ois_count
# verify that the opaqueId is recorded in an IncidentsUpdated table
record_updated = IncidentsUpdated.query.filter_by(opaque_id=use_id).first()
assert record_updated is not None
assert record_updated.incident_type == "ois"
assert record_updated.department_id == department.id
# post to the heartbeat URL to start the update
response = testapp.post_json("/data/heartbeat", params={"heartbeat": "heartbeat"})
# Create 1 new fake incident
new_data = test_client.make_ois(count=1, short_name=department.short_name)
# give it the same opaqueId as the first batch
new_data[0]['opaqueId'] = use_id
# post the new incident
response = testapp.post_json("/data/OIS", params={'month': 0, 'year': 0, 'data': new_data})
# assert that we got the expected reponse
assert response.status_code == 200
# there is now only 1 incident row in the database
check_ois = OfficerInvolvedShootingBPD.query.all()
assert len(check_ois) == 1
# verify that the opaqueID posted matches that in the database
assert check_ois[0].opaque_id == use_id
# verify that the opaqueId is recorded in an IncidentsUpdated table
record_updated = IncidentsUpdated.query.filter_by(opaque_id=use_id).first()
assert record_updated is not None
assert record_updated.incident_type == "ois"
assert record_updated.department_id == department.id
def test_post_ois_data(self, testapp):
''' New and updated OIS data from the extractor is processed as expected.
'''
# Set up the extractor
department = Department.create(name="B Police Department", short_name="BPD", load_defaults=False)
extractor, _ = Extractor.from_department_and_password(department=department, password="******")
# Set the correct authorization
testapp.authorization = ('Basic', (extractor.username, 'password'))
# post to the heartbeat URL to start the update
response = testapp.post_json("/data/heartbeat", params={"heartbeat": "heartbeat"})
# Post 5 fake incidents to the OIS endpoint
ois_count = 5
test_client = JSONTestClient()
ois_data = test_client.make_ois(count=ois_count, short_name=department.short_name)
response = testapp.post_json("/data/OIS", params={'month': 0, 'year': 0, 'data': ois_data})
# assert that we got the expected reponse
assert response.status_code == 200
# there are 5 incident rows in the database
check_ois = OfficerInvolvedShootingBPD.query.all()
assert len(check_ois) == ois_count
for incident in ois_data:
# verify that the opaqueIDs posted match those in the database
assert OfficerInvolvedShootingBPD.query.filter_by(opaque_id=incident['opaqueId']).first() is not None
# verify that the opaqueIds are recorded in IncidentsUpdated tables
record_updated = IncidentsUpdated.query.filter_by(opaque_id=incident['opaqueId']).first()
assert record_updated is not None
assert record_updated.incident_type == "ois"
assert record_updated.department_id == department.id
# post to the heartbeat URL to start a new update
response = testapp.post_json("/data/heartbeat", params={"heartbeat": "heartbeat"})
# Create 5 more fake incidents
new_data = test_client.make_ois(count=ois_count, short_name=department.short_name)
# give them the same opaqueIds as the first batch
for idx, _ in enumerate(new_data):
new_data[idx]['opaqueId'] = ois_data[idx]['opaqueId']
# post the new incident rows
response = testapp.post_json("/data/OIS", params={'month': 0, 'year': 0, 'data': new_data})
# assert that we got the expected reponse
assert response.status_code == 200
# there are 5 incident rows in the database
check_ois = OfficerInvolvedShootingBPD.query.all()
assert len(check_ois) == ois_count
# verify that the opaqueIDs posted match those in the database
for incident in ois_data:
assert OfficerInvolvedShootingBPD.query.filter_by(opaque_id=incident['opaqueId']).first() is not None
# Create 5 more fake incidents
new_data = test_client.make_ois(count=ois_count, short_name=department.short_name)
# give them the same opaqueIds as the first batch
for idx, _ in enumerate(new_data):
new_data[idx]['opaqueId'] = ois_data[idx]['opaqueId']
# post the new incident rows without starting a new update
response = testapp.post_json("/data/OIS", params={'month': 0, 'year': 0, 'data': new_data})
# assert that we got the expected reponse
assert response.status_code == 200
# there are 10 incident rows in the database
check_ois = OfficerInvolvedShootingBPD.query.all()
assert len(check_ois) == ois_count * 2
def test_client():
''' Erase the database and load in a full suite of test data
'''
if not prompt_bool(
"Are you sure you want to destroy and recreate Comport's database?"
):
return
delete_everything()
# create a fake PD and admin user
department = Department.create(
name="Izquierda Metropolitan Police Department",
short_name="IMPD",
load_defaults=True)
user = User.create(username="******",
email="*****@*****.**",
password="******",
active=True,
is_admin=True)
user.departments.append(department)
user.save()
# create some fake officer out on service data
date_now = datetime.datetime.now()
date_step = date_now - relativedelta(months=30)
while date_step.year < date_now.year or date_step.month < date_now.month:
DenominatorValue.create(
department_id=department.id,
month=date_step.month,
year=date_step.year,
officers_out_on_service=(100000 + (randint(0, 46000) - 23000)))
date_step = date_step + relativedelta(months=1)
# create some fake demographic data
demo_template = [
dict(race="Asian", city_factor=0.0194, dept_factor=0.0013),
dict(race="Black", city_factor=0.2452, dept_factor=0.1402),
dict(race="Hispanic", city_factor=0.0861, dept_factor=0.0253),
dict(race="Other", city_factor=0.0699, dept_factor=0.0101),
dict(race="White", city_factor=0.5794, dept_factor=0.8231)
]
# for the city
city_population = 100000 + round(100000 * ((randint(0, 16) / 100) - .08))
for value in demo_template:
DemographicValue.create(department_id=department.id,
race=value["race"],
count=round(city_population *
value["city_factor"]),
department_value=False)
# for the department
dept_population = 1500 + round(1500 * ((randint(0, 16) / 100) - .08))
for value in demo_template:
DemographicValue.create(department_id=department.id,
race=value["race"],
count=round(dept_population *
value["dept_factor"]),
department_value=True)
# create a JSON test client and run it
test_client = JSONTestClient()
mutations = []
# mutations.append(MissingDataMutator())
# mutations.append(FuzzedDataMutator())
# mutations.append(KnownBadDataMutator())
# mutations.append(EmptyDataMutator())
# mutations.append(CasingMutator())
# mutations.append(CondenisngDateMutator())
# mutations.append(GapDateMutator())
test_client.run(department, mutations)
def test_client():
''' Erase the database and load in a full suite of test data
'''
if not prompt_bool("Are you sure you want to destroy and recreate Comport's database?"):
return
delete_everything()
# create a fake PD and admin user
department = Department.create(name="Izquierda Metropolitan Police Department", short_name="IMPD", load_defaults=True)
user = User.create(username="******", email="*****@*****.**", password="******", active=True, is_admin=True)
user.departments.append(department)
user.save()
# create some fake officer out on service data
date_now = datetime.datetime.now()
date_step = date_now - relativedelta(months=30)
while date_step.year < date_now.year or date_step.month < date_now.month:
DenominatorValue.create(
department_id=department.id,
month=date_step.month,
year=date_step.year,
officers_out_on_service=(100000 + (randint(0, 46000) - 23000))
)
date_step = date_step + relativedelta(months=1)
# create some fake demographic data
demo_template = [
dict(race="Asian", city_factor=0.0194, dept_factor=0.0013),
dict(race="Black", city_factor=0.2452, dept_factor=0.1402),
dict(race="Hispanic", city_factor=0.0861, dept_factor=0.0253),
dict(race="Other", city_factor=0.0699, dept_factor=0.0101),
dict(race="White", city_factor=0.5794, dept_factor=0.8231)
]
# for the city
city_population = 100000 + round(100000 * ((randint(0, 16) / 100) - .08))
for value in demo_template:
DemographicValue.create(
department_id=department.id,
race=value["race"],
count=round(city_population * value["city_factor"]),
department_value=False
)
# for the department
dept_population = 1500 + round(1500 * ((randint(0, 16) / 100) - .08))
for value in demo_template:
DemographicValue.create(
department_id=department.id,
race=value["race"],
count=round(dept_population * value["dept_factor"]),
department_value=True
)
# create a JSON test client and run it
test_client = JSONTestClient()
mutations = []
# mutations.append(MissingDataMutator())
# mutations.append(FuzzedDataMutator())
# mutations.append(KnownBadDataMutator())
# mutations.append(EmptyDataMutator())
# mutations.append(CasingMutator())
# mutations.append(CondenisngDateMutator())
# mutations.append(GapDateMutator())
test_client.run(department, mutations)
def test_update_ois_data(self, testapp):
''' Updated OIS data from the extractor is processed as expected.
'''
# Set up the extractor
department = Department.create(name="Good Police Department", short_name="GPD", load_defaults=False)
extractor, envs = Extractor.from_department_and_password(department=department, password="******")
# Set the correct authorization
testapp.authorization = ('Basic', (extractor.username, 'password'))
# Get a generated list of OIS descriptions from the JSON test client
test_client = JSONTestClient()
ois_data = test_client.get_prebaked_ois(last=1)
# post the json to the OIS URL
response = testapp.post_json("/data/OIS", params={'month': 0, 'year': 0, 'data': ois_data})
# assert that we got the expected reponse
assert response.status_code == 200
assert response.json_body['updated'] == 0
assert response.json_body['added'] == 1
# Get the second pre-baked ois incident
updated_ois_data = test_client.get_prebaked_ois(first=1, last=2)
# Swap in the opaque ID from the first ois incident
updated_ois_data[0]["opaqueId"] = ois_data[0]["opaqueId"]
# The ois incident won't be a match unless this field is the same
updated_ois_data[0]["officerIdentifier"] = ois_data[0]["officerIdentifier"]
# post the json to the ois URL
response = testapp.post_json("/data/OIS", params={'month': 0, 'year': 0, 'data': updated_ois_data})
# assert that we got the expected reponse
assert response.status_code == 200
assert response.json_body['updated'] == 1
assert response.json_body['added'] == 0
# There's only one complaint in the database.
all_ois = OfficerInvolvedShooting.query.all()
assert len(all_ois) == 1
# check the ois incident in the database against the updated data that was sent
cleaner = Cleaners()
sent_ois = updated_ois_data[0]
check_ois = OfficerInvolvedShooting.query.filter_by(opaque_id=sent_ois['opaqueId']).first()
assert check_ois.occured_date.strftime('%Y-%m-%d %-H:%-M:%S') == sent_ois['occuredDate']
assert check_ois.division == cleaner.capitalize(sent_ois['division'])
assert check_ois.precinct == cleaner.capitalize(sent_ois['precinct'])
assert check_ois.shift == cleaner.capitalize(sent_ois['shift'])
assert check_ois.beat == cleaner.capitalize(sent_ois['beat'])
assert check_ois.disposition == sent_ois['disposition']
assert check_ois.resident_race == cleaner.race(sent_ois['residentRace'])
assert check_ois.resident_sex == cleaner.sex(sent_ois['residentSex'])
assert check_ois.resident_age == cleaner.number_to_string(sent_ois['residentAge'])
assert check_ois.resident_weapon_used == cleaner.resident_weapon_used(sent_ois['residentWeaponUsed'])
assert check_ois.resident_condition == sent_ois['residentCondition']
assert check_ois.officer_identifier == sent_ois['officerIdentifier']
assert check_ois.officer_weapon_used == sent_ois['officerForceType']
assert check_ois.officer_race == cleaner.race(sent_ois['officerRace'])
assert check_ois.officer_sex == cleaner.sex(sent_ois['officerSex'])
assert check_ois.officer_age == cleaner.number_to_string(sent_ois['officerAge'])
assert check_ois.officer_years_of_service == cleaner.string_to_integer(sent_ois['officerYearsOfService'])
assert check_ois.officer_condition == sent_ois['officerCondition']
def test_update_complaint_data(self, testapp):
''' Updated complaint data from the extractor is processed as expected.
'''
# Set up the extractor
department = Department.create(name="IM Police Department", short_name="IMPD", load_defaults=False)
extractor, envs = Extractor.from_department_and_password(department=department, password="******")
# Set the correct authorization
testapp.authorization = ('Basic', (extractor.username, 'password'))
# Get a generated list of complaint descriptions from the JSON test client
test_client = JSONTestClient()
complaint_data = test_client.get_prebaked_complaints(last=1)
# post the json to the complaint URL
response = testapp.post_json("/data/complaints", params={'month': 0, 'year': 0, 'data': complaint_data})
# assert that we got the expected reponse
assert response.status_code == 200
assert response.json_body['updated'] == 0
assert response.json_body['added'] == 1
# Get the second pre-baked complaint
updated_complaint_data = test_client.get_prebaked_complaints(first=1, last=2)
# Swap in the opaque ID from the first complaint
updated_complaint_data[0]["opaqueId"] = complaint_data[0]["opaqueId"]
# The complaint won't be a match unless these fields are the same
updated_complaint_data[0]["allegationType"] = complaint_data[0]["allegationType"]
updated_complaint_data[0]["allegation"] = complaint_data[0]["allegation"]
updated_complaint_data[0]["officerIdentifier"] = complaint_data[0]["officerIdentifier"]
updated_complaint_data[0]["residentRace"] = complaint_data[0]["residentRace"]
updated_complaint_data[0]["residentSex"] = complaint_data[0]["residentSex"]
updated_complaint_data[0]["residentAge"] = complaint_data[0]["residentAge"]
# post the json to the complaint URL
response = testapp.post_json("/data/complaints", params={'month': 0, 'year': 0, 'data': updated_complaint_data})
# assert that we got the expected reponse
assert response.status_code == 200
assert response.json_body['updated'] == 1
assert response.json_body['added'] == 0
# There's only one complaint in the database.
all_complaints = CitizenComplaintIMPD.query.all()
assert len(all_complaints) == 1
# check the complaint incident in the database against the updated data that was sent
cleaner = Cleaners()
sent_complaint = cleaner.capitalize_incident(updated_complaint_data[0])
check_complaint = CitizenComplaintIMPD.query.filter_by(opaque_id=sent_complaint['opaqueId']).first()
assert check_complaint.occured_date.strftime('%Y-%m-%d %-H:%-M:%S') == sent_complaint['occuredDate']
assert check_complaint.division == sent_complaint['division']
assert check_complaint.precinct == sent_complaint['precinct']
assert check_complaint.shift == sent_complaint['shift']
assert check_complaint.beat == sent_complaint['beat']
assert check_complaint.disposition == sent_complaint['disposition']
assert check_complaint.service_type == sent_complaint['serviceType']
assert check_complaint.source == sent_complaint['source']
assert check_complaint.allegation_type == sent_complaint['allegationType']
assert check_complaint.allegation == sent_complaint['allegation']
assert check_complaint.resident_race == cleaner.race(sent_complaint['residentRace'])
assert check_complaint.resident_sex == cleaner.sex(sent_complaint['residentSex'])
assert check_complaint.resident_age == cleaner.number_to_string(sent_complaint['residentAge'])
assert check_complaint.officer_identifier == sent_complaint['officerIdentifier']
assert check_complaint.officer_race == cleaner.race(sent_complaint['officerRace'])
assert check_complaint.officer_sex == cleaner.sex(sent_complaint['officerSex'])
assert check_complaint.officer_age == cleaner.number_to_string(sent_complaint['officerAge'])
assert check_complaint.officer_years_of_service == cleaner.number_to_string(sent_complaint['officerYearsOfService'])
def test_post_uof_data(self, testapp):
''' New and updated UOF data from the extractor is processed as expected.
'''
# Set up the extractor
department = Department.create(name="B Police Department",
short_name="BPD",
load_defaults=False)
extractor, _ = Extractor.from_department_and_password(
department=department, password="******")
# Set the correct authorization
testapp.authorization = ('Basic', (extractor.username, 'password'))
# post to the heartbeat URL to start the update
response = testapp.post_json("/data/heartbeat",
params={"heartbeat": "heartbeat"})
# Post 5 fake incidents to the UOF endpoint
uof_count = 5
test_client = JSONTestClient()
uof_data = test_client.make_uof(count=uof_count,
short_name=department.short_name)
response = testapp.post_json("/data/UOF",
params={
'month': 0,
'year': 0,
'data': uof_data
})
# assert that we got the expected reponse
assert response.status_code == 200
# there are 5 incident rows in the database
check_uofs = UseOfForceIncidentBPD.query.all()
assert len(check_uofs) == uof_count
for incident in uof_data:
# verify that the opaqueIDs posted match those in the database
assert UseOfForceIncidentBPD.query.filter_by(
opaque_id=incident['opaqueId']).first() is not None
# verify that the opaqueIds are recorded in IncidentsUpdated tables
record_updated = IncidentsUpdated.query.filter_by(
opaque_id=incident['opaqueId']).first()
assert record_updated is not None
assert record_updated.department_id == department.id
assert record_updated.incident_type == "uof"
# post to the heartbeat URL to start the new update
response = testapp.post_json("/data/heartbeat",
params={"heartbeat": "heartbeat"})
# Create 5 more fake incidents
new_data = test_client.make_uof(count=uof_count,
short_name=department.short_name)
# give them the same opaqueIds as the first batch
for idx, _ in enumerate(new_data):
new_data[idx]['opaqueId'] = uof_data[idx]['opaqueId']
# post the new incident rows
response = testapp.post_json("/data/UOF",
params={
'month': 0,
'year': 0,
'data': new_data
})
# assert that we got the expected reponse
assert response.status_code == 200
# there are 5 incident rows in the database
check_uofs = UseOfForceIncidentBPD.query.all()
assert len(check_uofs) == uof_count
# verify that the opaqueIDs posted match those in the database
for incident in uof_data:
assert UseOfForceIncidentBPD.query.filter_by(
opaque_id=incident['opaqueId']).first() is not None
# Create 5 more fake incidents
new_data = test_client.make_uof(count=uof_count,
short_name=department.short_name)
# give them the same opaqueIds as the first batch
for idx, _ in enumerate(new_data):
new_data[idx]['opaqueId'] = uof_data[idx]['opaqueId']
# post the new incident rows without starting a new update
response = testapp.post_json("/data/UOF",
params={
'month': 0,
'year': 0,
'data': new_data
})
# assert that we got the expected reponse
assert response.status_code == 200
# there are 10 incident rows in the database
check_uofs = UseOfForceIncidentBPD.query.all()
assert len(check_uofs) == uof_count * 2
def test_all_ois_records_destroyed_when_new_record_posted(self, testapp):
''' Posting a new record with an id that matches a set of past records destroys all of them.
'''
# Set up the extractor
department = Department.create(name="B Police Department",
short_name="BPD",
load_defaults=False)
extractor, _ = Extractor.from_department_and_password(
department=department, password="******")
# Set the correct authorization
testapp.authorization = ('Basic', (extractor.username, 'password'))
# post to the heartbeat URL to start the update
response = testapp.post_json("/data/heartbeat",
params={"heartbeat": "heartbeat"})
# Post 5 fake incidents with an identical opaqueId to the OIS endpoint
ois_count = 5
test_client = JSONTestClient()
ois_data = test_client.make_ois(count=ois_count,
short_name=department.short_name)
use_id = ois_data[0]['opaqueId']
for idx, _ in enumerate(ois_data):
ois_data[idx]['opaqueId'] = use_id
response = testapp.post_json("/data/OIS",
params={
'month': 0,
'year': 0,
'data': ois_data
})
# assert that we got the expected reponse
assert response.status_code == 200
# there are 5 incident rows in the database
check_ois = OfficerInvolvedShootingBPD.query.all()
assert len(check_ois) == ois_count
# all the records in the database have the same id
ois_records = OfficerInvolvedShootingBPD.query.filter_by(
opaque_id=use_id).all()
assert len(ois_records) == ois_count
# verify that the opaqueId is recorded in an IncidentsUpdated table
record_updated = IncidentsUpdated.query.filter_by(
opaque_id=use_id).first()
assert record_updated is not None
assert record_updated.incident_type == "ois"
assert record_updated.department_id == department.id
# post to the heartbeat URL to start the update
response = testapp.post_json("/data/heartbeat",
params={"heartbeat": "heartbeat"})
# Create 1 new fake incident
new_data = test_client.make_ois(count=1,
short_name=department.short_name)
# give it the same opaqueId as the first batch
new_data[0]['opaqueId'] = use_id
# post the new incident
response = testapp.post_json("/data/OIS",
params={
'month': 0,
'year': 0,
'data': new_data
})
# assert that we got the expected reponse
assert response.status_code == 200
# there is now only 1 incident row in the database
check_ois = OfficerInvolvedShootingBPD.query.all()
assert len(check_ois) == 1
# verify that the opaqueID posted matches that in the database
assert check_ois[0].opaque_id == use_id
# verify that the opaqueId is recorded in an IncidentsUpdated table
record_updated = IncidentsUpdated.query.filter_by(
opaque_id=use_id).first()
assert record_updated is not None
assert record_updated.incident_type == "ois"
assert record_updated.department_id == department.id
def test_update_uof_data(self, testapp):
''' Updated UOF data from the extractor is processed as expected.
'''
# Set up the extractor
department = Department.create(name="IM Police Department", short_name="IMPD", load_defaults=False)
extractor, envs = Extractor.from_department_and_password(department=department, password="******")
# Set the correct authorization
testapp.authorization = ('Basic', (extractor.username, 'password'))
# Get a generated list of UOF descriptions from the JSON test client
test_client = JSONTestClient()
uof_data = test_client.get_prebaked_uof(last=1)
# post the json to the UOF URL
response = testapp.post_json("/data/UOF", params={'month': 0, 'year': 0, 'data': uof_data})
# assert that we got the expected reponse
assert response.status_code == 200
assert response.json_body['updated'] == 0
assert response.json_body['added'] == 1
# Get the second pre-baked uof incident
updated_uof_data = test_client.get_prebaked_uof(first=1, last=2)
# Swap in the opaque ID from the first uof incident
updated_uof_data[0]["opaqueId"] = uof_data[0]["opaqueId"]
# The uof incident won't be a match unless these fields are the same
updated_uof_data[0]["officerIdentifier"] = uof_data[0]["officerIdentifier"]
updated_uof_data[0]["officerForceType"] = uof_data[0]["officerForceType"]
# post the json to the uof URL
response = testapp.post_json("/data/UOF", params={'month': 0, 'year': 0, 'data': updated_uof_data})
# assert that we got the expected reponse
assert response.status_code == 200
assert response.json_body['updated'] == 1
assert response.json_body['added'] == 0
# There's only one complaint in the database.
all_uof = UseOfForceIncidentIMPD.query.all()
assert len(all_uof) == 1
# check the uof incident in the database against the updated data that was sent
cleaner = Cleaners()
sent_uof = updated_uof_data[0]
check_uof = UseOfForceIncidentIMPD.query.filter_by(opaque_id=sent_uof['opaqueId']).first()
assert check_uof.occured_date.strftime('%Y-%m-%d %-H:%-M:%S') == sent_uof['occuredDate']
assert check_uof.division == cleaner.capitalize(sent_uof['division'])
assert check_uof.precinct == cleaner.capitalize(sent_uof['precinct'])
assert check_uof.shift == cleaner.capitalize(sent_uof['shift'])
assert check_uof.beat == cleaner.capitalize(sent_uof['beat'])
assert check_uof.disposition == sent_uof['disposition']
assert check_uof.officer_force_type == cleaner.officer_force_type(sent_uof['officerForceType'])
assert check_uof.use_of_force_reason == sent_uof['useOfForceReason']
assert check_uof.service_type == sent_uof['serviceType']
assert check_uof.arrest_made == sent_uof['arrestMade']
assert check_uof.arrest_charges == sent_uof['arrestCharges']
assert check_uof.resident_weapon_used == sent_uof['residentWeaponUsed']
assert check_uof.resident_injured == sent_uof['residentInjured']
assert check_uof.resident_hospitalized == sent_uof['residentHospitalized']
assert check_uof.officer_injured == sent_uof['officerInjured']
assert check_uof.officer_hospitalized == sent_uof['officerHospitalized']
assert check_uof.resident_race == cleaner.race(sent_uof['residentRace'])
assert check_uof.resident_sex == cleaner.sex(sent_uof['residentSex'])
assert check_uof.resident_age == cleaner.number_to_string(sent_uof['residentAge'])
assert check_uof.resident_condition == sent_uof['residentCondition']
assert check_uof.officer_identifier == sent_uof['officerIdentifier']
assert check_uof.officer_race == cleaner.race(sent_uof['officerRace'])
assert check_uof.officer_sex == cleaner.sex(sent_uof['officerSex'])
assert check_uof.officer_age == cleaner.number_to_string(sent_uof['officerAge'])
assert check_uof.officer_years_of_service == cleaner.number_to_string(sent_uof['officerYearsOfService'])
assert check_uof.officer_condition == sent_uof['officerCondition']
