def test_post_mistyped_uof_data(self, testapp): ''' New UOF data from the extractor is processed as expected. ''' # Set up the extractor department = Department.create(name="Good Police Department", short_name="GPD", load_defaults=False) extractor, envs = Extractor.from_department_and_password(department=department, password="******") # Set the correct authorization testapp.authorization = ('Basic', (extractor.username, 'password')) # Get a generated list of UOF descriptions from the JSON test client test_client = JSONTestClient() uof_count = 1 uof_data = test_client.get_prebaked_uof(last=uof_count) # The app expects number values to be transmitted as strings. Let's change them to integers. uof_data[0]['residentAge'] = 28 uof_data[0]['officerAge'] = 46 uof_data[0]['officerYearsOfService'] = 17 # post the json to the UOF URL response = testapp.post_json("/data/UOF", params={'month': 0, 'year': 0, 'data': uof_data}) # assert that we got the expected reponse assert response.status_code == 200 assert response.json_body['updated'] == 0 assert response.json_body['added'] == uof_count # check the uof incident in the database against the data that was sent cleaner = Cleaners() sent_uof = uof_data[0] check_uof = UseOfForceIncident.query.filter_by(opaque_id=sent_uof['opaqueId']).first() assert check_uof.occured_date.strftime('%Y-%m-%d %-H:%-M:%S') == sent_uof['occuredDate'] assert check_uof.division == cleaner.capitalize(sent_uof['division']) assert check_uof.precinct == cleaner.capitalize(sent_uof['precinct']) assert check_uof.shift == cleaner.capitalize(sent_uof['shift']) assert check_uof.beat == cleaner.capitalize(sent_uof['beat']) assert check_uof.disposition == sent_uof['disposition'] assert check_uof.officer_force_type == cleaner.officer_force_type(sent_uof['officerForceType']) assert check_uof.use_of_force_reason == sent_uof['useOfForceReason'] assert check_uof.service_type == sent_uof['serviceType'] assert check_uof.arrest_made == sent_uof['arrestMade'] assert check_uof.arrest_charges == sent_uof['arrestCharges'] assert check_uof.resident_weapon_used == sent_uof['residentWeaponUsed'] assert check_uof.resident_injured == sent_uof['residentInjured'] assert check_uof.resident_hospitalized == sent_uof['residentHospitalized'] assert check_uof.officer_injured == sent_uof['officerInjured'] assert check_uof.officer_hospitalized == sent_uof['officerHospitalized'] assert check_uof.resident_race == cleaner.race(sent_uof['residentRace']) assert check_uof.resident_sex == cleaner.sex(sent_uof['residentSex']) assert check_uof.resident_age == cleaner.number_to_string(sent_uof['residentAge']) assert check_uof.resident_condition == sent_uof['residentCondition'] assert check_uof.officer_identifier == sent_uof['officerIdentifier'] assert check_uof.officer_race == cleaner.race(sent_uof['officerRace']) assert check_uof.officer_sex == cleaner.sex(sent_uof['officerSex']) assert check_uof.officer_age == cleaner.number_to_string(sent_uof['officerAge']) assert check_uof.officer_years_of_service == cleaner.number_to_string(sent_uof['officerYearsOfService']) assert check_uof.officer_condition == sent_uof['officerCondition']
def test_post_mistyped_ois_data(self, testapp): ''' New OIS data from the extractor is processed as expected. ''' # Set up the extractor department = Department.create(name="Good Police Department", short_name="GPD", load_defaults=False) extractor, envs = Extractor.from_department_and_password(department=department, password="******") # Set the correct authorization testapp.authorization = ('Basic', (extractor.username, 'password')) # Get a generated list of OIS descriptions from the JSON test client test_client = JSONTestClient() ois_count = 1 ois_data = test_client.get_prebaked_ois(last=ois_count) # The app expects number values to be transmitted as strings. Let's change them to integers. ois_data[0]['residentAge'] = 28 ois_data[0]['officerAge'] = 46 # And it expects this number value to be transmitted as a number, so let's make it a string. ois_data[0]['officerYearsOfService'] = "17" # post the json to the OIS URL response = testapp.post_json("/data/OIS", params={'month': 0, 'year': 0, 'data': ois_data}) # assert that we got the expected reponse assert response.status_code == 200 assert response.json_body['updated'] == 0 assert response.json_body['added'] == ois_count # check the ois incident in the database against the data that was sent cleaner = Cleaners() sent_ois = ois_data[0] check_ois = OfficerInvolvedShooting.query.filter_by(opaque_id=sent_ois['opaqueId']).first() assert check_ois.occured_date.strftime('%Y-%m-%d %-H:%-M:%S') == sent_ois['occuredDate'] assert check_ois.division == cleaner.capitalize(sent_ois['division']) assert check_ois.precinct == cleaner.capitalize(sent_ois['precinct']) assert check_ois.shift == cleaner.capitalize(sent_ois['shift']) assert check_ois.beat == cleaner.capitalize(sent_ois['beat']) assert check_ois.disposition == sent_ois['disposition'] assert check_ois.resident_race == cleaner.race(sent_ois['residentRace']) assert check_ois.resident_sex == cleaner.sex(sent_ois['residentSex']) assert check_ois.resident_age == cleaner.number_to_string(sent_ois['residentAge']) assert check_ois.resident_weapon_used == cleaner.resident_weapon_used(sent_ois['residentWeaponUsed']) assert check_ois.resident_condition == sent_ois['residentCondition'] assert check_ois.officer_identifier == sent_ois['officerIdentifier'] assert check_ois.officer_weapon_used == sent_ois['officerForceType'] assert check_ois.officer_race == cleaner.race(sent_ois['officerRace']) assert check_ois.officer_sex == cleaner.sex(sent_ois['officerSex']) assert check_ois.officer_age == cleaner.number_to_string(sent_ois['officerAge']) assert check_ois.officer_years_of_service == cleaner.string_to_integer(sent_ois['officerYearsOfService']) assert check_ois.officer_condition == sent_ois['officerCondition']
def test_post_mistyped_complaint_data(self, testapp): ''' New complaint data from the extractor with wrongly typed data is processed as expected. ''' # Set up the extractor department = Department.create(name="Good Police Department", short_name="GPD", load_defaults=False) extractor, envs = Extractor.from_department_and_password(department=department, password="******") # Set the correct authorization testapp.authorization = ('Basic', (extractor.username, 'password')) # Get a generated list of complaint descriptions from the JSON test client test_client = JSONTestClient() complaint_count = 1 complaint_data = test_client.get_prebaked_complaints(last=complaint_count) # The app expects number values to be transmitted as strings. Let's change them to integers. complaint_data[0]['residentAge'] = 28 complaint_data[0]['officerAge'] = 46 complaint_data[0]['officerYearsOfService'] = 17 # post the json to the complaint URL response = testapp.post_json("/data/complaints", params={'month': 0, 'year': 0, 'data': complaint_data}) # assert that we got the expected reponse assert response.status_code == 200 assert response.json_body['updated'] == 0 assert response.json_body['added'] == complaint_count # check the complaint incident in the database against the data that was sent cleaner = Cleaners() sent_complaint = cleaner.capitalize_incident(complaint_data[0]) check_complaint = CitizenComplaint.query.filter_by(opaque_id=sent_complaint['opaqueId']).first() assert check_complaint.occured_date.strftime('%Y-%m-%d %-H:%-M:%S') == sent_complaint['occuredDate'] assert check_complaint.division == sent_complaint['division'] assert check_complaint.precinct == sent_complaint['precinct'] assert check_complaint.shift == sent_complaint['shift'] assert check_complaint.beat == sent_complaint['beat'] assert check_complaint.disposition == sent_complaint['disposition'] assert check_complaint.service_type == sent_complaint['serviceType'] assert check_complaint.source == sent_complaint['source'] assert check_complaint.allegation_type == sent_complaint['allegationType'] assert check_complaint.allegation == sent_complaint['allegation'] assert check_complaint.resident_race == cleaner.race(sent_complaint['residentRace']) assert check_complaint.resident_sex == cleaner.sex(sent_complaint['residentSex']) assert check_complaint.resident_age == cleaner.number_to_string(sent_complaint['residentAge']) assert check_complaint.officer_identifier == sent_complaint['officerIdentifier'] assert check_complaint.officer_race == cleaner.race(sent_complaint['officerRace']) assert check_complaint.officer_sex == cleaner.sex(sent_complaint['officerSex']) assert check_complaint.officer_age == cleaner.number_to_string(sent_complaint['officerAge']) assert check_complaint.officer_years_of_service == cleaner.number_to_string(sent_complaint['officerYearsOfService'])
def test_skip_multiple_complaint_data(self, testapp): ''' Multiple complaint data from the extractor is skipped. ''' # Set up the extractor department = Department.create(name="Good Police Department", short_name="GPD", load_defaults=False) extractor, envs = Extractor.from_department_and_password(department=department, password="******") # Set the correct authorization testapp.authorization = ('Basic', (extractor.username, 'password')) # Get a generated list of complaint descriptions from the JSON test client test_client = JSONTestClient() complaint_data = test_client.get_prebaked_complaints(last=1) # post the json to the complaint URL response = testapp.post_json("/data/complaints", params={'month': 0, 'year': 0, 'data': complaint_data}) # assert that we got the expected reponse assert response.status_code == 200 assert response.json_body['updated'] == 0 assert response.json_body['added'] == 1 # Get the second pre-baked complaint multiple_complaint_data = test_client.get_prebaked_complaints(first=1, last=2) # Swap in the opaque ID from the first complaint multiple_complaint_data[0]["opaqueId"] = complaint_data[0]["opaqueId"] # The complaint will be skipped as a 'multiple' if these fields are the same multiple_complaint_data[0]["allegationType"] = complaint_data[0]["allegationType"] multiple_complaint_data[0]["allegation"] = complaint_data[0]["allegation"] multiple_complaint_data[0]["officerIdentifier"] = complaint_data[0]["officerIdentifier"] # post the json to the complaint URL response = testapp.post_json("/data/complaints", params={'month': 0, 'year': 0, 'data': multiple_complaint_data}) # assert that we got the expected reponse assert response.status_code == 200 assert response.json_body['updated'] == 0 assert response.json_body['added'] == 0 # There is one complaint in the database. all_complaints = CitizenComplaint.query.all() assert len(all_complaints) == 1
def test_post_ois_data_near_match_does_not_update(self, testapp): ''' OIS data with the same ID but different details creates a new record. ''' # Set up the extractor department = Department.create(name="Good Police Department", short_name="GPD", load_defaults=False) extractor, envs = Extractor.from_department_and_password(department=department, password="******") # Set the correct authorization testapp.authorization = ('Basic', (extractor.username, 'password')) # Get a generated list of OIS descriptions from the JSON test client test_client = JSONTestClient() ois_data = test_client.get_prebaked_ois(last=1) # post the json to the OIS URL response = testapp.post_json("/data/OIS", params={'month': 0, 'year': 0, 'data': ois_data}) # assert that we got the expected reponse assert response.status_code == 200 assert response.json_body['updated'] == 0 assert response.json_body['added'] == 1 # Get the second pre-baked ois incident updated_ois_data = test_client.get_prebaked_ois(first=1, last=2) # Swap in the opaque ID from the first ois incident updated_ois_data[0]["opaqueId"] = ois_data[0]["opaqueId"] # post the json to the ois URL response = testapp.post_json("/data/OIS", params={'month': 0, 'year': 0, 'data': updated_ois_data}) # assert that we got the expected reponse assert response.status_code == 200 assert response.json_body['updated'] == 0 assert response.json_body['added'] == 1 # There's only one complaint in the database. all_ois = OfficerInvolvedShooting.query.all() assert len(all_ois) == 2
def test_client(): ''' Erase the database and load in a full suite of test data ''' if not prompt_bool( "Are you sure you want to destroy and recreate Comport's database?" ): return delete_everything() # create a fake PD and admin user department = Department.create( name="Izquierda Metropolitan Police Department", short_name="IMPD", load_defaults=True) user = User.create(username="******", email="*****@*****.**", password="******", active=True, is_admin=True) user.departments.append(department) user.save() # create some fake officer out on service data date_now = datetime.datetime.now() date_step = date_now - relativedelta(months=30) while date_step.year < date_now.year or date_step.month < date_now.month: DenominatorValue.create( department_id=department.id, month=date_step.month, year=date_step.year, officers_out_on_service=(100000 + (randint(0, 46000) - 23000))) date_step = date_step + relativedelta(months=1) # create some fake demographic data demo_template = [ dict(race="Asian", city_factor=0.0194, dept_factor=0.0013), dict(race="Black", city_factor=0.2452, dept_factor=0.1402), dict(race="Hispanic", city_factor=0.0861, dept_factor=0.0253), dict(race="Other", city_factor=0.0699, dept_factor=0.0101), dict(race="White", city_factor=0.5794, dept_factor=0.8231) ] # for the city city_population = 100000 + round(100000 * ((randint(0, 16) / 100) - .08)) for value in demo_template: DemographicValue.create(department_id=department.id, race=value["race"], count=round(city_population * value["city_factor"]), department_value=False) # for the department dept_population = 1500 + round(1500 * ((randint(0, 16) / 100) - .08)) for value in demo_template: DemographicValue.create(department_id=department.id, race=value["race"], count=round(dept_population * value["dept_factor"]), department_value=True) # create a JSON test client and run it test_client = JSONTestClient() mutations = [] # mutations.append(MissingDataMutator()) # mutations.append(FuzzedDataMutator()) # mutations.append(KnownBadDataMutator()) # mutations.append(EmptyDataMutator()) # mutations.append(CasingMutator()) # mutations.append(CondenisngDateMutator()) # mutations.append(GapDateMutator()) test_client.run(department, mutations)
def test_update_ois_data(self, testapp): ''' Updated OIS data from the extractor is processed as expected. ''' # Set up the extractor department = Department.create(name="Good Police Department", short_name="GPD", load_defaults=False) extractor, envs = Extractor.from_department_and_password(department=department, password="******") # Set the correct authorization testapp.authorization = ('Basic', (extractor.username, 'password')) # Get a generated list of OIS descriptions from the JSON test client test_client = JSONTestClient() ois_data = test_client.get_prebaked_ois(last=1) # post the json to the OIS URL response = testapp.post_json("/data/OIS", params={'month': 0, 'year': 0, 'data': ois_data}) # assert that we got the expected reponse assert response.status_code == 200 assert response.json_body['updated'] == 0 assert response.json_body['added'] == 1 # Get the second pre-baked ois incident updated_ois_data = test_client.get_prebaked_ois(first=1, last=2) # Swap in the opaque ID from the first ois incident updated_ois_data[0]["opaqueId"] = ois_data[0]["opaqueId"] # The ois incident won't be a match unless this field is the same updated_ois_data[0]["officerIdentifier"] = ois_data[0]["officerIdentifier"] # post the json to the ois URL response = testapp.post_json("/data/OIS", params={'month': 0, 'year': 0, 'data': updated_ois_data}) # assert that we got the expected reponse assert response.status_code == 200 assert response.json_body['updated'] == 1 assert response.json_body['added'] == 0 # There's only one complaint in the database. all_ois = OfficerInvolvedShooting.query.all() assert len(all_ois) == 1 # check the ois incident in the database against the updated data that was sent cleaner = Cleaners() sent_ois = updated_ois_data[0] check_ois = OfficerInvolvedShooting.query.filter_by(opaque_id=sent_ois['opaqueId']).first() assert check_ois.occured_date.strftime('%Y-%m-%d %-H:%-M:%S') == sent_ois['occuredDate'] assert check_ois.division == cleaner.capitalize(sent_ois['division']) assert check_ois.precinct == cleaner.capitalize(sent_ois['precinct']) assert check_ois.shift == cleaner.capitalize(sent_ois['shift']) assert check_ois.beat == cleaner.capitalize(sent_ois['beat']) assert check_ois.disposition == sent_ois['disposition'] assert check_ois.resident_race == cleaner.race(sent_ois['residentRace']) assert check_ois.resident_sex == cleaner.sex(sent_ois['residentSex']) assert check_ois.resident_age == cleaner.number_to_string(sent_ois['residentAge']) assert check_ois.resident_weapon_used == cleaner.resident_weapon_used(sent_ois['residentWeaponUsed']) assert check_ois.resident_condition == sent_ois['residentCondition'] assert check_ois.officer_identifier == sent_ois['officerIdentifier'] assert check_ois.officer_weapon_used == sent_ois['officerForceType'] assert check_ois.officer_race == cleaner.race(sent_ois['officerRace']) assert check_ois.officer_sex == cleaner.sex(sent_ois['officerSex']) assert check_ois.officer_age == cleaner.number_to_string(sent_ois['officerAge']) assert check_ois.officer_years_of_service == cleaner.string_to_integer(sent_ois['officerYearsOfService']) assert check_ois.officer_condition == sent_ois['officerCondition']
def test_post_uof_data(self, testapp): ''' New and updated UOF data from the extractor is processed as expected. ''' # Set up the extractor department = Department.create(name="B Police Department", short_name="BPD", load_defaults=False) extractor, _ = Extractor.from_department_and_password( department=department, password="******") # Set the correct authorization testapp.authorization = ('Basic', (extractor.username, 'password')) # post to the heartbeat URL to start the update response = testapp.post_json("/data/heartbeat", params={"heartbeat": "heartbeat"}) # Post 5 fake incidents to the UOF endpoint uof_count = 5 test_client = JSONTestClient() uof_data = test_client.make_uof(count=uof_count, short_name=department.short_name) response = testapp.post_json("/data/UOF", params={ 'month': 0, 'year': 0, 'data': uof_data }) # assert that we got the expected reponse assert response.status_code == 200 # there are 5 incident rows in the database check_uofs = UseOfForceIncidentBPD.query.all() assert len(check_uofs) == uof_count for incident in uof_data: # verify that the opaqueIDs posted match those in the database assert UseOfForceIncidentBPD.query.filter_by( opaque_id=incident['opaqueId']).first() is not None # verify that the opaqueIds are recorded in IncidentsUpdated tables record_updated = IncidentsUpdated.query.filter_by( opaque_id=incident['opaqueId']).first() assert record_updated is not None assert record_updated.department_id == department.id assert record_updated.incident_type == "uof" # post to the heartbeat URL to start the new update response = testapp.post_json("/data/heartbeat", params={"heartbeat": "heartbeat"}) # Create 5 more fake incidents new_data = test_client.make_uof(count=uof_count, short_name=department.short_name) # give them the same opaqueIds as the first batch for idx, _ in enumerate(new_data): new_data[idx]['opaqueId'] = uof_data[idx]['opaqueId'] # post the new incident rows response = testapp.post_json("/data/UOF", params={ 'month': 0, 'year': 0, 'data': new_data }) # assert that we got the expected reponse assert response.status_code == 200 # there are 5 incident rows in the database check_uofs = UseOfForceIncidentBPD.query.all() assert len(check_uofs) == uof_count # verify that the opaqueIDs posted match those in the database for incident in uof_data: assert UseOfForceIncidentBPD.query.filter_by( opaque_id=incident['opaqueId']).first() is not None # Create 5 more fake incidents new_data = test_client.make_uof(count=uof_count, short_name=department.short_name) # give them the same opaqueIds as the first batch for idx, _ in enumerate(new_data): new_data[idx]['opaqueId'] = uof_data[idx]['opaqueId'] # post the new incident rows without starting a new update response = testapp.post_json("/data/UOF", params={ 'month': 0, 'year': 0, 'data': new_data }) # assert that we got the expected reponse assert response.status_code == 200 # there are 10 incident rows in the database check_uofs = UseOfForceIncidentBPD.query.all() assert len(check_uofs) == uof_count * 2
def test_all_ois_records_destroyed_when_new_record_posted(self, testapp): ''' Posting a new record with an id that matches a set of past records destroys all of them. ''' # Set up the extractor department = Department.create(name="B Police Department", short_name="BPD", load_defaults=False) extractor, _ = Extractor.from_department_and_password( department=department, password="******") # Set the correct authorization testapp.authorization = ('Basic', (extractor.username, 'password')) # post to the heartbeat URL to start the update response = testapp.post_json("/data/heartbeat", params={"heartbeat": "heartbeat"}) # Post 5 fake incidents with an identical opaqueId to the OIS endpoint ois_count = 5 test_client = JSONTestClient() ois_data = test_client.make_ois(count=ois_count, short_name=department.short_name) use_id = ois_data[0]['opaqueId'] for idx, _ in enumerate(ois_data): ois_data[idx]['opaqueId'] = use_id response = testapp.post_json("/data/OIS", params={ 'month': 0, 'year': 0, 'data': ois_data }) # assert that we got the expected reponse assert response.status_code == 200 # there are 5 incident rows in the database check_ois = OfficerInvolvedShootingBPD.query.all() assert len(check_ois) == ois_count # all the records in the database have the same id ois_records = OfficerInvolvedShootingBPD.query.filter_by( opaque_id=use_id).all() assert len(ois_records) == ois_count # verify that the opaqueId is recorded in an IncidentsUpdated table record_updated = IncidentsUpdated.query.filter_by( opaque_id=use_id).first() assert record_updated is not None assert record_updated.incident_type == "ois" assert record_updated.department_id == department.id # post to the heartbeat URL to start the update response = testapp.post_json("/data/heartbeat", params={"heartbeat": "heartbeat"}) # Create 1 new fake incident new_data = test_client.make_ois(count=1, short_name=department.short_name) # give it the same opaqueId as the first batch new_data[0]['opaqueId'] = use_id # post the new incident response = testapp.post_json("/data/OIS", params={ 'month': 0, 'year': 0, 'data': new_data }) # assert that we got the expected reponse assert response.status_code == 200 # there is now only 1 incident row in the database check_ois = OfficerInvolvedShootingBPD.query.all() assert len(check_ois) == 1 # verify that the opaqueID posted matches that in the database assert check_ois[0].opaque_id == use_id # verify that the opaqueId is recorded in an IncidentsUpdated table record_updated = IncidentsUpdated.query.filter_by( opaque_id=use_id).first() assert record_updated is not None assert record_updated.incident_type == "ois" assert record_updated.department_id == department.id