def test_top_percentile_type_not_found(self):
officer = OfficerFactory(id=1, appointed_date=date(2016, 1, 1))
OfficerAllegationFactory(officer=officer,
allegation__incident_date=datetime(
2013, 1, 1, tzinfo=pytz.utc),
start_date=datetime(2014,
1,
1,
tzinfo=pytz.utc),
allegation__is_officer_complaint=False)
with self.assertRaisesRegex(ValueError, 'group is invalid'):
officer_percentile.top_percentile(2017,
percentile_groups=['not_exist'])
def build_cached_yearly_percentiles():
percentile_groups = [
PERCENTILE_ALLEGATION_GROUP,
PERCENTILE_ALLEGATION_INTERNAL_CIVILIAN_GROUP, PERCENTILE_TRR_GROUP
]
def _not_retired(officer):
return not officer.resignation_date or officer.year <= officer.resignation_date.year
results = []
for yr in tqdm(range(MIN_VISUAL_TOKEN_YEAR, MAX_VISUAL_TOKEN_YEAR + 1),
desc='Prepare percentile data'):
officers = officer_percentile.top_percentile(
yr, percentile_groups=percentile_groups)
results.extend(filter(_not_retired, officers))
cursor = connection.cursor()
cursor.execute(f'TRUNCATE TABLE {OfficerYearlyPercentile._meta.db_table}')
OfficerYearlyPercentile.objects.bulk_create(
OfficerYearlyPercentile(
officer=result,
year=result.year,
percentile_trr=getattr(result, 'percentile_trr', None),
percentile_allegation=getattr(result, 'percentile_allegation',
None),
percentile_allegation_civilian=getattr(
result, 'percentile_allegation_civilian', None),
percentile_allegation_internal=getattr(
result, 'percentile_allegation_internal', None),
) for result in results)
def populate_top_percentile_dict(self):
self.yearly_top_percentile = dict()
for yr in range(MIN_VISUAL_TOKEN_YEAR, MAX_VISUAL_TOKEN_YEAR + 1):
for officer in officer_percentile.top_percentile(
yr, percentile_groups=self.percentile_groups):
if officer.resignation_date and yr > officer.resignation_date.year:
continue
officer_list = self.yearly_top_percentile.setdefault(
officer.id, [])
officer_dict = {'id': officer.id, 'year': yr}
for attr in [
'percentile_trr', 'percentile_allegation',
'percentile_allegation_civilian',
'percentile_allegation_internal'
]:
if hasattr(officer, attr):
officer_dict[attr] = f'{getattr(officer, attr):.4f}'
officer_list.append(officer_dict)
def test_honorable_mention_percentile(self):
self._create_dataset_for_honorable_mention_percentile()
OfficerFactory(id=3, appointed_date=date(2015, 3, 15))
OfficerFactory(id=4, appointed_date=date(2015, 1, 1))
new_percentile_map = officer_percentile.create_percentile_map()
with patch('data.officer_percentile.PERCENTILE_MAP',
new_percentile_map):
# current year
annotated_officers = officer_percentile.top_percentile(
percentile_groups=[PERCENTILE_HONORABLE_MENTION_GROUP])
expect(annotated_officers).to.have.length(4)
expected_result = {
1: {
'officer_id': 1,
'metric_honorable_mention': 0.625,
'percentile_honorable_mention': 50.0,
},
2: {
'officer_id': 2,
'metric_honorable_mention': 1.875,
'percentile_honorable_mention': 75.0,
},
3: {
'officer_id': 3,
'metric_honorable_mention': 0,
'percentile_honorable_mention': 0,
},
4: {
'officer_id': 4,
'metric_honorable_mention': 0,
'percentile_honorable_mention': 0,
},
}
for officer in annotated_officers:
validate_object(officer, expected_result[officer.officer_id])
def test_top_percentile_not_enough_service_year(self):
officer1 = OfficerFactory(id=1, appointed_date=date(1990, 3, 14))
officer2 = OfficerFactory(id=2,
appointed_date=date(1990, 3, 14),
resignation_date=date(2014, 5, 1))
officer3 = OfficerFactory(id=3, appointed_date=date(2013, 3, 14))
officer4 = OfficerFactory(id=4,
appointed_date=date(1990, 3, 14),
resignation_date=date(2015, 5, 1))
# officer1 have all data
OfficerAllegationFactory.create_batch(
2,
officer=officer1,
allegation__incident_date=datetime(2013, 12, 31, tzinfo=pytz.utc),
)
OfficerAllegationFactory.create_batch(
3,
officer=officer1,
allegation__incident_date=datetime(2014, 7, 2, tzinfo=pytz.utc),
allegation__is_officer_complaint=True)
OfficerAllegationFactory.create_batch(
4,
officer=officer1,
allegation__incident_date=datetime(2014, 7, 2, tzinfo=pytz.utc),
allegation__is_officer_complaint=False)
TRRFactory.create_batch(2,
officer=officer1,
trr_datetime=datetime(2015,
2,
1,
tzinfo=pytz.utc))
# officer2 don't have trr
OfficerAllegationFactory(
officer=officer2,
allegation__incident_date=datetime(2013, 12, 31, tzinfo=pytz.utc),
)
OfficerAllegationFactory(officer=officer2,
allegation__incident_date=datetime(
2014, 2, 2, tzinfo=pytz.utc),
allegation__is_officer_complaint=True)
OfficerAllegationFactory(officer=officer2,
allegation__incident_date=datetime(
2014, 3, 2, tzinfo=pytz.utc),
allegation__is_officer_complaint=False)
# officer3 don't have allegation in ALLEGATION_MIN - ALLEGATION_MAX
OfficerAllegationFactory.create_batch(
2,
officer=officer3,
allegation__incident_date=datetime(2014, 7, 2, tzinfo=pytz.utc),
allegation__is_officer_complaint=True)
OfficerAllegationFactory.create_batch(
3,
officer=officer3,
allegation__incident_date=datetime(2014, 7, 2, tzinfo=pytz.utc),
allegation__is_officer_complaint=False)
TRRFactory.create_batch(3,
officer=officer3,
trr_datetime=datetime(2015,
2,
1,
tzinfo=pytz.utc))
# officer4 don't have allegation in INTERNAL_CIVILIAN_ALLEGATION year range
OfficerAllegationFactory.create_batch(
5,
officer=officer4,
allegation__incident_date=datetime(2013, 12, 31, tzinfo=pytz.utc),
)
TRRFactory.create_batch(6,
officer=officer4,
trr_datetime=datetime(2015,
2,
1,
tzinfo=pytz.utc))
expected_dict = {
1: {
'officer_id': 1,
'year': 2016,
'metric_allegation': 2,
'metric_allegation_civilian': 4,
'metric_allegation_internal': 3,
'metric_trr': 2,
'percentile_allegation': 33.3333,
'percentile_allegation_civilian': 66.6667,
'percentile_allegation_internal': 66.6667,
'percentile_trr': 0.0
},
2: {
'officer_id': 2,
'year': 2016,
'metric_allegation': 1,
'metric_allegation_civilian': None,
'metric_allegation_internal': None,
'metric_trr': None,
'percentile_allegation': 0.0,
'percentile_allegation_civilian': None,
'percentile_allegation_internal': None,
'percentile_trr': None
},
3: {
'officer_id': 3,
'year': 2016,
'metric_allegation': None,
'metric_allegation_civilian': 3,
'metric_allegation_internal': 2,
'metric_trr': 3,
'percentile_allegation': None,
'percentile_allegation_civilian': 33.3333,
'percentile_allegation_internal': 33.3333,
'percentile_trr': 50.0
},
4: {
'officer_id': 4,
'year': 2016,
'metric_allegation': 5,
'metric_allegation_civilian': 0.0,
'metric_allegation_internal': 0.0,
'metric_trr': None,
'percentile_allegation': 66.6667,
'percentile_allegation_civilian': 0.0,
'percentile_allegation_internal': 0.0,
'percentile_trr': None
}
}
officers = officer_percentile.top_percentile(2016)
for officer in officers:
validate_object(officer, expected_dict[officer.id])
def test_top_percentile_with_types(self):
officer1 = OfficerFactory(id=1, appointed_date=date(1990, 3, 14))
officer2 = OfficerFactory(id=2, appointed_date=date(1990, 3, 14))
officer3 = OfficerFactory(id=3, appointed_date=date(1990, 3, 14))
officer4 = OfficerFactory(id=4, appointed_date=date(1990, 3, 14))
# officer1 have all data
OfficerAllegationFactory.create_batch(
2,
officer=officer1,
allegation__incident_date=datetime(2013, 12, 31, tzinfo=pytz.utc),
)
OfficerAllegationFactory.create_batch(
3,
officer=officer1,
allegation__incident_date=datetime(2015, 7, 2, tzinfo=pytz.utc),
allegation__is_officer_complaint=True)
OfficerAllegationFactory.create_batch(
4,
officer=officer1,
allegation__incident_date=datetime(2015, 7, 2, tzinfo=pytz.utc),
allegation__is_officer_complaint=False)
TRRFactory.create_batch(2,
officer=officer1,
trr_datetime=datetime(2015,
2,
1,
tzinfo=pytz.utc))
# officer2 don't have trr
OfficerAllegationFactory(
officer=officer2,
allegation__incident_date=datetime(2013, 12, 31, tzinfo=pytz.utc),
)
OfficerAllegationFactory(officer=officer2,
allegation__incident_date=datetime(
2015, 7, 2, tzinfo=pytz.utc),
allegation__is_officer_complaint=True)
OfficerAllegationFactory(officer=officer2,
allegation__incident_date=datetime(
2015, 7, 2, tzinfo=pytz.utc),
allegation__is_officer_complaint=False)
# officer3 don't have allegation in ALLEGATION_MIN - ALLEGATION_MAX
OfficerAllegationFactory.create_batch(
2,
officer=officer3,
allegation__incident_date=datetime(2015, 7, 2, tzinfo=pytz.utc),
allegation__is_officer_complaint=True)
OfficerAllegationFactory.create_batch(
3,
officer=officer3,
allegation__incident_date=datetime(2015, 7, 2, tzinfo=pytz.utc),
allegation__is_officer_complaint=False)
TRRFactory.create_batch(3,
officer=officer3,
trr_datetime=datetime(2015,
2,
1,
tzinfo=pytz.utc))
# officer4 don't have allegation in INTERNAL_CIVILIAN_ALLEGATION year range
OfficerAllegationFactory.create_batch(
5,
officer=officer4,
allegation__incident_date=datetime(2013, 12, 31, tzinfo=pytz.utc),
)
TRRFactory.create_batch(6,
officer=officer4,
trr_datetime=datetime(2015,
2,
1,
tzinfo=pytz.utc))
expected_dict = {
1: {
'officer_id': 1,
'year': 2016,
'metric_allegation_civilian': 4,
'metric_allegation_internal': 3,
'metric_trr': 2,
'percentile_allegation': None,
'percentile_allegation_civilian': 75.0,
'percentile_allegation_internal': 75.0,
'percentile_trr': 25.0
},
2: {
'officer_id': 2,
'year': 2016,
'metric_allegation_civilian': 1,
'metric_allegation_internal': 1,
'metric_trr': 0.0,
'percentile_allegation': None,
'percentile_allegation_civilian': 25.0,
'percentile_allegation_internal': 25.0,
'percentile_trr': 0.0
},
3: {
'officer_id': 3,
'year': 2016,
'metric_allegation_civilian': 3,
'metric_allegation_internal': 2,
'metric_trr': 3,
'percentile_allegation': None,
'percentile_allegation_civilian': 50.0,
'percentile_allegation_internal': 50.0,
'percentile_trr': 50.0
},
4: {
'officer_id': 4,
'year': 2016,
'metric_allegation_civilian': 0.0,
'metric_allegation_internal': 0.0,
'metric_trr': 6,
'percentile_allegation': None,
'percentile_allegation_civilian': 0.0,
'percentile_allegation_internal': 0.0,
'percentile_trr': 75.0
}
}
visual_token_percentile_groups = [
PERCENTILE_ALLEGATION_INTERNAL_CIVILIAN_GROUP, PERCENTILE_TRR_GROUP
]
officers = officer_percentile.top_percentile(
2016, percentile_groups=visual_token_percentile_groups)
for officer in officers:
validate_object(officer, expected_dict[officer.id])
def test_top_percentile(self):
officer1 = OfficerFactory(id=1, appointed_date=date(1990, 3, 14))
officer2 = OfficerFactory(id=2, appointed_date=date(1990, 3, 14))
officer3 = OfficerFactory(id=3, appointed_date=date(1990, 3, 14))
# officer1 have all data
OfficerAllegationFactory.create_batch(
2,
officer=officer1,
allegation__incident_date=datetime(2013, 12, 31, tzinfo=pytz.utc),
)
OfficerAllegationFactory.create_batch(
3,
officer=officer1,
allegation__incident_date=datetime(2015, 7, 2, tzinfo=pytz.utc),
allegation__is_officer_complaint=True)
OfficerAllegationFactory.create_batch(
4,
officer=officer1,
allegation__incident_date=datetime(2015, 7, 2, tzinfo=pytz.utc),
allegation__is_officer_complaint=False)
TRRFactory.create_batch(2,
officer=officer1,
trr_datetime=datetime(2015,
2,
1,
tzinfo=pytz.utc))
# officer2 don't have trr
OfficerAllegationFactory(
officer=officer2,
allegation__incident_date=datetime(2013, 12, 31, tzinfo=pytz.utc),
)
OfficerAllegationFactory(officer=officer2,
allegation__incident_date=datetime(
2015, 7, 2, tzinfo=pytz.utc),
allegation__is_officer_complaint=True)
OfficerAllegationFactory(officer=officer2,
allegation__incident_date=datetime(
2015, 7, 2, tzinfo=pytz.utc),
allegation__is_officer_complaint=False)
# officer3 don't have allegation in ALLEGATION_MIN - ALLEGATION_MAX
OfficerAllegationFactory.create_batch(
2,
officer=officer3,
allegation__incident_date=datetime(2015, 7, 2, tzinfo=pytz.utc),
allegation__is_officer_complaint=True)
OfficerAllegationFactory.create_batch(
3,
officer=officer3,
allegation__incident_date=datetime(2015, 7, 2, tzinfo=pytz.utc),
allegation__is_officer_complaint=False)
TRRFactory.create_batch(3,
officer=officer3,
trr_datetime=datetime(2015,
2,
1,
tzinfo=pytz.utc))
expected_dict = {
1: {
'officer_id': 1,
'year': 2016,
'metric_allegation': 2,
'metric_allegation_civilian': 4,
'metric_allegation_internal': 3,
'metric_trr': 2,
'percentile_allegation': 66.6667,
'percentile_allegation_civilian': 66.6667,
'percentile_allegation_internal': 66.6667,
'percentile_trr': 33.3333
},
2: {
'officer_id': 2,
'year': 2016,
'metric_allegation': 1,
'metric_allegation_civilian': 1,
'metric_allegation_internal': 1,
'metric_trr': 0.0,
'percentile_allegation': 33.3333,
'percentile_allegation_civilian': 0.0000,
'percentile_allegation_internal': 0.0000,
'percentile_trr': 0.0
},
3: {
'officer_id': 3,
'year': 2016,
'metric_allegation': 0.0,
'metric_allegation_civilian': 3,
'metric_allegation_internal': 2,
'metric_trr': 3,
'percentile_allegation': 0.0,
'percentile_allegation_civilian': 33.3333,
'percentile_allegation_internal': 33.3333,
'percentile_trr': 66.6667,
}
}
officers = officer_percentile.top_percentile(2016)
for officer in officers:
validate_object(officer, expected_dict[officer.id])