def setUp(self):
violation_category = AllegationCategoryFactory(category='Operation/Personnel Violations')
use_of_force_category = AllegationCategoryFactory(category='Use Of Force')
illegal_search_category = AllegationCategoryFactory(category='Illegal Search')
false_arrest_category = AllegationCategoryFactory(category='False Arrest')
before_min_date = datetime(1987, 12, 31, tzinfo=pytz.utc)
valid_date = datetime(1988, 12, 31, tzinfo=pytz.utc)
today = datetime.now(pytz.utc)
for date in [before_min_date, valid_date, today]:
OfficerAllegationFactory.create_batch(
4,
allegation_category=violation_category,
allegation__incident_date=date,
disciplined=True)
OfficerAllegationFactory.create_batch(
3,
allegation_category=use_of_force_category,
allegation__incident_date=date,
disciplined=True)
OfficerAllegationFactory.create_batch(
2,
allegation_category=illegal_search_category,
allegation__incident_date=date,
disciplined=False)
OfficerAllegationFactory(
allegation_category=false_arrest_category,
allegation__incident_date=date,
final_outcome='900')
def test_extract_datum_most_common_category(self):
allegation = AllegationFactory()
cat1 = AllegationCategoryFactory(
category='Use Of Forces',
allegation_name='Sub Force'
)
cat2 = AllegationCategoryFactory(
category='Traffic',
allegation_name='Sub traffic'
)
OfficerAllegationFactory.create_batch(
2,
allegation=allegation,
allegation_category=cat1
)
OfficerAllegationFactory(
allegation=allegation,
allegation_category=cat2
)
rows = self.extract_data()
expect(rows).to.have.length(1)
expect(rows[0]['most_common_category']).to.eq({
'category': 'Use Of Forces',
'allegation_name': 'Sub Force'
})
def test_get_category_names(self):
allegation = AllegationFactory()
category1 = AllegationCategoryFactory(category='Use of Force')
category2 = AllegationCategoryFactory(category='Illegal Search')
OfficerAllegationFactory(allegation=allegation, allegation_category=category1)
OfficerAllegationFactory(allegation=allegation, allegation_category=category2)
expect(allegation.category_names).to.eq(['Illegal Search', 'Use of Force'])
OfficerAllegationFactory(allegation=allegation, allegation_category=None)
expect(allegation.category_names).to.eq(['Illegal Search', 'Unknown', 'Use of Force'])
def test_update_video_thumbnail(self, _, vimeo_api, crawl_copa, __):
logger = logging.getLogger('crawler.crawl_copa_data')
crawl_copa.return_value = [{
'allegation': {
'crid':
'123',
'incident_date':
datetime(2013, 4, 30, 21, 30, tzinfo=pytz.utc),
'attachment_files': [{
'file_type':
'video',
'title':
'video file',
'url':
'https://player.vimeo.com/video/288225991',
'original_url':
'https://player.vimeo.com/video/288225991',
'tag':
'Video',
'source_type':
'PORTAL_COPA',
'external_last_updated':
datetime(2018, 10, 30, 15, 0, 3, tzinfo=pytz.utc),
}],
'subjects': ['Subject1', 'Unknown']
},
'allegation_category': {
'category': 'Incident',
'allegation_name': 'Allegation Name'
},
'police_shooting': True
}]
vimeo_api.return_value = {
'id': 288225991,
'title': 'Log# 1082195 3rd Party Clip',
'description': 'Log# 1082195 3rd Party Clip',
'url': 'https://vimeo.com/307768537',
'upload_date': '2018-12-21 15:47:48',
'thumbnail_small':
'https://i.vimeocdn.com/video/747800241_100x75.webp',
'thumbnail_medium':
'https://i.vimeocdn.com/video/747800241_200x150.webp',
'thumbnail_large':
'https://i.vimeocdn.com/video/747800241_640.webp',
}
AllegationCategoryFactory(category='Incident',
allegation_name='Allegation Name')
attachment_file = AttachmentFileFactory(
allegation__crid='123',
title='old_title',
source_type=AttachmentSourceType.PORTAL_COPA,
external_id='288225991',
original_url='https://player.vimeo.com/video/288225991',
preview_image_url=None)
CopaPortalAttachmentImporter(logger).crawl_and_update_attachments()
expect(AttachmentFile.objects.get(
pk=attachment_file.pk).title).to.eq('video file')
expect(AttachmentFile.objects.get(pk=attachment_file.pk).preview_image_url). \
to.eq('https://i.vimeocdn.com/video/747800241_100x75.webp')
def test_serialization(self):
with freeze_time(datetime(2018, 4, 3, 12, 0, 10, tzinfo=pytz.utc)):
allegation_1 = AllegationFactory(
crid='111111',
most_common_category=AllegationCategoryFactory(
category='Use Of Force'),
incident_date=datetime(2001, 1, 1, tzinfo=pytz.utc),
)
allegation_2 = AllegationFactory(
crid='222222',
incident_date=datetime(2002, 2, 2, tzinfo=pytz.utc),
)
expect(AllegationSerializer(allegation_1).data).to.eq({
'crid':
'111111',
'category':
'Use Of Force',
'incident_date':
'2001-01-01',
})
expect(AllegationSerializer(allegation_2).data).to.eq({
'crid':
'222222',
'category':
'Unknown',
'incident_date':
'2002-02-02',
})
def test_not_update_video_thumbnail_when_source_is_not_vimeo(
self, _, crawl_copa):
logger = logging.getLogger('crawler.crawl_copa_data')
crawl_copa.return_value = [{
'attachments': [{
'type': 'video',
'link': 'https://player.fake_video.org/video/288225991',
'title': 'video file',
'last_updated': '2018-10-30T15:00:03+00:00',
}],
'date':
'04-30-2013',
'log_number':
'123',
'time':
'04-30-2013 9:30 pm',
'type':
'Allegation Name',
'subjects': ['Subject', '', 'Unknown'],
}]
AllegationCategoryFactory(category='Incident',
allegation_name='Allegation Name')
attachment_file = AttachmentFileFactory(
allegation__crid='123',
title='old_title',
source_type=AttachmentSourceType.PORTAL_COPA_DOCUMENTCLOUD,
external_id='288225991',
original_url='https://player.fake_video.org/video/288225991',
preview_image_url=None)
CopaPortalAttachmentImporter(logger).crawl_and_update_attachments()
expect(
AttachmentFile.objects.get(
pk=attachment_file.pk).preview_image_url).be.none()
def test_retrieve_recent_search_items(self):
OfficerFactory(
id=8562,
first_name='Jerome',
last_name='Finnigan',
current_badge='123456',
allegation_count=20,
sustained_count=5,
birth_year=1980,
race='White',
gender='M',
rank='Police Officer'
)
allegation_category = AllegationCategoryFactory(category='Use of Force')
AllegationFactory(
crid='C12345',
incident_date=datetime(2007, 1, 1, tzinfo=pytz.utc),
most_common_category=allegation_category,
)
trr = TRRFactory(id=123, trr_datetime=datetime(2007, 1, 1, tzinfo=pytz.utc))
ActionResponseFactory(trr=trr, force_type='Physical Force - Stunning', action_sub_category='4')
ActionResponseFactory(trr=trr, force_type='Taser', action_sub_category='5.1')
ActionResponseFactory(trr=trr, force_type='Impact Weapon', action_sub_category='5.2')
ActionResponseFactory(trr=trr, force_type='Taser Display', action_sub_category='3')
url = reverse('api:suggestion-recent-search-items')
response = self.client.get(url, {
'officer_ids[]': 8562,
'crids[]': 'C12345',
'trr_ids[]': 123,
})
expect(response.status_code).to.eq(status.HTTP_200_OK)
expect(response.data).to.eq([
{
'id': 8562,
'name': 'Jerome Finnigan',
'race': 'White',
'gender': 'Male',
'rank': 'Police Officer',
'allegation_count': 20,
'sustained_count': 5,
'birth_year': 1980,
'type': 'OFFICER',
},
{
'id': 'C12345',
'crid': 'C12345',
'category': 'Use of Force',
'incident_date': '2007-01-01',
'type': 'CR',
},
{
'id': 123,
'trr_datetime': '2007-01-01',
'force_type': 'Impact Weapon',
'type': 'TRR',
}
])
def test_category(self):
officer_allegation = OfficerAllegationFactory(allegation_category=None)
expect(officer_allegation.category).to.eq(None)
allegation_category = AllegationCategoryFactory(category='category')
officer_allegation = OfficerAllegationFactory(
allegation_category=allegation_category)
expect(officer_allegation.category).to.eq('category')
def test_serialization(self):
pinned_officer = OfficerFactory(
id=1,
rank='Police Officer',
first_name='Jerome',
last_name='Finnigan',
trr_percentile='11.11',
complaint_percentile='22.22',
civilian_allegation_percentile='33.33',
internal_allegation_percentile='44.44',
allegation_count=2,
)
relevant_allegation = AllegationFactory(
crid='1',
incident_date=datetime(2002, 2, 21, tzinfo=pytz.utc),
most_common_category=AllegationCategoryFactory(
category='Operation/Personnel Violations',
allegation_name='Subcategory'),
point=Point([0.01, 0.02]))
VictimFactory(gender='M',
race='Black',
age=35,
allegation=relevant_allegation)
pinboard = PinboardFactory(
id='66ef1560',
title='Test pinboard',
description='Test description',
)
pinboard.officers.set([pinned_officer])
OfficerAllegationFactory(officer=pinned_officer,
allegation=relevant_allegation)
expect(pinboard.relevant_complaints.count()).to.eq(1)
expect(
AllegationMobileSerializer(
pinboard.relevant_complaints.first()).data).to.eq({
'crid':
'1',
'category':
'Operation/Personnel Violations',
'incident_date':
'2002-02-21',
'officers': [{
'id': 1,
'rank': 'Police Officer',
'full_name': 'Jerome Finnigan',
'percentile_trr': '11.1100',
'percentile_allegation': '22.2200',
'percentile_allegation_civilian': '33.3300',
'percentile_allegation_internal': '44.4400',
}],
'point': {
'lon': 0.01,
'lat': 0.02
},
})
def test_subcategory(self):
officer_allegation = OfficerAllegationFactory(allegation_category=None)
expect(officer_allegation.subcategory).to.eq(None)
allegation_category = AllegationCategoryFactory(
allegation_name='subcategory')
officer_allegation = OfficerAllegationFactory(
allegation_category=allegation_category)
expect(officer_allegation.subcategory).to.eq('subcategory')
def test_new_timeline_item_no_join(self):
officer = OfficerFactory(id=123, appointed_date=None, rank='Police Officer')
unit = PoliceUnitFactory(unit_name='001', description='unit_001')
OfficerHistoryFactory(officer=officer, unit=unit, effective_date=date(2010, 1, 1), end_date=date(2011, 12, 31))
allegation = AllegationFactory(
crid='123456',
coaccused_count=4,
incident_date=datetime(2011, 8, 23, tzinfo=pytz.utc)
)
OfficerAllegationFactory(
final_finding='UN', final_outcome='Unknown',
officer=officer, allegation=allegation,
allegation_category=AllegationCategoryFactory(category='category', allegation_name='sub category')
)
OfficerAllegationFactory.create_batch(3, allegation=allegation)
SalaryFactory(officer=officer, year=2001, rank='Police Officer', spp_date=date(2001, 9, 23))
cache_managers.cache_all()
response = self.client.get(reverse('api-v2:officers-new-timeline-items', kwargs={'pk': 123}))
expect(response.status_code).to.eq(status.HTTP_200_OK)
expect(response.data).to.eq([
{
'date': '2011-08-23',
'kind': 'CR',
'crid': '123456',
'category': 'category',
'subcategory': 'sub category',
'finding': 'Unfounded',
'outcome': 'Unknown',
'coaccused': 4,
'unit_name': '001',
'unit_description': 'unit_001',
'rank': 'Police Officer',
'victims': [],
'attachments': [],
}, {
'date': '2010-01-01',
'kind': 'UNIT_CHANGE',
'unit_name': '001',
'unit_description': 'unit_001',
'rank': 'Police Officer',
}, {
'date': '2001-09-23',
'kind': 'RANK_CHANGE',
'unit_name': '',
'unit_description': '',
'rank': 'Police Officer',
}
])
def test_complaint_category_aggregation(self):
unit = PoliceUnitFactory()
officer = OfficerFactory()
OfficerHistoryFactory(unit=unit, officer=officer)
OfficerAllegationFactory(officer=officer,
allegation_category=AllegationCategoryFactory(
category='Use of Force'),
final_finding='NS')
expect(unit.complaint_category_aggregation).to.eq([{
'name': 'Use of Force',
'count': 1,
'sustained_count': 0
}])
def test_update_PORTAL_COPA_DOCUMENTCLOUD_file(self, _, crawl_copa):
logger = logging.getLogger('crawler.crawl_copa_data')
crawl_copa.return_value = [{
'allegation': {
'crid':
'123',
'incident_date':
datetime(2013, 4, 30, 21, 30, tzinfo=pytz.utc),
'attachment_files': [{
'file_type':
'document',
'title':
'pdf file',
'url':
'http://chicagocopa.org/document.pdf',
'original_url':
'http://chicagocopa.org/document.pdf',
'tag':
'Document',
'source_type':
'PORTAL_COPA_DOCUMENTCLOUD',
'external_last_updated':
datetime(2018, 10, 30, 15, 0, 3, tzinfo=pytz.utc),
}],
'subjects': ['Subject1', 'Unknown']
},
'allegation_category': {
'category': 'Incident',
'allegation_name': 'Allegation Name'
},
'police_shooting': True
}]
AllegationCategoryFactory(category='Incident',
allegation_name='Allegation Name')
attachment_file = AttachmentFileFactory(
allegation__crid='123',
title='old_title',
source_type=AttachmentSourceType.PORTAL_COPA_DOCUMENTCLOUD,
external_id='document.pdf',
original_url='http://chicagocopa.org/document.pdf',
external_last_updated=datetime(2017, 10, 30, tzinfo=pytz.utc))
new_attachments = CopaPortalAttachmentImporter(
logger).crawl_and_update_attachments()
expect(new_attachments).to.be.empty()
updated_attachment_file = AttachmentFile.objects.get(
pk=attachment_file.pk)
expect(updated_attachment_file.title).to.eq('CRID 123 CR pdf file')
expect(updated_attachment_file.external_last_updated).to.eq(
datetime(2018, 10, 30, 15, 0, 3, tzinfo=pytz.utc))
def test_serialization(self):
allegation_category = AllegationCategoryFactory(category='Use of Force')
allegation = AllegationFactory(
crid='C12345',
incident_date=datetime(2007, 1, 1, tzinfo=pytz.utc),
most_common_category=allegation_category,
)
expect(AllegationSerializer(allegation).data).to.eq({
'id': 'C12345',
'crid': 'C12345',
'incident_date': '2007-01-01',
'type': 'CR',
'category': 'Use of Force',
})
def test_serialization_no_point(self):
category = AllegationCategoryFactory(category='Use of Force',
allegation_name='Miscellaneous')
allegation = AllegationFactory(
crid=123,
most_common_category=category,
incident_date=datetime(2002, 1, 1, tzinfo=pytz.utc),
point=None,
coaccused_count=12,
)
expect(CRSerializer(allegation).data).to.eq({
'date': '2002-01-01',
'crid': '123',
'category': 'Use of Force',
'kind': 'CR',
})
def test_serialization(self):
category = AllegationCategoryFactory(category='Use of Force',
allegation_name='Subcategory')
allegation = AllegationFactory(
crid=123,
incident_date=datetime(2002, 1, 1, tzinfo=pytz.utc),
most_common_category=category,
coaccused_count=12,
point=Point(-35.5, 68.9),
old_complaint_address='34XX Douglas Blvd')
officer = OfficerFactory(id=1,
first_name='Jerome',
last_name='Finnigan',
allegation_count=20,
sustained_count=10,
birth_year=1980,
race='Asian',
gender='M',
rank='Police Officer',
resignation_date=datetime(2000,
1,
1,
tzinfo=pytz.utc),
trr_percentile=80,
complaint_percentile=85,
civilian_allegation_percentile=90,
internal_allegation_percentile=95)
OfficerAllegationFactory(officer=officer,
allegation=allegation,
recc_outcome='Separation',
final_outcome='30 Day Suspension',
final_finding='UN',
allegation_category=category,
disciplined=True)
expected_data = {
'date': '2002-01-01',
'crid': '123',
'category': 'Use of Force',
'kind': 'CR',
'point': {
'lon': -35.5,
'lat': 68.9
},
}
expect(CRSerializer(allegation).data).to.eq(expected_data)
def test_complaint_category_aggregation_with_duplicated_allegation(self):
unit = PoliceUnitFactory()
officer1 = OfficerFactory()
officer2 = OfficerFactory()
allegation = AllegationFactory()
allegation_category = AllegationCategoryFactory(category='Use of Force')
OfficerHistoryFactory(officer=officer1, unit=unit)
OfficerHistoryFactory(officer=officer2, unit=unit)
OfficerAllegationFactory(
officer=officer1, allegation=allegation, allegation_category=allegation_category, final_finding='NS'
)
OfficerAllegationFactory(
officer=officer2, allegation=allegation, allegation_category=allegation_category, final_finding='NS'
)
expect(unit.complaint_category_aggregation).to.eq([{
'name': 'Use of Force',
'count': 1,
'sustained_count': 0
}])
def test_search_with_apostrophe(self):
allegation_category = AllegationCategoryFactory(category='Use of Force')
allegation_1 = AllegationFactory(
crid='C12345',
incident_date=datetime(2007, 1, 1, tzinfo=pytz.utc),
most_common_category=allegation_category,
)
allegation_2 = AllegationFactory(
crid='C12346',
incident_date=datetime(2007, 1, 1, tzinfo=pytz.utc),
most_common_category=allegation_category,
)
allegation_3 = AllegationFactory(
crid='C12347',
incident_date=datetime(2007, 1, 1, tzinfo=pytz.utc),
most_common_category=allegation_category,
)
allegation_4 = AllegationFactory(
crid='C12348',
incident_date=datetime(2007, 1, 1, tzinfo=pytz.utc),
most_common_category=allegation_category,
)
AttachmentFileFactory(text_content='Name: OBrien', allegation=allegation_1)
AttachmentFileFactory(text_content='Name: O\'Brien', allegation=allegation_2)
AttachmentFileFactory(text_content='Name: O Brien', allegation=allegation_3)
AttachmentFileFactory(text_content='Name: Jim', allegation=allegation_4)
self.rebuild_index()
self.refresh_index()
response = self.client.get(reverse('api:suggestion-list'), {
'term': 'O\'Brien',
})
sorted_cr_results = list(sorted(response.data['CR'], key=lambda cr: cr['crid']))
print(sorted_cr_results)
expect(len(sorted_cr_results)).to.eq(3)
expect(sorted_cr_results[0]['crid']).to.eq('C12345')
expect(sorted_cr_results[0]['highlight']['text_content']).to.eq(['Name: O<em>Brien</em>'])
expect(sorted_cr_results[1]['crid']).to.eq('C12346')
expect(sorted_cr_results[1]['highlight']['text_content']).to.eq(['Name: O\'<em>Brien</em>'])
expect(sorted_cr_results[2]['crid']).to.eq('C12347')
expect(sorted_cr_results[2]['highlight']['text_content']).to.eq(['Name: O <em>Brien</em>'])
def test_complaint_category_aggregation(self):
officer = OfficerFactory()
allegation_category = AllegationCategoryFactory(
category='Use of Force')
OfficerAllegationFactory(officer=officer,
allegation=AllegationFactory(),
allegation_category=allegation_category,
start_date=None,
final_finding='NS')
OfficerAllegationFactory(officer=officer,
allegation=AllegationFactory(),
allegation_category=allegation_category,
start_date=date(2010, 1, 1),
final_finding='NS')
OfficerAllegationFactory(officer=officer,
allegation=AllegationFactory(),
allegation_category=allegation_category,
start_date=date(2011, 1, 1),
final_finding='SU')
expect(officer.complaint_category_aggregation).to.eq([{
'name':
'Use of Force',
'count':
3,
'sustained_count':
1,
'items': [{
'year': 2010,
'count': 1,
'sustained_count': 0,
'name': 'Use of Force'
}, {
'year': 2011,
'count': 1,
'sustained_count': 1,
'name': 'Use of Force'
}]
}])
def test_retrieve_recent_search_items(self):
OfficerFactory(id=8562,
first_name='Jerome',
last_name='Finnigan',
current_badge='123456')
allegation_category = AllegationCategoryFactory(
category='Use of Force')
AllegationFactory(
crid='C12345',
incident_date=datetime(2007, 1, 1, tzinfo=pytz.utc),
most_common_category=allegation_category,
)
TRRFactory(id=123)
url = reverse('api-v2:search-mobile-recent-search-items')
response = self.client.get(url, {
'officer_ids[]': 8562,
'crids[]': 'C12345',
'trr_ids[]': 123,
})
expect(response.status_code).to.eq(status.HTTP_200_OK)
expect(response.data).to.eq([{
'id': 8562,
'name': 'Jerome Finnigan',
'badge': '123456',
'type': 'OFFICER',
}, {
'id': 'C12345',
'crid': 'C12345',
'incident_date': '2007-01-01',
'category': 'Use of Force',
'type': 'CR',
}, {
'id': 123,
'type': 'TRR',
}])
def test_serialization(self):
category = AllegationCategoryFactory(category='Use of Force',
allegation_name='Subcategory')
allegation = AllegationFactory(
crid=123,
old_complaint_address='16XX N TALMAN AVE, CHICAGO IL',
most_common_category=category,
incident_date=datetime(2002, 1, 1, tzinfo=pytz.utc),
coaccused_count=12,
point=Point(-35.5, 68.9),
)
expect(AllegationMobileSerializer(allegation).data).to.eq({
'crid':
'123',
'category':
'Use of Force',
'incident_date':
'2002-01-01',
'point': {
'lon': -35.5,
'lat': 68.9
},
})
def test_request_complaint_summary(self):
allegation = AllegationFactory(crid='11',
incident_date=datetime(2002,
2,
28,
tzinfo=pytz.utc),
summary='Summary')
category = AllegationCategoryFactory(category='Use of Force')
OfficerAllegationFactory(
allegation=allegation,
officer=OfficerFactory(appointed_date=date(2001, 1, 1)),
start_date=date(2003, 2, 28),
end_date=date(2004, 4, 28),
allegation_category=category)
OfficerAllegationFactory(
allegation=allegation,
officer=OfficerFactory(appointed_date=date(2001, 1, 1)),
start_date=date(2003, 2, 28),
end_date=date(2004, 4, 28),
allegation_category=None)
officer_cache_manager.build_cached_yearly_percentiles()
officer_cache_manager.build_cached_columns()
allegation_cache_manager.cache_data()
response = self.client.get(reverse('api-v2:cr-complaint-summaries'))
expect(response.status_code).to.eq(status.HTTP_200_OK)
expect(response.data).to.eq([{
'crid':
'11',
'category_names': ['Unknown', 'Use of Force'],
'incident_date':
'2002-02-28',
'summary':
'Summary'
}])
def test_serialization(self):
category = AllegationCategoryFactory(category='Use of Force', allegation_name='Subcategory')
allegation = AllegationFactory(
crid=123,
old_complaint_address='16XX N TALMAN AVE, CHICAGO IL',
most_common_category=category,
incident_date=datetime(2002, 1, 1, tzinfo=pytz.utc),
coaccused_count=12,
point=Point(-35.5, 68.9),
)
VictimFactory(
gender='M',
race='Black',
age=35,
allegation=allegation
)
expect(AllegationSerializer(allegation).data).to.eq({
'crid': '123',
'address': '16XX N TALMAN AVE, CHICAGO IL',
'category': 'Use of Force',
'incident_date': '2002-01-01',
'victims': [
{
'gender': 'Male',
'race': 'Black',
'age': 35
}
],
'point': {
'lon': -35.5,
'lat': 68.9
},
'to': '/complaint/123/',
'sub_category': 'Subcategory',
})
def test_serialization(self):
pinned_officer = OfficerFactory(
id=1,
rank='Police Officer',
first_name='Jerome',
last_name='Finnigan',
allegation_count=10,
trr_percentile='99.99',
complaint_percentile='88.88',
civilian_allegation_percentile='77.77',
internal_allegation_percentile='66.66',
)
relevant_allegation = AllegationFactory(
crid='1',
incident_date=datetime(2002, 2, 21, tzinfo=pytz.utc),
most_common_category=AllegationCategoryFactory(
category='Operation/Personnel Violations'),
point=Point([0.01, 0.02]),
)
AttachmentFileFactory(
id=1,
file_type='document',
title='relevant document 1',
allegation=relevant_allegation,
show=True,
preview_image_url=
"https://assets.documentcloud.org/CRID-1-CR-p1-normal.gif",
url='http://cr-1-document.com/',
)
pinboard = PinboardFactory(
id='66ef1560',
title='Test pinboard',
description='Test description',
)
pinboard.officers.set([pinned_officer])
OfficerAllegationFactory(officer=pinned_officer,
allegation=relevant_allegation)
expect(pinboard.relevant_documents.count()).to.eq(1)
expect(
AllegationSerializer(
pinboard.relevant_documents[0].allegation).data).to.eq({
'crid':
'1',
'category':
'Operation/Personnel Violations',
'incident_date':
'2002-02-21',
'coaccused': [{
'id': 1,
'rank': 'Police Officer',
'full_name': 'Jerome Finnigan',
'percentile_allegation': '88.8800',
'percentile_allegation_civilian': '77.7700',
'percentile_allegation_internal': '66.6600',
'percentile_trr': '99.9900',
'allegation_count': 10,
}],
'point': {
'lon': 0.01,
'lat': 0.02,
},
})
expect(DocumentSerializer(
pinboard.relevant_documents.first()).data).to.eq({
'id':
1,
'preview_image_url':
"https://assets.documentcloud.org/CRID-1-CR-p1-normal.gif",
'url':
'http://cr-1-document.com/',
'allegation': {
'crid':
'1',
'category':
'Operation/Personnel Violations',
'incident_date':
'2002-02-21',
'coaccused': [{
'id': 1,
'rank': 'Police Officer',
'full_name': 'Jerome Finnigan',
'allegation_count': 10,
'percentile_allegation': '88.8800',
'percentile_allegation_civilian': '77.7700',
'percentile_allegation_internal': '66.6600',
'percentile_trr': '99.9900',
}],
'point': {
'lon': 0.01,
'lat': 0.02,
},
}
})
def test_crawl_and_update_attachments(self, aws_mock, crawl_copa, _):
logger = logging.getLogger('crawler.crawl_copa_data')
crawl_copa.return_value = [{
'allegation': {
'crid':
'123',
'incident_date':
datetime(2013, 4, 30, 21, 30, tzinfo=pytz.utc),
'attachment_files': [{
'file_type':
'document',
'title':
'Some Document',
'url':
'http://chicagocopa.org/document_link.pdf',
'original_url':
'http://chicagocopa.org/document_link.pdf',
'tag':
'Document',
'source_type':
'SUMMARY_REPORTS_COPA',
'external_last_updated':
datetime(2018, 10, 30, 15, 0, 3, tzinfo=pytz.utc),
}, {
'file_type':
'document',
'title':
'Some Document',
'url':
'http://chicagocopa.org/document.pdf',
'original_url':
'http://chicagocopa.org/document.pdf',
'tag':
'Document',
'source_type':
'SUMMARY_REPORTS_COPA',
'external_last_updated':
datetime(2017, 10, 30, 15, 0, 3, tzinfo=pytz.utc),
}],
'subjects': ['Subject1', 'Unknown']
},
'allegation_category': {
'category': 'Incident',
'allegation_name': 'Allegation Name'
},
'police_shooting': True
}, {
'allegation': {
'crid':
'456',
'incident_date':
datetime(2013, 4, 30, 21, 30, tzinfo=pytz.utc),
'attachment_files': [{
'file_type':
'document',
'title':
'Some PDF',
'url':
'http://chicagocopa.org/other.pdf',
'original_url':
'http://chicagocopa.org/other.pdf',
'tag':
'Document',
'source_type':
'SUMMARY_REPORTS_COPA',
'external_last_updated':
datetime(2017, 10, 30, 15, 0, 3, tzinfo=pytz.utc),
}],
'subjects': ['Subject1', 'Unknown']
},
'allegation_category': {
'category': 'Incident',
'allegation_name': 'Allegation Name'
},
'police_shooting': True
}]
AllegationCategoryFactory(category='Incident',
allegation_name='Allegation Name')
with freeze_time(datetime(2018, 4, 2, 12, 0, 1, tzinfo=pytz.utc)):
DocumentCrawlerFactory(
source_type=AttachmentSourceType.SUMMARY_REPORTS_COPA,
status=DOCUMENT_CRAWLER_SUCCESS,
num_documents=5,
num_new_documents=1,
num_updated_documents=4,
num_successful_run=1,
)
with freeze_time(datetime(2018, 4, 3, 12, 0, 1, tzinfo=pytz.utc)):
DocumentCrawlerFactory(
source_type=AttachmentSourceType.SUMMARY_REPORTS_COPA,
status=DOCUMENT_CRAWLER_FAILED,
num_successful_run=1,
)
allegation = AllegationFactory(crid='123')
attachment_file = AttachmentFileFactory(
allegation=allegation,
source_type='',
external_id='document.pdf',
original_url='http://chicagocopa.org/document.pdf')
expect(expect(DocumentCrawler.objects.count())).to.eq(2)
expect(Allegation.objects.count()).to.eq(1)
expect(Allegation.objects.get(
crid='123').attachment_files.count()).to.eq(1)
with freeze_time(datetime(2018, 4, 4, 12, 0, 1, tzinfo=pytz.utc)):
new_attachments = CopaSummaryReportsAttachmentImporter(
logger).crawl_and_update_attachments()
expect(Allegation.objects.count()).to.eq(1)
expect(AttachmentFile.objects.filter(
allegation=allegation).count()).to.eq(2)
expect(AttachmentFile.objects.get(
pk=attachment_file.pk).source_type).to.eq(
AttachmentSourceType.SUMMARY_REPORTS_COPA)
expect(DocumentCrawler.objects.count()).to.eq(3)
crawler_log = DocumentCrawler.objects.order_by('-created_at').first()
expect(crawler_log.source_type).to.eq(
AttachmentSourceType.SUMMARY_REPORTS_COPA)
expect(crawler_log.status).to.eq(DOCUMENT_CRAWLER_SUCCESS)
expect(crawler_log.num_documents).to.eq(2)
expect(crawler_log.num_new_documents).to.eq(1)
expect(crawler_log.num_updated_documents).to.eq(1)
expect(crawler_log.num_successful_run).to.eq(2)
expect(new_attachments).to.have.length(1)
expect(new_attachments[0].url).to.eq(
'http://chicagocopa.org/document_link.pdf')
log_content = b'\nCreating 1 attachments' \
b'\nUpdating 1 attachments' \
b'\nCurrent Total summary_reports_copa attachments: 2' \
b'\nDone importing!'
log_args = aws_mock.s3.put_object.call_args[1]
expect(len(log_args)).to.eq(4)
expect(log_args['Body']).to.contain(log_content)
expect(log_args['Bucket']).to.contain('crawler_logs_bucket')
expect(log_args['Key']).to.contain(
'summary_reports_copa/summary-reports-copa-2018-04-04-120001.txt')
expect(log_args['ContentType']).to.eq('text/plain')
def test_new_timeline_item(self):
officer = OfficerFactory(id=123, appointed_date=date(2000, 1, 1), rank='Police Officer')
unit1 = PoliceUnitFactory(unit_name='001', description='unit_001')
unit2 = PoliceUnitFactory(unit_name='002', description='unit_002')
OfficerHistoryFactory(officer=officer, unit=unit1, effective_date=date(2010, 1, 1), end_date=date(2011, 12, 31))
OfficerHistoryFactory(officer=officer, unit=unit2, effective_date=date(2012, 1, 1), end_date=None)
AwardFactory(officer=officer, start_date=date(2011, 3, 23), award_type='Honorable Mention')
AwardFactory(officer=officer, start_date=date(2015, 3, 23), award_type='Complimentary Letter')
AwardFactory(officer=officer, start_date=date(2011, 3, 23), award_type='Life Saving Award')
allegation = AllegationFactory(
crid='123456',
coaccused_count=4,
incident_date=datetime(2011, 8, 23, tzinfo=pytz.utc)
)
VictimFactory(allegation=allegation, gender='M', race='White', age=34)
OfficerAllegationFactory(
final_finding='UN', final_outcome='Unknown',
officer=officer, allegation=allegation,
allegation_category=AllegationCategoryFactory(category='category', allegation_name='sub category')
)
OfficerAllegationFactory.create_batch(3, allegation=allegation)
allegation2 = AllegationFactory(
crid='654321',
point=Point(35.5, 68.9),
coaccused_count=1,
incident_date=datetime(2015, 8, 23, tzinfo=pytz.utc)
)
OfficerAllegationFactory(
final_finding='UN', final_outcome='9 Day Suspension',
officer=officer, allegation=allegation2,
allegation_category=AllegationCategoryFactory(category='Use of Force', allegation_name='sub category')
)
trr2011 = TRRFactory(
officer=officer,
trr_datetime=datetime(2011, 9, 23, tzinfo=pytz.utc),
taser=True,
firearm_used=False
)
trr2015 = TRRFactory(
officer=officer,
trr_datetime=datetime(2015, 9, 23, tzinfo=pytz.utc),
taser=False,
firearm_used=False
)
SalaryFactory(officer=officer, year=2001, rank='Police Officer', spp_date=date(2001, 9, 23))
SalaryFactory(officer=officer, year=2000, rank='Junior Police Officer', spp_date=date(2000, 1, 1))
cache_managers.cache_all()
response = self.client.get(reverse('api-v2:officers-new-timeline-items', kwargs={'pk': 123}))
expect(response.status_code).to.eq(status.HTTP_200_OK)
expect(response.data).to.eq([
{
'trr_id': trr2015.id,
'date': '2015-09-23',
'kind': 'FORCE',
'taser': False,
'firearm_used': False,
'unit_name': '002',
'unit_description': 'unit_002',
'rank': 'Police Officer',
}, {
'date': '2015-08-23',
'kind': 'CR',
'crid': '654321',
'category': 'Use of Force',
'subcategory': 'sub category',
'finding': 'Unfounded',
'outcome': '9 Day Suspension',
'coaccused': 1,
'unit_name': '002',
'unit_description': 'unit_002',
'rank': 'Police Officer',
'point': {
'lon': 35.5,
'lat': 68.9
},
'victims': [],
'attachments': []
}, {
'date': '2012-01-01',
'kind': 'UNIT_CHANGE',
'unit_name': '002',
'unit_description': 'unit_002',
'rank': 'Police Officer',
}, {
'trr_id': trr2011.id,
'date': '2011-09-23',
'kind': 'FORCE',
'taser': True,
'firearm_used': False,
'unit_name': '001',
'unit_description': 'unit_001',
'rank': 'Police Officer',
}, {
'date': '2011-08-23',
'kind': 'CR',
'crid': '123456',
'category': 'category',
'subcategory': 'sub category',
'finding': 'Unfounded',
'outcome': 'Unknown',
'coaccused': 4,
'unit_name': '001',
'unit_description': 'unit_001',
'rank': 'Police Officer',
'victims': [
{
'race': 'White',
'age': 34,
'gender': 'Male',
}
],
'attachments': [],
}, {
'date': '2011-03-23',
'kind': 'AWARD',
'award_type': 'Life Saving Award',
'unit_name': '001',
'unit_description': 'unit_001',
'rank': 'Police Officer',
}, {
'date': '2010-01-01',
'kind': 'UNIT_CHANGE',
'unit_name': '001',
'unit_description': 'unit_001',
'rank': 'Police Officer',
}, {
'date': '2001-09-23',
'kind': 'RANK_CHANGE',
'unit_name': '',
'unit_description': '',
'rank': 'Police Officer',
}, {
'date': '2000-01-01',
'kind': 'JOINED',
'unit_name': '',
'unit_description': '',
'rank': 'Junior Police Officer',
},
])
def test_summary(self):
officer = OfficerFactory(
tags=[],
first_name='Kevin', last_name='Kerl', id=123, race='White', gender='M',
appointed_date=date(2017, 2, 27), rank='PO', resignation_date=date(2017, 12, 27),
active=ACTIVE_YES_CHOICE, birth_year=1910, complaint_percentile=32.5,
sustained_count=1, allegation_count=1, discipline_count=1, trr_count=1,
civilian_compliment_count=1, honorable_mention_count=1, major_award_count=1,
last_unit_id=1, current_badge='123456', current_salary=90000, has_unique_name=True
)
allegation = AllegationFactory()
allegation_category = AllegationCategoryFactory(category='Use of Force')
OfficerHistoryFactory(officer=officer, unit=PoliceUnitFactory(id=1, unit_name='CAND', description=''))
ComplainantFactory(allegation=allegation, race='White', age=18, gender='F')
OfficerBadgeNumberFactory(officer=officer, star='123456', current=True)
OfficerAllegationFactory(
officer=officer, allegation=allegation, allegation_category=allegation_category,
final_finding='SU', start_date=date(2000, 1, 1), disciplined=True
)
AwardFactory(officer=officer, award_type='Complimentary Letter')
AwardFactory(officer=officer, award_type='Honored Police Star')
AwardFactory(officer=officer, award_type='Honorable Mention')
SalaryFactory(officer=officer, salary=50000, year=2015)
SalaryFactory(officer=officer, salary=90000, year=2017)
TRRFactory(officer=officer)
officer_cache_manager.build_cached_columns()
allegation_cache_manager.cache_data()
response = self.client.get(reverse('api-v2:officers-summary', kwargs={'pk': 123}))
expect(response.status_code).to.eq(status.HTTP_200_OK)
expected_data = {
'id': 123,
'unit': {
'id': 1,
'unit_name': 'CAND',
'description': '',
'long_unit_name': 'Unit CAND',
},
'date_of_appt': '2017-02-27',
'date_of_resignation': '2017-12-27',
'active': 'Active',
'rank': 'PO',
'full_name': 'Kevin Kerl',
'race': 'White',
'badge': '123456',
'historic_units': [{
'id': 1,
'unit_name': 'CAND',
'description': '',
'long_unit_name': 'Unit CAND'
}],
'gender': 'Male',
'birth_year': 1910,
'sustained_count': 1,
'civilian_compliment_count': 1,
'allegation_count': 1,
'discipline_count': 1,
'honorable_mention_count': 1,
'to': '/officer/123/kevin-kerl/',
'url': 'http://cpdb.lvh.me/officer/kevin-kerl/123',
'current_salary': 90000,
'trr_count': 1,
'major_award_count': 1,
'unsustained_count': 0,
'percentile_allegation': '32.5000',
'coaccusals': [],
'percentiles': [],
'tags': [],
'historic_badges': [],
'has_unique_name': True
}
expect(response.data).to.eq(expected_data)
def test_crawl_and_update_attachments(self, aws_mock, crawl_copa, _):
logger = logging.getLogger('crawler.crawl_copa_data')
crawl_copa.return_value = [{
'allegation': {
'crid':
'123',
'incident_date':
datetime(2013, 4, 30, 21, 30, tzinfo=pytz.utc),
'attachment_files': [{
'file_type':
'audio',
'title':
'Audio Clip',
'url':
'http://chicagocopa.org/audio_link.mp3',
'original_url':
'http://chicagocopa.org/audio_link.mp3',
'tag':
'Audio',
'source_type':
'PORTAL_COPA',
'external_last_updated':
datetime(2018, 10, 30, 15, 0, 3, tzinfo=pytz.utc),
}, {
'file_type':
'document',
'title':
'Document',
'url':
'http://chicagocopa.org/document.pdf',
'original_url':
'http://chicagocopa.org/document.pdf',
'tag':
'Document',
'source_type':
'PORTAL_COPA',
'external_last_updated':
datetime(2017, 10, 30, 15, 0, 3, tzinfo=pytz.utc),
}],
'subjects': ['Subject', '', 'Unknown'],
},
'allegation_category': {
'category': 'Incident',
'allegation_name': 'Allegation Name'
},
'police_shooting': True
}, {
'allegation': {
'crid':
'456',
'incident_date':
datetime(2013, 4, 30, 21, 30, tzinfo=pytz.utc),
'attachment_files': [{
'file_type':
'document',
'title':
'Document',
'url':
'http://chicagocopa.org/other.pdf',
'original_url':
'http://chicagocopa.org/other.pdf',
'tag':
'Document',
'source_type':
'PORTAL_COPA',
'external_last_updated':
datetime(2017, 10, 30, 15, 0, 3, tzinfo=pytz.utc),
}],
'subjects': ['Subject 2'],
},
'allegation_category': {
'category': 'Incident',
'allegation_name': 'Allegation Name'
},
'police_shooting': True
}]
AllegationCategoryFactory(category='Incident',
allegation_name='Allegation Name')
allegation = AllegationFactory(crid='123')
attachment_file = AttachmentFileFactory(
allegation=allegation,
source_type='',
external_id='document.pdf',
original_url='http://chicagocopa.org/document.pdf')
expect(DocumentCrawler.objects.count()).to.eq(0)
expect(Allegation.objects.count()).to.eq(1)
expect(Allegation.objects.get(
crid='123').attachment_files.count()).to.eq(1)
with freeze_time(datetime(2018, 4, 4, 12, 0, 1, tzinfo=pytz.utc)):
new_attachments = CopaPortalAttachmentImporter(
logger).crawl_and_update_attachments()
expect(Allegation.objects.count()).to.eq(1)
expect(Allegation.objects.get(crid='123').subjects).to.eq(['Subject'])
expect(AttachmentFile.objects.filter(
allegation=allegation).count()).to.eq(2)
expect(AttachmentFile.objects.get(
pk=attachment_file.pk).source_type).to.eq(
AttachmentSourceType.PORTAL_COPA)
expect(DocumentCrawler.objects.count()).to.eq(1)
crawler_log = DocumentCrawler.objects.first()
expect(crawler_log.source_type).to.eq(AttachmentSourceType.PORTAL_COPA)
expect(crawler_log.num_documents).to.eq(2)
expect(crawler_log.num_new_documents).to.eq(1)
expect(crawler_log.num_updated_documents).to.eq(1)
expect(new_attachments).to.have.length(1)
expect(new_attachments[0].title).to.eq('Audio Clip')
expect(new_attachments[0].url).to.eq(
'http://chicagocopa.org/audio_link.mp3')
log_content = b'Creating 1 attachments' \
b'\nUpdating 1 attachments' \
b'\nCurrent Total portal_copa attachments: 2' \
b'\nDone importing!'
log_args = aws_mock.s3.put_object.call_args[1]
expect(len(log_args)).to.eq(4)
expect(log_args['Body']).to.contain(log_content)
expect(log_args['Bucket']).to.eq('crawler_logs_bucket')
expect(log_args['Key']).to.eq(
'portal_copa/portal-copa-2018-04-04-120001.txt')
expect(log_args['ContentType']).to.eq('text/plain')
def test_serialization(self):
with freeze_time(datetime(2018, 4, 3, 12, 0, 10, tzinfo=pytz.utc)):
officer_1 = OfficerFactory(
first_name='Jerome',
last_name='Finnigan',
allegation_count=0,
complaint_percentile='0.0000',
trr_percentile='0.0000',
civilian_allegation_percentile='0.0000',
internal_allegation_percentile='0.0000',
)
officer_2 = OfficerFactory(
first_name='Joe',
last_name='Parker',
allegation_count=5,
complaint_percentile='50.0000',
trr_percentile='50.0000',
civilian_allegation_percentile='50.0000',
internal_allegation_percentile='50.0000',
)
officer_3 = OfficerFactory(
first_name='John',
last_name='Hurley',
allegation_count=10,
complaint_percentile='99.9999',
trr_percentile='99.9999',
civilian_allegation_percentile='99.9999',
internal_allegation_percentile='99.9999',
)
allegation_1 = AllegationFactory(
crid='111111',
most_common_category=AllegationCategoryFactory(
category='Use Of Force'),
incident_date=datetime(2001, 1, 1, tzinfo=pytz.utc),
)
allegation_2 = AllegationFactory(
crid='222222',
incident_date=datetime(2002, 2, 2, tzinfo=pytz.utc),
)
trr_1 = TRRFactory(id='111',
trr_datetime=datetime(2001,
1,
1,
tzinfo=pytz.utc))
ActionResponseFactory(trr=trr_1, force_type='Use Of Force')
trr_2 = TRRFactory(id='222',
trr_datetime=datetime(2002,
2,
2,
tzinfo=pytz.utc))
pinboard = PinboardFactory(
id='aaaa1111',
title='Pinboard 1',
description='Pinboard description 1',
officers=[officer_1, officer_2, officer_3],
allegations=[allegation_1, allegation_2],
trrs=[trr_1, trr_2],
)
setattr(pinboard, 'child_pinboard_count', 2)
expect(PinboardSerializer(pinboard).data).to.eq({
'id':
'aaaa1111',
'title':
'Pinboard 1',
'description':
'Pinboard description 1',
'created_at':
'2018-04-03T12:00:10Z',
'officers_count':
3,
'allegations_count':
2,
'trrs_count':
2,
'child_pinboard_count':
2,
'officers': [
{
'id': officer_3.id,
'name': 'John Hurley',
'count': 10,
'percentile_allegation': '99.9999',
'percentile_trr': '99.9999',
'percentile_allegation_civilian': '99.9999',
'percentile_allegation_internal': '99.9999',
},
{
'id': officer_2.id,
'name': 'Joe Parker',
'count': 5,
'percentile_allegation': '50.0000',
'percentile_trr': '50.0000',
'percentile_allegation_civilian': '50.0000',
'percentile_allegation_internal': '50.0000',
},
{
'id': officer_1.id,
'name': 'Jerome Finnigan',
'count': 0,
'percentile_allegation': '0.0000',
'percentile_trr': '0.0000',
'percentile_allegation_civilian': '0.0000',
'percentile_allegation_internal': '0.0000',
},
],
'allegations': [
{
'crid': '222222',
'category': 'Unknown',
'incident_date': '2002-02-02',
},
{
'crid': '111111',
'category': 'Use Of Force',
'incident_date': '2001-01-01',
},
],
'trrs': [{
'id': 222,
'trr_datetime': '2002-02-02',
'category': 'Unknown',
}, {
'id': 111,
'trr_datetime': '2001-01-01',
'category': 'Use Of Force',
}],
})
def test_serialization(self):
category = AllegationCategoryFactory(
category='Use of Force',
allegation_name='Improper Search Of Person')
allegation = AllegationFactory(
crid='123',
is_officer_complaint=True,
incident_date=datetime(2005, 12, 31, tzinfo=pytz.utc),
most_common_category=category,
old_complaint_address='34XX Douglas Blvd',
)
attachment = AttachmentFileFactory(tag='TRR',
allegation=allegation,
title='CR document',
id='123456',
url='http://cr-document.com/',
file_type=MEDIA_TYPE_DOCUMENT)
officer = OfficerFactory(id=8562,
first_name='Jerome',
last_name='Finnigan',
allegation_count=5,
trr_percentile=80,
complaint_percentile=85,
civilian_allegation_percentile=90,
internal_allegation_percentile=95)
officer_allegation = OfficerAllegationFactory(
id=1,
officer=officer,
allegation=allegation,
recc_outcome='10 Day Suspension',
final_finding='SU',
final_outcome='Separation',
disciplined=True,
allegation_category=category)
VictimFactory(gender='M', race='Black', age=35, allegation=allegation)
setattr(allegation, 'prefetch_filtered_attachment_files', [attachment])
allegation.officerallegation_set.set([officer_allegation])
expect(SocialGraphCRDetailSerializer(allegation).data).to.eq({
'kind':
'CR',
'crid':
'123',
'to':
'/complaint/123/',
'category':
'Use of Force',
'subcategory':
'Improper Search Of Person',
'incident_date':
'2005-12-31',
'address':
'34XX Douglas Blvd',
'victims': [{
'gender': 'Male',
'race': 'Black',
'age': 35
}],
'coaccused': [{
'id': 8562,
'full_name': 'Jerome Finnigan',
'allegation_count': 5,
'percentile_allegation': '85.0000',
'percentile_allegation_civilian': '90.0000',
'percentile_allegation_internal': '95.0000',
'percentile_trr': '80.0000',
}],
'attachments': [{
'id': '123456',
'title': 'CR document',
'url': 'http://cr-document.com/',
'file_type': MEDIA_TYPE_DOCUMENT,
}],
'officer_ids': [8562],
})
