def test_next_document_id_is_none_if_all_documents_have_tags(self):
allegation = AllegationFactory(crid='456')
with freeze_time('2017-08-04'):
AttachmentFileFactory(
allegation=allegation,
file_type='document',
tags=['tag123'],
)
with freeze_time('2017-09-04'):
AttachmentFileFactory(
allegation=allegation,
file_type='document',
tags=['tag124'],
)
with freeze_time('2017-10-05'):
attachment = AttachmentFileFactory(
allegation=allegation,
file_type='document',
tags=['tag125'],
)
with freeze_time('2017-08-05'):
AttachmentFileFactory(
allegation=allegation,
file_type='audio',
)
next_document_id = AuthenticatedAttachmentFileSerializer(
attachment).data['next_document_id']
expect(next_document_id).to.eq(None)
def setUp(self):
self.attachment_file_admin = AttachmentFileAdmin(
AttachmentFile, AdminSite())
self.request = RequestFactory()
allegation_1 = AllegationFactory(
incident_date=datetime(2005, 12, 31, tzinfo=pytz.utc))
allegation_2 = AllegationFactory(
incident_date=datetime(2007, 12, 31, tzinfo=pytz.utc))
with freeze_time(datetime(2018, 4, 4, 12, 0, 1, tzinfo=pytz.utc)):
self.attachment_file_1 = AttachmentFileFactory(
id=1,
allegation=allegation_1,
title='Title 1',
source_type='DOCUMENTCLOUD',
)
self.attachment_file_1.tags.set('Tactical', 'Complaint', 'Taser')
with freeze_time(datetime(2018, 5, 5, 12, 0, 1, tzinfo=pytz.utc)):
self.attachment_file_2 = AttachmentFileFactory(
id=2,
allegation=allegation_2,
title='Title 2',
source_type='PORTAL_COPA',
)
self.attachment_file_2.tags.set('Taser')
def test_not_update_when_cannot_save_documentcloud_document(self):
self.mock_doc.save.side_effect = HTTPError('', '404', '', None, None)
attachment = AttachmentFileFactory(external_id=1,
source_type='DOCUMENTCLOUD')
attachment.update_to_documentcloud('title', 'some title')
expect(self.mock_doc.title).to.be.eq('some title')
expect(self.mock_logger.error).to.be.called_with(
'Cannot save document with external id 1 on DocumentCloud')
def test_not_update_when_cannot_get_documentcloud_document(self):
self.mock_client.documents.get.side_effect = DoesNotExistError()
attachment = AttachmentFileFactory(external_id=1,
source_type='DOCUMENTCLOUD')
attachment.update_to_documentcloud('title', 'some title')
expect(self.mock_doc.save).not_to.be.called()
expect(self.mock_logger.error).to.be.called_with(
'Cannot find document with external id 1 on DocumentCloud')
def test_update_successfully(self):
attachment = AttachmentFileFactory(external_id=1,
source_type='DOCUMENTCLOUD')
attachment.update_to_documentcloud('title', 'some title')
expect(self.mock_client.documents.get).to.be.called_with(1)
expect(self.mock_doc.save).to.be.called()
expect(self.mock_logger.error).not_to.be.called()
expect(self.mock_doc.title).to.be.eq('some title')
def test_attachment_shown_manager(self):
AttachmentFileFactory(id=1)
AttachmentFileFactory(id=2, show=False)
AttachmentFileFactory(id=3, show=False)
shown_attachments = AttachmentFile.showing.all()
expect(shown_attachments).to.have.length(1)
expect(shown_attachments[0].id).to.eq(1)
def test_relevant_documents_via_accused_officers(self):
pinned_officer_1 = OfficerFactory(id=1)
pinned_officer_2 = OfficerFactory(id=2)
pinned_officer_3 = OfficerFactory(id=3)
relevant_allegation_1 = AllegationFactory(crid='1',
incident_date=datetime(
2002,
2,
21,
tzinfo=pytz.utc))
relevant_allegation_2 = AllegationFactory(crid='2',
incident_date=datetime(
2002,
2,
22,
tzinfo=pytz.utc))
not_relevant_allegation = AllegationFactory(crid='not relevant')
relevant_document_1 = AttachmentFileFactory(
id=1,
file_type='document',
allegation=relevant_allegation_1,
show=True)
relevant_document_2 = AttachmentFileFactory(
id=2,
file_type='document',
allegation=relevant_allegation_2,
show=True)
AttachmentFileFactory(id=998,
file_type='document',
title='relevant but not show',
allegation=relevant_allegation_1,
show=False)
AttachmentFileFactory(id=999,
file_type='document',
title='not relevant',
allegation=not_relevant_allegation,
show=True)
pinboard = PinboardFactory(
title='Test pinboard',
description='Test description',
)
pinboard.officers.set(
[pinned_officer_1, pinned_officer_2, pinned_officer_3])
OfficerAllegationFactory(officer=pinned_officer_1,
allegation=relevant_allegation_1)
OfficerAllegationFactory(officer=pinned_officer_2,
allegation=relevant_allegation_2)
relevant_documents = list(pinboard.relevant_documents)
expect(relevant_documents).to.have.length(2)
expect(relevant_documents[0].id).to.eq(relevant_document_2.id)
expect(relevant_documents[0].allegation.crid).to.eq('2')
expect(relevant_documents[1].id).to.eq(relevant_document_1.id)
expect(relevant_documents[1].allegation.crid).to.eq('1')
def test_attachments(self):
allegation = AllegationFactory()
officer_allegation = OfficerAllegationFactory(allegation=allegation)
attachment_1 = AttachmentFileFactory(allegation=allegation)
attachment_2 = AttachmentFileFactory(allegation=allegation)
result = list(officer_allegation.attachments)
expect(result).to.have.length(2)
expect(result).to.contain(attachment_1)
expect(result).to.contain(attachment_2)
def test_not_update_when_no_change(self):
attachment = AttachmentFileFactory(external_id=1,
source_type='DOCUMENTCLOUD',
title='no changed title')
setattr(self.mock_doc, 'title', 'no changed title')
attachment.update_to_documentcloud('title', 'no changed title')
expect(self.mock_client.documents.get).to.be.called_with(1)
expect(self.mock_doc.save).not_to.be.called()
def test_handle(self, send_email_mock, summary_reports_importer_mock,
portal_importer_mock):
attachment_file_1 = AttachmentFileFactory(
source_type=AttachmentSourceType.PORTAL_COPA)
attachment_file_2 = AttachmentFileFactory(
source_type=AttachmentSourceType.SUMMARY_REPORTS_COPA)
portal_importer_mock.return_value = [attachment_file_1]
summary_reports_importer_mock.return_value = [attachment_file_2]
call_command('crawl_copa_data')
expect(send_email_mock).to.be.called_with(
[attachment_file_1, attachment_file_2])
def test_filtered_attachment_files(self):
allegation = AllegationFactory()
attachment1 = AttachmentFileFactory(tag='Other', allegation=allegation, file_type=MEDIA_TYPE_DOCUMENT)
AttachmentFileFactory(
tag='Other', allegation=allegation, file_type=MEDIA_TYPE_DOCUMENT, show=False)
AttachmentFileFactory(
tag='Other', title='CR Arrest Report Herron', allegation=allegation, file_type=MEDIA_TYPE_DOCUMENT
)
AttachmentFileFactory(
tag='OCIR', allegation=allegation, file_type=MEDIA_TYPE_DOCUMENT)
AttachmentFileFactory(tag='AR', allegation=allegation, file_type=MEDIA_TYPE_DOCUMENT)
expect(list(allegation.filtered_attachment_files)).to.eq([attachment1])
def setUp(self):
self.attachment_file_admin = AttachmentFileAdmin(
AttachmentFile, AdminSite())
self.request = RequestFactory()
allegation_1 = AllegationFactory(
incident_date=datetime(2005, 12, 31, tzinfo=pytz.utc))
allegation_2 = AllegationFactory(
incident_date=datetime(2007, 12, 31, tzinfo=pytz.utc))
self.attachment_file_1 = AttachmentFileFactory(allegation=allegation_1)
self.attachment_file_2 = AttachmentFileFactory(allegation=allegation_2)
self.attachment_file_3 = AttachmentFileFactory(allegation=allegation_2)
self.attachment_file_1.tags.set('Tactical', 'Complaint', 'Taser')
self.attachment_file_2.tags.set('Taser')
def test_request_reprocess_missing_text_update_count_even_if_status_code_is_not_200(
self, _):
AttachmentFileFactory(
file_type='document',
text_content='',
external_id='DOCUMENTCLOUD_empty_text_id',
source_type=AttachmentSourceType.DOCUMENTCLOUD,
)
AttachmentFileFactory(
file_type='document',
text_content='',
external_id='PORTAL_COPA_DOCUMENTCLOUD_empty_text_id',
source_type=AttachmentSourceType.PORTAL_COPA_DOCUMENTCLOUD,
reprocess_text_count=1,
)
AttachmentFileFactory(
file_type='document',
text_content='',
external_id='DOCUMENTCLOUD_tries_enough_id',
source_type=AttachmentSourceType.DOCUMENTCLOUD,
reprocess_text_count=3,
)
with patch(
'document_cloud.documentcloud_session.DocumentCloudSession.post',
return_value=Mock(status_code=200)):
session = DocumentCloudSession(self.log_func)
with patch(
'document_cloud.documentcloud_session.DocumentCloudSession.post',
return_value=Mock(status_code=404,
json=Mock(return_value=''))):
session.request_reprocess_missing_text_documents()
expect(self.log_func).to.be.any_call(
'Sent reprocessing text requests: 0 success, 2 failure, 1 skipped for 3 no-text documents'
)
expect(
AttachmentFile.objects.get(
external_id='DOCUMENTCLOUD_empty_text_id').reprocess_text_count
).to.equal(1)
expect(
AttachmentFile.objects.get(
external_id='PORTAL_COPA_DOCUMENTCLOUD_empty_text_id').
reprocess_text_count).to.equal(2)
expect(
AttachmentFile.objects.get(
external_id='DOCUMENTCLOUD_tries_enough_id').
reprocess_text_count).to.equal(3)
def test_invoke_create_zip(self, aws_mock):
exception = botocore.exceptions.ClientError(
error_response={'Error': {
'Code': 'NoSuchKey'
}},
operation_name='get_object')
aws_mock.s3.get_object.side_effect = exception
allegation = AllegationFactory(crid='1')
allegation_456 = AllegationFactory(crid='456')
AttachmentFileFactory(allegation=allegation,
source_type='DOCUMENTCLOUD',
external_id='ABC',
title='allegation 1 attachment')
AttachmentFileFactory(allegation=allegation, source_type='COPA')
AttachmentFileFactory(allegation=allegation_456,
source_type='DOCUMENTCLOUD')
AttachmentFileFactory(allegation=allegation_456,
source_type='PORTAL_COPA_DOCUMENTCLOUD')
officer = OfficerFactory(id=1,
first_name='Jerome',
last_name='Finnigan')
OfficerAllegationFactory(officer=officer, allegation=allegation)
allegation_2 = AllegationFactory(crid='2')
allegation_789 = AllegationFactory(crid='789')
AttachmentFileFactory(allegation=allegation_2,
source_type='DOCUMENTCLOUD',
external_id='XYZ',
title='allegation 2 attachment')
AttachmentFileFactory(allegation=allegation_2, source_type='COPA')
AttachmentFileFactory(allegation=allegation_789,
source_type='DOCUMENTCLOUD')
AttachmentFileFactory(allegation=allegation_789,
source_type='PORTAL_COPA_DOCUMENTCLOUD')
investigator = InvestigatorFactory(officer=officer)
InvestigatorAllegationFactory(allegation=allegation_2,
investigator=investigator)
officer.invoke_create_zip(with_docs=True)
expect(aws_mock.s3.get_object).to.be.called_with(
Bucket='officer_content_bucket',
Key='zip_with_docs/Jerome_Finnigan_with_docs.zip')
_, kwargs = aws_mock.lambda_client.invoke_async.call_args
expect(kwargs['FunctionName']).to.eq('createOfficerZipFileTest')
expect(json.loads(kwargs['InvokeArgs'])).to.eq({
'key': 'zip_with_docs/Jerome_Finnigan_with_docs.zip',
'bucket': 'officer_content_bucket',
'file_map': {
'xlsx/1/accused.xlsx': 'accused.xlsx',
'xlsx/1/use_of_force.xlsx': 'use_of_force.xlsx',
'xlsx/1/investigator.xlsx': 'investigator.xlsx',
'xlsx/1/documents.xlsx': 'documents.xlsx',
'pdf/ABC': f'documents/allegation 1 attachment.pdf',
'pdf/XYZ': f'investigators/allegation 2 attachment.pdf'
}
})
def test_investigator_attachments(self):
allegation = AllegationFactory(crid='123')
allegation_456 = AllegationFactory(crid='456')
attachment_1 = AttachmentFileFactory(allegation=allegation,
source_type='DOCUMENTCLOUD')
attachment_2 = AttachmentFileFactory(
allegation=allegation, source_type='PORTAL_COPA_DOCUMENTCLOUD')
AttachmentFileFactory(allegation=allegation,
source_type='PORTAL_COPA_DOCUMENTCLOUD',
show=False)
AttachmentFileFactory(allegation=allegation, source_type='COPA')
AttachmentFileFactory(allegation=allegation,
source_type='COPA',
show=False)
AttachmentFileFactory(allegation=allegation_456,
source_type='DOCUMENTCLOUD')
AttachmentFileFactory(allegation=allegation_456,
source_type='PORTAL_COPA_DOCUMENTCLOUD')
AttachmentFileFactory(allegation=allegation_456,
source_type='PORTAL_COPA_DOCUMENTCLOUD',
show=False)
investigator = InvestigatorFactory(officer=OfficerFactory(id=1))
InvestigatorAllegationFactory(allegation=allegation,
investigator=investigator)
expect({
attachment.id
for attachment in investigator.officer.investigator_attachments
}).to.eq({attachment_1.id, attachment_2.id})
def test_update_allegation_summary_not_update_if_do_not_have_text_content(
self):
allegation = AllegationFactory(summary='')
attachment_file = AttachmentFileFactory(
source_type=AttachmentSourceType.
SUMMARY_REPORTS_COPA_DOCUMENTCLOUD,
text_content='',
allegation=allegation,
)
expect(attachment_file.update_allegation_summary()).to.be.false()
expect(allegation.summary).to.eq('')
expect(allegation.is_extracted_summary).to.be.false()
def test_update_allegation_summary_not_update_if_can_not_parse_summary_from_text_content(
self):
text_content = 'This is invalid text content.'
allegation = AllegationFactory(summary='')
attachment_file = AttachmentFileFactory(
source_type=AttachmentSourceType.
SUMMARY_REPORTS_COPA_DOCUMENTCLOUD,
text_content=text_content,
allegation=allegation,
)
expect(attachment_file.update_allegation_summary()).to.be.false()
expect(allegation.summary).to.eq('')
expect(allegation.is_extracted_summary).to.be.false()
def test_update_allegation_summary_not_update_if_allegation_already_has_summary(
self):
text_content = \
'CIVILIAN OFFICE OF POLICE ACCOUNTABILITY ' \
'\nSUMMARY REPORT OF INVESTIGATION1' \
'\nI. EXECUTIVE SUMMARY' \
'\nDate of Incident: September 25, 2015' \
'\nTime of Incident: 8:53 pm.' \
'\nLocation of Incident: N. Central Park Avenue, Chicago, IL' \
'\nDate of COPA Notification: September 25, 2015' \
'\nTime of COPA Notification: 9:15 pm.' \
'\nOn September 25, 2015, at approximately 8:50 pm, Officers A and responded to a' \
'\ncall of a disturbance with a mentally ill subject, Involved Civilian 1 (Involved Civilian 1), at ' \
'\nN. Central Park Avenue, Chicago, IL. Upon arrival, the officers met with Involved Civilian' \
'\nmother, Involved Civilian 2 (Involved Civilian 2), who stated that Involved Civilian 1 was acting' \
'\ncrazy, had a knife, and would not come out of his bedroom. Officers A and B, along with assisting' \
'\nOfficers and D, entered the residence and knocked on Involved Civilian bedroom door.' \
'\nInvolved Civilian 1 opened the bedroom door while holding a knife in his hand. The officers' \
'\nordered Involved Civilian 1 to drop the knife, but he did not comply. Involved Civilian 1 exited' \
'\nhis bedroom and approached Officer A as he stood in the kitchen, which was adjacent to the' \
'\nbedroom. Officer attempted to tase Involved Civilian 1, but the Taser did not appear to have any' \
'\neffect on Involved Civilian 1. Involved Civilian 1 continued to approach Officer A, while still' \
'\nholding the knife in his hand, at which time Officer A discharged his firearm five times, striking' \
'\nInvolved Civilian 1 several times about the body. Involved Civilian 1 was declared dead at 2133' \
'\nhours at Mt. Sinai hospital. investigation demonstrates that Officer use of deadly' \
'\nforce complied with Chicago Police Department rules and directives.' \
'\nII. INVOLVED PARTIES' \
'\nInvolved Officer Officer A, star Employee Date of' \
'\nAppointment: Chicago Police Officer, Unit of' \
'\nAssignment: XX, DOB: 1983, Male White.' \
'\nInvolved Individual#1: Involved Civilian 1, DOB: 1982, Male, Black.' \
'\n1 On September 15, 2017, the Civilian Office of Police Accountability (COPA) replaced the ' \
'\nIndependent Police' \
allegation = AllegationFactory(
summary='This allegation already has summary.')
attachment_file = AttachmentFileFactory(
source_type=AttachmentSourceType.
SUMMARY_REPORTS_COPA_DOCUMENTCLOUD,
text_content=text_content,
allegation=allegation,
)
expect(attachment_file.update_allegation_summary()).to.be.false()
expect(
allegation.summary).to.eq('This allegation already has summary.')
expect(allegation.is_extracted_summary).to.be.false()
def test_upload_summary_reports_copa_documents(self, DocumentCloudMock):
logger = logging.getLogger('crawler.crawl_copa_data')
DocumentCloudMock().documents.upload.return_value = PropertyMock(
id='5396984-crid-123-cr-tactical-response-report',
title='CRID 123 CR Tactical Response Report',
canonical_url=
'https://www.documentcloud.org/documents/5396984-tactical-response-report.html',
normal_image_url=
'https://www.documentcloud.org/documents/tactical-response-report-p1-normal.gif',
created_at=datetime(2017, 8, 4, 14, 30, 00, tzinfo=pytz.utc),
updated_at=datetime(2017, 8, 5, 14, 30, 00, tzinfo=pytz.utc),
resources=None)
allegation = AllegationFactory(crid='123')
AttachmentFileFactory(
external_id='123-OCIR-Redacted.pdf',
allegation=allegation,
source_type=AttachmentSourceType.SUMMARY_REPORTS_COPA,
file_type=MEDIA_TYPE_DOCUMENT,
title='COPA Summary Report',
original_url=
'https://www.chicagocopa.org/wp-content/uploads/2017/10/Log-1086285-TRR-Redacted.pdf',
upload_fail_attempts=1,
)
CopaSummaryReportsAttachmentImporter(logger).upload_to_documentcloud()
AttachmentFile.objects.get(pending_documentcloud_id='5396984')
expect(DocumentCloudMock().documents.upload).to.be.called_with(
'https://www.chicagocopa.org/wp-content/uploads/2017/10/Log-1086285-TRR-Redacted.pdf',
title='CRID 123 CR COPA Summary Report',
description=AttachmentSourceType.
SUMMARY_REPORTS_COPA_DOCUMENTCLOUD,
access='public',
force_ocr=True)
def test_logging_when_sending_cr_attachment_available_email_raise_error(
self, mock_send_email, mock_logger):
mock_send_email.side_effect = SMTPException('Sending failed')
EmailTemplateFactory(
subject='To {name}',
body='This message is related to crid {pk} with url {url}',
from_email='*****@*****.**',
type=CR_ATTACHMENT_AVAILABLE)
allegation_123 = AllegationFactory(crid='123')
AttachmentRequestFactory(allegation=allegation_123,
email='*****@*****.**',
noti_email_sent=False)
AttachmentFileFactory(allegation=allegation_123)
new_attachments = AttachmentFile.objects.all()
send_cr_attachment_available_email(new_attachments)
expect(AttachmentRequest.objects.filter(
noti_email_sent=True).count()).to.eq(0)
expect(
AttachmentRequest.objects.filter(
noti_email_sent=False).count()).to.eq(1)
expect(mock_logger.info).to.be.called_with(
'Cannot send notification email for crid 123 to [email protected]'
)
def test_update_video_thumbnail(self, _, vimeo_api, crawl_copa, __):
logger = logging.getLogger('crawler.crawl_copa_data')
crawl_copa.return_value = [{
'allegation': {
'crid':
'123',
'incident_date':
datetime(2013, 4, 30, 21, 30, tzinfo=pytz.utc),
'attachment_files': [{
'file_type':
'video',
'title':
'video file',
'url':
'https://player.vimeo.com/video/288225991',
'original_url':
'https://player.vimeo.com/video/288225991',
'tag':
'Video',
'source_type':
'PORTAL_COPA',
'external_last_updated':
datetime(2018, 10, 30, 15, 0, 3, tzinfo=pytz.utc),
}],
'subjects': ['Subject1', 'Unknown']
},
'allegation_category': {
'category': 'Incident',
'allegation_name': 'Allegation Name'
},
'police_shooting': True
}]
vimeo_api.return_value = {
'id': 288225991,
'title': 'Log# 1082195 3rd Party Clip',
'description': 'Log# 1082195 3rd Party Clip',
'url': 'https://vimeo.com/307768537',
'upload_date': '2018-12-21 15:47:48',
'thumbnail_small':
'https://i.vimeocdn.com/video/747800241_100x75.webp',
'thumbnail_medium':
'https://i.vimeocdn.com/video/747800241_200x150.webp',
'thumbnail_large':
'https://i.vimeocdn.com/video/747800241_640.webp',
}
AllegationCategoryFactory(category='Incident',
allegation_name='Allegation Name')
attachment_file = AttachmentFileFactory(
allegation__crid='123',
title='old_title',
source_type=AttachmentSourceType.PORTAL_COPA,
external_id='288225991',
original_url='https://player.vimeo.com/video/288225991',
preview_image_url=None)
CopaPortalAttachmentImporter(logger).crawl_and_update_attachments()
expect(AttachmentFile.objects.get(
pk=attachment_file.pk).title).to.eq('video file')
expect(AttachmentFile.objects.get(pk=attachment_file.pk).preview_image_url). \
to.eq('https://i.vimeocdn.com/video/747800241_100x75.webp')
def test_not_update_video_thumbnail_when_source_is_not_vimeo(
self, _, crawl_copa):
logger = logging.getLogger('crawler.crawl_copa_data')
crawl_copa.return_value = [{
'attachments': [{
'type': 'video',
'link': 'https://player.fake_video.org/video/288225991',
'title': 'video file',
'last_updated': '2018-10-30T15:00:03+00:00',
}],
'date':
'04-30-2013',
'log_number':
'123',
'time':
'04-30-2013 9:30 pm',
'type':
'Allegation Name',
'subjects': ['Subject', '', 'Unknown'],
}]
AllegationCategoryFactory(category='Incident',
allegation_name='Allegation Name')
attachment_file = AttachmentFileFactory(
allegation__crid='123',
title='old_title',
source_type=AttachmentSourceType.PORTAL_COPA_DOCUMENTCLOUD,
external_id='288225991',
original_url='https://player.fake_video.org/video/288225991',
preview_image_url=None)
CopaPortalAttachmentImporter(logger).crawl_and_update_attachments()
expect(
AttachmentFile.objects.get(
pk=attachment_file.pk).preview_image_url).be.none()
def test_create_success(self):
allegation = AllegationFactory(crid=123456)
attachment_file = AttachmentFileFactory(
id=123,
allegation=allegation,
title='CR document 10',
tag='CR',
url='https://cr-document.com/10',
file_type=MEDIA_TYPE_DOCUMENT,
preview_image_url='http://preview.com/url3',
)
expect(AttachmentTracking.objects.count()).to.eq(0)
response = self.client.post(reverse(
'api-v2:attachment-tracking-list'),
{'accessed_from_page': 'CR', 'app': 'frontend', 'attachment_id': 123}
)
expect(response.status_code).to.eq(status.HTTP_200_OK)
expect(AttachmentTracking.objects.count()).to.eq(1)
attachment_tracking = AttachmentTracking.objects.first()
expect(attachment_tracking.created_at).to.eq(datetime(2018, 4, 4, 12, 0, 1, tzinfo=pytz.utc))
expect(attachment_tracking.attachment_file).to.eq(attachment_file)
expect(attachment_tracking.accessed_from_page).to.eq('CR')
expect(attachment_tracking.app).to.eq('frontend')
def test_upload_pdf(self, aws_mock):
AttachmentFileFactory(
external_id='00000105',
source_type='PORTAL_COPA',
url=
'http://www.chicagocopa.org/wp-content/uploads/2016/05/CHI-R-00000105.pdf'
)
AttachmentFileFactory(
external_id='123',
source_type='DOCUMENTCLOUD',
url='https://www.documentcloud.org/documents/2-CRID-123-CR.html')
AttachmentFileFactory(
external_id='456',
source_type='PORTAL_COPA_DOCUMENTCLOUD',
url='https://www.documentcloud.org/documents/2-CRID-456-CR.html')
AttachmentFileFactory(
external_id='789',
source_type='SUMMARY_REPORTS_COPA_DOCUMENTCLOUD',
url='https://www.documentcloud.org/documents/3-CRID-789-CR.html')
call_command('upload_pdf')
expect(aws_mock.lambda_client.invoke_async.call_count).to.eq(3)
expect(aws_mock.lambda_client.invoke_async).to.be.any_call(
FunctionName='uploadPdfTest',
InvokeArgs=json.dumps({
'url':
'https://www.documentcloud.org/documents/2-CRID-123-CR.html',
'bucket': 'officer_content_bucket',
'key': 'pdf/123'
}))
expect(aws_mock.lambda_client.invoke_async).to.be.any_call(
FunctionName='uploadPdfTest',
InvokeArgs=json.dumps({
'url':
'https://www.documentcloud.org/documents/2-CRID-456-CR.html',
'bucket': 'officer_content_bucket',
'key': 'pdf/456'
}))
expect(aws_mock.lambda_client.invoke_async).to.be.any_call(
FunctionName='uploadPdfTest',
InvokeArgs=json.dumps({
'url':
'https://www.documentcloud.org/documents/3-CRID-789-CR.html',
'bucket': 'officer_content_bucket',
'key': 'pdf/789'
}))
def test_upload_to_s3(self, mock_aws):
attachment = AttachmentFileFactory(
external_id='123',
url=
'http://www.chicagocopa.org/wp-content/uploads/2016/05/CHI-R-00000105.pdf'
)
attachment.upload_to_s3()
expect(mock_aws.lambda_client.invoke_async).to.be.any_call(
FunctionName='uploadPdfTest',
InvokeArgs=json.dumps({
'url':
'http://www.chicagocopa.org/wp-content/uploads/2016/05/CHI-R-00000105.pdf',
'bucket': 'officer-content-test',
'key': 'pdf/123'
}))
def test_allegation_attachments(self):
allegation = AllegationFactory(crid='123')
attachment_1 = AttachmentFileFactory(allegation=allegation,
source_type='DOCUMENTCLOUD')
attachment_2 = AttachmentFileFactory(
allegation=allegation, source_type='PORTAL_COPA_DOCUMENTCLOUD')
AttachmentFileFactory(allegation=allegation,
source_type='PORTAL_COPA_DOCUMENTCLOUD',
show=False)
AttachmentFileFactory(allegation=allegation, source_type='COPA')
allegation_456 = AllegationFactory(crid='456')
AttachmentFileFactory(allegation=allegation_456,
source_type='DOCUMENTCLOUD')
AttachmentFileFactory(allegation=allegation_456,
source_type='PORTAL_COPA_DOCUMENTCLOUD')
AttachmentFileFactory(allegation=allegation_456,
source_type='PORTAL_COPA_DOCUMENTCLOUD',
show=False)
officer = OfficerFactory(id=1)
OfficerAllegationFactory(officer=officer, allegation=allegation)
expect({
attachment.id
for attachment in officer.allegation_attachments
}).to.eq({attachment_1.id, attachment_2.id})
def test_paginate_es_query(self):
allegation = AllegationFactory(crid=123456)
attachment_1 = AttachmentFileFactory(id=1,
allegation=allegation,
title='Document Title 1',
text_content='Text Content 1')
attachment_2 = AttachmentFileFactory(id=2,
allegation=allegation,
title='Document Title 2',
text_content='Text Content 2')
AttachmentFileFactory(id=3,
allegation=allegation,
title='Document Title 3',
text_content='Text Content 3')
queryset = AttachmentFile.objects.all()
class MockObject(object):
pass
returned_value_1 = MockObject()
returned_value_2 = MockObject()
setattr(returned_value_1, 'id', 1)
setattr(returned_value_2, 'id', 2)
request = Mock()
request.query_params = {'limit': 20, 'offset': 30}
search_result = Mock()
search_result.hits = Mock()
search_result.hits.total = 50
search_result.__iter__ = Mock(
return_value=iter([returned_value_1, returned_value_2]))
query = Mock()
query.__getitem__ = Mock(return_value=query)
query.execute.return_value = search_result
pagination = ESQuerysetPagination()
paginated_query = pagination.paginate_es_query(query, request,
queryset)
expect(list(paginated_query)).to.eq([attachment_1, attachment_2])
expect(pagination.count).to.eq(50)
expect(pagination.limit).to.eq(20)
expect(pagination.offset).to.eq(30)
expect(pagination.request).to.eq(request)
def test_search_with_apostrophe(self):
allegation_category = AllegationCategoryFactory(category='Use of Force')
allegation_1 = AllegationFactory(
crid='C12345',
incident_date=datetime(2007, 1, 1, tzinfo=pytz.utc),
most_common_category=allegation_category,
)
allegation_2 = AllegationFactory(
crid='C12346',
incident_date=datetime(2007, 1, 1, tzinfo=pytz.utc),
most_common_category=allegation_category,
)
allegation_3 = AllegationFactory(
crid='C12347',
incident_date=datetime(2007, 1, 1, tzinfo=pytz.utc),
most_common_category=allegation_category,
)
allegation_4 = AllegationFactory(
crid='C12348',
incident_date=datetime(2007, 1, 1, tzinfo=pytz.utc),
most_common_category=allegation_category,
)
AttachmentFileFactory(text_content='Name: OBrien', allegation=allegation_1)
AttachmentFileFactory(text_content='Name: O\'Brien', allegation=allegation_2)
AttachmentFileFactory(text_content='Name: O Brien', allegation=allegation_3)
AttachmentFileFactory(text_content='Name: Jim', allegation=allegation_4)
self.rebuild_index()
self.refresh_index()
response = self.client.get(reverse('api:suggestion-list'), {
'term': 'O\'Brien',
})
sorted_cr_results = list(sorted(response.data['CR'], key=lambda cr: cr['crid']))
print(sorted_cr_results)
expect(len(sorted_cr_results)).to.eq(3)
expect(sorted_cr_results[0]['crid']).to.eq('C12345')
expect(sorted_cr_results[0]['highlight']['text_content']).to.eq(['Name: O<em>Brien</em>'])
expect(sorted_cr_results[1]['crid']).to.eq('C12346')
expect(sorted_cr_results[1]['highlight']['text_content']).to.eq(['Name: O\'<em>Brien</em>'])
expect(sorted_cr_results[2]['crid']).to.eq('C12347')
expect(sorted_cr_results[2]['highlight']['text_content']).to.eq(['Name: O <em>Brien</em>'])
def test_lastmod(self):
attachment = AttachmentFileFactory(id=123)
with freeze_time(datetime(2018, 4, 4, 12, 0, 1, tzinfo=pytz.utc)):
attachment.title = 'title'
attachment.save()
attachment.refresh_from_db()
expect(AttachmentSitemap().lastmod(attachment)).to.eq(
datetime(2018, 4, 4, 12, 0, 1, tzinfo=pytz.utc))
def test_sitemap_attachment_view(self):
AttachmentFileFactory(id=1, file_type='Video', show=True)
AttachmentFileFactory(id=9, file_type='Audio', show=True)
AttachmentFileFactory(id=2, file_type='document', show=False)
AttachmentFileFactory(id=3, file_type='document', show=True)
AttachmentFileFactory(id=5, file_type='document', show=True)
AttachmentFileFactory(id=4, file_type='document', show=True)
response = self.client.get('/sitemap-attachment.xml')
expect(response.status_code).to.eq(status.HTTP_200_OK)
expect(response.content.decode('utf8')).to.eq(
'<?xml version="1.0" encoding="UTF-8"?>\n'
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
'<url>'
'<loc>https://example.com/document/3/</loc>'
'<lastmod>2019-03-18</lastmod>'
'<changefreq>daily</changefreq>'
'<priority>0.5</priority>'
'</url>'
'<url>'
'<loc>https://example.com/document/4/</loc>'
'<lastmod>2019-03-18</lastmod>'
'<changefreq>daily</changefreq>'
'<priority>0.5</priority>'
'</url>'
'<url>'
'<loc>https://example.com/document/5/</loc>'
'<lastmod>2019-03-18</lastmod>'
'<changefreq>daily</changefreq>'
'<priority>0.5</priority>'
'</url>\n'
'</urlset>\n'
)
