def test_agency_merge(self): """Test agency merging""" good_agency = AgencyFactory(status="approved", email=None, fax=None) bad_agency = AgencyFactory(status="approved", email=None, fax=None) appeal_agency = AgencyFactory(appeal_agency=bad_agency) foia = FOIARequestFactory(agency=bad_agency) composer = FOIAComposerFactory(agencies=[bad_agency]) user = UserFactory() email = EmailAddressFactory() AgencyEmailFactory(agency=good_agency, email=email, email_type="to") AgencyEmailFactory(agency=bad_agency, email=email, email_type="cc") fax1 = PhoneNumberFactory() fax2 = PhoneNumberFactory() AgencyPhoneFactory(agency=good_agency, phone=fax1, request_type="primary") AgencyPhoneFactory(agency=bad_agency, phone=fax2, request_type="primary") good_agency.merge(bad_agency, user) bad_agency.refresh_from_db() appeal_agency.refresh_from_db() foia.refresh_from_db() composer.refresh_from_db() eq_(bad_agency.status, "rejected") eq_(foia.agency, good_agency) eq_(composer.agencies.first(), good_agency) eq_(appeal_agency.appeal_agency, good_agency) # email that already exists is not copied over eq_(good_agency.emails.count(), 1) eq_(good_agency.agencyemail_set.first().email_type, "to") # phone number that doesnt exist is copied over eq_(good_agency.phones.count(), 2) # existing phone number is unaffected ok_( good_agency.agencyphone_set.filter( phone=fax1, request_type="primary" ).exists() ) # its type is set to none when copied over ok_( good_agency.agencyphone_set.filter(phone=fax2, request_type="none").exists() ) assert_in(good_agency.name, bad_agency.notes) assert_in(str(good_agency.pk), bad_agency.notes) assert_in(user.username, bad_agency.notes)
class TestAgencyImporter(TestCase): """Tests for the mass agency importer""" def setUp(self): """Create some existing jurisdictions and agencies for the tests""" local = LocalJurisdictionFactory() state = local.parent federal = state.parent self.cia = AgencyFactory( name="Central Intelligence Agency", jurisdiction=federal, email=None, fax=None, ) self.governor = AgencyFactory( name="Governor's Office", jurisdiction=state, ) self.police = AgencyFactory( name="Boston Police Department", jurisdiction=local, ) def test_match(self): """Test different instances of matching existing agencies""" reader = PyReader([ # case insensitive match { "agency": "central intelligence agency", "jurisdiction": "united states of america", }, # matches abbrev, fuzzy name match { "agency": "Center Intelligence Agency", "jurisdiction": "USA" }, # matches abbrev { "agency": "Governor's Office", "jurisdiction": "MA" }, # matches state name, fuzzy { "agency": "Governors Office", "jurisdiction": "Massachusetts" }, # local jurisdiction matches { "agency": "Boston Police Department", "jurisdiction": "Boston, MA" }, # fuzzy match, full state name { "agency": "The Police Department", "jurisdiction": "Boston, Massachusetts", }, # bad jurisdiction { "agency": "The Police Department", "jurisdiction": "Springfield, ZZ" }, # bad agency { "agency": "Sheriff's Secret Police", "jurisdiction": "Boston, MA" }, # blank agency { "agency": "", "jurisdiction": "Boston, MA" }, # missing agency { "jurisdiction": "Boston, MA" }, # missing agency, blank jurisdiction { "jurisdiction": "" }, ]) importer = Importer(reader) data = list(importer.match()) eq_(data[0]["match_agency"], self.cia) eq_(data[0]["agency_status"], "exact match") eq_(data[1]["match_agency"], self.cia) assert_greater_equal(data[1]["match_agency_score"], 83) eq_(data[1]["agency_status"], "fuzzy match") eq_(data[2]["match_agency"], self.governor) eq_(data[2]["agency_status"], "exact match") eq_(data[3]["match_agency"], self.governor) assert_greater_equal(data[3]["match_agency_score"], 83) eq_(data[3]["agency_status"], "fuzzy match") eq_(data[4]["match_agency"], self.police) eq_(data[4]["agency_status"], "exact match") eq_(data[5]["match_agency"], self.police) assert_greater_equal(data[5]["match_agency_score"], 83) eq_(data[5]["agency_status"], "fuzzy match") assert_not_in("match_agency", data[6]) eq_(data[6]["jurisdiction_status"], "no jurisdiction") assert_not_in("match_agency", data[7]) eq_(data[7]["agency_status"], "no agency") eq_("missing agency", data[8]["agency_status"]) eq_("missing agency", data[9]["agency_status"]) eq_("missing agency", data[10]["agency_status"]) eq_("missing jurisdiction", data[10]["jurisdiction_status"]) def test_import_update(self): """An import test where we are updating the contact information for an existing agency """ reader = PyReader([ { "agency": "central intelligence agency", "jurisdiction": "united states of america", "email": "*****@*****.**", "fax": "617-555-0001", "phone": "617-555-0000", "address_city": "Washington", "address_state": "DC", "address_zip": "20505", "portal_url": "https://www.cia.gov/portal/", "portal_type": "foiaonline", "aliases": "CIA", "foia_website": "https://www.cia.gov/foia/", "website": "https://www.cia.gov/", }, ]) importer = Importer(reader) data = list(importer.import_()) self.cia.refresh_from_db() eq_(data[0]["agency_status"], "exact match") eq_(self.cia.email.email, "*****@*****.**") eq_(data[0]["email_status"], "set primary") eq_(self.cia.fax.number, "+1 617-555-0001") eq_(data[0]["fax_status"], "set primary") ok_(self.cia.get_phones().filter(number="+1 617-555-0000").exists()) eq_(data[0]["phone_status"], "set") eq_(self.cia.address.zip_code, "20505") eq_(self.cia.address.city, "Washington") eq_(self.cia.address.state, "DC") eq_(data[0]["address_status"], "set primary") eq_(self.cia.portal.url, "https://www.cia.gov/portal/") eq_(self.cia.portal.type, "foiaonline") eq_(data[0]["portal_status"], "set") eq_(self.cia.aliases, "CIA") eq_(data[0]["aliases_status"], "set") eq_(self.cia.url, "https://www.cia.gov/foia/") eq_(data[0]["foia_website_status"], "set") eq_(self.cia.website, "https://www.cia.gov/") eq_(data[0]["website_status"], "set") def test_import_update_invalid(self): """Test import with some invalid data""" reader = PyReader([ { "agency": "central intelligence agency", "jurisdiction": "united states of america", "email": "foia@cia", "fax": "617-555-001", "phone": "foobar", "address_city": "Washington", "address_state": "foobar", "address_zip": "0123", "portal_url": "not a url", "portal_type": "not a portal", "foia_website": "www.cia.gov/foia/", "website": "foo.bar", }, ]) importer = Importer(reader) data = list(importer.import_()) eq_(data[0]["agency_status"], "exact match") eq_(data[0]["email_status"], "error") eq_(data[0]["fax_status"], "error") eq_(data[0]["phone_status"], "error") eq_(data[0]["address_status"], "error") eq_(data[0]["portal_status"], "error") eq_(data[0]["foia_website_status"], "error") eq_(data[0]["website_status"], "error") def test_import_update_duplicate(self): """Test an import with data already on the agency""" agency_phone = AgencyPhoneFactory(agency=self.police) reader = PyReader([ { "agency": "Boston Police Department", "jurisdiction": "Boston, MA", "email": self.police.email.email, "fax": self.police.fax.number.as_national, "phone": agency_phone.phone.number.as_national, }, ]) importer = Importer(reader) data = list(importer.import_()) eq_(data[0]["email_status"], "already set") eq_(data[0]["fax_status"], "already set") eq_(data[0]["phone_status"], "already set") def test_import_update_redundant(self): """Test an update with data different from the data already on the agency""" AgencyAddress.objects.create( agency=self.police, address=AddressFactory(), request_type="primary", ) self.police.portal = Portal.objects.create( url="https://www.example.com", name="Test Portal", type="other", ) self.police.save() reader = PyReader([ { "agency": "Boston Police Department", "jurisdiction": "Boston, MA", "email": "*****@*****.**", "cc_emails": "[email protected], [email protected]", "fax": "617-555-0001", "address_city": "Washington", "address_state": "DC", "address_zip": "01233", "portal_url": "https://www.cia.gov/portal/", "portal_type": "foiaonline", }, ]) importer = Importer(reader) data = list(importer.import_()) eq_(data[0]["email_status"], "set other") ok_(self.police.emails.filter(email="*****@*****.**").exists()) ok_(self.police.emails.filter(email="*****@*****.**").exists()) ok_(self.police.emails.filter(email="*****@*****.**").exists()) eq_(data[0]["fax_status"], "set other") ok_(self.police.phones.filter(number="617-555-0001").exists()) eq_(data[0]["address_status"], "set other") ok_(self.police.addresses.filter(city="Washington").exists()) eq_(data[0]["portal_status"], "not set, existing") eq_(self.police.portal.name, "Test Portal") def test_create(self): """Test creating a new agency""" reader = PyReader([ { "agency": "Foobar", "jurisdiction": "united states of america", "email": "*****@*****.**", "cc_emails": "[email protected], [email protected]", "fax": "617-555-0001", "phone": "617-555-0000", "address_street": "123 Main St", "address_city": "Washington", "address_state": "DC", "address_zip": "20505", "portal_url": "https://www.new-agency.gov/portal/", "portal_type": "nextrequest", "foia_website": "https://www.new-agency.gov/foia/", "website": "https://www.new-agency.gov/", "requires_proxy": "true", }, ]) importer = Importer(reader) data = list(importer.import_()) eq_(data[0]["agency_status"], "created") agency = data[0]["match_agency"] eq_(agency.name, "Foobar") eq_(agency.email.email, "*****@*****.**") eq_( sorted(e.email for e in agency.get_emails(request_type="primary", email_type="cc")), ["*****@*****.**", "*****@*****.**"], ) eq_(data[0]["email_status"], "set primary") eq_(agency.fax.number, "+1 617-555-0001") eq_(data[0]["fax_status"], "set primary") ok_(agency.get_phones().filter(number="+1 617-555-0000").exists()) eq_(data[0]["phone_status"], "set") eq_(agency.address.street, "123 Main St") eq_(agency.address.zip_code, "20505") eq_(agency.address.city, "Washington") eq_(agency.address.state, "DC") eq_(data[0]["address_status"], "set primary") eq_(agency.portal.url, "https://www.new-agency.gov/portal/") eq_(agency.portal.type, "nextrequest") eq_(data[0]["portal_status"], "set") assert_not_in("aliases_status", data[0]) eq_(agency.url, "https://www.new-agency.gov/foia/") eq_(data[0]["foia_website_status"], "set") eq_(agency.website, "https://www.new-agency.gov/") eq_(data[0]["website_status"], "set") ok_(agency.requires_proxy) eq_(data[0]["requires_proxy_status"], "set true") def test_create_minimal(self): """Test a creation with minimal contact information supplied""" reader = PyReader([ { "agency": "Foobar", "jurisdiction": "united states of america", }, ]) importer = Importer(reader) data = list(importer.import_()) eq_(data[0]["agency_status"], "created") agency = data[0]["match_agency"] eq_(agency.name, "Foobar") def test_create_bad_jurisdiction(self): """Test creating an agency in a bad jurisdiction""" reader = PyReader([ { "agency": "Foobar", "jurisdiction": "Foobar", }, ]) importer = Importer(reader) data = list(importer.import_()) eq_(data[0]["jurisdiction_status"], "no jurisdiction") assert_not_in("agency_status", data[0])