def test_exception_unknown_collection_on_create_identifier(self): commons = models.Organization.objects.create(name="The Commons") importer = PopItImporter() with self.assertRaisesRegexp( Exception, r"Unknown collection 'not-a-collection'"): importer.create_identifier('not-a-collection', 'commons', commons)
def test_inline_area_parent_relationship(self): input_json = ''' { "organizations": [ { "id": "sp", "name": "Scottish Parliament", "area": { "id": "subarea-1", "name": "Scotland", "identifier": "sco", "classification": "country", "parent_id": "area-1" } } ], "areas": [ { "id": "area-1", "name": "United Kingdom", "identifier": "uk", "classification": "country" } ] } ''' data = json.loads(input_json) importer = PopItImporter() importer.import_from_export_json_data(data) self.assertEqual(models.Area.objects.count(), 2) child = models.Area.objects.get(name='Scotland') parent = models.Area.objects.get(name='United Kingdom') self.assertEqual(child.parent, parent)
def test_link_and_source_without_a_note(self): input_json = ''' { "persons": [ { "id": "a1b2", "name": "Alice", "sources": [ { "url": "http://example.org/source-of-alice-data" } ], "links": [ { "url": "http://example.org/a-link-without-a-note" } ] } ] } ''' data = json.loads(input_json) importer = PopItImporter() importer.import_from_export_json_data(data) self.assertEqual(models.Person.objects.count(), 1) person = models.Person.objects.get() self.assertEqual(person.name, "Alice") self.assertEqual( person.sources.get().url, "http://example.org/source-of-alice-data") self.assertEqual( person.links.get().url, "http://example.org/a-link-without-a-note")
def test_all_top_level_optional(self): # This just check that you don't get an exception when # importing empty Popolo JSON. input_json = '{}' data = json.loads(input_json) importer = PopItImporter() importer.import_from_export_json_data(data)
def test_update_existing_organization(self): existing = models.Organization.objects.create(name="The Commons") existing.identifiers.create( scheme="popit-organization", identifier="commons") input_json = ''' { "organizations": [ { "id": "commons", "name": "House of Commons" } ] } ''' data = json.loads(input_json) importer = PopItImporter() importer.import_from_export_json_data(data) self.assertEqual(models.Organization.objects.count(), 1) organization = models.Organization.objects.get() self.assertEqual(existing, organization) self.assertEqual(organization.name, "House of Commons") self.assertEqual( organization.identifiers.get(scheme='popit-organization').identifier, "commons" )
def test_organization_with_identifiers(self): input_json = ''' { "organizations": [ { "id": "commons", "name": "House of Commons", "identifiers": [ { "scheme": "big-db-of-parliaments", "identifier": "uk-commons" } ] } ] } ''' data = json.loads(input_json) importer = PopItImporter() importer.import_from_export_json_data(data) self.assertEqual(models.Organization.objects.count(), 1) organization = models.Organization.objects.get() self.assertEqual(organization.name, "House of Commons") self.assertEqual( organization.identifiers.get(scheme='popit-organization').identifier, "commons" ) self.assertEqual( organization.identifiers.get(scheme='big-db-of-parliaments').identifier, "uk-commons" )
def test_truncation_warn(self): long_name = ('Albert ' * 100).strip() input_json = ''' {{ "persons": [ {{ "id": "a1b2", "name": "{0}" }} ] }} '''.format(long_name) data = json.loads(input_json) # With truncate='warn' the field should be truncated, but # print a warning to stderr: importer = PopItImporter(truncate='warn') with capture_output() as (out, err): importer.import_from_export_json_data(data) output = err.getvalue().strip() self.assertIn('Warning: truncating Albert', output) self.assertIn('Albert to a length of 512', output) person = models.Person.objects.get() max_length = person._meta.get_field('name').max_length truncated_name = long_name[:max_length] self.assertEqual(person.name, truncated_name)
def test_post_creation(self): input_json = ''' { "posts": [ { "id": "65808", "label": "Member of Parliament for Dulwich and West Norwood", "role": "Member of Parliament", "organization_id": "commons" } ], "organizations": [ { "id": "commons", "name": "House of Commons" } ] } ''' data = json.loads(input_json) importer = PopItImporter() importer.import_from_export_json_data(data) self.assertEqual(models.Post.objects.count(), 1) post = models.Post.objects.get() self.assertEqual(post.label, "Member of Parliament for Dulwich and West Norwood") self.assertEqual(post.role, "Member of Parliament") self.assertEqual(models.Organization.objects.count(), 1)
def test_remove_identifiers_when_empty_array_specified(self): existing_person = models.Person.objects.create(name='Algernon') existing_person.identifiers.create( scheme='popolo:person', identifier="a1b2" ) existing_person.identifiers.create( scheme='ignorable', identifier="some-data-we-care-nothing-for" ) input_json = ''' { "persons": [ { "id": "a1b2", "name": "Alice", "identifiers": [] } ] } ''' data = json.loads(input_json) importer = PopItImporter(id_prefix='popolo:') importer.import_from_export_json_data(data) self.assertEqual(models.Person.objects.count(), 1) # Reget the person from the database: person = models.Person.objects.get(pk=existing_person.id) self.assertEqual(person.name, 'Alice') self.assertSequenceEqual( person.identifiers.order_by('scheme').values_list('scheme', 'identifier'), [(u'popolo:person', u'a1b2')] )
def test_organisation_with_external_area(self): input_json = ''' { "organizations": [ { "id": "commons", "name": "House of Commons", "area_id": "area-1" } ], "areas": [ { "id": "area-1", "name": "The United Kingdom of Great Britain and Northern Ireland", "identifier": "uk", "classification": "country" } ] } ''' data = json.loads(input_json) importer = PopItImporter() importer.import_from_export_json_data(data) self.assertEqual(models.Organization.objects.count(), 1) self.assertEqual(models.Area.objects.count(), 1) area = models.Area.objects.get() self.assertEqual( area.name, 'The United Kingdom of Great Britain and Northern Ireland') self.assertEqual(area.identifier, 'uk') self.assertEqual(area.classification, 'country')
def test_area_update(self): existing = models.Area.objects.create( name="Alba", identifier="blah-sco", classification="country", ) existing.other_identifiers.create( scheme='popit-area', identifier='subarea-1', ) input_json = ''' { "areas": [ { "id": "subarea-1", "name": "Scotland", "identifier": "sco", "classification": "country" } ] } ''' data = json.loads(input_json) importer = PopItImporter() importer.import_from_export_json_data(data) self.assertEqual(models.Area.objects.count(), 1) area = models.Area.objects.get() self.assertEqual(area.name, 'Scotland') self.assertEqual(area.identifier, 'sco') self.assertEqual(area.classification, 'country')
def test_custom_identifier_prefix(self): input_json = ''' { "persons": [ { "id": "a1b2", "name": "Alice" } ], "organizations": [ { "id": "commons", "name": "House of Commons" } ] } ''' data = json.loads(input_json) importer = PopItImporter(id_prefix='popolo:') importer.import_from_export_json_data(data) self.assertEqual(models.Person.objects.count(), 1) self.assertEqual(models.Organization.objects.count(), 1) person = models.Person.objects.get() organization = models.Organization.objects.get() person_identifier = person.identifiers.get() organization_identifier = organization.identifiers.get() self.assertEqual(person_identifier.scheme, 'popolo:person') self.assertEqual(person_identifier.identifier, 'a1b2') self.assertEqual(organization_identifier.scheme, 'popolo:organization') self.assertEqual(organization_identifier.identifier, 'commons')
def test_membership_no_dates_but_some_in_legislative_period(self): data = json.loads(self.input_json) importer = PopItImporter() importer.import_from_export_json_data(data) self.assertEqual(models.Membership.objects.count(), 2) earlier, later = models.Membership.objects.order_by('start_date') self.assertEqual(earlier.start_date, '2010-05-06') self.assertEqual(earlier.end_date, '2011-02-08') self.assertEqual(later.start_date, '2015-05-08') self.assertEqual(later.end_date, '')
def test_related_objects_for_person(self): input_json = ''' { "persons": [ { "id": "a1b2", "name": "Alice", "identifiers": [ { "identifier": "123456789", "scheme": "yournextmp-candidate" } ], "contact_details": [ { "contact_type": "twitter", "label": "", "note": "", "value": "sometwitterusernameorother" } ], "links": [ { "note": "homepage", "url": "http://example.com/alice" } ] } ] } ''' data = json.loads(input_json) importer = PopItImporter() importer.import_from_export_json_data(data) self.assertEqual(models.Person.objects.count(), 1) person = models.Person.objects.get() self.assertEqual(person.name, "Alice") self.assertEqual(person.identifiers.count(), 2) self.assertEqual( person.identifiers.get(scheme='popit-person').identifier, "a1b2") self.assertEqual( person.identifiers.get(scheme='yournextmp-candidate').identifier, "123456789") self.assertEqual(person.contact_details.count(), 1) contact_detail = person.contact_details.get() self.assertEqual(contact_detail.contact_type, 'twitter') self.assertEqual(contact_detail.value, 'sometwitterusernameorother') self.assertEqual(person.links.count(), 1) link = person.links.get() self.assertEqual(link.note, 'homepage') self.assertEqual(link.url, 'http://example.com/alice')
def handle(self, **options): self.importer = PopItImporter() self.generate_party_sets() with open(options['CSV-FILENAME']) as f: csv_reader = csv.reader(f) for party_data in csv_reader: self.update_party(party_data)
def test_truncation_none(self): long_name = ('Albert ' * 100).strip() input_json = ''' {{ "persons": [ {{ "id": "a1b2", "name": "{0}" }} ] }} '''.format(long_name) data = json.loads(input_json) # With truncate='yes', there should be no exception: importer = PopItImporter(truncate='yes') importer.import_from_export_json_data(data) person = models.Person.objects.get() max_length = person._meta.get_field('name').max_length truncated_name = long_name[:max_length] self.assertEqual(person.name, truncated_name)
def test_creates_new_person_if_not_found(self): existing_person = models.Person.objects.create(name='Algernon') input_json = ''' { "persons": [ { "id": "a1b2", "name": "Alice" } ] } ''' data = json.loads(input_json) importer = PopItImporter() importer.import_from_export_json_data(data) self.assertEqual(models.Person.objects.count(), 2) new_person = models.Person.objects.exclude(pk=existing_person.id).get() new_person_identifier = new_person.identifiers.get() self.assertEqual(new_person_identifier.scheme, 'popit-person') self.assertEqual(new_person_identifier.identifier, 'a1b2')
def test_person_with_membership(self): input_json = ''' { "persons": [ { "id": "a1b2", "name": "Alice" } ], "organizations": [ { "id": "commons", "name": "House of Commons" } ], "memberships": [ { "person_id": "a1b2", "organization_id": "commons" } ] } ''' data = json.loads(input_json) importer = PopItImporter() importer.import_from_export_json_data(data) self.assertEqual(models.Membership.objects.count(), 1) self.assertEqual(models.Person.objects.count(), 1) self.assertEqual(models.Organization.objects.count(), 1) person = models.Person.objects.get() self.assertEqual(person.name, "Alice") organization = models.Organization.objects.get() self.assertEqual(organization.name, "House of Commons") self.assertEqual( organization.identifiers.get(scheme='popit-organization').identifier, "commons" ) membership = models.Membership.objects.get() self.assertEqual(membership.person, person) self.assertEqual(membership.organization, organization)
def test_post_update(self): existing_org = models.Organization.objects.create(name='House of Commons') existing_org.identifiers.create( scheme='popit-organization', identifier='commons') existing_post = models.Post.objects.create( label="MP for Dulwich and West Norwood", role="MP", organization=existing_org ) models.Identifier.objects.create( scheme='popit-post', identifier='65808', content_object=existing_post, ) # Now import JSON that updates that post: input_json = ''' { "posts": [ { "id": "65808", "label": "Member of Parliament for Dulwich and West Norwood", "role": "Member of Parliament", "organization_id": "commons" } ], "organizations": [ { "id": "commons", "name": "House of Commons" } ] } ''' data = json.loads(input_json) importer = PopItImporter() importer.import_from_export_json_data(data) self.assertEqual(models.Post.objects.count(), 1) post = models.Post.objects.get() self.assertEqual(post.label, "Member of Parliament for Dulwich and West Norwood") self.assertEqual(post.role, "Member of Parliament") self.assertEqual(models.Organization.objects.count(), 1)
def test_values_in_related_objects_are_truncated(self): input_json = ''' { "persons": [ { "id": "a1b2", "name": "Alice", "contact_details": [ { "type": "official-work-email", "value": "*****@*****.**" } ] } ] } ''' data = json.loads(input_json) importer = PopItImporter(truncate='yes') importer.import_from_export_json_data(data) cd = models.ContactDetail.objects.get() self.assertEqual(cd.contact_type, 'official-wor') # Also test that they can be updated without creating a # duplicate. (This could happen if the implementation uses the # overlong version for checking existence, but the truncated # version for setting.) importer.import_from_export_json_data(data) self.assertEqual(models.ContactDetail.objects.count(), 1) cd = models.ContactDetail.objects.get() self.assertEqual(cd.contact_type, 'official-wor')
def test_simplest_person(self): input_json = ''' { "persons": [ { "id": "a1b2", "name": "Alice" } ], "organizations": [], "memberships": [] } ''' data = json.loads(input_json) importer = PopItImporter() importer.import_from_export_json_data(data) self.assertEqual(models.Person.objects.count(), 1) person = models.Person.objects.get() self.assertEqual(person.name, "Alice") self.assertEqual( person.identifiers.get(scheme='popit-person').identifier, "a1b2")
def test_organization_parent_relationship(self): input_json = ''' { "organizations": [ { "id": "co", "name": "Cabinet Office", "parent_id": "cs" }, { "id": "cs", "name": "Civil Service" } ] } ''' data = json.loads(input_json) importer = PopItImporter() importer.import_from_export_json_data(data) self.assertEqual(models.Organization.objects.count(), 2) child = models.Organization.objects.get(name='Cabinet Office') parent = models.Organization.objects.get(name='Civil Service') self.assertEqual(child.parent, parent)
def test_exception_from_inline_area_missing_id(self): input_json = ''' { "organizations": [ { "id": "commons", "name": "House of Commons", "area": { "name": "The United Kingdom of Great Britain and Northern Ireland", "identifier": "uk", "classification": "country" } } ] } ''' data = json.loads(input_json) importer = PopItImporter() with capture_output() as (out, err): with self.assertRaisesRegexp( ValueError, 'Found inline area data, but with no "id" attribute'): importer.import_from_export_json_data(data)
def test_extra_preserved_identifiers(self): existing_person = models.Person.objects.create(name='Algernon') existing_person.identifiers.create( scheme='popolo:person', identifier="a1b2" ) existing_person.identifiers.create( scheme='ignorable', identifier="some-data-we-care-nothing-for" ) existing_person.identifiers.create( scheme='preserve-me', identifier="data-that-should-be-kept" ) input_json = ''' { "persons": [ { "id": "a1b2", "name": "Alice", "identifiers": [] } ] } ''' data = json.loads(input_json) importer = PopItImporter( id_prefix='popolo:', id_schemes_to_preserve={'person': {'preserve-me'}}) importer.import_from_export_json_data(data) self.assertEqual(models.Person.objects.count(), 1) # Reget the person from the database: person = models.Person.objects.get(pk=existing_person.id) self.assertEqual(person.name, 'Alice') self.assertSequenceEqual( person.identifiers.order_by('scheme').values_list('scheme', 'identifier'), [(u'popolo:person', u'a1b2'), (u'preserve-me', 'data-that-should-be-kept')] )
def test_import_from_a_file(self): input_json = ''' { "persons": [ { "id": "a1b2", "name": "Alice" } ], "organizations": [ { "id": "commons", "name": "House of Commons" } ], "memberships": [ { "person_id": "a1b2", "organization_id": "commons" } ] } ''' try: with NamedTemporaryFile('w', delete=False, suffix='.json') as ntf: ntf.write(input_json) importer = PopItImporter() importer.import_from_export_json(ntf.name) finally: os.remove(ntf.name) # Now check everything was created: self.assertEqual(models.Membership.objects.count(), 1) self.assertEqual(models.Person.objects.count(), 1) self.assertEqual(models.Organization.objects.count(), 1) self.assertEqual(models.Identifier.objects.count(), 3)
def test_truncation_exception(self): long_name = ('Albert ' * 100).strip() input_json = ''' {{ "persons": [ {{ "id": "a1b2", "name": "{0}" }} ] }} '''.format(long_name) data = json.loads(input_json) # With the default, truncate='exception', there should be an # exception raised: importer = PopItImporter() with self.assertRaisesRegexp( ValidationError, 'Ensure this value has at most 512 characters'): # Capture the output just to reduce noise in the test # output - this would include output from # show_data_on_error otherwise. with capture_output() as (out, err): importer.import_from_export_json_data(data)
def test_updates_person_if_found(self): existing_person = models.Person.objects.create(name='Algernon') existing_person.identifiers.create( scheme='popolo:person', identifier="a1b2" ) input_json = ''' { "persons": [ { "id": "a1b2", "name": "Alice" } ] } ''' data = json.loads(input_json) importer = PopItImporter(id_prefix='popolo:') mock_observer = Mock() importer.add_observer(mock_observer) importer.import_from_export_json_data(data) self.assertEqual(models.Person.objects.count(), 1) # Reget the person from the database: person = models.Person.objects.get(pk=existing_person.id) self.assertEqual(person.name, 'Alice') # Check that the observer was called with created=False: self.assertEqual(mock_observer.notify.call_count, 1) self.assertEqual( mock_observer.notify.call_args, call( 'person', existing_person, False, { "id": "a1b2", "name": "Alice" } ) )
def handle(self, **options): self.importer = PopItImporter() party_set = self.get_party_set(options['party_set']) with open(options['JSON-FILENAME']) as f: for party_data in json.load(f): self.update_party(party_data, party_set)
def test_truncation_unknown_option(self): with self.assertRaises(ValueError): PopItImporter(truncate='dunno')
def test_dont_recreate_related_objects(self): input_json = ''' { "persons": [ { "id": "a1b2", "name": "Alice", "identifiers": [ { "identifier": "123456789", "scheme": "yournextmp-candidate" } ], "contact_details": [ { "contact_type": "twitter", "label": "", "note": "", "value": "sometwitterusernameorother" } ], "links": [ { "note": "homepage", "url": "http://example.com/alice" } ] } ] } ''' data = json.loads(input_json) importer = PopItImporter() importer.import_from_export_json_data(data) self.assertEqual(models.Person.objects.count(), 1) original_person = models.Person.objects.get() self.assertEqual(original_person.name, "Alice") self.assertEqual(original_person.identifiers.count(), 2) original_identifier_a = original_person.identifiers.get(scheme='popit-person') original_identifier_b = original_person.identifiers.get(scheme='yournextmp-candidate') self.assertEqual(original_identifier_a.identifier, "a1b2") self.assertEqual(original_identifier_b.identifier, "123456789") self.assertEqual(original_person.contact_details.count(), 1) original_contact_detail = original_person.contact_details.get() # Now try importing again, and refetch those objects: importer = PopItImporter() importer.import_from_export_json_data(data) self.assertEqual(models.Person.objects.count(), 1) new_person = models.Person.objects.get() self.assertEqual(new_person.name, "Alice") self.assertEqual(new_person.identifiers.count(), 2) new_identifier_a = new_person.identifiers.get(scheme='popit-person') new_identifier_b = new_person.identifiers.get(scheme='yournextmp-candidate') self.assertEqual(new_identifier_a.identifier, "a1b2") self.assertEqual(new_identifier_b.identifier, "123456789") self.assertEqual(new_person.contact_details.count(), 1) new_contact_detail = new_person.contact_details.get() # Now check that these objects haven't changed ID: self.assertEqual(original_person.id, new_person.id) self.assertEqual(original_identifier_a.id, new_identifier_a.id) self.assertEqual(original_identifier_b.id, new_identifier_b.id) self.assertEqual(original_contact_detail.id, new_contact_detail.id)