def test_exception_unknown_collection_on_create_identifier(self):
commons = models.Organization.objects.create(name="The Commons")
importer = PopItImporter()
with self.assertRaisesRegexp(
Exception,
r"Unknown collection 'not-a-collection'"):
importer.create_identifier('not-a-collection', 'commons', commons)
def test_inline_area_parent_relationship(self):
input_json = '''
{
"organizations": [
{
"id": "sp",
"name": "Scottish Parliament",
"area": {
"id": "subarea-1",
"name": "Scotland",
"identifier": "sco",
"classification": "country",
"parent_id": "area-1"
}
}
],
"areas": [
{
"id": "area-1",
"name": "United Kingdom",
"identifier": "uk",
"classification": "country"
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter()
importer.import_from_export_json_data(data)
self.assertEqual(models.Area.objects.count(), 2)
child = models.Area.objects.get(name='Scotland')
parent = models.Area.objects.get(name='United Kingdom')
self.assertEqual(child.parent, parent)
def test_link_and_source_without_a_note(self):
input_json = '''
{
"persons": [
{
"id": "a1b2",
"name": "Alice",
"sources": [
{
"url": "http://example.org/source-of-alice-data"
}
],
"links": [
{
"url": "http://example.org/a-link-without-a-note"
}
]
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter()
importer.import_from_export_json_data(data)
self.assertEqual(models.Person.objects.count(), 1)
person = models.Person.objects.get()
self.assertEqual(person.name, "Alice")
self.assertEqual(
person.sources.get().url,
"http://example.org/source-of-alice-data")
self.assertEqual(
person.links.get().url,
"http://example.org/a-link-without-a-note")
def test_all_top_level_optional(self):
# This just check that you don't get an exception when
# importing empty Popolo JSON.
input_json = '{}'
data = json.loads(input_json)
importer = PopItImporter()
importer.import_from_export_json_data(data)
def test_update_existing_organization(self):
existing = models.Organization.objects.create(name="The Commons")
existing.identifiers.create(
scheme="popit-organization", identifier="commons")
input_json = '''
{
"organizations": [
{
"id": "commons",
"name": "House of Commons"
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter()
importer.import_from_export_json_data(data)
self.assertEqual(models.Organization.objects.count(), 1)
organization = models.Organization.objects.get()
self.assertEqual(existing, organization)
self.assertEqual(organization.name, "House of Commons")
self.assertEqual(
organization.identifiers.get(scheme='popit-organization').identifier,
"commons"
)
def test_organization_with_identifiers(self):
input_json = '''
{
"organizations": [
{
"id": "commons",
"name": "House of Commons",
"identifiers": [
{
"scheme": "big-db-of-parliaments",
"identifier": "uk-commons"
}
]
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter()
importer.import_from_export_json_data(data)
self.assertEqual(models.Organization.objects.count(), 1)
organization = models.Organization.objects.get()
self.assertEqual(organization.name, "House of Commons")
self.assertEqual(
organization.identifiers.get(scheme='popit-organization').identifier,
"commons"
)
self.assertEqual(
organization.identifiers.get(scheme='big-db-of-parliaments').identifier,
"uk-commons"
)
def test_truncation_warn(self):
long_name = ('Albert ' * 100).strip()
input_json = '''
{{
"persons": [
{{
"id": "a1b2",
"name": "{0}"
}}
]
}}
'''.format(long_name)
data = json.loads(input_json)
# With truncate='warn' the field should be truncated, but
# print a warning to stderr:
importer = PopItImporter(truncate='warn')
with capture_output() as (out, err):
importer.import_from_export_json_data(data)
output = err.getvalue().strip()
self.assertIn('Warning: truncating Albert', output)
self.assertIn('Albert to a length of 512', output)
person = models.Person.objects.get()
max_length = person._meta.get_field('name').max_length
truncated_name = long_name[:max_length]
self.assertEqual(person.name, truncated_name)
def test_post_creation(self):
input_json = '''
{
"posts": [
{
"id": "65808",
"label": "Member of Parliament for Dulwich and West Norwood",
"role": "Member of Parliament",
"organization_id": "commons"
}
],
"organizations": [
{
"id": "commons",
"name": "House of Commons"
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter()
importer.import_from_export_json_data(data)
self.assertEqual(models.Post.objects.count(), 1)
post = models.Post.objects.get()
self.assertEqual(post.label, "Member of Parliament for Dulwich and West Norwood")
self.assertEqual(post.role, "Member of Parliament")
self.assertEqual(models.Organization.objects.count(), 1)
def test_remove_identifiers_when_empty_array_specified(self):
existing_person = models.Person.objects.create(name='Algernon')
existing_person.identifiers.create(
scheme='popolo:person',
identifier="a1b2"
)
existing_person.identifiers.create(
scheme='ignorable',
identifier="some-data-we-care-nothing-for"
)
input_json = '''
{
"persons": [
{
"id": "a1b2",
"name": "Alice",
"identifiers": []
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter(id_prefix='popolo:')
importer.import_from_export_json_data(data)
self.assertEqual(models.Person.objects.count(), 1)
# Reget the person from the database:
person = models.Person.objects.get(pk=existing_person.id)
self.assertEqual(person.name, 'Alice')
self.assertSequenceEqual(
person.identifiers.order_by('scheme').values_list('scheme', 'identifier'),
[(u'popolo:person', u'a1b2')]
)
def test_organisation_with_external_area(self):
input_json = '''
{
"organizations": [
{
"id": "commons",
"name": "House of Commons",
"area_id": "area-1"
}
],
"areas": [
{
"id": "area-1",
"name": "The United Kingdom of Great Britain and Northern Ireland",
"identifier": "uk",
"classification": "country"
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter()
importer.import_from_export_json_data(data)
self.assertEqual(models.Organization.objects.count(), 1)
self.assertEqual(models.Area.objects.count(), 1)
area = models.Area.objects.get()
self.assertEqual(
area.name,
'The United Kingdom of Great Britain and Northern Ireland')
self.assertEqual(area.identifier, 'uk')
self.assertEqual(area.classification, 'country')
def test_area_update(self):
existing = models.Area.objects.create(
name="Alba",
identifier="blah-sco",
classification="country",
)
existing.other_identifiers.create(
scheme='popit-area',
identifier='subarea-1',
)
input_json = '''
{
"areas": [
{
"id": "subarea-1",
"name": "Scotland",
"identifier": "sco",
"classification": "country"
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter()
importer.import_from_export_json_data(data)
self.assertEqual(models.Area.objects.count(), 1)
area = models.Area.objects.get()
self.assertEqual(area.name, 'Scotland')
self.assertEqual(area.identifier, 'sco')
self.assertEqual(area.classification, 'country')
def test_custom_identifier_prefix(self):
input_json = '''
{
"persons": [
{
"id": "a1b2",
"name": "Alice"
}
],
"organizations": [
{
"id": "commons",
"name": "House of Commons"
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter(id_prefix='popolo:')
importer.import_from_export_json_data(data)
self.assertEqual(models.Person.objects.count(), 1)
self.assertEqual(models.Organization.objects.count(), 1)
person = models.Person.objects.get()
organization = models.Organization.objects.get()
person_identifier = person.identifiers.get()
organization_identifier = organization.identifiers.get()
self.assertEqual(person_identifier.scheme, 'popolo:person')
self.assertEqual(person_identifier.identifier, 'a1b2')
self.assertEqual(organization_identifier.scheme, 'popolo:organization')
self.assertEqual(organization_identifier.identifier, 'commons')
def test_membership_no_dates_but_some_in_legislative_period(self):
data = json.loads(self.input_json)
importer = PopItImporter()
importer.import_from_export_json_data(data)
self.assertEqual(models.Membership.objects.count(), 2)
earlier, later = models.Membership.objects.order_by('start_date')
self.assertEqual(earlier.start_date, '2010-05-06')
self.assertEqual(earlier.end_date, '2011-02-08')
self.assertEqual(later.start_date, '2015-05-08')
self.assertEqual(later.end_date, '')
def test_related_objects_for_person(self):
input_json = '''
{
"persons": [
{
"id": "a1b2",
"name": "Alice",
"identifiers": [
{
"identifier": "123456789",
"scheme": "yournextmp-candidate"
}
],
"contact_details": [
{
"contact_type": "twitter",
"label": "",
"note": "",
"value": "sometwitterusernameorother"
}
],
"links": [
{
"note": "homepage",
"url": "http://example.com/alice"
}
]
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter()
importer.import_from_export_json_data(data)
self.assertEqual(models.Person.objects.count(), 1)
person = models.Person.objects.get()
self.assertEqual(person.name, "Alice")
self.assertEqual(person.identifiers.count(), 2)
self.assertEqual(
person.identifiers.get(scheme='popit-person').identifier,
"a1b2")
self.assertEqual(
person.identifiers.get(scheme='yournextmp-candidate').identifier,
"123456789")
self.assertEqual(person.contact_details.count(), 1)
contact_detail = person.contact_details.get()
self.assertEqual(contact_detail.contact_type, 'twitter')
self.assertEqual(contact_detail.value, 'sometwitterusernameorother')
self.assertEqual(person.links.count(), 1)
link = person.links.get()
self.assertEqual(link.note, 'homepage')
self.assertEqual(link.url, 'http://example.com/alice')
def handle(self, **options):
self.importer = PopItImporter()
self.generate_party_sets()
with open(options['CSV-FILENAME']) as f:
csv_reader = csv.reader(f)
for party_data in csv_reader:
self.update_party(party_data)
def test_truncation_none(self):
long_name = ('Albert ' * 100).strip()
input_json = '''
{{
"persons": [
{{
"id": "a1b2",
"name": "{0}"
}}
]
}}
'''.format(long_name)
data = json.loads(input_json)
# With truncate='yes', there should be no exception:
importer = PopItImporter(truncate='yes')
importer.import_from_export_json_data(data)
person = models.Person.objects.get()
max_length = person._meta.get_field('name').max_length
truncated_name = long_name[:max_length]
self.assertEqual(person.name, truncated_name)
def test_creates_new_person_if_not_found(self):
existing_person = models.Person.objects.create(name='Algernon')
input_json = '''
{
"persons": [
{
"id": "a1b2",
"name": "Alice"
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter()
importer.import_from_export_json_data(data)
self.assertEqual(models.Person.objects.count(), 2)
new_person = models.Person.objects.exclude(pk=existing_person.id).get()
new_person_identifier = new_person.identifiers.get()
self.assertEqual(new_person_identifier.scheme, 'popit-person')
self.assertEqual(new_person_identifier.identifier, 'a1b2')
def test_person_with_membership(self):
input_json = '''
{
"persons": [
{
"id": "a1b2",
"name": "Alice"
}
],
"organizations": [
{
"id": "commons",
"name": "House of Commons"
}
],
"memberships": [
{
"person_id": "a1b2",
"organization_id": "commons"
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter()
importer.import_from_export_json_data(data)
self.assertEqual(models.Membership.objects.count(), 1)
self.assertEqual(models.Person.objects.count(), 1)
self.assertEqual(models.Organization.objects.count(), 1)
person = models.Person.objects.get()
self.assertEqual(person.name, "Alice")
organization = models.Organization.objects.get()
self.assertEqual(organization.name, "House of Commons")
self.assertEqual(
organization.identifiers.get(scheme='popit-organization').identifier,
"commons"
)
membership = models.Membership.objects.get()
self.assertEqual(membership.person, person)
self.assertEqual(membership.organization, organization)
def test_post_update(self):
existing_org = models.Organization.objects.create(name='House of Commons')
existing_org.identifiers.create(
scheme='popit-organization', identifier='commons')
existing_post = models.Post.objects.create(
label="MP for Dulwich and West Norwood",
role="MP",
organization=existing_org
)
models.Identifier.objects.create(
scheme='popit-post',
identifier='65808',
content_object=existing_post,
)
# Now import JSON that updates that post:
input_json = '''
{
"posts": [
{
"id": "65808",
"label": "Member of Parliament for Dulwich and West Norwood",
"role": "Member of Parliament",
"organization_id": "commons"
}
],
"organizations": [
{
"id": "commons",
"name": "House of Commons"
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter()
importer.import_from_export_json_data(data)
self.assertEqual(models.Post.objects.count(), 1)
post = models.Post.objects.get()
self.assertEqual(post.label, "Member of Parliament for Dulwich and West Norwood")
self.assertEqual(post.role, "Member of Parliament")
self.assertEqual(models.Organization.objects.count(), 1)
def test_values_in_related_objects_are_truncated(self):
input_json = '''
{
"persons": [
{
"id": "a1b2",
"name": "Alice",
"contact_details": [
{
"type": "official-work-email",
"value": "*****@*****.**"
}
]
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter(truncate='yes')
importer.import_from_export_json_data(data)
cd = models.ContactDetail.objects.get()
self.assertEqual(cd.contact_type, 'official-wor')
# Also test that they can be updated without creating a
# duplicate. (This could happen if the implementation uses the
# overlong version for checking existence, but the truncated
# version for setting.)
importer.import_from_export_json_data(data)
self.assertEqual(models.ContactDetail.objects.count(), 1)
cd = models.ContactDetail.objects.get()
self.assertEqual(cd.contact_type, 'official-wor')
def test_simplest_person(self):
input_json = '''
{
"persons": [
{
"id": "a1b2",
"name": "Alice"
}
],
"organizations": [],
"memberships": []
}
'''
data = json.loads(input_json)
importer = PopItImporter()
importer.import_from_export_json_data(data)
self.assertEqual(models.Person.objects.count(), 1)
person = models.Person.objects.get()
self.assertEqual(person.name, "Alice")
self.assertEqual(
person.identifiers.get(scheme='popit-person').identifier,
"a1b2")
def test_organization_parent_relationship(self):
input_json = '''
{
"organizations": [
{
"id": "co",
"name": "Cabinet Office",
"parent_id": "cs"
},
{
"id": "cs",
"name": "Civil Service"
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter()
importer.import_from_export_json_data(data)
self.assertEqual(models.Organization.objects.count(), 2)
child = models.Organization.objects.get(name='Cabinet Office')
parent = models.Organization.objects.get(name='Civil Service')
self.assertEqual(child.parent, parent)
def test_exception_from_inline_area_missing_id(self):
input_json = '''
{
"organizations": [
{
"id": "commons",
"name": "House of Commons",
"area": {
"name": "The United Kingdom of Great Britain and Northern Ireland",
"identifier": "uk",
"classification": "country"
}
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter()
with capture_output() as (out, err):
with self.assertRaisesRegexp(
ValueError,
'Found inline area data, but with no "id" attribute'):
importer.import_from_export_json_data(data)
def test_extra_preserved_identifiers(self):
existing_person = models.Person.objects.create(name='Algernon')
existing_person.identifiers.create(
scheme='popolo:person',
identifier="a1b2"
)
existing_person.identifiers.create(
scheme='ignorable',
identifier="some-data-we-care-nothing-for"
)
existing_person.identifiers.create(
scheme='preserve-me',
identifier="data-that-should-be-kept"
)
input_json = '''
{
"persons": [
{
"id": "a1b2",
"name": "Alice",
"identifiers": []
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter(
id_prefix='popolo:', id_schemes_to_preserve={'person': {'preserve-me'}})
importer.import_from_export_json_data(data)
self.assertEqual(models.Person.objects.count(), 1)
# Reget the person from the database:
person = models.Person.objects.get(pk=existing_person.id)
self.assertEqual(person.name, 'Alice')
self.assertSequenceEqual(
person.identifiers.order_by('scheme').values_list('scheme', 'identifier'),
[(u'popolo:person', u'a1b2'), (u'preserve-me', 'data-that-should-be-kept')]
)
def test_import_from_a_file(self):
input_json = '''
{
"persons": [
{
"id": "a1b2",
"name": "Alice"
}
],
"organizations": [
{
"id": "commons",
"name": "House of Commons"
}
],
"memberships": [
{
"person_id": "a1b2",
"organization_id": "commons"
}
]
}
'''
try:
with NamedTemporaryFile('w', delete=False, suffix='.json') as ntf:
ntf.write(input_json)
importer = PopItImporter()
importer.import_from_export_json(ntf.name)
finally:
os.remove(ntf.name)
# Now check everything was created:
self.assertEqual(models.Membership.objects.count(), 1)
self.assertEqual(models.Person.objects.count(), 1)
self.assertEqual(models.Organization.objects.count(), 1)
self.assertEqual(models.Identifier.objects.count(), 3)
def test_truncation_exception(self):
long_name = ('Albert ' * 100).strip()
input_json = '''
{{
"persons": [
{{
"id": "a1b2",
"name": "{0}"
}}
]
}}
'''.format(long_name)
data = json.loads(input_json)
# With the default, truncate='exception', there should be an
# exception raised:
importer = PopItImporter()
with self.assertRaisesRegexp(
ValidationError,
'Ensure this value has at most 512 characters'):
# Capture the output just to reduce noise in the test
# output - this would include output from
# show_data_on_error otherwise.
with capture_output() as (out, err):
importer.import_from_export_json_data(data)
def test_updates_person_if_found(self):
existing_person = models.Person.objects.create(name='Algernon')
existing_person.identifiers.create(
scheme='popolo:person',
identifier="a1b2"
)
input_json = '''
{
"persons": [
{
"id": "a1b2",
"name": "Alice"
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter(id_prefix='popolo:')
mock_observer = Mock()
importer.add_observer(mock_observer)
importer.import_from_export_json_data(data)
self.assertEqual(models.Person.objects.count(), 1)
# Reget the person from the database:
person = models.Person.objects.get(pk=existing_person.id)
self.assertEqual(person.name, 'Alice')
# Check that the observer was called with created=False:
self.assertEqual(mock_observer.notify.call_count, 1)
self.assertEqual(
mock_observer.notify.call_args,
call(
'person',
existing_person,
False,
{
"id": "a1b2",
"name": "Alice"
}
)
)
def handle(self, **options):
self.importer = PopItImporter()
party_set = self.get_party_set(options['party_set'])
with open(options['JSON-FILENAME']) as f:
for party_data in json.load(f):
self.update_party(party_data, party_set)
def test_truncation_unknown_option(self):
with self.assertRaises(ValueError):
PopItImporter(truncate='dunno')
def test_dont_recreate_related_objects(self):
input_json = '''
{
"persons": [
{
"id": "a1b2",
"name": "Alice",
"identifiers": [
{
"identifier": "123456789",
"scheme": "yournextmp-candidate"
}
],
"contact_details": [
{
"contact_type": "twitter",
"label": "",
"note": "",
"value": "sometwitterusernameorother"
}
],
"links": [
{
"note": "homepage",
"url": "http://example.com/alice"
}
]
}
]
}
'''
data = json.loads(input_json)
importer = PopItImporter()
importer.import_from_export_json_data(data)
self.assertEqual(models.Person.objects.count(), 1)
original_person = models.Person.objects.get()
self.assertEqual(original_person.name, "Alice")
self.assertEqual(original_person.identifiers.count(), 2)
original_identifier_a = original_person.identifiers.get(scheme='popit-person')
original_identifier_b = original_person.identifiers.get(scheme='yournextmp-candidate')
self.assertEqual(original_identifier_a.identifier, "a1b2")
self.assertEqual(original_identifier_b.identifier, "123456789")
self.assertEqual(original_person.contact_details.count(), 1)
original_contact_detail = original_person.contact_details.get()
# Now try importing again, and refetch those objects:
importer = PopItImporter()
importer.import_from_export_json_data(data)
self.assertEqual(models.Person.objects.count(), 1)
new_person = models.Person.objects.get()
self.assertEqual(new_person.name, "Alice")
self.assertEqual(new_person.identifiers.count(), 2)
new_identifier_a = new_person.identifiers.get(scheme='popit-person')
new_identifier_b = new_person.identifiers.get(scheme='yournextmp-candidate')
self.assertEqual(new_identifier_a.identifier, "a1b2")
self.assertEqual(new_identifier_b.identifier, "123456789")
self.assertEqual(new_person.contact_details.count(), 1)
new_contact_detail = new_person.contact_details.get()
# Now check that these objects haven't changed ID:
self.assertEqual(original_person.id, new_person.id)
self.assertEqual(original_identifier_a.id, new_identifier_a.id)
self.assertEqual(original_identifier_b.id, new_identifier_b.id)
self.assertEqual(original_contact_detail.id, new_contact_detail.id)