def update_popit_person(self, popit_person_id, ppc_data, image_filename):
from candidates.models import PopItPerson
from ..images import image_uploaded_already
# Get the existing data first:
person_data, _ = self.get_person(popit_person_id)
previous_versions = person_data.pop("versions")
new_person_data = self.get_person_data_from_ppc(ppc_data)
# Remove any empty keys, we don't want to overwrite exiting
# data with nothing:
keys = new_person_data.keys()
warnings = []
for key in keys:
if not new_person_data[key]:
del new_person_data[key]
# Also make sure that we don't overwrite any existing
# fields that are filled in with different values:
if key not in ("standing_in", "party_memberships"):
new_person_data_value = new_person_data.get(key)
person_data_value = person_data.get(key)
if (person_data_value and new_person_data_value
and new_person_data_value != person_data_value):
if key_value_appeared_in_previous_version(
key, new_person_data_value, previous_versions):
warning_message = "[{0}] it looks as if a previous "
warning_message += "version had {1}, so not "
warning_message += "overwriting the current value {2}"
warnings.append(
warning_message.format(key, new_person_data_value,
person_data_value))
del new_person_data[key]
else:
warnings.append("[{}] replacing {}".format(
key, person_data_value))
warnings.append("[{}] with new value {}".format(
key, new_person_data_value))
if warnings:
print("Warnings for person/{} {}".format(
popit_person_id, person_data["name"]).encode("utf-8"))
for warning in warnings:
print(" ...", warning.encode("utf-8"))
merged_person_data = merge_person_data(person_data, new_person_data)
change_metadata = get_change_metadata(
None,
"Updated candidate from official PPC data ({})".format(
ppc_data["party_slug"]),
)
person = PopItPerson.create_from_reduced_json(merged_person_data)
person.record_version(change_metadata)
person_id = person.save_to_popit(self.api)
if image_filename:
if image_uploaded_already(self.api.persons, person_id,
image_filename):
print("That image has already been uploaded!")
else:
print("Uploading image...")
self.upload_person_image(person_id, image_filename,
ppc_data["image_url"])
person.invalidate_cache_entries()
return person_id
def update_popit_person(self, popit_person_id, ppc_data, image_filename):
from candidates.models import PopItPerson
from ..images import image_uploaded_already
# Get the existing data first:
person_data, _ = self.get_person(popit_person_id)
previous_versions = person_data.pop('versions')
new_person_data = self.get_person_data_from_ppc(ppc_data)
# Remove any empty keys, we don't want to overwrite exiting
# data with nothing:
keys = new_person_data.keys()
warnings = []
for key in keys:
if not new_person_data[key]:
del new_person_data[key]
# Also make sure that we don't overwrite any existing
# fields that are filled in with different values:
if key not in ('standing_in', 'party_memberships'):
new_person_data_value = new_person_data.get(key)
person_data_value = person_data.get(key)
if person_data_value and new_person_data_value and new_person_data_value != person_data_value:
if key_value_appeared_in_previous_version(
key,
new_person_data_value,
previous_versions
):
warning_message = "[{0}] it looks as if a previous "
warning_message += "version had {1}, so not "
warning_message += "overwriting the current value {2}"
warnings.append(warning_message.format(
key,
new_person_data_value,
person_data_value
))
del new_person_data[key]
else:
warnings.append("[{0}] replacing {1}".format(key, person_data_value))
warnings.append("[{0}] with new value {1}".format(key, new_person_data_value))
if warnings:
print("Warnings for person/{0} {1}".format(
popit_person_id, person_data['name']
).encode('utf-8'))
for warning in warnings:
print(" ...", warning.encode('utf-8'))
merged_person_data = merge_person_data(person_data, new_person_data)
change_metadata = get_change_metadata(
None,
'Updated candidate from official PPC data ({0})'.format(ppc_data['party_slug']),
)
person = PopItPerson.create_from_reduced_json(merged_person_data)
person.record_version(change_metadata)
person_id = person.save_to_popit(self.api)
if image_filename:
if image_uploaded_already(self.api.persons, person_id, image_filename):
print("That image has already been uploaded!")
else:
print("Uploading image...")
self.upload_person_image(person_id, image_filename, ppc_data['image_url'])
person.invalidate_cache_entries()
return person_id
def add_popit_person(self, ppc_data, image_filename):
change_metadata = get_change_metadata(
None,
'Created new candidate from official PPC data ({0})'.format(ppc_data['party_slug']),
)
person_data = self.get_person_data_from_ppc(ppc_data)
person = PopItPerson.create_from_reduced_json(person_data)
person.record_version(change_metadata)
person_id = person.save_to_popit(self.api)
if image_filename:
self.upload_person_image(person_id, image_filename, ppc_data['image_url'])
person.invalidate_cache_entries()
return person_id
def test_get_person_data_from_dict_clear_email(self):
form_data = {
'name': 'John Doe',
'email': '',
'birth_date': '',
'wikipedia_url': 'http://en.wikipedia.org/wiki/John_Doe',
'homepage_url': '',
'twitter_username': '******',
'facebook_personal_url': '',
'facebook_page_url': '',
'party_ppc_page_url': '',
}
expected_result = {
'birth_date':
None,
'contact_details': [{
'type': 'twitter',
'value': 'foobar'
}],
'email':
'',
'gender':
'',
'honorific_prefix':
'',
'honorific_suffix':
'',
'id':
None,
'links': [{
'note': 'wikipedia',
'url': 'http://en.wikipedia.org/wiki/John_Doe'
}],
'memberships': [],
'name':
u'John Doe',
'party_memberships': {},
'standing_in': {},
}
p = PopItPerson.create_from_reduced_json(form_data)
self.assertEqual(p.popit_data, expected_result)
def test_get_person_data_from_dict_clear_email(self):
form_data = {
'name': 'John Doe',
'email': '',
'birth_date': '',
'wikipedia_url': 'http://en.wikipedia.org/wiki/John_Doe',
'homepage_url': '',
'twitter_username': '******',
'facebook_personal_url': '',
'facebook_page_url': '',
'party_ppc_page_url': '',
}
expected_result = {
'birth_date': None,
'contact_details': [
{
'type': 'twitter',
'value': 'foobar'
}
],
'email': '',
'gender': '',
'honorific_prefix': '',
'honorific_suffix': '',
'id': None,
'links': [
{
'note': 'wikipedia', 'url': 'http://en.wikipedia.org/wiki/John_Doe'
}
],
'memberships': [],
'name': u'John Doe',
'party_memberships': {},
'standing_in': {},
}
p = PopItPerson.create_from_reduced_json(form_data)
self.assertEqual(
p.popit_data,
expected_result
)
def test_update_tessa_jowell(self, mock_invalidate_person,
mock_invalidate_posts, mocked_put):
mock_api = MagicMock()
mock_api.persons = FakePersonCollection
old_person_data = {
"birth_date": '1947',
"email": "*****@*****.**",
"facebook_page_url": "",
"facebook_personal_url": "",
"gender": "",
"homepage_url": "http://foo.example.org",
"honorific_prefix": "",
"honorific_suffix": "",
"id": "2009",
"identifiers": [],
"image": None,
"linkedin_url": "",
"name": "Tessa Jowell",
"other_names": [],
"party_memberships": {
"2010": {
"id": "party:53",
"name": "Labour Party"
},
"2015": {
"id": "party:53",
"name": "Labour Party"
}
},
"party_ppc_page_url": "",
"proxy_image": None,
"standing_in": {
"2010": {
"mapit_url": "http://mapit.mysociety.org/area/65808",
"name": "Dulwich and West Norwood",
"post_id": "65808",
},
"2015": {
"mapit_url": "http://mapit.mysociety.org/area/65913",
"name": "Camberwell and Peckham",
"post_id": "65913",
}
},
"twitter_username": "******",
"wikipedia_url": "",
}
new_person_data = deepcopy(old_person_data)
new_person_data['standing_in']['2015'] = {
"mapit_url": "http://mapit.mysociety.org/area/65808",
"name": "Dulwich and West Norwood",
"post_id": "65808",
}
previous_version = {
'data': {
"name": "Tessa Jowell",
"standing_in": {
"2010": {
"mapit_url": "http://mapit.mysociety.org/area/65808",
"name": "Dulwich and West Norwood",
"post_id": "65808",
},
"2015": {
"mapit_url": "http://mapit.mysociety.org/area/65913",
"name": "Camberwell and Peckham",
"post_id": "65913",
}
},
}
}
person = PopItPerson.create_from_reduced_json(old_person_data)
person.update_from_reduced_json(new_person_data)
person.versions = [previous_version]
person.record_version(
{
'information_source': 'A change made for testing purposes',
'username': '******',
'version_id': '6054aa38b30b4418',
'timestamp': '2014-09-28T14:02:44.567413',
}, )
person.save_to_popit(mock_api)
self.assertEqual(2, len(mocked_put.call_args_list))
first_put_call_args = {
'birth_date':
'1947',
'contact_details': [],
'email':
u'*****@*****.**',
'gender':
'',
'honorific_prefix':
'',
'honorific_suffix':
'',
'id':
'2009',
'identifiers': [],
'name':
u'Tessa Jowell',
'links': [],
'other_names': [],
'party_memberships':
None,
'standing_in':
None,
'versions': [{
'username': '******',
'information_source': 'A change made for testing purposes',
'version_id': '6054aa38b30b4418',
'timestamp': '2014-09-28T14:02:44.567413',
'data': new_person_data
}, previous_version],
}
second_put_call_args = {
'birth_date':
'1947',
'contact_details': [{
'type': 'twitter',
'value': 'jowellt'
}],
'email':
u'*****@*****.**',
'gender':
'',
'honorific_prefix':
'',
'honorific_suffix':
'',
'id':
'2009',
'identifiers': [],
'name':
u'Tessa Jowell',
'links': [{
'note': 'homepage',
'url': 'http://foo.example.org'
}],
'other_names': [],
'party_memberships': {
'2010': {
'id': 'party:53',
'name': 'Labour Party'
},
'2015': {
'id': 'party:53',
'name': 'Labour Party'
}
},
'standing_in': {
'2015': {
'name': 'Dulwich and West Norwood',
'mapit_url': 'http://mapit.mysociety.org/area/65808',
'post_id': '65808',
},
'2010': {
'name': 'Dulwich and West Norwood',
'mapit_url': 'http://mapit.mysociety.org/area/65808',
'post_id': '65808',
}
},
'versions': [{
'username': '******',
'information_source': 'A change made for testing purposes',
'version_id': '6054aa38b30b4418',
'timestamp': '2014-09-28T14:02:44.567413',
'data': new_person_data
}, previous_version],
}
self.assertTrue(
equal_call_args(
[first_put_call_args],
mocked_put.call_args_list[0][0],
),
"Unexpected first PUT (the one blanking out standing_in and party_memberships",
)
self.assertTrue(
equal_call_args(
[second_put_call_args],
mocked_put.call_args_list[1][0],
),
"Unexpected second PUT (the one with real standing_in and party_memberships",
)
self.assertEqual(4, mock_api.memberships.post.call_count)
posted_memberships = [
c[0][0] for c in mock_api.memberships.post.call_args_list
]
self.assertEqual(posted_memberships, [
{
"election": "2015",
"end_date": "9999-12-31",
"person_id": "2009",
"post_id": "65808",
"role": "Candidate",
"start_date": "2010-05-07"
},
{
"election": "2010",
"end_date": "2010-05-06",
"person_id": "2009",
"post_id": "65808",
"role": "Candidate",
"start_date": "2005-05-06"
},
{
"end_date": "9999-12-31",
"organization_id": "party:53",
"person_id": "2009",
"start_date": "2010-05-07"
},
{
"end_date": "2010-05-06",
"organization_id": "party:53",
"person_id": "2009",
"start_date": "2005-05-06"
},
])
mock_invalidate_person.assert_called_with('2009')
mock_invalidate_posts.assert_called_with(set(['65808', '65913']))
def test_update_tessa_jowell(
self,
mock_invalidate_person,
mock_invalidate_posts,
mocked_put
):
mock_api = MagicMock()
mock_api.persons = FakePersonCollection
old_person_data = {
"birth_date": '1947',
"email": "*****@*****.**",
"facebook_page_url": "",
"facebook_personal_url": "",
"gender": "",
"homepage_url": "http://foo.example.org",
"honorific_prefix": "",
"honorific_suffix": "",
"id": "2009",
"identifiers": [],
"image": None,
"linkedin_url": "",
"name": "Tessa Jowell",
"other_names": [],
"party_memberships": {
"2010": {
"id": "party:53",
"name": "Labour Party"
},
"2015": {
"id": "party:53",
"name": "Labour Party"
}
},
"party_ppc_page_url": "",
"proxy_image": None,
"standing_in": {
"2010": {
"mapit_url": "http://mapit.mysociety.org/area/65808",
"name": "Dulwich and West Norwood",
"post_id": "65808",
},
"2015": {
"mapit_url": "http://mapit.mysociety.org/area/65913",
"name": "Camberwell and Peckham",
"post_id": "65913",
}
},
"twitter_username": "******",
"wikipedia_url": "",
}
new_person_data = deepcopy(old_person_data)
new_person_data['standing_in']['2015'] = {
"mapit_url": "http://mapit.mysociety.org/area/65808",
"name": "Dulwich and West Norwood",
"post_id": "65808",
}
previous_version = {
'data': {
"name": "Tessa Jowell",
"standing_in": {
"2010": {
"mapit_url": "http://mapit.mysociety.org/area/65808",
"name": "Dulwich and West Norwood",
"post_id": "65808",
},
"2015": {
"mapit_url": "http://mapit.mysociety.org/area/65913",
"name": "Camberwell and Peckham",
"post_id": "65913",
}
},
}
}
person = PopItPerson.create_from_reduced_json(old_person_data)
person.update_from_reduced_json(new_person_data)
person.versions = [previous_version]
person.record_version(
{
'information_source': 'A change made for testing purposes',
'username': '******',
'version_id': '6054aa38b30b4418',
'timestamp': '2014-09-28T14:02:44.567413',
},
)
person.save_to_popit(mock_api)
self.assertEqual(2, len(mocked_put.call_args_list))
first_put_call_args = {
'birth_date': '1947',
'contact_details': [],
'email': u'*****@*****.**',
'gender': '',
'honorific_prefix': '',
'honorific_suffix': '',
'id': '2009',
'identifiers': [],
'name': u'Tessa Jowell',
'links': [],
'other_names': [],
'party_memberships': None,
'standing_in': None,
'versions': [
{
'username': '******',
'information_source': 'A change made for testing purposes',
'version_id': '6054aa38b30b4418',
'timestamp': '2014-09-28T14:02:44.567413',
'data': new_person_data
},
previous_version
],
}
second_put_call_args = {
'birth_date': '1947',
'contact_details': [
{
'type': 'twitter',
'value': 'jowellt'
}
],
'email': u'*****@*****.**',
'gender': '',
'honorific_prefix': '',
'honorific_suffix': '',
'id': '2009',
'identifiers': [],
'name': u'Tessa Jowell',
'links': [
{
'note': 'homepage',
'url': 'http://foo.example.org'
}
],
'other_names': [],
'party_memberships': {
'2010': {
'id': 'party:53',
'name': 'Labour Party'
},
'2015': {
'id': 'party:53',
'name': 'Labour Party'
}
},
'standing_in': {
'2015': {
'name': 'Dulwich and West Norwood',
'mapit_url': 'http://mapit.mysociety.org/area/65808',
'post_id': '65808',
},
'2010': {
'name': 'Dulwich and West Norwood',
'mapit_url': 'http://mapit.mysociety.org/area/65808',
'post_id': '65808',
}
},
'versions': [
{
'username': '******',
'information_source': 'A change made for testing purposes',
'version_id': '6054aa38b30b4418',
'timestamp': '2014-09-28T14:02:44.567413',
'data': new_person_data
},
previous_version
],
}
self.assertTrue(
equal_call_args(
[first_put_call_args],
mocked_put.call_args_list[0][0],
),
"Unexpected first PUT (the one blanking out standing_in and party_memberships",
)
self.assertTrue(
equal_call_args(
[second_put_call_args],
mocked_put.call_args_list[1][0],
),
"Unexpected second PUT (the one with real standing_in and party_memberships",
)
self.assertEqual(4, mock_api.memberships.post.call_count)
posted_memberships = [
c[0][0]
for c in mock_api.memberships.post.call_args_list
]
self.assertEqual(
posted_memberships,
[
{
"election": "2015",
"end_date": "9999-12-31",
"person_id": "2009",
"post_id": "65808",
"role": "Candidate",
"start_date": "2010-05-07"
},
{
"election": "2010",
"end_date": "2010-05-06",
"person_id": "2009",
"post_id": "65808",
"role": "Candidate",
"start_date": "2005-05-06"
},
{
"end_date": "9999-12-31",
"organization_id": "party:53",
"person_id": "2009",
"start_date": "2010-05-07"
},
{
"end_date": "2010-05-06",
"organization_id": "party:53",
"person_id": "2009",
"start_date": "2005-05-06"
},
]
)
mock_invalidate_person.assert_called_with('2009')
mock_invalidate_posts.assert_called_with(set(['65808', '65913']))