def determine_party(self, mp_info):
pa_list = mp_info['parties']
latest_pa = pa_list[0]
for pa in pa_list[1:]:
if pa['begin'] > latest_pa['begin']:
if not pg_to_party(pa['party']):
continue
latest_pa = pa
party_name = pg_to_party(latest_pa['party'])
if party_name:
party = Party.objects.get(abbreviation=party_name)
return party
return None
def determine_party(self, mp_info):
pa_list = mp_info['parties']
latest_pa = pa_list[0]
for pa in pa_list[1:]:
if pa['begin'] > latest_pa['begin']:
if not pg_to_party(pa['party']):
continue
latest_pa = pa
party_name = pg_to_party(latest_pa['party'])
if party_name:
party = Party.objects.get(abbreviation=party_name)
return party
return None
def save_member(self, mp_info):
# Member
try:
mp = Member.objects.get(origin_id=str(mp_info['id']))
if not self.replace:
return
except Member.DoesNotExist:
mp = Member(origin_id=str(mp_info['id']))
mp.name = mp_info['name']
mp.birth_date = mp_info['birthdate']
if 'birthplace' in mp_info:
mp.birth_place = mp_info['birthplace']
mp.given_names = mp_info['given_names']
mp.surname = mp_info['surname']
if 'email' in mp_info:
mp.email = mp_info['email']
else:
mp.email = None
if 'phone' in mp_info:
mp.phone = mp_info['phone']
else:
mp.phone = None
mp.info_link = mp_info['info_url']
# This is to support the hack where we add non-MP ministers
# as pseudo MPs. They do not get a PartyAssociation, only
# party attribute for purpose of counting ministers
if 'party' in mp_info:
mp.party = Party.objects.get(abbreviation=mp_info['party'])
else:
mp.party = self.determine_party(mp_info)
url = mp_info['portrait']
ext = url.split('.')[-1]
fname = slugify(mp.name) + '.' + ext
dir_name = os.path.join(mp.photo.field.upload_to, fname)
path = os.path.join(settings.MEDIA_ROOT, dir_name)
mp.photo = dir_name
if self.replace or not os.path.exists(path):
self.logger.debug("getting MP portrait")
s = self.open_url(url, 'members')
f = open(path, 'wb')
f.write(s)
f.close()
mp.mark_modified()
mp.save()
# Districts
DistrictAssociation.objects.filter(member=mp).delete()
for da in mp_info['districts']:
d_name = da['district'].replace(' vaalipiiri', '')
try:
district = District.objects.get(long_name=da['district'])
except District.DoesNotExist:
district = None
da_obj = DistrictAssociation(member=mp, begin=da['begin'])
da_obj.district = district
da_obj.name = d_name
if 'end' in da:
da_obj.end = da['end']
da_obj.save()
# Parties
PartyAssociation.objects.filter(member=mp).delete()
for pa in mp_info['parties']:
p_name = pg_to_party(pa['party'])
if p_name:
party = Party.objects.get(abbreviation=p_name)
else:
party = None
pa_obj = PartyAssociation(member=mp, begin=pa['begin'])
pa_obj.party = party
pa_obj.name = pa['party']
if 'end' in pa:
pa_obj.end = pa['end']
pa_obj.save()
def find_with_attrs(obj_list, arg_dict):
for o in obj_list:
for (key, val) in arg_dict.items():
oattr = getattr(o, key)
if oattr != val:
#print "mismatch: %s (%s) vs. %s (%s)" % (oattr, type(oattr), val, type(val))
break
else:
return o
ca_list = list(CommitteeAssociation.objects.filter(member=mp))
sa_list = list(SpeakerAssociation.objects.filter(member=mp))
ma_list = list(MinistryAssociation.objects.filter(member=mp))
# Memberships
for post in mp_info['posts']:
org_type = post.get('org_type', None)
if org_type in ('committee', 'speakers'):
if org_type == 'committee':
try:
committee = Committee.objects.get(name=post['org_name'])
# Update current committees
if committee.current == False and post['current'] == True:
committee.current = True
committee.save()
except Committee.DoesNotExist:
committee = Committee(name=post['org_name'],
current=post['current'])
committee.save()
# There can be several periods
periods = post['periods']
for period in periods:
# Role is only optional information
if 'role' in period:
role = period['role']
else:
role = None
args = dict(begin=period['begin'], end=period['end'], role=role)
# Differentiate between committee and speaker associations
if org_type == 'committee':
args['committee_id'] = committee.id
ca_obj = find_with_attrs(ca_list, args)
if not ca_obj:
args['member'] = mp
ca_obj = CommitteeAssociation(**args)
self.logger.debug("New committee association: %s" % ca_obj)
ca_obj.save()
else:
ca_obj.found = True
else:
sa_obj = find_with_attrs(sa_list, args)
if not sa_obj:
args['member'] = mp
sa_obj = SpeakerAssociation(**args)
self.logger.debug("New speaker association: %s" % sa_obj)
sa_obj.save()
else:
sa_obj.found = True
else:
args = dict(label=post['label'], role=post['role'], begin=post['begin'], end=post['end'])
ma_obj = find_with_attrs(ma_list, args)
if not ma_obj:
args['member'] = mp
ma_obj = MinistryAssociation(**args)
self.logger.debug("New ministry association: %s" % ma_obj)
ma_obj.save()
else:
ma_obj.found = True
for obj in ca_list + ma_list + sa_list:
if not getattr(obj, 'found', False):
self.logger.warning("Deleting removed association: %s" % obj)
obj.delete()
def save_member(self, mp_info):
# Member
try:
mp = Member.objects.get(origin_id=str(mp_info['id']))
if not self.replace:
return
except Member.DoesNotExist:
mp = Member(origin_id=str(mp_info['id']))
mp.name = mp_info['name']
mp.birth_date = mp_info['birthdate']
if 'birthplace' in mp_info:
mp.birth_place = mp_info['birthplace']
mp.given_names = mp_info['given_names']
mp.surname = mp_info['surname']
if 'email' in mp_info:
mp.email = mp_info['email']
else:
mp.email = None
if 'phone' in mp_info:
mp.phone = mp_info['phone']
else:
mp.phone = None
if 'gender' in mp_info and mp_info['gender']:
mp.gender = mp_info['gender']
mp.info_link = mp_info['info_url']
# This is to support the hack where we add non-MP ministers
# as pseudo MPs. They do not get a PartyAssociation, only
# party attribute for purpose of counting ministers
if 'party' in mp_info:
mp.party = Party.objects.get(abbreviation=mp_info['party'])
else:
mp.party = self.determine_party(mp_info)
url = mp_info['portrait']
fname = slugify(mp.name) + '.jpg'
dir_name = os.path.join(mp.photo.field.upload_to, fname)
path = os.path.join(settings.MEDIA_ROOT, dir_name)
mp.photo = dir_name
if self.replace or not os.path.exists(path):
self.logger.debug("getting MP portrait")
s = self.open_url(url, 'members')
f = open(path, 'wb')
f.write(s)
f.close()
mp.mark_modified()
mp.save()
# Districts
DistrictAssociation.objects.filter(member=mp).delete()
for da in mp_info['districts']:
d_name = da['district'].replace(' vaalipiiri', '')
try:
district = District.objects.get(long_name=da['district'])
except District.DoesNotExist:
district = None
da_obj = DistrictAssociation(member=mp, begin=da['begin'])
da_obj.district = district
da_obj.name = d_name
if 'end' in da:
da_obj.end = da['end']
da_obj.save()
# Parties
PartyAssociation.objects.filter(member=mp).delete()
for pa in mp_info['parties']:
p_name = pg_to_party(pa['party'])
if p_name:
party = Party.objects.get(abbreviation=p_name)
else:
party = None
pa_obj = PartyAssociation(member=mp, begin=pa['begin'])
pa_obj.party = party
pa_obj.name = pa['party']
if 'end' in pa:
pa_obj.end = pa['end']
pa_obj.save()
def find_with_attrs(obj_list, arg_dict):
for o in obj_list:
for (key, val) in list(arg_dict.items()):
oattr = getattr(o, key)
if oattr != val:
#print "mismatch: %s (%s) vs. %s (%s)" % (oattr, type(oattr), val, type(val))
break
else:
return o
ca_list = list(CommitteeAssociation.objects.filter(member=mp))
sa_list = list(SpeakerAssociation.objects.filter(member=mp))
ma_list = list(MinistryAssociation.objects.filter(member=mp))
# Memberships
for post in mp_info['posts']:
org_type = post.get('org_type', None)
if org_type in ('committee', 'speakers'):
if org_type == 'committee':
try:
committee = Committee.objects.get(
name=post['org_name'])
# Update current committees
if committee.current == False and post[
'current'] == True:
committee.current = True
committee.save()
except Committee.DoesNotExist:
committee = Committee(name=post['org_name'],
current=post['current'])
committee.save()
# There can be several periods
periods = post['periods']
for period in periods:
# Role is only optional information
if 'role' in period:
role = period['role']
else:
role = None
args = dict(begin=period['begin'],
end=period['end'],
role=role)
# Differentiate between committee and speaker associations
if org_type == 'committee':
args['committee_id'] = committee.id
ca_obj = find_with_attrs(ca_list, args)
if not ca_obj:
args['member'] = mp
ca_obj = CommitteeAssociation(**args)
self.logger.debug("New committee association: %s" %
ca_obj)
ca_obj.save()
else:
ca_obj.found = True
else:
sa_obj = find_with_attrs(sa_list, args)
if not sa_obj:
args['member'] = mp
sa_obj = SpeakerAssociation(**args)
self.logger.debug("New speaker association: %s" %
sa_obj)
sa_obj.save()
else:
sa_obj.found = True
else:
args = dict(label=post['label'],
role=post['role'],
begin=post['begin'],
end=post['end'])
ma_obj = find_with_attrs(ma_list, args)
if not ma_obj:
args['member'] = mp
ma_obj = MinistryAssociation(**args)
self.logger.debug("New ministry association: %s" % ma_obj)
ma_obj.save()
else:
ma_obj.found = True
for obj in ca_list + ma_list + sa_list:
if not getattr(obj, 'found', False):
self.logger.warning("Deleting removed association: %s" % obj)
obj.delete()
