def determine_party(self, mp_info): pa_list = mp_info['parties'] latest_pa = pa_list[0] for pa in pa_list[1:]: if pa['begin'] > latest_pa['begin']: if not pg_to_party(pa['party']): continue latest_pa = pa party_name = pg_to_party(latest_pa['party']) if party_name: party = Party.objects.get(abbreviation=party_name) return party return None
def save_member(self, mp_info): # Member try: mp = Member.objects.get(origin_id=str(mp_info['id'])) if not self.replace: return except Member.DoesNotExist: mp = Member(origin_id=str(mp_info['id'])) mp.name = mp_info['name'] mp.birth_date = mp_info['birthdate'] if 'birthplace' in mp_info: mp.birth_place = mp_info['birthplace'] mp.given_names = mp_info['given_names'] mp.surname = mp_info['surname'] if 'email' in mp_info: mp.email = mp_info['email'] else: mp.email = None if 'phone' in mp_info: mp.phone = mp_info['phone'] else: mp.phone = None mp.info_link = mp_info['info_url'] # This is to support the hack where we add non-MP ministers # as pseudo MPs. They do not get a PartyAssociation, only # party attribute for purpose of counting ministers if 'party' in mp_info: mp.party = Party.objects.get(abbreviation=mp_info['party']) else: mp.party = self.determine_party(mp_info) url = mp_info['portrait'] ext = url.split('.')[-1] fname = slugify(mp.name) + '.' + ext dir_name = os.path.join(mp.photo.field.upload_to, fname) path = os.path.join(settings.MEDIA_ROOT, dir_name) mp.photo = dir_name if self.replace or not os.path.exists(path): self.logger.debug("getting MP portrait") s = self.open_url(url, 'members') f = open(path, 'wb') f.write(s) f.close() mp.mark_modified() mp.save() # Districts DistrictAssociation.objects.filter(member=mp).delete() for da in mp_info['districts']: d_name = da['district'].replace(' vaalipiiri', '') try: district = District.objects.get(long_name=da['district']) except District.DoesNotExist: district = None da_obj = DistrictAssociation(member=mp, begin=da['begin']) da_obj.district = district da_obj.name = d_name if 'end' in da: da_obj.end = da['end'] da_obj.save() # Parties PartyAssociation.objects.filter(member=mp).delete() for pa in mp_info['parties']: p_name = pg_to_party(pa['party']) if p_name: party = Party.objects.get(abbreviation=p_name) else: party = None pa_obj = PartyAssociation(member=mp, begin=pa['begin']) pa_obj.party = party pa_obj.name = pa['party'] if 'end' in pa: pa_obj.end = pa['end'] pa_obj.save() def find_with_attrs(obj_list, arg_dict): for o in obj_list: for (key, val) in arg_dict.items(): oattr = getattr(o, key) if oattr != val: #print "mismatch: %s (%s) vs. %s (%s)" % (oattr, type(oattr), val, type(val)) break else: return o ca_list = list(CommitteeAssociation.objects.filter(member=mp)) sa_list = list(SpeakerAssociation.objects.filter(member=mp)) ma_list = list(MinistryAssociation.objects.filter(member=mp)) # Memberships for post in mp_info['posts']: org_type = post.get('org_type', None) if org_type in ('committee', 'speakers'): if org_type == 'committee': try: committee = Committee.objects.get(name=post['org_name']) # Update current committees if committee.current == False and post['current'] == True: committee.current = True committee.save() except Committee.DoesNotExist: committee = Committee(name=post['org_name'], current=post['current']) committee.save() # There can be several periods periods = post['periods'] for period in periods: # Role is only optional information if 'role' in period: role = period['role'] else: role = None args = dict(begin=period['begin'], end=period['end'], role=role) # Differentiate between committee and speaker associations if org_type == 'committee': args['committee_id'] = committee.id ca_obj = find_with_attrs(ca_list, args) if not ca_obj: args['member'] = mp ca_obj = CommitteeAssociation(**args) self.logger.debug("New committee association: %s" % ca_obj) ca_obj.save() else: ca_obj.found = True else: sa_obj = find_with_attrs(sa_list, args) if not sa_obj: args['member'] = mp sa_obj = SpeakerAssociation(**args) self.logger.debug("New speaker association: %s" % sa_obj) sa_obj.save() else: sa_obj.found = True else: args = dict(label=post['label'], role=post['role'], begin=post['begin'], end=post['end']) ma_obj = find_with_attrs(ma_list, args) if not ma_obj: args['member'] = mp ma_obj = MinistryAssociation(**args) self.logger.debug("New ministry association: %s" % ma_obj) ma_obj.save() else: ma_obj.found = True for obj in ca_list + ma_list + sa_list: if not getattr(obj, 'found', False): self.logger.warning("Deleting removed association: %s" % obj) obj.delete()
def save_member(self, mp_info): # Member try: mp = Member.objects.get(origin_id=str(mp_info['id'])) if not self.replace: return except Member.DoesNotExist: mp = Member(origin_id=str(mp_info['id'])) mp.name = mp_info['name'] mp.birth_date = mp_info['birthdate'] if 'birthplace' in mp_info: mp.birth_place = mp_info['birthplace'] mp.given_names = mp_info['given_names'] mp.surname = mp_info['surname'] if 'email' in mp_info: mp.email = mp_info['email'] else: mp.email = None if 'phone' in mp_info: mp.phone = mp_info['phone'] else: mp.phone = None if 'gender' in mp_info and mp_info['gender']: mp.gender = mp_info['gender'] mp.info_link = mp_info['info_url'] # This is to support the hack where we add non-MP ministers # as pseudo MPs. They do not get a PartyAssociation, only # party attribute for purpose of counting ministers if 'party' in mp_info: mp.party = Party.objects.get(abbreviation=mp_info['party']) else: mp.party = self.determine_party(mp_info) url = mp_info['portrait'] fname = slugify(mp.name) + '.jpg' dir_name = os.path.join(mp.photo.field.upload_to, fname) path = os.path.join(settings.MEDIA_ROOT, dir_name) mp.photo = dir_name if self.replace or not os.path.exists(path): self.logger.debug("getting MP portrait") s = self.open_url(url, 'members') f = open(path, 'wb') f.write(s) f.close() mp.mark_modified() mp.save() # Districts DistrictAssociation.objects.filter(member=mp).delete() for da in mp_info['districts']: d_name = da['district'].replace(' vaalipiiri', '') try: district = District.objects.get(long_name=da['district']) except District.DoesNotExist: district = None da_obj = DistrictAssociation(member=mp, begin=da['begin']) da_obj.district = district da_obj.name = d_name if 'end' in da: da_obj.end = da['end'] da_obj.save() # Parties PartyAssociation.objects.filter(member=mp).delete() for pa in mp_info['parties']: p_name = pg_to_party(pa['party']) if p_name: party = Party.objects.get(abbreviation=p_name) else: party = None pa_obj = PartyAssociation(member=mp, begin=pa['begin']) pa_obj.party = party pa_obj.name = pa['party'] if 'end' in pa: pa_obj.end = pa['end'] pa_obj.save() def find_with_attrs(obj_list, arg_dict): for o in obj_list: for (key, val) in list(arg_dict.items()): oattr = getattr(o, key) if oattr != val: #print "mismatch: %s (%s) vs. %s (%s)" % (oattr, type(oattr), val, type(val)) break else: return o ca_list = list(CommitteeAssociation.objects.filter(member=mp)) sa_list = list(SpeakerAssociation.objects.filter(member=mp)) ma_list = list(MinistryAssociation.objects.filter(member=mp)) # Memberships for post in mp_info['posts']: org_type = post.get('org_type', None) if org_type in ('committee', 'speakers'): if org_type == 'committee': try: committee = Committee.objects.get( name=post['org_name']) # Update current committees if committee.current == False and post[ 'current'] == True: committee.current = True committee.save() except Committee.DoesNotExist: committee = Committee(name=post['org_name'], current=post['current']) committee.save() # There can be several periods periods = post['periods'] for period in periods: # Role is only optional information if 'role' in period: role = period['role'] else: role = None args = dict(begin=period['begin'], end=period['end'], role=role) # Differentiate between committee and speaker associations if org_type == 'committee': args['committee_id'] = committee.id ca_obj = find_with_attrs(ca_list, args) if not ca_obj: args['member'] = mp ca_obj = CommitteeAssociation(**args) self.logger.debug("New committee association: %s" % ca_obj) ca_obj.save() else: ca_obj.found = True else: sa_obj = find_with_attrs(sa_list, args) if not sa_obj: args['member'] = mp sa_obj = SpeakerAssociation(**args) self.logger.debug("New speaker association: %s" % sa_obj) sa_obj.save() else: sa_obj.found = True else: args = dict(label=post['label'], role=post['role'], begin=post['begin'], end=post['end']) ma_obj = find_with_attrs(ma_list, args) if not ma_obj: args['member'] = mp ma_obj = MinistryAssociation(**args) self.logger.debug("New ministry association: %s" % ma_obj) ma_obj.save() else: ma_obj.found = True for obj in ca_list + ma_list + sa_list: if not getattr(obj, 'found', False): self.logger.warning("Deleting removed association: %s" % obj) obj.delete()