def parse_committee_vote_data(self, proposal_data, dossier_pk): title = u'%s vote on %s' % (proposal_data['committee'], proposal_data['doc']) changed = False try: proposal = Proposal.objects.get(title=title, dossier_id=dossier_pk) except Proposal.DoesNotExist: proposal = Proposal(title=title, dossier_id=dossier_pk) changed = True try: ref = RE_COMVOTE_REF.search(proposal_data['url']).group(1) except: logger.debug(u'Cannot find proposal reference for %s' % title) return data_map = dict(datetime=_parse_date(proposal_data['ts']), reference=ref, kind='committee-vote') position_map = {'For': '+', 'Against': '-', 'Abstain': '0'} data_map.update(self.parse_proposal_totals(proposal_data, position_map)) for key, value in data_map.items(): if value != getattr(proposal, key, None): setattr(proposal, key, value) changed = True if changed: proposal.save() if self.should_skip(proposal_data): logger.debug(u'Skipping votes for dossier %s', proposal_data.get('epref', title)) return self.parse_proposal_votes(proposal, proposal_data, position_map)
def parse_proposal_data(self, proposal_data, dossier_pk): """Get or Create a proposal model from raw data""" if 'issue_type' not in proposal_data.keys(): logger.debug('This proposal data without issue_type: %s', proposal_data['epref']) return changed = False try: proposal = Proposal.objects.get(title=proposal_data['title']) except Proposal.DoesNotExist: proposal = Proposal(title=proposal_data['title']) changed = True data_map = dict(title=proposal_data['title'], datetime=_parse_date(proposal_data['ts']), dossier_id=dossier_pk, reference=proposal_data.get('report'), kind=proposal_data.get('issue_type')) data_map.update(self.parse_proposal_totals(proposal_data)) for key, value in data_map.items(): if value != getattr(proposal, key, None): setattr(proposal, key, value) changed = True if changed: proposal.save() if self.should_skip(proposal_data): logger.debug(u'Skipping votes for dossier %s', proposal_data.get('epref', proposal_data['title'])) return self.parse_proposal_votes(proposal, proposal_data) return proposal
def parse_scrutin_data(self, data): ref = data['url'] if 'dossier_url' not in data: logger.debug('Cannot create proposal without dossier') return dossier = self.get_dossier(data['dossier_url']) if dossier is None: logger.debug('Cannot create proposal for unknown dossier %s' % data['dossier_url']) return changed = False try: proposal = Proposal.objects.get(reference=ref) except Proposal.DoesNotExist: proposal = Proposal(reference=ref, total_for=0, total_against=0, total_abstain=0) logger.debug('Created proposal %s' % ref) changed = True values = dict( title=_get_unique_title(proposal.pk, data["objet"]), datetime=_parse_date(data["date"]), dossier_id=dossier.pk, kind='dossier' ) for key, value in values.items(): if value != getattr(proposal, key, None): logger.debug('Changed proposal %s to %s' % (key, value)) setattr(proposal, key, value) changed = True if changed: logger.debug('Updated proposal %s' % ref) proposal.save()
def parse_proposal_data(self, proposal_data, dossier_pk): """Get or Create a proposal model from raw data""" proposal_display = "{} ({})".format( proposal_data["title"].encode("utf-8"), proposal_data.get("report", "").encode("utf-8") ) if "issue_type" not in proposal_data.keys(): logger.debug("This proposal data without issue_type: %s", proposal_data["epref"]) return changed = False try: proposal = Proposal.objects.get(title=proposal_data["title"]) except Proposal.DoesNotExist: proposal = Proposal(title=proposal_data["title"]) changed = True data_map = dict( title=proposal_data["title"], datetime=_parse_date(proposal_data["ts"]), dossier_id=dossier_pk, reference=proposal_data.get("report"), kind=proposal_data.get("issue_type"), ) for position in ("For", "Abstain", "Against"): position_data = proposal_data.get(position, {}) position_total = position_data.get("total", 0) if isinstance(position_total, str) and position_total.isdigit(): position_total = int(position_total) data_map["total_%s" % position.lower()] = position_total for key, value in data_map.items(): if value != getattr(proposal, key, None): setattr(proposal, key, value) changed = True if changed: proposal.save() if self.should_skip(proposal_data): logger.debug("Skipping dossier %s", proposal_data.get("epref", proposal_data["title"])) return positions = ["For", "Abstain", "Against"] logger.info("Looking for votes in proposal {}".format(proposal_display)) for position in positions: for group_vote_data in proposal_data.get(position, {}).get("groups", {}): for vote_data in group_vote_data["votes"]: if not isinstance(vote_data, dict): logger.error("Skipping vote data %s for proposal %s", vote_data, proposal_data["_id"]) continue representative_pk = self.get_representative(vote_data) if representative_pk is None: logger.error("Could not find mep for %s", vote_data) continue representative_name = vote_data.get("orig", "") changed = False try: vote = Vote.objects.get(representative_id=representative_pk, proposal_id=proposal.pk) except Vote.DoesNotExist: vote = Vote(proposal_id=proposal.pk, representative_id=representative_pk) changed = True if vote.position != position.lower(): changed = True vote.position = position.lower() if vote.representative_name != representative_name: changed = True vote.representative_name = representative_name if changed: vote.save() logger.debug( "Save vote %s for MEP %s on %s #%s to %s", vote.pk, representative_pk, proposal_data["title"], proposal.pk, position, ) return proposal
def parse_proposal_data(self, proposal_data, dossier_pk): """Get or Create a proposal model from raw data""" proposal_display = '{} ({})'.format( proposal_data['title'].encode('utf-8'), proposal_data.get('report', '').encode('utf-8')) if 'issue_type' not in proposal_data.keys(): logger.debug('This proposal data without issue_type: %s', proposal_data['epref']) return changed = False try: proposal = Proposal.objects.get(title=proposal_data['title']) except Proposal.DoesNotExist: proposal = Proposal(title=proposal_data['title']) changed = True data_map = dict(title=proposal_data['title'], datetime=_parse_date(proposal_data['ts']), dossier_id=dossier_pk, reference=proposal_data.get('report'), kind=proposal_data.get('issue_type')) for position in ('For', 'Abstain', 'Against'): position_data = proposal_data.get(position, {}) position_total = position_data.get('total', 0) if isinstance(position_total, str) and position_total.isdigit(): position_total = int(position_total) data_map['total_%s' % position.lower()] = position_total for key, value in data_map.items(): if value != getattr(proposal, key, None): setattr(proposal, key, value) changed = True if changed: proposal.save() if self.should_skip(proposal_data): logger.debug('Skipping dossier %s', proposal_data.get('epref', proposal_data['title'])) return positions = ['For', 'Abstain', 'Against'] logger.info( 'Looking for votes in proposal {}'.format(proposal_display)) for position in positions: for group_vote_data in proposal_data.get(position, {}).get('groups', {}): for vote_data in group_vote_data['votes']: if not isinstance(vote_data, dict): logger.error('Skipping vote data %s for proposal %s', vote_data, proposal_data['_id']) continue representative_pk = self.get_representative(vote_data) if representative_pk is None: logger.error('Could not find mep for %s', vote_data) continue representative_name = vote_data.get('orig', '') changed = False try: vote = Vote.objects.get( representative_id=representative_pk, proposal_id=proposal.pk) except Vote.DoesNotExist: vote = Vote(proposal_id=proposal.pk, representative_id=representative_pk) changed = True if vote.position != position.lower(): changed = True vote.position = position.lower() if vote.representative_name != representative_name: changed = True vote.representative_name = representative_name if changed: vote.save() logger.debug('Save vote %s for MEP %s on %s #%s to %s', vote.pk, representative_pk, proposal_data['title'], proposal.pk, position) return proposal
def parse_proposal_data(self, proposal_data, dossier_pk): """Get or Create a proposal model from raw data""" proposal_display = '{} ({})'.format(proposal_data['title'].encode( 'utf-8'), proposal_data.get('report', '').encode('utf-8')) if 'issue_type' not in proposal_data.keys(): logger.debug('This proposal data without issue_type: %s', proposal_data['epref']) return changed = False try: proposal = Proposal.objects.get(title=proposal_data['title']) except Proposal.DoesNotExist: proposal = Proposal(title=proposal_data['title']) changed = True data_map = dict( title=proposal_data['title'], datetime=_parse_date(proposal_data['ts']), dossier_id=dossier_pk, reference=proposal_data.get('report'), kind=proposal_data.get('issue_type') ) for position in ('For', 'Abstain', 'Against'): position_data = proposal_data.get(position, {}) position_total = position_data.get('total', 0) if isinstance(position_total, str) and position_total.isdigit(): position_total = int(position_total) data_map['total_%s' % position.lower()] = position_total for key, value in data_map.items(): if value != getattr(proposal, key, None): setattr(proposal, key, value) changed = True if changed: proposal.save() responses = vote_pre_import.send(sender=self, vote_data=proposal_data) for receiver, response in responses: if response is False: logger.debug( 'Skipping dossier %s', proposal_data.get( 'epref', proposal_data['title'])) return positions = ['For', 'Abstain', 'Against'] logger.info( 'Looking for votes in proposal {}'.format(proposal_display)) for position in positions: for group_vote_data in proposal_data.get( position, {}).get( 'groups', {}): for vote_data in group_vote_data['votes']: if not isinstance(vote_data, dict): logger.error('Skipping vote data %s for proposal %s', vote_data, proposal_data['_id']) continue representative_pk = self.get_representative(vote_data) if representative_pk is None: logger.error('Could not find mep for %s', vote_data) continue representative_name = vote_data.get('orig', '') changed = False try: vote = Vote.objects.get( representative_id=representative_pk, proposal_id=proposal.pk) except Vote.DoesNotExist: vote = Vote(proposal_id=proposal.pk, representative_id=representative_pk) changed = True if vote.position != position.lower(): changed = True vote.position = position.lower() if vote.representative_name != representative_name: changed = True vote.representative_name = representative_name if changed: vote.save() logger.debug('Save vote %s for MEP %s on %s #%s to %s', vote.pk, representative_pk, proposal_data['title'], proposal.pk, position) return proposal